Files
mtusi2026lab3/scripts/audit-lab3.py
T
2026-11-06 18:54:28 +03:00

551 lines
18 KiB
Python

"""Воспроизведение замечаний к lab3. Только текущий локальный Compose-стенд.
Временные переименования, pause и замены контейнеров восстанавливаются в finally.
Секреты не печатаются. Все созданные пользователи и сообщения синтетические.
"""
import argparse
import base64
import http.client
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlsplit
from replicas import (
ROOT,
compose,
containers,
describe,
docker,
instance_id,
wait_ready,
wait_visible,
)
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
EVIDENCE = ROOT / "evidence"
def sql(query):
return subprocess.check_output(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"sh",
"-c",
'exec psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At',
],
cwd=ROOT,
input=query,
text=True,
timeout=15,
).strip()
def wire(method, path, actor=None, body=None, timeout=7):
url = urlsplit(os.environ.get("BASE_URL", "http://localhost:8080"))
headers = {"Content-Type": "application/json"}
if actor:
headers["Authorization"] = "Bearer " + actor["token"]
conn = http.client.HTTPConnection(url.hostname, url.port, timeout=timeout)
before = time.monotonic()
try:
conn.request(
method,
path,
body=None if body is None else json.dumps(body).encode(),
headers=headers,
)
response = conn.getresponse()
response.read()
return {
"status": response.status,
"seconds": round(time.monotonic() - before, 3),
"instance": response.getheader("X-Instance-Id"),
}
finally:
conn.close()
def direct_ready(cid):
code = """import urllib.request,urllib.error
try: r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=6)
except urllib.error.HTTPError as e: r=e
with r: print(r.status)
"""
return int(docker("exec", cid, "python", "-c", code))
def restored():
for cid in containers():
wait_visible(wait_ready(cid))
def contract():
client = Client()
alice, bob, eve, chat = client.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/messages"
for endpoint in ["/api/v1/chats", path]:
for query in [
"cursor=W10",
"cursor=bnVsbA",
"limit=abc",
"limit=0",
"limit=101",
]:
client.call("GET", endpoint + "?" + query, 400, actor=alice)
# Unicode Basic password including colon and no body on login.
registration = {
"username": "audit_utf_" + uuid.uuid4().hex[:12],
"display_name": "Unicode",
"password": "пароль:😀с-пробелом 123",
}
client.call("POST", "/api/v1/users", 201, body=registration, schema="User")
basic = (
"Basic "
+ base64.b64encode(
(registration["username"] + ":" + registration["password"]).encode()
).decode()
)
client.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": basic},
schema="Session",
)
client.call(
"POST",
"/api/v1/auth/sessions",
400,
raw=b"{}",
headers={"Authorization": basic},
)
client.call(
"POST",
"/api/v1/chats",
400,
actor=alice,
body={
"title": "Unknown member",
"member_ids": [alice["id"], str(uuid.uuid4())],
},
)
# Empty page limit=7 is warmed before the successful write.
assert not client.call("GET", path + "?limit=7", actor=alice, schema="MessagePage")[
"json"
]["items"]
key = uuid.uuid4().hex
barrier = threading.Barrier(16)
clients = [Client() for _ in range(16)]
message = "Начало\u0000конец 👋"
def send(local):
barrier.wait()
return local.message(alice, chat, message, key=key)
with ThreadPoolExecutor(max_workers=16) as pool:
responses = list(pool.map(send, clients))
original = responses[0]["json"]
assert all(r["json"] == original for r in responses)
observed = {r["headers"]["X-Instance-Id"] for r in responses}
assert len(observed) >= 2
assert int(sql(f"SELECT count(*) FROM messages WHERE chat_id='{chat['id']}'")) == 1
assert (
int(sql(f"SELECT count(*) FROM message_requests WHERE chat_id='{chat['id']}'"))
== 1
)
assert (
int(
sql(
f"SELECT position(decode('00','hex') IN text) FROM messages WHERE id='{original['id']}'"
)
)
> 0
)
for limit in [1, 7, 10, 20, 50, 100]:
result = client.call(
"GET", path + "?limit=" + str(limit), actor=bob, schema="MessagePage"
)["json"]
assert result["items"] == [original], "Stale history or lost U+0000"
client.call(
"POST",
path,
409,
actor=alice,
body={"text": "changed"},
headers={"Idempotency-Key": key},
)
chat2 = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Other chat", "member_ids": [alice["id"]]},
schema="Chat",
)["json"]
client.message(alice, chat2, message, key=key)
client.message(bob, chat, message, key=key)
for bad in ["", "x" * 129, "bad key", "bad/key"]:
client.call(
"POST",
path,
400,
actor=alice,
body={"text": "bad key"},
headers={"Idempotency-Key": bad},
)
# Bind cursor to actual chat; limit=1 produces a nonempty cursor.
page = client.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")[
"json"
]
client.call(
"GET",
"/api/v1/chats/" + chat2["id"] + "/messages?cursor=" + page["next_cursor"],
400,
actor=alice,
)
large = client.call(
"POST",
path,
413,
actor=alice,
raw=b'{"text":"valid"}' + b" " * 70000,
headers={
"Content-Type": "application/json",
"Idempotency-Key": uuid.uuid4().hex,
},
)
assert large["headers"]["X-Instance-Id"] and large["headers"]["X-Request-Id"]
# A raw U+0000 is illegal JSON; escaped \u0000 is valid and must round-trip.
client.call(
"POST",
path,
400,
actor=alice,
raw=b'{"text":"bad\x00json"}',
headers={
"Content-Type": "application/json",
"Idempotency-Key": uuid.uuid4().hex,
},
)
client.call(
"POST",
path,
404,
actor=eve,
body={"text": message},
headers={"Idempotency-Key": key},
)
# Exercise boundary on Unicode code points, not UTF-8 byte count.
long_message = client.message(alice, chat, "\u0000" * 4000)["json"]
assert long_message["text"] == "\u0000" * 4000
replay = client.message(alice, chat, message, key=key)
assert replay["json"] == original
return {
"result": "PASS",
"simultaneous_requests": 16,
"observed_instances": sorted(observed),
"database_rows_for_first_key": 1,
"headers_413": "present",
"nul_roundtrip": "201, replay and history; 4000 code points supported",
"checks": [
"invalid cursor types/limit",
"UTF-8 Basic",
"login body rejected",
"unknown member 400",
"key scope/format",
"foreign cursor",
"fresh history limit=7 and other limits",
"outsider hidden",
"raw invalid JSON rejected",
],
}
def readiness():
client = Client()
actor = client.user("audit_ready")
evidence = []
for table in ["users", "chats", "chat_members", "messages", "message_requests"]:
hidden = "audit_hidden_" + table
try:
sql(f"ALTER TABLE {table} RENAME TO {hidden}")
statuses = {cid[:12]: direct_ready(cid) for cid in containers()}
assert set(statuses.values()) == {503}, statuses
deadline = time.monotonic() + 8
while wire("GET", "/api/v1/users/me", actor)["status"] != 503:
assert time.monotonic() < deadline, "LB still routes to unready API"
time.sleep(0.2)
results = []
for _ in range(8):
result = wire("GET", "/api/v1/users/me", actor)
assert result["status"] == 503, result
results.append(result)
registration = wire(
"POST",
"/api/v1/users",
body={
"username": "blocked_" + uuid.uuid4().hex[:12],
"display_name": "Must not route",
"password": "valid-audit-password-123",
},
)
assert registration["status"] == 503, registration
evidence.append(
{
"missing_table": table,
"direct_readiness": statuses,
"business_requests": results,
"registration": registration,
}
)
finally:
sql(f"ALTER TABLE IF EXISTS {hidden} RENAME TO {table}")
restored()
return {"result": "PASS", "experiments": evidence}
def frozen_stores():
client = Client()
actor = client.user("audit_pause")
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
evidence = []
for service in ["postgres", "redis"]:
try:
began = time.monotonic()
compose("pause", service)
transition = wire("GET", "/health/ready")
assert transition["status"] == 503, transition
assert transition["seconds"] < 7
# Allow active checks to exclude all API, then test the LB route,
# including registration that itself does not depend on Redis.
time.sleep(4)
result = wire("GET", "/api/v1/users/me", actor)
registration = wire(
"POST",
"/api/v1/users",
body={
"username": "paused_" + uuid.uuid4().hex[:12],
"display_name": "Paused store",
"password": "valid-audit-password-123",
},
)
assert result["status"] == registration["status"] == 503, (
result,
registration,
)
evidence.append(
{
"service": service,
"initial_ready_request": transition,
"business": result,
"registration": registration,
"elapsed_seconds": round(time.monotonic() - began, 3),
}
)
finally:
compose("unpause", service)
restored()
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
return {"result": "PASS", "experiments": evidence}
def changing_ips():
client = Client()
actor = client.user("audit_dns")
chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=actor,
body={"title": "Forced new IPs", "member_ids": [actor["id"]]},
schema="Chat",
)["json"]
originals = containers()
lb = compose("ps", "-q", "lb")
lb_started = describe(lb)["StartedAt"]
holders = []
events = []
occupied = []
try:
for old in originals:
networks = json.loads(
docker("inspect", "--format", "{{json .NetworkSettings.Networks}}", old)
)
assert len(networks) == 1
network, state = next(iter(networks.items()))
ip = state["IPAddress"]
alias = instance_id(old)
docker("stop", "--time", "10", old)
docker("rm", old)
holder = "lab3-audit-ip-" + uuid.uuid4().hex[:10]
docker(
"run",
"-d",
"--name",
holder,
"--network",
network,
"--ip",
ip,
"--cpus",
".05",
"--memory",
"32m",
"--read-only",
"--cap-drop",
"ALL",
"--entrypoint",
"python",
"mtusi-messenger:lab3",
"-c",
"import time; time.sleep(900)",
)
holders.append(holder)
occupied.append(ip)
before = set(containers())
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api")
added = set(containers()) - before
assert len(added) == 1
new = added.pop()
new_alias = wait_ready(new)
new_ip = docker(
"inspect",
"--format",
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
new,
)
assert new_ip not in occupied
began = time.monotonic()
wait_visible(new_alias, timeout=10)
message = client.message(actor, chat, "DNS " + new_alias)["json"]
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages",
actor=actor,
schema="MessagePage",
)["json"]
assert message in page["items"]
events.append(
{
"old_container": old[:12],
"old_ip": ip,
"old_instance": alias,
"new_container": new[:12],
"new_ip": new_ip,
"new_instance": new_alias,
"discovery_after_ready_seconds": round(time.monotonic() - began, 3),
}
)
observed = {
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")[
"headers"
]["X-Instance-Id"]
for _ in range(40)
}
assert observed == {e["new_instance"] for e in events}, observed
assert describe(lb)["StartedAt"] == lb_started, "HAProxy was restarted"
return {
"result": "PASS",
"events": events,
"observed_instances_after": sorted(observed),
"lb_restarted": False,
"old_ips_kept_occupied_during_test": occupied,
}
finally:
for holder in holders:
docker("rm", "-f", holder)
compose("up", "-d", "--no-deps", "--wait", "api")
restored()
def limits_and_evidence():
limits = {}
for service in ["api", "postgres", "redis", "lb", "migration"]:
limits[service] = []
for cid in compose("ps", "--all", "-q", service).split():
state = json.loads(
docker("inspect", "--format", "{{json .HostConfig}}", cid)
)
record = {
"container": cid[:12],
"cpu": state["NanoCpus"] / 1e9,
"memory_bytes": state["Memory"],
}
assert record["cpu"] > 0 and record["memory_bytes"] > 0
if service == "api":
assert not state["PortBindings"]
limits[service].append(record)
rows = json.loads((EVIDENCE / "lab3-research/results.json").read_text())
for series in ["fixed", "growing"]:
for count in [1, 2, 4]:
assert any(
r["job"]["series"] == series and r["job"]["replicas"] == count
for r in rows
)
fixed = [r for r in rows if r["job"]["series"] == "fixed"]
assert {r["job"]["replicas"] * r["job"]["cpus"] for r in fixed} == {2.0}
assert {r["job"]["replicas"] * int(r["job"]["memory"][:-1]) for r in fixed} == {
1024
}
assert any(r["job"]["series"] == "improved-db" for r in rows)
for name in ["results.json", "summary.csv", "scaling.png", "bottleneck.png"]:
assert (EVIDENCE / "lab3-research" / name).stat().st_size > 0
# The rolling command explicitly invokes Python and needs no executable bit.
assert "$(PYTHON) scripts/rolling-update.py" in (ROOT / "Makefile").read_text()
for file in ["scripts/restart.sh", "scripts/stop-one.sh"]:
assert (ROOT / file).stat().st_mode & 0o111
return {
"result": "PASS",
"limits": limits,
"fixed_api_budget": {"cpu": 2, "memory_mib": 1024},
"research_runs": len(rows),
"rolling_command": "python3 scripts/rolling-update.py",
"shell_hooks_executable": True,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--phase", choices=["all", "contract", "infrastructure"], default="all"
)
args = parser.parse_args()
checks = [("contract", contract)] if args.phase == "contract" else []
if args.phase == "all":
checks.append(("contract", contract))
if args.phase in ["all", "infrastructure"]:
checks += [
("readiness", readiness),
("frozen-stores", frozen_stores),
("changing-ips", changing_ips),
("limits-evidence", limits_and_evidence),
]
summary = {}
for name, check in checks:
print("Checking " + name, flush=True)
result = check()
(EVIDENCE / ("lab3-audit-" + name + ".json")).write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
)
summary[name] = "PASS"
print(name + " PASS", flush=True)
(EVIDENCE / ("lab3-audit-summary-" + args.phase + ".json")).write_text(
json.dumps(summary, indent=2) + "\n"
)
if __name__ == "__main__":
main()