"""Reproduce grade-3 evidence against the lab3 Compose stack; no credentials in output.""" import argparse from datetime import datetime, timezone import json import os from pathlib import Path import subprocess import sys import threading import time from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure COMPOSE = ["docker", "compose", "--env-file", ".env"] def command(args, capture=False): result = subprocess.run(args, cwd=ROOT, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE if capture else sys.stdout, stderr=subprocess.STDOUT, timeout=180) if result.returncode: raise RuntimeError("Command failed: " + args[0]) return result.stdout def sql(query): return command(COMPOSE + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query], True).strip() def state(): result = {} for service in ("api1", "api2", "postgres", "lb"): cid = command(COMPOSE + ["ps", "-a", "-q", service], True).strip() data = json.loads(command(["docker", "inspect", cid], True))[0] result[service] = {"id": cid[:12], "status": data["State"]["Status"], "started_at": data["State"]["StartedAt"], "volumes": [m["Name"] for m in data["Mounts"] if m["Type"] == "volume"], "ports": data["HostConfig"]["PortBindings"], "memory_limit": data["HostConfig"]["Memory"], "nano_cpus": data["HostConfig"]["NanoCpus"]} return result def wait_replicas(c, actor): deadline, seen = time.monotonic() + 60, set() while time.monotonic() < deadline: try: seen.add(c.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"]) if len(seen) >= 2: return seen except Failure: pass time.sleep(0.1) raise AssertionError("Two replicas did not become ready") def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--base-url", default="http://127.0.0.1:8080") p.add_argument("--output") args = p.parse_args() if args.output: sys.stdout = open(ROOT / args.output, "w", encoding="utf-8", buffering=1) sys.stderr = sys.stdout os.environ.update(BASE_URL=args.base_url, NO_PROXY="localhost,127.0.0.1", no_proxy="localhost,127.0.0.1") print("UTC " + datetime.now(timezone.utc).isoformat(), flush=True) print("COMMAND python scripts/verify.py --base-url " + args.base_url, flush=True) command(["docker", "version", "--format", "Client={{.Client.Version}} Server={{.Server.Version}}"]) command(["docker", "info", "--format", "Docker CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}"]) command(COMPOSE + ["config", "--quiet"]) command([sys.executable, "scripts/check_contract.py"]) command([sys.executable, "tests/smoke.py", "--base-url", args.base_url]) command([sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-v"]) c = Client(args.base_url, timeout=2) a, b, _, chat = c.bootstrap() seen = wait_replicas(c, a) print("OBSERVED_INSTANCES " + json.dumps(sorted(seen)), flush=True) first = c.message(a, chat, "survives restart", key="restart-key")["json"] second = c.message(b, chat, "cursor after restart")["json"] path = "/api/v1/chats/" + chat["id"] + "/messages" page = c.call("GET", path + "?limit=1", actor=a, schema="MessagePage")["json"] counts = "SELECT (SELECT count(*) FROM users),(SELECT count(*) FROM chats),(SELECT count(*) FROM memberships),(SELECT count(*) FROM messages),(SELECT count(*) FROM sessions),(SELECT count(*) FROM message_keys),(SELECT count(*) FROM schema_migrations)" before_counts = sql(counts) for _ in range(2): command(COMPOSE + ["run", "--rm", "--no-deps", "migrate"]) assert sql(counts) == before_counts print("PASS: repeated migrations preserve all table counts " + before_counts, flush=True) before = state() print("BEFORE_RESTART " + json.dumps(before), flush=True) hook = "scripts/restart.cmd" if os.name == "nt" else "scripts/restart.sh" command([sys.executable, "tests/resilience.py", "--mode", "persistence", "--action-script", hook, "--base-url", args.base_url]) after = state() print("AFTER_RESTART " + json.dumps(after), flush=True) for service in before: assert before[service]["started_at"] != after[service]["started_at"] assert before[service]["volumes"] == after[service]["volumes"] assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first rest = c.call("GET", path + "?limit=1&cursor=" + quote(page["next_cursor"]), actor=a, schema="MessagePage")["json"] assert page["items"] == [first] and rest["items"] == [second] print("PASS: four containers restarted; volume, sessions, cursor and idempotency result preserved", flush=True) stopped = set() hook = "scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh" # Run the unchanged public test until it has selected each real replica. # Read concurrently with its POST stream to also measure read availability. for attempt in range(6): if stopped == {"api1", "api2"}: break command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True) wait_replicas(c, a) done, reads = threading.Event(), {"ok": 0, "errors": 0} def read_during_failure(): reader = Client(args.base_url, timeout=2) while not done.is_set(): try: result = reader.call("GET", path, actor=b, schema="MessagePage")["json"] assert first in result["items"] reads["ok"] += 1 except Failure: reads["errors"] += 1 done.wait(0.1) worker = threading.Thread(target=read_during_failure) worker.start() try: print("FAILOVER_RUN " + str(attempt + 1), flush=True) command([sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", hook, "--base-url", args.base_url]) current = state() down = {s for s in ("api1", "api2") if current[s]["status"] == "exited"} assert len(down) == 1 stopped.update(down) print("AFTER_FAILOVER " + json.dumps(current), flush=True) assert reads["ok"] >= 5 print("CONCURRENT_READS " + json.dumps(reads), flush=True) assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first finally: done.set() worker.join(timeout=5) command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True) assert stopped == {"api1", "api2"}, "Both replicas must be tested" wait_replicas(c, a) assert sql("SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM memberships m WHERE m.chat_id=c.id)") == "0" print("PASS: each API stopped separately; reads/writes and original session survive; both replicas restored", flush=True) print("PASS: grade-3 verification complete", flush=True) if __name__ == "__main__": main()