"""Reproduce grade-3 evidence against the lab4 Compose stack; no credentials in output.""" import argparse from datetime import datetime, timezone import json import os from pathlib import Path import subprocess import sys import threading import time from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure COMPOSE = ["docker", "compose", "--env-file", ".env"] def command(args, capture=False): result = subprocess.run(args, cwd=ROOT, text=True, encoding="utf-8", errors="replace", stdout=subprocess.PIPE if capture else sys.stdout, stderr=subprocess.STDOUT, timeout=180) if result.returncode: raise RuntimeError("Command failed: " + args[0]) return result.stdout def sql(query): return command(COMPOSE + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query], True).strip() def state(): result = {} for service in ("api1", "api2", "postgres", "s3", "lb"): cid = command(COMPOSE + ["ps", "-a", "-q", service], True).strip() data = json.loads(command(["docker", "inspect", cid], True))[0] result[service] = {"id": cid[:12], "status": data["State"]["Status"], "started_at": data["State"]["StartedAt"], "volumes": [m["Name"] for m in data["Mounts"] if m["Type"] == "volume"], "ports": data["HostConfig"]["PortBindings"], "memory_limit": data["HostConfig"]["Memory"], "nano_cpus": data["HostConfig"]["NanoCpus"]} return result def wait_replicas(c, actor): deadline, seen = time.monotonic() + 60, set() while time.monotonic() < deadline: try: seen.add(c.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"]) if len(seen) >= 2: return seen except Failure: pass time.sleep(0.1) raise AssertionError("Two replicas did not become ready") def verify_upload_failover(c, a, b, chat): import hashlib content = (ROOT / "tests/fixtures/sample.png").read_bytes() for service in ("api1", "api2"): wait_replicas(c, a) for _ in range(30): response = c.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=failover.png", 201, actor=a, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment") target = response["headers"]["X-Instance-Id"] if target.startswith(service + "-"): break else: raise AssertionError("Upload did not reach expected replica") att = response["json"] message = c.message(a, chat, "", attachments=[att["id"]])["json"] env = dict(os.environ, TARGET_INSTANCE_ID=target) subprocess.run([sys.executable, "scripts/stop_one.py"], cwd=ROOT, env=env, check=True, timeout=20, stdout=sys.stdout, stderr=subprocess.STDOUT) try: assert state()[service]["status"] == "exited" result = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=b) assert result["bytes"] == content assert result["headers"]["X-Instance-Id"] != target assert c.call("GET", "/api/v1/attachments/" + att["id"], actor=b, schema="Attachment")["json"] == att print("UPLOAD_FAILOVER " + json.dumps({"stopped": target, "download_instance": result["headers"]["X-Instance-Id"], "sha256": hashlib.sha256(result["bytes"]).hexdigest(), "message_id": message["id"]}), flush=True) finally: command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True) # Inspect mounts: original bytes are never persisted in an API volume. assert all(not state()[service]["volumes"] for service in ("api1", "api2")) key = sql("SELECT object_key FROM attachments WHERE id='" + att["id"] + "'") probe = """import urllib.request, urllib.error, sys, os try: urllib.request.urlopen(os.environ['S3_ENDPOINT'] + '/' + os.environ['S3_BUCKET'] + '/' + sys.argv[1],timeout=3) raise AssertionError('Anonymous S3 GET allowed') except urllib.error.HTTPError as exc: assert exc.code == 403, exc.code print('PASS: anonymous S3 GET of existing object denied (403)') """ command(COMPOSE + ["exec", "-T", "api1", "python", "-c", probe, key]) def verify_s3_failure(c, a, b, chat): att, content = c.attachment(a, chat) before = sql("SELECT count(*) FROM attachments") # Execute inside the API network namespace: LB correctly removes unready APIs. # Program travels through stdin; bearer never appears in command arguments/output. program = """import urllib.request, urllib.error, json, base64 actor = ACTOR attachment = ATTACHMENT chat_id = CHAT_ID content = base64.b64decode(CONTENT) def request(path, expected, data=None, mime=None): headers = {'Authorization': 'Bearer ' + actor['token']} if mime: headers['Content-Type'] = mime req = urllib.request.Request('http://127.0.0.1:8080' + path, data=data, headers=headers) try: response = urllib.request.urlopen(req, timeout=8) except urllib.error.HTTPError as exc: response = exc with response: assert response.code == expected, (path, response.code) result = response.read() return result request('/health/live', 200) request('/health/ready', 503) request('/api/v1/attachments/' + attachment['id'], 200) request('/api/v1/attachments/' + attachment['id'] + '/content', 503) request('/api/v1/chats/' + chat_id + '/attachments?filename=outage.png', 503, content, 'image/png') print('PASS: S3 down: live=200 ready=503 metadata=200 download=503 upload=503') """ import base64 program = program.replace("ACTOR", repr({"token": a["token"]})).replace("ATTACHMENT", repr(att)).replace("CHAT_ID", repr(chat["id"])).replace("CONTENT", repr(base64.b64encode(content).decode())) command(COMPOSE + ["stop", "--timeout", "10", "s3"], True) try: result = subprocess.run(COMPOSE + ["exec", "-T", "api1", "python", "-"], cwd=ROOT, input=program, text=True, capture_output=True, timeout=30) assert result.returncode == 0, "S3 outage check failed (details suppressed to protect session)" print(result.stdout.strip(), flush=True) assert sql("SELECT count(*) FROM attachments") == before finally: command(COMPOSE + ["start", "s3"], True) wait_replicas(c, a) assert c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=b)["bytes"] == content print("PASS: S3 recovered; original bytes preserved; failed upload created no metadata", flush=True) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument("--base-url", default="http://127.0.0.1:8080") p.add_argument("--output") args = p.parse_args() if args.output: sys.stdout = open(ROOT / args.output, "w", encoding="utf-8", buffering=1) sys.stderr = sys.stdout os.environ.update(BASE_URL=args.base_url, NO_PROXY="localhost,127.0.0.1", no_proxy="localhost,127.0.0.1") print("UTC " + datetime.now(timezone.utc).isoformat(), flush=True) print("COMMAND python scripts/verify.py --base-url " + args.base_url, flush=True) command(["docker", "version", "--format", "Client={{.Client.Version}} Server={{.Server.Version}}"]) command(["docker", "info", "--format", "Docker CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}"]) command(COMPOSE + ["config", "--quiet"]) command([sys.executable, "scripts/check_contract.py"]) command([sys.executable, "tests/smoke.py", "--base-url", args.base_url]) command([sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-v"]) c = Client(args.base_url, timeout=5) a, b, _, chat = c.bootstrap() seen = wait_replicas(c, a) print("OBSERVED_INSTANCES " + json.dumps(sorted(seen)), flush=True) first = c.message(a, chat, "survives restart", key="restart-key")["json"] second = c.message(b, chat, "cursor after restart")["json"] path = "/api/v1/chats/" + chat["id"] + "/messages" page = c.call("GET", path + "?limit=1", actor=a, schema="MessagePage")["json"] counts = "SELECT (SELECT count(*) FROM users),(SELECT count(*) FROM chats),(SELECT count(*) FROM memberships),(SELECT count(*) FROM messages),(SELECT count(*) FROM sessions),(SELECT count(*) FROM message_keys),(SELECT count(*) FROM schema_migrations),(SELECT count(*) FROM attachments)" before_counts = sql(counts) for _ in range(2): command(COMPOSE + ["run", "--rm", "--no-deps", "migrate"]) assert sql(counts) == before_counts print("PASS: repeated migrations preserve all table counts " + before_counts, flush=True) before = state() print("BEFORE_RESTART " + json.dumps(before), flush=True) hook = "scripts/restart.cmd" if os.name == "nt" else "scripts/restart.sh" command([sys.executable, "tests/resilience.py", "--mode", "persistence", "--action-script", hook, "--base-url", args.base_url]) after = state() print("AFTER_RESTART " + json.dumps(after), flush=True) for service in before: assert before[service]["started_at"] != after[service]["started_at"] assert before[service]["volumes"] == after[service]["volumes"] assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first rest = c.call("GET", path + "?limit=1&cursor=" + quote(page["next_cursor"]), actor=a, schema="MessagePage")["json"] assert page["items"] == [first] and rest["items"] == [second] print("PASS: five containers restarted; volume, sessions, cursor and idempotency result preserved", flush=True) verify_upload_failover(c, a, b, chat) verify_s3_failure(c, a, b, chat) stopped = set() hook = "scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh" # Run the unchanged public test until it has selected each real replica. # Read concurrently with its POST stream to also measure read availability. for attempt in range(6): if stopped == {"api1", "api2"}: break command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True) wait_replicas(c, a) done, reads = threading.Event(), {"ok": 0, "errors": 0} def read_during_failure(): reader = Client(args.base_url, timeout=2) while not done.is_set(): try: result = reader.call("GET", path, actor=b, schema="MessagePage")["json"] assert first in result["items"] reads["ok"] += 1 except Failure: reads["errors"] += 1 done.wait(0.1) worker = threading.Thread(target=read_during_failure) worker.start() try: print("FAILOVER_RUN " + str(attempt + 1), flush=True) command([sys.executable, "tests/resilience.py", "--mode", "failover", "--action-script", hook, "--base-url", args.base_url]) current = state() down = {s for s in ("api1", "api2") if current[s]["status"] == "exited"} assert len(down) == 1 stopped.update(down) print("AFTER_FAILOVER " + json.dumps(current), flush=True) assert reads["ok"] >= 5 print("CONCURRENT_READS " + json.dumps(reads), flush=True) assert c.message(a, chat, "survives restart", key="restart-key")["json"] == first finally: done.set() worker.join(timeout=5) command(COMPOSE + ["up", "-d", "--wait", "api1", "api2"], True) assert stopped == {"api1", "api2"}, "Both replicas must be tested" wait_replicas(c, a) assert sql("SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM memberships m WHERE m.chat_id=c.id)") == "0" print("PASS: each API stopped separately; reads/writes and original session survive; both replicas restored; S3 bytes checked", flush=True) print("PASS: grade-3 verification complete", flush=True) if __name__ == "__main__": main()