#!/usr/bin/env python3 """Lab5 level-4 failure scenarios against a RUNNING compose stack (stdlib only; run from the repo root). python3 student-tests/level4_failures.py all # or: worker_down | commit_publish_gap | crash_after_s3 | replay | s3_outage Each scenario drives `docker compose` (never `down -v`: volumes stay) and prints PASS/FAIL with the observed facts. """ import base64, json, os, subprocess, sys, time, urllib.error, urllib.request, uuid from pathlib import Path BASE = os.getenv("BASE_URL", "http://localhost:8080") PNG = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes() DC = ["docker", "compose", "--env-file", ".env"] def sh(*args, env=None, check=True): r = subprocess.run(list(args), capture_output=True, text=True, env={**os.environ, **(env or {})}) if check and r.returncode: raise SystemExit(f"command failed: {' '.join(args)}\n{r.stderr[-500:]}") return r.stdout.strip() def sql(query): return sh(*DC, "exec", "-T", "postgres", "sh", "-c", f'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c "{query}"') def http(method, path, token=None, body=None, headers=None, raw=None): h = dict(headers or {}) if token: h["Authorization"] = "Bearer " + token data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) if body is not None: h["Content-Type"] = "application/json" req = urllib.request.Request(BASE + path, data=data, method=method, headers=h) try: with urllib.request.urlopen(req, timeout=30) as r: payload = r.read() return r.status, (json.loads(payload) if payload and "json" in r.headers.get("Content-Type", "") else payload) except urllib.error.HTTPError as e: payload = e.read() try: return e.code, json.loads(payload) except ValueError: return e.code, payload def setup(): name = "u" + uuid.uuid4().hex[:10] pw = "correct-horse-battery" st, user = http("POST", "/api/v1/users", body={"username": name, "display_name": "Tester", "password": pw}) assert st == 201, (st, user) cred = base64.b64encode(f"{name}:{pw}".encode()).decode() st, sess = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) token = sess["token"] st, chat = http("POST", "/api/v1/chats", token, {"title": "lab5", "member_ids": [user["id"]]}) return token, chat["id"] def upload(token, chat): st, att = http("POST", f"/api/v1/chats/{chat}/attachments?filename=sample.png", token, raw=PNG, headers={"Content-Type": "image/png"}) assert st == 202 and att["status"] == "queued", (st, att) return att["id"] def wait(token, aid, want, timeout=90): end, last = time.time() + timeout, None while time.time() < end: st, att = http("GET", f"/api/v1/attachments/{aid}", token) last = att if st == 200 and att["status"] == want: return att time.sleep(1) raise AssertionError(f"attachment {aid} did not reach {want}; last={last}") def wait_ready_api(timeout=90): end = time.time() + timeout while time.time() < end: if http("GET", "/health/ready")[0] == 200: return time.sleep(1) raise AssertionError("API not ready") def check_variants(att): v = {x["name"]: x for x in att["variants"]} assert (v["thumbnail"]["width"], v["thumbnail"]["height"], v["thumbnail"]["content_type"]) == (256, 160, "image/jpeg"), v["thumbnail"] assert (v["crop"]["width"], v["crop"]["height"]) == (128, 128), v["crop"] assert v["optimized"]["content_type"] == "image/webp" def events(aid): return sql(f"SELECT string_agg(event, ',' ORDER BY id) FROM job_events e JOIN attachment_jobs j USING (job_id) WHERE j.attachment_id='{aid}'") def worker_down(): token, chat = setup() sh(*DC, "stop", "worker") aid = upload(token, chat) st, _ = http("GET", f"/api/v1/attachments/{aid}/content", token) assert st == 409, st # content of a queued attachment st, msg = http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "hi", "attachment_ids": [aid]}, {"Idempotency-Key": "k1"}) assert st == 201, (st, msg) # other features keep working; queued attachment can be attached sh(*DC, "restart", "api"); wait_ready_api() assert http("GET", f"/api/v1/attachments/{aid}", token)[1]["status"] == "queued" # accepted work survived the API restart sh(*DC, "up", "-d", "worker") att = wait(token, aid, "ready"); check_variants(att) print("PASS worker_down:", aid, "events:", events(aid)) def commit_publish_gap(): token, chat = setup() sh(*DC, "up", "-d", "dispatcher", env={"FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH": "1"}, check=True) sh(*DC, "exec", "-T", "dispatcher", "rm", "-f", "/tmp/dispatcher_fault_done", check=False) sh(*DC, "restart", "dispatcher") # fresh container process: the fault fires once, on the first outbox batch time.sleep(3) aid = upload(token, chat) # COMMIT happens; the dispatcher dies before publishing att = wait(token, aid, "ready", 90); check_variants(att) sh(*DC, "up", "-d", "dispatcher") # back to the normal configuration (.env: fault off) print("PASS commit_publish_gap:", aid, "events:", events(aid)) def crash_after_s3(): token, chat = setup() env = {"FAULT_WORKER_EXIT_AFTER_S3": "1", "JOB_TIMEOUT_SECONDS": "10", "LEASE_GRACE_SECONDS": "5"} sh(*DC, "up", "-d", "worker", env=env) time.sleep(5) aid = upload(token, chat) att = wait(token, aid, "ready", 150); check_variants(att) n = sql(f"SELECT count(*) FROM attachment_variants WHERE attachment_id='{aid}'") assert n == "3", n # the retry completed the SAME result, no second set of variants sh(*DC, "up", "-d", "worker") # restore normal worker config print("PASS crash_after_s3:", aid, "events:", events(aid)) def replay(): token, chat = setup() sh(*DC, "stop", "worker") aid = upload(token, chat) good = sql(f"SELECT sha256 FROM attachments WHERE id='{aid}'") sql(f"UPDATE attachments SET sha256=repeat('0',64) WHERE id='{aid}'") # simulate corrupted/tampered staging bytes sh(*DC, "up", "-d", "worker") att = wait(token, aid, "failed", 60) assert att["error_code"] == "integrity_error", att # permanent error: failed at once, no retries st, _ = http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "", "attachment_ids": [aid]}, {"Idempotency-Key": "k2"}) assert st == 409, st sql(f"UPDATE attachments SET sha256='{good}' WHERE id='{aid}'") # operator fixes the cause job = sql(f"SELECT job_id FROM attachment_jobs WHERE attachment_id='{aid}'") print(sh(*DC, "exec", "-T", "worker", "python", "scripts/replay_failed.py", "--job-id", job, "--reason", "student-test")) att = wait(token, aid, "ready", 60); check_variants(att) again = sh(*DC, "exec", "-T", "worker", "python", "scripts/replay_failed.py", "--job-id", job, check=False) assert "REFUSED" in again, again # replay of a successful job is refused print("PASS replay:", aid, "events:", events(aid)) def s3_outage(): token, chat = setup() sh(*DC, "stop", "worker") aid = upload(token, chat) sh(*DC, "stop", "s3") sh(*DC, "up", "-d", "--no-deps", "worker") time.sleep(15) ev = events(aid) assert "retry_scheduled" in ev, ev # temporary error => backoff with jitter, not failed sh(*DC, "start", "s3"); wait_ready_api() att = wait(token, aid, "ready", 120); check_variants(att) print("PASS s3_outage:", aid, "events:", events(aid)) SCENARIOS = {"worker_down": worker_down, "commit_publish_gap": commit_publish_gap, "crash_after_s3": crash_after_s3, "replay": replay, "s3_outage": s3_outage} if __name__ == "__main__": which = sys.argv[1] if len(sys.argv) > 1 else "all" for name, fn in SCENARIOS.items(): if which in ("all", name): fn()