#!/usr/bin/env python3 """Grade-4 failure scenarios for the lab5 attachment pipeline. Each scenario is explicit and reproducible: 1. worker-stopped accept: 202 while no worker, resume -> ready; 2. outbox gap: broker down at COMMIT -> job stays unpublished -> API restart -> broker returns -> ready (no new task needed); 3. retry/replay: repeated transient failures -> failed(retry_exhausted) -> operator replay after recovery -> ready; 4. redelivery of a ready job is a no-op (variants not duplicated); 5. lease recovery: a stuck 'processing' job is requeued by the watchdog. Run from the repository root (docker CLI + running lab5 stand). """ from __future__ import annotations import argparse import base64 import io import json import os import struct import subprocess import time import urllib.error import urllib.request import uuid import zlib BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") def _png_chunk(tag, data): return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) def make_png(width=320, height=200): raw = bytearray() pixels = os.urandom(width * height * 3) for row in range(height): raw.append(0) raw += pixels[row * width * 3 : (row + 1) * width * 3] ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) return b"\x89PNG\r\n\x1a\n" + _png_chunk(b"IHDR", ihdr) + _png_chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _png_chunk(b"IEND", b"") def http(method, path, token=None, body=None, raw=None, content_type=None): headers = {"Accept": "application/json"} data = raw if body is not None: data = json.dumps(body).encode() headers["Content-Type"] = "application/json" if content_type: headers["Content-Type"] = content_type if token: headers["Authorization"] = "Bearer " + token request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) try: with urllib.request.urlopen(request, timeout=20) as response: return response.status, response.read() except urllib.error.HTTPError as error: return error.code, error.read() except (urllib.error.URLError, TimeoutError, OSError): return 0, b"" def compose(*args, check=True): return subprocess.run(["docker", "compose", *args], check=check, capture_output=True, text=True) def psql(sql): result = subprocess.run( ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAc", sql], capture_output=True, text=True, ) return result.stdout.strip() def service_running(name: str) -> bool: result = subprocess.run( ["docker", "compose", "ps", "--status", "running", "--format", "{{.Service}}"], capture_output=True, text=True, ) return name in result.stdout.split() def storage_reachable() -> bool: # Raw TCP probe (bypasses the app's circuit breaker, which cannot be used to # decide whether the store is really down). result = subprocess.run( ["docker", "compose", "exec", "-T", "api", "python", "-c", "import socket,sys; socket.create_connection(('s3',8333),1); sys.exit(0)"], capture_output=True, text=True, timeout=30, ) return result.returncode == 0 def wait_storage_down(timeout: float = 60.0) -> None: deadline = time.time() + timeout while time.time() < deadline: if not storage_reachable(): return time.sleep(1) raise SystemExit("S3 still reachable; cannot run the failure scenario") def wait_ready(timeout=90.0): deadline = time.time() + timeout while time.time() < deadline: if http("GET", "/health/ready")[0] == 200: return time.sleep(1) raise SystemExit("stand did not become ready") def new_actor(): username = "fail_" + uuid.uuid4().hex[:8] password = "T3st-" + uuid.uuid4().hex http("POST", "/api/v1/users", body={"username": username, "display_name": "Fail", "password": password}) credential = base64.b64encode(f"{username}:{password}".encode()).decode() request = urllib.request.Request( BASE + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credential, "Accept": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=20) as response: token = json.loads(response.read())["token"] user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"] chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "fail", "member_ids": [user_id]})[1]) return token, chat["id"] def upload(token, chat_id, data=None): data = data if data is not None else make_png() status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=x.png", token=token, raw=data, content_type="image/png") assert status == 202, (status, body) return json.loads(body)["id"] def status_of(token, attachment_id): return json.loads(http("GET", f"/api/v1/attachments/{attachment_id}", token=token)[1]) def wait_status(token, attachment_id, wanted, timeout=120.0): deadline = time.time() + timeout last = None while time.time() < deadline: last = status_of(token, attachment_id) if last["status"] == wanted: return last time.sleep(0.5) raise SystemExit(f"attachment {attachment_id} did not reach {wanted}: {last}") def scenario_worker_stopped(log): log("\n== scenario 1: accept while worker stopped, resume -> ready ==") compose("stop", "worker") try: token, chat_id = new_actor() att_id = upload(token, chat_id) time.sleep(2) state = status_of(token, att_id) log(f"with worker stopped: status={state['status']} (expected queued)") assert state["status"] == "queued", state compose("restart", "api") wait_ready() log("API restarted; work still queued (durable)") assert status_of(token, att_id)["status"] == "queued" finally: compose("start", "worker") ready = wait_status(token, att_id, "ready") names = sorted(v["name"] for v in ready["variants"]) log(f"worker resumed: status=ready variants={names}") assert {"thumbnail", "crop", "optimized"} <= set(names) def scenario_outbox_gap(log): log("\n== scenario 2: COMMIT/publish gap (broker down), API restart, then broker ==") compose("stop", "broker") try: token, chat_id = new_actor() att_id = upload(token, chat_id) time.sleep(2) job = psql(f"select published_at from attachment_jobs where attachment_id='{att_id}'") log(f"broker down: status={status_of(token, att_id)['status']} published_at={job!r} (expected empty)") assert status_of(token, att_id)["status"] in {"queued", "processing"} compose("restart", "api") wait_ready() log("API restarted with broker still down; intent preserved in PostgreSQL") finally: compose("start", "broker") ready = wait_status(token, att_id, "ready", timeout=180) log(f"broker back: status={ready['status']} (outbox republished without new task)") def scenario_retry_replay(log): log("\n== scenario 3: transient failures -> failed(retry_exhausted) -> replay ==") compose("stop", "worker") while service_running("worker"): time.sleep(0.5) token, chat_id = new_actor() att_id = upload(token, chat_id) original_key = psql(f"select staging_key from attachments where id='{att_id}'") # Fault injection: the durable original is (temporarily) unavailable, so the # worker's fetch fails and is retried with backoff until attempts are spent. psql(f"update attachments set staging_key='staging/missing-{att_id}' where id='{att_id}'") log("injected: staging object points to a missing key (transient fetch failure)") compose("start", "worker") failed = wait_status(token, att_id, "failed", timeout=180) log(f"status=failed error_code={failed['error_code']} (expected retry_exhausted)") assert failed["error_code"] == "retry_exhausted", failed # Operator repairs the cause and replays the failed job explicitly. psql(f"update attachments set staging_key='{original_key}' where id='{att_id}'") job_id = att_id_job(att_id) replayed = subprocess.run( ["docker", "compose", "exec", "-T", "api", "python", "-m", "src.replay", job_id], capture_output=True, text=True, ).stdout.strip() log(f"explicit replay after recovery: {replayed}") ready = wait_status(token, att_id, "ready", timeout=120) log(f"after replay: status=ready variants={sorted(v['name'] for v in ready['variants'])}") def att_id_job(attachment_id): return psql(f"select job_id from attachment_jobs where attachment_id='{attachment_id}'") def scenario_redelivery(log): log("\n== scenario 4: redelivery of a ready job is a no-op ==") token, chat_id = new_actor() att_id = upload(token, chat_id) ready = wait_status(token, att_id, "ready") before = sorted((v["name"], v["sha256"]) for v in ready["variants"]) # Republish the finished job (duplicate delivery). psql(f"update attachment_jobs set status='pending', published_at=null, available_at=now() where attachment_id='{att_id}'") log("re-queued the same job (duplicate delivery)") time.sleep(6) after = status_of(token, att_id) after_variants = sorted((v["name"], v["sha256"]) for v in after["variants"]) log(f"status={after['status']} variants={[n for n,_ in after_variants]} (must be unchanged)") assert after["status"] == "ready" assert after_variants == before, "duplicate delivery changed variants" def scenario_lease_recovery(log): log("\n== scenario 5: stuck processing -> lease recovery ==") compose("stop", "worker") token, chat_id = new_actor() att_id = upload(token, chat_id) job_id = att_id_job(att_id) # Simulate a worker that died mid-processing with an already-expired lease. psql(f"update attachments set status='processing' where id='{att_id}'") psql(f"update attachment_jobs set status='processing', lease_until=now() - interval '5 seconds' where job_id='{job_id}'") log("marked attachment+job processing with an expired lease") compose("start", "worker") ready = wait_status(token, att_id, "ready", timeout=120) log(f"watchdog requeued and finished the job: status={ready['status']}") def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--only", choices=["1", "2", "3", "4", "5"], help="run one scenario") args = parser.parse_args() wait_ready() def log(message): print(message, flush=True) scenarios = { "1": scenario_worker_stopped, "2": scenario_outbox_gap, "3": scenario_retry_replay, "4": scenario_redelivery, "5": scenario_lease_recovery, } if args.only: scenarios[args.only](log) else: for name in ("1", "2", "3", "4", "5"): scenarios[name](log) print("\nPASS: lab5 grade-4 failure scenarios" + (f" (scenario {args.only})" if args.only else "")) if __name__ == "__main__": main()