#!/usr/bin/env python3 """Drive the lab5 grade-4 failure scenarios and write sanitized evidence. This is an operator script for the local Compose stand, not part of the public black-box test suite. It deliberately restarts the worker, RabbitMQ and the API replicas, so run it only against the disposable lab environment. Scenarios: 1. worker stopped -> 202 queued -> message link -> worker start -> ready 2. RabbitMQ down -> commit/outbox gap -> broker start -> ready 3. crash after durable effect, before ack -> redelivery -> same result 4. transient failure -> jittered retry -> ready on attempt 2 5. permanent failure -> failed registry -> explicit replay -> ready 6. crash after claim -> lease expiry -> sweeper requeue -> ready """ from __future__ import annotations import argparse import hashlib import json import os import re import shlex import subprocess import sys import time import uuid from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check # noqa: E402 COMPOSE = ["docker", "compose", "--env-file", ".env"] WORKER = "mtusi2026lab-worker-1" def run(cmd, *, env=None, timeout=180, check_code=True): merged = os.environ.copy() if env: merged.update(env) proc = subprocess.run( cmd, cwd=ROOT, env=merged, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout, ) if check_code and proc.returncode != 0: raise RuntimeError( f"command failed ({proc.returncode}): {' '.join(shlex.quote(x) for x in cmd)}\n" f"stdout:\n{proc.stdout[-4000:]}\nstderr:\n{proc.stderr[-4000:]}" ) return proc def compose(*args, env=None, timeout=180, check_code=True): return run(COMPOSE + list(args), env=env, timeout=timeout, check_code=check_code) def compose_capture(*args, env=None, timeout=180): return compose(*args, env=env, timeout=timeout).stdout def psql(sql): command = ( 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" -v ON_ERROR_STOP=1 -c ' + shlex.quote(sql) ) out = compose_capture("exec", "-T", "postgres", "sh", "-c", command) return [line for line in out.splitlines() if line.strip()] def parse_ts(value): value = value.strip() if value.endswith("+00"): value += ":00" return datetime.fromisoformat(value.replace(" ", "T")) def inspect(container_id): try: out = run(["docker", "inspect", container_id]).stdout return json.loads(out)[0] except (RuntimeError, json.JSONDecodeError): return {} def service_ids(service): ids = compose_capture("ps", "-q", service).split() return [item for item in ids if item] def wait_healthy(service, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: ids = service_ids(service) if ids: states = [inspect(item) for item in ids] if all( state.get("State", {}).get("Health", {}).get("Status") == "healthy" for state in states ): return ids time.sleep(1) raise Failure(f"{service} did not become healthy within {timeout}s") def wait_stopped(service, timeout=30): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if not service_ids(service): return time.sleep(0.5) raise Failure(f"{service} did not stop within {timeout}s") def recreate_worker(extra_env=None): compose( "up", "-d", "--no-build", "--force-recreate", "worker", env=extra_env or {}, timeout=240, ) wait_healthy("worker") def recreate_api(extra_env=None): compose( "up", "-d", "--no-build", "--force-recreate", "api", env=extra_env or {}, timeout=240, ) wait_healthy("api") # HAProxy resolves api through Docker DNS; give its health checks a moment # to converge after both replica addresses change. time.sleep(3) def restore_worker(): recreate_worker() def restore_api(): recreate_api() def job_row(attachment_id): rows = psql( "SELECT j.job_id, j.status, j.attempt, j.error_code, j.created_at, " "j.processing_started_at, j.finished_at, j.published_at, j.replay_count, " "a.staging_key, a.object_key " "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id " f"WHERE j.attachment_id = '{attachment_id}'" ) if len(rows) != 1: raise Failure(f"expected one job row for {attachment_id}, got {len(rows)}") parts = rows[0].split("|") return { "job_id": parts[0], "status": parts[1], "attempt": int(parts[2]), "error_code": None if parts[3] == "" else parts[3], "created_at": parse_ts(parts[4]), "processing_started_at": None if parts[5] == "" else parse_ts(parts[5]), "finished_at": None if parts[6] == "" else parse_ts(parts[6]), "published_at": None if parts[7] == "" else parse_ts(parts[7]), "replay_count": int(parts[8]), "staging_key": parts[9], "object_key": parts[10], } def wait_job_status(attachment_id, wanted, timeout=90): deadline = time.monotonic() + timeout last = None while time.monotonic() < deadline: last = job_row(attachment_id) if last["status"] == wanted: return last if wanted != "failed" and last["status"] == "failed": raise Failure(f"job {attachment_id} failed unexpectedly: {last['error_code']}") time.sleep(0.4) raise Failure(f"job {attachment_id} did not reach {wanted}; last={last}") def s3_variant_keys(prefix): script = ( 'export HOME=/tmp; ' 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; ' 'rc ls -r --json --no-progress "app/$S3_BUCKET/$PREFIX"' ) proc = compose( "run", "--rm", "--no-deps", "-e", f"PREFIX={prefix}", "--entrypoint", "sh", "s3-init", "-c", script, check_code=False, timeout=120, ) if proc.returncode != 0: raise Failure(f"rc ls failed: {proc.stderr[-1000:]}") payload = json.loads(proc.stdout) return sorted(item["key"] for item in payload.get("items", [])) def s3_object_exists(key): script = ( 'export HOME=/tmp; ' 'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; ' 'rc head --quiet "app/$S3_BUCKET/$KEY" >/dev/null' ) proc = compose( "run", "--rm", "--no-deps", "-e", f"KEY={key}", "--entrypoint", "sh", "s3-init", "-c", script, check_code=False, timeout=120, ) return proc.returncode == 0 def jpeg_size(data): if not data.startswith(b"\xff\xd8"): raise Failure("variant is not a JPEG") index = 2 while index + 9 < len(data): if data[index] != 0xFF: index += 1 continue marker = data[index + 1] index += 2 if marker in (0xD8, 0xD9): continue length = int.from_bytes(data[index : index + 2], "big") if marker in (0xC0, 0xC1, 0xC2): height = int.from_bytes(data[index + 3 : index + 5], "big") width = int.from_bytes(data[index + 5 : index + 7], "big") return width, height index += length raise Failure("JPEG dimensions not found") def png_size(data): if not data.startswith(b"\x89PNG\r\n\x1a\n"): raise Failure("variant is not a PNG") width = int.from_bytes(data[16:20], "big") height = int.from_bytes(data[20:24], "big") return width, height def verify_ready_attachment(client, actor, attachment, original): check(attachment["status"] == "ready", "attachment is not ready") check(attachment["error_code"] is None, "ready attachment has an error_code") variants = {item["name"]: item for item in attachment["variants"]} check(set(variants) == {"thumbnail", "crop", "optimized"}, "wrong variant set") check( (variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160), "thumbnail is not 256x160 for the 320x200 fixture", ) check( (variants["crop"]["width"], variants["crop"]["height"]) == (128, 128), "crop is not 128x128", ) check( (variants["optimized"]["width"], variants["optimized"]["height"]) == (320, 200), "optimized variant upscaled the small fixture", ) downloaded = client.call( "GET", f"/api/v1/attachments/{attachment['id']}/content", actor=actor, )["bytes"] check(downloaded == original, "original download differs from the upload") check( hashlib.sha256(downloaded).hexdigest() == attachment["sha256"], "original download checksum differs", ) checks = { "original": (downloaded, "image/png", png_size), "thumbnail": ( client.call( "GET", f"/api/v1/attachments/{attachment['id']}/content?variant=thumbnail", actor=actor, )["bytes"], "image/jpeg", jpeg_size, ), "crop": ( client.call( "GET", f"/api/v1/attachments/{attachment['id']}/content?variant=crop", actor=actor, )["bytes"], "image/jpeg", jpeg_size, ), "optimized": ( client.call( "GET", f"/api/v1/attachments/{attachment['id']}/content?variant=optimized", actor=actor, )["bytes"], "image/webp", None, ), } observed = {} for name, (data, media_type, size_parser) in checks.items(): if media_type == "image/jpeg": check(size_parser(data) == ( variants[name]["width"], variants[name]["height"], ), f"{name} bytes do not match metadata dimensions") if media_type == "image/webp": check(data.startswith(b"RIFF") and data[8:12] == b"WEBP", "optimized is not WebP") if name != "original": check( hashlib.sha256(data).hexdigest() == variants[name]["sha256"], f"{name} checksum differs", ) check(len(data) == variants[name]["size_bytes"], f"{name} size differs") observed[name] = { "content_type": media_type, "size_bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(), "width": variants[name]["width"] if name != "original" else 320, "height": variants[name]["height"] if name != "original" else 200, } return observed def upload(client, actor, chat, content, name): return client.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}", 202, actor=actor, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment", )["json"] def attach_message(client, actor, chat, attachment_id, expected=201): return client.call( "POST", f"/api/v1/chats/{chat['id']}/messages", expected, actor=actor, body={"text": "", "attachment_ids": [attachment_id]}, headers={"Idempotency-Key": uuid.uuid4().hex}, ) def worker_logs(): return compose_capture("logs", "--no-color", "--tail=160", "worker") def relevant_logs(text, tokens): lines = [] for line in text.splitlines(): if any(token and token in line for token in tokens): lines.append(line) return lines def scenario_worker_stopped(client, actor, chat, content, evidence): compose("stop", "worker") wait_stopped("worker") attachment = upload(client, actor, chat, content, "grade4-worker-stopped.png") check(attachment["status"] == "queued", "upload did not return queued") attach_message(client, actor, chat, attachment["id"]) content_status = client.call( "GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=actor, )["json"] check(content_status["error"]["code"] == "conflict", "queued content is not 409") compose("start", "worker") wait_healthy("worker") ready = client.wait_ready(actor, attachment, wait_seconds=60) observed = verify_ready_attachment(client, actor, ready, content) row = job_row(attachment["id"]) check(row["status"] == "ready" and row["staging_key"] == "", "ready row/staging cleanup mismatch") evidence["worker_stopped"] = { "attachment_id": attachment["id"], "queued_status": attachment["status"], "content_while_queued": 409, "ready_after_worker_start": True, "queue_seconds": round( (row["processing_started_at"] - row["created_at"]).total_seconds(), 3 ), "evidence": observed, "row": { "attempt": row["attempt"], "published": row["published_at"] is not None, "staging_cleared": row["staging_key"] == "", }, } return attachment["id"] def scenario_broker_outage(client, actor, chat, content, evidence): compose("stop", "rabbitmq") wait_stopped("rabbitmq") attachment = upload(client, actor, chat, content, "grade4-broker-down.png") row = job_row(attachment["id"]) check(row["status"] == "queued", "job is not queued during broker outage") check(row["published_at"] is None, "job unexpectedly published without broker") time.sleep(3) row_again = job_row(attachment["id"]) check(row_again["published_at"] is None, "outbox published while broker was down") healthy = client.call("GET", "/health/ready", schema="Health") check(healthy["json"]["status"] == "ok", "API readiness depends on RabbitMQ") compose("start", "rabbitmq") wait_healthy("rabbitmq") ready = client.wait_ready(actor, attachment, wait_seconds=90) verify_ready_attachment(client, actor, ready, content) row_ready = job_row(attachment["id"]) check(row_ready["published_at"] is not None, "job was not published after broker recovery") evidence["broker_outage"] = { "attachment_id": attachment["id"], "202_accepted_without_broker": attachment["status"] == "queued", "unpublished_during_outage": True, "api_ready_during_broker_outage": healthy["json"]["status"] == "ok", "published_after_recovery": True, "attempt": row_ready["attempt"], "queue_seconds": round( (row_ready["processing_started_at"] - row_ready["created_at"]).total_seconds(), 3 ), } return attachment["id"] def scenario_crash_after_effect(client, actor, chat, content, evidence): recreate_worker({"WORKER_CRASH_AFTER_EFFECT": "true"}) worker_id = service_ids("worker")[0] restarts_before = inspect(worker_id)["RestartCount"] attachment = upload(client, actor, chat, content, "grade4-crash-after-effect.png") ready = client.wait_ready(actor, attachment, wait_seconds=90) observed = verify_ready_attachment(client, actor, ready, content) deadline = time.monotonic() + 30 restarts_after = restarts_before while time.monotonic() < deadline and restarts_after <= restarts_before: time.sleep(0.5) ids = service_ids("worker") if not ids: continue restarts_after = inspect(ids[0]).get("RestartCount", restarts_after) row = job_row(attachment["id"]) keys = s3_variant_keys(row["object_key"]) check(len(keys) == 3, f"expected exactly three variant objects, got {len(keys)}") logs = relevant_logs( worker_logs(), [attachment["id"], row["job_id"], "fault injection", "attachment ready"], ) check(any("after durable effect before ack" in line for line in logs), "crash hook did not log") evidence["crash_after_effect"] = { "attachment_id": attachment["id"], "worker_restart_count_before": restarts_before, "worker_restart_count_after": restarts_after, "ready_after_redelivery": True, "variant_objects": len(keys), "evidence": observed, "log_excerpt": logs, } restore_worker() return attachment["id"] def scenario_transient_retry(client, actor, chat, content, evidence): recreate_worker( { "WORKER_FAULT_MODE": "transient-once", "JOB_RETRY_BASE_SECONDS": "2", "JOB_RETRY_MAX_SECONDS": "8", } ) attachment = upload(client, actor, chat, content, "grade4-transient-retry.png") ready = client.wait_ready(actor, attachment, wait_seconds=90) verify_ready_attachment(client, actor, ready, content) row = job_row(attachment["id"]) check(row["status"] == "ready", "transient retry did not finish ready") check(row["attempt"] >= 2, "transient retry did not use a second attempt") logs = relevant_logs( worker_logs(), [attachment["id"], row["job_id"], "transient job failure", "processing attachment"], ) check(any("transient job failure" in line for line in logs), "retry log missing") evidence["transient_retry"] = { "attachment_id": attachment["id"], "attempts": row["attempt"], "status": row["status"], "log_excerpt": logs, } restore_worker() return attachment["id"] def scenario_permanent_failure_and_replay(client, actor, chat, content, evidence): recreate_worker({"WORKER_FAULT_MODE": "permanent-once"}) attachment = upload(client, actor, chat, content, "grade4-permanent-failure.png") failed = wait_job_status(attachment["id"], "failed", timeout=45) check(failed["error_code"] == "invalid_image", "unexpected permanent error code") snapshot = client.call( "GET", f"/api/v1/attachments/{attachment['id']}", actor=actor, schema="Attachment", )["json"] check(snapshot["status"] == "failed", "attachment did not expose failed") client.call( "GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=actor, ) conflict = attach_message(client, actor, chat, attachment["id"], expected=409)["json"] check(conflict["error"]["code"] == "conflict", "failed attachment was attachable") failure_log = relevant_logs( worker_logs(), [attachment["id"], failed["job_id"], "permanent job failure", "invalid_image"], ) restore_worker() replay = compose( "exec", "-T", "worker", "/usr/local/bin/worker", "-replay", attachment["id"], check_code=False, ) check(replay.returncode == 0, f"replay command failed: {replay.stderr[-1000:]}") ready = client.wait_ready(actor, attachment, wait_seconds=90) observed = verify_ready_attachment(client, actor, ready, content) row = job_row(attachment["id"]) check(row["status"] == "ready" and row["replay_count"] >= 1, "replay did not finish ready") keys = s3_variant_keys(row["object_key"]) check(len(keys) == 3, f"replay created a second variant set: {keys}") check(s3_object_exists(row["object_key"]), "original object is missing after replay") after_replay_log = relevant_logs( worker_logs(), [attachment["id"], row["job_id"], "replay", "attachment ready"], ) logs = failure_log + after_replay_log evidence["permanent_failure_replay"] = { "attachment_id": attachment["id"], "failed_code": failed["error_code"], "failed_content_http": 409, "failed_message_link_http": 409, "replay_exit": replay.returncode, "replay_output": (replay.stdout + replay.stderr).strip().splitlines(), "replay_count": row["replay_count"], "variant_objects_after_replay": len(keys), "original_object_present": True, "evidence": observed, "log_excerpt": logs, } return attachment["id"] def scenario_lease_recovery(client, actor, chat, content, evidence): compose("stop", "worker") wait_stopped("worker") attachment = upload(client, actor, chat, content, "grade4-lease-recovery.png") # The one-shot worker claims the job, then exits before any side effect. # A short lease keeps the demonstration quick without changing the default. one_shot = compose( "run", "--rm", "--no-deps", "-e", "WORKER_CRASH_AFTER_CLAIM=true", "-e", "JOB_LEASE_SECONDS=10", "worker", check_code=False, timeout=120, ) check(one_shot.returncode == 10, f"crash-after-claim exit={one_shot.returncode}") claimed = job_row(attachment["id"]) check( claimed["status"] == "processing" and claimed["processing_started_at"] is not None, "job was not left in processing", ) recreate_worker({"JOB_LEASE_SECONDS": "10", "LEASE_SWEEP_SECONDS": "2"}) ready = client.wait_ready(actor, attachment, wait_seconds=90) observed = verify_ready_attachment(client, actor, ready, content) row = job_row(attachment["id"]) check(row["status"] == "ready", "lease recovery did not finish ready") logs = relevant_logs( worker_logs(), [attachment["id"], row["job_id"], "recovered expired", "processing attachment"], ) check(any("recovered expired job leases" in line for line in logs), "sweeper log missing") evidence["lease_recovery"] = { "attachment_id": attachment["id"], "one_shot_exit": one_shot.returncode, "processing_before_expiry": True, "attempts": row["attempt"], "ready_after_sweep": True, "evidence": observed, "log_excerpt": logs, } restore_worker() return attachment["id"] def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--out", type=Path, default=ROOT / "evidence/lab5-grade4.json") parser.add_argument("--markdown", type=Path, default=ROOT / "evidence/lab5-grade4.md") parser.add_argument( "--scenario", choices=( "all", "worker-stopped", "broker-outage", "crash-after-effect", "transient-retry", "permanent-failure-replay", "lease-recovery", ), default="all", ) args = parser.parse_args() content = (ROOT / "tests/fixtures/sample.png").read_bytes() client = Client(args.base_url) actor, _, _, chat = client.bootstrap() evidence = { "generated_at": datetime.now(timezone.utc).isoformat(), "fixture": { "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest(), "width": 320, "height": 200, }, "scenarios": {}, } selected = [ ("worker-stopped", scenario_worker_stopped), ("broker-outage", scenario_broker_outage), ("crash-after-effect", scenario_crash_after_effect), ("transient-retry", scenario_transient_retry), ("permanent-failure-replay", scenario_permanent_failure_and_replay), ("lease-recovery", scenario_lease_recovery), ] try: for name, function in selected: if args.scenario not in ("all", name): continue print(f"== {name}", flush=True) function(client, actor, chat, content, evidence["scenarios"]) print(f"== {name}: OK", flush=True) finally: restore_worker() restore_api() args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(json.dumps(evidence, indent=2) + "\n") lines = [ "# Lab5 grade-4 runtime evidence", "", f"Generated: `{evidence['generated_at']}`", "", "Synthetic fixture: 320x200 PNG, public contract fixture; no tokens or credentials are stored.", "", "| Scenario | Observation |", "| --- | --- |", ] s = evidence["scenarios"] if "worker_stopped" in s: lines.append( "| Worker stopped during upload | 202 queued, content 409, message linked while queued, " f"ready after worker start; queue {s['worker_stopped']['queue_seconds']} s, " f"attempt {s['worker_stopped']['row']['attempt']} |" ) if "broker_outage" in s: lines.append( "| Broker outage (COMMIT/publish gap) | 202 accepted with `published_at IS NULL`, " "API readiness stayed 200, job published and ready after RabbitMQ restart |" ) if "crash_after_effect" in s: lines.append( "| Crash after effect, before ack | worker restart count " f"{s['crash_after_effect']['worker_restart_count_before']} -> " f"{s['crash_after_effect']['worker_restart_count_after']}, " f"{s['crash_after_effect']['variant_objects']} deterministic variant objects, ready |" ) if "transient_retry" in s: lines.append( "| Transient error + jittered retry | failed once, attempt " f"{s['transient_retry']['attempts']}, ready |" ) if "permanent_failure_replay" in s: lines.append( "| Permanent error + explicit replay | failed `invalid_image`, content/link 409, " f"replay_count {s['permanent_failure_replay']['replay_count']}, ready with " f"{s['permanent_failure_replay']['variant_objects_after_replay']} variant objects |" ) if "lease_recovery" in s: lines.append( "| Crash after claim / expired lease | one-shot exit 10, processing, sweeper recovered, " f"attempt {s['lease_recovery']['attempts']}, ready |" ) lines.extend(["", "## Sanitized log excerpts", ""]) for name, payload in s.items(): excerpt = payload.get("log_excerpt") if not excerpt: continue lines.append(f"### {name}") lines.append("") lines.append("```text") lines.extend(line for line in excerpt[:12]) lines.append("```") lines.append("") args.markdown.write_text("\n".join(lines) + "\n") print(json.dumps({"ok": True, "out": str(args.out), "markdown": str(args.markdown)})) if __name__ == "__main__": try: main() except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc: print("FAIL:", exc) raise SystemExit(1)