diff --git a/migrations/007_replay_audit.sql b/migrations/007_replay_audit.sql new file mode 100644 index 0000000..cbe4c37 --- /dev/null +++ b/migrations/007_replay_audit.sql @@ -0,0 +1,16 @@ +-- A replay starts a new bounded run without deleting earlier outbox history. +ALTER TABLE attachment_jobs + ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1 + CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt); + +CREATE TABLE attachment_job_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + previous_attempt integer NOT NULL CHECK (previous_attempt > 0), + new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt), + previous_error_code text NOT NULL, + operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80), + reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, new_attempt) +); diff --git a/scripts/measure-image-pipeline.py b/scripts/measure-image-pipeline.py new file mode 100644 index 0000000..6b27caa --- /dev/null +++ b/scripts/measure-image-pipeline.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Small synthetic image workload; reports upload and time-to-ready separately.""" + +import argparse +import hashlib +import json +import math +import os +import struct +import sys +import time +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png_noise(width, height): + raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")) + + +def percentile(values, percentile_value): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * percentile_value) - 1)], 3) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--per-size", type=int, default=4) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100: + parser.error("invalid workload size") + + setup = Client(args.base_url) + alice, _bob, _eve, chat = setup.bootstrap() + images = { + "small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(), + "noise_800x600": png_noise(800, 600), + } + items = [(name, data) for name, data in images.items() for _ in range(args.per_size)] + + def work(item): + name, data = item + client = Client(args.base_url, timeout=20) + start = time.monotonic() + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}.png", + 202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + accepted = time.monotonic() + attachment = response["json"] + if attachment["sha256"] != hashlib.sha256(data).hexdigest(): + raise Failure("accepted checksum mismatch") + client.wait_ready(alice, attachment, wait_seconds=180) + ready = time.monotonic() + return {"profile": name, "size_bytes": len(data), + "upload_ms": round((accepted - start) * 1000, 3), + "ready_ms": round((ready - start) * 1000, 3)} + + started = time.monotonic() + completed, failures = [], [] + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + for future in as_completed([executor.submit(work, item) for item in items]): + try: + completed.append(future.result()) + except (Failure, OSError, ValueError) as exc: + failures.append(type(exc).__name__) + elapsed = time.monotonic() - started + result = { + "lab": 6, "workload": "synthetic PNG upload and poll until ready", + "concurrency": args.concurrency, "per_size": args.per_size, + "elapsed_seconds": round(elapsed, 3), "accepted_ready": len(completed), + "failed": len(failures), "failure_types": sorted(set(failures)), + "profiles": {}, + } + for name, data in images.items(): + rows = [item for item in completed if item["profile"] == name] + result["profiles"][name] = { + "size_bytes": len(data), "successful": len(rows), + "upload_ms": {f"p{p}": percentile([r["upload_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + "ready_ms": {f"p{p}": percentile([r["ready_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + } + Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + if failures or len(completed) != len(items): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py index 47b2342..4be6206 100644 --- a/src/pipeline_jobs.py +++ b/src/pipeline_jobs.py @@ -155,7 +155,7 @@ def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): if max_attempts < 1 or delay_seconds < 0: raise ValueError("invalid retry configuration") - failed = permanent or job["attempt"] >= max_attempts + failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts if failed: code = error_code if permanent else "retry_exhausted" conn.execute( diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..96e7596 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,64 @@ +"""Explicit, audited replay of one failed current-version job.""" + +import argparse +from uuid import UUID + +from .db import pool + + +def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int: + if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200: + raise ValueError("operator and reason are required") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.status AS attachment_status," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j,a", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "failed" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attachment_status"] != "failed"): + raise ValueError("only a failed current-version job can be replayed") + attempt = job["attempt"] + 1 + conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s," + "next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s", + (attempt, attempt, job_id), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)", + (job_id, attempt), + ) + conn.execute( + "INSERT INTO attachment_job_replays " + "(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) " + "VALUES (%s,%s,%s,%s,%s,%s)", + (job_id, job["attempt"], attempt, job["error_code"], operator, reason), + ) + return attempt + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_id", type=UUID) + parser.add_argument("--operator", required=True) + parser.add_argument("--reason", required=True) + args = parser.parse_args() + try: + with pool().connection() as conn: + attempt = replay_failed(conn, args.job_id, args.operator, args.reason) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued job {args.job_id} attempt {attempt}") + + +if __name__ == "__main__": + main() diff --git a/src/reprocess.py b/src/reprocess.py new file mode 100644 index 0000000..f7f4354 --- /dev/null +++ b/src/reprocess.py @@ -0,0 +1,57 @@ +"""Start a new pipeline version from a verified, already ready original.""" + +import argparse +from uuid import UUID, uuid4 + +from .db import pool + + +def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]: + with conn.transaction(): + row = conn.execute( + "SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,) + ).fetchone() + if row is None or row["status"] != "ready": + raise ValueError("only a ready attachment can start a new version") + original = conn.execute( + "SELECT object_key,size_bytes,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name='original'", + (attachment_id, row["pipeline_version"]), + ).fetchone() + if (original is None or original["object_key"] != row["object_key"] or + original["size_bytes"] != row["size_bytes"] or + original["sha256"] != row["sha256"]): + raise ValueError("verified original is missing") + version = row["pipeline_version"] + 1 + job_id = uuid4() + conn.execute( + "UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL," + "updated_at=clock_timestamp() WHERE id=%s", + (version, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) " + "VALUES (%s,%s,%s)", + (job_id, attachment_id, version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + return version, job_id + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + try: + with pool().connection() as conn: + version, job_id = start_new_version(conn, args.attachment_id) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued attachment {args.attachment_id} version {version} job {job_id}") + + +if __name__ == "__main__": + main() diff --git a/src/worker.py b/src/worker.py index 364b58b..ddcede4 100644 --- a/src/worker.py +++ b/src/worker.py @@ -110,6 +110,13 @@ def _delay(attempt: int) -> float: return base + random.uniform(0, base / 2) +def _fault(point: str, job_id) -> None: + """Test-only process death at a named durable boundary for one job.""" + if (os.environ.get("LAB5_FAULT_POINT") == point and + os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)): + os._exit(72) + + @contextmanager def _deadline(seconds: int): def expired(_signal, _frame): @@ -130,6 +137,7 @@ def process_message(item: dict) -> str: if job is None: return "duplicate_or_stale" token = job["lease_token"] + _fault("after_claim_before_s3", job["id"]) try: with _deadline(LEASE_SECONDS): with tracer.start_as_current_span("s3.read_source", record_exception=False, set_status_on_exception=False): @@ -138,9 +146,11 @@ def process_message(item: dict) -> str: variants = render(data, job["content_type"]) with tracer.start_as_current_span("s3.write_variants", record_exception=False, set_status_on_exception=False): stored = _write_variants(job, variants) + _fault("after_s3_before_result_commit", job["id"]) with pool().connection() as conn: with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False): current = complete_job(conn, job["id"], token, stored) + _fault("after_result_commit_before_ack", job["id"]) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) diff --git a/student-tests/publish_gap_inner.py b/student-tests/publish_gap_inner.py new file mode 100644 index 0000000..e1e0559 --- /dev/null +++ b/student-tests/publish_gap_inner.py @@ -0,0 +1,59 @@ +"""Run in a one-shot publisher container to inject publish-before-confirm gap.""" + +import json +import os +from pathlib import Path +from uuid import UUID, uuid4 + +import pika + +from src.db import pool +from src.publisher import QUEUE, job_payload, publish_batch + + +attachment_id = UUID(os.environ["FAULT_ATTACHMENT_ID"]) +url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() +broker = pika.BlockingConnection(pika.URLParameters(url)) +try: + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + token = uuid4() + with pool().connection() as conn: + row = conn.execute( + "SELECT o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at " + "FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id " + "WHERE j.attachment_id=%s AND j.pipeline_version=1 AND o.attempt=1", + (attachment_id,), + ).fetchone() + assert row is not None + claimed = conn.execute( + "UPDATE attachment_outbox SET claim_token=%s," + "claim_expires_at=clock_timestamp()+interval '30 seconds' " + "WHERE id=%s AND claim_token IS NULL AND published_at IS NULL RETURNING id", + (token, row["id"]), + ).fetchone() + assert claimed is not None + body = json.dumps(job_payload(row), separators=(",", ":")).encode() + channel.basic_publish( + exchange="", routing_key=QUEUE, mandatory=True, body=body, + properties=pika.BasicProperties(content_type="application/json", delivery_mode=2), + ) + # Simulate process death after broker confirm, before PostgreSQL confirm. + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_outbox SET claim_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL", + (row["id"], token), + ) + published = publish_batch(channel) + with pool().connection() as conn: + outbox = conn.execute( + "SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox " + "WHERE id=%s", (row["id"],), + ).fetchone() + count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count + assert published >= 1 and outbox["confirmed"] and count >= 2 + print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count})) +finally: + broker.close() diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..63a8e8e --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Deterministic fault windows for the versioned attachment pipeline. + +Run in a one-shot worker image while the normal worker and publisher are stopped. +All rows and objects are synthetic and stay in the educational volumes. +""" + +import hashlib +import json +import os +import subprocess +import sys +import time +from io import BytesIO +from uuid import uuid4 + +from PIL import Image + +from src.attachments import BUCKET, storage +from src.db import pool +from src.pipeline_jobs import ( + claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload, +) +from src.replay import replay_failed +from src.reprocess import start_new_version +from src.worker import process_message + + +image = BytesIO() +Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG") +DATA = image.getvalue() +SHA = hashlib.sha256(DATA).hexdigest() + + +def create_case(): + user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4() + staging_key = f"staging/{attachment_id}" + with pool().connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)", + (user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"), + ) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture")) + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", + (chat_id, user_id), + ) + reserve_upload(conn, attachment_id, staging_key) + storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png") + with pool().connection() as conn: + _, job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id, + filename="fixture.png", staging_key=staging_key, + object_key=f"originals/{attachment_id}", content_type="image/png", + size_bytes=len(DATA), sha256=SHA, + ) + return attachment_id, job_id + + +def item(job_id, attempt=1): + return {"job_id": str(job_id), "attempt": attempt} + + +def state(attachment_id, version): + with pool().connection() as conn: + attachment = conn.execute( + "SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,) + ).fetchone() + rows = conn.execute( + "SELECT name,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name", + (attachment_id, version), + ).fetchall() + return attachment, {row["name"]: row["sha256"] for row in rows} + + +def assert_ready(attachment_id, version): + attachment, variants = state(attachment_id, version) + assert attachment["status"] == "ready" and attachment["pipeline_version"] == version + assert attachment["sha256"] == SHA + assert set(variants) == {"original", "thumbnail", "crop", "optimized"} + assert variants["original"] == SHA + return variants + + +def expire_and_recover(job_id): + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND status='processing'", + (job_id,), + ) + recovered = recover_expired(conn, max_attempts=5) + assert (job_id, "queued") in recovered + + +def crash(job_id, point): + child = subprocess.run( + [sys.executable, "-c", "from src.worker import process_message; " + f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"], + env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)}, + capture_output=True, text=True, + ) + assert child.returncode == 72, (point, child.returncode, child.stderr) + + +def main(): + results = {} + + attachment, job = create_case() + crash(job, "after_result_commit_before_ack") + first = assert_ready(attachment, 1) + assert process_message(item(job)) == "duplicate_or_stale" + assert assert_ready(attachment, 1) == first + results["after_result_commit_before_ack_duplicate"] = "pass" + + attachment, job = create_case() + crash(job, "after_claim_before_s3") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["before_side_effect_lease_recovery"] = "pass" + + attachment, job = create_case() + crash(job, "after_s3_before_result_commit") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["after_s3_before_result_commit"] = "pass" + + attachment, old_job = create_case() + assert process_message(item(old_job)) == "ready" + with pool().connection() as conn: + version, new_job = start_new_version(conn, attachment) + assert version == 2 + assert process_message(item(old_job)) == "duplicate_or_stale" + assert process_message(item(new_job)) == "ready" + newer = assert_ready(attachment, 2) + assert process_message(item(old_job)) == "duplicate_or_stale" + assert assert_ready(attachment, 2) == newer + assert len(state(attachment, 1)[1]) == 4 + results["duplicate_reorder_new_version"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=5, + error_code="invalid_image", permanent=True, + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay") + assert attempt == 2 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + with pool().connection() as conn: + count = conn.execute( + "SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,) + ).fetchone()["n"] + assert count == 1 + results["failed_registry_operator_replay"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + delay_seconds=0.01, error_code="storage_unavailable", + ) == "queued" + time.sleep(0.03) + with pool().connection() as conn: + claimed = claim_job(conn, job, 2) + assert claimed is not None + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + error_code="storage_unavailable", + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair") + assert attempt == 3 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + results["bounded_transient_retry_then_replay"] = "pass" + + print(json.dumps(results, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_publish_confirm_gap.py b/student-tests/test_publish_confirm_gap.py new file mode 100644 index 0000000..1006997 --- /dev/null +++ b/student-tests/test_publish_confirm_gap.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Show that broker confirm without outbox confirmation safely republishes.""" + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def compose(*args, input_data=None): + return subprocess.run( + ["docker", "compose", *args], cwd=ROOT, + input=input_data, check=True, + ) + + +def main(): + args = arguments("Lab6 publish/confirm gap") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher") + attachment = None + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, _original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not queued") + source = (ROOT / "student-tests/publish_gap_inner.py").read_bytes() + compose( + "run", "--rm", "--no-deps", "-T", + "-e", f"FAULT_ATTACHMENT_ID={attachment['id']}", "publisher", "python", "-", + input_data=source, + ) + finally: + compose("up", "-d", "publisher", "worker") + ready = client.wait_ready(bob, attachment) + check(ready["status"] == "ready", "republished job did not finish") + check(len(ready["variants"]) == 3, "duplicate delivery changed variant set") + print("PASS: confirmed broker copy republished after lost DB confirm; worker reached ready once") + + +if __name__ == "__main__": + run(main)