From ab88c5ba1b2c76c46aeea32e81fbc8f1c09065e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:44:24 +0300 Subject: [PATCH] Prove versioned pipeline recovery and audited replay (U4) --- evidence/pipeline-faults-2026-10-02.md | 36 +++++ migrations/007_replay_audit.sql | 16 +++ src/pipeline_jobs.py | 2 +- src/replay.py | 64 +++++++++ src/reprocess.py | 57 ++++++++ src/worker.py | 10 ++ student-tests/test_pipeline_faults.py | 188 +++++++++++++++++++++++++ 7 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 evidence/pipeline-faults-2026-10-02.md create mode 100644 migrations/007_replay_audit.sql create mode 100644 src/replay.py create mode 100644 src/reprocess.py create mode 100644 student-tests/test_pipeline_faults.py diff --git a/evidence/pipeline-faults-2026-10-02.md b/evidence/pipeline-faults-2026-10-02.md new file mode 100644 index 0000000..26743bf --- /dev/null +++ b/evidence/pipeline-faults-2026-10-02.md @@ -0,0 +1,36 @@ +# Versioned pipeline fault injection — 2026-10-02 + +## Hypothesis and method + +A process death before or after an S3 side effect, a duplicate delivery, an expired lease, and a reordered older-version delivery should leave one logical variant set per `(attachment_id, pipeline_version, name)`. A failed current job should replay only by an explicit audited operator action. RabbitMQ remains at-least-once: duplicate messages are expected; the durable PostgreSQL state and deterministic S3 keys make their effects idempotent. + +`student-tests/test_pipeline_faults.py` creates synthetic 320×200 PNG uploads in the same PostgreSQL/MinIO stand as the application. It runs the real `src.worker.process_message` in child processes with `LAB5_FAULT_POINT` and `LAB5_FAULT_JOB_ID`; `os._exit(72)` terminates the child at the selected durable boundary. The parent checks database state, expires a simulated dead worker lease, runs recovery, redelivers the same logical job with the next managed attempt, and compares variant rows/checksums. The normal worker and publisher are stopped while injecting faults so their consumers cannot race the test. The test also forces a bounded temporary failure, verifies terminal `failed`, then uses the audited replay operation. No broker message is fabricated by the test; separate handoff/outbox tests exercise publication and confirm. + +Fault points in worker code: `after_claim_before_s3`, `after_s3_before_result_commit`, `after_result_commit_before_ack`. Each is disabled unless both environment variables name a point and exact synthetic job ID. The test runs once on the existing educational volumes and once on a fresh, isolated Compose project. The isolated project had zero `attachment_jobs` and zero `attachment_job_replays` before the run; its volumes were removed afterward. Main lab5 volumes were preserved. + +## Reproduce + +```sh +# Existing local stand, with worker and publisher stopped: +docker compose --env-file .env stop worker publisher +docker compose --env-file .env run --rm --build --no-deps -T worker python - < student-tests/test_pipeline_faults.py + +# Fresh, isolated PostgreSQL and S3 volumes; no API port and no normal consumer: +docker compose -p mtusi2026lab5fresh --env-file .env up -d --build postgres migrate s3-bootstrap +docker compose -p mtusi2026lab5fresh --env-file .env run --rm --build --no-deps -T worker python - < student-tests/test_pipeline_faults.py +docker compose -p mtusi2026lab5fresh --env-file .env down -v +``` + +On this host Docker builds used the ignored `compose.local.yaml` DNS override via `COMPOSE_FILE=compose.yaml:compose.local.yaml`; it does not alter application behavior. The first attempt to use `up --wait` on the isolated project exited 1 because Docker Compose treated the completed one-shot `migrate` service as not running; inspection showed `migrate` exited 0, PostgreSQL/S3 were healthy, and both relevant tables contained zero rows before the fault test. + +## Observed result + +Both runs returned the same anonymized raw summary (one JSON line per run): + +```json +{"after_result_commit_before_ack_duplicate":"pass","after_s3_before_result_commit":"pass","before_side_effect_lease_recovery":"pass","bounded_transient_retry_then_replay":"pass","duplicate_reorder_new_version":"pass","failed_registry_operator_replay":"pass"} +``` + +For each ready version, the test found exactly the four expected variant names (`original`, `thumbnail`, `crop`, `optimized`), a matching original SHA-256, and the expected current `pipeline_version`. Replaying an already ready job returned `duplicate_or_stale`. After a second version became ready, an older delivery left its checksums and current version unchanged; version 1 rows remained available as history. The failed-job replay wrote one audit row and used a new attempt without deleting prior outbox history. The transient retry test exhausted a two-attempt budget, recorded `failed`, then succeeded only after replay reset that budget. + +The controlled process death checks durable boundaries, not the probability of such a crash under load. The local stand has one MinIO and one PostgreSQL instance, so this result does not prove service availability during their outage or exactly-once delivery. The broker outage/confirm boundary is tested separately; this matrix checks the worker effect boundary against real S3 and PostgreSQL. diff --git a/migrations/007_replay_audit.sql b/migrations/007_replay_audit.sql new file mode 100644 index 0000000..cbe4c37 --- /dev/null +++ b/migrations/007_replay_audit.sql @@ -0,0 +1,16 @@ +-- A replay starts a new bounded run without deleting earlier outbox history. +ALTER TABLE attachment_jobs + ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1 + CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt); + +CREATE TABLE attachment_job_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + previous_attempt integer NOT NULL CHECK (previous_attempt > 0), + new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt), + previous_error_code text NOT NULL, + operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80), + reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, new_attempt) +); diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py index 5fa477f..0e6f8fe 100644 --- a/src/pipeline_jobs.py +++ b/src/pipeline_jobs.py @@ -149,7 +149,7 @@ def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): if max_attempts < 1 or delay_seconds < 0: raise ValueError("invalid retry configuration") - failed = permanent or job["attempt"] >= max_attempts + failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts if failed: code = error_code if permanent else "retry_exhausted" conn.execute( diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..96e7596 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,64 @@ +"""Explicit, audited replay of one failed current-version job.""" + +import argparse +from uuid import UUID + +from .db import pool + + +def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int: + if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200: + raise ValueError("operator and reason are required") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.status AS attachment_status," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j,a", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "failed" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attachment_status"] != "failed"): + raise ValueError("only a failed current-version job can be replayed") + attempt = job["attempt"] + 1 + conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s," + "next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s", + (attempt, attempt, job_id), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)", + (job_id, attempt), + ) + conn.execute( + "INSERT INTO attachment_job_replays " + "(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) " + "VALUES (%s,%s,%s,%s,%s,%s)", + (job_id, job["attempt"], attempt, job["error_code"], operator, reason), + ) + return attempt + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_id", type=UUID) + parser.add_argument("--operator", required=True) + parser.add_argument("--reason", required=True) + args = parser.parse_args() + try: + with pool().connection() as conn: + attempt = replay_failed(conn, args.job_id, args.operator, args.reason) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued job {args.job_id} attempt {attempt}") + + +if __name__ == "__main__": + main() diff --git a/src/reprocess.py b/src/reprocess.py new file mode 100644 index 0000000..f7f4354 --- /dev/null +++ b/src/reprocess.py @@ -0,0 +1,57 @@ +"""Start a new pipeline version from a verified, already ready original.""" + +import argparse +from uuid import UUID, uuid4 + +from .db import pool + + +def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]: + with conn.transaction(): + row = conn.execute( + "SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,) + ).fetchone() + if row is None or row["status"] != "ready": + raise ValueError("only a ready attachment can start a new version") + original = conn.execute( + "SELECT object_key,size_bytes,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name='original'", + (attachment_id, row["pipeline_version"]), + ).fetchone() + if (original is None or original["object_key"] != row["object_key"] or + original["size_bytes"] != row["size_bytes"] or + original["sha256"] != row["sha256"]): + raise ValueError("verified original is missing") + version = row["pipeline_version"] + 1 + job_id = uuid4() + conn.execute( + "UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL," + "updated_at=clock_timestamp() WHERE id=%s", + (version, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) " + "VALUES (%s,%s,%s)", + (job_id, attachment_id, version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + return version, job_id + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + try: + with pool().connection() as conn: + version, job_id = start_new_version(conn, args.attachment_id) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued attachment {args.attachment_id} version {version} job {job_id}") + + +if __name__ == "__main__": + main() diff --git a/src/worker.py b/src/worker.py index b0c2f7f..f96f7a5 100644 --- a/src/worker.py +++ b/src/worker.py @@ -104,6 +104,13 @@ def _delay(attempt: int) -> float: return base + random.uniform(0, base / 2) +def _fault(point: str, job_id) -> None: + """Test-only process death at a named durable boundary for one job.""" + if (os.environ.get("LAB5_FAULT_POINT") == point and + os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)): + os._exit(72) + + @contextmanager def _deadline(seconds: int): def expired(_signal, _frame): @@ -124,13 +131,16 @@ def process_message(item: dict) -> str: if job is None: return "duplicate_or_stale" token = job["lease_token"] + _fault("after_claim_before_s3", job["id"]) try: with _deadline(LEASE_SECONDS): data = _source_bytes(job) variants = render(data, job["content_type"]) stored = _write_variants(job, variants) + _fault("after_s3_before_result_commit", job["id"]) with pool().connection() as conn: current = complete_job(conn, job["id"], token, stored) + _fault("after_result_commit_before_ack", job["id"]) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..63a8e8e --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Deterministic fault windows for the versioned attachment pipeline. + +Run in a one-shot worker image while the normal worker and publisher are stopped. +All rows and objects are synthetic and stay in the educational volumes. +""" + +import hashlib +import json +import os +import subprocess +import sys +import time +from io import BytesIO +from uuid import uuid4 + +from PIL import Image + +from src.attachments import BUCKET, storage +from src.db import pool +from src.pipeline_jobs import ( + claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload, +) +from src.replay import replay_failed +from src.reprocess import start_new_version +from src.worker import process_message + + +image = BytesIO() +Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG") +DATA = image.getvalue() +SHA = hashlib.sha256(DATA).hexdigest() + + +def create_case(): + user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4() + staging_key = f"staging/{attachment_id}" + with pool().connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)", + (user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"), + ) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture")) + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", + (chat_id, user_id), + ) + reserve_upload(conn, attachment_id, staging_key) + storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png") + with pool().connection() as conn: + _, job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id, + filename="fixture.png", staging_key=staging_key, + object_key=f"originals/{attachment_id}", content_type="image/png", + size_bytes=len(DATA), sha256=SHA, + ) + return attachment_id, job_id + + +def item(job_id, attempt=1): + return {"job_id": str(job_id), "attempt": attempt} + + +def state(attachment_id, version): + with pool().connection() as conn: + attachment = conn.execute( + "SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,) + ).fetchone() + rows = conn.execute( + "SELECT name,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name", + (attachment_id, version), + ).fetchall() + return attachment, {row["name"]: row["sha256"] for row in rows} + + +def assert_ready(attachment_id, version): + attachment, variants = state(attachment_id, version) + assert attachment["status"] == "ready" and attachment["pipeline_version"] == version + assert attachment["sha256"] == SHA + assert set(variants) == {"original", "thumbnail", "crop", "optimized"} + assert variants["original"] == SHA + return variants + + +def expire_and_recover(job_id): + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND status='processing'", + (job_id,), + ) + recovered = recover_expired(conn, max_attempts=5) + assert (job_id, "queued") in recovered + + +def crash(job_id, point): + child = subprocess.run( + [sys.executable, "-c", "from src.worker import process_message; " + f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"], + env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)}, + capture_output=True, text=True, + ) + assert child.returncode == 72, (point, child.returncode, child.stderr) + + +def main(): + results = {} + + attachment, job = create_case() + crash(job, "after_result_commit_before_ack") + first = assert_ready(attachment, 1) + assert process_message(item(job)) == "duplicate_or_stale" + assert assert_ready(attachment, 1) == first + results["after_result_commit_before_ack_duplicate"] = "pass" + + attachment, job = create_case() + crash(job, "after_claim_before_s3") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["before_side_effect_lease_recovery"] = "pass" + + attachment, job = create_case() + crash(job, "after_s3_before_result_commit") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["after_s3_before_result_commit"] = "pass" + + attachment, old_job = create_case() + assert process_message(item(old_job)) == "ready" + with pool().connection() as conn: + version, new_job = start_new_version(conn, attachment) + assert version == 2 + assert process_message(item(old_job)) == "duplicate_or_stale" + assert process_message(item(new_job)) == "ready" + newer = assert_ready(attachment, 2) + assert process_message(item(old_job)) == "duplicate_or_stale" + assert assert_ready(attachment, 2) == newer + assert len(state(attachment, 1)[1]) == 4 + results["duplicate_reorder_new_version"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=5, + error_code="invalid_image", permanent=True, + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay") + assert attempt == 2 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + with pool().connection() as conn: + count = conn.execute( + "SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,) + ).fetchone()["n"] + assert count == 1 + results["failed_registry_operator_replay"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + delay_seconds=0.01, error_code="storage_unavailable", + ) == "queued" + time.sleep(0.03) + with pool().connection() as conn: + claimed = claim_job(conn, job, 2) + assert claimed is not None + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + error_code="storage_unavailable", + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair") + assert attempt == 3 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + results["bounded_transient_retry_then_replay"] = "pass" + + print(json.dumps(results, sort_keys=True)) + + +if __name__ == "__main__": + main()