{ "suite": "lab5-pipeline-idempotency", "started_at": "2026-10-29T00:21:40.224842Z", "config": { "base_url": "http://localhost:18085", "level": 3, "only": "", "expect_variants": "thumbnail", "max_attempts": 5, "stall_seconds": 15.0, "compose": "docker compose", "compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml" }, "invariants": [ { "id": "terminal", "statement": "every 202 reaches its expected terminal state", "result": "PASS", "details": { "accepted": 5 } }, { "id": "single_effect", "statement": "one variant set per (attachment_id, pipeline_version, variant)", "result": "PASS", "details": { "rows": 0 } }, { "id": "no_ready_rollback", "statement": "ready never returns to queued or processing", "result": "PASS", "details": { "traced": 5 } }, { "id": "monotonic_publication", "statement": "published_version only moves forward", "result": "PASS", "details": { "sampled": 2 } }, { "id": "bytes_match_metadata", "statement": "sha256, size and real dimensions match served bytes", "result": "PASS", "details": { "verified": 4 } }, { "id": "bounded_attempts", "statement": "attempt <= JOB_MAX_ATTEMPTS", "result": "PASS", "details": { "rows": 0 } }, { "id": "no_orphans", "statement": "no pending outbox, messages, dead letters, staging, objects", "result": "PASS", "details": { "queues": { "attachments.process": [ 0, 0 ], "attachments.process.dlq": [ 1, 0 ] }, "s3": "SKIP: --s3-list is not set" } }, { "id": "no_lost_202", "statement": "every 202 is present in PostgreSQL and the API", "result": "PASS", "details": { "accepted": 5 } } ], "scenarios": [ { "name": "smoke_pipeline", "level": 3, "window": "upload, ready, variants match metadata", "seconds": 1.3, "result": "PASS", "observed": { "attachment_id": "01a18508-c6f6-774b-ae60-24aa728ad62c", "variants": [ "crop", "optimized", "thumbnail" ] } }, { "name": "worker_down_broker_restart", "level": 3, "window": "worker down, API and broker restart", "seconds": 24.0, "result": "PASS", "observed": { "attachment_id": "01a18508-cd97-737c-bb03-5409de83ff8e", "timeline": [ { "at": "2026-10-29T00:21:49.075959Z", "step": "worker stopped" }, { "at": "2026-10-29T00:21:49.095967Z", "step": "upload 202 queued", "attachment_id": "01a18508-cd97-737c-bb03-5409de83ff8e" }, { "at": "2026-10-29T00:21:49.099982Z", "step": "content 409 while queued" }, { "at": "2026-10-29T00:21:49.138263Z", "step": "message with attachment 201, other messages work", "message_id": "01a18508-cdad-708f-8324-18123e9c7758" }, { "at": "2026-10-29T00:21:54.782528Z", "step": "job is in the broker queue", "queues": { "attachments.process": [ 1, 0 ], "attachments.process.dlq": [ 1, 0 ] } }, { "at": "2026-10-29T00:22:00.869187Z", "step": "API replicas and relay restarted" }, { "at": "2026-10-29T00:22:09.698869Z", "step": "broker restarted with its volume, job survived", "queues": { "attachments.process": [ 1, 0 ], "attachments.process.dlq": [ 1, 0 ] } }, { "at": "2026-10-29T00:22:12.427256Z", "step": "worker started" }, { "at": "2026-10-29T00:22:12.707560Z", "step": "ready", "variants": [ "crop", "optimized", "thumbnail" ] } ] } }, { "name": "duplicate_after_ready", "level": 3, "window": "redelivery of a finished job", "seconds": 7.4, "result": "PASS", "observed": { "attachment_id": "01a18509-29ee-7116-b3df-b04caf1c48e7", "events": [ { "kind": "enqueued", "attempt": 1 }, { "kind": "claimed", "attempt": 1 }, { "kind": "done", "attempt": 1 }, { "kind": "duplicate_noop", "attempt": 1 } ] } }, { "name": "poison_image", "level": 3, "window": "permanent error without retries", "seconds": 0.0, "result": "PASS", "observed": { "rejected_by_api": 415 } }, { "name": "s3_outage_recovers", "level": 3, "window": "transient error with retry", "seconds": 11.6, "result": "PASS", "observed": { "attachment_id": "01a18509-4892-72ec-8f31-ea07c55239c7", "jobs": [ { "job_id": "01a18509-4898-70c0-9198-8af96f5da78b", "pipeline_version": 1, "status": "done", "attempt": 2 } ] } }, { "name": "retry_exhausted", "level": 3, "window": "transient error until JOB_MAX_ATTEMPTS", "seconds": 23.4, "result": "PASS", "observed": { "attachment_id": "01a18509-8268-718e-912d-c6718d5492e0", "jobs": [ { "job_id": "01a18509-826e-7736-8d1e-ba66a970dc8e", "pipeline_version": 1, "status": "failed", "attempt": 5 } ] } } ], "result": "PASS", "finished_at": "2026-10-29T00:23:06.519678Z" }