{ "suite": "lab5-pipeline-idempotency", "started_at": "2026-10-29T00:23:06.625217Z", "config": { "base_url": "http://localhost:18085", "level": 4, "only": "", "expect_variants": "thumbnail,crop,optimized", "max_attempts": 5, "stall_seconds": 15.0, "compose": "docker compose", "compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml" }, "invariants": [ { "id": "terminal", "statement": "every 202 reaches its expected terminal state", "result": "PASS", "details": { "accepted": 33 } }, { "id": "single_effect", "statement": "one variant set per (attachment_id, pipeline_version, variant)", "result": "PASS", "details": { "rows": 0 } }, { "id": "no_ready_rollback", "statement": "ready never returns to queued or processing", "result": "PASS", "details": { "traced": 33 } }, { "id": "monotonic_publication", "statement": "published_version only moves forward", "result": "PASS", "details": { "sampled": 8 } }, { "id": "bytes_match_metadata", "statement": "sha256, size and real dimensions match served bytes", "result": "PASS", "details": { "verified": 28 } }, { "id": "bounded_attempts", "statement": "attempt <= JOB_MAX_ATTEMPTS", "result": "PASS", "details": { "rows": 0 } }, { "id": "no_orphans", "statement": "no pending outbox, messages, dead letters, staging, objects", "result": "PASS", "details": { "queues": { "attachments.process": [ 0, 0 ], "attachments.process.dlq": [ 2, 0 ] }, "s3": "SKIP: --s3-list is not set" } }, { "id": "no_lost_202", "statement": "every 202 is present in PostgreSQL and the API", "result": "PASS", "details": { "accepted": 33 } } ], "scenarios": [ { "name": "smoke_pipeline", "level": 3, "window": "upload, ready, variants match metadata", "seconds": 2.3, "result": "PASS", "observed": { "attachment_id": "01a1850a-2a86-7412-ad79-91e2d1035be6", "variants": [ "crop", "optimized", "thumbnail" ] } }, { "name": "worker_down_broker_restart", "level": 3, "window": "worker down, API and broker restart", "seconds": 22.9, "result": "PASS", "observed": { "attachment_id": "01a1850a-35f4-7738-ae88-adcd8221f85d", "timeline": [ { "at": "2026-10-29T00:23:21.327338Z", "step": "worker stopped" }, { "at": "2026-10-29T00:23:21.352082Z", "step": "upload 202 queued", "attachment_id": "01a1850a-35f4-7738-ae88-adcd8221f85d" }, { "at": "2026-10-29T00:23:21.358202Z", "step": "content 409 while queued" }, { "at": "2026-10-29T00:23:21.411091Z", "step": "message with attachment 201, other messages work", "message_id": "01a1850a-3612-709d-812a-314668261056" }, { "at": "2026-10-29T00:23:24.139715Z", "step": "job is in the broker queue", "queues": { "attachments.process": [ 1, 0 ], "attachments.process.dlq": [ 1, 0 ] } }, { "at": "2026-10-29T00:23:30.980552Z", "step": "API replicas and relay restarted" }, { "at": "2026-10-29T00:23:40.688506Z", "step": "broker restarted with its volume, job survived", "queues": { "attachments.process": [ 1, 0 ], "attachments.process.dlq": [ 1, 0 ] } }, { "at": "2026-10-29T00:23:43.386331Z", "step": "worker started" }, { "at": "2026-10-29T00:23:43.590658Z", "step": "ready", "variants": [ "crop", "optimized", "thumbnail" ] } ] } }, { "name": "duplicate_after_ready", "level": 3, "window": "redelivery of a finished job", "seconds": 5.2, "result": "PASS", "observed": { "attachment_id": "01a1850a-8cee-7426-8784-3914f1216d71", "events": [ { "kind": "enqueued", "attempt": 1 }, { "kind": "claimed", "attempt": 1 }, { "kind": "done", "attempt": 1 }, { "kind": "duplicate_noop", "attempt": 1 } ] } }, { "name": "poison_image", "level": 3, "window": "permanent error without retries", "seconds": 0.0, "result": "PASS", "observed": { "rejected_by_api": 415 } }, { "name": "s3_outage_recovers", "level": 3, "window": "transient error with retry", "seconds": 14.3, "result": "PASS", "observed": { "attachment_id": "01a1850a-a267-7089-8d90-45cb5c521022", "jobs": [ { "job_id": "01a1850a-a26f-7791-88a6-29a9fa3d3aa4", "pipeline_version": 1, "status": "done", "attempt": 2 } ] } }, { "name": "retry_exhausted", "level": 3, "window": "transient error until JOB_MAX_ATTEMPTS", "seconds": 20.4, "result": "PASS", "observed": { "attachment_id": "01a1850a-e8d8-7122-ab63-13e85b328efe", "jobs": [ { "job_id": "01a1850a-e911-75b4-a3aa-7fd609efefa6", "pipeline_version": 1, "status": "failed", "attempt": 5 } ] } }, { "name": "image_matrix", "level": 4, "window": "limits and variants", "seconds": 12.6, "result": "PASS", "observed": { "images": [ { "file": "png-rgb-320x200.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-rgba-300x300.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-palette-trns-200x100.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-gray-17x1000.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-gray-alpha-640x480.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-gray16-300x200.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-rgba16-64x64.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-1x1.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-wide-3000x2.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-small-50x30.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "jpeg-rgb-1000x750.jpg", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "jpeg-gray-801x601.jpg", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "jpeg-cmyk-400x300.jpg", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "jpeg-exif6-400x200.jpg", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "jpeg-exif3-300x200.jpg", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "jpeg-rgb-4000x3000.jpg", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-exactly-10mib.png", "http": 202, "status": "ready", "error_code": null, "variants": [ "crop", "optimized", "thumbnail" ] }, { "file": "png-over-10mib.png", "http": "rejected" }, { "file": "png-over-pixels-8001x5000.png", "http": "rejected" }, { "file": "jpeg-over-pixels-10000x5000.jpg", "http": "rejected" }, { "file": "png-truncated.png", "http": "rejected" }, { "file": "png-corrupt-idat.png", "http": "rejected" }, { "file": "jpeg-truncated.jpg", "http": "rejected" }, { "file": "png-zero-width.png", "http": "rejected" }, { "file": "not-an-image.png", "http": "rejected" }, { "file": "gif-declared-as-png.png", "http": "rejected" } ] } }, { "name": "crash_commit_publish", "level": 4, "window": "after COMMIT, before publish", "seconds": 17.5, "result": "PASS", "observed": { "attachment_id": "01a1850b-7437-7757-92f0-862b8e3e430c", "exit_code": 137, "after_crash": { "outbox_published": false, "status": "queued" }, "variants": [ { "pipeline_version": 1, "name": "crop", "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", "size_bytes": 1820, "object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/crop-97bf1400b30a28c1.jpg" }, { "pipeline_version": 1, "name": "optimized", "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", "size_bytes": 2462, "object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/optimized-1090d84ed8411226.webp" }, { "pipeline_version": 1, "name": "thumbnail", "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", "size_bytes": 4679, "object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/thumbnail-7bc40bf315c8e18e.jpg" } ] } }, { "name": "crash_publish_confirm", "level": 4, "window": "after publish, before confirm", "seconds": 15.7, "result": "PASS", "observed": { "attachment_id": "01a1850b-acca-750e-8e22-5af17f32c47d", "exit_code": 137, "after_crash": { "outbox_published": [ false ], "queues": { "attachments.process": [ 0, 0 ], "attachments.process.dlq": [ 1, 0 ] } }, "variants": [ { "pipeline_version": 1, "name": "crop", "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", "size_bytes": 1820, "object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/crop-97bf1400b30a28c1.jpg" }, { "pipeline_version": 1, "name": "optimized", "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", "size_bytes": 2462, "object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/optimized-1090d84ed8411226.webp" }, { "pipeline_version": 1, "name": "thumbnail", "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", "size_bytes": 4679, "object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/thumbnail-7bc40bf315c8e18e.jpg" } ], "events": [ "enqueued", "claimed", "done", "duplicate_noop" ] } }, { "name": "crash_after_s3_before_commit", "level": 4, "window": "after S3 write, before COMMIT", "seconds": 23.1, "result": "PASS", "observed": { "attachment_id": "01a1850b-f488-7792-be43-3371a5179dda", "exit_code": 137, "after_crash": { "status": "processing", "variant_rows": 0 }, "variants": [ { "pipeline_version": 1, "name": "crop", "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", "size_bytes": 1820, "object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/crop-97bf1400b30a28c1.jpg" }, { "pipeline_version": 1, "name": "optimized", "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", "size_bytes": 2462, "object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/optimized-1090d84ed8411226.webp" }, { "pipeline_version": 1, "name": "thumbnail", "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", "size_bytes": 4679, "object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/thumbnail-7bc40bf315c8e18e.jpg" } ] } }, { "name": "crash_after_commit_before_ack", "level": 4, "window": "after COMMIT, before ack", "seconds": 17.5, "result": "PASS", "observed": { "attachment_id": "01a1850c-4e58-7510-a23a-f8ddb2214d77", "exit_code": 137, "after_crash": { "status": "ready", "variants": [ { "pipeline_version": 1, "name": "crop", "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", "size_bytes": 1820, "object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/crop-97bf1400b30a28c1.jpg" }, { "pipeline_version": 1, "name": "optimized", "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", "size_bytes": 2462, "object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/optimized-1090d84ed8411226.webp" }, { "pipeline_version": 1, "name": "thumbnail", "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", "size_bytes": 4679, "object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/thumbnail-7bc40bf315c8e18e.jpg" } ], "queues": { "attachments.process": [ 0, 1 ], "attachments.process.dlq": [ 1, 0 ] } }, "variants": [ { "pipeline_version": 1, "name": "crop", "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", "size_bytes": 1820, "object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/crop-97bf1400b30a28c1.jpg" }, { "pipeline_version": 1, "name": "optimized", "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", "size_bytes": 2462, "object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/optimized-1090d84ed8411226.webp" }, { "pipeline_version": 1, "name": "thumbnail", "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", "size_bytes": 4679, "object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/thumbnail-7bc40bf315c8e18e.jpg" } ] } }, { "name": "lease_expiry_fencing", "level": 4, "window": "stalled worker, lease expiry", "seconds": 40.0, "result": "PASS", "observed": { "attachment_id": "01a1850c-936d-7519-bba4-0322f1a3c0b8", "jobs": [ { "job_id": "01a1850c-9377-74d0-a5d9-48ed6ed9d65c", "pipeline_version": 1, "status": "done", "attempt": 2 } ], "events": [ { "kind": "enqueued", "attempt": 1 }, { "kind": "claimed", "attempt": 1 }, { "kind": "lease_expired", "attempt": 2 }, { "kind": "claimed", "attempt": 2 }, { "kind": "done", "attempt": 2 }, { "kind": "fenced", "attempt": 1 } ] } }, { "name": "replay_failed", "level": 4, "window": "operator replay", "seconds": 44.6, "result": "PASS", "observed": { "attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322", "replay": { "result": "replayed", "attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322", "operator": "cli" }, "replay_of_ready": { "result": "noop", "attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322", "operator": "cli" }, "events": [ { "kind": "enqueued", "attempt": 1 }, { "kind": "claimed", "attempt": 1 }, { "kind": "retry_scheduled", "attempt": 2 }, { "kind": "claimed", "attempt": 2 }, { "kind": "retry_scheduled", "attempt": 3 }, { "kind": "claimed", "attempt": 3 }, { "kind": "retry_scheduled", "attempt": 4 }, { "kind": "claimed", "attempt": 4 }, { "kind": "retry_scheduled", "attempt": 5 }, { "kind": "claimed", "attempt": 5 }, { "kind": "failed", "attempt": 5 }, { "kind": "replayed", "attempt": 1 }, { "kind": "claimed", "attempt": 1 }, { "kind": "done", "attempt": 1 }, { "kind": "replay_noop", "attempt": 1 } ] } }, { "name": "malformed_message_dead_lettered", "level": 4, "window": "message that is not a job goes to the DLQ", "seconds": 7.3, "result": "PASS", "observed": { "dead_letters": 1 } }, { "name": "worker_limits", "level": 4, "window": "render time and memory limits, 40 Mpx in the worker limit", "seconds": 27.8, "result": "PASS", "observed": { "image": { "file": "png-at-pixel-limit-8000x5000.png", "bytes": 116714, "pixels": 40000000 }, "render_timeout": { "seconds": 0.5, "attachment_id": "01a1850e-0649-7516-bedf-76fcced46675", "error_code": "processing_failed", "worker": "running" }, "render_memory": { "mib": 200, "attachment_id": "01a1850e-2203-76ad-94a5-d96a067b88d6", "error_code": "processing_failed", "worker": "running" }, "default_limits": { "attachment_id": "01a1850e-3e3d-75c8-b892-9a107c43d4b6", "variants": [ "crop", "optimized", "thumbnail" ], "memory_peak_mib": 378.8, "memory_max_mib": 1024.0 } } }, { "name": "retry_backoff_jitter", "level": 4, "window": "backoff with equal jitter between attempts", "seconds": 35.2, "result": "PASS", "observed": { "attachment_id": "01a1850e-85dc-7779-9d26-df88e9a5fe8b", "delays": [ { "attempt": 2, "delay": 0.694765 }, { "attempt": 3, "delay": 1.738794 }, { "attempt": 4, "delay": 2.834139 }, { "attempt": 5, "delay": 7.550992 } ], "share_of_ceiling": [ 0.695, 0.869, 0.709, 0.944 ], "jobs": [ { "job_id": "01a1850e-8623-7161-af00-316420cb53db", "pipeline_version": 1, "status": "failed", "attempt": 5 } ] } }, { "name": "failed_retention", "level": 4, "window": "failed staging kept until retention, then replay expires", "seconds": 41.8, "result": "PASS", "observed": { "attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69", "staging_before": "kept", "replay": { "result": "expired", "attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69", "operator": "cli" } } } ], "result": "PASS", "finished_at": "2026-10-29T00:29:20.963570Z" }