Files
mtusi2026lab5/evidence/level4.json
T
ArtyomandClaude Opus 5.5 573220e12a Record level 4 evidence and fill in the report
evidence/level4.json: all 17 scenarios of levels 3 and 4 and the eight
invariants pass on 0b85edf; evidence/operator.txt shows the failed
registry, a replay to ready, a noop replay, an expired replay and the
DLQ. Smoke, topology, level3, persistence, failover, lab 4 scenarios,
limits, student tests and contract checks are refreshed on the same
commit. REPORT.md describes the variants, worker limits, jitter, lease
renewal, crash points, replay and retention, adds rows 4.1-4.4, the
level 4 failure experiments and the updated limitations.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:39:20 +03:00

924 lines
24 KiB
JSON

{
"suite": "lab5-pipeline-idempotency",
"started_at": "2026-10-29T00:23:06.625217Z",
"config": {
"base_url": "http://localhost:18085",
"level": 4,
"only": "",
"expect_variants": "thumbnail,crop,optimized",
"max_attempts": 5,
"stall_seconds": 15.0,
"compose": "docker compose",
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
},
"invariants": [
{
"id": "terminal",
"statement": "every 202 reaches its expected terminal state",
"result": "PASS",
"details": {
"accepted": 33
}
},
{
"id": "single_effect",
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_ready_rollback",
"statement": "ready never returns to queued or processing",
"result": "PASS",
"details": {
"traced": 33
}
},
{
"id": "monotonic_publication",
"statement": "published_version only moves forward",
"result": "PASS",
"details": {
"sampled": 8
}
},
{
"id": "bytes_match_metadata",
"statement": "sha256, size and real dimensions match served bytes",
"result": "PASS",
"details": {
"verified": 28
}
},
{
"id": "bounded_attempts",
"statement": "attempt <= JOB_MAX_ATTEMPTS",
"result": "PASS",
"details": {
"rows": 0
}
},
{
"id": "no_orphans",
"statement": "no pending outbox, messages, dead letters, staging, objects",
"result": "PASS",
"details": {
"queues": {
"attachments.process": [
0,
0
],
"attachments.process.dlq": [
2,
0
]
},
"s3": "SKIP: --s3-list is not set"
}
},
{
"id": "no_lost_202",
"statement": "every 202 is present in PostgreSQL and the API",
"result": "PASS",
"details": {
"accepted": 33
}
}
],
"scenarios": [
{
"name": "smoke_pipeline",
"level": 3,
"window": "upload, ready, variants match metadata",
"seconds": 2.3,
"result": "PASS",
"observed": {
"attachment_id": "01a1850a-2a86-7412-ad79-91e2d1035be6",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
},
{
"name": "worker_down_broker_restart",
"level": 3,
"window": "worker down, API and broker restart",
"seconds": 22.9,
"result": "PASS",
"observed": {
"attachment_id": "01a1850a-35f4-7738-ae88-adcd8221f85d",
"timeline": [
{
"at": "2026-10-29T00:23:21.327338Z",
"step": "worker stopped"
},
{
"at": "2026-10-29T00:23:21.352082Z",
"step": "upload 202 queued",
"attachment_id": "01a1850a-35f4-7738-ae88-adcd8221f85d"
},
{
"at": "2026-10-29T00:23:21.358202Z",
"step": "content 409 while queued"
},
{
"at": "2026-10-29T00:23:21.411091Z",
"step": "message with attachment 201, other messages work",
"message_id": "01a1850a-3612-709d-812a-314668261056"
},
{
"at": "2026-10-29T00:23:24.139715Z",
"step": "job is in the broker queue",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
1,
0
]
}
},
{
"at": "2026-10-29T00:23:30.980552Z",
"step": "API replicas and relay restarted"
},
{
"at": "2026-10-29T00:23:40.688506Z",
"step": "broker restarted with its volume, job survived",
"queues": {
"attachments.process": [
1,
0
],
"attachments.process.dlq": [
1,
0
]
}
},
{
"at": "2026-10-29T00:23:43.386331Z",
"step": "worker started"
},
{
"at": "2026-10-29T00:23:43.590658Z",
"step": "ready",
"variants": [
"crop",
"optimized",
"thumbnail"
]
}
]
}
},
{
"name": "duplicate_after_ready",
"level": 3,
"window": "redelivery of a finished job",
"seconds": 5.2,
"result": "PASS",
"observed": {
"attachment_id": "01a1850a-8cee-7426-8784-3914f1216d71",
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "done",
"attempt": 1
},
{
"kind": "duplicate_noop",
"attempt": 1
}
]
}
},
{
"name": "poison_image",
"level": 3,
"window": "permanent error without retries",
"seconds": 0.0,
"result": "PASS",
"observed": {
"rejected_by_api": 415
}
},
{
"name": "s3_outage_recovers",
"level": 3,
"window": "transient error with retry",
"seconds": 14.3,
"result": "PASS",
"observed": {
"attachment_id": "01a1850a-a267-7089-8d90-45cb5c521022",
"jobs": [
{
"job_id": "01a1850a-a26f-7791-88a6-29a9fa3d3aa4",
"pipeline_version": 1,
"status": "done",
"attempt": 2
}
]
}
},
{
"name": "retry_exhausted",
"level": 3,
"window": "transient error until JOB_MAX_ATTEMPTS",
"seconds": 20.4,
"result": "PASS",
"observed": {
"attachment_id": "01a1850a-e8d8-7122-ab63-13e85b328efe",
"jobs": [
{
"job_id": "01a1850a-e911-75b4-a3aa-7fd609efefa6",
"pipeline_version": 1,
"status": "failed",
"attempt": 5
}
]
}
},
{
"name": "image_matrix",
"level": 4,
"window": "limits and variants",
"seconds": 12.6,
"result": "PASS",
"observed": {
"images": [
{
"file": "png-rgb-320x200.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-rgba-300x300.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-palette-trns-200x100.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-gray-17x1000.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-gray-alpha-640x480.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-gray16-300x200.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-rgba16-64x64.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-1x1.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-wide-3000x2.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-small-50x30.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "jpeg-rgb-1000x750.jpg",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "jpeg-gray-801x601.jpg",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "jpeg-cmyk-400x300.jpg",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "jpeg-exif6-400x200.jpg",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "jpeg-exif3-300x200.jpg",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "jpeg-rgb-4000x3000.jpg",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-exactly-10mib.png",
"http": 202,
"status": "ready",
"error_code": null,
"variants": [
"crop",
"optimized",
"thumbnail"
]
},
{
"file": "png-over-10mib.png",
"http": "rejected"
},
{
"file": "png-over-pixels-8001x5000.png",
"http": "rejected"
},
{
"file": "jpeg-over-pixels-10000x5000.jpg",
"http": "rejected"
},
{
"file": "png-truncated.png",
"http": "rejected"
},
{
"file": "png-corrupt-idat.png",
"http": "rejected"
},
{
"file": "jpeg-truncated.jpg",
"http": "rejected"
},
{
"file": "png-zero-width.png",
"http": "rejected"
},
{
"file": "not-an-image.png",
"http": "rejected"
},
{
"file": "gif-declared-as-png.png",
"http": "rejected"
}
]
}
},
{
"name": "crash_commit_publish",
"level": 4,
"window": "after COMMIT, before publish",
"seconds": 17.5,
"result": "PASS",
"observed": {
"attachment_id": "01a1850b-7437-7757-92f0-862b8e3e430c",
"exit_code": 137,
"after_crash": {
"outbox_published": false,
"status": "queued"
},
"variants": [
{
"pipeline_version": 1,
"name": "crop",
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
"size_bytes": 1820,
"object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/crop-97bf1400b30a28c1.jpg"
},
{
"pipeline_version": 1,
"name": "optimized",
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"size_bytes": 2462,
"object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/optimized-1090d84ed8411226.webp"
},
{
"pipeline_version": 1,
"name": "thumbnail",
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/thumbnail-7bc40bf315c8e18e.jpg"
}
]
}
},
{
"name": "crash_publish_confirm",
"level": 4,
"window": "after publish, before confirm",
"seconds": 15.7,
"result": "PASS",
"observed": {
"attachment_id": "01a1850b-acca-750e-8e22-5af17f32c47d",
"exit_code": 137,
"after_crash": {
"outbox_published": [
false
],
"queues": {
"attachments.process": [
0,
0
],
"attachments.process.dlq": [
1,
0
]
}
},
"variants": [
{
"pipeline_version": 1,
"name": "crop",
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
"size_bytes": 1820,
"object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/crop-97bf1400b30a28c1.jpg"
},
{
"pipeline_version": 1,
"name": "optimized",
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"size_bytes": 2462,
"object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/optimized-1090d84ed8411226.webp"
},
{
"pipeline_version": 1,
"name": "thumbnail",
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/thumbnail-7bc40bf315c8e18e.jpg"
}
],
"events": [
"enqueued",
"claimed",
"done",
"duplicate_noop"
]
}
},
{
"name": "crash_after_s3_before_commit",
"level": 4,
"window": "after S3 write, before COMMIT",
"seconds": 23.1,
"result": "PASS",
"observed": {
"attachment_id": "01a1850b-f488-7792-be43-3371a5179dda",
"exit_code": 137,
"after_crash": {
"status": "processing",
"variant_rows": 0
},
"variants": [
{
"pipeline_version": 1,
"name": "crop",
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
"size_bytes": 1820,
"object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/crop-97bf1400b30a28c1.jpg"
},
{
"pipeline_version": 1,
"name": "optimized",
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"size_bytes": 2462,
"object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/optimized-1090d84ed8411226.webp"
},
{
"pipeline_version": 1,
"name": "thumbnail",
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/thumbnail-7bc40bf315c8e18e.jpg"
}
]
}
},
{
"name": "crash_after_commit_before_ack",
"level": 4,
"window": "after COMMIT, before ack",
"seconds": 17.5,
"result": "PASS",
"observed": {
"attachment_id": "01a1850c-4e58-7510-a23a-f8ddb2214d77",
"exit_code": 137,
"after_crash": {
"status": "ready",
"variants": [
{
"pipeline_version": 1,
"name": "crop",
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
"size_bytes": 1820,
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/crop-97bf1400b30a28c1.jpg"
},
{
"pipeline_version": 1,
"name": "optimized",
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"size_bytes": 2462,
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/optimized-1090d84ed8411226.webp"
},
{
"pipeline_version": 1,
"name": "thumbnail",
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/thumbnail-7bc40bf315c8e18e.jpg"
}
],
"queues": {
"attachments.process": [
0,
1
],
"attachments.process.dlq": [
1,
0
]
}
},
"variants": [
{
"pipeline_version": 1,
"name": "crop",
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
"size_bytes": 1820,
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/crop-97bf1400b30a28c1.jpg"
},
{
"pipeline_version": 1,
"name": "optimized",
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
"size_bytes": 2462,
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/optimized-1090d84ed8411226.webp"
},
{
"pipeline_version": 1,
"name": "thumbnail",
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
"size_bytes": 4679,
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/thumbnail-7bc40bf315c8e18e.jpg"
}
]
}
},
{
"name": "lease_expiry_fencing",
"level": 4,
"window": "stalled worker, lease expiry",
"seconds": 40.0,
"result": "PASS",
"observed": {
"attachment_id": "01a1850c-936d-7519-bba4-0322f1a3c0b8",
"jobs": [
{
"job_id": "01a1850c-9377-74d0-a5d9-48ed6ed9d65c",
"pipeline_version": 1,
"status": "done",
"attempt": 2
}
],
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "lease_expired",
"attempt": 2
},
{
"kind": "claimed",
"attempt": 2
},
{
"kind": "done",
"attempt": 2
},
{
"kind": "fenced",
"attempt": 1
}
]
}
},
{
"name": "replay_failed",
"level": 4,
"window": "operator replay",
"seconds": 44.6,
"result": "PASS",
"observed": {
"attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322",
"replay": {
"result": "replayed",
"attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322",
"operator": "cli"
},
"replay_of_ready": {
"result": "noop",
"attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322",
"operator": "cli"
},
"events": [
{
"kind": "enqueued",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "retry_scheduled",
"attempt": 2
},
{
"kind": "claimed",
"attempt": 2
},
{
"kind": "retry_scheduled",
"attempt": 3
},
{
"kind": "claimed",
"attempt": 3
},
{
"kind": "retry_scheduled",
"attempt": 4
},
{
"kind": "claimed",
"attempt": 4
},
{
"kind": "retry_scheduled",
"attempt": 5
},
{
"kind": "claimed",
"attempt": 5
},
{
"kind": "failed",
"attempt": 5
},
{
"kind": "replayed",
"attempt": 1
},
{
"kind": "claimed",
"attempt": 1
},
{
"kind": "done",
"attempt": 1
},
{
"kind": "replay_noop",
"attempt": 1
}
]
}
},
{
"name": "malformed_message_dead_lettered",
"level": 4,
"window": "message that is not a job goes to the DLQ",
"seconds": 7.3,
"result": "PASS",
"observed": {
"dead_letters": 1
}
},
{
"name": "worker_limits",
"level": 4,
"window": "render time and memory limits, 40 Mpx in the worker limit",
"seconds": 27.8,
"result": "PASS",
"observed": {
"image": {
"file": "png-at-pixel-limit-8000x5000.png",
"bytes": 116714,
"pixels": 40000000
},
"render_timeout": {
"seconds": 0.5,
"attachment_id": "01a1850e-0649-7516-bedf-76fcced46675",
"error_code": "processing_failed",
"worker": "running"
},
"render_memory": {
"mib": 200,
"attachment_id": "01a1850e-2203-76ad-94a5-d96a067b88d6",
"error_code": "processing_failed",
"worker": "running"
},
"default_limits": {
"attachment_id": "01a1850e-3e3d-75c8-b892-9a107c43d4b6",
"variants": [
"crop",
"optimized",
"thumbnail"
],
"memory_peak_mib": 378.8,
"memory_max_mib": 1024.0
}
}
},
{
"name": "retry_backoff_jitter",
"level": 4,
"window": "backoff with equal jitter between attempts",
"seconds": 35.2,
"result": "PASS",
"observed": {
"attachment_id": "01a1850e-85dc-7779-9d26-df88e9a5fe8b",
"delays": [
{
"attempt": 2,
"delay": 0.694765
},
{
"attempt": 3,
"delay": 1.738794
},
{
"attempt": 4,
"delay": 2.834139
},
{
"attempt": 5,
"delay": 7.550992
}
],
"share_of_ceiling": [
0.695,
0.869,
0.709,
0.944
],
"jobs": [
{
"job_id": "01a1850e-8623-7161-af00-316420cb53db",
"pipeline_version": 1,
"status": "failed",
"attempt": 5
}
]
}
},
{
"name": "failed_retention",
"level": 4,
"window": "failed staging kept until retention, then replay expires",
"seconds": 41.8,
"result": "PASS",
"observed": {
"attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69",
"staging_before": "kept",
"replay": {
"result": "expired",
"attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69",
"operator": "cli"
}
}
}
],
"result": "PASS",
"finished_at": "2026-10-29T00:29:20.963570Z"
}