evidence/level4.json: all 17 scenarios of levels 3 and 4 and the eight invariants pass on 0b85edf; evidence/operator.txt shows the failed registry, a replay to ready, a noop replay, an expired replay and the DLQ. Smoke, topology, level3, persistence, failover, lab 4 scenarios, limits, student tests and contract checks are refreshed on the same commit. REPORT.md describes the variants, worker limits, jitter, lease renewal, crash points, replay and retention, adds rows 4.1-4.4, the level 4 failure experiments and the updated limitations. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
924 lines
24 KiB
JSON
924 lines
24 KiB
JSON
{
|
|
"suite": "lab5-pipeline-idempotency",
|
|
"started_at": "2026-10-29T00:23:06.625217Z",
|
|
"config": {
|
|
"base_url": "http://localhost:18085",
|
|
"level": 4,
|
|
"only": "",
|
|
"expect_variants": "thumbnail,crop,optimized",
|
|
"max_attempts": 5,
|
|
"stall_seconds": 15.0,
|
|
"compose": "docker compose",
|
|
"compose_faults": "docker compose -f compose.yaml -f infra/compose.faults.yaml"
|
|
},
|
|
"invariants": [
|
|
{
|
|
"id": "terminal",
|
|
"statement": "every 202 reaches its expected terminal state",
|
|
"result": "PASS",
|
|
"details": {
|
|
"accepted": 33
|
|
}
|
|
},
|
|
{
|
|
"id": "single_effect",
|
|
"statement": "one variant set per (attachment_id, pipeline_version, variant)",
|
|
"result": "PASS",
|
|
"details": {
|
|
"rows": 0
|
|
}
|
|
},
|
|
{
|
|
"id": "no_ready_rollback",
|
|
"statement": "ready never returns to queued or processing",
|
|
"result": "PASS",
|
|
"details": {
|
|
"traced": 33
|
|
}
|
|
},
|
|
{
|
|
"id": "monotonic_publication",
|
|
"statement": "published_version only moves forward",
|
|
"result": "PASS",
|
|
"details": {
|
|
"sampled": 8
|
|
}
|
|
},
|
|
{
|
|
"id": "bytes_match_metadata",
|
|
"statement": "sha256, size and real dimensions match served bytes",
|
|
"result": "PASS",
|
|
"details": {
|
|
"verified": 28
|
|
}
|
|
},
|
|
{
|
|
"id": "bounded_attempts",
|
|
"statement": "attempt <= JOB_MAX_ATTEMPTS",
|
|
"result": "PASS",
|
|
"details": {
|
|
"rows": 0
|
|
}
|
|
},
|
|
{
|
|
"id": "no_orphans",
|
|
"statement": "no pending outbox, messages, dead letters, staging, objects",
|
|
"result": "PASS",
|
|
"details": {
|
|
"queues": {
|
|
"attachments.process": [
|
|
0,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
2,
|
|
0
|
|
]
|
|
},
|
|
"s3": "SKIP: --s3-list is not set"
|
|
}
|
|
},
|
|
{
|
|
"id": "no_lost_202",
|
|
"statement": "every 202 is present in PostgreSQL and the API",
|
|
"result": "PASS",
|
|
"details": {
|
|
"accepted": 33
|
|
}
|
|
}
|
|
],
|
|
"scenarios": [
|
|
{
|
|
"name": "smoke_pipeline",
|
|
"level": 3,
|
|
"window": "upload, ready, variants match metadata",
|
|
"seconds": 2.3,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850a-2a86-7412-ad79-91e2d1035be6",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "worker_down_broker_restart",
|
|
"level": 3,
|
|
"window": "worker down, API and broker restart",
|
|
"seconds": 22.9,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850a-35f4-7738-ae88-adcd8221f85d",
|
|
"timeline": [
|
|
{
|
|
"at": "2026-10-29T00:23:21.327338Z",
|
|
"step": "worker stopped"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:21.352082Z",
|
|
"step": "upload 202 queued",
|
|
"attachment_id": "01a1850a-35f4-7738-ae88-adcd8221f85d"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:21.358202Z",
|
|
"step": "content 409 while queued"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:21.411091Z",
|
|
"step": "message with attachment 201, other messages work",
|
|
"message_id": "01a1850a-3612-709d-812a-314668261056"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:24.139715Z",
|
|
"step": "job is in the broker queue",
|
|
"queues": {
|
|
"attachments.process": [
|
|
1,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:30.980552Z",
|
|
"step": "API replicas and relay restarted"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:40.688506Z",
|
|
"step": "broker restarted with its volume, job survived",
|
|
"queues": {
|
|
"attachments.process": [
|
|
1,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:43.386331Z",
|
|
"step": "worker started"
|
|
},
|
|
{
|
|
"at": "2026-10-29T00:23:43.590658Z",
|
|
"step": "ready",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "duplicate_after_ready",
|
|
"level": 3,
|
|
"window": "redelivery of a finished job",
|
|
"seconds": 5.2,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850a-8cee-7426-8784-3914f1216d71",
|
|
"events": [
|
|
{
|
|
"kind": "enqueued",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "done",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "duplicate_noop",
|
|
"attempt": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "poison_image",
|
|
"level": 3,
|
|
"window": "permanent error without retries",
|
|
"seconds": 0.0,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"rejected_by_api": 415
|
|
}
|
|
},
|
|
{
|
|
"name": "s3_outage_recovers",
|
|
"level": 3,
|
|
"window": "transient error with retry",
|
|
"seconds": 14.3,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850a-a267-7089-8d90-45cb5c521022",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a1850a-a26f-7791-88a6-29a9fa3d3aa4",
|
|
"pipeline_version": 1,
|
|
"status": "done",
|
|
"attempt": 2
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "retry_exhausted",
|
|
"level": 3,
|
|
"window": "transient error until JOB_MAX_ATTEMPTS",
|
|
"seconds": 20.4,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850a-e8d8-7122-ab63-13e85b328efe",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a1850a-e911-75b4-a3aa-7fd609efefa6",
|
|
"pipeline_version": 1,
|
|
"status": "failed",
|
|
"attempt": 5
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "image_matrix",
|
|
"level": 4,
|
|
"window": "limits and variants",
|
|
"seconds": 12.6,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"images": [
|
|
{
|
|
"file": "png-rgb-320x200.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-rgba-300x300.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-palette-trns-200x100.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-gray-17x1000.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-gray-alpha-640x480.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-gray16-300x200.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-rgba16-64x64.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-1x1.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-wide-3000x2.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-small-50x30.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "jpeg-rgb-1000x750.jpg",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "jpeg-gray-801x601.jpg",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "jpeg-cmyk-400x300.jpg",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "jpeg-exif6-400x200.jpg",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "jpeg-exif3-300x200.jpg",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "jpeg-rgb-4000x3000.jpg",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-exactly-10mib.png",
|
|
"http": 202,
|
|
"status": "ready",
|
|
"error_code": null,
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
]
|
|
},
|
|
{
|
|
"file": "png-over-10mib.png",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "png-over-pixels-8001x5000.png",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "jpeg-over-pixels-10000x5000.jpg",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "png-truncated.png",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "png-corrupt-idat.png",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "jpeg-truncated.jpg",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "png-zero-width.png",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "not-an-image.png",
|
|
"http": "rejected"
|
|
},
|
|
{
|
|
"file": "gif-declared-as-png.png",
|
|
"http": "rejected"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "crash_commit_publish",
|
|
"level": 4,
|
|
"window": "after COMMIT, before publish",
|
|
"seconds": 17.5,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850b-7437-7757-92f0-862b8e3e430c",
|
|
"exit_code": 137,
|
|
"after_crash": {
|
|
"outbox_published": false,
|
|
"status": "queued"
|
|
},
|
|
"variants": [
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "crop",
|
|
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
|
|
"size_bytes": 1820,
|
|
"object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/crop-97bf1400b30a28c1.jpg"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "optimized",
|
|
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
|
"size_bytes": 2462,
|
|
"object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/optimized-1090d84ed8411226.webp"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "thumbnail",
|
|
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
|
"size_bytes": 4679,
|
|
"object_key": "variants/01a1850b-7437-7757-92f0-862b8e3e430c/1/thumbnail-7bc40bf315c8e18e.jpg"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "crash_publish_confirm",
|
|
"level": 4,
|
|
"window": "after publish, before confirm",
|
|
"seconds": 15.7,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850b-acca-750e-8e22-5af17f32c47d",
|
|
"exit_code": 137,
|
|
"after_crash": {
|
|
"outbox_published": [
|
|
false
|
|
],
|
|
"queues": {
|
|
"attachments.process": [
|
|
0,
|
|
0
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
"variants": [
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "crop",
|
|
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
|
|
"size_bytes": 1820,
|
|
"object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/crop-97bf1400b30a28c1.jpg"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "optimized",
|
|
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
|
"size_bytes": 2462,
|
|
"object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/optimized-1090d84ed8411226.webp"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "thumbnail",
|
|
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
|
"size_bytes": 4679,
|
|
"object_key": "variants/01a1850b-acca-750e-8e22-5af17f32c47d/1/thumbnail-7bc40bf315c8e18e.jpg"
|
|
}
|
|
],
|
|
"events": [
|
|
"enqueued",
|
|
"claimed",
|
|
"done",
|
|
"duplicate_noop"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "crash_after_s3_before_commit",
|
|
"level": 4,
|
|
"window": "after S3 write, before COMMIT",
|
|
"seconds": 23.1,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850b-f488-7792-be43-3371a5179dda",
|
|
"exit_code": 137,
|
|
"after_crash": {
|
|
"status": "processing",
|
|
"variant_rows": 0
|
|
},
|
|
"variants": [
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "crop",
|
|
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
|
|
"size_bytes": 1820,
|
|
"object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/crop-97bf1400b30a28c1.jpg"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "optimized",
|
|
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
|
"size_bytes": 2462,
|
|
"object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/optimized-1090d84ed8411226.webp"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "thumbnail",
|
|
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
|
"size_bytes": 4679,
|
|
"object_key": "variants/01a1850b-f488-7792-be43-3371a5179dda/1/thumbnail-7bc40bf315c8e18e.jpg"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "crash_after_commit_before_ack",
|
|
"level": 4,
|
|
"window": "after COMMIT, before ack",
|
|
"seconds": 17.5,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850c-4e58-7510-a23a-f8ddb2214d77",
|
|
"exit_code": 137,
|
|
"after_crash": {
|
|
"status": "ready",
|
|
"variants": [
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "crop",
|
|
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
|
|
"size_bytes": 1820,
|
|
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/crop-97bf1400b30a28c1.jpg"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "optimized",
|
|
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
|
"size_bytes": 2462,
|
|
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/optimized-1090d84ed8411226.webp"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "thumbnail",
|
|
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
|
"size_bytes": 4679,
|
|
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/thumbnail-7bc40bf315c8e18e.jpg"
|
|
}
|
|
],
|
|
"queues": {
|
|
"attachments.process": [
|
|
0,
|
|
1
|
|
],
|
|
"attachments.process.dlq": [
|
|
1,
|
|
0
|
|
]
|
|
}
|
|
},
|
|
"variants": [
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "crop",
|
|
"sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143",
|
|
"size_bytes": 1820,
|
|
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/crop-97bf1400b30a28c1.jpg"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "optimized",
|
|
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
|
"size_bytes": 2462,
|
|
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/optimized-1090d84ed8411226.webp"
|
|
},
|
|
{
|
|
"pipeline_version": 1,
|
|
"name": "thumbnail",
|
|
"sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594",
|
|
"size_bytes": 4679,
|
|
"object_key": "variants/01a1850c-4e58-7510-a23a-f8ddb2214d77/1/thumbnail-7bc40bf315c8e18e.jpg"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "lease_expiry_fencing",
|
|
"level": 4,
|
|
"window": "stalled worker, lease expiry",
|
|
"seconds": 40.0,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850c-936d-7519-bba4-0322f1a3c0b8",
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a1850c-9377-74d0-a5d9-48ed6ed9d65c",
|
|
"pipeline_version": 1,
|
|
"status": "done",
|
|
"attempt": 2
|
|
}
|
|
],
|
|
"events": [
|
|
{
|
|
"kind": "enqueued",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "lease_expired",
|
|
"attempt": 2
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 2
|
|
},
|
|
{
|
|
"kind": "done",
|
|
"attempt": 2
|
|
},
|
|
{
|
|
"kind": "fenced",
|
|
"attempt": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "replay_failed",
|
|
"level": 4,
|
|
"window": "operator replay",
|
|
"seconds": 44.6,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322",
|
|
"replay": {
|
|
"result": "replayed",
|
|
"attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322",
|
|
"operator": "cli"
|
|
},
|
|
"replay_of_ready": {
|
|
"result": "noop",
|
|
"attachment_id": "01a1850d-2be1-778f-8538-077b3e53d322",
|
|
"operator": "cli"
|
|
},
|
|
"events": [
|
|
{
|
|
"kind": "enqueued",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "retry_scheduled",
|
|
"attempt": 2
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 2
|
|
},
|
|
{
|
|
"kind": "retry_scheduled",
|
|
"attempt": 3
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 3
|
|
},
|
|
{
|
|
"kind": "retry_scheduled",
|
|
"attempt": 4
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 4
|
|
},
|
|
{
|
|
"kind": "retry_scheduled",
|
|
"attempt": 5
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 5
|
|
},
|
|
{
|
|
"kind": "failed",
|
|
"attempt": 5
|
|
},
|
|
{
|
|
"kind": "replayed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "claimed",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "done",
|
|
"attempt": 1
|
|
},
|
|
{
|
|
"kind": "replay_noop",
|
|
"attempt": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "malformed_message_dead_lettered",
|
|
"level": 4,
|
|
"window": "message that is not a job goes to the DLQ",
|
|
"seconds": 7.3,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"dead_letters": 1
|
|
}
|
|
},
|
|
{
|
|
"name": "worker_limits",
|
|
"level": 4,
|
|
"window": "render time and memory limits, 40 Mpx in the worker limit",
|
|
"seconds": 27.8,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"image": {
|
|
"file": "png-at-pixel-limit-8000x5000.png",
|
|
"bytes": 116714,
|
|
"pixels": 40000000
|
|
},
|
|
"render_timeout": {
|
|
"seconds": 0.5,
|
|
"attachment_id": "01a1850e-0649-7516-bedf-76fcced46675",
|
|
"error_code": "processing_failed",
|
|
"worker": "running"
|
|
},
|
|
"render_memory": {
|
|
"mib": 200,
|
|
"attachment_id": "01a1850e-2203-76ad-94a5-d96a067b88d6",
|
|
"error_code": "processing_failed",
|
|
"worker": "running"
|
|
},
|
|
"default_limits": {
|
|
"attachment_id": "01a1850e-3e3d-75c8-b892-9a107c43d4b6",
|
|
"variants": [
|
|
"crop",
|
|
"optimized",
|
|
"thumbnail"
|
|
],
|
|
"memory_peak_mib": 378.8,
|
|
"memory_max_mib": 1024.0
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "retry_backoff_jitter",
|
|
"level": 4,
|
|
"window": "backoff with equal jitter between attempts",
|
|
"seconds": 35.2,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850e-85dc-7779-9d26-df88e9a5fe8b",
|
|
"delays": [
|
|
{
|
|
"attempt": 2,
|
|
"delay": 0.694765
|
|
},
|
|
{
|
|
"attempt": 3,
|
|
"delay": 1.738794
|
|
},
|
|
{
|
|
"attempt": 4,
|
|
"delay": 2.834139
|
|
},
|
|
{
|
|
"attempt": 5,
|
|
"delay": 7.550992
|
|
}
|
|
],
|
|
"share_of_ceiling": [
|
|
0.695,
|
|
0.869,
|
|
0.709,
|
|
0.944
|
|
],
|
|
"jobs": [
|
|
{
|
|
"job_id": "01a1850e-8623-7161-af00-316420cb53db",
|
|
"pipeline_version": 1,
|
|
"status": "failed",
|
|
"attempt": 5
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "failed_retention",
|
|
"level": 4,
|
|
"window": "failed staging kept until retention, then replay expires",
|
|
"seconds": 41.8,
|
|
"result": "PASS",
|
|
"observed": {
|
|
"attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69",
|
|
"staging_before": "kept",
|
|
"replay": {
|
|
"result": "expired",
|
|
"attachment_id": "01a1850e-f5b0-731a-b6a5-e50c724f4f69",
|
|
"operator": "cli"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"result": "PASS",
|
|
"finished_at": "2026-10-29T00:29:20.963570Z"
|
|
}
|