forked from mtusi2026/mtusi2026lab5
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
209 lines
9.2 KiB
JSON
209 lines
9.2 KiB
JSON
{
|
|
"generated_at": "2026-11-14T21:23:08.494056+00:00",
|
|
"fixture": {
|
|
"bytes": 111817,
|
|
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
|
"width": 320,
|
|
"height": 200
|
|
},
|
|
"scenarios": {
|
|
"worker_stopped": {
|
|
"attachment_id": "3d1153db-d279-447c-8cda-e24fe2079119",
|
|
"queued_status": "queued",
|
|
"content_while_queued": 409,
|
|
"ready_after_worker_start": true,
|
|
"queue_seconds": 2.316,
|
|
"evidence": {
|
|
"original": {
|
|
"content_type": "image/png",
|
|
"size_bytes": 111817,
|
|
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
|
"width": 320,
|
|
"height": 200
|
|
},
|
|
"thumbnail": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 5653,
|
|
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
|
|
"width": 256,
|
|
"height": 160
|
|
},
|
|
"crop": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 2441,
|
|
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
|
|
"width": 128,
|
|
"height": 128
|
|
},
|
|
"optimized": {
|
|
"content_type": "image/webp",
|
|
"size_bytes": 2576,
|
|
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
|
|
"width": 320,
|
|
"height": 200
|
|
}
|
|
},
|
|
"row": {
|
|
"attempt": 1,
|
|
"published": true,
|
|
"staging_cleared": true
|
|
}
|
|
},
|
|
"broker_outage": {
|
|
"attachment_id": "bce30804-16d3-477c-a732-40bd336a0873",
|
|
"202_accepted_without_broker": true,
|
|
"unpublished_during_outage": true,
|
|
"api_ready_during_broker_outage": true,
|
|
"published_after_recovery": true,
|
|
"attempt": 1,
|
|
"queue_seconds": 9.005
|
|
},
|
|
"crash_after_effect": {
|
|
"attachment_id": "afc5a48d-7781-4627-a2b2-cf9be1d92ac5",
|
|
"worker_restart_count_before": 0,
|
|
"worker_restart_count_after": 1,
|
|
"ready_after_redelivery": true,
|
|
"variant_objects": 3,
|
|
"evidence": {
|
|
"original": {
|
|
"content_type": "image/png",
|
|
"size_bytes": 111817,
|
|
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
|
"width": 320,
|
|
"height": 200
|
|
},
|
|
"thumbnail": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 5653,
|
|
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
|
|
"width": 256,
|
|
"height": 160
|
|
},
|
|
"crop": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 2441,
|
|
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
|
|
"width": 128,
|
|
"height": 128
|
|
},
|
|
"optimized": {
|
|
"content_type": "image/webp",
|
|
"size_bytes": 2576,
|
|
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
|
|
"width": 320,
|
|
"height": 200
|
|
}
|
|
},
|
|
"log_excerpt": [
|
|
"worker-1 | time=2026-11-14T21:23:37.203Z level=INFO msg=\"processing attachment\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:23:37.392Z level=INFO msg=\"attachment ready\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:23:37.393Z level=ERROR msg=\"fault injection: exiting after durable effect before ack\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14",
|
|
"worker-1 | time=2026-11-14T21:23:37.932Z level=INFO msg=\"job is already finished or owned\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 terminal=true busy=false attempt=1"
|
|
]
|
|
},
|
|
"transient_retry": {
|
|
"attachment_id": "3b46dc74-f4cc-4c1b-89fc-f0e4023df603",
|
|
"attempts": 2,
|
|
"status": "ready",
|
|
"log_excerpt": [
|
|
"worker-1 | time=2026-11-14T21:23:54.733Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:23:54.733Z level=WARN msg=\"transient job failure; scheduling retry\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817 code=processing_failed err=\"injected transient failure\" retry_in=2.25377422s",
|
|
"worker-1 | time=2026-11-14T21:23:57.029Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:23:57.219Z level=INFO msg=\"attachment ready\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817"
|
|
]
|
|
},
|
|
"permanent_failure_replay": {
|
|
"attachment_id": "a1417cf6-4d8b-48f7-9767-e5b876f0bdd1",
|
|
"failed_code": "invalid_image",
|
|
"failed_content_http": 409,
|
|
"failed_message_link_http": 409,
|
|
"replay_exit": 0,
|
|
"replay_output": [
|
|
"2026/11/14 21:24:43 replay: attachment a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 requeued"
|
|
],
|
|
"replay_count": 1,
|
|
"variant_objects_after_replay": 3,
|
|
"original_object_present": true,
|
|
"evidence": {
|
|
"original": {
|
|
"content_type": "image/png",
|
|
"size_bytes": 111817,
|
|
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
|
"width": 320,
|
|
"height": 200
|
|
},
|
|
"thumbnail": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 5653,
|
|
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
|
|
"width": 256,
|
|
"height": 160
|
|
},
|
|
"crop": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 2441,
|
|
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
|
|
"width": 128,
|
|
"height": 128
|
|
},
|
|
"optimized": {
|
|
"content_type": "image/webp",
|
|
"size_bytes": 2576,
|
|
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
|
|
"width": 320,
|
|
"height": 200
|
|
}
|
|
},
|
|
"log_excerpt": [
|
|
"worker-1 | time=2026-11-14T21:24:33.736Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:24:33.736Z level=ERROR msg=\"permanent job failure\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 code=invalid_image err=\"injected permanent failure\"",
|
|
"worker-1 | time=2026-11-14T21:24:43.247Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:24:43.469Z level=INFO msg=\"attachment ready\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817"
|
|
]
|
|
},
|
|
"lease_recovery": {
|
|
"attachment_id": "6d5091ae-474d-4e94-a4a9-f673aa16c9c1",
|
|
"one_shot_exit": 10,
|
|
"processing_before_expiry": true,
|
|
"attempts": 2,
|
|
"ready_after_sweep": true,
|
|
"evidence": {
|
|
"original": {
|
|
"content_type": "image/png",
|
|
"size_bytes": 111817,
|
|
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
|
"width": 320,
|
|
"height": 200
|
|
},
|
|
"thumbnail": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 5653,
|
|
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
|
|
"width": 256,
|
|
"height": 160
|
|
},
|
|
"crop": {
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": 2441,
|
|
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
|
|
"width": 128,
|
|
"height": 128
|
|
},
|
|
"optimized": {
|
|
"content_type": "image/webp",
|
|
"size_bytes": 2576,
|
|
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
|
|
"width": 320,
|
|
"height": 200
|
|
}
|
|
},
|
|
"log_excerpt": [
|
|
"worker-1 | time=2026-11-14T21:24:51.033Z level=INFO msg=\"job is already finished or owned\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f terminal=false busy=true attempt=1",
|
|
"worker-1 | time=2026-11-14T21:24:59.019Z level=WARN msg=\"recovered expired job leases\" count=1",
|
|
"worker-1 | time=2026-11-14T21:24:59.229Z level=INFO msg=\"processing attachment\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817",
|
|
"worker-1 | time=2026-11-14T21:24:59.420Z level=INFO msg=\"attachment ready\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817"
|
|
]
|
|
}
|
|
}
|
|
}
|