{ "generated_at": "2026-11-14T21:23:08.494056+00:00", "fixture": { "bytes": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "width": 320, "height": 200 }, "scenarios": { "worker_stopped": { "attachment_id": "3d1153db-d279-447c-8cda-e24fe2079119", "queued_status": "queued", "content_while_queued": 409, "ready_after_worker_start": true, "queue_seconds": 2.316, "evidence": { "original": { "content_type": "image/png", "size_bytes": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "width": 320, "height": 200 }, "thumbnail": { "content_type": "image/jpeg", "size_bytes": 5653, "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", "width": 256, "height": 160 }, "crop": { "content_type": "image/jpeg", "size_bytes": 2441, "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", "width": 128, "height": 128 }, "optimized": { "content_type": "image/webp", "size_bytes": 2576, "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", "width": 320, "height": 200 } }, "row": { "attempt": 1, "published": true, "staging_cleared": true } }, "broker_outage": { "attachment_id": "bce30804-16d3-477c-a732-40bd336a0873", "202_accepted_without_broker": true, "unpublished_during_outage": true, "api_ready_during_broker_outage": true, "published_after_recovery": true, "attempt": 1, "queue_seconds": 9.005 }, "crash_after_effect": { "attachment_id": "afc5a48d-7781-4627-a2b2-cf9be1d92ac5", "worker_restart_count_before": 0, "worker_restart_count_after": 1, "ready_after_redelivery": true, "variant_objects": 3, "evidence": { "original": { "content_type": "image/png", "size_bytes": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "width": 320, "height": 200 }, "thumbnail": { "content_type": "image/jpeg", "size_bytes": 5653, "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", "width": 256, "height": 160 }, "crop": { "content_type": "image/jpeg", "size_bytes": 2441, "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", "width": 128, "height": 128 }, "optimized": { "content_type": "image/webp", "size_bytes": 2576, "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", "width": 320, "height": 200 } }, "log_excerpt": [ "worker-1 | time=2026-11-14T21:23:37.203Z level=INFO msg=\"processing attachment\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817", "worker-1 | time=2026-11-14T21:23:37.392Z level=INFO msg=\"attachment ready\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817", "worker-1 | time=2026-11-14T21:23:37.393Z level=ERROR msg=\"fault injection: exiting after durable effect before ack\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14", "worker-1 | time=2026-11-14T21:23:37.932Z level=INFO msg=\"job is already finished or owned\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 terminal=true busy=false attempt=1" ] }, "transient_retry": { "attachment_id": "3b46dc74-f4cc-4c1b-89fc-f0e4023df603", "attempts": 2, "status": "ready", "log_excerpt": [ "worker-1 | time=2026-11-14T21:23:54.733Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817", "worker-1 | time=2026-11-14T21:23:54.733Z level=WARN msg=\"transient job failure; scheduling retry\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817 code=processing_failed err=\"injected transient failure\" retry_in=2.25377422s", "worker-1 | time=2026-11-14T21:23:57.029Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817", "worker-1 | time=2026-11-14T21:23:57.219Z level=INFO msg=\"attachment ready\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817" ] }, "permanent_failure_replay": { "attachment_id": "a1417cf6-4d8b-48f7-9767-e5b876f0bdd1", "failed_code": "invalid_image", "failed_content_http": 409, "failed_message_link_http": 409, "replay_exit": 0, "replay_output": [ "2026/11/14 21:24:43 replay: attachment a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 requeued" ], "replay_count": 1, "variant_objects_after_replay": 3, "original_object_present": true, "evidence": { "original": { "content_type": "image/png", "size_bytes": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "width": 320, "height": 200 }, "thumbnail": { "content_type": "image/jpeg", "size_bytes": 5653, "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", "width": 256, "height": 160 }, "crop": { "content_type": "image/jpeg", "size_bytes": 2441, "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", "width": 128, "height": 128 }, "optimized": { "content_type": "image/webp", "size_bytes": 2576, "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", "width": 320, "height": 200 } }, "log_excerpt": [ "worker-1 | time=2026-11-14T21:24:33.736Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817", "worker-1 | time=2026-11-14T21:24:33.736Z level=ERROR msg=\"permanent job failure\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 code=invalid_image err=\"injected permanent failure\"", "worker-1 | time=2026-11-14T21:24:43.247Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817", "worker-1 | time=2026-11-14T21:24:43.469Z level=INFO msg=\"attachment ready\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817" ] }, "lease_recovery": { "attachment_id": "6d5091ae-474d-4e94-a4a9-f673aa16c9c1", "one_shot_exit": 10, "processing_before_expiry": true, "attempts": 2, "ready_after_sweep": true, "evidence": { "original": { "content_type": "image/png", "size_bytes": 111817, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "width": 320, "height": 200 }, "thumbnail": { "content_type": "image/jpeg", "size_bytes": 5653, "sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe", "width": 256, "height": 160 }, "crop": { "content_type": "image/jpeg", "size_bytes": 2441, "sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3", "width": 128, "height": 128 }, "optimized": { "content_type": "image/webp", "size_bytes": 2576, "sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c", "width": 320, "height": 200 } }, "log_excerpt": [ "worker-1 | time=2026-11-14T21:24:51.033Z level=INFO msg=\"job is already finished or owned\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f terminal=false busy=true attempt=1", "worker-1 | time=2026-11-14T21:24:59.019Z level=WARN msg=\"recovered expired job leases\" count=1", "worker-1 | time=2026-11-14T21:24:59.229Z level=INFO msg=\"processing attachment\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817", "worker-1 | time=2026-11-14T21:24:59.420Z level=INFO msg=\"attachment ready\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817" ] } } }