Files
Mikhail Verkovykh 9e43faa536 feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
2026-11-15 00:30:36 +03:00

209 lines
9.2 KiB
JSON

{
"generated_at": "2026-11-14T21:23:08.494056+00:00",
"fixture": {
"bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"width": 320,
"height": 200
},
"scenarios": {
"worker_stopped": {
"attachment_id": "3d1153db-d279-447c-8cda-e24fe2079119",
"queued_status": "queued",
"content_while_queued": 409,
"ready_after_worker_start": true,
"queue_seconds": 2.316,
"evidence": {
"original": {
"content_type": "image/png",
"size_bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"width": 320,
"height": 200
},
"thumbnail": {
"content_type": "image/jpeg",
"size_bytes": 5653,
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
"width": 256,
"height": 160
},
"crop": {
"content_type": "image/jpeg",
"size_bytes": 2441,
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
"width": 128,
"height": 128
},
"optimized": {
"content_type": "image/webp",
"size_bytes": 2576,
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
"width": 320,
"height": 200
}
},
"row": {
"attempt": 1,
"published": true,
"staging_cleared": true
}
},
"broker_outage": {
"attachment_id": "bce30804-16d3-477c-a732-40bd336a0873",
"202_accepted_without_broker": true,
"unpublished_during_outage": true,
"api_ready_during_broker_outage": true,
"published_after_recovery": true,
"attempt": 1,
"queue_seconds": 9.005
},
"crash_after_effect": {
"attachment_id": "afc5a48d-7781-4627-a2b2-cf9be1d92ac5",
"worker_restart_count_before": 0,
"worker_restart_count_after": 1,
"ready_after_redelivery": true,
"variant_objects": 3,
"evidence": {
"original": {
"content_type": "image/png",
"size_bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"width": 320,
"height": 200
},
"thumbnail": {
"content_type": "image/jpeg",
"size_bytes": 5653,
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
"width": 256,
"height": 160
},
"crop": {
"content_type": "image/jpeg",
"size_bytes": 2441,
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
"width": 128,
"height": 128
},
"optimized": {
"content_type": "image/webp",
"size_bytes": 2576,
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
"width": 320,
"height": 200
}
},
"log_excerpt": [
"worker-1 | time=2026-11-14T21:23:37.203Z level=INFO msg=\"processing attachment\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817",
"worker-1 | time=2026-11-14T21:23:37.392Z level=INFO msg=\"attachment ready\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 attachment_id=afc5a48d-7781-4627-a2b2-cf9be1d92ac5 attempt=1 image_bytes=111817",
"worker-1 | time=2026-11-14T21:23:37.393Z level=ERROR msg=\"fault injection: exiting after durable effect before ack\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14",
"worker-1 | time=2026-11-14T21:23:37.932Z level=INFO msg=\"job is already finished or owned\" job_id=fdd5c786-ca1e-4d04-98b6-75ce15ccda14 terminal=true busy=false attempt=1"
]
},
"transient_retry": {
"attachment_id": "3b46dc74-f4cc-4c1b-89fc-f0e4023df603",
"attempts": 2,
"status": "ready",
"log_excerpt": [
"worker-1 | time=2026-11-14T21:23:54.733Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817",
"worker-1 | time=2026-11-14T21:23:54.733Z level=WARN msg=\"transient job failure; scheduling retry\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=1 image_bytes=111817 code=processing_failed err=\"injected transient failure\" retry_in=2.25377422s",
"worker-1 | time=2026-11-14T21:23:57.029Z level=INFO msg=\"processing attachment\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817",
"worker-1 | time=2026-11-14T21:23:57.219Z level=INFO msg=\"attachment ready\" job_id=c18ef120-24a5-4344-899b-fcfc826bfbe7 attachment_id=3b46dc74-f4cc-4c1b-89fc-f0e4023df603 attempt=2 image_bytes=111817"
]
},
"permanent_failure_replay": {
"attachment_id": "a1417cf6-4d8b-48f7-9767-e5b876f0bdd1",
"failed_code": "invalid_image",
"failed_content_http": 409,
"failed_message_link_http": 409,
"replay_exit": 0,
"replay_output": [
"2026/11/14 21:24:43 replay: attachment a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 requeued"
],
"replay_count": 1,
"variant_objects_after_replay": 3,
"original_object_present": true,
"evidence": {
"original": {
"content_type": "image/png",
"size_bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"width": 320,
"height": 200
},
"thumbnail": {
"content_type": "image/jpeg",
"size_bytes": 5653,
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
"width": 256,
"height": 160
},
"crop": {
"content_type": "image/jpeg",
"size_bytes": 2441,
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
"width": 128,
"height": 128
},
"optimized": {
"content_type": "image/webp",
"size_bytes": 2576,
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
"width": 320,
"height": 200
}
},
"log_excerpt": [
"worker-1 | time=2026-11-14T21:24:33.736Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817",
"worker-1 | time=2026-11-14T21:24:33.736Z level=ERROR msg=\"permanent job failure\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817 code=invalid_image err=\"injected permanent failure\"",
"worker-1 | time=2026-11-14T21:24:43.247Z level=INFO msg=\"processing attachment\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817",
"worker-1 | time=2026-11-14T21:24:43.469Z level=INFO msg=\"attachment ready\" job_id=3f9c3f24-841e-44cd-8f8f-4adfef88d3ea attachment_id=a1417cf6-4d8b-48f7-9767-e5b876f0bdd1 attempt=1 image_bytes=111817"
]
},
"lease_recovery": {
"attachment_id": "6d5091ae-474d-4e94-a4a9-f673aa16c9c1",
"one_shot_exit": 10,
"processing_before_expiry": true,
"attempts": 2,
"ready_after_sweep": true,
"evidence": {
"original": {
"content_type": "image/png",
"size_bytes": 111817,
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"width": 320,
"height": 200
},
"thumbnail": {
"content_type": "image/jpeg",
"size_bytes": 5653,
"sha256": "4462c506c73f692d1a8da1f5a5d217bf29e7c7af0fe19f2c03a021055c8751fe",
"width": 256,
"height": 160
},
"crop": {
"content_type": "image/jpeg",
"size_bytes": 2441,
"sha256": "802a8482e119909b5716aea9f502185f9604d3992656e5e5baf72bc79afd31f3",
"width": 128,
"height": 128
},
"optimized": {
"content_type": "image/webp",
"size_bytes": 2576,
"sha256": "b0021a56abe58c6346f8fcae3a3e9820fa230e40286de35b1c50127d8f04e49c",
"width": 320,
"height": 200
}
},
"log_excerpt": [
"worker-1 | time=2026-11-14T21:24:51.033Z level=INFO msg=\"job is already finished or owned\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f terminal=false busy=true attempt=1",
"worker-1 | time=2026-11-14T21:24:59.019Z level=WARN msg=\"recovered expired job leases\" count=1",
"worker-1 | time=2026-11-14T21:24:59.229Z level=INFO msg=\"processing attachment\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817",
"worker-1 | time=2026-11-14T21:24:59.420Z level=INFO msg=\"attachment ready\" job_id=9201154a-7ddd-404b-8cc0-4ce24688c04f attachment_id=6d5091ae-474d-4e94-a4a9-f673aa16c9c1 attempt=2 image_bytes=111817"
]
}
}
}