forked from mtusi2026/mtusi2026lab5
133 lines
5.0 KiB
JSON
133 lines
5.0 KiB
JSON
{
|
|
"scenarios": [
|
|
{
|
|
"scenario": "worker_down",
|
|
"what": "worker stopped -> upload 202 -> message -> API + broker restart -> worker start -> ready",
|
|
"seconds": 31.1,
|
|
"observed": [
|
|
"worker: messenger-lab5-worker-1 exited Exited (0) Less than a second ago",
|
|
"queue before broker restart: {\"jobs\": {\"superseded\": 1, \"failed\": 6, \"done\": 33, \"queued\": 1}, \"outbox_unpublished\": 0, \"queue_ready_messages\": 1, \"queue_consumers\": 0, \"dlq_messages\": 1}",
|
|
"queue after broker restart (volume kept): {\"jobs\": {\"superseded\": 1, \"failed\": 6, \"done\": 33, \"queued\": 1}, \"outbox_unpublished\": 0, \"queue_ready_messages\": 1, \"queue_consumers\": 0, \"dlq_messages\": 1}",
|
|
"attachment 1f804983-862e-4fcc-b3ca-ee2af8564aba ready after worker start; thumbnail 256x160"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "outbox_gap",
|
|
"what": "COMMIT done, relay down (= crash between COMMIT and publish) -> relay restart publishes -> ready",
|
|
"seconds": 15.8,
|
|
"observed": [
|
|
"relay down: outbox [{'id': 60, 'attempt': '1', 'published': False, 'publish_count': 0}], attachment queued",
|
|
"after relay start: ready, events ['accepted', 'claimed', 'ready']"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "relay_crash_after_confirm",
|
|
"what": "relay dies after the publisher confirm but before marking the row -> duplicate publish -> worker no-op",
|
|
"seconds": 7.6,
|
|
"observed": [
|
|
"events: ['accepted', 'fault_injected', 'claimed', 'ready', 'duplicate_noop']"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "crash_after_claim",
|
|
"what": "crash before any side effect: lease expires, reaper re-enqueues attempt 2",
|
|
"seconds": 28.5,
|
|
"observed": [
|
|
"worker.after_claim: ready in 23.9s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2]"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "crash_after_s3",
|
|
"what": "crash after writing S3 objects, before COMMIT: retry overwrites the same keys, one set of variants",
|
|
"seconds": 29.0,
|
|
"observed": [
|
|
"worker.after_s3_before_commit: ready in 24.2s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2]"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "crash_after_commit",
|
|
"what": "crash after COMMIT, before ack: redelivery sees done -> ack as no-op, no second attempt",
|
|
"seconds": 6.6,
|
|
"observed": [
|
|
"worker.after_commit_before_ack: ready in 1.7s; events ['accepted', 'claimed', 'ready', 'fault_injected', 'duplicate_noop']; variant attempts [1]"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "transient_backoff",
|
|
"what": "two transient failures -> retries with exponential backoff + jitter -> ready on attempt 3",
|
|
"seconds": 12.8,
|
|
"observed": [
|
|
"retry delays [1.12, 3.15] (windows [1,2] and [2,4] s)"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "exhausted_then_replay",
|
|
"what": "5 transient failures -> failed retry_exhausted (registry) -> explicit replay -> ready; replay of ready refused",
|
|
"seconds": 38.3,
|
|
"observed": [
|
|
"failed after 5 attempts, registry entry present",
|
|
"replayed job 5ff64075-edfa-4472-afed-a003bdbb601d (attempt 6 next)",
|
|
"replay of ready: refused: job is done (attachment ready); replay applies only to failed jobs"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "long_running_not_hung",
|
|
"what": "a 30 s step longer than the 20 s lease: heartbeat renews the lease -> no second attempt",
|
|
"seconds": 37.4,
|
|
"observed": [
|
|
"30 s processing with 20 s lease: attempt=1, no lease_expired"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "duplicate",
|
|
"what": "3 raw duplicates of an already completed job: ack as no-op, nothing rewritten",
|
|
"seconds": 12.4,
|
|
"observed": [
|
|
"3 duplicates -> 3 duplicate_noop, updated_at unchanged"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "reorder",
|
|
"what": "relay stopped; deliveries arrive out of order: attempt 1, then 1 again and 4, then the outbox backlog",
|
|
"seconds": 20.2,
|
|
"observed": [
|
|
"events: ['accepted', 'claimed', 'fault_injected', 'retry_scheduled', 'stale_attempt_noop', 'stale_attempt_noop', 'stale_attempt_noop', 'claimed', 'ready']"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "new_version",
|
|
"what": "v1 ready -> reprocess v2 -> v2 published; replaying a v1 message afterwards changes nothing",
|
|
"seconds": 11.9,
|
|
"observed": [
|
|
"v1 optimized 1090d84ed841.. -> v2 f2878c4a9d89..; stale v1 message: duplicate_noop"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "stale_version_race",
|
|
"what": "v1 attempt is slow; v2 commits first; v1 then tries to commit -> superseded, v2 result kept",
|
|
"seconds": 16.3,
|
|
"observed": [
|
|
"commit order: [('ready', 'v2: thumbnail,crop,optimized'), ('superseded', 'published v2 is newer than v1')]; v1 objects are unreferenced leftovers for the sweeper"
|
|
]
|
|
},
|
|
{
|
|
"scenario": "poison_message",
|
|
"what": "a message that violates jobs.schema.json goes to the DLQ instead of looping",
|
|
"seconds": 6.4,
|
|
"observed": [
|
|
"DLQ 1 -> 2"
|
|
]
|
|
}
|
|
],
|
|
"final_stats": {
|
|
"jobs": {
|
|
"superseded": 2,
|
|
"failed": 6,
|
|
"done": 47
|
|
},
|
|
"outbox_unpublished": 0,
|
|
"queue_ready_messages": 0,
|
|
"queue_consumers": 1,
|
|
"dlq_messages": 2
|
|
}
|
|
} |