Files
mtusi2026lab5/evidence/faults.json

133 lines
5.0 KiB
JSON

{
"scenarios": [
{
"scenario": "worker_down",
"what": "worker stopped -> upload 202 -> message -> API + broker restart -> worker start -> ready",
"seconds": 31.1,
"observed": [
"worker: messenger-lab5-worker-1 exited Exited (0) Less than a second ago",
"queue before broker restart: {\"jobs\": {\"superseded\": 1, \"failed\": 6, \"done\": 33, \"queued\": 1}, \"outbox_unpublished\": 0, \"queue_ready_messages\": 1, \"queue_consumers\": 0, \"dlq_messages\": 1}",
"queue after broker restart (volume kept): {\"jobs\": {\"superseded\": 1, \"failed\": 6, \"done\": 33, \"queued\": 1}, \"outbox_unpublished\": 0, \"queue_ready_messages\": 1, \"queue_consumers\": 0, \"dlq_messages\": 1}",
"attachment 1f804983-862e-4fcc-b3ca-ee2af8564aba ready after worker start; thumbnail 256x160"
]
},
{
"scenario": "outbox_gap",
"what": "COMMIT done, relay down (= crash between COMMIT and publish) -> relay restart publishes -> ready",
"seconds": 15.8,
"observed": [
"relay down: outbox [{'id': 60, 'attempt': '1', 'published': False, 'publish_count': 0}], attachment queued",
"after relay start: ready, events ['accepted', 'claimed', 'ready']"
]
},
{
"scenario": "relay_crash_after_confirm",
"what": "relay dies after the publisher confirm but before marking the row -> duplicate publish -> worker no-op",
"seconds": 7.6,
"observed": [
"events: ['accepted', 'fault_injected', 'claimed', 'ready', 'duplicate_noop']"
]
},
{
"scenario": "crash_after_claim",
"what": "crash before any side effect: lease expires, reaper re-enqueues attempt 2",
"seconds": 28.5,
"observed": [
"worker.after_claim: ready in 23.9s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2]"
]
},
{
"scenario": "crash_after_s3",
"what": "crash after writing S3 objects, before COMMIT: retry overwrites the same keys, one set of variants",
"seconds": 29.0,
"observed": [
"worker.after_s3_before_commit: ready in 24.2s; events ['accepted', 'claimed', 'fault_injected', 'lease_held_noop', 'lease_expired', 'retry_scheduled', 'claimed', 'ready']; variant attempts [2]"
]
},
{
"scenario": "crash_after_commit",
"what": "crash after COMMIT, before ack: redelivery sees done -> ack as no-op, no second attempt",
"seconds": 6.6,
"observed": [
"worker.after_commit_before_ack: ready in 1.7s; events ['accepted', 'claimed', 'ready', 'fault_injected', 'duplicate_noop']; variant attempts [1]"
]
},
{
"scenario": "transient_backoff",
"what": "two transient failures -> retries with exponential backoff + jitter -> ready on attempt 3",
"seconds": 12.8,
"observed": [
"retry delays [1.12, 3.15] (windows [1,2] and [2,4] s)"
]
},
{
"scenario": "exhausted_then_replay",
"what": "5 transient failures -> failed retry_exhausted (registry) -> explicit replay -> ready; replay of ready refused",
"seconds": 38.3,
"observed": [
"failed after 5 attempts, registry entry present",
"replayed job 5ff64075-edfa-4472-afed-a003bdbb601d (attempt 6 next)",
"replay of ready: refused: job is done (attachment ready); replay applies only to failed jobs"
]
},
{
"scenario": "long_running_not_hung",
"what": "a 30 s step longer than the 20 s lease: heartbeat renews the lease -> no second attempt",
"seconds": 37.4,
"observed": [
"30 s processing with 20 s lease: attempt=1, no lease_expired"
]
},
{
"scenario": "duplicate",
"what": "3 raw duplicates of an already completed job: ack as no-op, nothing rewritten",
"seconds": 12.4,
"observed": [
"3 duplicates -> 3 duplicate_noop, updated_at unchanged"
]
},
{
"scenario": "reorder",
"what": "relay stopped; deliveries arrive out of order: attempt 1, then 1 again and 4, then the outbox backlog",
"seconds": 20.2,
"observed": [
"events: ['accepted', 'claimed', 'fault_injected', 'retry_scheduled', 'stale_attempt_noop', 'stale_attempt_noop', 'stale_attempt_noop', 'claimed', 'ready']"
]
},
{
"scenario": "new_version",
"what": "v1 ready -> reprocess v2 -> v2 published; replaying a v1 message afterwards changes nothing",
"seconds": 11.9,
"observed": [
"v1 optimized 1090d84ed841.. -> v2 f2878c4a9d89..; stale v1 message: duplicate_noop"
]
},
{
"scenario": "stale_version_race",
"what": "v1 attempt is slow; v2 commits first; v1 then tries to commit -> superseded, v2 result kept",
"seconds": 16.3,
"observed": [
"commit order: [('ready', 'v2: thumbnail,crop,optimized'), ('superseded', 'published v2 is newer than v1')]; v1 objects are unreferenced leftovers for the sweeper"
]
},
{
"scenario": "poison_message",
"what": "a message that violates jobs.schema.json goes to the DLQ instead of looping",
"seconds": 6.4,
"observed": [
"DLQ 1 -> 2"
]
}
],
"final_stats": {
"jobs": {
"superseded": 2,
"failed": 6,
"done": 47
},
"outbox_unpublished": 0,
"queue_ready_messages": 0,
"queue_consumers": 1,
"dlq_messages": 2
}
}