# 2026-11-13T18:06:13Z python3 student-tests/worker_faults.py --output evidence/worker-faults.json
[   7.54s] workers recreated with fault injection {"env": {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}}
[  40.79s] crash-after-claim {"events": ["enqueued", "claimed", "fault_injected", "busy_ignored", "lease_expired", "requeued_after_lease", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
[  76.26s] crash-after-s3-put {"events": ["enqueued", "claimed", "fault_injected", "busy_ignored", "lease_expired", "requeued_after_lease", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
[  80.62s] crash-after-commit {"events": ["enqueued", "claimed", "ready", "fault_injected", "duplicate_ignored", "staging_released"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
[  80.76s] worker containers after crashes (restart: unless-stopped) {"containers": ["/mtusi-lab5-fresh-worker-1 1 running", "/mtusi-lab5-fresh-worker-2 2 running"]}
[  85.08s] transient-2: retried with backoff {"delays": [0.968, 1.852], "bounds": [[0.5, 1], [1.0, 2]], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 3}}
[  87.18s] unexpected exception -> processing_failed -> retry -> ready {"attachment_id": "5c527c98-0276-4827-b652-d74ea3872885", "retry_error_codes": ["processing_failed"], "seconds_to_ready": 1.26, "events": ["enqueued", "claimed", "retry_scheduled", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
[ 102.16s] transient-always: failed after max attempts {"error_code": "retry_exhausted", "attempts": 5, "content": 409, "attach": 409, "registry_entry": {"attachment_id": "e1b414dd-1f9e-4982-9de1-e31e69f89523", "job_id": "a795d934-00a6-4e23-9be0-3371a2ebbd55", "error_code": "retry_exhausted", "attempt": 5, "max_attempts": 5, "replay_count": 0, "last_error": "last error: storage_unavailable injected transient error", "updated_at": "2026-11-13T18:07:54.386852Z", "replayable": true}}
[ 106.01s] replay -> ready; replay of ready refused {"attachment_id": "e1b414dd-1f9e-4982-9de1-e31e69f89523", "dry_run": "would_replay", "replay": "replayed", "second_replay": "refused_status_ready", "replay_count": 1, "events": ["enqueued", "claimed", "retry_scheduled", "claimed", "retry_scheduled", "claimed", "retry_scheduled", "claimed", "retry_scheduled", "claimed", "failed", "replayed", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 6}}
[ 115.41s] tampered staging -> failed without retries {"attachment_id": "03f82fee-70af-4267-a19d-a3a5d7f5ea17", "error_code": "staging_corrupted", "attempts": 1, "events": ["enqueued", "claimed", "failed"]}
[ 126.10s] poison message dead-lettered and purged {"dlq_before": 0, "dlq_after_publish": 1, "listed": [{"reason": "rejected", "delivery_count": 1, "action": "kept", "invalid": "extra fields"}], "purged": ["purged_invalid"], "dlq_after_purge": 0}
[ 143.08s] job deadline exceeded -> processing_timeout -> retry -> ready {"attachment_id": "176a0882-0837-439a-9766-d66c6147ac7d", "retry_error_codes": ["processing_timeout"], "events": ["enqueued", "claimed", "retry_scheduled", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
[ 149.87s] workers restored to .env configuration 
PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ
exit=0
