# 2026-11-10 14:23:04 UTC | Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease) | exit 0 | 126.3s
$ python scripts/ops.py pipeline-faults
2026-11-10T14:20:58.422Z pipeline-faults: recreating worker with compose.faults.yaml
 Container messenger-lab5-postgres-1 Running 
 Container messenger-lab5-s3-1 Running 
 Container messenger-lab5-api-1 Running 
 Container messenger-lab5-api-2 Running 
 Container messenger-lab5-redis-1 Running 
 Container messenger-lab5-api-fault-publish-1 Creating 
 Container messenger-lab5-api-fault-dup-1 Creating 
 Container messenger-lab5-api-fault-publish-1 Created 
 Container messenger-lab5-api-fault-dup-1 Created 
 Container messenger-lab5-s3-1 Waiting 
 Container messenger-lab5-postgres-1 Waiting 
 Container messenger-lab5-postgres-1 Healthy 
 Container messenger-lab5-migrate-1 Starting 
 Container messenger-lab5-s3-1 Healthy 
 Container messenger-lab5-s3-init-1 Starting 
 Container messenger-lab5-migrate-1 Started 
 Container messenger-lab5-s3-init-1 Started 
 Container messenger-lab5-migrate-1 Waiting 
 Container messenger-lab5-s3-init-1 Waiting 
 Container messenger-lab5-postgres-1 Waiting 
 Container messenger-lab5-redis-1 Waiting 
 Container messenger-lab5-redis-1 Healthy 
 Container messenger-lab5-postgres-1 Healthy 
 Container messenger-lab5-migrate-1 Exited 
 Container messenger-lab5-s3-init-1 Exited 
 Container messenger-lab5-api-1 Waiting 
 Container messenger-lab5-api-2 Waiting 
 Container messenger-lab5-api-1 Waiting 
 Container messenger-lab5-api-2 Waiting 
 Container messenger-lab5-api-1 Healthy 
 Container messenger-lab5-api-2 Healthy 
 Container messenger-lab5-api-fault-dup-1 Starting 
 Container messenger-lab5-api-1 Healthy 
 Container messenger-lab5-api-2 Healthy 
 Container messenger-lab5-api-fault-publish-1 Starting 
 Container messenger-lab5-api-fault-dup-1 Started 
 Container messenger-lab5-api-fault-publish-1 Started 
 Container messenger-lab5-api-run-f6d1d1271d68 Creating 
 Container messenger-lab5-api-run-f6d1d1271d68 Created 
ready: http://api-fault-publish:8080/health/ready
 Container messenger-lab5-api-run-0c827bfb627f Creating 
 Container messenger-lab5-api-run-0c827bfb627f Created 
ready: http://api-fault-dup:8080/health/ready
 Container messenger-lab5-api-run-6e712120049d Creating 
 Container messenger-lab5-api-run-6e712120049d Created 
test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... ok
test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... ok
test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... ok
test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... ok
test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... ok
test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... ok
test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... ok
test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... ok
test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
  datetime_now = datetime.datetime.utcnow()
ok
test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... ok

----------------------------------------------------------------------
Ran 10 tests in 90.529s

OK
2026-11-10T14:23:04.629Z pipeline-faults: worker restored to normal configuration
# 2026-11-10 14:23:06 UTC | Очередь после сценариев: задания, outbox, DLQ, lag | exit 0 | 1.7s
$ python scripts/ops.py jobs stats
 Container messenger-lab5-worker-run-e232ea69a3ca Creating 
 Container messenger-lab5-worker-run-e232ea69a3ca Created 
{
  "jobs": {
    "succeeded": 158,
    "failed": 2
  },
  "attachments": {
    "ready": 159,
    "failed": 2
  },
  "outbox": {
    "rows": 0,
    "due": 0,
    "oldest_age_s": 0.0
  },
  "dead_letters_unreplayed": 2,
  "oldest_unfinished_job_age_s": 0.0,
  "broker": {
    "stream": "attachments.jobs",
    "length": 4,
    "groups": [
      {
        "name": "attachment-workers",
        "consumers": 10,
        "pending": 0,
        "lag": 0,
        "last_delivered_id": "1794320565944-0"
      }
    ]
  }
}

# 2026-11-10 14:23:07 UTC | Реестр failed (DLQ) | exit 0 | 1.6s
$ python scripts/ops.py jobs failed --limit 10
 Container messenger-lab5-worker-run-85c598896740 Creating 
 Container messenger-lab5-worker-run-85c598896740 Created 
[
  {
    "attachment_id": "9f72cadb-e186-43cb-8456-ee1d5f3bde45",
    "job_id": "05c07c99-eda0-4dbe-b49e-98d1426f3933",
    "error_code": "retry_exhausted",
    "attempts": 5,
    "failed_at": "2026-11-10 14:21:41.960496+00:00",
    "replayed_at": "2026-11-10 14:21:42.086003+00:00",
    "replay_count": 1,
    "job_status": "succeeded",
    "replayable": false
  },
  {
    "attachment_id": "390e3a82-63cd-4a64-a50c-a200f68924c4",
    "job_id": "bfe81165-4f51-445a-bf37-07e61e527985",
    "error_code": "invalid_image",
    "attempts": 1,
    "failed_at": "2026-11-10 14:21:32.124484+00:00",
    "replayed_at": null,
    "replay_count": 0,
    "job_status": "failed",
    "replayable": true
  },
  {
    "attachment_id": "5c05fbd5-77b0-4554-87bd-5b315aea3b82",
    "job_id": "d9f58adb-d682-4d2f-a94c-16f2ce326ec7",
    "error_code": "retry_exhausted",
    "attempts": 5,
    "failed_at": "2026-11-10 14:09:44.857997+00:00",
    "replayed_at": "2026-11-10 14:09:44.981918+00:00",
    "replay_count": 1,
    "job_status": "succeeded",
    "replayable": false
  },
  {
    "attachment_id": "5a2d84b3-b13f-4c34-83e5-65f42b792752",
    "job_id": "41cdf50b-cfd4-4a88-864c-f5a918948014",
    "error_code": "invalid_image",
    "attempts": 1,
    "failed_at": "2026-11-10 14:09:34.398249+00:00",
    "replayed_at": null,
    "replay_count": 0,
    "job_status": "failed",
    "replayable": true
  }
]

