# Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung worker) + read-only audit of the invariants
# 2026-10-31T14:48:35Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT

$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe bench/pipeline_faults.py --output evidence/track2 --uploads 5
[2026-10-31T14:48:37Z] baseline_no_faults
[2026-10-31T14:48:54Z] duplicate_publish
[2026-10-31T14:49:25Z] duplicate_and_reverse_order
[2026-10-31T14:49:56Z] relay_crash_after_confirm_before_mark
[2026-10-31T14:50:27Z] worker_crash_before_side_effect
[2026-10-31T14:51:19Z] worker_crash_after_s3_before_commit
[2026-10-31T14:52:05Z] worker_crash_after_commit_before_ack
[2026-10-31T14:53:10Z] transient_errors_then_success
[2026-10-31T14:53:42Z] two_workers_duplicate_publish
[2026-10-31T14:54:22Z] versions_scenario
[2026-10-31T14:55:08Z] queued_reorder_scenario
[2026-10-31T14:56:08Z] hung_worker_scenario
bcd68c4d813f0d14a490156098130cab5e63ae21352e9fc620b8ac85f48df995
bcd68c4d813f0d14a490156098130cab5e63ae21352e9fc620b8ac85f48df995
scenario                                      uploads ready claims  problems
baseline_no_faults                                  5     5      5  0
duplicate_publish                                   5     5      5  0
duplicate_and_reverse_order                         5     5      5  0
relay_crash_after_confirm_before_mark               5     5      5  0
worker_crash_before_side_effect                     5     5     14  0
worker_crash_after_s3_before_commit                 5     5     10  0
worker_crash_after_commit_before_ack                5     5      5  0
transient_errors_then_success                       5     5     15  0
two_workers_duplicate_publish                       5     5      5  0
stale_version_and_reorder                           1     -      -  0
reordered_versions_in_queue                         1     -      2  0
hung_worker_docker_pause                            1     -      2  0
audit after all scenarios: ok=True violations=0 objects={'final': 523, 'staging': 2, 'published': 522, 'checksums_verified': True}
[exit 0, 515.1 s]

$ docker compose run --rm auditor --verify-checksums
{
 "ok": true,
 "violations": [],
 "notes": [],
 "attachments": {
  "failed": 4,
  "pending": 2,
  "ready": 129
 },
 "jobs": {
  "superseded": 2,
  "succeeded": 131,
  "failed": 6
 },
 "objects": {
  "final": 523,
  "staging": 2,
  "published": 522,
  "checksums_verified": true
 }
}
 Container mtusi-lab5-auditor-run-e2fdeabd6d0f Creating 
 Container mtusi-lab5-auditor-run-e2fdeabd6d0f Created
[exit 0, 7.2 s]

$ docker compose run --rm jobctl stats
{
 "jobs": {
  "failed": 6,
  "succeeded": 131,
  "superseded": 2
 },
 "outbox_unpublished": 0,
 "outbox_due": 0
}
 Container mtusi-lab5-jobctl-run-6f68d2c5bb69 Creating 
 Container mtusi-lab5-jobctl-run-6f68d2c5bb69 Created
[exit 0, 5.0 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.7 s: ['1fb1ea74a42b', '492b7b24bd1f']
[exit 0, 1.7 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
{
  "lab": 5,
  "workload": "closed-loop POST messages; no automatic retries",
  "requested_duration_seconds": 30.0,
  "elapsed_seconds": 30.066,
  "concurrency": 4,
  "successful_requests": 5515,
  "failed_requests": 0,
  "error_ratio": 0.0,
  "successful_rps": 183.429,
  "successful_latency_ms": {
    "p50": 17.96,
    "p95": 43.491,
    "p99": 58.653
  },
  "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
[exit 0, 31.5 s]

$ docker compose ps -a --format table {{.Service}}	{{.Status}}
SERVICE    STATUS
api        Up 10 minutes (healthy)
api        Up 11 minutes (healthy)
broker     Up 23 minutes (healthy)
lb         Up 30 minutes (healthy)
migrate    Exited (0) About a minute ago
postgres   Up 11 minutes (healthy)
relay      Up 2 minutes (healthy)
s3         Up 9 minutes (healthy)
s3-init    Exited (0) About a minute ago
valkey     Up 13 minutes (healthy)
worker     Up About a minute (healthy)
[exit 0, 0.4 s]
