# Level 4: failover series, rolling update under load, public load.py
# 2026-10-19T15:20:55Z, commit 2708732 fix: LB и на [::1] — localhost на Windows стоил 2 с на запрос; сдвиг цели failover

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe bench/failover_series.py --output evidence/failover
== stop mtusi-lab3-api-1 ...
{"failed_after_stop": 0, "last_failure_after_stop_s": 0.0, "target_last_served_after_stop_s": 1.283, "rejoin_serving_s": 3.35}
== stop mtusi-lab3-api-2 ...
{"failed_after_stop": 0, "last_failure_after_stop_s": 0.0, "target_last_served_after_stop_s": 1.235, "rejoin_serving_s": 3.07}
== kill mtusi-lab3-api-1 ...
{"failed_after_stop": 3, "last_failure_after_stop_s": 0.247, "target_last_served_after_stop_s": 0.22, "rejoin_serving_s": 2.8}
== kill mtusi-lab3-api-2 ...
{"failed_after_stop": 3, "last_failure_after_stop_s": 0.214, "target_last_served_after_stop_s": 0.174, "rejoin_serving_s": 3.64}
== pause mtusi-lab3-api-1 ...
{"failed_after_stop": 4, "last_failure_after_stop_s": 3.204, "target_last_served_after_stop_s": 0.148, "rejoin_serving_s": 2.31}
== pause mtusi-lab3-api-2 ...
{"failed_after_stop": 4, "last_failure_after_stop_s": 3.12, "target_last_served_after_stop_s": 0.098, "rejoin_serving_s": 1.27}
| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| stop | mtusi-lab3-api-1 | 3568 | 0  | 294.6 / 222.5 / 249.7 | 0.0 | 1.283 | 3.25 / 3.35 | 0 / 0 |
| stop | mtusi-lab3-api-2 | 3565 | 0  | 259.2 / 225.7 / 275.1 | 0.0 | 1.235 | 1.35 / 3.07 | 0 / 0 |
| kill | mtusi-lab3-api-1 | 2734 | 3 [0] | 270.9 / 214.7 / 245.5 | 0.247 | 0.22 | 1.17 / 2.8 | 0 / 0 |
| kill | mtusi-lab3-api-2 | 2297 | 3 [0] | 225.6 / 179.6 / 240.0 | 0.214 | 0.174 | 2.32 / 3.64 | 0 / 0 |
| pause | mtusi-lab3-api-1 | 1653 | 4 [0] | 271.3 / 135.2 / 244.6 | 3.204 | 0.148 | 2.21 / 2.31 | 0 / 0 |
| pause | mtusi-lab3-api-2 | 1787 | 4 [0] | 253.6 / 147.0 / 253.0 | 3.12 | 0.098 | 1.16 / 1.27 | 0 / 0 |
[exit 0, 179.0 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe bench/rolling_under_load.py --output evidence/rolling-update
{
  "script_exit_code": 0,
  "update_seconds": 57.1,
  "before": {
    "mtusi-lab3-api-1": "907f96037a59 revision=dev",
    "mtusi-lab3-api-2": "c3f64421973d revision=dev"
  },
  "after": {
    "mtusi-lab3-api-3": "97c3063d8125 revision=2708732-20261019T152401Z",
    "mtusi-lab3-api-4": "7a0d24d4d31c revision=2708732-20261019T152401Z"
  },
  "history": {
    "history_messages": 7119,
    "confirmed_messages": 7119,
    "lost_confirmed": 0,
    "altered_confirmed": 0,
    "duplicate_ids": 0,
    "duplicate_texts": 0,
    "unconfirmed_but_stored": 0
  }
}
{"seconds": 57.09, "requests": 11848, "ok": 11848, "failed": 0, "ok_rps": 207.5, "error_ratio": 0.0, "statuses": {"201": 5924, "200": 5924}, "instances": {"907f96037a59": 3080, "c3f64421973d": 4767, "97c3063d8125": 2891, "7a0d24d4d31c": 1110}, "p50_ms": 14.32, "p95_ms": 53.01, "p99_ms": 68.28}
[exit 0, 74.9 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.7 s: ['7a0d24d4d31c', '97c3063d8125']
[exit 0, 1.7 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
{
  "lab": 3,
  "workload": "closed-loop POST messages; no automatic retries",
  "requested_duration_seconds": 30.0,
  "elapsed_seconds": 30.093,
  "concurrency": 4,
  "successful_requests": 5247,
  "failed_requests": 0,
  "error_ratio": 0.0,
  "successful_rps": 174.357,
  "successful_latency_ms": {
    "p50": 18.617,
    "p95": 50.067,
    "p99": 65.226
  },
  "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
[exit 0, 30.8 s]

$ docker compose ps -a --format table {{.Service}}	{{.Status}}
SERVICE    STATUS
api        Up About a minute (healthy)
api        Up About a minute (healthy)
lb         Up 9 minutes (healthy)
migrate    Exited (0) 4 minutes ago
postgres   Up 5 minutes (healthy)
valkey     Up 6 minutes (healthy)
[exit 0, 0.3 s]
