# make test-failover with scripts/stop-one: which container is really stopped, restore, repeat
# 2026-10-25T11:42:11Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов

--- run 1; running API replicas BEFORE:
mtusi-lab4-api-1         host=8691c30aa702  running  exit=0   pid=33790  started=2026-10-25T11:41:59.187
mtusi-lab4-api-2         host=4026e296947c  running  exit=0   pid=33848  started=2026-10-25T11:41:59.568

$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" 
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 5.956}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 16.5 s]

AFTER (the stopped replica, left stopped by the hook):
mtusi-lab4-api-1         host=8691c30aa702  exited   exit=0   pid=0      started=2026-10-25T11:41:59.187

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api2   172.25.0.3      UP       mtusi-lab4-api-2         X-Instance-Id=4026e296947c
(other slots: MAINT, no container)
[exit 0, 0.7 s]

$ sh scripts/restore-api.sh
mtusi-lab4-api-1 Up 6 seconds (healthy)
mtusi-lab4-api-2 Up 37 seconds (healthy)
 Container mtusi-lab4-api-2 Running 
 Container mtusi-lab4-api-1 Starting 
 Container mtusi-lab4-api-1 Started 
 Container mtusi-lab4-api-1 Waiting 
 Container mtusi-lab4-api-2 Waiting 
 Container mtusi-lab4-api-1 Healthy 
 Container mtusi-lab4-api-2 Healthy
[exit 0, 6.9 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['4026e296947c', '8691c30aa702']
[exit 0, 1.4 s]

--- run 2; running API replicas BEFORE:
mtusi-lab4-api-1         host=8691c30aa702  running  exit=0   pid=35544  started=2026-10-25T11:42:30.519
mtusi-lab4-api-2         host=4026e296947c  running  exit=0   pid=33848  started=2026-10-25T11:41:59.568

$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" 
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 6.291}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 16.4 s]

AFTER (the stopped replica, left stopped by the hook):
mtusi-lab4-api-2         host=4026e296947c  exited   exit=0   pid=0      started=2026-10-25T11:41:59.568

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api1   172.25.0.2      UP       mtusi-lab4-api-1         X-Instance-Id=8691c30aa702
(other slots: MAINT, no container)
[exit 0, 0.8 s]

$ sh scripts/restore-api.sh
mtusi-lab4-api-1 Up 32 seconds (healthy)
mtusi-lab4-api-2 Up 6 seconds (healthy)
 Container mtusi-lab4-api-1 Running 
 Container mtusi-lab4-api-2 Starting 
 Container mtusi-lab4-api-2 Started 
 Container mtusi-lab4-api-1 Waiting 
 Container mtusi-lab4-api-2 Waiting 
 Container mtusi-lab4-api-1 Healthy 
 Container mtusi-lab4-api-2 Healthy
[exit 0, 6.9 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['4026e296947c', '8691c30aa702']
[exit 0, 1.5 s]

Replicas stopped by the public test across runs: ['mtusi-lab4-api-1', 'mtusi-lab4-api-2']
