# make test-failover with scripts/stop-one: which container is really stopped, restore, repeat
# 2026-10-31T14:21:25Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT

--- run 1; running API replicas BEFORE:
mtusi-lab5-api-1         host=d08185c0e3f3  running  exit=0   pid=91728  started=2026-10-31T14:21:03.235
mtusi-lab5-api-2         host=85c7cde066aa  running  exit=0   pid=91671  started=2026-10-31T14:21:02.858

$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" 
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 6.326}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 19.4 s]

AFTER (the stopped replica, left stopped by the hook):
mtusi-lab5-api-1         host=d08185c0e3f3  exited   exit=0   pid=0      started=2026-10-31T14:21:03.235

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api2   172.28.0.2      UP       mtusi-lab5-api-2         X-Instance-Id=85c7cde066aa
(other slots: MAINT, no container)
[exit 0, 0.9 s]

$ sh scripts/restore-api.sh
mtusi-lab5-api-1 Up 6 seconds (healthy)
mtusi-lab5-api-2 Up 50 seconds (healthy)
 Container mtusi-lab5-api-2 Running 
 Container mtusi-lab5-api-1 Starting 
 Container mtusi-lab5-api-1 Started 
 Container mtusi-lab5-api-2 Waiting 
 Container mtusi-lab5-api-1 Waiting 
 Container mtusi-lab5-api-2 Healthy 
 Container mtusi-lab5-api-1 Healthy
[exit 0, 6.9 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['85c7cde066aa', 'd08185c0e3f3']
[exit 0, 1.4 s]

--- run 2; running API replicas BEFORE:
mtusi-lab5-api-1         host=d08185c0e3f3  running  exit=0   pid=94963  started=2026-10-31T14:21:47.426
mtusi-lab5-api-2         host=85c7cde066aa  running  exit=0   pid=91671  started=2026-10-31T14:21:02.858

$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080" 
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 6.074}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
[exit 0, 18.0 s]

AFTER (the stopped replica, left stopped by the hook):
mtusi-lab5-api-2         host=85c7cde066aa  exited   exit=0   pid=0      started=2026-10-31T14:21:02.858

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
api1   172.28.0.3      UP       mtusi-lab5-api-1         X-Instance-Id=d08185c0e3f3
(other slots: MAINT, no container)
[exit 0, 0.7 s]

$ sh scripts/restore-api.sh
mtusi-lab5-api-1 Up 34 seconds (healthy)
mtusi-lab5-api-2 Up 6 seconds (healthy)
 Container mtusi-lab5-api-1 Running 
 Container mtusi-lab5-api-2 Starting 
 Container mtusi-lab5-api-2 Started 
 Container mtusi-lab5-api-2 Waiting 
 Container mtusi-lab5-api-1 Waiting 
 Container mtusi-lab5-api-2 Healthy 
 Container mtusi-lab5-api-1 Healthy
[exit 0, 7.4 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['85c7cde066aa', 'd08185c0e3f3']
[exit 0, 1.6 s]

Replicas stopped by the public test across runs: ['mtusi-lab5-api-1', 'mtusi-lab5-api-2']
