date: 2026-11-05T22:58:34Z  commit: 1aec062aba4560b51a6d9155571c9610bc777d7e
environment: {"COMPOSE_PROJECT_NAME": "hlpipe-lab4-grades-20261106", "COMPOSE_FILE": "compose.yaml", "LB_HTTP_PORT": "18084", "LB_STATS_PORT": "24844", "BASE_URL": "http://127.0.0.1:18084", "EVIDENCE_DIR": "evidence/sqlalchemy-2026-11-06/grade3", "NO_PROXY": "localhost,127.0.0.1,::1"}
$ sh scripts/failover-evidence.sh http://127.0.0.1:18084 0
date: 2026-11-05T22:58:34Z  commit: 1aec062aba4560b51a6d9155571c9610bc777d7e
== before: api replicas (hostname = X-Instance-Id), state, StartedAt, FinishedAt
cff4a17e0883 running started=2026-11-05T22:58:30.83452755Z finished=2026-11-05T22:58:24.503577672Z
f2e1260f6803 running started=2026-11-05T22:58:30.769256883Z finished=2026-11-05T22:58:24.505784755Z
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18084" 
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 2.826}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
== after: api replicas
cff4a17e0883 running started=2026-11-05T22:58:30.83452755Z finished=2026-11-05T22:58:24.503577672Z
f2e1260f6803 exited started=2026-11-05T22:58:30.769256883Z finished=2026-11-05T22:58:37.544193511Z
== stopped replica: f2e1260f6803; shutdown: signal, drain window with readiness 503, close, exit
{"event":"Handling signal: term","level":"info","ts":"2026-11-05T22:58:35.049678Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-05T22:58:35.051721Z"}
{"limit":2.0,"event":"service.draining","level":"info","ts":"2026-11-05T22:58:35.108386Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"187f845e94a643fcb766377e2e260da0","duration_ms":0.62,"event":"request.handled","level":"info","ts":"2026-11-05T22:58:36.012021Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"9e0cd8cdc6184fce8f9af5735cffb8f3","duration_ms":1.317,"event":"request.handled","level":"info","ts":"2026-11-05T22:58:36.521518Z"}
{"method":"GET","route":"/health/ready","status":503,"request_id":"55e0a45cd1e9478cb8203bb6cbfe0da1","duration_ms":49.175,"event":"request.handled","level":"info","ts":"2026-11-05T22:58:37.045066Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-05T22:58:37.055403Z"}
{"event":"Shutting down","level":"info","ts":"2026-11-05T22:58:37.110223Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-05T22:58:37.157015Z"}
{"event":"Finished server process [19]","level":"info","ts":"2026-11-05T22:58:37.157139Z"}
{"event":"Worker (pid:19) was sent SIGTERM!","level":"info","ts":"2026-11-05T22:58:37.207794Z"}
{"event":"service.stopped","level":"info","ts":"2026-11-05T22:58:37.213602Z"}
{"event":"Finished server process [20]","level":"info","ts":"2026-11-05T22:58:37.213742Z"}
{"event":"Worker (pid:20) was sent SIGTERM!","level":"info","ts":"2026-11-05T22:58:37.308719Z"}
{"event":"Shutting down: Master","level":"info","ts":"2026-11-05T22:58:37.414643Z"}
$ scripts/restore.sh
 Container hlpipe-lab4-grades-20261106-api-1  Running
 Container hlpipe-lab4-grades-20261106-api-2  Starting
 Container hlpipe-lab4-grades-20261106-api-2  Started
 Container hlpipe-lab4-grades-20261106-api-2  Waiting
 Container hlpipe-lab4-grades-20261106-api-1  Waiting
 Container hlpipe-lab4-grades-20261106-api-2  Healthy
 Container hlpipe-lab4-grades-20261106-api-1  Healthy
== after restore
cff4a17e0883 running started=2026-11-05T22:58:30.83452755Z finished=2026-11-05T22:58:24.503577672Z
f2e1260f6803 running started=2026-11-05T22:58:50.474934128Z finished=2026-11-05T22:58:37.544193511Z
exit_code: 0  elapsed_seconds: 18.126
