date: 2026-11-06T08:24:13.893821+00:00 commit: b063c8780eabf63cdb9da3802f82440750308d6b
$ sh -c 'RATE=100 sh scripts/failover-each.sh evidence/final-validation/failover-each && RATE=100 python3 scripts/probe.py load --expect replace --duration 90 --out evidence/final-validation/rolling -- scripts/rolling-update.sh'
 Container hlpipe-lab3-validation-20261106-api-1  Running
 Container hlpipe-lab3-validation-20261106-api-2  Running
 Container hlpipe-lab3-validation-20261106-api-2  Waiting
 Container hlpipe-lab3-validation-20261106-api-1  Waiting
 Container hlpipe-lab3-validation-20261106-api-2  Healthy
 Container hlpipe-lab3-validation-20261106-api-1  Healthy
{"label": "9aefe0725018", "action": ["sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh"], "action_exit": 0, "action_seconds": 15.167, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, "lb_samples_missing": 0, "detection_seconds": 0.863, "rejoin_seconds": 14.911, "probe": {"attempts": 667, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.043, "recovery_s": 0.0, "instances_before": {"ad1ce3e2864d": 126, "9aefe0725018": 118}, "instances_during_outage": {"ad1ce3e2864d": 130}, "instances_after": {"9aefe0725018": 151, "ad1ce3e2864d": 131}}, "history": {"confirmed": 667, "stored": 667, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 6001, "rps": 100.02, "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.33, "p95_ms": 4.08, "p99_ms": 4.83, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true}, "pass": true}
 Container hlpipe-lab3-validation-20261106-api-1  Running
 Container hlpipe-lab3-validation-20261106-api-2  Running
 Container hlpipe-lab3-validation-20261106-api-2  Waiting
 Container hlpipe-lab3-validation-20261106-api-1  Waiting
 Container hlpipe-lab3-validation-20261106-api-2  Healthy
 Container hlpipe-lab3-validation-20261106-api-1  Healthy
{"label": "ad1ce3e2864d", "action": ["sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh"], "action_exit": 0, "action_seconds": 16.258, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, "lb_samples_missing": 0, "detection_seconds": 1.59, "rejoin_seconds": 16.166, "probe": {"attempts": 664, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.102, "recovery_s": 0.0, "instances_before": {"ad1ce3e2864d": 126, "9aefe0725018": 114}, "instances_during_outage": {"9aefe0725018": 135}, "instances_after": {"9aefe0725018": 142, "ad1ce3e2864d": 131}}, "history": {"confirmed": 664, "stored": 664, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 6001, "rps": 100.02, "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.34, "p95_ms": 4.12, "p99_ms": 5.04, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true}, "pass": true}
 Container hlpipe-lab3-validation-20261106-api-1  Running
 Container hlpipe-lab3-validation-20261106-api-2  Running
 Container hlpipe-lab3-validation-20261106-api-2  Waiting
 Container hlpipe-lab3-validation-20261106-api-1  Waiting
 Container hlpipe-lab3-validation-20261106-api-2  Healthy
 Container hlpipe-lab3-validation-20261106-api-1  Healthy
{"label": "action", "action": ["scripts/rolling-update.sh"], "action_exit": 0, "action_seconds": 11.096, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 2, "lb_samples_missing": 0, "detection_seconds": null, "rejoin_seconds": null, "probe": {"attempts": 941, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.051, "recovery_s": 0.0, "instances_before": {"9aefe0725018": 124, "ad1ce3e2864d": 116}, "instances_during_outage": {}, "instances_after": {"8062857b40ae": 320, "a89ed03a52aa": 277}}, "history": {"confirmed": 941, "stored": 941, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 9001, "rps": 100.01, "ok_rps": 100.01, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.42, "p95_ms": 4.18, "p99_ms": 5.1, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "capacity_kept": true, "all_replaced": true}, "pass": true}

exit: 0
