Files
mtusi2026lab3/scripts/failover_series.py
2026-10-16 01:30:10 +03:00

85 lines
3.6 KiB
Python

#!/usr/bin/env python3
import argparse
import json
import threading
import time
from pathlib import Path
from lab3lib import WriteLoad, api_containers, ps_snapshot, sh, wait_healthy
def main():
p = argparse.ArgumentParser()
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--mode', choices=['stop', 'kill'], default='stop')
p.add_argument('--workers', type=int, default=8)
p.add_argument('--observe', type=float, default=12.0, help='seconds after the failure')
p.add_argument('--output', default=None)
a = p.parse_args()
replicas = [r for r in api_containers() if r['state'] == 'running']
assert len(replicas) >= 2, 'need >=2 running api replicas'
load = WriteLoad(a.base_url, workers=a.workers)
load.start()
time.sleep(5)
rounds = []
for r in replicas:
cid = r['id']
base = (time.monotonic() - 5, time.monotonic())
before = ps_snapshot()
t_fail = time.monotonic()
done = {}
def act():
sh('docker', a.mode, cid)
done['t'] = time.monotonic()
th = threading.Thread(target=act)
th.start()
time.sleep(a.observe)
th.join()
during = ps_snapshot()
t_restore = time.monotonic()
sh('docker', 'start', cid)
healthy = wait_healthy(cid, 60)
t_healthy = time.monotonic()
t_back = None
while time.monotonic() - t_healthy < 15:
if any(e[1] and e[2] == cid and e[0] > t_healthy for e in list(load.events)):
t_back = time.monotonic()
break
time.sleep(0.1)
time.sleep(3)
ev = [e for e in list(load.events) if t_fail <= e[0] < t_restore]
errs = [e for e in ev if not e[1]]
served_target = [e for e in ev if e[1] and e[2] == cid]
rounds.append({
'replica_name': r['name'], 'instance_id': cid, 'action': 'docker ' + a.mode,
'action_duration_s': round(done['t'] - t_fail, 2),
'baseline_5s': load.window(*base),
'after_failure': load.window(t_fail, t_restore),
'first_1s_after_failure': load.window(t_fail, t_fail + 1),
'last_error_after_failure_s': round(max(e[0] for e in errs) - t_fail, 2) if errs else None,
'target_last_served_after_failure_s': round(max(e[0] + e[3] for e in served_target) - t_fail, 2) if served_target else None,
'restored_healthy_after_s': round(t_healthy - t_restore, 2) if healthy else None,
'restored_back_in_rotation_after_s': round(t_back - t_restore, 2) if t_back else None,
'errors_sample': sorted({e[2] for e in errs})[:3],
'docker_ps_before': before.splitlines(), 'docker_ps_during': during.splitlines(),
})
print(json.dumps({k: v for k, v in rounds[-1].items() if not k.startswith('docker_ps')}, ensure_ascii=False))
load.stop()
result = {'mode': a.mode, 'workers': a.workers, 'client_timeout_s': load.timeout,
'total': load.window(load.t0, time.monotonic()), 'client_retries': load.retries,
'rounds': rounds, 'data_check': load.verify_history(),
'docker_ps_end': ps_snapshot().splitlines()}
print(json.dumps(result['data_check']))
ok = result['data_check']['lost_confirmed'] == 0 and result['data_check']['duplicate_texts'] == 0
print('PASS' if ok else 'FAIL', 'data integrity across', len(rounds), 'failures')
if a.output:
Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
raise SystemExit(0 if ok else 1)
if __name__ == '__main__':
main()