#!/usr/bin/env python3 import argparse import json import threading import time from pathlib import Path from lab3lib import WriteLoad, api_containers, ps_snapshot, sh, wait_healthy def main(): p = argparse.ArgumentParser() p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--mode', choices=['stop', 'kill'], default='stop') p.add_argument('--workers', type=int, default=8) p.add_argument('--observe', type=float, default=12.0, help='seconds after the failure') p.add_argument('--output', default=None) a = p.parse_args() replicas = [r for r in api_containers() if r['state'] == 'running'] assert len(replicas) >= 2, 'need >=2 running api replicas' load = WriteLoad(a.base_url, workers=a.workers) load.start() time.sleep(5) rounds = [] for r in replicas: cid = r['id'] base = (time.monotonic() - 5, time.monotonic()) before = ps_snapshot() t_fail = time.monotonic() done = {} def act(): sh('docker', a.mode, cid) done['t'] = time.monotonic() th = threading.Thread(target=act) th.start() time.sleep(a.observe) th.join() during = ps_snapshot() t_restore = time.monotonic() sh('docker', 'start', cid) healthy = wait_healthy(cid, 60) t_healthy = time.monotonic() t_back = None while time.monotonic() - t_healthy < 15: if any(e[1] and e[2] == cid and e[0] > t_healthy for e in list(load.events)): t_back = time.monotonic() break time.sleep(0.1) time.sleep(3) ev = [e for e in list(load.events) if t_fail <= e[0] < t_restore] errs = [e for e in ev if not e[1]] served_target = [e for e in ev if e[1] and e[2] == cid] rounds.append({ 'replica_name': r['name'], 'instance_id': cid, 'action': 'docker ' + a.mode, 'action_duration_s': round(done['t'] - t_fail, 2), 'baseline_5s': load.window(*base), 'after_failure': load.window(t_fail, t_restore), 'first_1s_after_failure': load.window(t_fail, t_fail + 1), 'last_error_after_failure_s': round(max(e[0] for e in errs) - t_fail, 2) if errs else None, 'target_last_served_after_failure_s': round(max(e[0] + e[3] for e in served_target) - t_fail, 2) if served_target else None, 'restored_healthy_after_s': round(t_healthy - t_restore, 2) if healthy else None, 'restored_back_in_rotation_after_s': round(t_back - t_restore, 2) if t_back else None, 'errors_sample': sorted({e[2] for e in errs})[:3], 'docker_ps_before': before.splitlines(), 'docker_ps_during': during.splitlines(), }) print(json.dumps({k: v for k, v in rounds[-1].items() if not k.startswith('docker_ps')}, ensure_ascii=False)) load.stop() result = {'mode': a.mode, 'workers': a.workers, 'client_timeout_s': load.timeout, 'total': load.window(load.t0, time.monotonic()), 'client_retries': load.retries, 'rounds': rounds, 'data_check': load.verify_history(), 'docker_ps_end': ps_snapshot().splitlines()} print(json.dumps(result['data_check'])) ok = result['data_check']['lost_confirmed'] == 0 and result['data_check']['duplicate_texts'] == 0 print('PASS' if ok else 'FAIL', 'data integrity across', len(rounds), 'failures') if a.output: Path(a.output).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') raise SystemExit(0 if ok else 1) if __name__ == '__main__': main()