#!/usr/bin/env python3 """Restart API replicas one at a time while the public load test is running.""" import argparse import json import os from pathlib import Path import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] os.environ.setdefault('PYTHONUTF8', '1') def run(command, capture=False, timeout=None): result = subprocess.run( command, cwd=ROOT, text=True, capture_output=capture, check=False, timeout=timeout, ) if result.returncode: detail = (result.stderr or result.stdout or '').strip() raise RuntimeError('Command failed: ' + ' '.join(command) + '\n' + detail) return result.stdout.strip() if capture else '' def wait_healthy(container_id, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: status = run( [ 'docker', 'inspect', '--format', '{{.State.Health.Status}}', container_id, ], capture=True, ) if status == 'healthy': return time.sleep(0.5) raise TimeoutError('API replica did not become healthy after restart') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--duration', type=int, default=40) parser.add_argument('--concurrency', type=int, default=16) parser.add_argument('--output', default='evidence/rolling-restart.json') args = parser.parse_args() if args.duration < 10 or args.concurrency < 1: parser.error('duration must be at least 10 seconds and concurrency must be positive') containers = run( ['docker', 'compose', 'ps', '-q', 'api'], capture=True ).splitlines() if len(containers) < 2: parser.error('At least two running API replicas are required') output = (ROOT / args.output).resolve() output.parent.mkdir(parents=True, exist_ok=True) load_output = output.with_name(output.stem + '-load.json') load_process = subprocess.Popen( [ sys.executable, str(ROOT / 'tests' / 'load.py'), '--base-url', args.base_url, '--duration', str(args.duration), '--concurrency', str(args.concurrency), '--output', str(load_output), ], cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) restarts = [] try: time.sleep(2) for container_id in containers: instance_id = run( [ 'docker', 'inspect', '--format', '{{.Config.Hostname}}', container_id, ], capture=True, ) started = time.monotonic() run(['docker', 'restart', '--timeout', '15', container_id], timeout=30) wait_healthy(container_id) restarts.append( { 'instance_id': instance_id, 'ready_seconds': round(time.monotonic() - started, 3), } ) stdout, stderr = load_process.communicate(timeout=max(90, args.duration + 30)) except Exception: if load_process.poll() is None: load_process.terminate() load_process.wait(timeout=5) raise if load_process.returncode: raise RuntimeError('Load test failed:\n' + (stderr or stdout).strip()) report = { 'operation': 'serial API replica restart under load; same image, no schema change', 'duration_seconds': args.duration, 'concurrency': args.concurrency, 'restarted_replicas': restarts, 'load': json.loads(load_output.read_text(encoding='utf-8')), 'load_raw_file': load_output.name, 'limitations': [ 'This verifies graceful process replacement, not a new-image rolling deployment.', 'PostgreSQL and Redis are single instances; their failure is not covered.', ], } output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print(json.dumps(report['load'], ensure_ascii=False, indent=2)) print('Saved serial restart evidence to ' + str(output)) if __name__ == '__main__': try: main() except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error: print('ERROR: ' + str(error), file=sys.stderr) raise SystemExit(1)