import json import os from pathlib import Path import subprocess import sys ROOT = Path(__file__).resolve().parents[1] def docker(*args): return subprocess.check_output(['docker', *args], cwd=ROOT, text=True).strip() def containers(): ids = docker('compose', 'ps', '-q', 'api').split() return json.loads(docker('inspect', *ids)) if ids else [] def main(): mode = sys.argv[1] if mode == 'restart': subprocess.run(['docker', 'compose', 'restart', 'postgres', 's3', 'api', 'lb'], cwd=ROOT, check=True) elif mode == 'stop': target = os.environ.get('TARGET_INSTANCE_ID') matches = [item for item in containers() if item['Config']['Hostname'] == target] if len(matches) != 1: raise SystemExit('Target must identify exactly one running API container') print('Stopping API container ' + matches[0]['Id'][:12]) docker('stop', '-t', '2', matches[0]['Id']) state = json.loads(docker('inspect', matches[0]['Id']))[0] event = {'target_instance': target, 'container_id': state['Id'][:12], 'name': state['Name'], 'status_after': state['State']['Status'], 'finished_at': state['State']['FinishedAt']} with (ROOT / 'evidence/stop-events.jsonl').open('a', encoding='utf-8') as output: output.write(json.dumps(event) + '\n') elif mode == 'rolling': import time import urllib.request for item in containers(): name = item['Name'].lstrip('/') started = time.time() stages = {} docker('stop', '-t', '6', item['Id']) stages['stopped_seconds'] = round(time.time() - started, 3) docker('rm', item['Id']) stages['removed_seconds'] = round(time.time() - started, 3) docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api') stages['created_seconds'] = round(time.time() - started, 3) deadline = time.monotonic() + 45 observations = [] previous = None while time.monotonic() < deadline: active = containers() state = [(c['Id'][:12], c.get('State', {}).get('Health', {}).get('Status')) for c in active] if state != previous: observations.append({'elapsed_seconds': round(time.time()-started, 3), 'containers': [{'id': c['Id'][:12], 'health': c.get('State', {}).get('Health', {})} for c in active]}) previous = state if len(active) == 2 and all(c.get('State', {}).get('Health', {}).get('Status') == 'healthy' for c in active): break time.sleep(0.5) else: raise SystemExit('Replacement did not become healthy: ' + name) stages['healthy_seconds'] = round(time.time() - started, 3) with urllib.request.urlopen(os.getenv('BASE_URL', 'http://localhost:8080').rstrip('/') + '/health/ready', timeout=3) as response: if response.status != 200: raise SystemExit('LB unavailable') time.sleep(3) print(json.dumps({'event': 'rolling_replacement', 'old_name': name, 'old_container': item['Id'][:12], 'started_epoch': started, 'finished_epoch': time.time(), 'stages': stages, 'health_observations': observations, 'new_containers': [c['Id'][:12] for c in active]}), flush=True) else: raise SystemExit('Unknown operation') if __name__ == '__main__': main()