#!/usr/bin/env python3 """Replace each API container using a healthy temporary extra replica (surge=1).""" import json import os from pathlib import Path import subprocess import time import urllib.error import urllib.request ROOT = Path(__file__).resolve().parents[1] COMPOSE = str(ROOT / 'scripts/compose.sh') def command(*args): return subprocess.check_output(args, cwd=ROOT, text=True).strip() def ids(): return command(COMPOSE, 'ps', '-q', 'api').split() def inspect(cid): result = json.loads(command('docker', 'inspect', cid))[0] return {'id': result['Id'], 'hostname': result['Config']['Hostname'], 'image_id': result['Image'], 'image': result['Config']['Image'], 'status': result['State']['Status'], 'health': result['State'].get('Health', {}).get('Status')} def main(): original = ids() if len(original) < 2: raise RuntimeError('rolling update requires at least two running replicas') base = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') evidence = {'before': [inspect(i) for i in original], 'steps': []} try: for old in original: before = set(ids()) subprocess.run([COMPOSE, 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=' + str(len(original)+1), 'api'], check=True, cwd=ROOT) added = set(ids()) - before if len(added) != 1: raise RuntimeError('expected exactly one spare container') spare = added.pop() deadline = time.monotonic() + 60 while True: info = inspect(spare) # Healthy container must actually be observed behind the public LB. seen = False if info['health'] == 'healthy': try: for _ in range(2 * (len(original) + 1)): with urllib.request.urlopen(base + '/health/ready', timeout=2) as response: if response.headers.get('X-Instance-Id') == info['hostname']: seen = True break except (OSError, urllib.error.URLError): pass if seen: break if time.monotonic() >= deadline: raise RuntimeError('spare did not become healthy and reachable via LB') time.sleep(.3) subprocess.run(['docker', 'stop', '--time', '12', old], check=True, stdout=subprocess.DEVNULL) stopped = inspect(old) subprocess.run(['docker', 'rm', old], check=True, stdout=subprocess.DEVNULL) evidence['steps'].append({'removed': stopped, 'replacement': info}) evidence['after'] = [inspect(i) for i in ids()] assert len(evidence['after']) == len(original) assert not set(original) & set(ids()) evidence['status'] = 'PASS' finally: dest = ROOT / 'evidence/rolling-containers.json' dest.write_text(json.dumps(evidence, indent=2) + '\n') print('PASS: all original API containers replaced; evidence/rolling-containers.json') if __name__ == '__main__': main()