#!/usr/bin/env python3 """Build a new API image and replace replicas one at a time under load.""" import argparse import json import os from pathlib import Path import shlex import subprocess import sys import time import uuid ROOT = Path(__file__).resolve().parents[1] BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080') COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose')) os.environ.setdefault('PYTHONUTF8', '1') def run(args, capture=True, timeout=None): result = subprocess.run( args, cwd=ROOT, text=True, capture_output=capture, check=False, timeout=timeout, ) if result.returncode: detail = (result.stderr or result.stdout or '').strip() raise RuntimeError('Operation failed: ' + ' '.join(args[:3]) + '\n' + detail) return result.stdout.strip() if capture else '' def compose(*args, capture=True, timeout=None): return run(COMPOSE + ['-f', str(ROOT / 'compose.yaml'), *args], capture, timeout) def inspect(container_id): return json.loads(run(['docker', 'inspect', container_id]))[0] def wait_healthy(container_id, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: state = inspect(container_id)['State'] if state['Status'] == 'exited': raise RuntimeError('Replacement API container exited before becoming healthy') if state.get('Health', {}).get('Status') == 'healthy': return time.sleep(0.5) raise TimeoutError('Replacement API container did not become healthy') def wait_routed_to(instance_id, base_url, timeout=20): sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure client = Client(base_url, timeout=5) deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: response = client.call( 'GET', '/api/v1/users/me', expected=401, schema='Error' ) if response['headers'].get('X-Instance-Id') == instance_id: return except Failure: pass time.sleep(0.1) raise TimeoutError('Nginx did not route a request to the new API container') def candidate_command(old, image_id, name): config = old['Config'] host_config = old['HostConfig'] networks = old['NetworkSettings']['Networks'] if len(networks) != 1: raise RuntimeError('Expected API to use exactly one private network') network = next(iter(networks)) args = [ 'docker', 'run', '--detach', '--name', name, '--network', network, '--network-alias', 'api', '--restart', 'no', '--stop-timeout', '15', ] if config.get('User'): args += ['--user', config['User']] if host_config.get('NanoCpus'): args += ['--cpus', str(host_config['NanoCpus'] / 1_000_000_000)] if host_config.get('Memory'): args += ['--memory', str(host_config['Memory'])] for env in config.get('Env', []): args += ['--env', env] for mount in old.get('Mounts', []): if mount['Destination'] != '/run/secrets/database_url': raise RuntimeError('Unexpected API mount; refusing to copy deployment state') args += [ '--mount', 'type=bind,source=%s,target=%s,readonly' % (mount['Source'], mount['Destination']), ] args.append(image_id) return args def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default=BASE_URL) parser.add_argument('--duration', type=int, default=90) parser.add_argument('--concurrency', type=int, default=16) parser.add_argument('--output', default='evidence/rolling-update.json') args = parser.parse_args() if args.duration < 30 or args.concurrency < 1: parser.error('duration must be at least 30 seconds and concurrency must be positive') if not (ROOT / 'secrets' / 'database_url').is_file(): parser.error('Run scripts/init-local-secrets.ps1 before the update') old_ids = compose('ps', '-q', 'api').splitlines() if len(old_ids) < 2: parser.error('At least two running API replicas are required') old_image_ids = {inspect(container_id)['Image'] for container_id in old_ids} image = inspect(old_ids[0])['Config']['Image'] revision = 'lab3-' + uuid.uuid4().hex[:12] run( [ 'docker', 'build', '--build-arg', 'BUILD_REVISION=' + revision, '--tag', image, '.', ], capture=False, timeout=300, ) image_id = run(['docker', 'image', 'inspect', '--format', '{{.Id}}', image]) if image_id in old_image_ids: raise RuntimeError('The candidate API image is unchanged; refusing a no-op update') compose('run', '--rm', 'migrate', capture=False, timeout=120) output = (ROOT / args.output).resolve() output.parent.mkdir(parents=True, exist_ok=True) load_output = output.with_name(output.stem + '-load.json') load_process = subprocess.Popen( [ sys.executable, str(ROOT / 'tests' / 'load.py'), '--base-url', args.base_url, '--duration', str(args.duration), '--concurrency', str(args.concurrency), '--output', str(load_output), ], cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) candidates = [] replacements = [] try: time.sleep(2) for index, old_id in enumerate(old_ids, start=1): old = inspect(old_id) instance_id = old['Config']['Hostname'] candidate_name = 'lab3-api-candidate-' + uuid.uuid4().hex[:10] started = time.monotonic() candidate_result = run( candidate_command(old, image_id, candidate_name), timeout=60, ) candidate_id = candidate_result.splitlines()[0] candidates.append(candidate_id) new_instance_id = inspect(candidate_id)['Config']['Hostname'] wait_healthy(candidate_id) wait_routed_to(new_instance_id, args.base_url) run(['docker', 'stop', '--timeout', '15', old_id], timeout=30) run(['docker', 'rm', old_id]) replacements.append( { 'old_instance_id': instance_id, 'new_instance_id': new_instance_id, 'ready_and_routed_seconds': round(time.monotonic() - started, 3), 'sequence': index, } ) compose( 'up', '--detach', '--scale', 'api=2', '--no-deps', 'api', capture=False, timeout=120, ) managed_ids = compose('ps', '-q', 'api').splitlines() if len(managed_ids) != 2: raise RuntimeError('Compose did not restore exactly two managed API replicas') for container_id in managed_ids: wait_healthy(container_id) for candidate_id in candidates: run(['docker', 'rm', '--force', candidate_id]) stdout, stderr = load_process.communicate(timeout=max(90, args.duration + 30)) except Exception: if load_process.poll() is None: load_process.terminate() load_process.wait(timeout=5) raise if load_process.returncode: raise RuntimeError('Load test failed:\n' + (stderr or stdout).strip()) report = { 'operation': 'new API image; serial container replacement with temporary capacity surge', 'duration_seconds': args.duration, 'concurrency': args.concurrency, 'new_image_id': image_id, 'build_revision': revision, 'old_image_ids': sorted(old_image_ids), 'replacements': replacements, 'load': json.loads(load_output.read_text(encoding='utf-8')), 'load_raw_file': load_output.name, 'limitations': [ 'The schema migration in this exercise is additive and backward-compatible.', 'PostgreSQL and Redis are single instances and remain SPOFs.', ], } output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print(json.dumps(report['load'], ensure_ascii=False, indent=2)) print('Saved rolling update evidence to ' + str(output)) if __name__ == '__main__': try: main() except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error: print('ERROR: ' + str(error), file=sys.stderr) raise SystemExit(1)