#!/usr/bin/env python3 """Explicit, bounded operator actions for this lab's Compose project only.""" import argparse import json import os from pathlib import Path import subprocess import time import uuid ROOT = Path(__file__).resolve().parents[1] SERVICES = ('api-a', 'api-b') def command(args, capture=False): result = subprocess.run(args, cwd=ROOT, check=True, text=True, stdout=subprocess.PIPE if capture else None) return result.stdout.strip() if capture else None def compose(*args, capture=False): return command(['docker', 'compose', '--env-file', '.env', *args], capture) def inspect(container): return json.loads(command(['docker', 'inspect', container], True))[0] def containers(service): return compose('ps', '-a', '-q', service, capture=True).splitlines() def start_services(*services): for service in services: for container in containers(service): command(['docker', 'start', container]) def wait_healthy(service, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: ids = containers(service) if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids): return time.sleep(.5) raise RuntimeError('%s did not become healthy' % service) def target_service(target): if not target: raise RuntimeError('TARGET_INSTANCE_ID is required') for service in SERVICES: for container in containers(service): data = inspect(container) if data['Config']['Hostname'] == target: return service, container raise RuntimeError('Unknown API instance alias') def set_backend(service, enabled): # Fixed allowlist; shell receives no untrusted instance or credential input. if service not in SERVICES: raise ValueError('Unknown service') token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex path = '/tmp/control-' + service compose('exec', '-T', 'nginx', 'sh', '-ec', "printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path) deadline = time.monotonic() + 10 expected = token + (' up' if enabled else ' down') while time.monotonic() < deadline: ack = compose('exec', '-T', 'nginx', 'sh', '-c', 'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True) if ack == expected: return time.sleep(.2) raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service) def stop_one(): service, container = target_service(os.environ.get('TARGET_INSTANCE_ID')) running = [c for name in SERVICES for c in containers(name) if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' and inspect(c)['State']['Running']] if len(running) < 2: raise RuntimeError('Restore the other replica before this experiment') print('Stopping API replica:', service, container[:12], flush=True) command(['docker', 'stop', '--time', '8', container]) if inspect(container)['State']['Running']: raise RuntimeError('Target is still running') def restart(): # Never start the one-shot migration service as part of persistence testing. compose('restart', 'postgres', 'redis', 's3', *SERVICES, 'nginx') for service in (*SERVICES, 'nginx'): wait_healthy(service) def rolling_update(): # Run additive migrations once before replacing either API version. compose('run', '--rm', '--no-deps', 'migrate') for service in SERVICES: other = SERVICES[1] if service == SERVICES[0] else SERVICES[0] wait_healthy(other) set_backend(other, True) before = containers(service) set_backend(service, False) # The old nginx worker can retain the old upstream table until it exits. # worker_shutdown_timeout=6s bounds that drain; no new work then reaches it. time.sleep(6.5) compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service) wait_healthy(service) set_backend(service, True) time.sleep(2) # DNS valid=1s + readiness before moving to the next replica. after = containers(service) if before == after: raise RuntimeError('Replica was not replaced') print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True) def restore(): start_services(*SERVICES) for service in SERVICES: wait_healthy(service) for service in SERVICES: set_backend(service, True) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status')) args = parser.parse_args() if args.action == 'stop-one': stop_one() elif args.action == 'restart': restart() elif args.action == 'rolling-update': rolling_update() elif args.action == 'restore': restore() else: for service in ('postgres', 'redis', *SERVICES, 'nginx'): for container in containers(service): data = inspect(container) print(json.dumps({'service': service, 'id': data['Id'][:12], 'instance': data['Config']['Hostname'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'health': data['State'].get('Health', {}).get('Status'), 'image_id': data['Image'], 'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9, 'memory_limit_bytes': data['HostConfig']['Memory'], 'published_ports': data['HostConfig'].get('PortBindings')})) if __name__ == '__main__': main()