#!/usr/bin/env python3 """Local Compose orchestration. No credentials, tokens or environment dumps in evidence.""" import argparse import datetime import json import os import re import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') def command(args, timeout=120): return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip() def compose(*args, timeout=120): return command(['docker', 'compose', *args], timeout=timeout) def inspect(containers): if not containers: return [] return json.loads(command(['docker','inspect',*containers])) def container_ids(service=None, all_containers=False): args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else []) return compose(*args).split() def probe(container, host='127.0.0.1', path='/health/live'): code = '''import json,sys,urllib.request,urllib.error try: r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4) except urllib.error.HTTPError as exc: r=exc print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')})) ''' return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8)) def replicas(): items = [] for data in inspect(container_ids('api')): if data['State']['Running']: result = probe(data['Id']) items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'), 'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']}) return items def snapshot(): items = [] for data in inspect(container_ids(all_containers=True)): service = data['Config']['Labels']['com.docker.compose.service'] item = { 'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'), 'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'), 'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'], 'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'], 'published_ports':data['HostConfig']['PortBindings'], 'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'), 'networks':sorted(data['NetworkSettings']['Networks']), } if service == 'api' and data['State']['Running']: result = probe(data['Id']) item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']}) items.append(item) return items def wait_healthy(container, timeout=60): deadline = time.monotonic()+timeout while time.monotonic() < deadline: data = inspect([container])[0] if data['State'].get('Health',{}).get('Status') == 'healthy': return time.sleep(0.3) raise RuntimeError('Container did not become healthy') def wait_cluster(count=2, timeout=60): deadline = time.monotonic()+timeout seen = set() while time.monotonic() < deadline: try: with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response: alias = response.headers.get('X-Instance-Id') if alias: seen.add(alias) if len(seen) >= count: return sorted(seen) except (OSError, urllib.error.URLError): pass time.sleep(0.1) raise RuntimeError('Load balancer did not discover required API instances') def stop_one(target): if not re.fullmatch(r'[a-f0-9]{32}', target): raise ValueError('Invalid target instance alias') matching = [item for item in replicas() if item['instance']==target] if len(matching) != 1: raise RuntimeError('Target must match exactly one running API process') item = matching[0] started = time.monotonic() command(['docker','stop','--time','7',item['container']],timeout=9) state = inspect([item['container']])[0]['State'] if state['Running'] or state['ExitCode'] != 0: raise RuntimeError('API did not stop gracefully') result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True} (ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n') return result def restart(): compose('restart','api','postgres','redis','minio','cleanup','lb',timeout=30) compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40) return wait_cluster(2,timeout=15) def front_address(container): networks = inspect([container])[0]['NetworkSettings']['Networks'] return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front')) def replace_one(container): before = set(container_ids('api')) if container not in before or len(before) != 2: raise RuntimeError('Replacement expects exactly two running API containers') old_address = front_address(container) # Start-first ensures a genuinely different IP while the old address is still occupied. compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api') expanded = set(container_ids('api')) added = expanded-before if len(added) != 1 or not before.issubset(expanded): raise RuntimeError('Start-first step unexpectedly changed an existing container') replacement = added.pop() wait_healthy(replacement) new_address = front_address(replacement) if new_address == old_address: raise RuntimeError('Replacement did not get a different IP') expected_three = {item['instance'] for item in replicas()} if set(wait_cluster(3)) != expected_three: raise RuntimeError('Load balancer did not discover all three live addresses') command(['docker','stop','--time','7',container],timeout=10) state = inspect([container])[0]['State'] if state['ExitCode'] != 0: raise RuntimeError('Old release did not terminate gracefully') command(['docker','rm',container]) compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api') after = set(container_ids('api')) if after != (before-{container}) | {replacement}: raise RuntimeError('Sequential replacement changed the wrong containers') expected = {item['instance'] for item in replicas()} if set(wait_cluster(2)) != expected: raise RuntimeError('Unexpected final backend aliases') return {'old_container':container,'new_container':replacement, 'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True, 'instances_during_start_first':sorted(expected_three), 'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']} def main(): parser = argparse.ArgumentParser() parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec']) parser.add_argument('arguments',nargs=argparse.REMAINDER) args = parser.parse_args() if args.action == 'wait': print(json.dumps({'instances':wait_cluster()})) elif args.action == 'snapshot': print(json.dumps(snapshot(),indent=2)) elif args.action == 'stop-one': print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID']))) elif args.action == 'exec': ids = container_ids('api') if not ids or not args.arguments: raise RuntimeError('A live API container and command are required') raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT)) elif args.action == 'restart': print(json.dumps({'instances':restart()})) else: compose('run','--rm','--no-deps','migrate') print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2)) if __name__ == '__main__': main()