197 lines
8.1 KiB
Python
197 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
|
|
|
|
|
def command(args, timeout=120):
|
|
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
|
|
|
|
|
def compose(*args, timeout=120):
|
|
return command(['docker', 'compose', *args], timeout=timeout)
|
|
|
|
|
|
def inspect(containers):
|
|
if not containers:
|
|
return []
|
|
return json.loads(command(['docker','inspect',*containers]))
|
|
|
|
|
|
def container_ids(service=None, all_containers=False):
|
|
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
|
return compose(*args).split()
|
|
|
|
|
|
def probe(container, host='127.0.0.1', path='/health/live'):
|
|
code = '''import json,sys,urllib.request,urllib.error
|
|
try:
|
|
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
|
except urllib.error.HTTPError as exc:
|
|
r=exc
|
|
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
|
'''
|
|
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
|
|
|
|
|
def replicas():
|
|
items = []
|
|
for data in inspect(container_ids('api')):
|
|
if data['State']['Running']:
|
|
result = probe(data['Id'])
|
|
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
|
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
|
return items
|
|
|
|
|
|
def snapshot():
|
|
items = []
|
|
for data in inspect(container_ids(all_containers=True)):
|
|
service = data['Config']['Labels']['com.docker.compose.service']
|
|
item = {
|
|
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
|
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
|
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
|
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
|
'published_ports':data['HostConfig']['PortBindings'],
|
|
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
|
'networks':sorted(data['NetworkSettings']['Networks']),
|
|
}
|
|
if service == 'api' and data['State']['Running']:
|
|
result = probe(data['Id'])
|
|
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
|
items.append(item)
|
|
return items
|
|
|
|
|
|
def wait_healthy(container, timeout=60):
|
|
deadline = time.monotonic()+timeout
|
|
while time.monotonic() < deadline:
|
|
data = inspect([container])[0]
|
|
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
|
return
|
|
time.sleep(0.3)
|
|
raise RuntimeError('Container did not become healthy')
|
|
|
|
|
|
def wait_cluster(count=2, timeout=60):
|
|
deadline = time.monotonic()+timeout
|
|
seen = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
|
alias = response.headers.get('X-Instance-Id')
|
|
if alias:
|
|
seen.add(alias)
|
|
if len(seen) >= count:
|
|
return sorted(seen)
|
|
except (OSError, urllib.error.URLError):
|
|
pass
|
|
time.sleep(0.1)
|
|
raise RuntimeError('Load balancer did not discover required API instances')
|
|
|
|
|
|
def stop_one(target):
|
|
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
|
raise ValueError('Invalid target instance alias')
|
|
matching = [item for item in replicas() if item['instance']==target]
|
|
if len(matching) != 1:
|
|
raise RuntimeError('Target must match exactly one running API process')
|
|
item = matching[0]
|
|
started = time.monotonic()
|
|
command(['docker','stop','--time','7',item['container']],timeout=9)
|
|
state = inspect([item['container']])[0]['State']
|
|
if state['Running'] or state['ExitCode'] != 0:
|
|
raise RuntimeError('API did not stop gracefully')
|
|
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
|
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
|
return result
|
|
|
|
|
|
def restart():
|
|
compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30)
|
|
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
|
return wait_cluster(2,timeout=15)
|
|
|
|
|
|
def front_address(container):
|
|
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
|
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
|
|
|
|
|
def replace_one(container):
|
|
before = set(container_ids('api'))
|
|
if container not in before or len(before) != 2:
|
|
raise RuntimeError('Replacement expects exactly two running API containers')
|
|
old_address = front_address(container)
|
|
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
|
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
|
expanded = set(container_ids('api'))
|
|
added = expanded-before
|
|
if len(added) != 1 or not before.issubset(expanded):
|
|
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
|
replacement = added.pop()
|
|
wait_healthy(replacement)
|
|
new_address = front_address(replacement)
|
|
if new_address == old_address:
|
|
raise RuntimeError('Replacement did not get a different IP')
|
|
expected_three = {item['instance'] for item in replicas()}
|
|
if set(wait_cluster(3)) != expected_three:
|
|
raise RuntimeError('Load balancer did not discover all three live addresses')
|
|
command(['docker','stop','--time','7',container],timeout=10)
|
|
state = inspect([container])[0]['State']
|
|
if state['ExitCode'] != 0:
|
|
raise RuntimeError('Old release did not terminate gracefully')
|
|
command(['docker','rm',container])
|
|
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
|
after = set(container_ids('api'))
|
|
if after != (before-{container}) | {replacement}:
|
|
raise RuntimeError('Sequential replacement changed the wrong containers')
|
|
expected = {item['instance'] for item in replicas()}
|
|
if set(wait_cluster(2)) != expected:
|
|
raise RuntimeError('Unexpected final backend aliases')
|
|
return {'old_container':container,'new_container':replacement,
|
|
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
|
'instances_during_start_first':sorted(expected_three),
|
|
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
|
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
|
args = parser.parse_args()
|
|
if args.action == 'wait':
|
|
print(json.dumps({'instances':wait_cluster()}))
|
|
elif args.action == 'snapshot':
|
|
print(json.dumps(snapshot(),indent=2))
|
|
elif args.action == 'stop-one':
|
|
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
|
elif args.action == 'exec':
|
|
ids = container_ids('api')
|
|
if not ids or not args.arguments:
|
|
raise RuntimeError('A live API container and command are required')
|
|
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
|
elif args.action == 'restart':
|
|
print(json.dumps({'instances':restart()}))
|
|
else:
|
|
compose('run','--rm','--no-deps','migrate')
|
|
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|