Files
mtusi2026lab6/scripts/cluster.py
T
2026-11-10 18:09:59 +03:00

197 lines
8.1 KiB
Python

#!/usr/bin/env python3
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
def command(args, timeout=120):
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
def compose(*args, timeout=120):
return command(['docker', 'compose', *args], timeout=timeout)
def inspect(containers):
if not containers:
return []
return json.loads(command(['docker','inspect',*containers]))
def container_ids(service=None, all_containers=False):
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
return compose(*args).split()
def probe(container, host='127.0.0.1', path='/health/live'):
code = '''import json,sys,urllib.request,urllib.error
try:
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
except urllib.error.HTTPError as exc:
r=exc
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
'''
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
def replicas():
items = []
for data in inspect(container_ids('api')):
if data['State']['Running']:
result = probe(data['Id'])
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
return items
def snapshot():
items = []
for data in inspect(container_ids(all_containers=True)):
service = data['Config']['Labels']['com.docker.compose.service']
item = {
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
'published_ports':data['HostConfig']['PortBindings'],
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
'networks':sorted(data['NetworkSettings']['Networks']),
}
if service == 'api' and data['State']['Running']:
result = probe(data['Id'])
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
items.append(item)
return items
def wait_healthy(container, timeout=60):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
data = inspect([container])[0]
if data['State'].get('Health',{}).get('Status') == 'healthy':
return
time.sleep(0.3)
raise RuntimeError('Container did not become healthy')
def wait_cluster(count=2, timeout=60):
deadline = time.monotonic()+timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
alias = response.headers.get('X-Instance-Id')
if alias:
seen.add(alias)
if len(seen) >= count:
return sorted(seen)
except (OSError, urllib.error.URLError):
pass
time.sleep(0.1)
raise RuntimeError('Load balancer did not discover required API instances')
def stop_one(target):
if not re.fullmatch(r'[a-f0-9]{32}', target):
raise ValueError('Invalid target instance alias')
matching = [item for item in replicas() if item['instance']==target]
if len(matching) != 1:
raise RuntimeError('Target must match exactly one running API process')
item = matching[0]
started = time.monotonic()
command(['docker','stop','--time','7',item['container']],timeout=9)
state = inspect([item['container']])[0]['State']
if state['Running'] or state['ExitCode'] != 0:
raise RuntimeError('API did not stop gracefully')
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
return result
def restart():
compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30)
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
return wait_cluster(2,timeout=15)
def front_address(container):
networks = inspect([container])[0]['NetworkSettings']['Networks']
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
def replace_one(container):
before = set(container_ids('api'))
if container not in before or len(before) != 2:
raise RuntimeError('Replacement expects exactly two running API containers')
old_address = front_address(container)
# Start-first ensures a genuinely different IP while the old address is still occupied.
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
expanded = set(container_ids('api'))
added = expanded-before
if len(added) != 1 or not before.issubset(expanded):
raise RuntimeError('Start-first step unexpectedly changed an existing container')
replacement = added.pop()
wait_healthy(replacement)
new_address = front_address(replacement)
if new_address == old_address:
raise RuntimeError('Replacement did not get a different IP')
expected_three = {item['instance'] for item in replicas()}
if set(wait_cluster(3)) != expected_three:
raise RuntimeError('Load balancer did not discover all three live addresses')
command(['docker','stop','--time','7',container],timeout=10)
state = inspect([container])[0]['State']
if state['ExitCode'] != 0:
raise RuntimeError('Old release did not terminate gracefully')
command(['docker','rm',container])
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
after = set(container_ids('api'))
if after != (before-{container}) | {replacement}:
raise RuntimeError('Sequential replacement changed the wrong containers')
expected = {item['instance'] for item in replicas()}
if set(wait_cluster(2)) != expected:
raise RuntimeError('Unexpected final backend aliases')
return {'old_container':container,'new_container':replacement,
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
'instances_during_start_first':sorted(expected_three),
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
parser.add_argument('arguments',nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.action == 'wait':
print(json.dumps({'instances':wait_cluster()}))
elif args.action == 'snapshot':
print(json.dumps(snapshot(),indent=2))
elif args.action == 'stop-one':
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
elif args.action == 'exec':
ids = container_ids('api')
if not ids or not args.arguments:
raise RuntimeError('A live API container and command are required')
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
elif args.action == 'restart':
print(json.dumps({'instances':restart()}))
else:
compose('run','--rm','--no-deps','migrate')
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
if __name__ == '__main__':
main()