forked from mtusi2026/mtusi2026lab3
ЛР 3 — БПИ2303 — Унру — на 3
This commit is contained in:
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def command(args, timeout=120):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120):
|
||||
return command(['docker', 'compose', *args], timeout=timeout)
|
||||
|
||||
|
||||
def inspect(containers):
|
||||
if not containers:
|
||||
return []
|
||||
return json.loads(command(['docker','inspect',*containers]))
|
||||
|
||||
|
||||
def container_ids(service=None, all_containers=False):
|
||||
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
||||
return compose(*args).split()
|
||||
|
||||
|
||||
def probe(container, host='127.0.0.1', path='/health/live'):
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
try:
|
||||
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
r=exc
|
||||
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
||||
'''
|
||||
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
||||
|
||||
|
||||
def replicas():
|
||||
items = []
|
||||
for data in inspect(container_ids('api')):
|
||||
if data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
||||
return items
|
||||
|
||||
|
||||
def snapshot():
|
||||
items = []
|
||||
for data in inspect(container_ids(all_containers=True)):
|
||||
service = data['Config']['Labels']['com.docker.compose.service']
|
||||
item = {
|
||||
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
||||
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
||||
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
||||
'published_ports':data['HostConfig']['PortBindings'],
|
||||
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
||||
'networks':sorted(data['NetworkSettings']['Networks']),
|
||||
}
|
||||
if service == 'api' and data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def wait_healthy(container, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
data = inspect([container])[0]
|
||||
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError('Container did not become healthy')
|
||||
|
||||
|
||||
def wait_cluster(count=2, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
||||
alias = response.headers.get('X-Instance-Id')
|
||||
if alias:
|
||||
seen.add(alias)
|
||||
if len(seen) >= count:
|
||||
return sorted(seen)
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError('Load balancer did not discover required API instances')
|
||||
|
||||
|
||||
def stop_one(target):
|
||||
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
||||
raise ValueError('Invalid target instance alias')
|
||||
matching = [item for item in replicas() if item['instance']==target]
|
||||
if len(matching) != 1:
|
||||
raise RuntimeError('Target must match exactly one running API process')
|
||||
item = matching[0]
|
||||
started = time.monotonic()
|
||||
command(['docker','stop','--time','7',item['container']],timeout=9)
|
||||
state = inspect([item['container']])[0]['State']
|
||||
if state['Running'] or state['ExitCode'] != 0:
|
||||
raise RuntimeError('API did not stop gracefully')
|
||||
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
||||
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
return result
|
||||
|
||||
|
||||
def restart():
|
||||
compose('restart','api','postgres','redis','lb',timeout=30)
|
||||
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
||||
return wait_cluster(2,timeout=15)
|
||||
|
||||
|
||||
def front_address(container):
|
||||
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
||||
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
||||
|
||||
|
||||
def replace_one(container):
|
||||
before = set(container_ids('api'))
|
||||
if container not in before or len(before) != 2:
|
||||
raise RuntimeError('Replacement expects exactly two running API containers')
|
||||
old_address = front_address(container)
|
||||
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
||||
expanded = set(container_ids('api'))
|
||||
added = expanded-before
|
||||
if len(added) != 1 or not before.issubset(expanded):
|
||||
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
||||
replacement = added.pop()
|
||||
wait_healthy(replacement)
|
||||
new_address = front_address(replacement)
|
||||
if new_address == old_address:
|
||||
raise RuntimeError('Replacement did not get a different IP')
|
||||
expected_three = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(3)) != expected_three:
|
||||
raise RuntimeError('Load balancer did not discover all three live addresses')
|
||||
command(['docker','stop','--time','7',container],timeout=10)
|
||||
state = inspect([container])[0]['State']
|
||||
if state['ExitCode'] != 0:
|
||||
raise RuntimeError('Old release did not terminate gracefully')
|
||||
command(['docker','rm',container])
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
||||
after = set(container_ids('api'))
|
||||
if after != (before-{container}) | {replacement}:
|
||||
raise RuntimeError('Sequential replacement changed the wrong containers')
|
||||
expected = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(2)) != expected:
|
||||
raise RuntimeError('Unexpected final backend aliases')
|
||||
return {'old_container':container,'new_container':replacement,
|
||||
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
||||
'instances_during_start_first':sorted(expected_three),
|
||||
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
||||
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'wait':
|
||||
print(json.dumps({'instances':wait_cluster()}))
|
||||
elif args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(),indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
||||
elif args.action == 'exec':
|
||||
ids = container_ids('api')
|
||||
if not ids or not args.arguments:
|
||||
raise RuntimeError('A live API container and command are required')
|
||||
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
||||
elif args.action == 'restart':
|
||||
print(json.dumps({'instances':restart()}))
|
||||
else:
|
||||
compose('run','--rm','--no-deps','migrate')
|
||||
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user