forked from mtusi2026/mtusi2026lab3
146 lines
5.9 KiB
Python
146 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Operate ONLY containers selected by the lab3 Compose project/service labels."""
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COMPOSE = ['docker', 'compose', '--env-file', '.env']
|
|
|
|
|
|
def command(args, env=None):
|
|
result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
|
if result.returncode:
|
|
# Commands never contain credential values. Never dump Docker inspection/env.
|
|
raise RuntimeError('Infrastructure command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1200:])
|
|
return result.stdout.strip()
|
|
|
|
|
|
def containers(service=None, running=False):
|
|
args = ['docker', 'ps'] + ([] if running else ['-a'])
|
|
args += ['--filter', 'label=com.docker.compose.project=mtusi-lab3']
|
|
if service:
|
|
args += ['--filter', 'label=com.docker.compose.service=' + service]
|
|
ids = command(args + ['-q']).split()
|
|
if not ids:
|
|
return []
|
|
result = []
|
|
for item in json.loads(command(['docker', 'inspect'] + ids)):
|
|
result.append({'id': item['Id'][:12], 'name': item['Name'].lstrip('/'),
|
|
'service': item['Config']['Labels']['com.docker.compose.service'],
|
|
'instance': item['Config']['Hostname'], 'image': item['Image'],
|
|
'status': item['State']['Status'], 'started': item['State']['StartedAt'],
|
|
'health': item['State'].get('Health', {}).get('Status'),
|
|
'nano_cpus': item['HostConfig']['NanoCpus'],
|
|
'memory_bytes': item['HostConfig']['Memory'],
|
|
'published_ports': item['HostConfig']['PortBindings']})
|
|
return sorted(result, key=lambda x: x['name'])
|
|
|
|
|
|
def base_url():
|
|
settings = dict(line.split('=', 1) for line in (ROOT / '.env').read_text().splitlines()
|
|
if line and not line.startswith('#') and '=' in line)
|
|
return os.getenv('BASE_URL', 'http://127.0.0.1:' + settings.get('HOST_PORT', '8083'))
|
|
|
|
|
|
def wait_instances(expected=2, timeout=90, required=None):
|
|
deadline = time.monotonic() + timeout
|
|
observed = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(base_url() + '/health/ready', timeout=2) as response:
|
|
observed.add(response.headers['X-Instance-Id'])
|
|
if len(observed) >= expected and (required is None or required in observed):
|
|
return sorted(observed)
|
|
except OSError:
|
|
observed.clear()
|
|
time.sleep(.1)
|
|
raise RuntimeError('LB did not expose the expected healthy replicas')
|
|
|
|
|
|
def record(kind, before, after, **extra):
|
|
value = {'action': kind, 'at_utc': dt.datetime.now(dt.timezone.utc).isoformat(),
|
|
'before': before, 'after': after, **extra}
|
|
folder = ROOT / 'evidence' / 'actions'
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
name = kind + '-' + str(time.time_ns()) + '.json'
|
|
(folder / name).write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
|
|
return value
|
|
|
|
|
|
def stop(target=None, hard=False):
|
|
target = target or os.environ.get('TARGET_INSTANCE_ID')
|
|
before = containers('api', running=True)
|
|
matches = [x for x in before if x['instance'] == target]
|
|
if len(matches) != 1 or len(before) < 2:
|
|
raise RuntimeError('Target must identify exactly one running lab3 API; another replica must remain')
|
|
victim = matches[0]
|
|
if hard:
|
|
command(['docker', 'kill', victim['id']])
|
|
else:
|
|
command(['docker', 'stop', '-t', '15', victim['id']])
|
|
after = containers('api')
|
|
assert next(x for x in after if x['id'] == victim['id'])['status'] == 'exited'
|
|
return record('kill' if hard else 'stop', before, after, target=target)
|
|
|
|
|
|
def restore():
|
|
command(COMPOSE + ['up', '-d', '--no-deps', '--scale', 'api=2', 'api'])
|
|
return wait_instances(2)
|
|
|
|
|
|
def restart():
|
|
before = containers()
|
|
command(COMPOSE + ['restart', 'postgres', 'redis', 'api', 'lb'])
|
|
wait_instances(2)
|
|
after = containers()
|
|
for old in before:
|
|
if old['service'] in ('postgres', 'redis', 'api', 'lb'):
|
|
new = next(x for x in after if x['id'] == old['id'])
|
|
assert new['started'] != old['started'], 'container was not restarted'
|
|
return record('restart', before, after)
|
|
|
|
|
|
def rolling():
|
|
originals = containers('api', running=True)
|
|
count = len(originals)
|
|
if not 2 <= count < 8:
|
|
raise RuntimeError('Rolling replacement requires 2..7 healthy replicas')
|
|
replacements = []
|
|
for old in originals:
|
|
before = containers('api', running=True)
|
|
command(COMPOSE + ['up', '-d', '--no-deps', '--no-recreate', '--scale', f'api={count+1}', 'api'])
|
|
added = [x for x in containers('api', running=True) if x['id'] not in {y['id'] for y in before}]
|
|
assert len(added) == 1, 'exactly one surge replica expected'
|
|
new = added[0]
|
|
wait_instances(count + 1, required=new['instance'])
|
|
command(['docker', 'stop', '-t', '15', old['id']])
|
|
command(['docker', 'rm', old['id']])
|
|
wait_instances(count, required=new['instance'])
|
|
replacements.append({'old': old, 'new': new})
|
|
after = containers('api', running=True)
|
|
assert len(after) == count
|
|
assert not {x['id'] for x in originals} & {x['id'] for x in after}
|
|
return record('rolling', originals, after, replacements=replacements)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('action', choices=['stop', 'kill', 'restart', 'restore', 'rolling', 'status'])
|
|
parser.add_argument('--target')
|
|
args = parser.parse_args()
|
|
if args.action in ('stop', 'kill'):
|
|
result = stop(args.target, args.action == 'kill')
|
|
else:
|
|
result = globals()[args.action]() if args.action != 'status' else containers()
|
|
print(json.dumps(result, ensure_ascii=True, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|