Files
mtusi2026lab4-fork/scripts/manage.py

158 lines
5.9 KiB
Python

#!/usr/bin/env python3
"""Explicit, bounded operator actions for this lab's Compose project only."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
SERVICES = ('api-a', 'api-b')
def command(args, capture=False):
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE if capture else None)
return result.stdout.strip() if capture else None
def compose(*args, capture=False):
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
def inspect(container):
return json.loads(command(['docker', 'inspect', container], True))[0]
def containers(service):
return compose('ps', '-a', '-q', service, capture=True).splitlines()
def start_services(*services):
for service in services:
for container in containers(service):
command(['docker', 'start', container])
def wait_healthy(service, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
ids = containers(service)
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
return
time.sleep(.5)
raise RuntimeError('%s did not become healthy' % service)
def target_service(target):
if not target:
raise RuntimeError('TARGET_INSTANCE_ID is required')
for service in SERVICES:
for container in containers(service):
data = inspect(container)
if data['Config']['Hostname'] == target:
return service, container
raise RuntimeError('Unknown API instance alias')
def set_backend(service, enabled):
# Fixed allowlist; shell receives no untrusted instance or credential input.
if service not in SERVICES:
raise ValueError('Unknown service')
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
path = '/tmp/control-' + service
compose('exec', '-T', 'nginx', 'sh', '-ec',
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
deadline = time.monotonic() + 10
expected = token + (' up' if enabled else ' down')
while time.monotonic() < deadline:
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
if ack == expected:
return
time.sleep(.2)
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
def stop_one():
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
running = [c for name in SERVICES for c in containers(name)
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
and inspect(c)['State']['Running']]
if len(running) < 2:
raise RuntimeError('Restore the other replica before this experiment')
print('Stopping API replica:', service, container[:12], flush=True)
command(['docker', 'stop', '--time', '8', container])
if inspect(container)['State']['Running']:
raise RuntimeError('Target is still running')
def restart():
# Never start the one-shot migration service as part of persistence testing.
compose('restart', 'postgres', 'redis', 's3', *SERVICES, 'nginx')
for service in (*SERVICES, 'nginx'):
wait_healthy(service)
def rolling_update():
# Run additive migrations once before replacing either API version.
compose('run', '--rm', '--no-deps', 'migrate')
for service in SERVICES:
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
wait_healthy(other)
set_backend(other, True)
before = containers(service)
set_backend(service, False)
# The old nginx worker can retain the old upstream table until it exits.
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
time.sleep(6.5)
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
wait_healthy(service)
set_backend(service, True)
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
after = containers(service)
if before == after:
raise RuntimeError('Replica was not replaced')
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
def restore():
start_services(*SERVICES)
for service in SERVICES:
wait_healthy(service)
for service in SERVICES:
set_backend(service, True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
args = parser.parse_args()
if args.action == 'stop-one':
stop_one()
elif args.action == 'restart':
restart()
elif args.action == 'rolling-update':
rolling_update()
elif args.action == 'restore':
restore()
else:
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
for container in containers(service):
data = inspect(container)
print(json.dumps({'service': service, 'id': data['Id'][:12],
'instance': data['Config']['Hostname'],
'running': data['State']['Running'],
'started_at': data['State']['StartedAt'],
'health': data['State'].get('Health', {}).get('Status'),
'image_id': data['Image'],
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
'memory_limit_bytes': data['HostConfig']['Memory'],
'published_ports': data['HostConfig'].get('PortBindings')}))
if __name__ == '__main__':
main()