158 lines
5.9 KiB
Python
158 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Explicit, bounded operator actions for this lab's Compose project only."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SERVICES = ('api-a', 'api-b')
|
|
|
|
|
|
def command(args, capture=False):
|
|
result = subprocess.run(args, cwd=ROOT, check=True, text=True,
|
|
stdout=subprocess.PIPE if capture else None)
|
|
return result.stdout.strip() if capture else None
|
|
|
|
|
|
def compose(*args, capture=False):
|
|
return command(['docker', 'compose', '--env-file', '.env', *args], capture)
|
|
|
|
|
|
def inspect(container):
|
|
return json.loads(command(['docker', 'inspect', container], True))[0]
|
|
|
|
|
|
def containers(service):
|
|
return compose('ps', '-a', '-q', service, capture=True).splitlines()
|
|
|
|
|
|
def start_services(*services):
|
|
for service in services:
|
|
for container in containers(service):
|
|
command(['docker', 'start', container])
|
|
|
|
|
|
def wait_healthy(service, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
ids = containers(service)
|
|
if ids and all(inspect(c)['State'].get('Health', {}).get('Status') == 'healthy' for c in ids):
|
|
return
|
|
time.sleep(.5)
|
|
raise RuntimeError('%s did not become healthy' % service)
|
|
|
|
|
|
def target_service(target):
|
|
if not target:
|
|
raise RuntimeError('TARGET_INSTANCE_ID is required')
|
|
for service in SERVICES:
|
|
for container in containers(service):
|
|
data = inspect(container)
|
|
if data['Config']['Hostname'] == target:
|
|
return service, container
|
|
raise RuntimeError('Unknown API instance alias')
|
|
|
|
|
|
def set_backend(service, enabled):
|
|
# Fixed allowlist; shell receives no untrusted instance or credential input.
|
|
if service not in SERVICES:
|
|
raise ValueError('Unknown service')
|
|
token = ('serve:' if enabled else 'drain:') + uuid.uuid4().hex
|
|
path = '/tmp/control-' + service
|
|
compose('exec', '-T', 'nginx', 'sh', '-ec',
|
|
"printf '%s\\n' '" + token + "' > " + path + '.next; mv -f ' + path + '.next ' + path)
|
|
deadline = time.monotonic() + 10
|
|
expected = token + (' up' if enabled else ' down')
|
|
while time.monotonic() < deadline:
|
|
ack = compose('exec', '-T', 'nginx', 'sh', '-c',
|
|
'cat /tmp/applied-' + service + ' 2>/dev/null || true', capture=True)
|
|
if ack == expected:
|
|
return
|
|
time.sleep(.2)
|
|
raise RuntimeError('Readiness controller did not acknowledge backend state for ' + service)
|
|
|
|
|
|
def stop_one():
|
|
service, container = target_service(os.environ.get('TARGET_INSTANCE_ID'))
|
|
running = [c for name in SERVICES for c in containers(name)
|
|
if inspect(c)['State'].get('Health', {}).get('Status') == 'healthy'
|
|
and inspect(c)['State']['Running']]
|
|
if len(running) < 2:
|
|
raise RuntimeError('Restore the other replica before this experiment')
|
|
print('Stopping API replica:', service, container[:12], flush=True)
|
|
command(['docker', 'stop', '--time', '8', container])
|
|
if inspect(container)['State']['Running']:
|
|
raise RuntimeError('Target is still running')
|
|
|
|
|
|
def restart():
|
|
# Never start the one-shot migration service as part of persistence testing.
|
|
compose('restart', 'postgres', 'redis', 's3', 'rabbitmq', *SERVICES, 'nginx', 'outbox-publisher', 'worker')
|
|
for service in (*SERVICES, 'nginx'):
|
|
wait_healthy(service)
|
|
|
|
|
|
def rolling_update():
|
|
# Run additive migrations once before replacing either API version.
|
|
compose('run', '--rm', '--no-deps', 'migrate')
|
|
for service in SERVICES:
|
|
other = SERVICES[1] if service == SERVICES[0] else SERVICES[0]
|
|
wait_healthy(other)
|
|
set_backend(other, True)
|
|
before = containers(service)
|
|
set_backend(service, False)
|
|
# The old nginx worker can retain the old upstream table until it exits.
|
|
# worker_shutdown_timeout=6s bounds that drain; no new work then reaches it.
|
|
time.sleep(6.5)
|
|
compose('up', '-d', '--no-deps', '--force-recreate', '--no-build', service)
|
|
wait_healthy(service)
|
|
set_backend(service, True)
|
|
time.sleep(2) # DNS valid=1s + readiness before moving to the next replica.
|
|
after = containers(service)
|
|
if before == after:
|
|
raise RuntimeError('Replica was not replaced')
|
|
print(json.dumps({'service': service, 'before': before, 'after': after}), flush=True)
|
|
|
|
|
|
def restore():
|
|
start_services(*SERVICES)
|
|
for service in SERVICES:
|
|
wait_healthy(service)
|
|
for service in SERVICES:
|
|
set_backend(service, True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('action', choices=('stop-one', 'restart', 'restore', 'rolling-update', 'status'))
|
|
args = parser.parse_args()
|
|
if args.action == 'stop-one':
|
|
stop_one()
|
|
elif args.action == 'restart':
|
|
restart()
|
|
elif args.action == 'rolling-update':
|
|
rolling_update()
|
|
elif args.action == 'restore':
|
|
restore()
|
|
else:
|
|
for service in ('postgres', 'redis', *SERVICES, 'nginx'):
|
|
for container in containers(service):
|
|
data = inspect(container)
|
|
print(json.dumps({'service': service, 'id': data['Id'][:12],
|
|
'instance': data['Config']['Hostname'],
|
|
'running': data['State']['Running'],
|
|
'started_at': data['State']['StartedAt'],
|
|
'health': data['State'].get('Health', {}).get('Status'),
|
|
'image_id': data['Image'],
|
|
'cpu_limit': data['HostConfig']['NanoCpus'] / 1e9,
|
|
'memory_limit_bytes': data['HostConfig']['Memory'],
|
|
'published_ports': data['HostConfig'].get('PortBindings')}))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|