137 lines
5.4 KiB
Python
137 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Operate only this Compose project; never remove a data volume or log secrets."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / 'evidence'
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8').strip()
|
|
|
|
|
|
def compose(*args):
|
|
return command('docker', 'compose', '--env-file', '.env', *args)
|
|
|
|
|
|
def containers(service=None, all_states=False):
|
|
args = ['ps', '-q'] + (['--all'] if all_states else []) + ([service] if service else [])
|
|
ids = compose(*args).split()
|
|
if not ids:
|
|
return []
|
|
raw = json.loads(command('docker', 'inspect', *ids))
|
|
result = []
|
|
for item in raw:
|
|
labels = item['Config']['Labels']
|
|
result.append(dict(id=item['Id'], name=item['Name'].lstrip('/'),
|
|
service=labels['com.docker.compose.service'],
|
|
number=int(labels.get('com.docker.compose.container-number', '1')),
|
|
hostname=item['Config']['Hostname'], pid=item['State']['Pid'],
|
|
started_at=item['State']['StartedAt'], status=item['State']['Status'],
|
|
health=item['State'].get('Health', {}).get('Status'),
|
|
image=item['Image'],
|
|
memory_limit=item['HostConfig']['Memory'],
|
|
cpu_limit=item['HostConfig']['NanoCpus'] / 1e9,
|
|
published_ports=item['NetworkSettings']['Ports'],
|
|
mounts=[dict(name=m.get('Name'), destination=m['Destination'])
|
|
for m in item['Mounts'] if m['Type'] == 'volume']))
|
|
return sorted(result, key=lambda x: (x['service'], x['number']))
|
|
|
|
|
|
def save(name, data):
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
(EVIDENCE / name).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
|
|
|
|
def wait_api(count=2, timeout=45):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
items = containers('api')
|
|
if len(items) == count and all(x['health'] == 'healthy' for x in items):
|
|
# DNS + one successful active health check; LB recovery is verified by HTTP tests.
|
|
time.sleep(3)
|
|
return items
|
|
time.sleep(.5)
|
|
raise RuntimeError('API containers did not become healthy')
|
|
|
|
|
|
def stop_one():
|
|
target = os.environ.get('TARGET_INSTANCE_ID')
|
|
items = containers('api')
|
|
if len(items) < 2:
|
|
raise RuntimeError('Need at least two live API replicas')
|
|
selected = [x for x in items if target and target.startswith(x['hostname'] + '-')]
|
|
if len(selected) != 1:
|
|
raise RuntimeError('TARGET_INSTANCE_ID must identify one live API process')
|
|
started = time.monotonic()
|
|
command('docker', 'stop', '--time', '8', selected[0]['id'])
|
|
after = containers('api', all_states=True)
|
|
assert next(x for x in after if x['id'] == selected[0]['id'])['status'] == 'exited'
|
|
assert len([x for x in after if x['status'] == 'running']) >= 1
|
|
save('stop-one.json', dict(target_instance=target, stopped=selected[0], after=after,
|
|
hook_seconds=time.monotonic() - started))
|
|
print('Stopped selected API process; other API remains running. evidence/stop-one.json')
|
|
|
|
|
|
def restart():
|
|
before = containers()
|
|
compose('restart', 'api', 'db', 'redis', 'lb')
|
|
after = containers()
|
|
for old in before:
|
|
if old['service'] == 'migrate':
|
|
continue
|
|
new = next(x for x in after if x['id'] == old['id'])
|
|
assert old['started_at'] != new['started_at'], old['name'] + ' did not restart'
|
|
assert old['mounts'] == new['mounts'], 'Volumes changed'
|
|
save('restart.json', dict(before=before, after=after))
|
|
print('Restarted both API processes, PostgreSQL, Redis and LB; volumes unchanged.')
|
|
|
|
|
|
def replace_one(identifier):
|
|
# Pin exact Compose-owned ID before stopping/removing. No storage containers.
|
|
old = next(x for x in containers('api') if x['id'] == identifier)
|
|
command('docker', 'stop', '--time', '8', old['id'])
|
|
command('docker', 'rm', old['id'])
|
|
# Keep the other running replica, even if a newly built image is available.
|
|
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
|
items = wait_api()
|
|
# Compose may assign a new replica number; verify a new process exists.
|
|
assert all(x['id'] != identifier for x in items)
|
|
return dict(old=old, after=items)
|
|
|
|
|
|
def rolling():
|
|
initial = containers('api')
|
|
assert len(initial) == 2, 'This grade 4 demonstration uses exactly two replicas'
|
|
changes = []
|
|
for old in initial:
|
|
changes.append(replace_one(old['id']))
|
|
save('rolling-processes.json', dict(before=initial, replacements=changes, after=containers('api')))
|
|
print('Replaced each API container in turn; both have new container IDs.')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('action', choices=['restart', 'stop-one', 'rolling', 'restore', 'state'])
|
|
action = parser.parse_args().action
|
|
if action == 'restart':
|
|
restart()
|
|
elif action == 'stop-one':
|
|
stop_one()
|
|
elif action == 'rolling':
|
|
rolling()
|
|
elif action == 'restore':
|
|
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
|
wait_api()
|
|
else:
|
|
print(json.dumps(containers(all_states=True), indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|