Files

75 lines
3.5 KiB
Python

import json
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.check_output(['docker', *args], cwd=ROOT, text=True).strip()
def containers():
ids = docker('compose', 'ps', '-q', 'api').split()
return json.loads(docker('inspect', *ids)) if ids else []
def main():
mode = sys.argv[1]
if mode == 'restart':
subprocess.run(['docker', 'compose', 'restart', 'postgres', 's3', 'api', 'lb'], cwd=ROOT, check=True)
elif mode == 'stop':
target = os.environ.get('TARGET_INSTANCE_ID')
matches = [item for item in containers() if item['Config']['Hostname'] == target]
if len(matches) != 1:
raise SystemExit('Target must identify exactly one running API container')
print('Stopping API container ' + matches[0]['Id'][:12])
docker('stop', '-t', '2', matches[0]['Id'])
state = json.loads(docker('inspect', matches[0]['Id']))[0]
event = {'target_instance': target, 'container_id': state['Id'][:12],
'name': state['Name'], 'status_after': state['State']['Status'],
'finished_at': state['State']['FinishedAt']}
with (ROOT / 'evidence/stop-events.jsonl').open('a', encoding='utf-8') as output:
output.write(json.dumps(event) + '\n')
elif mode == 'rolling':
import time
import urllib.request
for item in containers():
name = item['Name'].lstrip('/')
started = time.time()
stages = {}
docker('stop', '-t', '6', item['Id'])
stages['stopped_seconds'] = round(time.time() - started, 3)
docker('rm', item['Id'])
stages['removed_seconds'] = round(time.time() - started, 3)
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
stages['created_seconds'] = round(time.time() - started, 3)
deadline = time.monotonic() + 45
observations = []
previous = None
while time.monotonic() < deadline:
active = containers()
state = [(c['Id'][:12], c.get('State', {}).get('Health', {}).get('Status')) for c in active]
if state != previous:
observations.append({'elapsed_seconds': round(time.time()-started, 3), 'containers': [{'id': c['Id'][:12], 'health': c.get('State', {}).get('Health', {})} for c in active]})
previous = state
if len(active) == 2 and all(c.get('State', {}).get('Health', {}).get('Status') == 'healthy' for c in active):
break
time.sleep(0.5)
else:
raise SystemExit('Replacement did not become healthy: ' + name)
stages['healthy_seconds'] = round(time.time() - started, 3)
with urllib.request.urlopen(os.getenv('BASE_URL', 'http://localhost:8080').rstrip('/') + '/health/ready', timeout=3) as response:
if response.status != 200:
raise SystemExit('LB unavailable')
time.sleep(3)
print(json.dumps({'event': 'rolling_replacement', 'old_name': name, 'old_container': item['Id'][:12], 'started_epoch': started, 'finished_epoch': time.time(), 'stages': stages, 'health_observations': observations, 'new_containers': [c['Id'][:12] for c in active]}), flush=True)
else:
raise SystemExit('Unknown operation')
if __name__ == '__main__':
main()