forked from mtusi2026/mtusi2026lab3
81 lines
3.2 KiB
Python
81 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Replace each API container using a healthy temporary extra replica (surge=1)."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COMPOSE = str(ROOT / 'scripts/compose.sh')
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
|
|
|
|
|
def ids():
|
|
return command(COMPOSE, 'ps', '-q', 'api').split()
|
|
|
|
|
|
def inspect(cid):
|
|
result = json.loads(command('docker', 'inspect', cid))[0]
|
|
return {'id': result['Id'], 'hostname': result['Config']['Hostname'],
|
|
'image_id': result['Image'], 'image': result['Config']['Image'],
|
|
'status': result['State']['Status'],
|
|
'health': result['State'].get('Health', {}).get('Status')}
|
|
|
|
|
|
def main():
|
|
original = ids()
|
|
if len(original) < 2:
|
|
raise RuntimeError('rolling update requires at least two running replicas')
|
|
base = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
|
evidence = {'before': [inspect(i) for i in original], 'steps': []}
|
|
try:
|
|
for old in original:
|
|
before = set(ids())
|
|
subprocess.run([COMPOSE, 'up', '-d', '--no-deps', '--no-recreate',
|
|
'--scale', 'api=' + str(len(original)+1), 'api'], check=True, cwd=ROOT)
|
|
added = set(ids()) - before
|
|
if len(added) != 1:
|
|
raise RuntimeError('expected exactly one spare container')
|
|
spare = added.pop()
|
|
deadline = time.monotonic() + 60
|
|
while True:
|
|
info = inspect(spare)
|
|
# Healthy container must actually be observed behind the public LB.
|
|
seen = False
|
|
if info['health'] == 'healthy':
|
|
try:
|
|
for _ in range(2 * (len(original) + 1)):
|
|
with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
|
|
if response.headers.get('X-Instance-Id') == info['hostname']:
|
|
seen = True
|
|
break
|
|
except (OSError, urllib.error.URLError):
|
|
pass
|
|
if seen:
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError('spare did not become healthy and reachable via LB')
|
|
time.sleep(.3)
|
|
subprocess.run(['docker', 'stop', '--time', '12', old], check=True, stdout=subprocess.DEVNULL)
|
|
stopped = inspect(old)
|
|
subprocess.run(['docker', 'rm', old], check=True, stdout=subprocess.DEVNULL)
|
|
evidence['steps'].append({'removed': stopped, 'replacement': info})
|
|
evidence['after'] = [inspect(i) for i in ids()]
|
|
assert len(evidence['after']) == len(original)
|
|
assert not set(original) & set(ids())
|
|
evidence['status'] = 'PASS'
|
|
finally:
|
|
dest = ROOT / 'evidence/rolling-containers.json'
|
|
dest.write_text(json.dumps(evidence, indent=2) + '\n')
|
|
print('PASS: all original API containers replaced; evidence/rolling-containers.json')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|