80 lines
3.2 KiB
Python
Executable File
80 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Sequential replacement on a single Compose host; run build/migrate first.
|
|
No schema changes are applied here. Compatible old/new APIs are required.
|
|
"""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COMPOSE = [str(ROOT / 'scripts/compose.sh')]
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
|
|
|
|
|
|
def containers():
|
|
return command(*COMPOSE, 'ps', '-q', 'api').splitlines()
|
|
|
|
|
|
def instance(container):
|
|
return command('docker', 'exec', container, 'python', '-c',
|
|
'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/ready",timeout=2); print(r.headers["X-Instance-Id"])')
|
|
|
|
|
|
def rolling_update():
|
|
originals = containers()
|
|
if len(originals) != 2:
|
|
raise RuntimeError('Restore exactly two healthy API replicas first')
|
|
for container in originals:
|
|
instance(container)
|
|
port = dict(line.split('=', 1) for line in (ROOT / '.env').read_text().splitlines() if '=' in line)['PUBLIC_PORT']
|
|
events = []
|
|
for old in originals:
|
|
before = set(containers())
|
|
survivor = next(c for c in before if c != old)
|
|
instance(survivor)
|
|
old_alias = instance(old)
|
|
old_image = command("docker", "inspect", "--format", "{{.Image}}", old)
|
|
old_ip = command("docker", "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", old)
|
|
started = time.monotonic()
|
|
command('docker', 'stop', '--timeout', '6', old)
|
|
command('docker', 'rm', old)
|
|
# --no-recreate keeps the healthy survivor, even after an image build.
|
|
command(*COMPOSE, 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api')
|
|
after = set(containers())
|
|
assert survivor in after and old not in after
|
|
new = next(c for c in after if c not in before)
|
|
deadline = time.monotonic() + 45
|
|
seen = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
new_alias = instance(new)
|
|
with urllib.request.urlopen(f'http://127.0.0.1:{port}/health/ready', timeout=2) as r:
|
|
seen.add(r.headers['X-Instance-Id'])
|
|
if new_alias in seen and instance(survivor) in seen:
|
|
break
|
|
except Exception:
|
|
pass
|
|
time.sleep(.2)
|
|
else:
|
|
raise RuntimeError('Replacement did not join LB')
|
|
event = {'old_container': old, 'old_instance': old_alias,
|
|
'new_container': new, 'new_instance': new_alias,
|
|
'survivor_container': survivor,
|
|
'old_image': old_image,
|
|
'new_image': command('docker', 'inspect', '--format', '{{.Image}}', new),
|
|
'old_ip': old_ip,
|
|
'new_ip': command('docker', 'inspect', '--format', '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}', new),
|
|
'replacement_seconds': round(time.monotonic()-started, 3)}
|
|
events.append(event)
|
|
print(json.dumps(event), flush=True)
|
|
return events
|
|
|
|
|
|
if __name__ == '__main__':
|
|
rolling_update()
|