Files

122 lines
5.3 KiB
Bash
Executable File

#!/bin/sh
# Secure Compose surge rollout: inherited API identity/config/hardening, one replica at a time.
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
exec "${PYTHON:-python3}" - "$@" <<'PY'
import csv
import io
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import time
import uuid
sys.path.insert(0, str(Path.cwd() / 'scripts'))
sys.path.insert(0, str(Path.cwd() / 'tests'))
from lab7_environment import Environment, ROOT
from lab7_http import BASE_URL, CA_FILE, open_url
from client import Client
environment = Environment()
output = Path(os.environ.get('OUT', 'evidence/rolling-update'))
output.mkdir(parents=True, exist_ok=True)
image = os.environ.get('IMAGE', 'mtusi-messenger:lab7')
stats_url = os.environ.get('STATS', 'https://localhost:' + environment.value('LB_STATS_PORT', '8404') + '/stats;csv')
replicas = environment.compose('ps', '-q', 'api').stdout.split()
if len(replicas) < 2:
raise SystemExit('rolling update requires at least two running API replicas')
environment.run(['docker', 'image', 'inspect', image])
client = Client(BASE_URL, CA_FILE, timeout=5)
log = []
surge = []
loader = None
def rows():
with open_url(stats_url, operator=True, ca_file=CA_FILE) as response:
content = response.read().decode()
return list(csv.DictReader(io.StringIO(content.replace('# ', '', 1))))
def up_slots():
return sum(row['pxname'] == 'api_backend' and row['svname'] not in ('BACKEND', 'FRONTEND')
and row['status'].startswith('UP') for row in rows())
def wait_slots(expected):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if up_slots() == expected:
client.call('GET', '/health/ready', schema='Health')
return
time.sleep(1)
raise RuntimeError('LB did not converge to %d healthy business slots' % expected)
def wait_container(container):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
result = environment.run(['docker', 'exec', container, 'wget', '-qO-', '-T', '3',
'http://127.0.0.1:8084/health/ready'], check=False)
if result.returncode == 0:
return
time.sleep(1)
raise RuntimeError('surge replica did not become locally ready')
with tempfile.TemporaryDirectory(prefix='lab7-rollout-') as temporary:
override = Path(temporary) / 'compose.json'
override.write_text(json.dumps({'services': {'api': {'image': image}}}))
try:
wait_slots(len(replicas))
old_images = [environment.run(['docker', 'inspect', '--format', '{{.Image}}', container]).stdout.strip()
for container in replicas]
log.append('old image digests: ' + ', '.join(old_images))
with (output / 'load.txt').open('w') as load_log:
loader = subprocess.Popen(
[sys.executable, str(ROOT / 'student-tests/rolling_update_load.py'),
'--base-url', BASE_URL, '--ca-file', CA_FILE,
'--duration', os.environ.get('DURATION', '60'),
'--concurrency', os.environ.get('CONCURRENCY', '8'),
'--output', str(output / 'load.json')],
cwd=ROOT, env=environment.process_env, stdout=load_log, stderr=subprocess.STDOUT)
for old in replicas:
name = environment.project + '-rollout-' + uuid.uuid4().hex[:8]
environment.compose('-f', str(override), 'run', '-d', '--no-deps', '--use-aliases',
'--name', name, 'api')
surge.append(name)
wait_container(name)
wait_slots(len(replicas) + 1)
environment.run(['docker', 'stop', old])
environment.run(['docker', 'rm', old])
wait_slots(len(replicas))
log.append('ready surge %s replaced %s; healthy slots=%d' % (name, old, up_slots()))
result = loader.wait()
if result:
raise RuntimeError('load generator failed; inspect load.txt/load.json')
environment.compose('-f', str(override), 'up', '-d', '--no-deps', '--scale',
'api=' + str(len(replicas)), 'api')
wait_slots(len(replicas) + len(surge))
while surge:
name = surge[0]
environment.run(['docker', 'stop', name])
environment.run(['docker', 'rm', name])
surge.pop(0)
wait_slots(len(replicas) + len(surge))
log.append('Compose control restored; healthy slots=%d' % up_slots())
finally:
if loader is not None and loader.poll() is None:
loader.terminate()
loader.wait()
# Restore regular replicas before removing any surviving surge capacity.
environment.compose('-f', str(override), 'up', '-d', '--no-deps', '--scale',
'api=' + str(len(replicas)), 'api', check=False)
if surge:
for container in environment.compose('ps', '-q', 'api').stdout.split():
wait_container(container)
for name in surge:
environment.run(['docker', 'stop', name], check=False)
environment.run(['docker', 'rm', name], check=False)
(output / 'rolling-update.txt').write_text('\n'.join(log) + '\n')
print('\n'.join(log))
PY