#!/bin/sh # Secure Compose surge rollout: inherited API identity/config/hardening, one replica at a time. set -eu ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) cd "$ROOT_DIR" exec "${PYTHON:-python3}" - "$@" <<'PY' import csv import io import json import os from pathlib import Path import subprocess import sys import tempfile import time import uuid sys.path.insert(0, str(Path.cwd() / 'scripts')) sys.path.insert(0, str(Path.cwd() / 'tests')) from lab7_environment import Environment, ROOT from lab7_http import BASE_URL, CA_FILE, open_url from client import Client environment = Environment() output = Path(os.environ.get('OUT', 'evidence/rolling-update')) output.mkdir(parents=True, exist_ok=True) image = os.environ.get('IMAGE', 'mtusi-messenger:lab7') stats_url = os.environ.get('STATS', 'https://localhost:' + environment.value('LB_STATS_PORT', '8404') + '/stats;csv') replicas = environment.compose('ps', '-q', 'api').stdout.split() if len(replicas) < 2: raise SystemExit('rolling update requires at least two running API replicas') environment.run(['docker', 'image', 'inspect', image]) client = Client(BASE_URL, CA_FILE, timeout=5) log = [] surge = [] loader = None def rows(): with open_url(stats_url, operator=True, ca_file=CA_FILE) as response: content = response.read().decode() return list(csv.DictReader(io.StringIO(content.replace('# ', '', 1)))) def up_slots(): return sum(row['pxname'] == 'api_backend' and row['svname'] not in ('BACKEND', 'FRONTEND') and row['status'].startswith('UP') for row in rows()) def wait_slots(expected): deadline = time.monotonic() + 60 while time.monotonic() < deadline: if up_slots() == expected: client.call('GET', '/health/ready', schema='Health') return time.sleep(1) raise RuntimeError('LB did not converge to %d healthy business slots' % expected) def wait_container(container): deadline = time.monotonic() + 90 while time.monotonic() < deadline: result = environment.run(['docker', 'exec', container, 'wget', '-qO-', '-T', '3', 'http://127.0.0.1:8084/health/ready'], check=False) if result.returncode == 0: return time.sleep(1) raise RuntimeError('surge replica did not become locally ready') with tempfile.TemporaryDirectory(prefix='lab7-rollout-') as temporary: override = Path(temporary) / 'compose.json' override.write_text(json.dumps({'services': {'api': {'image': image}}})) try: wait_slots(len(replicas)) old_images = [environment.run(['docker', 'inspect', '--format', '{{.Image}}', container]).stdout.strip() for container in replicas] log.append('old image digests: ' + ', '.join(old_images)) with (output / 'load.txt').open('w') as load_log: loader = subprocess.Popen( [sys.executable, str(ROOT / 'student-tests/rolling_update_load.py'), '--base-url', BASE_URL, '--ca-file', CA_FILE, '--duration', os.environ.get('DURATION', '60'), '--concurrency', os.environ.get('CONCURRENCY', '8'), '--output', str(output / 'load.json')], cwd=ROOT, env=environment.process_env, stdout=load_log, stderr=subprocess.STDOUT) for old in replicas: name = environment.project + '-rollout-' + uuid.uuid4().hex[:8] environment.compose('-f', str(override), 'run', '-d', '--no-deps', '--use-aliases', '--name', name, 'api') surge.append(name) wait_container(name) wait_slots(len(replicas) + 1) environment.run(['docker', 'stop', old]) environment.run(['docker', 'rm', old]) wait_slots(len(replicas)) log.append('ready surge %s replaced %s; healthy slots=%d' % (name, old, up_slots())) result = loader.wait() if result: raise RuntimeError('load generator failed; inspect load.txt/load.json') environment.compose('-f', str(override), 'up', '-d', '--no-deps', '--scale', 'api=' + str(len(replicas)), 'api') wait_slots(len(replicas) + len(surge)) while surge: name = surge[0] environment.run(['docker', 'stop', name]) environment.run(['docker', 'rm', name]) surge.pop(0) wait_slots(len(replicas) + len(surge)) log.append('Compose control restored; healthy slots=%d' % up_slots()) finally: if loader is not None and loader.poll() is None: loader.terminate() loader.wait() # Restore regular replicas before removing any surviving surge capacity. environment.compose('-f', str(override), 'up', '-d', '--no-deps', '--scale', 'api=' + str(len(replicas)), 'api', check=False) if surge: for container in environment.compose('ps', '-q', 'api').stdout.split(): wait_container(container) for name in surge: environment.run(['docker', 'stop', name], check=False) environment.run(['docker', 'rm', name], check=False) (output / 'rolling-update.txt').write_text('\n'.join(log) + '\n') print('\n'.join(log)) PY