forked from mtusi2026/mtusi2026lab7
122 lines
5.3 KiB
Bash
Executable File
122 lines
5.3 KiB
Bash
Executable File
#!/bin/sh
|
|
# Secure Compose surge rollout: inherited API identity/config/hardening, one replica at a time.
|
|
set -eu
|
|
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
|
cd "$ROOT_DIR"
|
|
exec "${PYTHON:-python3}" - "$@" <<'PY'
|
|
import csv
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(Path.cwd() / 'scripts'))
|
|
sys.path.insert(0, str(Path.cwd() / 'tests'))
|
|
from lab7_environment import Environment, ROOT
|
|
from lab7_http import BASE_URL, CA_FILE, open_url
|
|
from client import Client
|
|
|
|
environment = Environment()
|
|
output = Path(os.environ.get('OUT', 'evidence/rolling-update'))
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
image = os.environ.get('IMAGE', 'mtusi-messenger:lab7')
|
|
stats_url = os.environ.get('STATS', 'https://localhost:' + environment.value('LB_STATS_PORT', '8404') + '/stats;csv')
|
|
replicas = environment.compose('ps', '-q', 'api').stdout.split()
|
|
if len(replicas) < 2:
|
|
raise SystemExit('rolling update requires at least two running API replicas')
|
|
environment.run(['docker', 'image', 'inspect', image])
|
|
client = Client(BASE_URL, CA_FILE, timeout=5)
|
|
log = []
|
|
surge = []
|
|
loader = None
|
|
|
|
def rows():
|
|
with open_url(stats_url, operator=True, ca_file=CA_FILE) as response:
|
|
content = response.read().decode()
|
|
return list(csv.DictReader(io.StringIO(content.replace('# ', '', 1))))
|
|
|
|
def up_slots():
|
|
return sum(row['pxname'] == 'api_backend' and row['svname'] not in ('BACKEND', 'FRONTEND')
|
|
and row['status'].startswith('UP') for row in rows())
|
|
|
|
def wait_slots(expected):
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
if up_slots() == expected:
|
|
client.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError('LB did not converge to %d healthy business slots' % expected)
|
|
|
|
def wait_container(container):
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
result = environment.run(['docker', 'exec', container, 'wget', '-qO-', '-T', '3',
|
|
'http://127.0.0.1:8084/health/ready'], check=False)
|
|
if result.returncode == 0:
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError('surge replica did not become locally ready')
|
|
|
|
with tempfile.TemporaryDirectory(prefix='lab7-rollout-') as temporary:
|
|
override = Path(temporary) / 'compose.json'
|
|
override.write_text(json.dumps({'services': {'api': {'image': image}}}))
|
|
try:
|
|
wait_slots(len(replicas))
|
|
old_images = [environment.run(['docker', 'inspect', '--format', '{{.Image}}', container]).stdout.strip()
|
|
for container in replicas]
|
|
log.append('old image digests: ' + ', '.join(old_images))
|
|
with (output / 'load.txt').open('w') as load_log:
|
|
loader = subprocess.Popen(
|
|
[sys.executable, str(ROOT / 'student-tests/rolling_update_load.py'),
|
|
'--base-url', BASE_URL, '--ca-file', CA_FILE,
|
|
'--duration', os.environ.get('DURATION', '60'),
|
|
'--concurrency', os.environ.get('CONCURRENCY', '8'),
|
|
'--output', str(output / 'load.json')],
|
|
cwd=ROOT, env=environment.process_env, stdout=load_log, stderr=subprocess.STDOUT)
|
|
for old in replicas:
|
|
name = environment.project + '-rollout-' + uuid.uuid4().hex[:8]
|
|
environment.compose('-f', str(override), 'run', '-d', '--no-deps', '--use-aliases',
|
|
'--name', name, 'api')
|
|
surge.append(name)
|
|
wait_container(name)
|
|
wait_slots(len(replicas) + 1)
|
|
environment.run(['docker', 'stop', old])
|
|
environment.run(['docker', 'rm', old])
|
|
wait_slots(len(replicas))
|
|
log.append('ready surge %s replaced %s; healthy slots=%d' % (name, old, up_slots()))
|
|
result = loader.wait()
|
|
if result:
|
|
raise RuntimeError('load generator failed; inspect load.txt/load.json')
|
|
environment.compose('-f', str(override), 'up', '-d', '--no-deps', '--scale',
|
|
'api=' + str(len(replicas)), 'api')
|
|
wait_slots(len(replicas) + len(surge))
|
|
while surge:
|
|
name = surge[0]
|
|
environment.run(['docker', 'stop', name])
|
|
environment.run(['docker', 'rm', name])
|
|
surge.pop(0)
|
|
wait_slots(len(replicas) + len(surge))
|
|
log.append('Compose control restored; healthy slots=%d' % up_slots())
|
|
finally:
|
|
if loader is not None and loader.poll() is None:
|
|
loader.terminate()
|
|
loader.wait()
|
|
# Restore regular replicas before removing any surviving surge capacity.
|
|
environment.compose('-f', str(override), 'up', '-d', '--no-deps', '--scale',
|
|
'api=' + str(len(replicas)), 'api', check=False)
|
|
if surge:
|
|
for container in environment.compose('ps', '-q', 'api').stdout.split():
|
|
wait_container(container)
|
|
for name in surge:
|
|
environment.run(['docker', 'stop', name], check=False)
|
|
environment.run(['docker', 'rm', name], check=False)
|
|
(output / 'rolling-update.txt').write_text('\n'.join(log) + '\n')
|
|
print('\n'.join(log))
|
|
PY
|