forked from mtusi2026/mtusi2026lab3
lab3
This commit is contained in:
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a new API image and replace replicas one at a time under load."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
||||
COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
|
||||
os.environ.setdefault('PYTHONUTF8', '1')
|
||||
|
||||
|
||||
def run(args, capture=True, timeout=None):
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
check=False,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode:
|
||||
detail = (result.stderr or result.stdout or '').strip()
|
||||
raise RuntimeError('Operation failed: ' + ' '.join(args[:3]) + '\n' + detail)
|
||||
return result.stdout.strip() if capture else ''
|
||||
|
||||
|
||||
def compose(*args, capture=True, timeout=None):
|
||||
return run(COMPOSE + ['-f', str(ROOT / 'compose.yaml'), *args], capture, timeout)
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
return json.loads(run(['docker', 'inspect', container_id]))[0]
|
||||
|
||||
|
||||
def wait_healthy(container_id, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
state = inspect(container_id)['State']
|
||||
if state['Status'] == 'exited':
|
||||
raise RuntimeError('Replacement API container exited before becoming healthy')
|
||||
if state.get('Health', {}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError('Replacement API container did not become healthy')
|
||||
|
||||
|
||||
def wait_routed_to(instance_id, base_url, timeout=20):
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
client = Client(base_url, timeout=5)
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.call(
|
||||
'GET', '/api/v1/users/me', expected=401, schema='Error'
|
||||
)
|
||||
if response['headers'].get('X-Instance-Id') == instance_id:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise TimeoutError('Nginx did not route a request to the new API container')
|
||||
|
||||
|
||||
def candidate_command(old, image_id, name):
|
||||
config = old['Config']
|
||||
host_config = old['HostConfig']
|
||||
networks = old['NetworkSettings']['Networks']
|
||||
if len(networks) != 1:
|
||||
raise RuntimeError('Expected API to use exactly one private network')
|
||||
network = next(iter(networks))
|
||||
args = [
|
||||
'docker', 'run', '--detach',
|
||||
'--name', name,
|
||||
'--network', network,
|
||||
'--network-alias', 'api',
|
||||
'--restart', 'no',
|
||||
'--stop-timeout', '15',
|
||||
]
|
||||
if config.get('User'):
|
||||
args += ['--user', config['User']]
|
||||
if host_config.get('NanoCpus'):
|
||||
args += ['--cpus', str(host_config['NanoCpus'] / 1_000_000_000)]
|
||||
if host_config.get('Memory'):
|
||||
args += ['--memory', str(host_config['Memory'])]
|
||||
for env in config.get('Env', []):
|
||||
args += ['--env', env]
|
||||
for mount in old.get('Mounts', []):
|
||||
if mount['Destination'] != '/run/secrets/database_url':
|
||||
raise RuntimeError('Unexpected API mount; refusing to copy deployment state')
|
||||
args += [
|
||||
'--mount',
|
||||
'type=bind,source=%s,target=%s,readonly'
|
||||
% (mount['Source'], mount['Destination']),
|
||||
]
|
||||
args.append(image_id)
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=BASE_URL)
|
||||
parser.add_argument('--duration', type=int, default=90)
|
||||
parser.add_argument('--concurrency', type=int, default=16)
|
||||
parser.add_argument('--output', default='evidence/rolling-update.json')
|
||||
args = parser.parse_args()
|
||||
if args.duration < 30 or args.concurrency < 1:
|
||||
parser.error('duration must be at least 30 seconds and concurrency must be positive')
|
||||
|
||||
if not (ROOT / 'secrets' / 'database_url').is_file():
|
||||
parser.error('Run scripts/init-local-secrets.ps1 before the update')
|
||||
old_ids = compose('ps', '-q', 'api').splitlines()
|
||||
if len(old_ids) < 2:
|
||||
parser.error('At least two running API replicas are required')
|
||||
|
||||
old_image_ids = {inspect(container_id)['Image'] for container_id in old_ids}
|
||||
image = inspect(old_ids[0])['Config']['Image']
|
||||
revision = 'lab3-' + uuid.uuid4().hex[:12]
|
||||
run(
|
||||
[
|
||||
'docker',
|
||||
'build',
|
||||
'--build-arg',
|
||||
'BUILD_REVISION=' + revision,
|
||||
'--tag',
|
||||
image,
|
||||
'.',
|
||||
],
|
||||
capture=False,
|
||||
timeout=300,
|
||||
)
|
||||
image_id = run(['docker', 'image', 'inspect', '--format', '{{.Id}}', image])
|
||||
if image_id in old_image_ids:
|
||||
raise RuntimeError('The candidate API image is unchanged; refusing a no-op update')
|
||||
compose('run', '--rm', 'migrate', capture=False, timeout=120)
|
||||
|
||||
output = (ROOT / args.output).resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load_output = output.with_name(output.stem + '-load.json')
|
||||
load_process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tests' / 'load.py'),
|
||||
'--base-url',
|
||||
args.base_url,
|
||||
'--duration',
|
||||
str(args.duration),
|
||||
'--concurrency',
|
||||
str(args.concurrency),
|
||||
'--output',
|
||||
str(load_output),
|
||||
],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
candidates = []
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(2)
|
||||
for index, old_id in enumerate(old_ids, start=1):
|
||||
old = inspect(old_id)
|
||||
instance_id = old['Config']['Hostname']
|
||||
candidate_name = 'lab3-api-candidate-' + uuid.uuid4().hex[:10]
|
||||
started = time.monotonic()
|
||||
candidate_result = run(
|
||||
candidate_command(old, image_id, candidate_name),
|
||||
timeout=60,
|
||||
)
|
||||
candidate_id = candidate_result.splitlines()[0]
|
||||
candidates.append(candidate_id)
|
||||
new_instance_id = inspect(candidate_id)['Config']['Hostname']
|
||||
wait_healthy(candidate_id)
|
||||
wait_routed_to(new_instance_id, args.base_url)
|
||||
run(['docker', 'stop', '--timeout', '15', old_id], timeout=30)
|
||||
run(['docker', 'rm', old_id])
|
||||
replacements.append(
|
||||
{
|
||||
'old_instance_id': instance_id,
|
||||
'new_instance_id': new_instance_id,
|
||||
'ready_and_routed_seconds': round(time.monotonic() - started, 3),
|
||||
'sequence': index,
|
||||
}
|
||||
)
|
||||
|
||||
compose(
|
||||
'up', '--detach', '--scale', 'api=2', '--no-deps', 'api',
|
||||
capture=False, timeout=120,
|
||||
)
|
||||
managed_ids = compose('ps', '-q', 'api').splitlines()
|
||||
if len(managed_ids) != 2:
|
||||
raise RuntimeError('Compose did not restore exactly two managed API replicas')
|
||||
for container_id in managed_ids:
|
||||
wait_healthy(container_id)
|
||||
for candidate_id in candidates:
|
||||
run(['docker', 'rm', '--force', candidate_id])
|
||||
stdout, stderr = load_process.communicate(timeout=max(90, args.duration + 30))
|
||||
except Exception:
|
||||
if load_process.poll() is None:
|
||||
load_process.terminate()
|
||||
load_process.wait(timeout=5)
|
||||
raise
|
||||
if load_process.returncode:
|
||||
raise RuntimeError('Load test failed:\n' + (stderr or stdout).strip())
|
||||
|
||||
report = {
|
||||
'operation': 'new API image; serial container replacement with temporary capacity surge',
|
||||
'duration_seconds': args.duration,
|
||||
'concurrency': args.concurrency,
|
||||
'new_image_id': image_id,
|
||||
'build_revision': revision,
|
||||
'old_image_ids': sorted(old_image_ids),
|
||||
'replacements': replacements,
|
||||
'load': json.loads(load_output.read_text(encoding='utf-8')),
|
||||
'load_raw_file': load_output.name,
|
||||
'limitations': [
|
||||
'The schema migration in this exercise is additive and backward-compatible.',
|
||||
'PostgreSQL and Redis are single instances and remain SPOFs.',
|
||||
],
|
||||
}
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps(report['load'], ensure_ascii=False, indent=2))
|
||||
print('Saved rolling update evidence to ' + str(output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error:
|
||||
print('ERROR: ' + str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user