feat: port lab3 backend foundation to lab4

This commit is contained in:
sinveria
2026-11-09 19:17:48 +03:00
parent ec540a1a72
commit aaa834e412
42 changed files with 2787 additions and 0 deletions
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Create charts from real Track 1 summary.csv data; does not invent measurements."""
import csv
from pathlib import Path
import matplotlib.pyplot as plt
root = Path(__file__).resolve().parents[1]
source = root / 'evidence' / 'scaling' / 'summary.csv'
if not source.exists():
raise SystemExit('Run scripts/run-scaling-experiment.py first')
with source.open(encoding='utf-8', newline='') as f:
rows = list(csv.DictReader(f))
if not rows:
raise SystemExit('summary.csv is empty')
out = source.parent
for key, title, ylabel, filename in [
('successful_rps', 'Track 1: successful throughput',
'Successful requests/sec', 'throughput.png'),
('p95_ms', 'Track 1: p95 latency',
'Milliseconds', 'p95-latency.png'),
('error_ratio', 'Track 1: error ratio',
'Error ratio', 'errors.png'),
]:
labels = [r['scenario'] for r in rows]
values = [float(r[key]) if r[key] else 0 for r in rows]
plt.figure(figsize=(11, 5))
plt.bar(labels, values)
if key == 'error_ratio':
plt.ylim(0, 0.01)
for i, value in enumerate(values):
plt.text(
i, 0.0002, f'{value:.1%}',
ha='center', va='bottom'
)
plt.title(title)
plt.ylabel(ylabel)
plt.xticks(rotation=25, ha='right')
plt.tight_layout()
plt.savefig(out / filename, dpi=160)
plt.close()
print('Charts written to', out)
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Concurrent idempotency verification through the public load-balancer URL."""
from concurrent.futures import ThreadPoolExecutor
import uuid
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
def main():
def configure(p):
p.add_argument('--requests', type=int, default=20)
args = arguments('Concurrent cross-replica idempotency test', configure)
check(2 <= args.requests <= 100, 'requests must be 2..100')
c = Client(args.base_url, args.ca_file, timeout=10)
alice, bob, eve, chat = c.bootstrap()
key = 'race-' + uuid.uuid4().hex
body = 'same concurrent request ' + uuid.uuid4().hex
def send(_):
return Client(args.base_url, args.ca_file, timeout=10).message(alice, chat, body, key=key)
with ThreadPoolExecutor(max_workers=args.requests) as pool:
responses = list(pool.map(send, range(args.requests)))
messages = [r['json'] for r in responses]
check(all(m == messages[0] for m in messages), 'concurrent retries returned different messages')
instances = {r['headers'].get('X-Instance-Id') for r in responses}
check(None not in instances, 'missing X-Instance-Id')
check(len(instances) >= 2, 'load balancer did not expose at least two API replicas')
check(len({m['id'] for m in messages}) == 1, 'concurrent retries created duplicates')
conflict = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409,
actor=alice, body={'text': body + ' changed'},
headers={'Idempotency-Key': key})
print({'requests': args.requests, 'unique_message_ids': 1,
'observed_instances': len(instances), 'different_body_status': conflict['status']})
print('PASS: concurrent same-key requests produced exactly one message')
if __name__ == '__main__':
run(main)
+116
View File
@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Diagnostic closed-loop load test. Does not modify public tests."""
from concurrent.futures import ThreadPoolExecutor
from collections import Counter
from pathlib import Path
import json
import math
import re
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, LAB, arguments, check, run
def main():
def configure(p):
p.add_argument('--duration', type=float, default=90)
p.add_argument('--concurrency', type=int, default=8)
p.add_argument('--output', default='evidence/rolling-update-diagnostic.json')
args = arguments('Diagnostic synthetic POST message load', configure)
check(1 <= args.duration <= 3600, 'duration must be 1..3600 seconds')
check(1 <= args.concurrency <= 128, 'concurrency must be 1..128')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
barrier = threading.Barrier(args.concurrency)
def worker(worker_id):
local = Client(args.base_url, args.ca_file, timeout=5)
barrier.wait()
deadline = time.monotonic() + args.duration
latencies = []
errors = Counter()
examples = []
index = 0
while time.monotonic() < deadline:
start = time.monotonic()
try:
local.message(alice, chat, 'diagnostic-%s-%s' % (worker_id, index))
latencies.append(time.monotonic() - start)
except Failure as exc:
message = str(exc)
match = re.search(r'received (\d{3})', message)
if match:
category = 'http_' + match.group(1)
elif 'connection/TLS failure' in message:
kind = re.search(r'\(([^)]+)\)', message)
category = 'connection_' + (kind.group(1) if kind else 'unknown')
else:
category = 'other_failure'
errors[category] += 1
if len(examples) < 10:
examples.append(message[:300])
index += 1
return latencies, errors, examples
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
results = list(pool.map(worker, range(args.concurrency)))
elapsed = time.monotonic() - started
latencies = sorted(x for values, _, _ in results for x in values)
errors = Counter()
examples = []
for _, counts, samples in results:
errors.update(counts)
examples.extend(samples)
total_errors = sum(errors.values())
total = len(latencies) + total_errors
def percentile(p):
if not latencies:
return None
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
report = {
'lab': LAB,
'workload': 'closed-loop POST messages; no automatic retries',
'duration_seconds': round(elapsed, 3),
'concurrency': args.concurrency,
'successful_requests': len(latencies),
'failed_requests': total_errors,
'error_ratio': round(total_errors / total, 6) if total else None,
'errors_by_type': dict(errors),
'successful_rps': round(len(latencies) / elapsed, 3),
'successful_latency_ms': {
'p50': percentile(.50),
'p95': percentile(.95),
'p99': percentile(.99)
},
'error_examples': examples[:20],
'note': 'Latency percentiles cover successful requests only.'
}
text = json.dumps(report, ensure_ascii=False, indent=2)
dest = Path(args.output)
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(text + '\n', encoding='utf-8')
print(text)
check(bool(latencies), 'no successful writes')
if __name__ == '__main__':
run(main)
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
# Restart all stateful dependencies and all API replicas without touching volumes.
docker compose restart postgres redis api
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env bash
set -eu
# Compose does not provide a native rolling-update primitive. This script
# performs a surge replacement: add one new healthy replica, then remove one old.
desired="${API_REPLICAS:-2}"
load_duration="${ROLLING_LOAD_DURATION:-90}"
load_script="${LOAD_SCRIPT:-tests/load.py}"
load_output="${ROLLING_LOAD_OUTPUT:-evidence/rolling-update-load.json}"
load_log="${ROLLING_LOAD_LOG:-evidence/rolling-update-load.log}"
if [ "$desired" -lt 2 ]; then echo 'API_REPLICAS must be >= 2' >&2; exit 2; fi
docker compose build api
python "$load_script" --base-url http://localhost:8080 --duration "$load_duration" --concurrency 8 --output "$load_output" >"$load_log" 2>&1 &
load_pid=$!
cleanup() { kill "$load_pid" 2>/dev/null || true; wait "$load_pid" 2>/dev/null || true; }
trap cleanup EXIT INT TERM
# Capture the old replica ids. Each iteration adds a new container before removing an old one.
old_ids=$(docker compose ps -q api)
for old_id in $old_ids; do
before=$(docker compose ps -q api | sort)
docker compose up -d --no-deps --no-recreate --scale "api=$((desired + 1))" api
new_id=''
attempt=0
while [ "$attempt" -lt 60 ]; do
after=$(docker compose ps -q api | sort)
new_id=$(comm -13 <(printf '%s\n' "$before") <(printf '%s\n' "$after") | head -n 1 || true)
if [ -n "$new_id" ]; then
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$new_id")
if [ "$health" = healthy ]; then break; fi
fi
attempt=$((attempt + 1)); sleep 2
done
if [ -z "$new_id" ] || [ "${health:-}" != healthy ]; then
echo 'New replica did not become healthy; leaving old replicas running' >&2
exit 1
fi
docker exec "$old_id" sh -c 'touch /tmp/api-draining'
sleep 10
docker stop --timeout 20 "$old_id" >/dev/null
docker rm "$old_id" >/dev/null
docker compose up -d --no-deps --no-recreate --scale "api=$desired" api
python scripts/wait-ready.py http://localhost:8080/health/ready 30
done
wait "$load_pid" || { echo 'Load test failed; inspect evidence/rolling-update-load.log' >&2; exit 1; }
trap - EXIT INT TERM
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env python3
"""Run Track 1 scenarios on the user's Docker host; records only measured results."""
import argparse
import csv
import json
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'evidence' / 'scaling'
BASE_ENV = ROOT / '.env'
def run(cmd, env=None, check=True, capture=False):
return subprocess.run(cmd, cwd=ROOT, env=env, check=check,
text=True, stdout=subprocess.PIPE if capture else None,
stderr=subprocess.STDOUT if capture else None)
def stats():
p = run(['docker', 'stats', '--no-stream', '--format', '{{.Name}};{{.CPUPerc}};{{.MemUsage}}'], capture=True)
return p.stdout.strip().splitlines()
def env_value(source, key, default):
prefix = key + '='
for line in source.splitlines():
if line.startswith(prefix):
return line[len(prefix):].strip().strip('\"').strip("'")
return default
def db_activity(env_path, db_user, db_name):
query = "SELECT count(*) FILTER (WHERE state='active'), count(*), current_setting('max_connections') FROM pg_stat_activity"
result = run(['docker', 'compose', '--env-file', str(env_path), 'exec', '-T', 'postgres',
'psql', '-U', db_user, '-d', db_name, '-At', '-F', ',', '-c', query],
capture=True, check=False)
return result.stdout.strip() if result.returncode == 0 else 'unavailable'
def scenario(name, replicas, total_cpu, total_mem, duration, concurrency):
source = BASE_ENV.read_text(encoding='utf-8')
db_user = env_value(source, 'POSTGRES_USER', 'messenger')
db_name = env_value(source, 'POSTGRES_DB', 'messenger')
api_cpu = total_cpu / replicas
api_mem_mb = total_mem // replicas
env_text = source.rstrip() + f'\nAPI_CPU_LIMIT={api_cpu:.3f}\nAPI_MEMORY_LIMIT={api_mem_mb}M\n'
env_path = OUT / ('.env-' + name)
env_path.write_text(env_text, encoding='utf-8')
compose = ['docker', 'compose', '--env-file', str(env_path), 'up', '-d', '--build', '--scale', f'api={replicas}']
run(compose)
deadline = time.time() + 120
while time.time() < deadline:
result = run(['docker', 'compose', '--env-file', str(env_path), 'ps', '-q', 'api'], capture=True, check=False)
ids = [line.strip() for line in result.stdout.splitlines() if line.strip()]
healthy = 0
for container_id in ids:
inspected = run(['docker', 'inspect', '--format', '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}', container_id], capture=True, check=False)
if inspected.returncode == 0 and inspected.stdout.strip() == 'healthy':
healthy += 1
if len(ids) >= replicas and healthy >= replicas:
break
time.sleep(2)
else:
raise RuntimeError(f'{name}: API replicas did not become healthy')
output = OUT / (name + '.json')
cmd = [sys.executable, 'tests/load.py', '--base-url', 'http://localhost:8080',
'--duration', str(duration), '--concurrency', str(concurrency), '--output', str(output)]
proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
samples = []
while proc.poll() is None:
timestamp = time.time()
try:
samples.append({'timestamp_epoch': timestamp, 'containers': stats(),
'postgres_active_total_max_connections': db_activity(env_path, db_user, db_name)})
except Exception as exc:
samples.append({'timestamp_epoch': timestamp, 'stats_error': type(exc).__name__})
time.sleep(1)
stdout = proc.stdout.read() if proc.stdout else ''
if proc.returncode != 0:
raise RuntimeError(f'load failed for {name}: {stdout}')
metrics = json.loads(output.read_text(encoding='utf-8'))
metrics.update({'scenario': name, 'api_replicas': replicas,
'total_api_cpu_budget': total_cpu,
'total_api_memory_budget_mb': total_mem,
'per_replica_cpu_limit': api_cpu,
'per_replica_memory_limit_mb': api_mem_mb,
'container_stats_samples': samples})
output.write_text(json.dumps(metrics, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
run(['docker', 'compose', '--env-file', str(env_path), 'down'])
return metrics
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--duration', type=int, default=30)
p.add_argument('--concurrency', type=int, default=8)
p.add_argument('--fixed-cpu', type=float, default=2.0)
p.add_argument('--fixed-memory-mb', type=int, default=2048)
p.add_argument('--expanded-cpu', type=float, nargs=3, default=[2.0, 4.0, 8.0])
p.add_argument('--expanded-memory-mb', type=int, nargs=3, default=[2048, 4096, 8192])
args = p.parse_args()
if not BASE_ENV.exists():
p.error('Create .env from .env.example before running')
OUT.mkdir(parents=True, exist_ok=True)
results = []
for n in (1, 2, 4):
results.append(scenario(f'fixed_{n}replicas', n, args.fixed_cpu,
args.fixed_memory_mb, args.duration, args.concurrency))
for cpu, mem in zip(args.expanded_cpu, args.expanded_memory_mb):
results.append(scenario(f'expanded_4replicas_{cpu:g}cpu_{mem}mb', 4, cpu,
mem, args.duration, args.concurrency))
with (OUT / 'summary.csv').open('w', newline='', encoding='utf-8') as f:
fields = ['scenario', 'api_replicas', 'total_api_cpu_budget', 'total_api_memory_budget_mb',
'successful_rps', 'failed_requests', 'error_ratio', 'p50_ms', 'p95_ms', 'p99_ms']
writer = csv.DictWriter(f, fieldnames=fields)
writer.writeheader()
for x in results:
lat = x['successful_latency_ms']
writer.writerow({'scenario': x['scenario'], 'api_replicas': x['api_replicas'],
'total_api_cpu_budget': x['total_api_cpu_budget'],
'total_api_memory_budget_mb': x['total_api_memory_budget_mb'],
'successful_rps': x['successful_rps'], 'failed_requests': x['failed_requests'],
'error_ratio': x['error_ratio'], 'p50_ms': lat['p50'], 'p95_ms': lat['p95'], 'p99_ms': lat['p99']})
print(f'Wrote measured results to {OUT}')
if __name__ == '__main__':
main()
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
project="${COMPOSE_PROJECT_NAME:-mtusi-lab3}"
found=""
for id in $(docker ps -q --filter "label=com.docker.compose.project=${project}" --filter "label=com.docker.compose.service=api"); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
found="$id"
break
fi
done
if [ -z "$found" ]; then
echo "No running API container matches TARGET_INSTANCE_ID" >&2
exit 1
fi
docker stop --time 3 "$found" >/dev/null
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env python3
import sys
import time
import urllib.request
url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8080/health/ready'
timeout = int(sys.argv[2]) if len(sys.argv) > 2 else 30
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
raise SystemExit(0)
except Exception:
time.sleep(0.5)
raise SystemExit('Readiness timeout: ' + url)