feat: port lab3 backend foundation to lab4
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""Create charts from real Track 1 summary.csv data; does not invent measurements."""
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
source = root / 'evidence' / 'scaling' / 'summary.csv'
|
||||
|
||||
if not source.exists():
|
||||
raise SystemExit('Run scripts/run-scaling-experiment.py first')
|
||||
|
||||
with source.open(encoding='utf-8', newline='') as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
|
||||
if not rows:
|
||||
raise SystemExit('summary.csv is empty')
|
||||
|
||||
out = source.parent
|
||||
|
||||
for key, title, ylabel, filename in [
|
||||
('successful_rps', 'Track 1: successful throughput',
|
||||
'Successful requests/sec', 'throughput.png'),
|
||||
('p95_ms', 'Track 1: p95 latency',
|
||||
'Milliseconds', 'p95-latency.png'),
|
||||
('error_ratio', 'Track 1: error ratio',
|
||||
'Error ratio', 'errors.png'),
|
||||
]:
|
||||
labels = [r['scenario'] for r in rows]
|
||||
values = [float(r[key]) if r[key] else 0 for r in rows]
|
||||
|
||||
plt.figure(figsize=(11, 5))
|
||||
plt.bar(labels, values)
|
||||
|
||||
if key == 'error_ratio':
|
||||
plt.ylim(0, 0.01)
|
||||
for i, value in enumerate(values):
|
||||
plt.text(
|
||||
i, 0.0002, f'{value:.1%}',
|
||||
ha='center', va='bottom'
|
||||
)
|
||||
|
||||
plt.title(title)
|
||||
plt.ylabel(ylabel)
|
||||
plt.xticks(rotation=25, ha='right')
|
||||
plt.tight_layout()
|
||||
plt.savefig(out / filename, dpi=160)
|
||||
plt.close()
|
||||
|
||||
print('Charts written to', out)
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Concurrent idempotency verification through the public load-balancer URL."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--requests', type=int, default=20)
|
||||
args = arguments('Concurrent cross-replica idempotency test', configure)
|
||||
check(2 <= args.requests <= 100, 'requests must be 2..100')
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
key = 'race-' + uuid.uuid4().hex
|
||||
body = 'same concurrent request ' + uuid.uuid4().hex
|
||||
|
||||
def send(_):
|
||||
return Client(args.base_url, args.ca_file, timeout=10).message(alice, chat, body, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.requests) as pool:
|
||||
responses = list(pool.map(send, range(args.requests)))
|
||||
messages = [r['json'] for r in responses]
|
||||
check(all(m == messages[0] for m in messages), 'concurrent retries returned different messages')
|
||||
instances = {r['headers'].get('X-Instance-Id') for r in responses}
|
||||
check(None not in instances, 'missing X-Instance-Id')
|
||||
check(len(instances) >= 2, 'load balancer did not expose at least two API replicas')
|
||||
check(len({m['id'] for m in messages}) == 1, 'concurrent retries created duplicates')
|
||||
conflict = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409,
|
||||
actor=alice, body={'text': body + ' changed'},
|
||||
headers={'Idempotency-Key': key})
|
||||
print({'requests': args.requests, 'unique_message_ids': 1,
|
||||
'observed_instances': len(instances), 'different_body_status': conflict['status']})
|
||||
print('PASS: concurrent same-key requests produced exactly one message')
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diagnostic closed-loop load test. Does not modify public tests."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, LAB, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--duration', type=float, default=90)
|
||||
p.add_argument('--concurrency', type=int, default=8)
|
||||
p.add_argument('--output', default='evidence/rolling-update-diagnostic.json')
|
||||
|
||||
args = arguments('Diagnostic synthetic POST message load', configure)
|
||||
check(1 <= args.duration <= 3600, 'duration must be 1..3600 seconds')
|
||||
check(1 <= args.concurrency <= 128, 'concurrency must be 1..128')
|
||||
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
|
||||
def worker(worker_id):
|
||||
local = Client(args.base_url, args.ca_file, timeout=5)
|
||||
barrier.wait()
|
||||
deadline = time.monotonic() + args.duration
|
||||
latencies = []
|
||||
errors = Counter()
|
||||
examples = []
|
||||
index = 0
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
local.message(alice, chat, 'diagnostic-%s-%s' % (worker_id, index))
|
||||
latencies.append(time.monotonic() - start)
|
||||
except Failure as exc:
|
||||
message = str(exc)
|
||||
match = re.search(r'received (\d{3})', message)
|
||||
|
||||
if match:
|
||||
category = 'http_' + match.group(1)
|
||||
elif 'connection/TLS failure' in message:
|
||||
kind = re.search(r'\(([^)]+)\)', message)
|
||||
category = 'connection_' + (kind.group(1) if kind else 'unknown')
|
||||
else:
|
||||
category = 'other_failure'
|
||||
|
||||
errors[category] += 1
|
||||
if len(examples) < 10:
|
||||
examples.append(message[:300])
|
||||
|
||||
index += 1
|
||||
|
||||
return latencies, errors, examples
|
||||
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(worker, range(args.concurrency)))
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
latencies = sorted(x for values, _, _ in results for x in values)
|
||||
errors = Counter()
|
||||
examples = []
|
||||
|
||||
for _, counts, samples in results:
|
||||
errors.update(counts)
|
||||
examples.extend(samples)
|
||||
|
||||
total_errors = sum(errors.values())
|
||||
total = len(latencies) + total_errors
|
||||
|
||||
def percentile(p):
|
||||
if not latencies:
|
||||
return None
|
||||
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
|
||||
|
||||
report = {
|
||||
'lab': LAB,
|
||||
'workload': 'closed-loop POST messages; no automatic retries',
|
||||
'duration_seconds': round(elapsed, 3),
|
||||
'concurrency': args.concurrency,
|
||||
'successful_requests': len(latencies),
|
||||
'failed_requests': total_errors,
|
||||
'error_ratio': round(total_errors / total, 6) if total else None,
|
||||
'errors_by_type': dict(errors),
|
||||
'successful_rps': round(len(latencies) / elapsed, 3),
|
||||
'successful_latency_ms': {
|
||||
'p50': percentile(.50),
|
||||
'p95': percentile(.95),
|
||||
'p99': percentile(.99)
|
||||
},
|
||||
'error_examples': examples[:20],
|
||||
'note': 'Latency percentiles cover successful requests only.'
|
||||
}
|
||||
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
dest = Path(args.output)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(text + '\n', encoding='utf-8')
|
||||
print(text)
|
||||
check(bool(latencies), 'no successful writes')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart all stateful dependencies and all API replicas without touching volumes.
|
||||
docker compose restart postgres redis api
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
# Compose does not provide a native rolling-update primitive. This script
|
||||
# performs a surge replacement: add one new healthy replica, then remove one old.
|
||||
desired="${API_REPLICAS:-2}"
|
||||
load_duration="${ROLLING_LOAD_DURATION:-90}"
|
||||
load_script="${LOAD_SCRIPT:-tests/load.py}"
|
||||
load_output="${ROLLING_LOAD_OUTPUT:-evidence/rolling-update-load.json}"
|
||||
load_log="${ROLLING_LOAD_LOG:-evidence/rolling-update-load.log}"
|
||||
if [ "$desired" -lt 2 ]; then echo 'API_REPLICAS must be >= 2' >&2; exit 2; fi
|
||||
|
||||
docker compose build api
|
||||
python "$load_script" --base-url http://localhost:8080 --duration "$load_duration" --concurrency 8 --output "$load_output" >"$load_log" 2>&1 &
|
||||
load_pid=$!
|
||||
cleanup() { kill "$load_pid" 2>/dev/null || true; wait "$load_pid" 2>/dev/null || true; }
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Capture the old replica ids. Each iteration adds a new container before removing an old one.
|
||||
old_ids=$(docker compose ps -q api)
|
||||
for old_id in $old_ids; do
|
||||
before=$(docker compose ps -q api | sort)
|
||||
docker compose up -d --no-deps --no-recreate --scale "api=$((desired + 1))" api
|
||||
new_id=''
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 60 ]; do
|
||||
after=$(docker compose ps -q api | sort)
|
||||
new_id=$(comm -13 <(printf '%s\n' "$before") <(printf '%s\n' "$after") | head -n 1 || true)
|
||||
if [ -n "$new_id" ]; then
|
||||
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$new_id")
|
||||
if [ "$health" = healthy ]; then break; fi
|
||||
fi
|
||||
attempt=$((attempt + 1)); sleep 2
|
||||
done
|
||||
if [ -z "$new_id" ] || [ "${health:-}" != healthy ]; then
|
||||
echo 'New replica did not become healthy; leaving old replicas running' >&2
|
||||
exit 1
|
||||
fi
|
||||
docker exec "$old_id" sh -c 'touch /tmp/api-draining'
|
||||
sleep 10
|
||||
docker stop --timeout 20 "$old_id" >/dev/null
|
||||
docker rm "$old_id" >/dev/null
|
||||
docker compose up -d --no-deps --no-recreate --scale "api=$desired" api
|
||||
python scripts/wait-ready.py http://localhost:8080/health/ready 30
|
||||
done
|
||||
wait "$load_pid" || { echo 'Load test failed; inspect evidence/rolling-update-load.log' >&2; exit 1; }
|
||||
trap - EXIT INT TERM
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run Track 1 scenarios on the user's Docker host; records only measured results."""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / 'evidence' / 'scaling'
|
||||
BASE_ENV = ROOT / '.env'
|
||||
|
||||
|
||||
def run(cmd, env=None, check=True, capture=False):
|
||||
return subprocess.run(cmd, cwd=ROOT, env=env, check=check,
|
||||
text=True, stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.STDOUT if capture else None)
|
||||
|
||||
|
||||
def stats():
|
||||
p = run(['docker', 'stats', '--no-stream', '--format', '{{.Name}};{{.CPUPerc}};{{.MemUsage}}'], capture=True)
|
||||
return p.stdout.strip().splitlines()
|
||||
|
||||
|
||||
def env_value(source, key, default):
|
||||
prefix = key + '='
|
||||
for line in source.splitlines():
|
||||
if line.startswith(prefix):
|
||||
return line[len(prefix):].strip().strip('\"').strip("'")
|
||||
return default
|
||||
|
||||
|
||||
def db_activity(env_path, db_user, db_name):
|
||||
query = "SELECT count(*) FILTER (WHERE state='active'), count(*), current_setting('max_connections') FROM pg_stat_activity"
|
||||
result = run(['docker', 'compose', '--env-file', str(env_path), 'exec', '-T', 'postgres',
|
||||
'psql', '-U', db_user, '-d', db_name, '-At', '-F', ',', '-c', query],
|
||||
capture=True, check=False)
|
||||
return result.stdout.strip() if result.returncode == 0 else 'unavailable'
|
||||
|
||||
|
||||
def scenario(name, replicas, total_cpu, total_mem, duration, concurrency):
|
||||
source = BASE_ENV.read_text(encoding='utf-8')
|
||||
db_user = env_value(source, 'POSTGRES_USER', 'messenger')
|
||||
db_name = env_value(source, 'POSTGRES_DB', 'messenger')
|
||||
api_cpu = total_cpu / replicas
|
||||
api_mem_mb = total_mem // replicas
|
||||
env_text = source.rstrip() + f'\nAPI_CPU_LIMIT={api_cpu:.3f}\nAPI_MEMORY_LIMIT={api_mem_mb}M\n'
|
||||
env_path = OUT / ('.env-' + name)
|
||||
env_path.write_text(env_text, encoding='utf-8')
|
||||
compose = ['docker', 'compose', '--env-file', str(env_path), 'up', '-d', '--build', '--scale', f'api={replicas}']
|
||||
run(compose)
|
||||
deadline = time.time() + 120
|
||||
while time.time() < deadline:
|
||||
result = run(['docker', 'compose', '--env-file', str(env_path), 'ps', '-q', 'api'], capture=True, check=False)
|
||||
ids = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
||||
healthy = 0
|
||||
for container_id in ids:
|
||||
inspected = run(['docker', 'inspect', '--format', '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}', container_id], capture=True, check=False)
|
||||
if inspected.returncode == 0 and inspected.stdout.strip() == 'healthy':
|
||||
healthy += 1
|
||||
if len(ids) >= replicas and healthy >= replicas:
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
raise RuntimeError(f'{name}: API replicas did not become healthy')
|
||||
|
||||
output = OUT / (name + '.json')
|
||||
cmd = [sys.executable, 'tests/load.py', '--base-url', 'http://localhost:8080',
|
||||
'--duration', str(duration), '--concurrency', str(concurrency), '--output', str(output)]
|
||||
proc = subprocess.Popen(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
samples = []
|
||||
while proc.poll() is None:
|
||||
timestamp = time.time()
|
||||
try:
|
||||
samples.append({'timestamp_epoch': timestamp, 'containers': stats(),
|
||||
'postgres_active_total_max_connections': db_activity(env_path, db_user, db_name)})
|
||||
except Exception as exc:
|
||||
samples.append({'timestamp_epoch': timestamp, 'stats_error': type(exc).__name__})
|
||||
time.sleep(1)
|
||||
stdout = proc.stdout.read() if proc.stdout else ''
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f'load failed for {name}: {stdout}')
|
||||
metrics = json.loads(output.read_text(encoding='utf-8'))
|
||||
metrics.update({'scenario': name, 'api_replicas': replicas,
|
||||
'total_api_cpu_budget': total_cpu,
|
||||
'total_api_memory_budget_mb': total_mem,
|
||||
'per_replica_cpu_limit': api_cpu,
|
||||
'per_replica_memory_limit_mb': api_mem_mb,
|
||||
'container_stats_samples': samples})
|
||||
output.write_text(json.dumps(metrics, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
run(['docker', 'compose', '--env-file', str(env_path), 'down'])
|
||||
return metrics
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--duration', type=int, default=30)
|
||||
p.add_argument('--concurrency', type=int, default=8)
|
||||
p.add_argument('--fixed-cpu', type=float, default=2.0)
|
||||
p.add_argument('--fixed-memory-mb', type=int, default=2048)
|
||||
p.add_argument('--expanded-cpu', type=float, nargs=3, default=[2.0, 4.0, 8.0])
|
||||
p.add_argument('--expanded-memory-mb', type=int, nargs=3, default=[2048, 4096, 8192])
|
||||
args = p.parse_args()
|
||||
if not BASE_ENV.exists():
|
||||
p.error('Create .env from .env.example before running')
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
results = []
|
||||
for n in (1, 2, 4):
|
||||
results.append(scenario(f'fixed_{n}replicas', n, args.fixed_cpu,
|
||||
args.fixed_memory_mb, args.duration, args.concurrency))
|
||||
for cpu, mem in zip(args.expanded_cpu, args.expanded_memory_mb):
|
||||
results.append(scenario(f'expanded_4replicas_{cpu:g}cpu_{mem}mb', 4, cpu,
|
||||
mem, args.duration, args.concurrency))
|
||||
with (OUT / 'summary.csv').open('w', newline='', encoding='utf-8') as f:
|
||||
fields = ['scenario', 'api_replicas', 'total_api_cpu_budget', 'total_api_memory_budget_mb',
|
||||
'successful_rps', 'failed_requests', 'error_ratio', 'p50_ms', 'p95_ms', 'p99_ms']
|
||||
writer = csv.DictWriter(f, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for x in results:
|
||||
lat = x['successful_latency_ms']
|
||||
writer.writerow({'scenario': x['scenario'], 'api_replicas': x['api_replicas'],
|
||||
'total_api_cpu_budget': x['total_api_cpu_budget'],
|
||||
'total_api_memory_budget_mb': x['total_api_memory_budget_mb'],
|
||||
'successful_rps': x['successful_rps'], 'failed_requests': x['failed_requests'],
|
||||
'error_ratio': x['error_ratio'], 'p50_ms': lat['p50'], 'p95_ms': lat['p95'], 'p99_ms': lat['p99']})
|
||||
print(f'Wrote measured results to {OUT}')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
project="${COMPOSE_PROJECT_NAME:-mtusi-lab3}"
|
||||
found=""
|
||||
for id in $(docker ps -q --filter "label=com.docker.compose.project=${project}" --filter "label=com.docker.compose.service=api"); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found="$id"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$found" ]; then
|
||||
echo "No running API container matches TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop --time 3 "$found" >/dev/null
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8080/health/ready'
|
||||
timeout = int(sys.argv[2]) if len(sys.argv) > 2 else 30
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
raise SystemExit(0)
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
raise SystemExit('Readiness timeout: ' + url)
|
||||
Reference in New Issue
Block a user