forked from mtusi2026/mtusi2026lab3
@@ -0,0 +1,124 @@
|
||||
"""Build the track 1 summary and figures from recorded JSON/CSV, without Docker."""
|
||||
from collections import defaultdict
|
||||
import csv
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
from statistics import mean, median
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
DATA = Path(__file__).resolve().parents[1] / 'evidence' / 'track1'
|
||||
|
||||
|
||||
def summarize(path):
|
||||
run = json.loads(path.read_text())
|
||||
resources = json.loads(path.with_name(path.stem + '-resources.json').read_text())
|
||||
case = resources['case']
|
||||
start = datetime.fromisoformat(run['workload_started_at'])
|
||||
first = start + timedelta(seconds=run['warmup_seconds']+2)
|
||||
last = start + timedelta(seconds=run['warmup_seconds']+run['requested_seconds'])
|
||||
cores = defaultdict(list)
|
||||
for sample in resources['resource_samples']:
|
||||
if first <= datetime.fromisoformat(sample['at']) <= last:
|
||||
totals = defaultdict(float)
|
||||
for container in sample['containers']:
|
||||
service = container['Name'].removeprefix('mtusi2026lab3-track1-').split('-')[0]
|
||||
totals[service] += float(container['CPUPerc'].rstrip('%'))/100
|
||||
for service, value in totals.items():
|
||||
cores[service].append(value)
|
||||
active = locked = connections = samples = 0
|
||||
for sample in run['db_samples']:
|
||||
if run['warmup_seconds'] <= sample['second'] <= run['warmup_seconds']+run['requested_seconds']:
|
||||
samples += 1
|
||||
for state in sample['states']:
|
||||
connections += state['connections']
|
||||
if state['state'] == 'active':
|
||||
active += state['connections']
|
||||
if state['wait_type'] == 'Lock':
|
||||
locked += state['connections']
|
||||
requests = run['successes'] + run['failed_requests']
|
||||
row = {'name': run['name'], 'replicas': case['replicas'], 'api_cpu_total': case['cpu']*case['replicas'],
|
||||
'api_ram_total_mib': int(case['memory'].rstrip('m'))*case['replicas'],
|
||||
'concurrency': run['concurrency'], 'rps': run['successful_rps'],
|
||||
'errors': run['failed_requests'], 'error_percent': round(100*run['failed_requests']/requests, 3),
|
||||
**run['latency_ms'], 'db_lock_percent_active': round(100*locked/active, 2) if active else 0,
|
||||
'db_connections_mean': round(connections/samples, 2),
|
||||
**{service+'_cpu_cores_mean': round(mean(cores[service]), 3) if cores[service] else None
|
||||
for service in ['api', 'nginx', 'postgres', 'redis', 'bench']}}
|
||||
# Verify aggregates against the request-level source data.
|
||||
with path.with_name(path.stem+'-requests.csv').open(newline='') as stream:
|
||||
raw = list(csv.DictReader(stream))
|
||||
assert len(raw) == requests
|
||||
assert sum(bool(r['error']) for r in raw) == row['errors']
|
||||
assert run['confirmed_messages_preserved']
|
||||
return row
|
||||
|
||||
|
||||
def interval(rows, field):
|
||||
values = [row[field] for row in rows]
|
||||
mid = median(values)
|
||||
return mid, mid-min(values), max(values)-mid
|
||||
|
||||
|
||||
def main():
|
||||
paths = [p for p in sorted(DATA.glob('*.json'))
|
||||
if p.stem.startswith(('fixed-', 'growing-', 'saturation-', 'improvement-'))
|
||||
and not p.stem.endswith('-resources')]
|
||||
rows = [summarize(path) for path in paths]
|
||||
with (DATA / 'summary.csv').open('w', newline='') as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
plt.rcParams.update({'font.size': 11, 'axes.grid': True, 'grid.alpha': .25})
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
|
||||
for series, label in [('fixed', 'API budget: 2 CPU / 1024 MiB'), ('growing', 'Each API: 1 CPU / 512 MiB')]:
|
||||
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
|
||||
groups = [[r for r in rows if r['name'].startswith(f'{series}-n{n}-')] for n in [1, 2, 4]]
|
||||
if not all(groups):
|
||||
continue
|
||||
values = [interval(group, field) for group in groups]
|
||||
axis.errorbar([1, 2, 4], [v[0] for v in values],
|
||||
yerr=[[v[1] for v in values], [v[2] for v in values]],
|
||||
marker='o', capsize=5, label=label)
|
||||
axis.set(xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
|
||||
axis.legend(fontsize=9)
|
||||
fig.suptitle('Same workload: 8 clients, one chat; median and min–max of 3 runs')
|
||||
fig.savefig(DATA / 'scaling.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
saturation = sorted([r for r in rows if r['name'].startswith('saturation-')], key=lambda r: r['concurrency'])
|
||||
fig, axes = plt.subplots(1, 3, figsize=(13, 4), layout='constrained')
|
||||
for axis, field, ylabel in zip(axes, ['rps', 'p95', 'db_lock_percent_active'],
|
||||
['Successful messages/s', 'p95 latency, ms', 'Active DB samples waiting for locks, %']):
|
||||
axis.plot([r['concurrency'] for r in saturation], [r[field] for r in saturation], marker='o')
|
||||
axis.set(xlabel='Concurrent clients', ylabel=ylabel, xscale='log', xticks=[1, 4, 8, 16, 32, 64])
|
||||
axis.set_xticklabels([1, 4, 8, 16, 32, 64])
|
||||
for row in saturation:
|
||||
if row['errors']:
|
||||
axes[0].annotate(f"{row['error_percent']:.1f}% errors", (row['concurrency'], row['rps']),
|
||||
xytext=(-4, 10), textcoords='offset points', ha='right', fontsize=8)
|
||||
fig.suptitle('Saturation: 4 API × 1 CPU; one run per load level')
|
||||
fig.savefig(DATA / 'saturation.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
groups = [[r for r in rows if r['name'].startswith(f'improvement-{v}-')] for v in ['before', 'after']]
|
||||
if all(groups):
|
||||
fig, axes = plt.subplots(1, 2, figsize=(9, 4), layout='constrained')
|
||||
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
|
||||
values = [interval(group, field) for group in groups]
|
||||
axis.bar(['Before', 'After'], [v[0] for v in values], color=['#627d98', '#159c80'],
|
||||
yerr=[[v[1] for v in values], [v[2] for v in values]], capsize=5)
|
||||
axis.set(ylabel=ylabel)
|
||||
for index, value in enumerate(values):
|
||||
axis.text(index, value[0]/2, f'{value[0]:.1f}', ha='center', color='white', fontsize=14)
|
||||
fig.suptitle('4 × 0.5 CPU → 2 × 1 CPU; same total budget, 3 pairs × 45 s')
|
||||
fig.savefig(DATA / 'improvement.png', dpi=160)
|
||||
plt.close(fig)
|
||||
print(f'Wrote summary.csv and figures for {len(rows)} runs')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Run one reproducible track 1 case in a separate Compose project."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = ['docker', 'compose', '-p', 'mtusi2026lab3-track1',
|
||||
'-f', 'compose.yaml', '-f', 'infra/scaling.compose.yaml']
|
||||
|
||||
|
||||
def command(*args, env=None, quiet_errors=False):
|
||||
return subprocess.check_output(args, cwd=ROOT, env=env, text=True, timeout=180,
|
||||
stderr=subprocess.DEVNULL if quiet_errors else None).strip()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('name')
|
||||
parser.add_argument('--replicas', type=int, choices=[1, 2, 4], default=2)
|
||||
parser.add_argument('--cpu', type=float, default=1)
|
||||
parser.add_argument('--memory', default='512m')
|
||||
parser.add_argument('--concurrency', type=int, default=32)
|
||||
parser.add_argument('--duration', type=int, default=15)
|
||||
args = parser.parse_args()
|
||||
assert Path(args.name).name == args.name
|
||||
env = dict(os.environ, TRACK1_REPLICAS=str(args.replicas), TRACK1_CPU=str(args.cpu),
|
||||
TRACK1_MEMORY=args.memory)
|
||||
output = ROOT / 'evidence' / 'track1'
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
command(*COMPOSE, 'up', '-d', '--no-build', '--wait', '--wait-timeout', '90', env=env)
|
||||
time.sleep(3) # nginx DNS cache expires in two seconds.
|
||||
ids = command(*COMPOSE, 'ps', '-q', env=env).splitlines()
|
||||
template = '{{json .Name}} {{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{.Image}}'
|
||||
limits = command('docker', 'inspect', '--format', template, *ids).splitlines()
|
||||
api_limits = [line.split() for line in limits if '/mtusi2026lab3-track1-api-' in line]
|
||||
assert len(api_limits) == args.replicas
|
||||
assert all(int(row[1]) == int(args.cpu*1e9)
|
||||
and int(row[2]) == int(args.memory.rstrip('m'))*1024*1024 for row in api_limits)
|
||||
metadata = {'case': vars(args), 'started_at': datetime.now(timezone.utc).isoformat(),
|
||||
'container_limits': limits, 'docker': command('docker', 'version', '--format', '{{.Server.Version}}'),
|
||||
'compose': command('docker', 'compose', 'version', '--short'),
|
||||
'host': command('docker', 'info', '--format', '{{.NCPU}} CPUs; {{.MemTotal}} bytes RAM'),
|
||||
'resource_samples': []}
|
||||
bench_name = 'mtusi2026lab3-track1-bench'
|
||||
process = subprocess.Popen([*COMPOSE, 'run', '--rm', '--no-deps', '--name', bench_name,
|
||||
'bench', 'python', '-u', '-m', 'scripts.scaling_load',
|
||||
'--name', args.name, '--concurrency', str(args.concurrency),
|
||||
'--duration', str(args.duration)], cwd=ROOT, env=env)
|
||||
while process.poll() is None:
|
||||
try:
|
||||
stats = command('docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids, bench_name,
|
||||
quiet_errors=True)
|
||||
metadata['resource_samples'].append({'at': datetime.now(timezone.utc).isoformat(),
|
||||
'containers': [json.loads(line) for line in stats.splitlines()]})
|
||||
except subprocess.CalledProcessError:
|
||||
pass # The one-shot benchmark can exit between poll and stats.
|
||||
time.sleep(1)
|
||||
(output / (args.name + '-resources.json')).write_text(json.dumps(metadata, indent=2)+'\n')
|
||||
if process.returncode:
|
||||
raise SystemExit(process.returncode)
|
||||
run = json.loads((output / (args.name + '.json')).read_text())
|
||||
assert len(run['instances']) == args.replicas, 'Not every API handled the workload'
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Closed-loop POST benchmark for the isolated track1 database, through nginx only."""
|
||||
import argparse
|
||||
import csv
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def db():
|
||||
return psycopg.connect(host=os.environ['POSTGRES_HOST'], dbname=os.environ['POSTGRES_DB'],
|
||||
user=os.environ['POSTGRES_USER'], password=os.environ['POSTGRES_PASSWORD'],
|
||||
autocommit=True, application_name='track1-monitor', connect_timeout=3)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--name', required=True)
|
||||
parser.add_argument('--concurrency', type=int, default=32)
|
||||
parser.add_argument('--duration', type=float, default=15)
|
||||
parser.add_argument('--warmup', type=float, default=3)
|
||||
args = parser.parse_args()
|
||||
assert 1 <= args.concurrency <= 128 and 1 <= args.duration <= 120
|
||||
assert os.environ.get('TRACK1_BENCHMARK') == '1'
|
||||
assert Path(args.name).name == args.name
|
||||
# Never reset the normal lab database. This project has separate synthetic volumes.
|
||||
with db() as conn:
|
||||
assert conn.execute('SELECT current_database()').fetchone()[0] == 'track1'
|
||||
conn.execute('TRUNCATE message_idempotency, messages, chat_members, chats, users')
|
||||
client = Client('http://nginx:8080', timeout=5)
|
||||
actor = client.user('scale')
|
||||
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
||||
body={'title': 'Scaling', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
||||
payload = 'm' * 64
|
||||
for _ in range(100):
|
||||
client.message(actor, chat, payload)
|
||||
stop_monitor = threading.Event()
|
||||
waits = []
|
||||
started = None
|
||||
started_at = None
|
||||
|
||||
def start_clock():
|
||||
nonlocal started, started_at
|
||||
started = time.monotonic()
|
||||
started_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
gate = threading.Barrier(args.concurrency + 1, action=start_clock)
|
||||
|
||||
def monitor():
|
||||
with db() as conn:
|
||||
while not stop_monitor.wait(0.5):
|
||||
rows = conn.execute("""
|
||||
SELECT state, coalesce(wait_event_type, 'CPU'), coalesce(wait_event, ''), count(*)
|
||||
FROM pg_stat_activity WHERE datname=current_database()
|
||||
AND application_name <> 'track1-monitor' AND backend_type='client backend'
|
||||
GROUP BY state, wait_event_type, wait_event
|
||||
""").fetchall()
|
||||
waits.append({'second': round(time.monotonic() - started, 3),
|
||||
'states': [{'state': s, 'wait_type': t, 'wait_event': e, 'connections': n}
|
||||
for s, t, e, n in rows]})
|
||||
|
||||
def worker(_):
|
||||
local = Client('http://nginx:8080', timeout=5)
|
||||
gate.wait(timeout=120)
|
||||
events, confirmations, instances = [], set(), Counter()
|
||||
attempts = 0
|
||||
while time.monotonic() - started < args.warmup + args.duration:
|
||||
begin = time.monotonic()
|
||||
attempts += 1
|
||||
try:
|
||||
response = local.message(actor, chat, payload, key=uuid4().hex)
|
||||
confirmations.add(response['json']['id'])
|
||||
instances[response['headers']['X-Instance-Id']] += 1
|
||||
error = None
|
||||
except Failure as exc:
|
||||
error = str(exc) # Client deliberately omits credentials and bodies.
|
||||
end = time.monotonic()
|
||||
if begin - started >= args.warmup:
|
||||
events.append((begin-started-args.warmup, end-begin, error))
|
||||
return events, confirmations, instances, attempts
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = [pool.submit(worker, i) for i in range(args.concurrency)]
|
||||
gate.wait(timeout=120)
|
||||
monitor_thread = threading.Thread(target=monitor)
|
||||
monitor_thread.start()
|
||||
results = [future.result() for future in futures]
|
||||
elapsed = time.monotonic() - started
|
||||
stop_monitor.set()
|
||||
monitor_thread.join()
|
||||
events = [event for result in results for event in result[0]]
|
||||
successful = sorted(duration for _, duration, error in events if error is None)
|
||||
errors = Counter(error for _, _, error in events if error is not None)
|
||||
confirmed = set().union(*(result[1] for result in results))
|
||||
instances = sum((result[2] for result in results), Counter())
|
||||
with db() as conn:
|
||||
stored = {str(row[0]) for row in conn.execute('SELECT id FROM messages WHERE chat_id=%s', (chat['id'],))}
|
||||
keys = conn.execute('SELECT count(*) FROM message_idempotency WHERE chat_id=%s', (chat['id'],)).fetchone()[0]
|
||||
assert confirmed <= stored, 'Confirmed message lost'
|
||||
assert len(stored) == keys, 'Message/idempotency count mismatch'
|
||||
assert len(stored) <= 100 + sum(result[3] for result in results), 'Duplicate messages'
|
||||
if len(confirmed) == sum(result[3] for result in results):
|
||||
assert len(stored) == 100 + len(confirmed)
|
||||
seconds = []
|
||||
for second in range(math.ceil(args.duration)):
|
||||
subset = [e for e in events if second <= e[0] < second+1]
|
||||
seconds.append({'second': second, 'successes': sum(e[2] is None for e in subset),
|
||||
'errors': sum(e[2] is not None for e in subset)})
|
||||
measured = elapsed - args.warmup
|
||||
def percentile(p):
|
||||
return round(successful[max(0, math.ceil(len(successful)*p)-1)]*1000, 3) if successful else None
|
||||
report = {'name': args.name, 'concurrency': args.concurrency, 'warmup_seconds': args.warmup,
|
||||
'workload_started_at': started_at,
|
||||
'requested_seconds': args.duration, 'elapsed_measurement_seconds': round(measured, 3),
|
||||
'successes': len(successful), 'errors': dict(errors), 'failed_requests': sum(errors.values()),
|
||||
'successful_rps': round(len(successful)/measured, 3),
|
||||
'latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)},
|
||||
'instances': dict(instances), 'confirmed_messages_preserved': True,
|
||||
'message_count': len(stored), 'seconds': seconds, 'db_samples': waits,
|
||||
'note': 'Closed loop; 64-byte text; 100 initial messages; one chat; no retries; success-only latency; drain included.'}
|
||||
Path('/results', args.name+'.json').write_text(json.dumps(report, indent=2)+'\n')
|
||||
with Path('/results', args.name+'-requests.csv').open('w', newline='') as stream:
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(['start_seconds', 'latency_ms', 'error'])
|
||||
writer.writerows((round(s, 6), round(d*1000, 3), e or '') for s, d, e in sorted(events))
|
||||
print(json.dumps({k: report[k] for k in ('name','successful_rps','failed_requests','latency_ms','instances')}), flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
"""The case order used in the track 1 report, on one unchanged application image."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('phase', choices=['saturation', 'scaling', 'improvement'])
|
||||
args = parser.parse_args()
|
||||
runner = Path(__file__).with_name('scaling.py')
|
||||
|
||||
def run(name, replicas=4, cpu=1, memory='512m', concurrency=32, duration=15):
|
||||
subprocess.run([sys.executable, str(runner), name, '--replicas', str(replicas),
|
||||
'--cpu', str(cpu), '--memory', memory, '--concurrency', str(concurrency),
|
||||
'--duration', str(duration)], check=True)
|
||||
|
||||
if args.phase == 'saturation':
|
||||
for concurrency in [1, 4, 8, 16, 32, 64]:
|
||||
run('saturation-c' + str(concurrency), concurrency=concurrency)
|
||||
elif args.phase == 'scaling':
|
||||
for repeat, order in enumerate([[1, 2, 4], [4, 2, 1], [2, 1, 4]], 1):
|
||||
for replicas in order:
|
||||
# Alternate which series runs first to reduce order bias.
|
||||
for series in (['fixed', 'growing'] if repeat % 2 else ['growing', 'fixed']):
|
||||
run(f'{series}-n{replicas}-r{repeat}', replicas=replicas,
|
||||
cpu=2/replicas if series == 'fixed' else 1,
|
||||
memory=f'{1024//replicas}m' if series == 'fixed' else '512m', concurrency=8)
|
||||
else:
|
||||
for repeat, order in enumerate([['before', 'after'], ['after', 'before'], ['before', 'after']], 1):
|
||||
for version in order:
|
||||
replicas = 4 if version == 'before' else 2
|
||||
run(f'improvement-{version}-r{repeat}', replicas=replicas,
|
||||
cpu=2/replicas, memory=f'{1024//replicas}m', concurrency=8, duration=45)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user