Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-10-23 17:50:43 +03:00
co-authored by Cursor
parent 1d85938fb4
commit 3417e22ba0
102 changed files with 112776 additions and 171 deletions
+124
View File
@@ -0,0 +1,124 @@
"""Build the track 1 summary and figures from recorded JSON/CSV, without Docker."""
from collections import defaultdict
import csv
from datetime import datetime, timedelta
import json
from pathlib import Path
from statistics import mean, median
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
DATA = Path(__file__).resolve().parents[1] / 'evidence' / 'track1'
def summarize(path):
run = json.loads(path.read_text())
resources = json.loads(path.with_name(path.stem + '-resources.json').read_text())
case = resources['case']
start = datetime.fromisoformat(run['workload_started_at'])
first = start + timedelta(seconds=run['warmup_seconds']+2)
last = start + timedelta(seconds=run['warmup_seconds']+run['requested_seconds'])
cores = defaultdict(list)
for sample in resources['resource_samples']:
if first <= datetime.fromisoformat(sample['at']) <= last:
totals = defaultdict(float)
for container in sample['containers']:
service = container['Name'].removeprefix('mtusi2026lab3-track1-').split('-')[0]
totals[service] += float(container['CPUPerc'].rstrip('%'))/100
for service, value in totals.items():
cores[service].append(value)
active = locked = connections = samples = 0
for sample in run['db_samples']:
if run['warmup_seconds'] <= sample['second'] <= run['warmup_seconds']+run['requested_seconds']:
samples += 1
for state in sample['states']:
connections += state['connections']
if state['state'] == 'active':
active += state['connections']
if state['wait_type'] == 'Lock':
locked += state['connections']
requests = run['successes'] + run['failed_requests']
row = {'name': run['name'], 'replicas': case['replicas'], 'api_cpu_total': case['cpu']*case['replicas'],
'api_ram_total_mib': int(case['memory'].rstrip('m'))*case['replicas'],
'concurrency': run['concurrency'], 'rps': run['successful_rps'],
'errors': run['failed_requests'], 'error_percent': round(100*run['failed_requests']/requests, 3),
**run['latency_ms'], 'db_lock_percent_active': round(100*locked/active, 2) if active else 0,
'db_connections_mean': round(connections/samples, 2),
**{service+'_cpu_cores_mean': round(mean(cores[service]), 3) if cores[service] else None
for service in ['api', 'nginx', 'postgres', 'redis', 'bench']}}
# Verify aggregates against the request-level source data.
with path.with_name(path.stem+'-requests.csv').open(newline='') as stream:
raw = list(csv.DictReader(stream))
assert len(raw) == requests
assert sum(bool(r['error']) for r in raw) == row['errors']
assert run['confirmed_messages_preserved']
return row
def interval(rows, field):
values = [row[field] for row in rows]
mid = median(values)
return mid, mid-min(values), max(values)-mid
def main():
paths = [p for p in sorted(DATA.glob('*.json'))
if p.stem.startswith(('fixed-', 'growing-', 'saturation-', 'improvement-'))
and not p.stem.endswith('-resources')]
rows = [summarize(path) for path in paths]
with (DATA / 'summary.csv').open('w', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
plt.rcParams.update({'font.size': 11, 'axes.grid': True, 'grid.alpha': .25})
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
for series, label in [('fixed', 'API budget: 2 CPU / 1024 MiB'), ('growing', 'Each API: 1 CPU / 512 MiB')]:
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
groups = [[r for r in rows if r['name'].startswith(f'{series}-n{n}-')] for n in [1, 2, 4]]
if not all(groups):
continue
values = [interval(group, field) for group in groups]
axis.errorbar([1, 2, 4], [v[0] for v in values],
yerr=[[v[1] for v in values], [v[2] for v in values]],
marker='o', capsize=5, label=label)
axis.set(xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
axis.legend(fontsize=9)
fig.suptitle('Same workload: 8 clients, one chat; median and min–max of 3 runs')
fig.savefig(DATA / 'scaling.png', dpi=160)
plt.close(fig)
saturation = sorted([r for r in rows if r['name'].startswith('saturation-')], key=lambda r: r['concurrency'])
fig, axes = plt.subplots(1, 3, figsize=(13, 4), layout='constrained')
for axis, field, ylabel in zip(axes, ['rps', 'p95', 'db_lock_percent_active'],
['Successful messages/s', 'p95 latency, ms', 'Active DB samples waiting for locks, %']):
axis.plot([r['concurrency'] for r in saturation], [r[field] for r in saturation], marker='o')
axis.set(xlabel='Concurrent clients', ylabel=ylabel, xscale='log', xticks=[1, 4, 8, 16, 32, 64])
axis.set_xticklabels([1, 4, 8, 16, 32, 64])
for row in saturation:
if row['errors']:
axes[0].annotate(f"{row['error_percent']:.1f}% errors", (row['concurrency'], row['rps']),
xytext=(-4, 10), textcoords='offset points', ha='right', fontsize=8)
fig.suptitle('Saturation: 4 API × 1 CPU; one run per load level')
fig.savefig(DATA / 'saturation.png', dpi=160)
plt.close(fig)
groups = [[r for r in rows if r['name'].startswith(f'improvement-{v}-')] for v in ['before', 'after']]
if all(groups):
fig, axes = plt.subplots(1, 2, figsize=(9, 4), layout='constrained')
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
values = [interval(group, field) for group in groups]
axis.bar(['Before', 'After'], [v[0] for v in values], color=['#627d98', '#159c80'],
yerr=[[v[1] for v in values], [v[2] for v in values]], capsize=5)
axis.set(ylabel=ylabel)
for index, value in enumerate(values):
axis.text(index, value[0]/2, f'{value[0]:.1f}', ha='center', color='white', fontsize=14)
fig.suptitle('4 × 0.5 CPU → 2 × 1 CPU; same total budget, 3 pairs × 45 s')
fig.savefig(DATA / 'improvement.png', dpi=160)
plt.close(fig)
print(f'Wrote summary.csv and figures for {len(rows)} runs')
if __name__ == '__main__':
main()
+70
View File
@@ -0,0 +1,70 @@
"""Run one reproducible track 1 case in a separate Compose project."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import time
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ['docker', 'compose', '-p', 'mtusi2026lab3-track1',
'-f', 'compose.yaml', '-f', 'infra/scaling.compose.yaml']
def command(*args, env=None, quiet_errors=False):
return subprocess.check_output(args, cwd=ROOT, env=env, text=True, timeout=180,
stderr=subprocess.DEVNULL if quiet_errors else None).strip()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('name')
parser.add_argument('--replicas', type=int, choices=[1, 2, 4], default=2)
parser.add_argument('--cpu', type=float, default=1)
parser.add_argument('--memory', default='512m')
parser.add_argument('--concurrency', type=int, default=32)
parser.add_argument('--duration', type=int, default=15)
args = parser.parse_args()
assert Path(args.name).name == args.name
env = dict(os.environ, TRACK1_REPLICAS=str(args.replicas), TRACK1_CPU=str(args.cpu),
TRACK1_MEMORY=args.memory)
output = ROOT / 'evidence' / 'track1'
output.mkdir(parents=True, exist_ok=True)
command(*COMPOSE, 'up', '-d', '--no-build', '--wait', '--wait-timeout', '90', env=env)
time.sleep(3) # nginx DNS cache expires in two seconds.
ids = command(*COMPOSE, 'ps', '-q', env=env).splitlines()
template = '{{json .Name}} {{.HostConfig.NanoCpus}} {{.HostConfig.Memory}} {{.Image}}'
limits = command('docker', 'inspect', '--format', template, *ids).splitlines()
api_limits = [line.split() for line in limits if '/mtusi2026lab3-track1-api-' in line]
assert len(api_limits) == args.replicas
assert all(int(row[1]) == int(args.cpu*1e9)
and int(row[2]) == int(args.memory.rstrip('m'))*1024*1024 for row in api_limits)
metadata = {'case': vars(args), 'started_at': datetime.now(timezone.utc).isoformat(),
'container_limits': limits, 'docker': command('docker', 'version', '--format', '{{.Server.Version}}'),
'compose': command('docker', 'compose', 'version', '--short'),
'host': command('docker', 'info', '--format', '{{.NCPU}} CPUs; {{.MemTotal}} bytes RAM'),
'resource_samples': []}
bench_name = 'mtusi2026lab3-track1-bench'
process = subprocess.Popen([*COMPOSE, 'run', '--rm', '--no-deps', '--name', bench_name,
'bench', 'python', '-u', '-m', 'scripts.scaling_load',
'--name', args.name, '--concurrency', str(args.concurrency),
'--duration', str(args.duration)], cwd=ROOT, env=env)
while process.poll() is None:
try:
stats = command('docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids, bench_name,
quiet_errors=True)
metadata['resource_samples'].append({'at': datetime.now(timezone.utc).isoformat(),
'containers': [json.loads(line) for line in stats.splitlines()]})
except subprocess.CalledProcessError:
pass # The one-shot benchmark can exit between poll and stats.
time.sleep(1)
(output / (args.name + '-resources.json')).write_text(json.dumps(metadata, indent=2)+'\n')
if process.returncode:
raise SystemExit(process.returncode)
run = json.loads((output / (args.name + '.json')).read_text())
assert len(run['instances']) == args.replicas, 'Not every API handled the workload'
if __name__ == '__main__':
main()
+142
View File
@@ -0,0 +1,142 @@
"""Closed-loop POST benchmark for the isolated track1 database, through nginx only."""
import argparse
import csv
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import math
import os
from pathlib import Path
import sys
import threading
import time
from uuid import uuid4
import psycopg
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure
def db():
return psycopg.connect(host=os.environ['POSTGRES_HOST'], dbname=os.environ['POSTGRES_DB'],
user=os.environ['POSTGRES_USER'], password=os.environ['POSTGRES_PASSWORD'],
autocommit=True, application_name='track1-monitor', connect_timeout=3)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--name', required=True)
parser.add_argument('--concurrency', type=int, default=32)
parser.add_argument('--duration', type=float, default=15)
parser.add_argument('--warmup', type=float, default=3)
args = parser.parse_args()
assert 1 <= args.concurrency <= 128 and 1 <= args.duration <= 120
assert os.environ.get('TRACK1_BENCHMARK') == '1'
assert Path(args.name).name == args.name
# Never reset the normal lab database. This project has separate synthetic volumes.
with db() as conn:
assert conn.execute('SELECT current_database()').fetchone()[0] == 'track1'
conn.execute('TRUNCATE message_idempotency, messages, chat_members, chats, users')
client = Client('http://nginx:8080', timeout=5)
actor = client.user('scale')
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Scaling', 'member_ids': [actor['id']]}, schema='Chat')['json']
payload = 'm' * 64
for _ in range(100):
client.message(actor, chat, payload)
stop_monitor = threading.Event()
waits = []
started = None
started_at = None
def start_clock():
nonlocal started, started_at
started = time.monotonic()
started_at = datetime.now(timezone.utc).isoformat()
gate = threading.Barrier(args.concurrency + 1, action=start_clock)
def monitor():
with db() as conn:
while not stop_monitor.wait(0.5):
rows = conn.execute("""
SELECT state, coalesce(wait_event_type, 'CPU'), coalesce(wait_event, ''), count(*)
FROM pg_stat_activity WHERE datname=current_database()
AND application_name <> 'track1-monitor' AND backend_type='client backend'
GROUP BY state, wait_event_type, wait_event
""").fetchall()
waits.append({'second': round(time.monotonic() - started, 3),
'states': [{'state': s, 'wait_type': t, 'wait_event': e, 'connections': n}
for s, t, e, n in rows]})
def worker(_):
local = Client('http://nginx:8080', timeout=5)
gate.wait(timeout=120)
events, confirmations, instances = [], set(), Counter()
attempts = 0
while time.monotonic() - started < args.warmup + args.duration:
begin = time.monotonic()
attempts += 1
try:
response = local.message(actor, chat, payload, key=uuid4().hex)
confirmations.add(response['json']['id'])
instances[response['headers']['X-Instance-Id']] += 1
error = None
except Failure as exc:
error = str(exc) # Client deliberately omits credentials and bodies.
end = time.monotonic()
if begin - started >= args.warmup:
events.append((begin-started-args.warmup, end-begin, error))
return events, confirmations, instances, attempts
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = [pool.submit(worker, i) for i in range(args.concurrency)]
gate.wait(timeout=120)
monitor_thread = threading.Thread(target=monitor)
monitor_thread.start()
results = [future.result() for future in futures]
elapsed = time.monotonic() - started
stop_monitor.set()
monitor_thread.join()
events = [event for result in results for event in result[0]]
successful = sorted(duration for _, duration, error in events if error is None)
errors = Counter(error for _, _, error in events if error is not None)
confirmed = set().union(*(result[1] for result in results))
instances = sum((result[2] for result in results), Counter())
with db() as conn:
stored = {str(row[0]) for row in conn.execute('SELECT id FROM messages WHERE chat_id=%s', (chat['id'],))}
keys = conn.execute('SELECT count(*) FROM message_idempotency WHERE chat_id=%s', (chat['id'],)).fetchone()[0]
assert confirmed <= stored, 'Confirmed message lost'
assert len(stored) == keys, 'Message/idempotency count mismatch'
assert len(stored) <= 100 + sum(result[3] for result in results), 'Duplicate messages'
if len(confirmed) == sum(result[3] for result in results):
assert len(stored) == 100 + len(confirmed)
seconds = []
for second in range(math.ceil(args.duration)):
subset = [e for e in events if second <= e[0] < second+1]
seconds.append({'second': second, 'successes': sum(e[2] is None for e in subset),
'errors': sum(e[2] is not None for e in subset)})
measured = elapsed - args.warmup
def percentile(p):
return round(successful[max(0, math.ceil(len(successful)*p)-1)]*1000, 3) if successful else None
report = {'name': args.name, 'concurrency': args.concurrency, 'warmup_seconds': args.warmup,
'workload_started_at': started_at,
'requested_seconds': args.duration, 'elapsed_measurement_seconds': round(measured, 3),
'successes': len(successful), 'errors': dict(errors), 'failed_requests': sum(errors.values()),
'successful_rps': round(len(successful)/measured, 3),
'latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)},
'instances': dict(instances), 'confirmed_messages_preserved': True,
'message_count': len(stored), 'seconds': seconds, 'db_samples': waits,
'note': 'Closed loop; 64-byte text; 100 initial messages; one chat; no retries; success-only latency; drain included.'}
Path('/results', args.name+'.json').write_text(json.dumps(report, indent=2)+'\n')
with Path('/results', args.name+'-requests.csv').open('w', newline='') as stream:
writer = csv.writer(stream)
writer.writerow(['start_seconds', 'latency_ms', 'error'])
writer.writerows((round(s, 6), round(d*1000, 3), e or '') for s, d, e in sorted(events))
print(json.dumps({k: report[k] for k in ('name','successful_rps','failed_requests','latency_ms','instances')}), flush=True)
if __name__ == '__main__':
main()
+39
View File
@@ -0,0 +1,39 @@
"""The case order used in the track 1 report, on one unchanged application image."""
import argparse
from pathlib import Path
import subprocess
import sys
def main():
parser = argparse.ArgumentParser()
parser.add_argument('phase', choices=['saturation', 'scaling', 'improvement'])
args = parser.parse_args()
runner = Path(__file__).with_name('scaling.py')
def run(name, replicas=4, cpu=1, memory='512m', concurrency=32, duration=15):
subprocess.run([sys.executable, str(runner), name, '--replicas', str(replicas),
'--cpu', str(cpu), '--memory', memory, '--concurrency', str(concurrency),
'--duration', str(duration)], check=True)
if args.phase == 'saturation':
for concurrency in [1, 4, 8, 16, 32, 64]:
run('saturation-c' + str(concurrency), concurrency=concurrency)
elif args.phase == 'scaling':
for repeat, order in enumerate([[1, 2, 4], [4, 2, 1], [2, 1, 4]], 1):
for replicas in order:
# Alternate which series runs first to reduce order bias.
for series in (['fixed', 'growing'] if repeat % 2 else ['growing', 'fixed']):
run(f'{series}-n{replicas}-r{repeat}', replicas=replicas,
cpu=2/replicas if series == 'fixed' else 1,
memory=f'{1024//replicas}m' if series == 'fixed' else '512m', concurrency=8)
else:
for repeat, order in enumerate([['before', 'after'], ['after', 'before'], ['before', 'after']], 1):
for version in order:
replicas = 4 if version == 'before' else 2
run(f'improvement-{version}-r{repeat}', replicas=replicas,
cpu=2/replicas, memory=f'{1024//replicas}m', concurrency=8, duration=45)
if __name__ == '__main__':
main()