Files
2026-11-06 19:22:27 +03:00

150 lines
7.1 KiB
Python

"""Track 1: reproducible local benchmark and overload/recovery proof. Uses Docker Compose."""
import argparse
import csv
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import platform
import subprocess
import tempfile
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
PROBE = (ROOT / 'student-tests/queue_probe.py').read_text(encoding='utf-8')
def run(args, input=None):
result = subprocess.run(args, cwd=ROOT, input=input, text=True, encoding='utf-8',
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if result.returncode:
raise RuntimeError('Command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-4000:])
return result.stdout.strip()
def compose(*args, override=None, input=None):
command = ['docker', 'compose', '-f', str(ROOT / 'compose.yaml')]
if override:
command += ['-f', str(override)]
return run(command + list(args), input)
def ready():
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
with urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) as response:
if response.status == 200:
return
except OSError:
pass
time.sleep(0.5)
raise RuntimeError('API did not become ready')
def probe(*args, service='worker'):
return json.loads(compose('exec', '-T', service, 'python', '-', *map(str, args), input=PROBE))
def configure(path, concurrency, limit, api=False):
path.write_text(json.dumps({'services': {
'worker': {'environment': {'WORKER_CONCURRENCY': str(concurrency)}},
'api': {'environment': {'MAX_PENDING_ATTACHMENTS': str(limit)}},
}}), encoding='utf-8')
services = ['worker', 'api'] if api else ['worker']
compose('up', '-d', '--no-deps', '--force-recreate', *services, override=path)
ready()
def write_results(data):
path = ROOT / 'evidence/queue-load.json'
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
fields = ['repeat', 'concurrency', 'prefetch_per_consumer', 'width', 'height', 'bytes',
'requests', 'clients', 'elapsed_s', 'throughput_per_s', 'ready_p95_s',
'queue_wait_p95_s', 'queue_lag_max_jobs', 'queue_oldest_max_s',
'processing_max', 'rss_peak_mib']
with (ROOT / 'evidence/queue-load.csv').open('w', encoding='utf-8', newline='') as output:
writer = csv.DictWriter(output, fieldnames=fields, extrasaction='ignore')
writer.writeheader()
writer.writerows(data['runs'])
def source_hash():
digest = hashlib.sha256()
paths = [ROOT / 'compose.yaml', ROOT / 'scripts/queue_experiment.py', ROOT / 'student-tests/queue_probe.py']
paths += list((ROOT / 'src').rglob('*.py')) + list((ROOT / 'migrations').glob('*.sql'))
for path in sorted(paths):
digest.update(path.relative_to(ROOT).as_posix().encode())
digest.update(path.read_bytes().replace(b'\r\n', b'\n'))
return digest.hexdigest()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--count', type=int, default=18)
parser.add_argument('--repeats', type=int, default=2)
args = parser.parse_args()
if not 6 <= args.count <= 48 or args.repeats < 1:
parser.error('count must be 6..48; repeats must be positive')
ready()
active = compose('exec', '-T', 'api', 'python', '-c',
"from src.database import create_db_engine; from sqlalchemy import text; "
"e=create_db_engine(); c=e.connect(); print(c.execute(text(\"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')\")).scalar())")
if active != '0':
raise RuntimeError('Wait for existing queued/processing work before this isolated experiment')
data = {
'started_at': datetime.now(timezone.utc).isoformat(),
'base_commit': run(['git', 'rev-parse', 'HEAD']), 'source_sha256': source_hash(),
'host_platform': platform.platform(),
'docker': json.loads(run(['docker', 'info', '--format', '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'])),
'image_id': run(['docker', 'image', 'inspect', 'mtusi2026lab5-api:local', '--format', '{{.Id}}']),
'profile': {'sizes': [[320, 200], [1280, 800], [2560, 1600]], 'concurrency': [1, 2, 4],
'prefetch_per_consumer': 1, 'requests_per_run': args.count, 'clients': 6,
'repeats': args.repeats, 'seed': 202605, 'jpeg_quality': 85,
'worker_cpus': 2, 'worker_memory_mib': 2048, 'sampling_seconds': 0.1,
'baseline_admission_limit': 64, 'overload_admission_limit': 12},
'runs': [],
}
with tempfile.TemporaryDirectory(prefix='lab5-queue-') as directory:
override = Path(directory) / 'compose.json'
try:
for repeat in range(1, args.repeats + 1):
order = [1, 2, 4] if repeat % 2 else [4, 2, 1]
sizes = data['profile']['sizes'] if repeat % 2 else list(reversed(data['profile']['sizes']))
for concurrency in order:
print(f'Configuration: repeat={repeat}, concurrency={concurrency}, prefetch=1', flush=True)
configure(override, concurrency, 64, api=repeat == 1 and concurrency == 1)
probe('measure', '--count', 6, '--clients', 6) # Same warmup for every configuration.
for width, height in sizes:
result = probe('measure', '--width', width, '--height', height, '--count', args.count)
assert result['processing_max'] <= concurrency
result.update(repeat=repeat, concurrency=concurrency, prefetch_per_consumer=1)
data['runs'].append(result)
write_results(data)
print(f" {width}x{height}: {result['throughput_per_s']:.2f}/s, "
f"p95={result['ready_p95_s']:.2f}s, RSS={result['rss_peak_mib']:.1f} MiB", flush=True)
print('Overload: stop worker, 24 concurrent uploads, shared limit=12', flush=True)
configure(override, 2, 12, api=True)
compose('stop', 'worker', 'cleanup')
overloaded = probe('overload', '--count', 24, '--clients', 12, '--limit', 12, service='api')
data['overload'] = overloaded
write_results(data)
# Accepted work must survive restarts while the worker is still stopped.
compose('restart', 'api', 'broker')
ready()
compose('start', 'worker')
data['recovery'] = probe('recover', '--expected', json.dumps(overloaded))
data['completed_at'] = datetime.now(timezone.utc).isoformat()
write_results(data)
print('PASS: 12 accepted, 12 rejected with Retry-After; all accepted IDs recovered; new upload accepted', flush=True)
finally:
# Restore the user's base Compose settings, including any values in .env.
compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker')
compose('start', 'cleanup')
ready()
if __name__ == '__main__':
main()