150 lines
7.1 KiB
Python
150 lines
7.1 KiB
Python
"""Track 1: reproducible local benchmark and overload/recovery proof. Uses Docker Compose."""
|
|
import argparse
|
|
import csv
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import platform
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROBE = (ROOT / 'student-tests/queue_probe.py').read_text(encoding='utf-8')
|
|
|
|
|
|
def run(args, input=None):
|
|
result = subprocess.run(args, cwd=ROOT, input=input, text=True, encoding='utf-8',
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if result.returncode:
|
|
raise RuntimeError('Command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-4000:])
|
|
return result.stdout.strip()
|
|
|
|
|
|
def compose(*args, override=None, input=None):
|
|
command = ['docker', 'compose', '-f', str(ROOT / 'compose.yaml')]
|
|
if override:
|
|
command += ['-f', str(override)]
|
|
return run(command + list(args), input)
|
|
|
|
|
|
def ready():
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) as response:
|
|
if response.status == 200:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.5)
|
|
raise RuntimeError('API did not become ready')
|
|
|
|
|
|
def probe(*args, service='worker'):
|
|
return json.loads(compose('exec', '-T', service, 'python', '-', *map(str, args), input=PROBE))
|
|
|
|
|
|
def configure(path, concurrency, limit, api=False):
|
|
path.write_text(json.dumps({'services': {
|
|
'worker': {'environment': {'WORKER_CONCURRENCY': str(concurrency)}},
|
|
'api': {'environment': {'MAX_PENDING_ATTACHMENTS': str(limit)}},
|
|
}}), encoding='utf-8')
|
|
services = ['worker', 'api'] if api else ['worker']
|
|
compose('up', '-d', '--no-deps', '--force-recreate', *services, override=path)
|
|
ready()
|
|
|
|
|
|
def write_results(data):
|
|
path = ROOT / 'evidence/queue-load.json'
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
fields = ['repeat', 'concurrency', 'prefetch_per_consumer', 'width', 'height', 'bytes',
|
|
'requests', 'clients', 'elapsed_s', 'throughput_per_s', 'ready_p95_s',
|
|
'queue_wait_p95_s', 'queue_lag_max_jobs', 'queue_oldest_max_s',
|
|
'processing_max', 'rss_peak_mib']
|
|
with (ROOT / 'evidence/queue-load.csv').open('w', encoding='utf-8', newline='') as output:
|
|
writer = csv.DictWriter(output, fieldnames=fields, extrasaction='ignore')
|
|
writer.writeheader()
|
|
writer.writerows(data['runs'])
|
|
|
|
|
|
def source_hash():
|
|
digest = hashlib.sha256()
|
|
paths = [ROOT / 'compose.yaml', ROOT / 'scripts/queue_experiment.py', ROOT / 'student-tests/queue_probe.py']
|
|
paths += list((ROOT / 'src').rglob('*.py')) + list((ROOT / 'migrations').glob('*.sql'))
|
|
for path in sorted(paths):
|
|
digest.update(path.relative_to(ROOT).as_posix().encode())
|
|
digest.update(path.read_bytes().replace(b'\r\n', b'\n'))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--count', type=int, default=18)
|
|
parser.add_argument('--repeats', type=int, default=2)
|
|
args = parser.parse_args()
|
|
if not 6 <= args.count <= 48 or args.repeats < 1:
|
|
parser.error('count must be 6..48; repeats must be positive')
|
|
ready()
|
|
active = compose('exec', '-T', 'api', 'python', '-c',
|
|
"from src.database import create_db_engine; from sqlalchemy import text; "
|
|
"e=create_db_engine(); c=e.connect(); print(c.execute(text(\"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')\")).scalar())")
|
|
if active != '0':
|
|
raise RuntimeError('Wait for existing queued/processing work before this isolated experiment')
|
|
data = {
|
|
'started_at': datetime.now(timezone.utc).isoformat(),
|
|
'base_commit': run(['git', 'rev-parse', 'HEAD']), 'source_sha256': source_hash(),
|
|
'host_platform': platform.platform(),
|
|
'docker': json.loads(run(['docker', 'info', '--format', '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'])),
|
|
'image_id': run(['docker', 'image', 'inspect', 'mtusi2026lab5-api:local', '--format', '{{.Id}}']),
|
|
'profile': {'sizes': [[320, 200], [1280, 800], [2560, 1600]], 'concurrency': [1, 2, 4],
|
|
'prefetch_per_consumer': 1, 'requests_per_run': args.count, 'clients': 6,
|
|
'repeats': args.repeats, 'seed': 202605, 'jpeg_quality': 85,
|
|
'worker_cpus': 2, 'worker_memory_mib': 2048, 'sampling_seconds': 0.1,
|
|
'baseline_admission_limit': 64, 'overload_admission_limit': 12},
|
|
'runs': [],
|
|
}
|
|
with tempfile.TemporaryDirectory(prefix='lab5-queue-') as directory:
|
|
override = Path(directory) / 'compose.json'
|
|
try:
|
|
for repeat in range(1, args.repeats + 1):
|
|
order = [1, 2, 4] if repeat % 2 else [4, 2, 1]
|
|
sizes = data['profile']['sizes'] if repeat % 2 else list(reversed(data['profile']['sizes']))
|
|
for concurrency in order:
|
|
print(f'Configuration: repeat={repeat}, concurrency={concurrency}, prefetch=1', flush=True)
|
|
configure(override, concurrency, 64, api=repeat == 1 and concurrency == 1)
|
|
probe('measure', '--count', 6, '--clients', 6) # Same warmup for every configuration.
|
|
for width, height in sizes:
|
|
result = probe('measure', '--width', width, '--height', height, '--count', args.count)
|
|
assert result['processing_max'] <= concurrency
|
|
result.update(repeat=repeat, concurrency=concurrency, prefetch_per_consumer=1)
|
|
data['runs'].append(result)
|
|
write_results(data)
|
|
print(f" {width}x{height}: {result['throughput_per_s']:.2f}/s, "
|
|
f"p95={result['ready_p95_s']:.2f}s, RSS={result['rss_peak_mib']:.1f} MiB", flush=True)
|
|
print('Overload: stop worker, 24 concurrent uploads, shared limit=12', flush=True)
|
|
configure(override, 2, 12, api=True)
|
|
compose('stop', 'worker', 'cleanup')
|
|
overloaded = probe('overload', '--count', 24, '--clients', 12, '--limit', 12, service='api')
|
|
data['overload'] = overloaded
|
|
write_results(data)
|
|
# Accepted work must survive restarts while the worker is still stopped.
|
|
compose('restart', 'api', 'broker')
|
|
ready()
|
|
compose('start', 'worker')
|
|
data['recovery'] = probe('recover', '--expected', json.dumps(overloaded))
|
|
data['completed_at'] = datetime.now(timezone.utc).isoformat()
|
|
write_results(data)
|
|
print('PASS: 12 accepted, 12 rejected with Retry-After; all accepted IDs recovered; new upload accepted', flush=True)
|
|
finally:
|
|
# Restore the user's base Compose settings, including any values in .env.
|
|
compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker')
|
|
compose('start', 'cleanup')
|
|
ready()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|