Files
mtusi2026lab5/scripts/track1/run_track1.py

231 lines
9.3 KiB
Python

#!/usr/bin/env python3
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
sys.path.insert(0, str(ROOT / 'student-tests'))
from client import Client, Failure
from lab5ops import MemSampler, gradient_png, psql, queues
import imagegen
def compose(*args, env=None):
e = dict(os.environ)
e.update(env or {})
r = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, env=e,
capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('compose %s: %s' % (args[:2], r.stderr[-300:]))
def noisy_png(w, h):
row_noise = lambda: b'\x00' + os.urandom(3 * w)
row_same = b'\x02' + bytes(3 * w)
raw = b''.join(row_noise() if y % 8 == 0 else row_same for y in range(h))
return imagegen.png(w, h, raw=raw, level=6)
SIZES = {
'S_320x200': lambda: (ROOT / 'tests/fixtures/sample.png').read_bytes(),
'M_2000x1500': lambda: noisy_png(2000, 1500),
'L_5000x4000': lambda: gradient_png(5000, 4000),
}
COUNTS = {'S_320x200': 120, 'M_2000x1500': 40, 'L_5000x4000': 16}
CONCURRENCY = [1, 2, 4]
def pct(values, p):
if not values:
return None
s = sorted(values)
return round(s[min(len(s) - 1, int(p * len(s)))], 3)
class LagSampler(threading.Thread):
def __init__(self, chat_id):
super().__init__(daemon=True)
self.chat_id = chat_id
self.samples = []
self.stop = threading.Event()
def run(self):
t0 = time.monotonic()
while not self.stop.is_set():
try:
r = psql("SELECT count(*) FILTER (WHERE j.status IN ('queued','processing')), "
"COALESCE(extract(epoch FROM now() - min(j.created_at) FILTER (WHERE j.status IN ('queued','processing'))), 0) "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % self.chat_id)
active, oldest = r.split('|')
q = queues().get('attachment.process.v1', {})
self.samples.append({'t': round(time.monotonic() - t0, 1), 'active': int(active),
'mq_ready': q.get('ready'), 'mq_unacked': q.get('unacked'),
'oldest_age_s': round(float(oldest), 2)})
except Exception:
pass
self.stop.wait(0.5)
def burst(c, actor, chat, data, n, threads=8):
codes = []
lock = threading.Lock()
def one(i):
try:
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=t1-%d.png' % i, [202, 503],
actor=actor, raw=data, headers={'Content-Type': 'image/png'})
code = r['status']
ra = r['headers'].get('Retry-After')
except Failure as exc:
code, ra = 'error:' + str(exc)[-40:], None
with lock:
codes.append((code, ra))
t0 = time.monotonic()
with ThreadPoolExecutor(threads) as pool:
list(pool.map(one, range(n)))
return codes, time.monotonic() - t0
def wait_drained(chat_id, timeout=900):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
n = int(psql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
"WHERE a.chat_id = '%s' AND j.status IN ('queued','processing')" % chat_id))
if n == 0:
return True
time.sleep(1)
return False
def job_stats(chat_id):
rows = psql("SELECT j.status, extract(epoch FROM j.created_at), extract(epoch FROM j.finished_at), j.attempt "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % chat_id)
jobs = [r.split('|') for r in rows.splitlines()]
ready = [(float(c), float(f)) for s, c, f, _ in jobs if s == 'ready']
ttr = [f - c for c, f in ready]
span = (max(f for _, f in ready) - min(c for c, _ in ready)) if ready else 0
return {'jobs': len(jobs), 'ready': len(ready), 'failed': sum(1 for j in jobs if j[0] == 'failed'),
'retried_jobs': sum(1 for j in jobs if int(j[3]) > 1),
'throughput_jobs_per_s': round(len(ready) / span, 2) if span else None,
'pipeline_span_s': round(span, 2),
'time_to_ready_s': {'p50': pct(ttr, .5), 'p95': pct(ttr, .95), 'max': round(max(ttr), 3) if ttr else None}}
def set_worker(concurrency):
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_CONCURRENCY': str(concurrency)})
time.sleep(2)
def one_run(base_url, size, conc):
c = Client(base_url, timeout=60)
alice, _bob, _eve, chat = c.bootstrap()
data = SIZES[size]()
n = COUNTS[size]
lag = LagSampler(chat['id'])
with MemSampler('worker') as mem:
lag.start()
codes, upload_s = burst(c, alice, chat, data, n)
drained = wait_drained(chat['id'])
time.sleep(1)
lag.stop.set()
lag.join()
st = job_stats(chat['id'])
s = lag.samples
return {'size': size, 'image_bytes': len(data), 'worker_concurrency_prefetch': conc, 'uploads': n,
'accepted_202': sum(1 for x, _ in codes if x == 202),
'rejected': sum(1 for x, _ in codes if x != 202),
'upload_burst_s': round(upload_s, 2), 'drained': drained, **st,
'queue_lag': {'max_active_jobs': max((x['active'] for x in s), default=None),
'max_mq_ready': max((x['mq_ready'] or 0 for x in s), default=None),
'max_oldest_job_age_s': max((x['oldest_age_s'] for x in s), default=None)},
'worker_peak_rss_mib': round(mem.peak, 1),
'lag_timeline': s[::max(1, len(s) // 40)]}
def cmd_matrix(a):
runs = []
try:
for conc in CONCURRENCY:
set_worker(conc)
for size in SIZES:
r = one_run(a.base_url, size, conc)
print(json.dumps({k: r[k] for k in ('size', 'worker_concurrency_prefetch', 'throughput_jobs_per_s',
'time_to_ready_s', 'worker_peak_rss_mib', 'queue_lag')}), flush=True)
runs.append(r)
finally:
set_worker(2)
return {'runs': runs}
def cmd_backpressure(a):
bound = 20
compose('up', '-d', '--no-deps', '--wait', 'api', env={'MAX_ACTIVE_JOBS': str(bound)})
set_worker(1)
try:
c = Client(a.base_url, timeout=60)
alice, _bob, _eve, chat = c.bootstrap()
data = SIZES['M_2000x1500']()
lag = LagSampler(chat['id'])
lag.start()
t0 = time.monotonic()
codes, upload_s = burst(c, alice, chat, data, 80)
drained = wait_drained(chat['id'])
drain_s = time.monotonic() - t0
t1 = time.monotonic()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after.png', 202, actor=alice,
raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
while True:
st = c.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']['status']
if st in ('ready', 'failed'):
break
time.sleep(0.2)
recovery_ready_s = time.monotonic() - t1
time.sleep(1)
lag.stop.set()
lag.join()
s = lag.samples
return {'max_active_jobs_bound': bound, 'worker_concurrency': 1, 'uploads': 80,
'accepted_202': sum(1 for x, _ in codes if x == 202),
'rejected_503': sum(1 for x, _ in codes if x == 503),
'retry_after_header': sorted({ra for x, ra in codes if x == 503}),
'other': [x for x, _ in codes if x not in (202, 503)],
'upload_burst_s': round(upload_s, 2), 'drained': drained, 'drain_s': round(drain_s, 1),
'max_active_observed': max((x['active'] for x in s), default=None),
'jobs': job_stats(chat['id']), 'after_overload_upload_status': st,
'after_overload_seconds_to_ready': round(recovery_ready_s, 2),
'lag_timeline': s[::max(1, len(s) // 40)]}
finally:
compose('up', '-d', '--no-deps', '--wait', 'api')
set_worker(2)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('command', choices=['matrix', 'backpressure'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
a = p.parse_args()
env = {'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'worker_limits': 'cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB',
'host': 'Windows 11, Docker Desktop 29.7.2 (WSL2)'}
res = {'experiment': a.command, **env, **(cmd_matrix(a) if a.command == 'matrix' else cmd_backpressure(a))}
text = json.dumps(res, ensure_ascii=False, indent=1)
if a.output:
(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
print(text[:3000])
if __name__ == '__main__':
main()