forked from mtusi2026/mtusi2026lab5
231 lines
9.3 KiB
Python
231 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
sys.path.insert(0, str(ROOT / 'student-tests'))
|
|
from client import Client, Failure
|
|
from lab5ops import MemSampler, gradient_png, psql, queues
|
|
import imagegen
|
|
|
|
|
|
def compose(*args, env=None):
|
|
e = dict(os.environ)
|
|
e.update(env or {})
|
|
r = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, env=e,
|
|
capture_output=True, text=True, encoding='utf-8')
|
|
if r.returncode != 0:
|
|
raise RuntimeError('compose %s: %s' % (args[:2], r.stderr[-300:]))
|
|
|
|
|
|
def noisy_png(w, h):
|
|
row_noise = lambda: b'\x00' + os.urandom(3 * w)
|
|
row_same = b'\x02' + bytes(3 * w)
|
|
raw = b''.join(row_noise() if y % 8 == 0 else row_same for y in range(h))
|
|
return imagegen.png(w, h, raw=raw, level=6)
|
|
|
|
|
|
SIZES = {
|
|
'S_320x200': lambda: (ROOT / 'tests/fixtures/sample.png').read_bytes(),
|
|
'M_2000x1500': lambda: noisy_png(2000, 1500),
|
|
'L_5000x4000': lambda: gradient_png(5000, 4000),
|
|
}
|
|
COUNTS = {'S_320x200': 120, 'M_2000x1500': 40, 'L_5000x4000': 16}
|
|
CONCURRENCY = [1, 2, 4]
|
|
|
|
|
|
def pct(values, p):
|
|
if not values:
|
|
return None
|
|
s = sorted(values)
|
|
return round(s[min(len(s) - 1, int(p * len(s)))], 3)
|
|
|
|
|
|
class LagSampler(threading.Thread):
|
|
def __init__(self, chat_id):
|
|
super().__init__(daemon=True)
|
|
self.chat_id = chat_id
|
|
self.samples = []
|
|
self.stop = threading.Event()
|
|
|
|
def run(self):
|
|
t0 = time.monotonic()
|
|
while not self.stop.is_set():
|
|
try:
|
|
r = psql("SELECT count(*) FILTER (WHERE j.status IN ('queued','processing')), "
|
|
"COALESCE(extract(epoch FROM now() - min(j.created_at) FILTER (WHERE j.status IN ('queued','processing'))), 0) "
|
|
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % self.chat_id)
|
|
active, oldest = r.split('|')
|
|
q = queues().get('attachment.process.v1', {})
|
|
self.samples.append({'t': round(time.monotonic() - t0, 1), 'active': int(active),
|
|
'mq_ready': q.get('ready'), 'mq_unacked': q.get('unacked'),
|
|
'oldest_age_s': round(float(oldest), 2)})
|
|
except Exception:
|
|
pass
|
|
self.stop.wait(0.5)
|
|
|
|
|
|
def burst(c, actor, chat, data, n, threads=8):
|
|
codes = []
|
|
lock = threading.Lock()
|
|
|
|
def one(i):
|
|
try:
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=t1-%d.png' % i, [202, 503],
|
|
actor=actor, raw=data, headers={'Content-Type': 'image/png'})
|
|
code = r['status']
|
|
ra = r['headers'].get('Retry-After')
|
|
except Failure as exc:
|
|
code, ra = 'error:' + str(exc)[-40:], None
|
|
with lock:
|
|
codes.append((code, ra))
|
|
|
|
t0 = time.monotonic()
|
|
with ThreadPoolExecutor(threads) as pool:
|
|
list(pool.map(one, range(n)))
|
|
return codes, time.monotonic() - t0
|
|
|
|
|
|
def wait_drained(chat_id, timeout=900):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
n = int(psql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
|
"WHERE a.chat_id = '%s' AND j.status IN ('queued','processing')" % chat_id))
|
|
if n == 0:
|
|
return True
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def job_stats(chat_id):
|
|
rows = psql("SELECT j.status, extract(epoch FROM j.created_at), extract(epoch FROM j.finished_at), j.attempt "
|
|
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE a.chat_id = '%s'" % chat_id)
|
|
jobs = [r.split('|') for r in rows.splitlines()]
|
|
ready = [(float(c), float(f)) for s, c, f, _ in jobs if s == 'ready']
|
|
ttr = [f - c for c, f in ready]
|
|
span = (max(f for _, f in ready) - min(c for c, _ in ready)) if ready else 0
|
|
return {'jobs': len(jobs), 'ready': len(ready), 'failed': sum(1 for j in jobs if j[0] == 'failed'),
|
|
'retried_jobs': sum(1 for j in jobs if int(j[3]) > 1),
|
|
'throughput_jobs_per_s': round(len(ready) / span, 2) if span else None,
|
|
'pipeline_span_s': round(span, 2),
|
|
'time_to_ready_s': {'p50': pct(ttr, .5), 'p95': pct(ttr, .95), 'max': round(max(ttr), 3) if ttr else None}}
|
|
|
|
|
|
def set_worker(concurrency):
|
|
compose('up', '-d', '--no-deps', '--wait', 'worker', env={'WORKER_CONCURRENCY': str(concurrency)})
|
|
time.sleep(2)
|
|
|
|
|
|
def one_run(base_url, size, conc):
|
|
c = Client(base_url, timeout=60)
|
|
alice, _bob, _eve, chat = c.bootstrap()
|
|
data = SIZES[size]()
|
|
n = COUNTS[size]
|
|
lag = LagSampler(chat['id'])
|
|
with MemSampler('worker') as mem:
|
|
lag.start()
|
|
codes, upload_s = burst(c, alice, chat, data, n)
|
|
drained = wait_drained(chat['id'])
|
|
time.sleep(1)
|
|
lag.stop.set()
|
|
lag.join()
|
|
st = job_stats(chat['id'])
|
|
s = lag.samples
|
|
return {'size': size, 'image_bytes': len(data), 'worker_concurrency_prefetch': conc, 'uploads': n,
|
|
'accepted_202': sum(1 for x, _ in codes if x == 202),
|
|
'rejected': sum(1 for x, _ in codes if x != 202),
|
|
'upload_burst_s': round(upload_s, 2), 'drained': drained, **st,
|
|
'queue_lag': {'max_active_jobs': max((x['active'] for x in s), default=None),
|
|
'max_mq_ready': max((x['mq_ready'] or 0 for x in s), default=None),
|
|
'max_oldest_job_age_s': max((x['oldest_age_s'] for x in s), default=None)},
|
|
'worker_peak_rss_mib': round(mem.peak, 1),
|
|
'lag_timeline': s[::max(1, len(s) // 40)]}
|
|
|
|
|
|
def cmd_matrix(a):
|
|
runs = []
|
|
try:
|
|
for conc in CONCURRENCY:
|
|
set_worker(conc)
|
|
for size in SIZES:
|
|
r = one_run(a.base_url, size, conc)
|
|
print(json.dumps({k: r[k] for k in ('size', 'worker_concurrency_prefetch', 'throughput_jobs_per_s',
|
|
'time_to_ready_s', 'worker_peak_rss_mib', 'queue_lag')}), flush=True)
|
|
runs.append(r)
|
|
finally:
|
|
set_worker(2)
|
|
return {'runs': runs}
|
|
|
|
|
|
def cmd_backpressure(a):
|
|
bound = 20
|
|
compose('up', '-d', '--no-deps', '--wait', 'api', env={'MAX_ACTIVE_JOBS': str(bound)})
|
|
set_worker(1)
|
|
try:
|
|
c = Client(a.base_url, timeout=60)
|
|
alice, _bob, _eve, chat = c.bootstrap()
|
|
data = SIZES['M_2000x1500']()
|
|
lag = LagSampler(chat['id'])
|
|
lag.start()
|
|
t0 = time.monotonic()
|
|
codes, upload_s = burst(c, alice, chat, data, 80)
|
|
drained = wait_drained(chat['id'])
|
|
drain_s = time.monotonic() - t0
|
|
t1 = time.monotonic()
|
|
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after.png', 202, actor=alice,
|
|
raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
|
|
while True:
|
|
st = c.call('GET', '/api/v1/attachments/' + att['id'], actor=alice, schema='Attachment')['json']['status']
|
|
if st in ('ready', 'failed'):
|
|
break
|
|
time.sleep(0.2)
|
|
recovery_ready_s = time.monotonic() - t1
|
|
time.sleep(1)
|
|
lag.stop.set()
|
|
lag.join()
|
|
s = lag.samples
|
|
return {'max_active_jobs_bound': bound, 'worker_concurrency': 1, 'uploads': 80,
|
|
'accepted_202': sum(1 for x, _ in codes if x == 202),
|
|
'rejected_503': sum(1 for x, _ in codes if x == 503),
|
|
'retry_after_header': sorted({ra for x, ra in codes if x == 503}),
|
|
'other': [x for x, _ in codes if x not in (202, 503)],
|
|
'upload_burst_s': round(upload_s, 2), 'drained': drained, 'drain_s': round(drain_s, 1),
|
|
'max_active_observed': max((x['active'] for x in s), default=None),
|
|
'jobs': job_stats(chat['id']), 'after_overload_upload_status': st,
|
|
'after_overload_seconds_to_ready': round(recovery_ready_s, 2),
|
|
'lag_timeline': s[::max(1, len(s) // 40)]}
|
|
finally:
|
|
compose('up', '-d', '--no-deps', '--wait', 'api')
|
|
set_worker(2)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument('command', choices=['matrix', 'backpressure'])
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--output')
|
|
a = p.parse_args()
|
|
env = {'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
'worker_limits': 'cpus=2.0 memory=768M GOMEMLIMIT=600MiB decode budget 384MiB',
|
|
'host': 'Windows 11, Docker Desktop 29.7.2 (WSL2)'}
|
|
res = {'experiment': a.command, **env, **(cmd_matrix(a) if a.command == 'matrix' else cmd_backpressure(a))}
|
|
text = json.dumps(res, ensure_ascii=False, indent=1)
|
|
if a.output:
|
|
(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
|
|
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
|
|
print(text[:3000])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|