forked from mtusi2026/mtusi2026lab5
172 lines
10 KiB
Python
172 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded track-1 experiment, stdlib only. Run on a dedicated Compose project.
|
|
JOB_MAX_INFLIGHT=12 must be configured in API. Six batches, 16 uploads each.
|
|
"""
|
|
import concurrent.futures
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import zlib
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, check
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
OUTPUT=ROOT/'evidence/overload-results.json'
|
|
CONFIGS=((1,1),(2,2),(4,8))
|
|
|
|
def dc(*args, env=None):
|
|
result=subprocess.run(['docker','compose',*args],env=env,capture_output=True,text=True,timeout=90)
|
|
if result.returncode: raise RuntimeError('compose failed: '+' '.join(args[:2])+': '+result.stderr[-600:])
|
|
return result.stdout.strip()
|
|
|
|
def sql(query):
|
|
return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query)
|
|
|
|
def png(w,h):
|
|
# Repeat deterministic textured rows: compact upload, real image decode/WebP work.
|
|
rng=random.Random(42)
|
|
rows=[rng.randbytes(w*3) for _ in range(32)]
|
|
raw=b''.join(b'\x00'+rows[y%32] for y in range(h))
|
|
def chunk(name,data):return struct.pack('!I',len(data))+name+data+struct.pack('!I',zlib.crc32(name+data)&0xffffffff)
|
|
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',w,h,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
|
|
|
|
def percentile(values,p=.95):
|
|
import math
|
|
return sorted(values)[max(0,math.ceil(len(values)*p)-1)]
|
|
|
|
def queue():
|
|
return [int(dc('exec','-T','broker','valkey-cli','LLEN',name)) for name in ('attachments:pending','attachments:active')]
|
|
|
|
def live_overload():
|
|
data=json.loads(OUTPUT.read_text())
|
|
c=Client(); actor,_,_,chat=c.bootstrap()
|
|
content=png(2000,1400)
|
|
samples=[]; stop=threading.Event()
|
|
def monitor():
|
|
while not stop.is_set():
|
|
samples.append(int(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")))
|
|
stop.wait(.1)
|
|
def upload(_):
|
|
return Client().call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=live.png',
|
|
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
thread=threading.Thread(target=monitor);thread.start();start=time.monotonic()
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
|
replies=list(executor.map(upload,range(32)))
|
|
ids=[r['json']['id'] for r in replies if r['status']==202]
|
|
rejected=sum(r['status']==429 for r in replies)
|
|
check(rejected>0,'live burst did not create backpressure')
|
|
deadline=time.monotonic()+75
|
|
while sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")!='0':
|
|
check(time.monotonic()<deadline,'live burst did not drain');time.sleep(.2)
|
|
for aid in ids:
|
|
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
|
|
check(meta['status']=='ready','accepted live task failed/lost')
|
|
check(max(samples)<=12,'global cap exceeded with live worker')
|
|
probe,_=c.attachment(actor,chat);c.wait_ready(actor,probe)
|
|
data['live_overload']={'offered':32,'clients':8,'accepted':len(ids),'rejected_429':rejected,
|
|
'max_unfinished':max(samples),'unfinished_samples':samples,'elapsed_s':round(time.monotonic()-start,3),
|
|
'all_accepted_ready':True,'recovery_probe_ready':True,'concurrency':4,'prefetch':8}
|
|
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
|
|
print('PASS live overload: '+json.dumps({k:v for k,v in data['live_overload'].items() if k!='unfinished_samples'}),flush=True)
|
|
finally:stop.set();thread.join()
|
|
|
|
|
|
def run():
|
|
c=Client(); actor,_,_,chat=c.bootstrap()
|
|
data={'started_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'configs':CONFIGS,
|
|
'cap':12,'offered_per_batch':16,'samples_interval_s':.25,'results':[],
|
|
'host':subprocess.check_output(['uname','-sm'],text=True).strip(),
|
|
'docker':json.loads(subprocess.check_output(['docker','info','--format','{{json .}}'],text=True))}
|
|
# Keep only non-sensitive machine capacity metadata.
|
|
data['docker']={k:data['docker'].get(k) for k in ('Architecture','NCPU','MemTotal','OperatingSystem','ServerVersion')}
|
|
for concurrency,prefetch in CONFIGS:
|
|
env=dict(os.environ,WORKER_CONCURRENCY=str(concurrency),WORKER_PREFETCH=str(prefetch))
|
|
dc('up','-d','--no-deps','worker',env=env)
|
|
# Warm one render per setting; its data is excluded from measurements.
|
|
warm,_=c.attachment(actor,chat); c.wait_ready(actor,warm)
|
|
for w,h in ((320,200),(2000,1400)):
|
|
dc('stop','worker')
|
|
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='0','stand is not isolated/drained')
|
|
content=png(w,h)
|
|
def upload(_):
|
|
client=Client()
|
|
return client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=study.png',
|
|
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
t0=time.monotonic()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
|
replies=list(executor.map(upload,range(16)))
|
|
accepted=[r['json']['id'] for r in replies if r['status']==202]
|
|
rejected=sum(r['status']==429 for r in replies)
|
|
check(len(accepted)==12 and rejected==4,'global admission cap was not enforced')
|
|
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='12','cap exceeded')
|
|
ids=','.join("'"+value+"'" for value in accepted)
|
|
# Give dispatcher <=2s to publish; admission remains durable even before publish.
|
|
until=time.monotonic()+3
|
|
while queue()[0]<12 and time.monotonic()<until:time.sleep(.1)
|
|
samples=[]; stop=threading.Event()
|
|
def monitor():
|
|
while not stop.is_set():
|
|
try:
|
|
rss=int(dc('exec','-T','worker','sh','-c',"awk '/^VmRSS:/ {s+=$2} END {print s+0}' /proc/[0-9]*/status"))
|
|
pending,active=queue()
|
|
samples.append({'t_s':round(time.monotonic()-release,3),'rss_kib':rss,'pending':pending,'active':active})
|
|
except RuntimeError:pass
|
|
stop.wait(.25)
|
|
release=time.monotonic(); release_utc=dt.datetime.now(dt.timezone.utc)
|
|
dc('start','worker')
|
|
thread=threading.Thread(target=monitor);thread.start()
|
|
try:
|
|
deadline=time.monotonic()+75
|
|
while True:
|
|
rows=json.loads(sql("SELECT json_agg(row_to_json(t)) FROM (SELECT id,status,attempts,extract(epoch FROM (processing_started_at-created_at)) AS lag,extract(epoch FROM (updated_at-created_at)) AS latency,updated_at FROM attachments WHERE id IN ("+ids+")) t"))
|
|
check(not any(r['status']=='failed' for r in rows),'failed accepted job')
|
|
if all(r['status']=='ready' for r in rows):break
|
|
check(time.monotonic()<deadline,'drain exceeded 75s budget')
|
|
time.sleep(.15)
|
|
# Verify durable effects for every accepted task through authenticated API.
|
|
import hashlib
|
|
for aid in accepted:
|
|
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
|
|
check({v['name'] for v in meta['variants']}=={'thumbnail','crop','optimized'},'variants missing')
|
|
check(meta['sha256']==hashlib.sha256(content).hexdigest(),'original checksum mismatch')
|
|
deadline=time.monotonic()+5
|
|
while queue()!=[0,0] and time.monotonic()<deadline:time.sleep(.1)
|
|
check(queue()==[0,0],'broker did not drain')
|
|
finally:
|
|
stop.set();thread.join()
|
|
# New upload is accepted after overload and its real objects are downloadable.
|
|
probe,probe_bytes=c.attachment(actor,chat)
|
|
probe=c.wait_ready(actor,probe)
|
|
path='/api/v1/attachments/'+probe['id']+'/content'
|
|
check(c.call('GET',path,actor=actor)['bytes']==probe_bytes,'recovery original changed')
|
|
for variant in probe['variants']:
|
|
actual=c.call('GET',path+'?variant='+variant['name'],actor=actor)['bytes']
|
|
check(hashlib.sha256(actual).hexdigest()==variant['sha256'],'recovery variant missing/corrupt')
|
|
last=max(dt.datetime.fromisoformat(r['updated_at'].replace('Z','+00:00')) for r in rows)
|
|
drain=(last-release_utc).total_seconds()
|
|
result={'concurrency':concurrency,'prefetch':prefetch,'dimensions':[w,h],'bytes':len(content),
|
|
'accepted':len(accepted),'rejected_429':rejected,'admission_s':round(release-t0,3),
|
|
'drain_s':round(drain,3),'throughput_jobs_s':round(12/drain,3),
|
|
'queue_lag_p95_s':round(percentile([float(r['lag']) for r in rows]),3),
|
|
'ready_p95_s':round(percentile([float(r['latency']) for r in rows]),3),
|
|
'peak_rss_mib':round(max(s['rss_kib'] for s in samples)/1024,2),
|
|
'peak_active':max(s['active'] for s in samples),'final_queue':queue(),'recovery_probe_ready':True,
|
|
'attempts': [r['attempts'] for r in rows], 'samples':samples}
|
|
check(result['peak_active']<=prefetch,'prefetch exceeded')
|
|
data['results'].append(result)
|
|
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
|
|
print(json.dumps({k:v for k,v in result.items() if k not in ('samples','attempts')}),flush=True)
|
|
print('PASS: six batches drained; accepted work preserved; cap and prefetch enforced',flush=True)
|
|
|
|
if __name__=='__main__':
|
|
if '--live-only' not in sys.argv:run()
|
|
live_overload()
|