forked from mtusi2026/mtusi2026lab5
ЛР 5: асинхронная обработка изображений, уровень 5
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bounded track-1 experiment, stdlib only. Run on a dedicated Compose project.
|
||||
JOB_MAX_INFLIGHT=12 must be configured in API. Six batches, 16 uploads each.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
OUTPUT=ROOT/'evidence/overload-results.json'
|
||||
CONFIGS=((1,1),(2,2),(4,8))
|
||||
|
||||
def dc(*args, env=None):
|
||||
result=subprocess.run(['docker','compose',*args],env=env,capture_output=True,text=True,timeout=90)
|
||||
if result.returncode: raise RuntimeError('compose failed: '+' '.join(args[:2])+': '+result.stderr[-600:])
|
||||
return result.stdout.strip()
|
||||
|
||||
def sql(query):
|
||||
return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query)
|
||||
|
||||
def png(w,h):
|
||||
# Repeat deterministic textured rows: compact upload, real image decode/WebP work.
|
||||
rng=random.Random(42)
|
||||
rows=[rng.randbytes(w*3) for _ in range(32)]
|
||||
raw=b''.join(b'\x00'+rows[y%32] for y in range(h))
|
||||
def chunk(name,data):return struct.pack('!I',len(data))+name+data+struct.pack('!I',zlib.crc32(name+data)&0xffffffff)
|
||||
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',w,h,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
|
||||
|
||||
def percentile(values,p=.95):
|
||||
import math
|
||||
return sorted(values)[max(0,math.ceil(len(values)*p)-1)]
|
||||
|
||||
def queue():
|
||||
return [int(dc('exec','-T','broker','valkey-cli','LLEN',name)) for name in ('attachments:pending','attachments:active')]
|
||||
|
||||
def live_overload():
|
||||
data=json.loads(OUTPUT.read_text())
|
||||
c=Client(); actor,_,_,chat=c.bootstrap()
|
||||
content=png(2000,1400)
|
||||
samples=[]; stop=threading.Event()
|
||||
def monitor():
|
||||
while not stop.is_set():
|
||||
samples.append(int(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")))
|
||||
stop.wait(.1)
|
||||
def upload(_):
|
||||
return Client().call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=live.png',
|
||||
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
thread=threading.Thread(target=monitor);thread.start();start=time.monotonic()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
|
||||
replies=list(executor.map(upload,range(32)))
|
||||
ids=[r['json']['id'] for r in replies if r['status']==202]
|
||||
rejected=sum(r['status']==429 for r in replies)
|
||||
check(rejected>0,'live burst did not create backpressure')
|
||||
deadline=time.monotonic()+75
|
||||
while sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")!='0':
|
||||
check(time.monotonic()<deadline,'live burst did not drain');time.sleep(.2)
|
||||
for aid in ids:
|
||||
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
|
||||
check(meta['status']=='ready','accepted live task failed/lost')
|
||||
check(max(samples)<=12,'global cap exceeded with live worker')
|
||||
probe,_=c.attachment(actor,chat);c.wait_ready(actor,probe)
|
||||
data['live_overload']={'offered':32,'clients':8,'accepted':len(ids),'rejected_429':rejected,
|
||||
'max_unfinished':max(samples),'unfinished_samples':samples,'elapsed_s':round(time.monotonic()-start,3),
|
||||
'all_accepted_ready':True,'recovery_probe_ready':True,'concurrency':4,'prefetch':8}
|
||||
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
|
||||
print('PASS live overload: '+json.dumps({k:v for k,v in data['live_overload'].items() if k!='unfinished_samples'}),flush=True)
|
||||
finally:stop.set();thread.join()
|
||||
|
||||
|
||||
def run():
|
||||
c=Client(); actor,_,_,chat=c.bootstrap()
|
||||
data={'started_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'configs':CONFIGS,
|
||||
'cap':12,'offered_per_batch':16,'samples_interval_s':.25,'results':[],
|
||||
'host':subprocess.check_output(['uname','-sm'],text=True).strip(),
|
||||
'docker':json.loads(subprocess.check_output(['docker','info','--format','{{json .}}'],text=True))}
|
||||
# Keep only non-sensitive machine capacity metadata.
|
||||
data['docker']={k:data['docker'].get(k) for k in ('Architecture','NCPU','MemTotal','OperatingSystem','ServerVersion')}
|
||||
for concurrency,prefetch in CONFIGS:
|
||||
env=dict(os.environ,WORKER_CONCURRENCY=str(concurrency),WORKER_PREFETCH=str(prefetch))
|
||||
dc('up','-d','--no-deps','worker',env=env)
|
||||
# Warm one render per setting; its data is excluded from measurements.
|
||||
warm,_=c.attachment(actor,chat); c.wait_ready(actor,warm)
|
||||
for w,h in ((320,200),(2000,1400)):
|
||||
dc('stop','worker')
|
||||
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='0','stand is not isolated/drained')
|
||||
content=png(w,h)
|
||||
def upload(_):
|
||||
client=Client()
|
||||
return client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=study.png',
|
||||
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
t0=time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
|
||||
replies=list(executor.map(upload,range(16)))
|
||||
accepted=[r['json']['id'] for r in replies if r['status']==202]
|
||||
rejected=sum(r['status']==429 for r in replies)
|
||||
check(len(accepted)==12 and rejected==4,'global admission cap was not enforced')
|
||||
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='12','cap exceeded')
|
||||
ids=','.join("'"+value+"'" for value in accepted)
|
||||
# Give dispatcher <=2s to publish; admission remains durable even before publish.
|
||||
until=time.monotonic()+3
|
||||
while queue()[0]<12 and time.monotonic()<until:time.sleep(.1)
|
||||
samples=[]; stop=threading.Event()
|
||||
def monitor():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
rss=int(dc('exec','-T','worker','sh','-c',"awk '/^VmRSS:/ {s+=$2} END {print s+0}' /proc/[0-9]*/status"))
|
||||
pending,active=queue()
|
||||
samples.append({'t_s':round(time.monotonic()-release,3),'rss_kib':rss,'pending':pending,'active':active})
|
||||
except RuntimeError:pass
|
||||
stop.wait(.25)
|
||||
release=time.monotonic(); release_utc=dt.datetime.now(dt.timezone.utc)
|
||||
dc('start','worker')
|
||||
thread=threading.Thread(target=monitor);thread.start()
|
||||
try:
|
||||
deadline=time.monotonic()+75
|
||||
while True:
|
||||
rows=json.loads(sql("SELECT json_agg(row_to_json(t)) FROM (SELECT id,status,attempts,extract(epoch FROM (processing_started_at-created_at)) AS lag,extract(epoch FROM (updated_at-created_at)) AS latency,updated_at FROM attachments WHERE id IN ("+ids+")) t"))
|
||||
check(not any(r['status']=='failed' for r in rows),'failed accepted job')
|
||||
if all(r['status']=='ready' for r in rows):break
|
||||
check(time.monotonic()<deadline,'drain exceeded 75s budget')
|
||||
time.sleep(.15)
|
||||
# Verify durable effects for every accepted task through authenticated API.
|
||||
import hashlib
|
||||
for aid in accepted:
|
||||
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
|
||||
check({v['name'] for v in meta['variants']}=={'thumbnail','crop','optimized'},'variants missing')
|
||||
check(meta['sha256']==hashlib.sha256(content).hexdigest(),'original checksum mismatch')
|
||||
deadline=time.monotonic()+5
|
||||
while queue()!=[0,0] and time.monotonic()<deadline:time.sleep(.1)
|
||||
check(queue()==[0,0],'broker did not drain')
|
||||
finally:
|
||||
stop.set();thread.join()
|
||||
# New upload is accepted after overload and its real objects are downloadable.
|
||||
probe,probe_bytes=c.attachment(actor,chat)
|
||||
probe=c.wait_ready(actor,probe)
|
||||
path='/api/v1/attachments/'+probe['id']+'/content'
|
||||
check(c.call('GET',path,actor=actor)['bytes']==probe_bytes,'recovery original changed')
|
||||
for variant in probe['variants']:
|
||||
actual=c.call('GET',path+'?variant='+variant['name'],actor=actor)['bytes']
|
||||
check(hashlib.sha256(actual).hexdigest()==variant['sha256'],'recovery variant missing/corrupt')
|
||||
last=max(dt.datetime.fromisoformat(r['updated_at'].replace('Z','+00:00')) for r in rows)
|
||||
drain=(last-release_utc).total_seconds()
|
||||
result={'concurrency':concurrency,'prefetch':prefetch,'dimensions':[w,h],'bytes':len(content),
|
||||
'accepted':len(accepted),'rejected_429':rejected,'admission_s':round(release-t0,3),
|
||||
'drain_s':round(drain,3),'throughput_jobs_s':round(12/drain,3),
|
||||
'queue_lag_p95_s':round(percentile([float(r['lag']) for r in rows]),3),
|
||||
'ready_p95_s':round(percentile([float(r['latency']) for r in rows]),3),
|
||||
'peak_rss_mib':round(max(s['rss_kib'] for s in samples)/1024,2),
|
||||
'peak_active':max(s['active'] for s in samples),'final_queue':queue(),'recovery_probe_ready':True,
|
||||
'attempts': [r['attempts'] for r in rows], 'samples':samples}
|
||||
check(result['peak_active']<=prefetch,'prefetch exceeded')
|
||||
data['results'].append(result)
|
||||
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
|
||||
print(json.dumps({k:v for k,v in result.items() if k not in ('samples','attempts')}),flush=True)
|
||||
print('PASS: six batches drained; accepted work preserved; cap and prefetch enforced',flush=True)
|
||||
|
||||
if __name__=='__main__':
|
||||
if '--live-only' not in sys.argv:run()
|
||||
live_overload()
|
||||
Reference in New Issue
Block a user