ЛР 5: асинхронная обработка изображений, уровень 5

This commit is contained in:
Nurlygayanova Ekaterina
2026-11-12 20:32:55 +03:00
parent 276dbdad8a
commit 7b3eab5473
115 changed files with 8847 additions and 34 deletions
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Bounded track-1 experiment, stdlib only. Run on a dedicated Compose project.
JOB_MAX_INFLIGHT=12 must be configured in API. Six batches, 16 uploads each.
"""
import concurrent.futures
import datetime as dt
import json
import os
from pathlib import Path
import random
import struct
import subprocess
import sys
import threading
import time
import zlib
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
ROOT=Path(__file__).resolve().parents[1]
OUTPUT=ROOT/'evidence/overload-results.json'
CONFIGS=((1,1),(2,2),(4,8))
def dc(*args, env=None):
result=subprocess.run(['docker','compose',*args],env=env,capture_output=True,text=True,timeout=90)
if result.returncode: raise RuntimeError('compose failed: '+' '.join(args[:2])+': '+result.stderr[-600:])
return result.stdout.strip()
def sql(query):
return dc('exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-v','ON_ERROR_STOP=1','-c',query)
def png(w,h):
# Repeat deterministic textured rows: compact upload, real image decode/WebP work.
rng=random.Random(42)
rows=[rng.randbytes(w*3) for _ in range(32)]
raw=b''.join(b'\x00'+rows[y%32] for y in range(h))
def chunk(name,data):return struct.pack('!I',len(data))+name+data+struct.pack('!I',zlib.crc32(name+data)&0xffffffff)
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!2I5B',w,h,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(raw,1))+chunk(b'IEND',b'')
def percentile(values,p=.95):
import math
return sorted(values)[max(0,math.ceil(len(values)*p)-1)]
def queue():
return [int(dc('exec','-T','broker','valkey-cli','LLEN',name)) for name in ('attachments:pending','attachments:active')]
def live_overload():
data=json.loads(OUTPUT.read_text())
c=Client(); actor,_,_,chat=c.bootstrap()
content=png(2000,1400)
samples=[]; stop=threading.Event()
def monitor():
while not stop.is_set():
samples.append(int(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")))
stop.wait(.1)
def upload(_):
return Client().call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=live.png',
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
thread=threading.Thread(target=monitor);thread.start();start=time.monotonic()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
replies=list(executor.map(upload,range(32)))
ids=[r['json']['id'] for r in replies if r['status']==202]
rejected=sum(r['status']==429 for r in replies)
check(rejected>0,'live burst did not create backpressure')
deadline=time.monotonic()+75
while sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")!='0':
check(time.monotonic()<deadline,'live burst did not drain');time.sleep(.2)
for aid in ids:
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
check(meta['status']=='ready','accepted live task failed/lost')
check(max(samples)<=12,'global cap exceeded with live worker')
probe,_=c.attachment(actor,chat);c.wait_ready(actor,probe)
data['live_overload']={'offered':32,'clients':8,'accepted':len(ids),'rejected_429':rejected,
'max_unfinished':max(samples),'unfinished_samples':samples,'elapsed_s':round(time.monotonic()-start,3),
'all_accepted_ready':True,'recovery_probe_ready':True,'concurrency':4,'prefetch':8}
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
print('PASS live overload: '+json.dumps({k:v for k,v in data['live_overload'].items() if k!='unfinished_samples'}),flush=True)
finally:stop.set();thread.join()
def run():
c=Client(); actor,_,_,chat=c.bootstrap()
data={'started_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'configs':CONFIGS,
'cap':12,'offered_per_batch':16,'samples_interval_s':.25,'results':[],
'host':subprocess.check_output(['uname','-sm'],text=True).strip(),
'docker':json.loads(subprocess.check_output(['docker','info','--format','{{json .}}'],text=True))}
# Keep only non-sensitive machine capacity metadata.
data['docker']={k:data['docker'].get(k) for k in ('Architecture','NCPU','MemTotal','OperatingSystem','ServerVersion')}
for concurrency,prefetch in CONFIGS:
env=dict(os.environ,WORKER_CONCURRENCY=str(concurrency),WORKER_PREFETCH=str(prefetch))
dc('up','-d','--no-deps','worker',env=env)
# Warm one render per setting; its data is excluded from measurements.
warm,_=c.attachment(actor,chat); c.wait_ready(actor,warm)
for w,h in ((320,200),(2000,1400)):
dc('stop','worker')
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='0','stand is not isolated/drained')
content=png(w,h)
def upload(_):
client=Client()
return client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=study.png',
expected=[202,429],actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')
t0=time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
replies=list(executor.map(upload,range(16)))
accepted=[r['json']['id'] for r in replies if r['status']==202]
rejected=sum(r['status']==429 for r in replies)
check(len(accepted)==12 and rejected==4,'global admission cap was not enforced')
check(sql("SELECT count(*) FROM attachments WHERE status IN ('queued','processing')")=='12','cap exceeded')
ids=','.join("'"+value+"'" for value in accepted)
# Give dispatcher <=2s to publish; admission remains durable even before publish.
until=time.monotonic()+3
while queue()[0]<12 and time.monotonic()<until:time.sleep(.1)
samples=[]; stop=threading.Event()
def monitor():
while not stop.is_set():
try:
rss=int(dc('exec','-T','worker','sh','-c',"awk '/^VmRSS:/ {s+=$2} END {print s+0}' /proc/[0-9]*/status"))
pending,active=queue()
samples.append({'t_s':round(time.monotonic()-release,3),'rss_kib':rss,'pending':pending,'active':active})
except RuntimeError:pass
stop.wait(.25)
release=time.monotonic(); release_utc=dt.datetime.now(dt.timezone.utc)
dc('start','worker')
thread=threading.Thread(target=monitor);thread.start()
try:
deadline=time.monotonic()+75
while True:
rows=json.loads(sql("SELECT json_agg(row_to_json(t)) FROM (SELECT id,status,attempts,extract(epoch FROM (processing_started_at-created_at)) AS lag,extract(epoch FROM (updated_at-created_at)) AS latency,updated_at FROM attachments WHERE id IN ("+ids+")) t"))
check(not any(r['status']=='failed' for r in rows),'failed accepted job')
if all(r['status']=='ready' for r in rows):break
check(time.monotonic()<deadline,'drain exceeded 75s budget')
time.sleep(.15)
# Verify durable effects for every accepted task through authenticated API.
import hashlib
for aid in accepted:
meta=c.call('GET','/api/v1/attachments/'+aid,actor=actor,schema='Attachment')['json']
check({v['name'] for v in meta['variants']}=={'thumbnail','crop','optimized'},'variants missing')
check(meta['sha256']==hashlib.sha256(content).hexdigest(),'original checksum mismatch')
deadline=time.monotonic()+5
while queue()!=[0,0] and time.monotonic()<deadline:time.sleep(.1)
check(queue()==[0,0],'broker did not drain')
finally:
stop.set();thread.join()
# New upload is accepted after overload and its real objects are downloadable.
probe,probe_bytes=c.attachment(actor,chat)
probe=c.wait_ready(actor,probe)
path='/api/v1/attachments/'+probe['id']+'/content'
check(c.call('GET',path,actor=actor)['bytes']==probe_bytes,'recovery original changed')
for variant in probe['variants']:
actual=c.call('GET',path+'?variant='+variant['name'],actor=actor)['bytes']
check(hashlib.sha256(actual).hexdigest()==variant['sha256'],'recovery variant missing/corrupt')
last=max(dt.datetime.fromisoformat(r['updated_at'].replace('Z','+00:00')) for r in rows)
drain=(last-release_utc).total_seconds()
result={'concurrency':concurrency,'prefetch':prefetch,'dimensions':[w,h],'bytes':len(content),
'accepted':len(accepted),'rejected_429':rejected,'admission_s':round(release-t0,3),
'drain_s':round(drain,3),'throughput_jobs_s':round(12/drain,3),
'queue_lag_p95_s':round(percentile([float(r['lag']) for r in rows]),3),
'ready_p95_s':round(percentile([float(r['latency']) for r in rows]),3),
'peak_rss_mib':round(max(s['rss_kib'] for s in samples)/1024,2),
'peak_active':max(s['active'] for s in samples),'final_queue':queue(),'recovery_probe_ready':True,
'attempts': [r['attempts'] for r in rows], 'samples':samples}
check(result['peak_active']<=prefetch,'prefetch exceeded')
data['results'].append(result)
OUTPUT.write_text(json.dumps(data,indent=2)+'\n')
print(json.dumps({k:v for k,v in result.items() if k not in ('samples','attempts')}),flush=True)
print('PASS: six batches drained; accepted work preserved; cap and prefetch enforced',flush=True)
if __name__=='__main__':
if '--live-only' not in sys.argv:run()
live_overload()