Files
mtusi2026lab4/tests/upload_load.py
T
2026-11-06 17:03:03 +03:00

69 lines
4.7 KiB
Python
Executable File

#!/usr/bin/env python3
"""10-way bounded-memory image upload experiment; output contains no credentials."""
from concurrent.futures import ThreadPoolExecutor, as_completed
import hashlib, json, math, os, pty, random, re, struct, subprocess, threading, time, zlib
from datetime import datetime, timezone
from pathlib import Path
from client import Client, LAB, arguments, check, run
def png_noise(width, height, seed):
rng=random.Random(seed); raw=bytearray()
for _ in range(height): raw.append(0); raw.extend(rng.randbytes(width*3))
def chunk(tag,data): return struct.pack('>I',len(data))+tag+data+struct.pack('>I',zlib.crc32(tag+data)&0xffffffff)
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(bytes(raw),1))+chunk(b'IEND',b'')
STATS_RE=re.compile(r'(mtusi_lab4_solution-api-[^|\s]+)\|([\d.]+)(GiB|MiB|KiB|B) / [^|\s]+\|([\d.]+)%')
def parse_stats_text(text):
vals={}
for name,number,unit,cpu in STATS_RE.findall(text):
scale={'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/(1024*1024)}[unit]
vals[name]={'memory_mib':float(number)*scale,'cpu_percent':float(cpu)}
return vals
def main():
a=arguments('Synthetic image upload experiment with sampled Docker API memory')
check(LAB==4,'experiment targets synchronous lab4 uploads')
c=Client(a.base_url,a.ca_file,timeout=30); alice,bob,eve,chat=c.bootstrap()
profiles=[('small',512,512),('large',2048,1536)]
result={'date_utc':datetime.now(timezone.utc).isoformat(),'concurrency':10,'profiles':[],'method':'10 workers start together; small profile repeats 10 times per worker and large 3 times; same generated PNG payload per profile; Docker stats streamed in a pseudo-terminal at Docker default cadence (~1s); no retries'}
for name,w,h in profiles:
repetitions=10 if name=='small' else 3
payload=png_noise(w,h,20261106+w)
barrier=threading.Barrier(10); samples=[]; stop=threading.Event()
master,slave=pty.openpty()
monitor_process=subprocess.Popen(['docker','stats','--format','{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}'],stdin=slave,stdout=slave,stderr=slave,close_fds=True)
os.close(slave)
def monitor():
buf=''
while not stop.is_set():
try: chunk=os.read(master,8192)
except OSError: break
if not chunk: break
buf=(buf+chunk.decode('utf-8','ignore'))[-16384:]
point=parse_stats_text(buf)
if point: samples.append(point)
mt=threading.Thread(target=monitor,daemon=True); mt.start()
def upload(i):
local=Client(a.base_url,a.ca_file,timeout=30); barrier.wait(); time.sleep(.2); out=[]
for j in range(repetitions):
t=time.monotonic()
try:
local.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name+'-'+str(i)+'-'+str(j)+'.png',201,actor=alice,raw=payload,headers={'Content-Type':'image/png'},schema='Attachment')
out.append({'ok':True,'latency_ms':(time.monotonic()-t)*1000})
except Exception as e:
out.append({'ok':False,'error_type':type(e).__name__,'latency_ms':(time.monotonic()-t)*1000})
return out
started=time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool: items=[item for group in pool.map(upload,range(10)) for item in group]
elapsed=time.monotonic()-started; stop.set(); monitor_process.terminate(); monitor_process.wait(timeout=3); mt.join(timeout=3); os.close(master)
lat=sorted(x['latency_ms'] for x in items if x['ok'])
peak={}
for snap in samples:
for service,m in snap.items(): peak[service]=max(peak.get(service,0),m['memory_mib'])
def pct(p): return round(lat[max(0,math.ceil(p*len(lat))-1)],2) if lat else None
result['profiles'].append({'name':name,'dimensions':[w,h],'pixels':w*h,'payload_bytes':len(payload),'sha256':hashlib.sha256(payload).hexdigest(),'uploads_per_worker':repetitions,'successes':sum(x['ok'] for x in items),'failures':sum(not x['ok'] for x in items),'elapsed_seconds':round(elapsed,3),'successful_rps':round(sum(x['ok'] for x in items)/elapsed,3),'latency_ms':{'p50':pct(.50),'p95':pct(.95),'p99':pct(.99)},'api_peak_memory_mib_sampled':peak,'sampling_interval_seconds':1,'sample_count':len(samples)})
dest=Path('evidence/upload-load.json'); dest.parent.mkdir(parents=True,exist_ok=True); dest.write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result,indent=2))
check(all(x['failures']==0 and x['successes']==10*x['uploads_per_worker'] for x in result['profiles']),'one or more uploads failed')
if __name__=='__main__': run(main)