69 lines
4.7 KiB
Python
Executable File
69 lines
4.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""10-way bounded-memory image upload experiment; output contains no credentials."""
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
import hashlib, json, math, os, pty, random, re, struct, subprocess, threading, time, zlib
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from client import Client, LAB, arguments, check, run
|
|
|
|
def png_noise(width, height, seed):
|
|
rng=random.Random(seed); raw=bytearray()
|
|
for _ in range(height): raw.append(0); raw.extend(rng.randbytes(width*3))
|
|
def chunk(tag,data): return struct.pack('>I',len(data))+tag+data+struct.pack('>I',zlib.crc32(tag+data)&0xffffffff)
|
|
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,2,0,0,0))+chunk(b'IDAT',zlib.compress(bytes(raw),1))+chunk(b'IEND',b'')
|
|
|
|
STATS_RE=re.compile(r'(mtusi_lab4_solution-api-[^|\s]+)\|([\d.]+)(GiB|MiB|KiB|B) / [^|\s]+\|([\d.]+)%')
|
|
def parse_stats_text(text):
|
|
vals={}
|
|
for name,number,unit,cpu in STATS_RE.findall(text):
|
|
scale={'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/(1024*1024)}[unit]
|
|
vals[name]={'memory_mib':float(number)*scale,'cpu_percent':float(cpu)}
|
|
return vals
|
|
|
|
def main():
|
|
a=arguments('Synthetic image upload experiment with sampled Docker API memory')
|
|
check(LAB==4,'experiment targets synchronous lab4 uploads')
|
|
c=Client(a.base_url,a.ca_file,timeout=30); alice,bob,eve,chat=c.bootstrap()
|
|
profiles=[('small',512,512),('large',2048,1536)]
|
|
result={'date_utc':datetime.now(timezone.utc).isoformat(),'concurrency':10,'profiles':[],'method':'10 workers start together; small profile repeats 10 times per worker and large 3 times; same generated PNG payload per profile; Docker stats streamed in a pseudo-terminal at Docker default cadence (~1s); no retries'}
|
|
for name,w,h in profiles:
|
|
repetitions=10 if name=='small' else 3
|
|
payload=png_noise(w,h,20261106+w)
|
|
barrier=threading.Barrier(10); samples=[]; stop=threading.Event()
|
|
master,slave=pty.openpty()
|
|
monitor_process=subprocess.Popen(['docker','stats','--format','{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}'],stdin=slave,stdout=slave,stderr=slave,close_fds=True)
|
|
os.close(slave)
|
|
def monitor():
|
|
buf=''
|
|
while not stop.is_set():
|
|
try: chunk=os.read(master,8192)
|
|
except OSError: break
|
|
if not chunk: break
|
|
buf=(buf+chunk.decode('utf-8','ignore'))[-16384:]
|
|
point=parse_stats_text(buf)
|
|
if point: samples.append(point)
|
|
mt=threading.Thread(target=monitor,daemon=True); mt.start()
|
|
def upload(i):
|
|
local=Client(a.base_url,a.ca_file,timeout=30); barrier.wait(); time.sleep(.2); out=[]
|
|
for j in range(repetitions):
|
|
t=time.monotonic()
|
|
try:
|
|
local.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+name+'-'+str(i)+'-'+str(j)+'.png',201,actor=alice,raw=payload,headers={'Content-Type':'image/png'},schema='Attachment')
|
|
out.append({'ok':True,'latency_ms':(time.monotonic()-t)*1000})
|
|
except Exception as e:
|
|
out.append({'ok':False,'error_type':type(e).__name__,'latency_ms':(time.monotonic()-t)*1000})
|
|
return out
|
|
started=time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=10) as pool: items=[item for group in pool.map(upload,range(10)) for item in group]
|
|
elapsed=time.monotonic()-started; stop.set(); monitor_process.terminate(); monitor_process.wait(timeout=3); mt.join(timeout=3); os.close(master)
|
|
lat=sorted(x['latency_ms'] for x in items if x['ok'])
|
|
peak={}
|
|
for snap in samples:
|
|
for service,m in snap.items(): peak[service]=max(peak.get(service,0),m['memory_mib'])
|
|
def pct(p): return round(lat[max(0,math.ceil(p*len(lat))-1)],2) if lat else None
|
|
result['profiles'].append({'name':name,'dimensions':[w,h],'pixels':w*h,'payload_bytes':len(payload),'sha256':hashlib.sha256(payload).hexdigest(),'uploads_per_worker':repetitions,'successes':sum(x['ok'] for x in items),'failures':sum(not x['ok'] for x in items),'elapsed_seconds':round(elapsed,3),'successful_rps':round(sum(x['ok'] for x in items)/elapsed,3),'latency_ms':{'p50':pct(.50),'p95':pct(.95),'p99':pct(.99)},'api_peak_memory_mib_sampled':peak,'sampling_interval_seconds':1,'sample_count':len(samples)})
|
|
dest=Path('evidence/upload-load.json'); dest.parent.mkdir(parents=True,exist_ok=True); dest.write_text(json.dumps(result,indent=2)+'\n')
|
|
print(json.dumps(result,indent=2))
|
|
check(all(x['failures']==0 and x['successes']==10*x['uploads_per_worker'] for x in result['profiles']),'one or more uploads failed')
|
|
if __name__=='__main__': run(main)
|