forked from mtusi2026/mtusi2026lab4
90 lines
4.8 KiB
Python
90 lines
4.8 KiB
Python
"""Ten simultaneous uploads, two file sizes; sampled cgroup memory and CPU."""
|
|
import concurrent.futures
|
|
import json
|
|
import math
|
|
import random
|
|
import re
|
|
import statistics
|
|
import struct
|
|
import threading
|
|
import time
|
|
import zlib
|
|
from common import Client, check, command, compose, save, wait_ready
|
|
from attachments import chunk
|
|
|
|
|
|
def image(side):
|
|
rng=random.Random(104729+side)
|
|
pixels=b''.join(b'\0'+rng.randbytes(side*3) for _ in range(side))
|
|
return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!IIBBBBB',side,side,8,2,0,0,0))+
|
|
chunk(b'IDAT',zlib.compress(pixels,1))+chunk(b'IEND',b''))
|
|
|
|
|
|
def bytes_value(value):
|
|
number,unit=re.fullmatch(r'([0-9.]+)\s*([A-Za-z]+)',value.strip()).groups()
|
|
return round(float(number)*{'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3,'kB':1000,'MB':1000**2,'GB':1000**3}[unit])
|
|
|
|
|
|
def main():
|
|
c=Client(timeout=40);alice,bob,eve,chat=c.bootstrap()
|
|
ids=compose('ps','-q','api').split()
|
|
check(len(ids)>=2,'need two API containers')
|
|
cases=[]
|
|
for side in (256,1536):
|
|
compose('up','-d','--no-deps','--force-recreate','--scale','api=2','api');wait_ready(count=2)
|
|
ids=compose('ps','-q','api').split()
|
|
baseline_cgroup=[json.loads(command(['docker','exec',i,'/messenger','memory-snapshot'])) for i in ids]
|
|
payload=image(side);samples=[];sampling_errors=[];stop=threading.Event();timing={}
|
|
barrier=threading.Barrier(11,action=lambda:timing.update(start=time.monotonic()))
|
|
def sample():
|
|
while not stop.is_set():
|
|
try:
|
|
raw=command(['docker','stats','--no-stream','--format','{{json .}}',*ids],timeout=20)
|
|
except Exception as exc:
|
|
sampling_errors.append(type(exc).__name__);return
|
|
rows=[json.loads(line) for line in raw.splitlines()]
|
|
samples.append({'t':time.monotonic(),'total_bytes':sum(bytes_value(r['MemUsage'].split('/')[0]) for r in rows),
|
|
'containers':[{k:r[k] for k in ('Name','MemUsage','CPUPerc')} for r in rows]})
|
|
stop.wait(.1)
|
|
monitor=threading.Thread(target=sample,daemon=True);monitor.start()
|
|
while not samples:
|
|
if sampling_errors:raise RuntimeError('Docker memory sampler failed')
|
|
time.sleep(.05)
|
|
def upload(_):
|
|
client=Client(timeout=40);barrier.wait(timeout=90)
|
|
start=time.monotonic()
|
|
result=client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=memory.png',[201,503],
|
|
actor=alice,raw=payload,headers={'Content-Type':'image/png'})
|
|
return result['status'],time.monotonic()-start
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
|
|
pending=[pool.submit(upload,i) for i in range(10)]
|
|
barrier.wait(timeout=90);results=[f.result() for f in pending]
|
|
elapsed=time.monotonic()-timing['start']
|
|
# Keep sampling after the burst to catch the next Docker sample.
|
|
time.sleep(2)
|
|
finally:
|
|
stop.set();monitor.join(timeout=20)
|
|
successes=[latency for status,latency in results if status==201]
|
|
check(not sampling_errors,'Docker memory sampling failed')
|
|
peaks=[json.loads(command(['docker','exec',i,'/messenger','memory-snapshot'])) for i in ids]
|
|
check(len(successes)==10,'all ten simultaneous uploads must complete successfully')
|
|
ordered=sorted(successes)
|
|
percentile=lambda q:ordered[min(len(ordered)-1,max(0,math.ceil(len(ordered)*q)-1))]
|
|
cases.append({'file_bytes':len(payload),'pixels':side*side,'concurrency':10,'requests':10,
|
|
'success':len(successes),'backpressure_503':len(results)-len(successes),
|
|
'error_fraction':(len(results)-len(successes))/len(results),
|
|
'successful_rps':len(successes)/elapsed,'duration_seconds':elapsed,
|
|
'p50_seconds':percentile(.5),'p95_seconds':percentile(.95),'p99_seconds':percentile(.99),
|
|
'baseline_bytes':samples[0]['total_bytes'],'sampled_peak_bytes':max(x['total_bytes'] for x in samples),
|
|
'baseline_cgroups':baseline_cgroup,'peak_cgroups':peaks,
|
|
'sum_cgroup_peaks_bytes':sum(x['memory.peak'] for x in peaks),
|
|
'samples':samples})
|
|
save('upload-memory.json',{'complete':True,'method':'10 simultaneous clients, no retries; docker stats CPU/RAM plus kernel cgroup-v2 memory.peak; API cgroups recreated per case. Sum of individual peaks is an upper bound on simultaneous total peak.',
|
|
'cases':cases})
|
|
print('PASS 10-client memory experiment at two sizes (see upload-memory.json)')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|