Files
mtusi2026lab4/student-tests/memory.py
T

90 lines
4.8 KiB
Python

"""Ten simultaneous uploads, two file sizes; sampled cgroup memory and CPU."""
import concurrent.futures
import json
import math
import random
import re
import statistics
import struct
import threading
import time
import zlib
from common import Client, check, command, compose, save, wait_ready
from attachments import chunk
def image(side):
rng=random.Random(104729+side)
pixels=b''.join(b'\0'+rng.randbytes(side*3) for _ in range(side))
return (b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('!IIBBBBB',side,side,8,2,0,0,0))+
chunk(b'IDAT',zlib.compress(pixels,1))+chunk(b'IEND',b''))
def bytes_value(value):
number,unit=re.fullmatch(r'([0-9.]+)\s*([A-Za-z]+)',value.strip()).groups()
return round(float(number)*{'B':1,'KiB':1024,'MiB':1024**2,'GiB':1024**3,'kB':1000,'MB':1000**2,'GB':1000**3}[unit])
def main():
c=Client(timeout=40);alice,bob,eve,chat=c.bootstrap()
ids=compose('ps','-q','api').split()
check(len(ids)>=2,'need two API containers')
cases=[]
for side in (256,1536):
compose('up','-d','--no-deps','--force-recreate','--scale','api=2','api');wait_ready(count=2)
ids=compose('ps','-q','api').split()
baseline_cgroup=[json.loads(command(['docker','exec',i,'/messenger','memory-snapshot'])) for i in ids]
payload=image(side);samples=[];sampling_errors=[];stop=threading.Event();timing={}
barrier=threading.Barrier(11,action=lambda:timing.update(start=time.monotonic()))
def sample():
while not stop.is_set():
try:
raw=command(['docker','stats','--no-stream','--format','{{json .}}',*ids],timeout=20)
except Exception as exc:
sampling_errors.append(type(exc).__name__);return
rows=[json.loads(line) for line in raw.splitlines()]
samples.append({'t':time.monotonic(),'total_bytes':sum(bytes_value(r['MemUsage'].split('/')[0]) for r in rows),
'containers':[{k:r[k] for k in ('Name','MemUsage','CPUPerc')} for r in rows]})
stop.wait(.1)
monitor=threading.Thread(target=sample,daemon=True);monitor.start()
while not samples:
if sampling_errors:raise RuntimeError('Docker memory sampler failed')
time.sleep(.05)
def upload(_):
client=Client(timeout=40);barrier.wait(timeout=90)
start=time.monotonic()
result=client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=memory.png',[201,503],
actor=alice,raw=payload,headers={'Content-Type':'image/png'})
return result['status'],time.monotonic()-start
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
pending=[pool.submit(upload,i) for i in range(10)]
barrier.wait(timeout=90);results=[f.result() for f in pending]
elapsed=time.monotonic()-timing['start']
# Keep sampling after the burst to catch the next Docker sample.
time.sleep(2)
finally:
stop.set();monitor.join(timeout=20)
successes=[latency for status,latency in results if status==201]
check(not sampling_errors,'Docker memory sampling failed')
peaks=[json.loads(command(['docker','exec',i,'/messenger','memory-snapshot'])) for i in ids]
check(len(successes)==10,'all ten simultaneous uploads must complete successfully')
ordered=sorted(successes)
percentile=lambda q:ordered[min(len(ordered)-1,max(0,math.ceil(len(ordered)*q)-1))]
cases.append({'file_bytes':len(payload),'pixels':side*side,'concurrency':10,'requests':10,
'success':len(successes),'backpressure_503':len(results)-len(successes),
'error_fraction':(len(results)-len(successes))/len(results),
'successful_rps':len(successes)/elapsed,'duration_seconds':elapsed,
'p50_seconds':percentile(.5),'p95_seconds':percentile(.95),'p99_seconds':percentile(.99),
'baseline_bytes':samples[0]['total_bytes'],'sampled_peak_bytes':max(x['total_bytes'] for x in samples),
'baseline_cgroups':baseline_cgroup,'peak_cgroups':peaks,
'sum_cgroup_peaks_bytes':sum(x['memory.peak'] for x in peaks),
'samples':samples})
save('upload-memory.json',{'complete':True,'method':'10 simultaneous clients, no retries; docker stats CPU/RAM plus kernel cgroup-v2 memory.peak; API cgroups recreated per case. Sum of individual peaks is an upper bound on simultaneous total peak.',
'cases':cases})
print('PASS 10-client memory experiment at two sizes (see upload-memory.json)')
if __name__=='__main__':
main()