#!/usr/bin/env python3 """Ten simultaneous proxy uploads, checksums, latency and cgroup memory samples.""" from concurrent.futures import ThreadPoolExecutor import hashlib import json import math from pathlib import Path import sys import threading import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT/'tests/own')) from helpers import Client, check, containers, docker, png, save, upload def memory(ids): result = {} for container in ids: raw = docker('exec',container,'cat','/sys/fs/cgroup/memory.current').stdout result[container] = int(raw) return result def percentile(values, p): return sorted(values)[max(0, math.ceil(len(values)*p)-1)] def kernel_peak(ids): return {container:int(docker('exec',container,'cat','/sys/fs/cgroup/memory.peak').stdout) for container in ids} def main(): c = Client(timeout=55) alice,bob,_,chat = c.bootstrap() upload(c,alice,chat,png()) ids = containers('api') results = {'concurrency':10, 'warmup_uploads':1, 'memory_method':'cgroup v2 memory.current sampled every >=100ms; values include file cache', 'runs':[]} for width,height in ((512,512),(1024,768)): content = png(width,height,noisy=True) sha = hashlib.sha256(content).hexdigest() baseline = memory(ids) baseline_kernel_peak = kernel_peak(ids) samples = [] barrier = threading.Barrier(11) def worker(number): local = Client(timeout=55) barrier.wait() return upload(local,alice,chat,content,filename='concurrent-'+str(number)+'.png') started = time.monotonic() with ThreadPoolExecutor(max_workers=10) as pool: futures = [pool.submit(worker,n) for n in range(10)] barrier.wait() while not all(f.done() for f in futures): samples.append({'seconds':round(time.monotonic()-started,3), 'bytes':memory(ids)}) time.sleep(.1) responses = [f.result() for f in futures] elapsed = time.monotonic()-started samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)}) for response in responses: att = response['json'] check(att['sha256']==sha,'concurrent upload checksum mismatch') downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob) check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed') latency = [r['elapsed']*1000 for r in responses] peak = {container:max(sample['bytes'][container] for sample in samples) for container in ids} run = {'width':width, 'height':height, 'size_bytes':len(content), 'sha256':sha, 'requests':10, 'successful':len(responses), 'errors':0, 'duration_seconds':round(elapsed,3), 'successful_rps':round(len(responses)/elapsed,3), 'latency_ms':{str(p):round(percentile(latency,p),3) for p in (.5,.95,.99)}, 'baseline_bytes':baseline, 'sampled_peak_bytes':peak, 'kernel_lifetime_peak_before_bytes':baseline_kernel_peak, 'kernel_lifetime_peak_after_bytes':kernel_peak(ids), 'sampled_peak_sum_bytes':max(sum(sample['bytes'].values()) for sample in samples), 'samples':samples} check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit') results['runs'].append(run) print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True) save('lab4-upload-memory.json',results) print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.') if __name__=='__main__': main()