forked from mtusi2026/mtusi2026lab4
105 lines
4.4 KiB
Python
105 lines
4.4 KiB
Python
"""Measure sampled cgroup memory during ten simultaneous HTTP uploads."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
from io import BytesIO
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
from threading import Barrier
|
|
import time
|
|
|
|
from PIL import Image
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
COUNT = 10
|
|
SAMPLER = """
|
|
import select, sys, time
|
|
from pathlib import Path
|
|
usage = Path('/sys/fs/cgroup/memory.current')
|
|
limit = Path('/sys/fs/cgroup/memory.max')
|
|
cpu = Path('/sys/fs/cgroup/cpu.stat')
|
|
print(limit.read_text().strip(), flush=True)
|
|
while True:
|
|
stats = dict(line.split() for line in cpu.read_text().splitlines())
|
|
print(usage.read_text().strip(), stats['usage_usec'], flush=True)
|
|
if select.select([sys.stdin], [], [], .05)[0]:
|
|
break
|
|
"""
|
|
|
|
|
|
def main():
|
|
args = arguments(__doc__)
|
|
client = Client(args.base_url, args.ca_file, timeout=60)
|
|
alice, _, _, chat = client.bootstrap()
|
|
width = 1700
|
|
source = Image.frombytes('RGB', (width, width), random.Random(2026).randbytes(width * width * 3))
|
|
buffer = BytesIO()
|
|
source.save(buffer, format='PNG', compress_level=0)
|
|
payload = buffer.getvalue()
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
check(len(payload) <= 10 * 1024 * 1024, 'fixture exceeds upload size limit')
|
|
path = f"/api/v1/chats/{chat['id']}/attachments?filename=load.png"
|
|
barrier = Barrier(COUNT + 1)
|
|
samplers = {}
|
|
|
|
def upload():
|
|
local = Client(args.base_url, args.ca_file, timeout=60)
|
|
barrier.wait(timeout=30)
|
|
start = time.monotonic()
|
|
try:
|
|
reply = local.call('POST', path, 201, actor=alice, raw=payload,
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
check(reply['json']['sha256'] == digest, 'upload hash mismatch')
|
|
return {'instance': reply['headers']['X-Instance-Id'], 'id': reply['json']['id'],
|
|
'seconds': time.monotonic() - start}
|
|
except Failure as exc:
|
|
return {'error': str(exc), 'seconds': time.monotonic() - start}
|
|
|
|
try:
|
|
for service in ('api-a', 'api-b'):
|
|
proc = subprocess.Popen(
|
|
['docker', 'compose', 'exec', '-T', service, 'python', '-u', '-c', SAMPLER],
|
|
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
|
|
samplers[service] = proc
|
|
with ThreadPoolExecutor(max_workers=COUNT) as pool:
|
|
futures = [pool.submit(upload) for _ in range(COUNT)]
|
|
limits = {service: int(proc.stdout.readline()) for service, proc in samplers.items()}
|
|
initial = {service: tuple(map(int, proc.stdout.readline().split()))
|
|
for service, proc in samplers.items()}
|
|
start = time.monotonic()
|
|
barrier.wait(timeout=30)
|
|
requests = [future.result() for future in futures]
|
|
elapsed = time.monotonic() - start
|
|
finally:
|
|
for proc in samplers.values():
|
|
if proc.poll() is None:
|
|
proc.stdin.close()
|
|
resources = {}
|
|
for service, proc in samplers.items():
|
|
samples = [initial[service]] + [tuple(map(int, line.split())) for line in proc.stdout]
|
|
check(proc.wait(timeout=10) == 0 and len(samples) > 1, 'memory sampler failed')
|
|
resources[service] = {
|
|
'baseline_bytes': samples[0][0], 'peak_sample_bytes': max(row[0] for row in samples),
|
|
'limit_bytes': limits[service], 'samples': len(samples),
|
|
'cpu_seconds': (samples[-1][1] - samples[0][1]) / 1e6,
|
|
}
|
|
successes = [row for row in requests if 'id' in row]
|
|
durations = sorted(row['seconds'] for row in successes)
|
|
percentiles = {f'p{p}': durations[math.ceil(p / 100 * len(durations)) - 1]
|
|
for p in (50, 95, 99)} if durations else {}
|
|
print(json.dumps({'concurrency': COUNT, 'payload_bytes': len(payload), 'sha256': digest,
|
|
'elapsed_seconds': elapsed, 'successful_rps': len(successes) / elapsed,
|
|
'latency_seconds': percentiles, 'requests': requests,
|
|
'memory_sample_interval_seconds': .05, 'resources': resources}, indent=2))
|
|
check(len({row['id'] for row in successes}) == COUNT, 'not all uploads succeeded uniquely')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|