Files
mtusi2026lab4/scripts/upload-load.py
T

130 lines
4.7 KiB
Python
Executable File

#!/usr/bin/env python3
"""Concurrent upload experiment with API memory measurement; creates synthetic data."""
import json
import math
import shutil
import subprocess
import sys
import threading
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
def engine():
return 'podman' if shutil.which('podman') else 'docker'
def api_containers(project):
result = subprocess.run(
[engine(), 'ps',
'--filter', 'label=com.docker.compose.project=' + project,
'--filter', 'label=com.docker.compose.service=api',
'--format', '{{.Names}}'],
capture_output=True, text=True, check=False)
check(result.returncode == 0, 'cannot list api containers: ' + result.stderr.strip())
return sorted(line.strip() for line in result.stdout.splitlines() if line.strip())
def peak_bytes(container):
for path in ('/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes'):
result = subprocess.run([engine(), 'exec', container, 'cat', path], capture_output=True, text=True, check=False)
if result.returncode == 0 and result.stdout.strip().isdigit():
return int(result.stdout.strip())
return None
def main():
def configure(p):
p.add_argument('--concurrency', type=int, default=10)
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/large.png'))
p.add_argument('--project', default='messenger-lab4')
p.add_argument('--output', default=None)
args = arguments('Concurrent attachment uploads with peak memory measurement', configure)
check(1 <= args.concurrency <= 64, 'concurrency must be 1..64')
source = Path(args.file)
check(source.is_file(), 'upload fixture not found: ' + str(source))
content = source.read_bytes()
containers = api_containers(args.project)
check(bool(containers), 'no api containers found for project ' + args.project)
baseline = {name: peak_bytes(name) for name in containers}
c = Client()
alice, _, _, chat = c.bootstrap()
actor = {'id': alice['id'], 'token': alice['token']}
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + source.name
barrier = threading.Barrier(args.concurrency)
lock = threading.Lock()
latencies, failures = [], []
def worker(index):
local = Client(timeout=120)
barrier.wait()
started = time.monotonic()
try:
response = local.call('POST', path, 201, actor=actor, raw=content,
headers={'Content-Type': 'image/png'}, schema='Attachment')
with lock:
latencies.append(time.monotonic() - started)
check(response['json']['size_bytes'] == len(content), 'uploaded size mismatch')
except Failure as exc:
with lock:
failures.append(str(exc))
started = time.monotonic()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
elapsed = time.monotonic() - started
peaks = {name: peak_bytes(name) for name in containers}
peak_delta = {}
for name in containers:
before = baseline[name]
after = peaks[name]
if before is None or after is None:
peak_delta[name] = None
else:
peak_delta[name] = after - before
ordered = sorted(latencies)
def percentile(p):
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1) if ordered else None
report = {
'concurrency': args.concurrency,
'uploads': args.concurrency,
'fixture': source.name,
'fixture_bytes': len(content),
'elapsed_seconds': round(elapsed, 3),
'successful': len(latencies),
'failed': len(failures),
'failures': failures[:3],
'latency_ms': {'p50': percentile(0.50), 'p95': percentile(0.95), 'p99': percentile(0.99)},
'api_containers': containers,
'peak_bytes_before': baseline,
'peak_bytes_after': peaks,
'peak_delta_bytes': peak_delta,
'peak_max_mb': round(max(v for v in peaks.values() if v is not None) / (1024 * 1024), 2) if any(v is not None for v in peaks.values()) else None,
}
text = json.dumps(report, indent=2)
if args.output:
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(text + '\n')
print(text)
check(not failures, 'some uploads failed')
check(len(latencies) == args.concurrency, 'not all uploads succeeded')
if __name__ == '__main__':
run(main)