#!/usr/bin/env python3 """Concurrent upload experiment with API memory measurement; creates synthetic data.""" import json import math import shutil import subprocess import sys import threading import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) from client import Client, Failure, arguments, check, run ROOT = Path(__file__).resolve().parents[1] def engine(): return 'podman' if shutil.which('podman') else 'docker' def api_containers(project): result = subprocess.run( [engine(), 'ps', '--filter', 'label=com.docker.compose.project=' + project, '--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}}'], capture_output=True, text=True, check=False) check(result.returncode == 0, 'cannot list api containers: ' + result.stderr.strip()) return sorted(line.strip() for line in result.stdout.splitlines() if line.strip()) def peak_bytes(container): for path in ('/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes'): result = subprocess.run([engine(), 'exec', container, 'cat', path], capture_output=True, text=True, check=False) if result.returncode == 0 and result.stdout.strip().isdigit(): return int(result.stdout.strip()) return None def main(): def configure(p): p.add_argument('--concurrency', type=int, default=10) p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/large.png')) p.add_argument('--project', default='messenger-lab4') p.add_argument('--output', default=None) args = arguments('Concurrent attachment uploads with peak memory measurement', configure) check(1 <= args.concurrency <= 64, 'concurrency must be 1..64') source = Path(args.file) check(source.is_file(), 'upload fixture not found: ' + str(source)) content = source.read_bytes() containers = api_containers(args.project) check(bool(containers), 'no api containers found for project ' + args.project) baseline = {name: peak_bytes(name) for name in containers} c = Client() alice, _, _, chat = c.bootstrap() actor = {'id': alice['id'], 'token': alice['token']} path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + source.name barrier = threading.Barrier(args.concurrency) lock = threading.Lock() latencies, failures = [], [] def worker(index): local = Client(timeout=120) barrier.wait() started = time.monotonic() try: response = local.call('POST', path, 201, actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') with lock: latencies.append(time.monotonic() - started) check(response['json']['size_bytes'] == len(content), 'uploaded size mismatch') except Failure as exc: with lock: failures.append(str(exc)) started = time.monotonic() threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)] for thread in threads: thread.start() for thread in threads: thread.join() elapsed = time.monotonic() - started peaks = {name: peak_bytes(name) for name in containers} peak_delta = {} for name in containers: before = baseline[name] after = peaks[name] if before is None or after is None: peak_delta[name] = None else: peak_delta[name] = after - before ordered = sorted(latencies) def percentile(p): return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1) if ordered else None report = { 'concurrency': args.concurrency, 'uploads': args.concurrency, 'fixture': source.name, 'fixture_bytes': len(content), 'elapsed_seconds': round(elapsed, 3), 'successful': len(latencies), 'failed': len(failures), 'failures': failures[:3], 'latency_ms': {'p50': percentile(0.50), 'p95': percentile(0.95), 'p99': percentile(0.99)}, 'api_containers': containers, 'peak_bytes_before': baseline, 'peak_bytes_after': peaks, 'peak_delta_bytes': peak_delta, 'peak_max_mb': round(max(v for v in peaks.values() if v is not None) / (1024 * 1024), 2) if any(v is not None for v in peaks.values()) else None, } text = json.dumps(report, indent=2) if args.output: destination = Path(args.output) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(text + '\n') print(text) check(not failures, 'some uploads failed') check(len(latencies) == args.concurrency, 'not all uploads succeeded') if __name__ == '__main__': run(main)