#!/usr/bin/env python3 """Concurrent attachment uploads with peak RSS of API containers. Generates a valid PNG at runtime (not stored in Git). Measures docker stats while N workers POST the same image. Writes a sanitized JSON summary. """ from __future__ import annotations import argparse import hashlib import json import os import platform import subprocess import sys import threading import time import uuid import zlib from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure # noqa: E402 def make_png(width: int, height: int) -> bytes: rows = [] for _y in range(height): row = bytearray(1 + width * 3) row[0] = 0 row[1:] = os.urandom(width * 3) rows.append(bytes(row)) raw = b''.join(rows) compressed = zlib.compress(raw, 1) def chunk(tag: bytes, data: bytes) -> bytes: crc = zlib.crc32(tag + data) & 0xffffffff return len(data).to_bytes(4, 'big') + tag + data + crc.to_bytes(4, 'big') ihdr = width.to_bytes(4, 'big') + height.to_bytes(4, 'big') + bytes([8, 2, 0, 0, 0]) return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', compressed) + chunk(b'IEND', b'') def parse_mib(usage: str) -> float: # "45.23MiB / 256MiB" or "123.4MB / 256MB" left = usage.split('/', 1)[0].strip() number = '' unit = '' for char in left: if char.isdigit() or char in '.': number += char else: unit += char value = float(number or '0') unit = unit.strip().lower() if unit in ('gib', 'gb'): return value * 1024 if unit in ('kib', 'kb'): return value / 1024 if unit in ('b',): return value / (1024 * 1024) return value def git_commit() -> str: try: return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=str(ROOT), text=True).strip() except (OSError, subprocess.CalledProcessError): return 'unknown' def docker_stats() -> list: try: ids = subprocess.check_output( ['docker', 'ps', '-q', '--filter', 'name=messenger-lab4-api'], cwd=str(ROOT), text=True, timeout=5, ).split() if not ids: return [] output = subprocess.check_output( ['docker', 'stats', '--no-stream', '--format', '{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}', *ids], cwd=str(ROOT), text=True, timeout=8, ) except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): return [] rows = [] for line in output.splitlines(): parts = line.split('\t') if len(parts) < 2: continue name, usage = parts[0], parts[1] rows.append({'name': name, 'mem_usage': usage, 'rss_mib': round(parse_mib(usage), 2)}) return rows def main() -> None: parser = argparse.ArgumentParser(description='Concurrent upload memory experiment') parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) parser.add_argument('--concurrency', type=int, default=10) parser.add_argument('--width', type=int, default=900) parser.add_argument('--height', type=int, default=700) parser.add_argument('--output', default=str(ROOT / 'evidence' / 'upload-concurrency.json')) args = parser.parse_args() if args.concurrency < 10: raise SystemExit('concurrency must be >= 10') image = make_png(args.width, args.height) size = len(image) digest = hashlib.sha256(image).hexdigest() client = Client(args.base_url, timeout=60) alice, _bob, _eve, chat = client.bootstrap() path = '/api/v1/chats/%s/attachments?filename=load.png' % chat['id'] stop = threading.Event() samples = [] def watch(): while not stop.is_set(): snapshot = docker_stats() if snapshot: samples.append({'ts': time.time(), 'containers': snapshot}) stop.wait(0.05) watcher = threading.Thread(target=watch, daemon=True) watcher.start() baseline = docker_stats() if baseline: samples.append({'ts': time.time(), 'containers': baseline, 'phase': 'baseline'}) started = time.monotonic() latencies = [] errors = 0 def one(_index: int): response = client.call( 'POST', path, 201, actor=alice, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment', ) body = response['json'] if body.get('sha256') != digest or body.get('size_bytes') != size: raise Failure('checksum mismatch on concurrent upload') return response['elapsed'] try: with ThreadPoolExecutor(max_workers=args.concurrency) as pool: futs = [pool.submit(one, i) for i in range(args.concurrency)] for fut in as_completed(futs): try: latencies.append(fut.result()) except Exception: errors += 1 finally: stop.set() watcher.join(timeout=2) elapsed = time.monotonic() - started latencies.sort() def pct(p): if not latencies: return None index = min(len(latencies) - 1, max(0, int(round((p / 100) * (len(latencies) - 1))))) return round(latencies[index], 4) peak = 0.0 per_container = {} for sample in samples: for row in sample['containers']: peak = max(peak, row['rss_mib']) prev = per_container.get(row['name'], 0.0) per_container[row['name']] = max(prev, row['rss_mib']) naive_bound_mib = round((args.concurrency * size) / (1024 * 1024), 2) summary = { 'hypothesis': ( 'Ten concurrent ~%.1f MiB PNG uploads stay inside the 256 MiB API limit and ' 'peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.' % (size / (1024 * 1024)) ), 'when': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), 'commit': git_commit(), 'host': { 'system': platform.system(), 'machine': platform.machine(), 'processor': platform.processor(), }, 'compose_limits': {'api_memory': '256M', 'replicas': 2}, 'profile': { 'concurrency': args.concurrency, 'width': args.width, 'height': args.height, 'size_bytes': size, 'sha256': digest, }, 'results': { 'ok': len(latencies), 'errors': errors, 'elapsed_s': round(elapsed, 3), 'p50_s': pct(50), 'p95_s': pct(95), 'p99_s': pct(99), 'peak_api_rss_mib': round(peak, 2), 'peak_rss_by_container_mib': per_container, 'samples': len(samples), 'naive_n_times_size_mib': naive_bound_mib, }, 'limits_of_measurement': ( 'docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. ' 'Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client.' ), } out = Path(args.output) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print('ok=%s errors=%s peak_rss_mib=%s naive_N_x_size_mib=%s file=%s' % ( len(latencies), errors, summary['results']['peak_api_rss_mib'], naive_bound_mib, out, )) if errors: raise SystemExit(1) if __name__ == '__main__': try: main() except Failure as exc: print('FAIL:', exc) raise SystemExit(1)