Files
2026-11-10 19:24:33 +03:00

239 lines
7.8 KiB
Python

#!/usr/bin/env python3
"""Concurrent attachment uploads with peak RSS of API containers.
Generates a valid PNG at runtime (not stored in Git). Measures docker stats while
N workers POST the same image. Writes a sanitized JSON summary.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import subprocess
import sys
import threading
import time
import uuid
import zlib
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure # noqa: E402
def make_png(width: int, height: int) -> bytes:
rows = []
for _y in range(height):
row = bytearray(1 + width * 3)
row[0] = 0
row[1:] = os.urandom(width * 3)
rows.append(bytes(row))
raw = b''.join(rows)
compressed = zlib.compress(raw, 1)
def chunk(tag: bytes, data: bytes) -> bytes:
crc = zlib.crc32(tag + data) & 0xffffffff
return len(data).to_bytes(4, 'big') + tag + data + crc.to_bytes(4, 'big')
ihdr = width.to_bytes(4, 'big') + height.to_bytes(4, 'big') + bytes([8, 2, 0, 0, 0])
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', compressed) + chunk(b'IEND', b'')
def parse_mib(usage: str) -> float:
# "45.23MiB / 256MiB" or "123.4MB / 256MB"
left = usage.split('/', 1)[0].strip()
number = ''
unit = ''
for char in left:
if char.isdigit() or char in '.':
number += char
else:
unit += char
value = float(number or '0')
unit = unit.strip().lower()
if unit in ('gib', 'gb'):
return value * 1024
if unit in ('kib', 'kb'):
return value / 1024
if unit in ('b',):
return value / (1024 * 1024)
return value
def git_commit() -> str:
try:
return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=str(ROOT), text=True).strip()
except (OSError, subprocess.CalledProcessError):
return 'unknown'
def docker_stats() -> list:
try:
ids = subprocess.check_output(
['docker', 'ps', '-q', '--filter', 'name=messenger-lab4-api'],
cwd=str(ROOT),
text=True,
timeout=5,
).split()
if not ids:
return []
output = subprocess.check_output(
['docker', 'stats', '--no-stream', '--format', '{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}', *ids],
cwd=str(ROOT),
text=True,
timeout=8,
)
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
return []
rows = []
for line in output.splitlines():
parts = line.split('\t')
if len(parts) < 2:
continue
name, usage = parts[0], parts[1]
rows.append({'name': name, 'mem_usage': usage, 'rss_mib': round(parse_mib(usage), 2)})
return rows
def main() -> None:
parser = argparse.ArgumentParser(description='Concurrent upload memory experiment')
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080'))
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--width', type=int, default=900)
parser.add_argument('--height', type=int, default=700)
parser.add_argument('--output', default=str(ROOT / 'evidence' / 'upload-concurrency.json'))
args = parser.parse_args()
if args.concurrency < 10:
raise SystemExit('concurrency must be >= 10')
image = make_png(args.width, args.height)
size = len(image)
digest = hashlib.sha256(image).hexdigest()
client = Client(args.base_url, timeout=60)
alice, _bob, _eve, chat = client.bootstrap()
path = '/api/v1/chats/%s/attachments?filename=load.png' % chat['id']
stop = threading.Event()
samples = []
def watch():
while not stop.is_set():
snapshot = docker_stats()
if snapshot:
samples.append({'ts': time.time(), 'containers': snapshot})
stop.wait(0.05)
watcher = threading.Thread(target=watch, daemon=True)
watcher.start()
baseline = docker_stats()
if baseline:
samples.append({'ts': time.time(), 'containers': baseline, 'phase': 'baseline'})
started = time.monotonic()
latencies = []
errors = 0
def one(_index: int):
response = client.call(
'POST',
path,
201,
actor=alice,
raw=image,
headers={'Content-Type': 'image/png'},
schema='Attachment',
)
body = response['json']
if body.get('sha256') != digest or body.get('size_bytes') != size:
raise Failure('checksum mismatch on concurrent upload')
return response['elapsed']
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futs = [pool.submit(one, i) for i in range(args.concurrency)]
for fut in as_completed(futs):
try:
latencies.append(fut.result())
except Exception:
errors += 1
finally:
stop.set()
watcher.join(timeout=2)
elapsed = time.monotonic() - started
latencies.sort()
def pct(p):
if not latencies:
return None
index = min(len(latencies) - 1, max(0, int(round((p / 100) * (len(latencies) - 1)))))
return round(latencies[index], 4)
peak = 0.0
per_container = {}
for sample in samples:
for row in sample['containers']:
peak = max(peak, row['rss_mib'])
prev = per_container.get(row['name'], 0.0)
per_container[row['name']] = max(prev, row['rss_mib'])
naive_bound_mib = round((args.concurrency * size) / (1024 * 1024), 2)
summary = {
'hypothesis': (
'Ten concurrent ~%.1f MiB PNG uploads stay inside the 256 MiB API limit and '
'peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.'
% (size / (1024 * 1024))
),
'when': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
'commit': git_commit(),
'host': {
'system': platform.system(),
'machine': platform.machine(),
'processor': platform.processor(),
},
'compose_limits': {'api_memory': '256M', 'replicas': 2},
'profile': {
'concurrency': args.concurrency,
'width': args.width,
'height': args.height,
'size_bytes': size,
'sha256': digest,
},
'results': {
'ok': len(latencies),
'errors': errors,
'elapsed_s': round(elapsed, 3),
'p50_s': pct(50),
'p95_s': pct(95),
'p99_s': pct(99),
'peak_api_rss_mib': round(peak, 2),
'peak_rss_by_container_mib': per_container,
'samples': len(samples),
'naive_n_times_size_mib': naive_bound_mib,
},
'limits_of_measurement': (
'docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. '
'Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client.'
),
}
out = Path(args.output)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print('ok=%s errors=%s peak_rss_mib=%s naive_N_x_size_mib=%s file=%s' % (
len(latencies), errors, summary['results']['peak_api_rss_mib'], naive_bound_mib, out,
))
if errors:
raise SystemExit(1)
if __name__ == '__main__':
try:
main()
except Failure as exc:
print('FAIL:', exc)
raise SystemExit(1)