forked from mtusi2026/mtusi2026lab4
239 lines
7.8 KiB
Python
239 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Concurrent attachment uploads with peak RSS of API containers.
|
|
|
|
Generates a valid PNG at runtime (not stored in Git). Measures docker stats while
|
|
N workers POST the same image. Writes a sanitized JSON summary.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
|
|
def make_png(width: int, height: int) -> bytes:
|
|
rows = []
|
|
for _y in range(height):
|
|
row = bytearray(1 + width * 3)
|
|
row[0] = 0
|
|
row[1:] = os.urandom(width * 3)
|
|
rows.append(bytes(row))
|
|
raw = b''.join(rows)
|
|
compressed = zlib.compress(raw, 1)
|
|
|
|
def chunk(tag: bytes, data: bytes) -> bytes:
|
|
crc = zlib.crc32(tag + data) & 0xffffffff
|
|
return len(data).to_bytes(4, 'big') + tag + data + crc.to_bytes(4, 'big')
|
|
|
|
ihdr = width.to_bytes(4, 'big') + height.to_bytes(4, 'big') + bytes([8, 2, 0, 0, 0])
|
|
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', compressed) + chunk(b'IEND', b'')
|
|
|
|
|
|
def parse_mib(usage: str) -> float:
|
|
# "45.23MiB / 256MiB" or "123.4MB / 256MB"
|
|
left = usage.split('/', 1)[0].strip()
|
|
number = ''
|
|
unit = ''
|
|
for char in left:
|
|
if char.isdigit() or char in '.':
|
|
number += char
|
|
else:
|
|
unit += char
|
|
value = float(number or '0')
|
|
unit = unit.strip().lower()
|
|
if unit in ('gib', 'gb'):
|
|
return value * 1024
|
|
if unit in ('kib', 'kb'):
|
|
return value / 1024
|
|
if unit in ('b',):
|
|
return value / (1024 * 1024)
|
|
return value
|
|
|
|
|
|
def git_commit() -> str:
|
|
try:
|
|
return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=str(ROOT), text=True).strip()
|
|
except (OSError, subprocess.CalledProcessError):
|
|
return 'unknown'
|
|
|
|
|
|
def docker_stats() -> list:
|
|
try:
|
|
ids = subprocess.check_output(
|
|
['docker', 'ps', '-q', '--filter', 'name=messenger-lab4-api'],
|
|
cwd=str(ROOT),
|
|
text=True,
|
|
timeout=5,
|
|
).split()
|
|
if not ids:
|
|
return []
|
|
output = subprocess.check_output(
|
|
['docker', 'stats', '--no-stream', '--format', '{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}', *ids],
|
|
cwd=str(ROOT),
|
|
text=True,
|
|
timeout=8,
|
|
)
|
|
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
|
|
return []
|
|
rows = []
|
|
for line in output.splitlines():
|
|
parts = line.split('\t')
|
|
if len(parts) < 2:
|
|
continue
|
|
name, usage = parts[0], parts[1]
|
|
rows.append({'name': name, 'mem_usage': usage, 'rss_mib': round(parse_mib(usage), 2)})
|
|
return rows
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description='Concurrent upload memory experiment')
|
|
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080'))
|
|
parser.add_argument('--concurrency', type=int, default=10)
|
|
parser.add_argument('--width', type=int, default=900)
|
|
parser.add_argument('--height', type=int, default=700)
|
|
parser.add_argument('--output', default=str(ROOT / 'evidence' / 'upload-concurrency.json'))
|
|
args = parser.parse_args()
|
|
if args.concurrency < 10:
|
|
raise SystemExit('concurrency must be >= 10')
|
|
|
|
image = make_png(args.width, args.height)
|
|
size = len(image)
|
|
digest = hashlib.sha256(image).hexdigest()
|
|
client = Client(args.base_url, timeout=60)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
path = '/api/v1/chats/%s/attachments?filename=load.png' % chat['id']
|
|
|
|
stop = threading.Event()
|
|
samples = []
|
|
|
|
def watch():
|
|
while not stop.is_set():
|
|
snapshot = docker_stats()
|
|
if snapshot:
|
|
samples.append({'ts': time.time(), 'containers': snapshot})
|
|
stop.wait(0.05)
|
|
|
|
watcher = threading.Thread(target=watch, daemon=True)
|
|
watcher.start()
|
|
baseline = docker_stats()
|
|
if baseline:
|
|
samples.append({'ts': time.time(), 'containers': baseline, 'phase': 'baseline'})
|
|
started = time.monotonic()
|
|
latencies = []
|
|
errors = 0
|
|
|
|
def one(_index: int):
|
|
response = client.call(
|
|
'POST',
|
|
path,
|
|
201,
|
|
actor=alice,
|
|
raw=image,
|
|
headers={'Content-Type': 'image/png'},
|
|
schema='Attachment',
|
|
)
|
|
body = response['json']
|
|
if body.get('sha256') != digest or body.get('size_bytes') != size:
|
|
raise Failure('checksum mismatch on concurrent upload')
|
|
return response['elapsed']
|
|
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futs = [pool.submit(one, i) for i in range(args.concurrency)]
|
|
for fut in as_completed(futs):
|
|
try:
|
|
latencies.append(fut.result())
|
|
except Exception:
|
|
errors += 1
|
|
finally:
|
|
stop.set()
|
|
watcher.join(timeout=2)
|
|
|
|
elapsed = time.monotonic() - started
|
|
latencies.sort()
|
|
|
|
def pct(p):
|
|
if not latencies:
|
|
return None
|
|
index = min(len(latencies) - 1, max(0, int(round((p / 100) * (len(latencies) - 1)))))
|
|
return round(latencies[index], 4)
|
|
|
|
peak = 0.0
|
|
per_container = {}
|
|
for sample in samples:
|
|
for row in sample['containers']:
|
|
peak = max(peak, row['rss_mib'])
|
|
prev = per_container.get(row['name'], 0.0)
|
|
per_container[row['name']] = max(prev, row['rss_mib'])
|
|
|
|
naive_bound_mib = round((args.concurrency * size) / (1024 * 1024), 2)
|
|
summary = {
|
|
'hypothesis': (
|
|
'Ten concurrent ~%.1f MiB PNG uploads stay inside the 256 MiB API limit and '
|
|
'peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.'
|
|
% (size / (1024 * 1024))
|
|
),
|
|
'when': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
|
'commit': git_commit(),
|
|
'host': {
|
|
'system': platform.system(),
|
|
'machine': platform.machine(),
|
|
'processor': platform.processor(),
|
|
},
|
|
'compose_limits': {'api_memory': '256M', 'replicas': 2},
|
|
'profile': {
|
|
'concurrency': args.concurrency,
|
|
'width': args.width,
|
|
'height': args.height,
|
|
'size_bytes': size,
|
|
'sha256': digest,
|
|
},
|
|
'results': {
|
|
'ok': len(latencies),
|
|
'errors': errors,
|
|
'elapsed_s': round(elapsed, 3),
|
|
'p50_s': pct(50),
|
|
'p95_s': pct(95),
|
|
'p99_s': pct(99),
|
|
'peak_api_rss_mib': round(peak, 2),
|
|
'peak_rss_by_container_mib': per_container,
|
|
'samples': len(samples),
|
|
'naive_n_times_size_mib': naive_bound_mib,
|
|
},
|
|
'limits_of_measurement': (
|
|
'docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. '
|
|
'Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client.'
|
|
),
|
|
}
|
|
out = Path(args.output)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print('ok=%s errors=%s peak_rss_mib=%s naive_N_x_size_mib=%s file=%s' % (
|
|
len(latencies), errors, summary['results']['peak_api_rss_mib'], naive_bound_mib, out,
|
|
))
|
|
if errors:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Failure as exc:
|
|
print('FAIL:', exc)
|
|
raise SystemExit(1)
|