Files
mtusi2026lab4/student-tests/memory_experiment.py
T
alenchha a856128b45 Исправить замечания ревью: битый JPEG, память nginx, 502 вместо 413
JPEG проверяется через djpeg -strict: обрезанный поток с дописанным EOI
теперь даёт 415. Память nginx больше не зависит от тел файлов: загрузка
идёт потоком до API, скачивание без временных файлов, воркеров 2. API не
закрывает соединение при недочитанном теле, что давало HTML 502; nginx
сам отклоняет тела больше лимитов API. Лимит памяти API поднят до 512 MiB
под проверку CMYK-JPEG на 40 Мпикс. Добавлен стресс-тест через LB.
2026-11-05 13:53:08 +03:00

94 lines
3.7 KiB
Python

#!/usr/bin/env python3
"""Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers.
python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3"""
import argparse
import json
import os
import re
import statistics
import subprocess
import sys
import threading
import time
import zlib
import struct
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def png(width, height):
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
def chunk(kind, body):
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824}
def mib(text):
value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups()
return float(value) * UNITS[unit] / 1048576
def sample(stop, out):
while not stop.is_set():
result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
cwd=ROOT, capture_output=True, text=True)
for line in result.stdout.splitlines():
name, usage = line.split(" ", 1)
if "api-" in name:
out.setdefault(name, []).append(mib(usage.split("/")[0].strip()))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--rounds", type=int, default=3)
args = parser.parse_args()
body = png(1700, 1700)
print("payload bytes:", len(body))
c = Client(timeout=60)
alice = c.user("mem")
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"]
path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"]
def one(_):
r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"})
return r["status"], r["elapsed"]
idle, peaks, statuses, times = {}, [], [], []
probe = {}
stop = threading.Event()
t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join()
idle = {k: max(v) for k, v in probe.items()}
for round_no in range(args.rounds):
probe = {}
stop = threading.Event()
t = threading.Thread(target=sample, args=(stop, probe)); t.start()
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(one, range(args.concurrency)))
stop.set(); t.join()
peak = {k: max(v) for k, v in probe.items()}
peaks.append(peak)
statuses += [s for s, _ in results]
times += [e for _, e in results]
print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results))))
summary = {
"payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds,
"idle_mib": {k: round(v, 1) for k, v in idle.items()},
"peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle},
"container_limit_mib": 512,
"statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))},
"upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2),
}
print(json.dumps(summary, indent=2))
main()