Files
mtusi2026lab4/student-tests/memory_experiment.py
T
alenchha 078a8341b5 ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API
с потоковой проверкой целостности, метаданные в PostgreSQL и байты в
приватном bucket, отдельный S3-пользователь с ограниченными правами,
идемпотентный bootstrap bucket, компенсация и reconciler (трек 2),
тесты отказов, эксперименты по памяти и согласованности.
2026-11-02 16:59:17 +03:00

94 lines
3.7 KiB
Python

#!/usr/bin/env python3
"""Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers.
python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3"""
import argparse
import json
import os
import re
import statistics
import subprocess
import sys
import threading
import time
import zlib
import struct
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def png(width, height):
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
def chunk(kind, body):
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824}
def mib(text):
value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups()
return float(value) * UNITS[unit] / 1048576
def sample(stop, out):
while not stop.is_set():
result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
cwd=ROOT, capture_output=True, text=True)
for line in result.stdout.splitlines():
name, usage = line.split(" ", 1)
if "api-" in name:
out.setdefault(name, []).append(mib(usage.split("/")[0].strip()))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--rounds", type=int, default=3)
args = parser.parse_args()
body = png(1700, 1700)
print("payload bytes:", len(body))
c = Client(timeout=60)
alice = c.user("mem")
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"]
path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"]
def one(_):
r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"})
return r["status"], r["elapsed"]
idle, peaks, statuses, times = {}, [], [], []
probe = {}
stop = threading.Event()
t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join()
idle = {k: max(v) for k, v in probe.items()}
for round_no in range(args.rounds):
probe = {}
stop = threading.Event()
t = threading.Thread(target=sample, args=(stop, probe)); t.start()
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(one, range(args.concurrency)))
stop.set(); t.join()
peak = {k: max(v) for k, v in probe.items()}
peaks.append(peak)
statuses += [s for s, _ in results]
times += [e for _, e in results]
print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results))))
summary = {
"payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds,
"idle_mib": {k: round(v, 1) for k, v in idle.items()},
"peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle},
"container_limit_mib": 256,
"statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))},
"upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2),
}
print(json.dumps(summary, indent=2))
main()