Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
94 lines
3.7 KiB
Python
94 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers.
|
|
python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import zlib
|
|
import struct
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402
|
|
|
|
|
|
def png(width, height):
|
|
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
|
|
def chunk(kind, body):
|
|
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
|
|
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
|
+ chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
|
|
|
|
|
|
UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824}
|
|
|
|
|
|
def mib(text):
|
|
value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups()
|
|
return float(value) * UNITS[unit] / 1048576
|
|
|
|
|
|
def sample(stop, out):
|
|
while not stop.is_set():
|
|
result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
for line in result.stdout.splitlines():
|
|
name, usage = line.split(" ", 1)
|
|
if "api-" in name:
|
|
out.setdefault(name, []).append(mib(usage.split("/")[0].strip()))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--concurrency", type=int, default=10)
|
|
parser.add_argument("--rounds", type=int, default=3)
|
|
args = parser.parse_args()
|
|
body = png(1700, 1700)
|
|
print("payload bytes:", len(body))
|
|
c = Client(timeout=60)
|
|
alice = c.user("mem")
|
|
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"]
|
|
path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"]
|
|
|
|
def one(_):
|
|
r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"})
|
|
return r["status"], r["elapsed"]
|
|
|
|
idle, peaks, statuses, times = {}, [], [], []
|
|
probe = {}
|
|
stop = threading.Event()
|
|
t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join()
|
|
idle = {k: max(v) for k, v in probe.items()}
|
|
for round_no in range(args.rounds):
|
|
probe = {}
|
|
stop = threading.Event()
|
|
t = threading.Thread(target=sample, args=(stop, probe)); t.start()
|
|
with ThreadPoolExecutor(args.concurrency) as pool:
|
|
results = list(pool.map(one, range(args.concurrency)))
|
|
stop.set(); t.join()
|
|
peak = {k: max(v) for k, v in probe.items()}
|
|
peaks.append(peak)
|
|
statuses += [s for s, _ in results]
|
|
times += [e for _, e in results]
|
|
print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results))))
|
|
summary = {
|
|
"payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds,
|
|
"idle_mib": {k: round(v, 1) for k, v in idle.items()},
|
|
"peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle},
|
|
"container_limit_mib": 256,
|
|
"statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))},
|
|
"upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2),
|
|
}
|
|
print(json.dumps(summary, indent=2))
|
|
|
|
|
|
main()
|