Files
mtusi2026lab4/student-tests/upload_memory.py
Ilya a97c35fc12 выполнение ЛР4 на 4
- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket,
  отдельные ограниченные ключи API, служебные порты закрыты;
  идемпотентный s3-init с проверкой приватности и прогревом записи
- вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс,
  тип по содержимому, проверка filename, скачивание только участникам
- согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper
  с grace и dry-run; attachment_ids в сообщениях, FK по чату
- back-pressure на S3 PUT после найденного OOM SeaweedFS
- тесты student-tests, evidence, REPORT.md, DEMO.md
2026-10-28 18:06:07 +03:00

108 lines
4.4 KiB
Python

#!/usr/bin/env python3
"""Experiment: N simultaneous uploads through the LB; peak memory of every API container (cgroup v2).
Hypothesis: with streaming into a spooled temp file (UPLOAD_SPOOL_BYTES in RAM per upload) and DECODE_CONCURRENCY=1,
peak anonymous memory of a replica grows by O(spool x N + one decoded image), not O(file size x N).
"""
import argparse
import json
import statistics
import subprocess
import sys
import threading
import time
import uuid
from stack import ROOT, dc, noise_png, png
from client import Client, Failure
MB = 1024 * 1024
def sample(cid):
out = subprocess.run(["docker", "exec", cid, "sh", "-c",
"cat /sys/fs/cgroup/memory.current; grep -E '^(anon|file) ' /sys/fs/cgroup/memory.stat"],
capture_output=True, text=True).stdout.split()
return {"current": int(out[0]), "anon": int(out[2]), "file": int(out[4])}
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--base-url", default="http://localhost:8080")
p.add_argument("--concurrency", type=int, default=10)
p.add_argument("--kind", choices=["noise", "pixels"], default="noise",
help="noise: ~9.6 MiB incompressible PNG; pixels: 7000x5714 RGB (39.99 Mpx, tiny file, 120 MB decoded)")
p.add_argument("--label", default="")
p.add_argument("--output")
a = p.parse_args()
data = noise_png(1790, 1790) if a.kind == "noise" else png(7000, 5714, 2)
c = Client(a.base_url, timeout=120)
alice, bob, eve, chat = c.bootstrap()
cids = dc("ps", "-q", "api").stdout.split() + dc("ps", "-q", "s3").stdout.split()
names = {cid: subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True).stdout.strip("/\n")
for cid in cids}
base = {cid: sample(cid) for cid in cids}
peaks = {cid: dict(base[cid]) for cid in cids}
stop = threading.Event()
def poll(cid):
while not stop.is_set():
s = sample(cid)
for k in s:
peaks[cid][k] = max(peaks[cid][k], s[k])
pollers = [threading.Thread(target=poll, args=(cid,), daemon=True) for cid in cids]
for t in pollers:
t.start()
time.sleep(1)
results = []
lock = threading.Lock()
def one(i):
t = time.monotonic()
try:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=m%d.png" % (chat["id"], i), [201, 413, 503], actor=alice,
raw=data, headers={"Content-Type": "image/png"})
status, inst = r["status"], r["headers"].get("X-Instance-Id")
except Failure as exc:
status, inst = str(exc)[:60], None
with lock:
results.append({"status": status, "seconds": round(time.monotonic() - t, 3), "instance": inst})
started = time.monotonic()
workers = [threading.Thread(target=one, args=(i,)) for i in range(a.concurrency)]
for w in workers:
w.start()
for w in workers:
w.join()
wall = time.monotonic() - started
time.sleep(1.5)
stop.set()
for t in pollers:
t.join()
lat = sorted(r["seconds"] for r in results)
report = {
"label": a.label, "kind": a.kind, "file_bytes": len(data), "concurrency": a.concurrency,
"statuses": {str(s): sum(1 for r in results if r["status"] == s) for s in {r["status"] for r in results}},
"per_instance": {i: sum(1 for r in results if r["instance"] == i) for i in {r["instance"] for r in results}},
"wall_seconds": round(wall, 2), "latency_s": {"p50": statistics.median(lat), "max": lat[-1]},
"memory_mib": {names[cid]: {
"baseline_anon": round(base[cid]["anon"] / MB, 1), "peak_anon": round(peaks[cid]["anon"] / MB, 1),
"delta_anon": round((peaks[cid]["anon"] - base[cid]["anon"]) / MB, 1),
"baseline_current": round(base[cid]["current"] / MB, 1), "peak_current": round(peaks[cid]["current"] / MB, 1),
"peak_file_cache": round(peaks[cid]["file"] / MB, 1)} for cid in cids},
"api_env": {k: dc("exec", "-T", "api", "printenv", k, check=False).stdout.strip() or "default"
for k in ("UPLOAD_SPOOL_BYTES", "UPLOAD_CONCURRENCY", "DECODE_CONCURRENCY")},
}
text = json.dumps(report, ensure_ascii=False, indent=1)
print(text)
if a.output:
(ROOT / a.output).write_text(text + "\n")
sys.exit(0 if all(r["status"] == 201 for r in results) else 1)
if __name__ == "__main__":
main()