Files
mtusi2026lab5/student-tests/upload_memory.py
T
zemiles 13bec4ab8b ЛР 5 — БПИ2303 — Лазарев — на 4
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher
confirms, manual ack, DLQ), transactional outbox, отдельный worker.

- upload: оригинал в staging S3, затем одной транзакцией queued + jobs +
  outbox -> 202; API к брокеру не подключается
- relay: публикация outbox с confirm, reaper истёкших аренд, redrive
- worker: claim по строке jobs, аренда с heartbeat и fencing по
  lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом,
  thumbnail 256, crop 128, WebP 1280 по детерминированным ключам
- retry с backoff и jitter, постоянные/временные ошибки, реестр failed,
  ops replay (ready не повторяется), DLQ для невалидных сообщений
- janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4
- тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack,
  заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
2026-11-13 16:28:18 +03:00

178 lines
8.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Пик памяти API при одновременных загрузках крупных файлов (уровень 4).
python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json
Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT
частями по 5 MiB и draft-декодирование JPEG держат RAM реплики ограниченной и не
зависящей линейно от числа одновременных загрузок; 10+ загрузок по ~10 MiB не
приближают процесс к лимиту контейнера 256 MiB.
Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой
реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и
anon/file из cgroup memory.stat. Нагрузка — concurrency потоков, каждый грузит свой
файл через LB; файлы: несжимаемый PNG ~9.99 MiB и (если на хосте есть Pillow) JPEG
6000x6000 = 36 Мпикс. В конце — VmHWM (пиковый RSS процесса) и memory.peak cgroup.
file-память cgroup — page cache временных файлов, она вытесняема и OOM не вызывает.
"""
import argparse
import hashlib
import io
import json
import os
import subprocess
import sys
import threading
import time
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "student-tests"))
from client import Client # noqa: E402
from attachments_negative import noise_png # noqa: E402
SAMPLER = ("while true; do r=$(grep VmRSS /proc/1/status | tr -s ' ' | cut -d' ' -f2); "
"a=$(grep '^anon ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); "
"f=$(grep '^file ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); echo \"$r $a $f\"; sleep 0.05; done")
def sh(*cmd):
return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=True).stdout
def containers():
return sh("docker", "compose", "ps", "-q", "api").split()
def proc_status(cid, field):
out = sh("docker", "exec", cid, "grep", field, "/proc/1/status")
return int(out.split()[1]) // 1024 # kB -> MiB
class Sampler:
def __init__(self, cid):
self.cid, self.samples = cid, []
self.p = subprocess.Popen(["docker", "exec", cid, "sh", "-c", SAMPLER], stdout=subprocess.PIPE, text=True)
threading.Thread(target=self._read, daemon=True).start()
def _read(self):
for line in self.p.stdout:
parts = line.split()
if len(parts) == 3:
self.samples.append((time.monotonic(), int(parts[0]) / 1024, int(parts[1]) / 2**20, int(parts[2]) / 2**20))
def stop(self):
self.p.terminate()
def make_jpeg():
try:
from PIL import Image
except ImportError:
return None
im = Image.effect_noise((6000, 6000), 40).convert("RGB")
for q in (60, 45, 30, 20):
b = io.BytesIO()
im.save(b, "JPEG", quality=q)
if b.tell() <= 10 * 1024 * 1024:
return b.getvalue()
return None
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--concurrency", type=int, default=12)
ap.add_argument("--rounds", type=int, default=3)
ap.add_argument("--no-restart", action="store_true")
ap.add_argument("--output")
args = ap.parse_args()
files = [("image/png", noise_png(10 * 1024 * 1024))]
jpeg = make_jpeg()
if jpeg:
files.append(("image/jpeg", jpeg))
if not args.no_restart:
sh("docker", "compose", "restart", "api")
sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api")
time.sleep(4)
c = Client(args.base_url, timeout=120)
alice, _bob, _eve, chat = c.bootstrap()
# прогрев: один upload каждого типа (импорт Pillow, пулы соединений)
for ctype, data in files:
c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data,
headers={"Content-Type": ctype})
cids = containers()
idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")}
for cid in cids}
s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0]
s3_before = {"rss_mib": proc_status(s3_cid, "VmRSS"), "oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip()}
samplers = [Sampler(cid) for cid in cids]
s3_sampler = Sampler(s3_cid)
time.sleep(1)
results, lock = [], threading.Lock()
def worker(i):
ctype, data = files[i % len(files)]
t = time.monotonic()
try:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=m%d" % (chat["id"], i), [202, 503], actor=alice,
raw=data, headers={"Content-Type": ctype}, schema=None)
st, inst = r["status"], r["headers"].get("X-Instance-Id")
except Exception as exc: # noqa: BLE001
st, inst = type(exc).__name__, None
with lock:
results.append({"type": ctype, "status": st, "instance": inst, "s": round(time.monotonic() - t, 2)})
started = time.monotonic()
for _ in range(args.rounds):
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for th in threads:
th.start()
for th in threads:
th.join()
wall = time.monotonic() - started
time.sleep(1)
for s in samplers + [s3_sampler]:
s.stop()
per = {}
for cid, s in zip(cids, samplers):
k = cid[:12]
per[k] = dict(idle[k],
samples=len(s.samples),
peak_rss_mib_sampled=round(max(x[1] for x in s.samples), 1),
peak_anon_mib=round(max(x[2] for x in s.samples), 1),
peak_file_cache_mib=round(max(x[3] for x in s.samples), 1),
vmhwm_mib=proc_status(cid, "VmHWM"),
cgroup_memory_peak_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak")) / 2**20, 1),
memory_limit_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20),
uploads_served=sum(1 for r in results if r["instance"] == k))
durations = sorted(r["s"] for r in results if r["status"] == 202)
summary = {
"concurrency": args.concurrency, "rounds": args.rounds,
"files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files],
"uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k)
for k in {r["status"] for r in results}},
"megabytes_uploaded_ok": round(sum(len(files[i % len(files)][1]) for i, r in enumerate(results)
if r["status"] == 202) / 2**20, 1),
"wall_s": round(wall, 1),
"latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None,
"replicas": per,
"s3": dict(s3_before,
peak_rss_mib_sampled=round(max(x[1] for x in s3_sampler.samples), 1) if s3_sampler.samples else None,
peak_anon_mib=round(max(x[2] for x in s3_sampler.samples), 1) if s3_sampler.samples else None,
restarts=int(sh("docker", "inspect", "-f", "{{.RestartCount}}", s3_cid)),
oom_killed_after=sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip(),
memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)),
"oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(),
}
print(json.dumps(summary, ensure_ascii=False, indent=1))
if args.output:
Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1))
return 0
if __name__ == "__main__":
sys.exit(main())