forked from mtusi2026/mtusi2026lab5
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
178 lines
8.3 KiB
Python
178 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
||
"""Пик памяти API при одновременных загрузках крупных файлов (уровень 4).
|
||
|
||
python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json
|
||
|
||
Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT
|
||
частями по 5 MiB и draft-декодирование JPEG держат RAM реплики ограниченной и не
|
||
зависящей линейно от числа одновременных загрузок; 10+ загрузок по ~10 MiB не
|
||
приближают процесс к лимиту контейнера 256 MiB.
|
||
|
||
Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой
|
||
реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и
|
||
anon/file из cgroup memory.stat. Нагрузка — concurrency потоков, каждый грузит свой
|
||
файл через LB; файлы: несжимаемый PNG ~9.99 MiB и (если на хосте есть Pillow) JPEG
|
||
6000x6000 = 36 Мпикс. В конце — VmHWM (пиковый RSS процесса) и memory.peak cgroup.
|
||
file-память cgroup — page cache временных файлов, она вытесняема и OOM не вызывает.
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import zlib
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "tests"))
|
||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||
from client import Client # noqa: E402
|
||
from attachments_negative import noise_png # noqa: E402
|
||
|
||
SAMPLER = ("while true; do r=$(grep VmRSS /proc/1/status | tr -s ' ' | cut -d' ' -f2); "
|
||
"a=$(grep '^anon ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); "
|
||
"f=$(grep '^file ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); echo \"$r $a $f\"; sleep 0.05; done")
|
||
|
||
|
||
def sh(*cmd):
|
||
return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=True).stdout
|
||
|
||
|
||
def containers():
|
||
return sh("docker", "compose", "ps", "-q", "api").split()
|
||
|
||
|
||
def proc_status(cid, field):
|
||
out = sh("docker", "exec", cid, "grep", field, "/proc/1/status")
|
||
return int(out.split()[1]) // 1024 # kB -> MiB
|
||
|
||
|
||
class Sampler:
|
||
def __init__(self, cid):
|
||
self.cid, self.samples = cid, []
|
||
self.p = subprocess.Popen(["docker", "exec", cid, "sh", "-c", SAMPLER], stdout=subprocess.PIPE, text=True)
|
||
threading.Thread(target=self._read, daemon=True).start()
|
||
|
||
def _read(self):
|
||
for line in self.p.stdout:
|
||
parts = line.split()
|
||
if len(parts) == 3:
|
||
self.samples.append((time.monotonic(), int(parts[0]) / 1024, int(parts[1]) / 2**20, int(parts[2]) / 2**20))
|
||
|
||
def stop(self):
|
||
self.p.terminate()
|
||
|
||
|
||
def make_jpeg():
|
||
try:
|
||
from PIL import Image
|
||
except ImportError:
|
||
return None
|
||
im = Image.effect_noise((6000, 6000), 40).convert("RGB")
|
||
for q in (60, 45, 30, 20):
|
||
b = io.BytesIO()
|
||
im.save(b, "JPEG", quality=q)
|
||
if b.tell() <= 10 * 1024 * 1024:
|
||
return b.getvalue()
|
||
return None
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||
ap.add_argument("--concurrency", type=int, default=12)
|
||
ap.add_argument("--rounds", type=int, default=3)
|
||
ap.add_argument("--no-restart", action="store_true")
|
||
ap.add_argument("--output")
|
||
args = ap.parse_args()
|
||
files = [("image/png", noise_png(10 * 1024 * 1024))]
|
||
jpeg = make_jpeg()
|
||
if jpeg:
|
||
files.append(("image/jpeg", jpeg))
|
||
if not args.no_restart:
|
||
sh("docker", "compose", "restart", "api")
|
||
sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api")
|
||
time.sleep(4)
|
||
c = Client(args.base_url, timeout=120)
|
||
alice, _bob, _eve, chat = c.bootstrap()
|
||
# прогрев: один upload каждого типа (импорт Pillow, пулы соединений)
|
||
for ctype, data in files:
|
||
c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data,
|
||
headers={"Content-Type": ctype})
|
||
cids = containers()
|
||
idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")}
|
||
for cid in cids}
|
||
s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0]
|
||
s3_before = {"rss_mib": proc_status(s3_cid, "VmRSS"), "oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip()}
|
||
samplers = [Sampler(cid) for cid in cids]
|
||
s3_sampler = Sampler(s3_cid)
|
||
time.sleep(1)
|
||
results, lock = [], threading.Lock()
|
||
|
||
def worker(i):
|
||
ctype, data = files[i % len(files)]
|
||
t = time.monotonic()
|
||
try:
|
||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=m%d" % (chat["id"], i), [202, 503], actor=alice,
|
||
raw=data, headers={"Content-Type": ctype}, schema=None)
|
||
st, inst = r["status"], r["headers"].get("X-Instance-Id")
|
||
except Exception as exc: # noqa: BLE001
|
||
st, inst = type(exc).__name__, None
|
||
with lock:
|
||
results.append({"type": ctype, "status": st, "instance": inst, "s": round(time.monotonic() - t, 2)})
|
||
|
||
started = time.monotonic()
|
||
for _ in range(args.rounds):
|
||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
||
for th in threads:
|
||
th.start()
|
||
for th in threads:
|
||
th.join()
|
||
wall = time.monotonic() - started
|
||
time.sleep(1)
|
||
for s in samplers + [s3_sampler]:
|
||
s.stop()
|
||
per = {}
|
||
for cid, s in zip(cids, samplers):
|
||
k = cid[:12]
|
||
per[k] = dict(idle[k],
|
||
samples=len(s.samples),
|
||
peak_rss_mib_sampled=round(max(x[1] for x in s.samples), 1),
|
||
peak_anon_mib=round(max(x[2] for x in s.samples), 1),
|
||
peak_file_cache_mib=round(max(x[3] for x in s.samples), 1),
|
||
vmhwm_mib=proc_status(cid, "VmHWM"),
|
||
cgroup_memory_peak_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak")) / 2**20, 1),
|
||
memory_limit_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20),
|
||
uploads_served=sum(1 for r in results if r["instance"] == k))
|
||
durations = sorted(r["s"] for r in results if r["status"] == 202)
|
||
summary = {
|
||
"concurrency": args.concurrency, "rounds": args.rounds,
|
||
"files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files],
|
||
"uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k)
|
||
for k in {r["status"] for r in results}},
|
||
"megabytes_uploaded_ok": round(sum(len(files[i % len(files)][1]) for i, r in enumerate(results)
|
||
if r["status"] == 202) / 2**20, 1),
|
||
"wall_s": round(wall, 1),
|
||
"latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None,
|
||
"replicas": per,
|
||
"s3": dict(s3_before,
|
||
peak_rss_mib_sampled=round(max(x[1] for x in s3_sampler.samples), 1) if s3_sampler.samples else None,
|
||
peak_anon_mib=round(max(x[2] for x in s3_sampler.samples), 1) if s3_sampler.samples else None,
|
||
restarts=int(sh("docker", "inspect", "-f", "{{.RestartCount}}", s3_cid)),
|
||
oom_killed_after=sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip(),
|
||
memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)),
|
||
"oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(),
|
||
}
|
||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||
if args.output:
|
||
Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|