Files
mtusi2026lab4/bench/upload_memory.py
T
Alexey BodanyukandClaude Opus 5.5 a2d867bec6 lab4 трек 2: reconciler S3/PostgreSQL, тесты вложений, отказов и согласованности
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key;
  dry-run read-only конструктивно (default_transaction_read_only + S3 без записи),
  --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо),
  перепроверка под FOR UPDATE, предохранитель от массового удаления;
- attachment_ids хранится в строке сообщения (история остаётся одним index range scan),
  message_attachments — для FK и обратного поиска;
- семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY);
- тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки,
  приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler;
- бенчмарки памяти загрузок и стоимости обхода reconciler.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 14:05:03 +03:00

212 lines
8.8 KiB
Python

#!/usr/bin/env python3
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
Method (repeatable, the stand must be up):
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
counter memory.peak starts from an idle process (it never decreases during a container's life);
* idle memory.current of each replica is read; then N clients upload at the same moment through
the LB (a barrier releases them together), each a different file;
* during the run memory.current of every replica is sampled every 50 ms inside the container;
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
PID 1 (the Python server alone) are read;
* the normal configuration is restored at the end.
python bench/upload_memory.py --output evidence/memory
"""
from __future__ import annotations
import argparse
import io
import json
import os
import platform
import statistics
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
MiB = 1024 * 1024
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
env={**os.environ, **(env or {})}).stdout
def replicas() -> list[str]:
return sh("docker", "compose", "ps", "-q", "api").split()
def cg(cid: str, name: str) -> int:
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
def vmhwm(cid: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
if line.startswith("VmHWM:"):
return int(line.split()[1]) * 1024
return 0
def recreate(env: dict) -> None:
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
time.sleep(3)
class Sampler(threading.Thread):
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
def __init__(self, cid: str):
super().__init__(daemon=True)
self.cid = cid
self.values: list[int] = []
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
stdout=subprocess.PIPE, text=True)
def run(self) -> None:
for line in self.proc.stdout:
if line.strip().isdigit():
self.values.append(int(line))
def stop(self) -> None:
self.proc.kill()
def actor(client: httpx.Client) -> tuple[str, str]:
name = "mem_" + uuid.uuid4().hex[:10]
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
import base64
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "mem", "member_ids": [me["id"]]}).json()
return token, chat["id"]
def big_jpeg(width: int, height: int, seed: int) -> bytes:
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
out = io.BytesIO()
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
return out.getvalue()
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
recreate(env)
ids = replicas()
idle = {cid: cg(cid, "memory.current") for cid in ids}
client = httpx.Client(base_url=BASE, timeout=120)
token, chat_id = actor(client)
barrier = threading.Barrier(len(files))
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
def upload(i: int, data: bytes, ctype: str) -> None:
c = httpx.Client(base_url=BASE, timeout=120)
barrier.wait()
started = time.perf_counter()
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"))
c.close()
samplers = [Sampler(cid) for cid in ids]
for s in samplers:
s.start()
time.sleep(0.5)
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
wall = time.perf_counter()
for t in threads:
t.start()
for t in threads:
t.join()
wall = time.perf_counter() - wall
time.sleep(0.5)
for s in samplers:
s.stop()
per_replica = {}
for cid, s in zip(ids, samplers):
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
per_replica[hostname] = {
"idle_mib": round(idle[cid] / MiB, 1),
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
"uploads_served": sum(1 for r in results if r[2] == hostname),
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
}
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
- per_replica[hostname]["idle_mib"], 1)
lat = sorted(r[1] for r in results if r[0] == 201)
codes: dict[str, int] = {}
for r in results:
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
summary = {
"scenario": name, "settings": env, "clients": len(files),
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
"types": sorted({t for _, t in files}), "status_codes": codes, "wall_seconds": round(wall, 2),
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
"replicas": per_replica,
}
print(json.dumps(summary, ensure_ascii=False), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--clients", type=int, default=16)
p.add_argument("--output", default="evidence/memory")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
print("generating files ...", flush=True)
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(args.clients)]
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
scenarios = [
("png-9.5MiB x16, slots=4 (default)", base, pngs),
("png-9.5MiB x16, slots=16 (no bound)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
("jpeg-40Mpx-progressive x16, slots=4, verify=1 (default)", base, jpegs),
("jpeg-40Mpx-progressive x16, slots=4, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
]
results = []
try:
for name, env, files in scenarios:
results.append(run_scenario(name, env, files))
finally:
recreate({}) # back to the compose defaults
meta = {
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
replicas()[0]).strip(),
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
}
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
print(json.dumps(meta, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())