Files
mtusi2026lab2/bench/run.py
T
ricoromaandClaude Opus 5.5 7b775a29a9 chore: carry over lab 1 solution into lab 2 skeleton
Copy the committed lab 1 sources as-is: src/, student-tests/, bench/,
Dockerfile, docker-compose.yml, pyproject.toml, uv.lock,
.python-version, config_example.toml, AGENTS.md, CLAUDE.md, .claude/,
openspec/, .graphifyignore and KNOWN_ISSUES.md. Lab 1 evidence stays
in the lab 1 repository.

Merge the new skeleton files with own sections: Makefile (lab 2
targets plus own run, docker, bench and common targets), .gitignore,
.dockerignore and .env.example. Rename the image to
mtusi-messenger:lab2 and point AGENTS.md at lab 2.

The application is still the in-memory lab 1 implementation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 22:11:46 +03:00

1274 lines
44 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Нагрузочный стенд трека 1: матрица размеров истории и конкуренции.
Запуск — `make bench` / `make bench_quick`. Методика — design change
`load-bench-harness`, инварианты — `.claude/rules/bench.md`. Только
stdlib: приложение не импортируется, всё через docker CLI и k6.
"""
import argparse
import csv
import json
import os
import platform
import re
import shutil
import signal
import statistics
import subprocess
import sys
import threading
import time
from dataclasses import asdict, dataclass, replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
K6_IMAGE = (
"grafana/k6:2.3.0@sha256:"
"9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34"
)
APP_IMAGE = "mtusi-messenger:bench"
NETWORK = "mtusi-bench"
APP = "mtusi-bench-api"
K6 = "mtusi-bench-k6"
READY_CHECK = (
"import urllib.request; urllib.request.urlopen("
"'http://127.0.0.1:8080/health/ready', timeout=2)"
)
SUCCESS_METRIC = "http_req_duration{expected_response:true}"
# предупреждение о хосте: swap занят больше, МиБ
SWAP_WARN_MIB = 1024
RUN_FIELDS = [
"run",
"scenario",
"size",
"concurrency",
"repeat",
"duration_s",
"requests_ok",
"requests_failed",
"rps",
"error_ratio",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"app_cpu_cores",
"rss_start_mib",
"rss_end_mib",
"rss_max_mib",
"mem_cgroup_end_mib",
"store_messages_start",
"store_messages_end",
"generator_cpu_cores",
"flags",
"check_failed",
]
PREFILL_FIELDS = [
"size",
"chunk",
"messages_before",
"messages_after",
"duration_s",
"requests_ok",
"requests_failed",
"rps",
"error_ratio",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"app_cpu_cores",
"rss_end_mib",
"mem_cgroup_end_mib",
"generator_cpu_cores",
"flags",
]
@dataclass(frozen=True)
class Profile:
name: str
sizes: tuple[int, ...]
concurrency: tuple[int, ...]
repeats: int
read_warmup_s: int
read_measure_s: int
write_warmup: int
write_measure: int
prefill_chunk: int = 50_000
prefill_vus: int = 32
text_length: int = 64
page_limit: int = 50
walk_limit: int = 100
cursor_pool: int = 1000
members: int = 10
seed: int = 20260930
# зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно
# те страницы, с которых начнётся замер
warmup_seed_shift: int = 1_000_003
generator_cpu_limit: float = 4.0
cpus: str = "1"
memory: str = "2g"
PROFILES = {
"full": Profile(
name="full",
sizes=(10_000, 100_000, 1_000_000),
concurrency=(1, 10, 50),
repeats=3,
read_warmup_s=5,
read_measure_s=20,
write_warmup=500,
write_measure=5000,
),
"quick": Profile(
name="quick",
sizes=(1_000, 5_000, 10_000),
concurrency=(1, 4, 16),
repeats=1,
read_warmup_s=2,
read_measure_s=5,
write_warmup=100,
write_measure=1000,
),
}
class BenchError(Exception):
"""Стенд не может продолжать: сообщение для человека."""
# ---------------------------------------------------------------- утилиты
def sh(*args: str, check: bool = True) -> str:
"""Команда без shell; stdout без хвостовых пробелов.
`LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод
разбирается регулярками с точкой.
"""
proc = subprocess.run(
args,
capture_output=True,
text=True,
check=False,
env={**os.environ, "LC_ALL": "C"},
)
if check and proc.returncode != 0:
raise BenchError(
f"{' '.join(args[:4])}…: exit {proc.returncode}\n"
f"{(proc.stdout + proc.stderr)[-2000:]}",
)
return proc.stdout.strip()
def human(n: int) -> str:
"""10000 → 10k, 1000000 → 1M."""
if n >= 1_000_000 and n % 1_000_000 == 0:
return f"{n // 1_000_000}M"
if n >= 1000 and n % 1000 == 0:
return f"{n // 1000}k"
return str(n)
def fmt_left(seconds: float) -> str:
seconds = max(0, int(seconds))
hours, rest = divmod(seconds, 3600)
minutes, secs = divmod(rest, 60)
if hours:
return f"{hours}h{minutes:02d}m"
if minutes:
return f"{minutes}m{secs:02d}s"
return f"{secs}s"
def mib(kib: float | None) -> float | None:
return None if kib is None else round(kib / 1024, 1)
def mib_bytes(value: int | None) -> float | None:
return None if value is None else round(value / 2**20, 1)
def fmt(value: float | None, digits: int = 0) -> str:
return "—" if value is None else f"{value:.{digits}f}"
def rounded(value: float | None, digits: int = 3) -> float | None:
return None if value is None else round(value, digits)
# ------------------------------------------------------------ хост и паспорт
def host_state() -> dict[str, Any]:
"""Swap, свободная память и питание (macOS); на другой ОС — null."""
state: dict[str, Any] = {
"swap_used_mib": None,
"swap_total_mib": None,
"memory_free_percent": None,
"power": None,
"battery_percent": None,
}
if platform.system() != "Darwin":
return state
swap = sh("sysctl", "-n", "vm.swapusage", check=False)
for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")):
match = re.search(rf"{key} = ([\d.,]+)M", swap)
if match:
state[name] = float(match.group(1).replace(",", "."))
pressure = sh("memory_pressure", check=False)
match = re.search(r"free percentage: (\d+)%", pressure)
if match:
state["memory_free_percent"] = int(match.group(1))
batt = sh("pmset", "-g", "batt", check=False)
match = re.search(r"drawing from '([^']+)'", batt)
if match:
state["power"] = match.group(1)
match = re.search(r"(\d+)%;", batt)
if match:
state["battery_percent"] = int(match.group(1))
return state
def host_warnings(state: dict[str, Any]) -> list[str]:
warnings = []
if state["power"] == "Battery Power":
warnings.append("host on battery power: CPU may be throttled")
used = state["swap_used_mib"]
if used is not None and used > SWAP_WARN_MIB:
warnings.append(
f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: "
"reboot before a full run",
)
return warnings
def git_info() -> dict[str, Any]:
commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD")
status = sh("git", "-C", str(ROOT), "status", "--porcelain")
return {"commit": commit, "dirty": bool(status)}
def static_passport(profile: Profile) -> dict[str, Any]:
docker = json.loads(sh("docker", "info", "--format", "{{json .}}"))
versions = sh(
"docker",
"run",
"--rm",
"--entrypoint",
"python",
APP_IMAGE,
"-c",
"import importlib.metadata as m, json, platform; print(json.dumps("
"{'python': platform.python_version(), **{p: m.version(p) for p in "
"('fastapi', 'starlette', 'pydantic', 'uvicorn')}}))",
)
k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version")
host: dict[str, Any] = {"os": platform.system(), "machine": None}
if platform.system() == "Darwin":
host["os"] = f"macOS {platform.mac_ver()[0]}"
host["machine"] = sh(
"sysctl",
"-n",
"machdep.cpu.brand_string",
check=False,
)
memsize = sh("sysctl", "-n", "hw.memsize", check=False)
host["memory_gib"] = (
round(int(memsize) / 2**30, 1) if memsize.isdigit() else None
)
return {
"app_image": {
"tag": APP_IMAGE,
"id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE),
"versions": json.loads(versions),
},
"generator": {"image": K6_IMAGE, "version": k6_version},
"docker": {
"server": docker.get("ServerVersion"),
"vm_os": docker.get("OperatingSystem"),
"vm_cpus": docker.get("NCPU"),
"vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1),
"cgroup": docker.get("CgroupVersion"),
"logging_driver": docker.get("LoggingDriver"),
},
"host": host,
"profile": asdict(profile),
}
# ------------------------------------------------------------------ стенд
class Watcher:
"""Раз в секунду, пока идёт запуск k6: VmRSS приложения и жив ли оно.
Приложение умерло (OOM) — k6 останавливается: иначе каждый его запрос
ждал бы таймаута, и наполнение тянулось бы часами.
"""
def __init__(self) -> None:
self.samples: list[int] = []
self.app_died = False
self._stop = threading.Event()
self._thread = threading.Thread(target=self._loop, daemon=True)
def _loop(self) -> None:
while not self._stop.wait(1.0):
out = sh(
"docker",
"exec",
APP,
"grep",
"^VmRSS:",
"/proc/1/status",
check=False,
)
match = re.search(r"(\d+) kB", out)
if match:
self.samples.append(int(match.group(1)))
continue
running = sh(
"docker",
"inspect",
"-f",
"{{.State.Running}}",
APP,
check=False,
)
if running != "true":
self.app_died = True
sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False)
return
def __enter__(self) -> "Watcher":
self._thread.start()
return self
def __exit__(self, *_: object) -> None:
self._stop.set()
self._thread.join()
class Stand:
"""Сеть, контейнер приложения и контейнер k6 на время одного размера."""
def __init__(self, profile: Profile, label_dir: Path) -> None:
self.profile = profile
self.label_dir = label_dir
self.work_dir = label_dir / ".work"
def check_free(self) -> None:
names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split()
busy = [name for name in (APP, K6) if name in names]
if busy:
raise BenchError(
f"containers {', '.join(busy)} already exist: another bench "
"is running or a crashed one left them; remove with "
f"docker rm -f {' '.join(busy)}",
)
def build_image(self) -> str:
sh(
"docker",
"build",
"-q",
"-t",
APP_IMAGE,
str(ROOT),
)
return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE)
def start(self) -> None:
self.work_dir.mkdir(parents=True, exist_ok=True)
sh("docker", "network", "create", NETWORK)
sh(
"docker",
"run",
"-d",
"--name",
APP,
"--network",
NETWORK,
f"--cpus={self.profile.cpus}",
f"--memory={self.profile.memory}",
f"--memory-swap={self.profile.memory}",
"--no-healthcheck",
APP_IMAGE,
)
sh(
"docker",
"run",
"-d",
"--name",
K6,
"--network",
NETWORK,
"-v",
f"{ROOT / 'bench' / 'k6'}:/scripts:ro",
"-v",
f"{self.label_dir}:/out",
"-v",
f"{self.work_dir}:/work",
"--entrypoint",
"sleep",
K6_IMAGE,
"infinity",
)
self.wait_ready()
def wait_ready(self, timeout: float = 60) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
probe = subprocess.run(
["docker", "exec", APP, "python", "-c", READY_CHECK],
capture_output=True,
check=False,
)
if probe.returncode == 0:
return
state = self.app_state()
if not state["running"]:
reason = "oom" if state["oom"] else "exited"
raise BenchError(f"{APP} died during start ({reason})")
time.sleep(0.25)
raise BenchError(f"{APP} not ready in {timeout:.0f}s")
def stop(self) -> None:
subprocess.run(
["docker", "rm", "-f", APP, K6],
capture_output=True,
check=False,
)
subprocess.run(
["docker", "network", "rm", NETWORK],
capture_output=True,
check=False,
)
def limits(self) -> dict[str, Any]:
host = json.loads(
sh("docker", "inspect", "-f", "{{json .HostConfig}}", APP),
)
config = json.loads(
sh("docker", "inspect", "-f", "{{json .Config}}", APP),
)
return {
"nano_cpus": host.get("NanoCpus"),
"memory": host.get("Memory"),
"memory_swap": host.get("MemorySwap"),
"published_ports": host.get("PortBindings") or {},
"healthcheck": (config.get("Healthcheck") or {}).get("Test"),
"log_driver": (host.get("LogConfig") or {}).get("Type"),
}
def app_state(self) -> dict[str, Any]:
state = json.loads(
sh("docker", "inspect", "-f", "{{json .State}}", APP),
)
restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", APP)
return {
"running": state.get("Running"),
"oom": state.get("OOMKilled"),
"restarts": int(restarts or 0),
}
def snapshot(self) -> dict[str, Any]:
"""CPU и память приложения, CPU генератора.
Если приложение уже мертво (OOM), его поля — None: снимок нужен и
для записи проваленного прогона.
"""
app = sh(
"docker",
"exec",
APP,
"sh",
"-c",
"cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; "
"grep -E '^(VmRSS|VmHWM):' /proc/1/status",
check=False,
)
k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat")
def number(pattern: str, text: str) -> int | None:
match = re.search(pattern, text, re.MULTILINE)
return int(match.group(1)) if match else None
return {
"app_usec": number(r"^usage_usec (\d+)$", app),
"mem_bytes": number(r"^(\d+)$", app),
"rss_kib": number(r"^VmRSS:\s+(\d+) kB", app),
"hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app),
"k6_usec": number(r"^usage_usec (\d+)$", k6),
}
def k6(
self,
script: str,
env: dict[str, Any],
*,
check: bool = True,
) -> subprocess.CompletedProcess[str]:
common = {
"TEXT_LENGTH": self.profile.text_length,
"PAGE_LIMIT": self.profile.page_limit,
}
flags = [
item
for key, value in {**common, **env}.items()
for item in ("-e", f"{key}={value}")
]
proc = subprocess.run(
[
"docker",
"exec",
K6,
"k6",
"run",
"--quiet",
"--no-color",
"--log-format",
"raw",
*flags,
f"/scripts/{script}",
],
capture_output=True,
text=True,
check=False,
)
if check and proc.returncode != 0:
raise BenchError(k6_failure(script, proc))
return proc
def work_json(self, name: str) -> Any:
return json.loads((self.work_dir / name).read_text(encoding="utf-8"))
# -------------------------------------------------------------- измерение
def parse_summary(path: Path) -> dict[str, Any]:
"""Итог k6: успехи, ошибки, RPS и перцентили успешных запросов.
Нет файла (k6 остановлен сторожем до итога) — нулевой итог.
"""
if not path.exists():
return {
"duration_s": 0,
"requests_ok": 0,
"requests_failed": 0,
"check_failed": 0,
"rps": None,
"error_ratio": None,
"p50_ms": None,
"p95_ms": None,
"p99_ms": None,
"max_ms": None,
}
data = json.loads(path.read_text(encoding="utf-8"))
metrics = data["metrics"]
failed = metrics.get("http_req_failed", {}).get("values", {})
ok = int(failed.get("fails", 0))
bad = int(failed.get("passes", 0))
checks = metrics.get("checks", {}).get("values", {})
success = metrics.get(SUCCESS_METRIC, {}).get("values", {})
duration = data["state"]["testRunDurationMs"] / 1000
return {
"duration_s": round(duration, 3),
"requests_ok": ok,
"requests_failed": bad,
"check_failed": int(checks.get("fails", 0)),
"rps": round(ok / duration, 1) if duration else None,
"error_ratio": round(bad / (ok + bad), 6) if ok + bad else None,
"p50_ms": rounded(success.get("p(50)")),
"p95_ms": rounded(success.get("p(95)")),
"p99_ms": rounded(success.get("p(99)")),
"max_ms": rounded(success.get("max")),
}
@dataclass
class Measured:
summary: dict[str, Any]
before: dict[str, Any]
after: dict[str, Any]
rss_max_kib: int | None
def cores(self, key: str) -> float | None:
"""Среднее число занятых ядер за прогон по счётчику cgroup."""
duration = self.summary["duration_s"]
start, end = self.before[key], self.after[key]
if not duration or start is None or end is None:
return None
return round((end - start) / 1e6 / duration, 3)
@property
def app_cpu(self) -> float | None:
return self.cores("app_usec")
@property
def generator_cpu(self) -> float | None:
return self.cores("k6_usec")
def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str:
tail = (proc.stdout + proc.stderr)[-3000:]
return f"k6 {script} failed ({proc.returncode}):\n{tail}"
def measure(
stand: Stand,
script: str,
env: dict[str, Any],
raw: Path,
) -> Measured:
"""Один запуск k6 со снимками до и после, опросом RSS и сторожем."""
before = stand.snapshot()
with Watcher() as watcher:
proc = stand.k6(script, env, check=False)
# если приложение умерло, k6 остановлен сторожем: прогон пишется с
# пометкой, а не падает здесь
if proc.returncode != 0 and not watcher.app_died:
raise BenchError(k6_failure(script, proc))
after = stand.snapshot()
values = [
v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v
]
return Measured(
summary=parse_summary(raw),
before=before,
after=after,
rss_max_kib=max(values) if values else None,
)
def run_flags(
profile: Profile,
measured: Measured,
app: dict[str, Any],
) -> list[str]:
flags = []
if app["oom"]:
flags.append("oom")
if not app["running"] or app["restarts"]:
flags.append("app_down")
summary = measured.summary
if summary["requests_failed"] or summary["check_failed"]:
flags.append("errors")
cpu = measured.generator_cpu
if cpu is not None and cpu >= profile.generator_cpu_limit:
flags.append("generator")
return flags
# ------------------------------------------------------------ результаты
class Results:
"""Файлы метки: runs.csv, prefill.csv, meta.json, progress.json."""
def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None:
self.dir = label_dir
self.meta = meta
self.runs: list[dict[str, Any]] = []
self.prefill: list[dict[str, Any]] = []
self.sizes: list[dict[str, Any]] = []
(label_dir / "raw").mkdir(parents=True)
self.write_meta()
def append(
self,
name: str,
fields: list[str],
row: dict[str, Any],
) -> None:
path = self.dir / name
new = not path.exists()
with path.open("a", newline="", encoding="utf-8") as file:
writer = csv.DictWriter(
file,
fieldnames=fields,
lineterminator="\n",
)
if new:
writer.writeheader()
writer.writerow({key: row.get(key) for key in fields})
file.flush()
def add_run(self, row: dict[str, Any]) -> None:
self.runs.append(row)
self.append("runs.csv", RUN_FIELDS, row)
def add_prefill(self, row: dict[str, Any]) -> None:
self.prefill.append(row)
self.append("prefill.csv", PREFILL_FIELDS, row)
def write_meta(self) -> None:
self.meta["sizes"] = self.sizes
text = json.dumps(self.meta, ensure_ascii=False, indent=2)
(self.dir / "meta.json").write_text(text + "\n", encoding="utf-8")
def write_progress(self, progress: dict[str, Any]) -> None:
progress["updated_at"] = datetime.now(UTC).isoformat(
timespec="seconds",
)
text = json.dumps(progress, ensure_ascii=False, indent=2)
(self.dir / "progress.json").write_text(text + "\n", encoding="utf-8")
def write_summary(self) -> None:
text = build_summary(self.meta, self.runs, self.prefill, self.sizes)
(self.dir / "summary.md").write_text(text, encoding="utf-8")
def cell(values: list[float | None], digits: int = 1) -> str:
"""Медиана и min–max повторов; один повтор — просто значение."""
present = [v for v in values if v is not None]
if not present:
return "—"
median = statistics.median(present)
if len(present) == 1:
return f"{median:.{digits}f}"
return (
f"{median:.{digits}f} ({min(present):.{digits}f}–"
f"{max(present):.{digits}f})"
)
def build_summary(
meta: dict[str, Any],
runs: list[dict[str, Any]],
prefill: list[dict[str, Any]],
sizes: list[dict[str, Any]],
) -> str:
profile = meta["profile"]
lines = [
f"# Нагрузочный замер: метка `{meta['label']}`",
"",
f"- Статус: **{meta['status']}**"
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
f"- Commit: `{meta['git']['commit']}`"
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
f"- Профиль `{profile['name']}`: размеры "
f"{', '.join(human(s) for s in profile['sizes'])}; клиенты "
f"{', '.join(map(str, profile['concurrency']))}; повторов "
f"{profile['repeats']}; чтение {profile['read_measure_s']} с после "
f"{profile['read_warmup_s']} с прогрева; запись "
f"{profile['write_measure']} после {profile['write_warmup']}",
f"- Лимиты приложения: {profile['cpus']} CPU, {profile['memory']} "
"без swap; значения — медиана (min–max) по повторам",
"- Посторонние контейнеры в VM во время замера: "
+ (", ".join(meta.get("other_containers") or []) or "нет"),
"",
]
for scenario in ("read", "write"):
rows = [r for r in runs if r["scenario"] == scenario]
if not rows:
continue
lines += [
f"## Сценарий `{scenario}`",
"",
"| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | "
"max, мс | CPU, ядер | RSS конец, МиБ | Сообщений в начале | "
"Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|",
]
cells = sorted({(r["size"], r["concurrency"]) for r in rows})
for size, conc in cells:
group = [
r
for r in rows
if r["size"] == size and r["concurrency"] == conc
]
flags = sorted({f for r in group for f in r["flags"].split()})
lines.append(
f"| {human(size)} | {conc} | "
f"{cell([r['rps'] for r in group], 0)} | "
f"{cell([r['p50_ms'] for r in group], 2)} | "
f"{cell([r['p95_ms'] for r in group], 2)} | "
f"{cell([r['p99_ms'] for r in group], 2)} | "
f"{cell([r['max_ms'] for r in group], 1)} | "
f"{cell([r['app_cpu_cores'] for r in group], 2)} | "
f"{cell([r['rss_end_mib'] for r in group], 1)} | "
f"{cell([r['store_messages_start'] for r in group], 0)} | "
f"{' '.join(flags) or '—'} |",
)
lines.append("")
if sizes:
lines += [
"## Память и наполнение по размерам",
"",
"| Размер | RSS до, МиБ | RSS после, МиБ | Байт на сообщение | "
"Наполнение, сообщ/с | Курсоров в пуле | Пик RSS за размер, МиБ |",
"|---|---|---|---|---|---|---|",
]
for info in sizes:
lines.append(
f"| {human(info['size'])} | {info['rss_before_mib']} | "
f"{info['rss_after_mib']} | {info['bytes_per_message']} | "
f"{info['prefill_rate']} | {info['cursors']} | "
f"{info.get('rss_peak_mib', '—')} |",
)
lines.append("")
if prefill:
largest = max(r["size"] for r in prefill)
curve = [r for r in prefill if r["size"] == largest]
lines += [
f"## Кривая наполнения {human(largest)}: запись при росте "
f"хранилища, {profile['prefill_vus']} клиента",
"",
"| Порция | Сообщений после | RPS | p50, мс | p95, мс | "
"p99, мс | max, мс | CPU, ядер | RSS, МиБ | Пометки |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for r in curve:
lines.append(
f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | "
f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | "
f"{r['max_ms']} | {r['app_cpu_cores']} | {r['rss_end_mib']} "
f"| {r['flags'] or '—'} |",
)
lines.append("")
if meta.get("host_warnings"):
lines += ["## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
lines.append("")
return "\n".join(lines)
# ---------------------------------------------------------------- прогон
class Bench:
def __init__(
self,
profile: Profile,
label: str,
results: Results,
) -> None:
self.profile = profile
self.label = label
self.results = results
self.stand = Stand(profile, results.dir)
self.started = time.monotonic()
self.runs_total = (
len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats
)
self.runs_done = 0
self.stage = "start"
self.size: int | None = None
# скорости стенда для оценки оставшегося времени
self.prefill_rate = 1500.0
self.write_rps: dict[int, float] = {}
self.run_overhead = 3.0
self.done_sizes = 0
self.prefilled = 0
# текущий размер наполнен и курсоры собраны
self.prepared = False
# ------------------------------------------------------------ вывод
def say(self, text: str) -> None:
elapsed = int(time.monotonic() - self.started)
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
print(f"[{stamp}] {text}", flush=True)
def eta(self) -> float:
p = self.profile
left = 0.0
for index, size in enumerate(p.sizes):
current = index == self.done_sizes
if index < self.done_sizes or (current and self.prepared):
continue
filled = self.prefilled if current else 0
left += (size - filled) / self.prefill_rate + size / 100 / 500
left += 15
per_size = len(p.concurrency) * p.repeats
reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes)
done_in_size = self.runs_done - self.done_sizes * per_size * 2
reads_left -= min(done_in_size, per_size)
writes_left -= max(0, done_in_size - per_size)
read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead
left += reads_left * read_time
rps = (
statistics.mean(self.write_rps.values())
if self.write_rps
else 1000
)
write_time = (p.write_warmup + p.write_measure) / rps
left += writes_left * (write_time + self.run_overhead)
return left
def progress(self, last: dict[str, Any] | None = None) -> None:
state = {
"label": self.label,
"profile": self.profile.name,
"stage": self.stage,
"size": self.size,
"runs_done": self.runs_done,
"runs_total": self.runs_total,
"elapsed_s": round(time.monotonic() - self.started),
"eta_s": round(self.eta()),
"warnings": self.results.meta.get("host_warnings", []),
}
if last is not None:
state["last_run"] = last
self.results.write_progress(state)
# --------------------------------------------------------- этапы
def run(self) -> None:
for index, size in enumerate(self.profile.sizes, start=1):
self.size = size
self.prefilled = 0
self.prepared = False
self.run_size(index, size)
self.done_sizes += 1
def run_size(self, index: int, size: int) -> None:
stand = self.stand
self.stage = f"size {human(size)}: start"
stand.start()
limits = stand.limits()
self.results.meta.setdefault("limits", limits)
if limits != self.results.meta["limits"]:
raise BenchError(f"limits changed between sizes: {limits}")
rss_start = stand.snapshot()["rss_kib"]
self.say(
f"size {human(size)} ({index}/{len(self.profile.sizes)}): "
f"app started, RSS {mib(rss_start)} MiB",
)
stand.k6("prepare.js", {"MEMBERS": self.profile.members})
self.progress()
rss_before = stand.snapshot()["rss_kib"]
prefill_seconds = self.run_prefill(size)
after = stand.snapshot()
rss_after = after["rss_kib"]
stand.k6(
"cursors.js",
{
"WALK_LIMIT": self.profile.walk_limit,
"POOL_SIZE": self.profile.cursor_pool,
},
)
pool = stand.work_json("cursors.json")
if pool["walked"] != size:
raise BenchError(
f"read chat has {pool['walked']} messages, expected {size}",
)
info = {
"size": size,
"rss_before_mib": mib(rss_before),
"rss_after_mib": mib(rss_after),
"bytes_per_message": round(
(rss_after - rss_before) * 1024 / size,
),
"prefill_seconds": round(prefill_seconds, 1),
"prefill_rate": round(size / prefill_seconds),
"cursors": len(pool["cursors"]),
}
self.results.sizes.append(info)
self.results.write_meta()
self.prepared = True
self.say(
f"size {human(size)}: prefill done, RSS {info['rss_after_mib']} "
f"MiB (+{info['bytes_per_message']} B/msg); "
f"cursors {info['cursors']}",
)
store = size
for scenario in ("read", "write"):
for conc in self.profile.concurrency:
for repeat in range(1, self.profile.repeats + 1):
store = self.run_one(scenario, size, conc, repeat, store)
if scenario == "read":
self.check_read_chat(size)
info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"])
self.results.write_meta()
stand.stop()
def check_read_chat(self, size: int) -> None:
"""После блока чтения в читаемом чате по-прежнему ровно N.
Новое сообщение встало бы в конец истории, поэтому достаточно
дочитать хвост от последнего курсора пула.
"""
self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit})
tail = self.stand.work_json("tail.json")
total = tail["from_after"] + tail["tail"]
if total != size:
raise BenchError(
f"read chat has {total} messages after the read block, "
f"expected {size}",
)
self.say(f"size {human(size)}: read chat unchanged, {total} messages")
def run_prefill(self, size: int) -> float:
chunk_size = self.profile.prefill_chunk
total_seconds = 0.0
for chunk, offset in enumerate(range(0, size, chunk_size), start=1):
count = min(chunk_size, size - offset)
name = f"prefill-{human(size)}-{chunk:02d}"
self.stage = f"size {human(size)}: prefill"
measured = measure(
self.stand,
"prefill.js",
{
"COUNT": count,
"OFFSET": offset,
"VUS": self.profile.prefill_vus,
"NAME": name,
},
self.results.dir / "raw" / f"{name}.json",
)
summary = measured.summary
flags = run_flags(self.profile, measured, self.stand.app_state())
self.results.add_prefill(
{
**summary,
"size": size,
"chunk": chunk,
"messages_before": offset,
"messages_after": offset + summary["requests_ok"],
"app_cpu_cores": measured.app_cpu,
"rss_end_mib": mib(measured.after["rss_kib"]),
"mem_cgroup_end_mib": mib_bytes(
measured.after["mem_bytes"],
),
"generator_cpu_cores": measured.generator_cpu,
"flags": " ".join(flags),
},
)
if summary["requests_ok"] != count:
raise BenchError(
f"prefill chunk {chunk}: {summary['requests_ok']} of "
f"{count} messages confirmed ({' '.join(flags)})",
)
total_seconds += summary["duration_s"]
self.prefilled = offset + count
self.prefill_rate = self.prefilled / total_seconds
self.say(
f"size {human(size)}: prefill {self.prefilled}/{size} "
f"({self.prefilled * 100 // size}%), "
f"{self.prefill_rate:.0f} msg/s, "
f"~{fmt_left((size - self.prefilled) / self.prefill_rate)} "
"left",
)
self.progress()
return total_seconds
def run_one(
self,
scenario: str,
size: int,
conc: int,
repeat: int,
store: int,
) -> int:
"""Прогрев и измеряемый прогон; возвращает сообщений в хранилище."""
p = self.profile
number = self.runs_done + 1
name = f"{scenario}-{human(size)}-c{conc}-r{repeat}"
self.stage = f"run {number}/{self.runs_total} {name}"
self.progress()
wall = time.monotonic()
if scenario == "read":
warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"}
self.stand.k6(
"read.js",
{**warm, "SEED": p.seed + p.warmup_seed_shift},
)
env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"}
env["SEED"] = p.seed
else:
self.stand.k6(
"write.js",
{"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store},
)
warm_ok = parse_summary(self.stand.work_dir / "last.json")
store += warm_ok["requests_ok"]
env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store}
measured = measure(
self.stand,
f"{scenario}.js",
{**env, "NAME": name},
self.results.dir / "raw" / f"{name}.json",
)
summary = measured.summary
app = self.stand.app_state()
flags = run_flags(p, measured, app)
end = store + (summary["requests_ok"] if scenario == "write" else 0)
row = {
**summary,
"run": number,
"scenario": scenario,
"size": size,
"concurrency": conc,
"repeat": repeat,
"app_cpu_cores": measured.app_cpu,
"rss_start_mib": mib(measured.before["rss_kib"]),
"rss_end_mib": mib(measured.after["rss_kib"]),
"rss_max_mib": mib(measured.rss_max_kib),
"mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]),
"store_messages_start": store,
"store_messages_end": end,
"generator_cpu_cores": measured.generator_cpu,
"flags": " ".join(flags),
}
self.results.add_run(row)
self.runs_done += 1
if scenario == "write" and summary["rps"]:
self.write_rps[conc] = summary["rps"]
overhead = time.monotonic() - wall - summary["duration_s"]
if scenario == "read":
overhead -= p.read_warmup_s
self.run_overhead = max(1.0, overhead)
errors = summary["requests_failed"] + summary["check_failed"]
self.say(
f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} "
f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps "
f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} "
f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} "
f"rss {row['rss_end_mib']} MiB err {errors} "
f"~{fmt_left(self.eta())} left"
+ (f" [{' '.join(flags)}]" if flags else ""),
)
self.progress(last=row)
if "oom" in flags or "app_down" in flags:
raise BenchError(
f"{APP} is down after {name} ({' '.join(flags)}): "
"matrix stopped",
)
return end
# ------------------------------------------------------------------ CLI
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--profile", choices=sorted(PROFILES), required=True)
parser.add_argument("--label", required=True)
# отладка стенда: переопределения пишутся в паспорт как overrides
parser.add_argument("--sizes", help="через запятую, например 120000")
parser.add_argument("--concurrency", help="через запятую")
parser.add_argument("--repeats", type=int)
parser.add_argument("--memory", help="лимит памяти, например 64m")
return parser.parse_args(argv)
def apply_overrides(
profile: Profile,
args: argparse.Namespace,
) -> tuple[Profile, dict[str, Any]]:
overrides: dict[str, Any] = {}
if args.sizes:
overrides["sizes"] = tuple(int(x) for x in args.sizes.split(","))
if args.concurrency:
overrides["concurrency"] = tuple(
int(x) for x in args.concurrency.split(",")
)
if args.repeats:
overrides["repeats"] = args.repeats
if args.memory:
overrides["memory"] = args.memory
return replace(profile, **overrides), overrides
def main(argv: list[str]) -> int:
args = parse_args(argv)
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label):
print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True)
return 2
label_dir = ROOT / "evidence" / "bench" / args.label
relative = label_dir.relative_to(ROOT)
if label_dir.exists():
print(f"label directory {relative} already exists", flush=True)
return 2
profile, overrides = apply_overrides(PROFILES[args.profile], args)
git = git_info()
state = host_state()
warnings = host_warnings(state)
meta: dict[str, Any] = {
"label": args.label,
"status": "incomplete",
"reason": "running",
"started_at": datetime.now(UTC).isoformat(timespec="seconds"),
"command": " ".join(["uv run python bench/run.py", *argv]),
"git": git,
"overrides": overrides,
"host_state": {"start": state},
"host_warnings": warnings,
# чужие контейнеры в той же VM: стенд их не трогает, но они шумят
"other_containers": sorted(
sh("docker", "ps", "--format", "{{.Names}}", check=False).split(),
),
}
stand_check = Stand(profile, label_dir)
started = time.monotonic()
def say(text: str) -> None:
elapsed = int(time.monotonic() - started)
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
print(f"[{stamp}] {text}", flush=True)
for warning in warnings:
say(f"WARNING: {warning}")
if overrides:
say(f"WARNING: profile overrides {overrides}: debug run")
if git["dirty"]:
say("NOTE: working tree has uncommitted changes (dirty: true)")
try:
stand_check.check_free()
image = stand_check.build_image()
except BenchError as exc:
print(f"bench failed: {exc}", flush=True)
return 1
say(f"image {APP_IMAGE} built ({image[:19]}…), network {NETWORK}")
meta.update(static_passport(profile))
results = Results(label_dir, meta)
bench = Bench(profile, args.label, results)
bench.started = started
code = 0
def terminate(*_: object) -> None:
raise KeyboardInterrupt
signal.signal(signal.SIGTERM, terminate)
try:
bench.run()
meta["status"] = "complete"
meta.pop("reason", None)
except KeyboardInterrupt:
meta["reason"] = "interrupted"
say("interrupted: partial results kept")
code = 130
except BenchError as exc:
meta["reason"] = str(exc).splitlines()[0]
say(f"bench failed: {exc}")
code = 1
finally:
# повторный Ctrl+C не должен оборвать уборку стенда
signal.signal(signal.SIGINT, signal.SIG_IGN)
bench.stand.stop()
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds")
meta["host_state"]["end"] = host_state()
results.write_meta()
results.write_summary()
bench.stage = "done" if code == 0 else meta.get("reason", "stopped")
bench.progress()
say(
f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, "
f"results in {relative}",
)
return code
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))