"""Нагрузочный стенд трека 1: матрица размеров истории и конкуренции. Запуск — `make bench` / `make bench_quick`. Методика — design change `load-bench-harness`, инварианты — `.claude/rules/bench.md`. Только stdlib: приложение не импортируется, всё через docker CLI и k6. """ import argparse import csv import json import os import platform import re import shutil import signal import statistics import subprocess import sys import threading import time from dataclasses import asdict, dataclass, replace from datetime import UTC, datetime from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] K6_IMAGE = ( "grafana/k6:2.3.0@sha256:" "9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34" ) APP_IMAGE = "mtusi-messenger:bench" NETWORK = "mtusi-bench" APP = "mtusi-bench-api" K6 = "mtusi-bench-k6" READY_CHECK = ( "import urllib.request; urllib.request.urlopen(" "'http://127.0.0.1:8080/health/ready', timeout=2)" ) SUCCESS_METRIC = "http_req_duration{expected_response:true}" # предупреждение о хосте: swap занят больше, МиБ SWAP_WARN_MIB = 1024 RUN_FIELDS = [ "run", "scenario", "size", "concurrency", "repeat", "duration_s", "requests_ok", "requests_failed", "rps", "error_ratio", "p50_ms", "p95_ms", "p99_ms", "max_ms", "app_cpu_cores", "rss_start_mib", "rss_end_mib", "rss_max_mib", "mem_cgroup_end_mib", "store_messages_start", "store_messages_end", "generator_cpu_cores", "flags", "check_failed", ] PREFILL_FIELDS = [ "size", "chunk", "messages_before", "messages_after", "duration_s", "requests_ok", "requests_failed", "rps", "error_ratio", "p50_ms", "p95_ms", "p99_ms", "max_ms", "app_cpu_cores", "rss_end_mib", "mem_cgroup_end_mib", "generator_cpu_cores", "flags", ] @dataclass(frozen=True) class Profile: name: str sizes: tuple[int, ...] concurrency: tuple[int, ...] repeats: int read_warmup_s: int read_measure_s: int write_warmup: int write_measure: int prefill_chunk: int = 50_000 prefill_vus: int = 32 text_length: int = 64 page_limit: int = 50 walk_limit: int = 100 cursor_pool: int = 1000 members: int = 10 seed: int = 20260930 # зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно # те страницы, с которых начнётся замер warmup_seed_shift: int = 1_000_003 generator_cpu_limit: float = 4.0 cpus: str = "1" memory: str = "2g" PROFILES = { "full": Profile( name="full", sizes=(10_000, 100_000, 1_000_000), concurrency=(1, 10, 50), repeats=3, read_warmup_s=5, read_measure_s=20, write_warmup=500, write_measure=5000, ), "quick": Profile( name="quick", sizes=(1_000, 5_000, 10_000), concurrency=(1, 4, 16), repeats=1, read_warmup_s=2, read_measure_s=5, write_warmup=100, write_measure=1000, ), } class BenchError(Exception): """Стенд не может продолжать: сообщение для человека.""" # ---------------------------------------------------------------- утилиты def sh(*args: str, check: bool = True) -> str: """Команда без shell; stdout без хвостовых пробелов. `LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод разбирается регулярками с точкой. """ proc = subprocess.run( args, capture_output=True, text=True, check=False, env={**os.environ, "LC_ALL": "C"}, ) if check and proc.returncode != 0: raise BenchError( f"{' '.join(args[:4])}…: exit {proc.returncode}\n" f"{(proc.stdout + proc.stderr)[-2000:]}", ) return proc.stdout.strip() def human(n: int) -> str: """10000 → 10k, 1000000 → 1M.""" if n >= 1_000_000 and n % 1_000_000 == 0: return f"{n // 1_000_000}M" if n >= 1000 and n % 1000 == 0: return f"{n // 1000}k" return str(n) def fmt_left(seconds: float) -> str: seconds = max(0, int(seconds)) hours, rest = divmod(seconds, 3600) minutes, secs = divmod(rest, 60) if hours: return f"{hours}h{minutes:02d}m" if minutes: return f"{minutes}m{secs:02d}s" return f"{secs}s" def mib(kib: float | None) -> float | None: return None if kib is None else round(kib / 1024, 1) def mib_bytes(value: int | None) -> float | None: return None if value is None else round(value / 2**20, 1) def fmt(value: float | None, digits: int = 0) -> str: return "—" if value is None else f"{value:.{digits}f}" def rounded(value: float | None, digits: int = 3) -> float | None: return None if value is None else round(value, digits) # ------------------------------------------------------------ хост и паспорт def host_state() -> dict[str, Any]: """Swap, свободная память и питание (macOS); на другой ОС — null.""" state: dict[str, Any] = { "swap_used_mib": None, "swap_total_mib": None, "memory_free_percent": None, "power": None, "battery_percent": None, } if platform.system() != "Darwin": return state swap = sh("sysctl", "-n", "vm.swapusage", check=False) for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")): match = re.search(rf"{key} = ([\d.,]+)M", swap) if match: state[name] = float(match.group(1).replace(",", ".")) pressure = sh("memory_pressure", check=False) match = re.search(r"free percentage: (\d+)%", pressure) if match: state["memory_free_percent"] = int(match.group(1)) batt = sh("pmset", "-g", "batt", check=False) match = re.search(r"drawing from '([^']+)'", batt) if match: state["power"] = match.group(1) match = re.search(r"(\d+)%;", batt) if match: state["battery_percent"] = int(match.group(1)) return state def host_warnings(state: dict[str, Any]) -> list[str]: warnings = [] if state["power"] == "Battery Power": warnings.append("host on battery power: CPU may be throttled") used = state["swap_used_mib"] if used is not None and used > SWAP_WARN_MIB: warnings.append( f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: " "reboot before a full run", ) return warnings def git_info() -> dict[str, Any]: commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD") status = sh("git", "-C", str(ROOT), "status", "--porcelain") return {"commit": commit, "dirty": bool(status)} def static_passport(profile: Profile) -> dict[str, Any]: docker = json.loads(sh("docker", "info", "--format", "{{json .}}")) versions = sh( "docker", "run", "--rm", "--entrypoint", "python", APP_IMAGE, "-c", "import importlib.metadata as m, json, platform; print(json.dumps(" "{'python': platform.python_version(), **{p: m.version(p) for p in " "('fastapi', 'starlette', 'pydantic', 'uvicorn')}}))", ) k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version") host: dict[str, Any] = {"os": platform.system(), "machine": None} if platform.system() == "Darwin": host["os"] = f"macOS {platform.mac_ver()[0]}" host["machine"] = sh( "sysctl", "-n", "machdep.cpu.brand_string", check=False, ) memsize = sh("sysctl", "-n", "hw.memsize", check=False) host["memory_gib"] = ( round(int(memsize) / 2**30, 1) if memsize.isdigit() else None ) return { "app_image": { "tag": APP_IMAGE, "id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE), "versions": json.loads(versions), }, "generator": {"image": K6_IMAGE, "version": k6_version}, "docker": { "server": docker.get("ServerVersion"), "vm_os": docker.get("OperatingSystem"), "vm_cpus": docker.get("NCPU"), "vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1), "cgroup": docker.get("CgroupVersion"), "logging_driver": docker.get("LoggingDriver"), }, "host": host, "profile": asdict(profile), } # ------------------------------------------------------------------ стенд class Watcher: """Раз в секунду, пока идёт запуск k6: VmRSS приложения и жив ли оно. Приложение умерло (OOM) — k6 останавливается: иначе каждый его запрос ждал бы таймаута, и наполнение тянулось бы часами. """ def __init__(self) -> None: self.samples: list[int] = [] self.app_died = False self._stop = threading.Event() self._thread = threading.Thread(target=self._loop, daemon=True) def _loop(self) -> None: while not self._stop.wait(1.0): out = sh( "docker", "exec", APP, "grep", "^VmRSS:", "/proc/1/status", check=False, ) match = re.search(r"(\d+) kB", out) if match: self.samples.append(int(match.group(1))) continue running = sh( "docker", "inspect", "-f", "{{.State.Running}}", APP, check=False, ) if running != "true": self.app_died = True sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False) return def __enter__(self) -> "Watcher": self._thread.start() return self def __exit__(self, *_: object) -> None: self._stop.set() self._thread.join() class Stand: """Сеть, контейнер приложения и контейнер k6 на время одного размера.""" def __init__(self, profile: Profile, label_dir: Path) -> None: self.profile = profile self.label_dir = label_dir self.work_dir = label_dir / ".work" def check_free(self) -> None: names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split() busy = [name for name in (APP, K6) if name in names] if busy: raise BenchError( f"containers {', '.join(busy)} already exist: another bench " "is running or a crashed one left them; remove with " f"docker rm -f {' '.join(busy)}", ) def build_image(self) -> str: sh( "docker", "build", "-q", "-t", APP_IMAGE, str(ROOT), ) return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE) def start(self) -> None: self.work_dir.mkdir(parents=True, exist_ok=True) sh("docker", "network", "create", NETWORK) sh( "docker", "run", "-d", "--name", APP, "--network", NETWORK, f"--cpus={self.profile.cpus}", f"--memory={self.profile.memory}", f"--memory-swap={self.profile.memory}", "--no-healthcheck", APP_IMAGE, ) sh( "docker", "run", "-d", "--name", K6, "--network", NETWORK, "-v", f"{ROOT / 'bench' / 'k6'}:/scripts:ro", "-v", f"{self.label_dir}:/out", "-v", f"{self.work_dir}:/work", "--entrypoint", "sleep", K6_IMAGE, "infinity", ) self.wait_ready() def wait_ready(self, timeout: float = 60) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: probe = subprocess.run( ["docker", "exec", APP, "python", "-c", READY_CHECK], capture_output=True, check=False, ) if probe.returncode == 0: return state = self.app_state() if not state["running"]: reason = "oom" if state["oom"] else "exited" raise BenchError(f"{APP} died during start ({reason})") time.sleep(0.25) raise BenchError(f"{APP} not ready in {timeout:.0f}s") def stop(self) -> None: subprocess.run( ["docker", "rm", "-f", APP, K6], capture_output=True, check=False, ) subprocess.run( ["docker", "network", "rm", NETWORK], capture_output=True, check=False, ) def limits(self) -> dict[str, Any]: host = json.loads( sh("docker", "inspect", "-f", "{{json .HostConfig}}", APP), ) config = json.loads( sh("docker", "inspect", "-f", "{{json .Config}}", APP), ) return { "nano_cpus": host.get("NanoCpus"), "memory": host.get("Memory"), "memory_swap": host.get("MemorySwap"), "published_ports": host.get("PortBindings") or {}, "healthcheck": (config.get("Healthcheck") or {}).get("Test"), "log_driver": (host.get("LogConfig") or {}).get("Type"), } def app_state(self) -> dict[str, Any]: state = json.loads( sh("docker", "inspect", "-f", "{{json .State}}", APP), ) restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", APP) return { "running": state.get("Running"), "oom": state.get("OOMKilled"), "restarts": int(restarts or 0), } def snapshot(self) -> dict[str, Any]: """CPU и память приложения, CPU генератора. Если приложение уже мертво (OOM), его поля — None: снимок нужен и для записи проваленного прогона. """ app = sh( "docker", "exec", APP, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; " "grep -E '^(VmRSS|VmHWM):' /proc/1/status", check=False, ) k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat") def number(pattern: str, text: str) -> int | None: match = re.search(pattern, text, re.MULTILINE) return int(match.group(1)) if match else None return { "app_usec": number(r"^usage_usec (\d+)$", app), "mem_bytes": number(r"^(\d+)$", app), "rss_kib": number(r"^VmRSS:\s+(\d+) kB", app), "hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app), "k6_usec": number(r"^usage_usec (\d+)$", k6), } def k6( self, script: str, env: dict[str, Any], *, check: bool = True, ) -> subprocess.CompletedProcess[str]: common = { "TEXT_LENGTH": self.profile.text_length, "PAGE_LIMIT": self.profile.page_limit, } flags = [ item for key, value in {**common, **env}.items() for item in ("-e", f"{key}={value}") ] proc = subprocess.run( [ "docker", "exec", K6, "k6", "run", "--quiet", "--no-color", "--log-format", "raw", *flags, f"/scripts/{script}", ], capture_output=True, text=True, check=False, ) if check and proc.returncode != 0: raise BenchError(k6_failure(script, proc)) return proc def work_json(self, name: str) -> Any: return json.loads((self.work_dir / name).read_text(encoding="utf-8")) # -------------------------------------------------------------- измерение def parse_summary(path: Path) -> dict[str, Any]: """Итог k6: успехи, ошибки, RPS и перцентили успешных запросов. Нет файла (k6 остановлен сторожем до итога) — нулевой итог. """ if not path.exists(): return { "duration_s": 0, "requests_ok": 0, "requests_failed": 0, "check_failed": 0, "rps": None, "error_ratio": None, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, } data = json.loads(path.read_text(encoding="utf-8")) metrics = data["metrics"] failed = metrics.get("http_req_failed", {}).get("values", {}) ok = int(failed.get("fails", 0)) bad = int(failed.get("passes", 0)) checks = metrics.get("checks", {}).get("values", {}) success = metrics.get(SUCCESS_METRIC, {}).get("values", {}) duration = data["state"]["testRunDurationMs"] / 1000 return { "duration_s": round(duration, 3), "requests_ok": ok, "requests_failed": bad, "check_failed": int(checks.get("fails", 0)), "rps": round(ok / duration, 1) if duration else None, "error_ratio": round(bad / (ok + bad), 6) if ok + bad else None, "p50_ms": rounded(success.get("p(50)")), "p95_ms": rounded(success.get("p(95)")), "p99_ms": rounded(success.get("p(99)")), "max_ms": rounded(success.get("max")), } @dataclass class Measured: summary: dict[str, Any] before: dict[str, Any] after: dict[str, Any] rss_max_kib: int | None def cores(self, key: str) -> float | None: """Среднее число занятых ядер за прогон по счётчику cgroup.""" duration = self.summary["duration_s"] start, end = self.before[key], self.after[key] if not duration or start is None or end is None: return None return round((end - start) / 1e6 / duration, 3) @property def app_cpu(self) -> float | None: return self.cores("app_usec") @property def generator_cpu(self) -> float | None: return self.cores("k6_usec") def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str: tail = (proc.stdout + proc.stderr)[-3000:] return f"k6 {script} failed ({proc.returncode}):\n{tail}" def measure( stand: Stand, script: str, env: dict[str, Any], raw: Path, ) -> Measured: """Один запуск k6 со снимками до и после, опросом RSS и сторожем.""" before = stand.snapshot() with Watcher() as watcher: proc = stand.k6(script, env, check=False) # если приложение умерло, k6 остановлен сторожем: прогон пишется с # пометкой, а не падает здесь if proc.returncode != 0 and not watcher.app_died: raise BenchError(k6_failure(script, proc)) after = stand.snapshot() values = [ v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v ] return Measured( summary=parse_summary(raw), before=before, after=after, rss_max_kib=max(values) if values else None, ) def run_flags( profile: Profile, measured: Measured, app: dict[str, Any], ) -> list[str]: flags = [] if app["oom"]: flags.append("oom") if not app["running"] or app["restarts"]: flags.append("app_down") summary = measured.summary if summary["requests_failed"] or summary["check_failed"]: flags.append("errors") cpu = measured.generator_cpu if cpu is not None and cpu >= profile.generator_cpu_limit: flags.append("generator") return flags # ------------------------------------------------------------ результаты class Results: """Файлы метки: runs.csv, prefill.csv, meta.json, progress.json.""" def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None: self.dir = label_dir self.meta = meta self.runs: list[dict[str, Any]] = [] self.prefill: list[dict[str, Any]] = [] self.sizes: list[dict[str, Any]] = [] (label_dir / "raw").mkdir(parents=True) self.write_meta() def append( self, name: str, fields: list[str], row: dict[str, Any], ) -> None: path = self.dir / name new = not path.exists() with path.open("a", newline="", encoding="utf-8") as file: writer = csv.DictWriter( file, fieldnames=fields, lineterminator="\n", ) if new: writer.writeheader() writer.writerow({key: row.get(key) for key in fields}) file.flush() def add_run(self, row: dict[str, Any]) -> None: self.runs.append(row) self.append("runs.csv", RUN_FIELDS, row) def add_prefill(self, row: dict[str, Any]) -> None: self.prefill.append(row) self.append("prefill.csv", PREFILL_FIELDS, row) def write_meta(self) -> None: self.meta["sizes"] = self.sizes text = json.dumps(self.meta, ensure_ascii=False, indent=2) (self.dir / "meta.json").write_text(text + "\n", encoding="utf-8") def write_progress(self, progress: dict[str, Any]) -> None: progress["updated_at"] = datetime.now(UTC).isoformat( timespec="seconds", ) text = json.dumps(progress, ensure_ascii=False, indent=2) (self.dir / "progress.json").write_text(text + "\n", encoding="utf-8") def write_summary(self) -> None: text = build_summary(self.meta, self.runs, self.prefill, self.sizes) (self.dir / "summary.md").write_text(text, encoding="utf-8") def cell(values: list[float | None], digits: int = 1) -> str: """Медиана и min–max повторов; один повтор — просто значение.""" present = [v for v in values if v is not None] if not present: return "—" median = statistics.median(present) if len(present) == 1: return f"{median:.{digits}f}" return ( f"{median:.{digits}f} ({min(present):.{digits}f}–" f"{max(present):.{digits}f})" ) def build_summary( meta: dict[str, Any], runs: list[dict[str, Any]], prefill: list[dict[str, Any]], sizes: list[dict[str, Any]], ) -> str: profile = meta["profile"] lines = [ f"# Нагрузочный замер: метка `{meta['label']}`", "", f"- Статус: **{meta['status']}**" + (f" ({meta['reason']})" if meta.get("reason") else ""), f"- Commit: `{meta['git']['commit']}`" + (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""), f"- Профиль `{profile['name']}`: размеры " f"{', '.join(human(s) for s in profile['sizes'])}; клиенты " f"{', '.join(map(str, profile['concurrency']))}; повторов " f"{profile['repeats']}; чтение {profile['read_measure_s']} с после " f"{profile['read_warmup_s']} с прогрева; запись " f"{profile['write_measure']} после {profile['write_warmup']}", f"- Лимиты приложения: {profile['cpus']} CPU, {profile['memory']} " "без swap; значения — медиана (min–max) по повторам", "- Посторонние контейнеры в VM во время замера: " + (", ".join(meta.get("other_containers") or []) or "нет"), "", ] for scenario in ("read", "write"): rows = [r for r in runs if r["scenario"] == scenario] if not rows: continue lines += [ f"## Сценарий `{scenario}`", "", "| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | " "max, мс | CPU, ядер | RSS конец, МиБ | Сообщений в начале | " "Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|", ] cells = sorted({(r["size"], r["concurrency"]) for r in rows}) for size, conc in cells: group = [ r for r in rows if r["size"] == size and r["concurrency"] == conc ] flags = sorted({f for r in group for f in r["flags"].split()}) lines.append( f"| {human(size)} | {conc} | " f"{cell([r['rps'] for r in group], 0)} | " f"{cell([r['p50_ms'] for r in group], 2)} | " f"{cell([r['p95_ms'] for r in group], 2)} | " f"{cell([r['p99_ms'] for r in group], 2)} | " f"{cell([r['max_ms'] for r in group], 1)} | " f"{cell([r['app_cpu_cores'] for r in group], 2)} | " f"{cell([r['rss_end_mib'] for r in group], 1)} | " f"{cell([r['store_messages_start'] for r in group], 0)} | " f"{' '.join(flags) or '—'} |", ) lines.append("") if sizes: lines += [ "## Память и наполнение по размерам", "", "| Размер | RSS до, МиБ | RSS после, МиБ | Байт на сообщение | " "Наполнение, сообщ/с | Курсоров в пуле | Пик RSS за размер, МиБ |", "|---|---|---|---|---|---|---|", ] for info in sizes: lines.append( f"| {human(info['size'])} | {info['rss_before_mib']} | " f"{info['rss_after_mib']} | {info['bytes_per_message']} | " f"{info['prefill_rate']} | {info['cursors']} | " f"{info.get('rss_peak_mib', '—')} |", ) lines.append("") if prefill: largest = max(r["size"] for r in prefill) curve = [r for r in prefill if r["size"] == largest] lines += [ f"## Кривая наполнения {human(largest)}: запись при росте " f"хранилища, {profile['prefill_vus']} клиента", "", "| Порция | Сообщений после | RPS | p50, мс | p95, мс | " "p99, мс | max, мс | CPU, ядер | RSS, МиБ | Пометки |", "|---|---|---|---|---|---|---|---|---|---|", ] for r in curve: lines.append( f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | " f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | " f"{r['max_ms']} | {r['app_cpu_cores']} | {r['rss_end_mib']} " f"| {r['flags'] or '—'} |", ) lines.append("") if meta.get("host_warnings"): lines += ["## Предупреждения о хосте", ""] lines += [f"- {w}" for w in meta["host_warnings"]] lines.append("") return "\n".join(lines) # ---------------------------------------------------------------- прогон class Bench: def __init__( self, profile: Profile, label: str, results: Results, ) -> None: self.profile = profile self.label = label self.results = results self.stand = Stand(profile, results.dir) self.started = time.monotonic() self.runs_total = ( len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats ) self.runs_done = 0 self.stage = "start" self.size: int | None = None # скорости стенда для оценки оставшегося времени self.prefill_rate = 1500.0 self.write_rps: dict[int, float] = {} self.run_overhead = 3.0 self.done_sizes = 0 self.prefilled = 0 # текущий размер наполнен и курсоры собраны self.prepared = False # ------------------------------------------------------------ вывод def say(self, text: str) -> None: elapsed = int(time.monotonic() - self.started) stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed)) print(f"[{stamp}] {text}", flush=True) def eta(self) -> float: p = self.profile left = 0.0 for index, size in enumerate(p.sizes): current = index == self.done_sizes if index < self.done_sizes or (current and self.prepared): continue filled = self.prefilled if current else 0 left += (size - filled) / self.prefill_rate + size / 100 / 500 left += 15 per_size = len(p.concurrency) * p.repeats reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes) done_in_size = self.runs_done - self.done_sizes * per_size * 2 reads_left -= min(done_in_size, per_size) writes_left -= max(0, done_in_size - per_size) read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead left += reads_left * read_time rps = ( statistics.mean(self.write_rps.values()) if self.write_rps else 1000 ) write_time = (p.write_warmup + p.write_measure) / rps left += writes_left * (write_time + self.run_overhead) return left def progress(self, last: dict[str, Any] | None = None) -> None: state = { "label": self.label, "profile": self.profile.name, "stage": self.stage, "size": self.size, "runs_done": self.runs_done, "runs_total": self.runs_total, "elapsed_s": round(time.monotonic() - self.started), "eta_s": round(self.eta()), "warnings": self.results.meta.get("host_warnings", []), } if last is not None: state["last_run"] = last self.results.write_progress(state) # --------------------------------------------------------- этапы def run(self) -> None: for index, size in enumerate(self.profile.sizes, start=1): self.size = size self.prefilled = 0 self.prepared = False self.run_size(index, size) self.done_sizes += 1 def run_size(self, index: int, size: int) -> None: stand = self.stand self.stage = f"size {human(size)}: start" stand.start() limits = stand.limits() self.results.meta.setdefault("limits", limits) if limits != self.results.meta["limits"]: raise BenchError(f"limits changed between sizes: {limits}") rss_start = stand.snapshot()["rss_kib"] self.say( f"size {human(size)} ({index}/{len(self.profile.sizes)}): " f"app started, RSS {mib(rss_start)} MiB", ) stand.k6("prepare.js", {"MEMBERS": self.profile.members}) self.progress() rss_before = stand.snapshot()["rss_kib"] prefill_seconds = self.run_prefill(size) after = stand.snapshot() rss_after = after["rss_kib"] stand.k6( "cursors.js", { "WALK_LIMIT": self.profile.walk_limit, "POOL_SIZE": self.profile.cursor_pool, }, ) pool = stand.work_json("cursors.json") if pool["walked"] != size: raise BenchError( f"read chat has {pool['walked']} messages, expected {size}", ) info = { "size": size, "rss_before_mib": mib(rss_before), "rss_after_mib": mib(rss_after), "bytes_per_message": round( (rss_after - rss_before) * 1024 / size, ), "prefill_seconds": round(prefill_seconds, 1), "prefill_rate": round(size / prefill_seconds), "cursors": len(pool["cursors"]), } self.results.sizes.append(info) self.results.write_meta() self.prepared = True self.say( f"size {human(size)}: prefill done, RSS {info['rss_after_mib']} " f"MiB (+{info['bytes_per_message']} B/msg); " f"cursors {info['cursors']}", ) store = size for scenario in ("read", "write"): for conc in self.profile.concurrency: for repeat in range(1, self.profile.repeats + 1): store = self.run_one(scenario, size, conc, repeat, store) if scenario == "read": self.check_read_chat(size) info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"]) self.results.write_meta() stand.stop() def check_read_chat(self, size: int) -> None: """После блока чтения в читаемом чате по-прежнему ровно N. Новое сообщение встало бы в конец истории, поэтому достаточно дочитать хвост от последнего курсора пула. """ self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit}) tail = self.stand.work_json("tail.json") total = tail["from_after"] + tail["tail"] if total != size: raise BenchError( f"read chat has {total} messages after the read block, " f"expected {size}", ) self.say(f"size {human(size)}: read chat unchanged, {total} messages") def run_prefill(self, size: int) -> float: chunk_size = self.profile.prefill_chunk total_seconds = 0.0 for chunk, offset in enumerate(range(0, size, chunk_size), start=1): count = min(chunk_size, size - offset) name = f"prefill-{human(size)}-{chunk:02d}" self.stage = f"size {human(size)}: prefill" measured = measure( self.stand, "prefill.js", { "COUNT": count, "OFFSET": offset, "VUS": self.profile.prefill_vus, "NAME": name, }, self.results.dir / "raw" / f"{name}.json", ) summary = measured.summary flags = run_flags(self.profile, measured, self.stand.app_state()) self.results.add_prefill( { **summary, "size": size, "chunk": chunk, "messages_before": offset, "messages_after": offset + summary["requests_ok"], "app_cpu_cores": measured.app_cpu, "rss_end_mib": mib(measured.after["rss_kib"]), "mem_cgroup_end_mib": mib_bytes( measured.after["mem_bytes"], ), "generator_cpu_cores": measured.generator_cpu, "flags": " ".join(flags), }, ) if summary["requests_ok"] != count: raise BenchError( f"prefill chunk {chunk}: {summary['requests_ok']} of " f"{count} messages confirmed ({' '.join(flags)})", ) total_seconds += summary["duration_s"] self.prefilled = offset + count self.prefill_rate = self.prefilled / total_seconds self.say( f"size {human(size)}: prefill {self.prefilled}/{size} " f"({self.prefilled * 100 // size}%), " f"{self.prefill_rate:.0f} msg/s, " f"~{fmt_left((size - self.prefilled) / self.prefill_rate)} " "left", ) self.progress() return total_seconds def run_one( self, scenario: str, size: int, conc: int, repeat: int, store: int, ) -> int: """Прогрев и измеряемый прогон; возвращает сообщений в хранилище.""" p = self.profile number = self.runs_done + 1 name = f"{scenario}-{human(size)}-c{conc}-r{repeat}" self.stage = f"run {number}/{self.runs_total} {name}" self.progress() wall = time.monotonic() if scenario == "read": warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"} self.stand.k6( "read.js", {**warm, "SEED": p.seed + p.warmup_seed_shift}, ) env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"} env["SEED"] = p.seed else: self.stand.k6( "write.js", {"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store}, ) warm_ok = parse_summary(self.stand.work_dir / "last.json") store += warm_ok["requests_ok"] env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store} measured = measure( self.stand, f"{scenario}.js", {**env, "NAME": name}, self.results.dir / "raw" / f"{name}.json", ) summary = measured.summary app = self.stand.app_state() flags = run_flags(p, measured, app) end = store + (summary["requests_ok"] if scenario == "write" else 0) row = { **summary, "run": number, "scenario": scenario, "size": size, "concurrency": conc, "repeat": repeat, "app_cpu_cores": measured.app_cpu, "rss_start_mib": mib(measured.before["rss_kib"]), "rss_end_mib": mib(measured.after["rss_kib"]), "rss_max_mib": mib(measured.rss_max_kib), "mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]), "store_messages_start": store, "store_messages_end": end, "generator_cpu_cores": measured.generator_cpu, "flags": " ".join(flags), } self.results.add_run(row) self.runs_done += 1 if scenario == "write" and summary["rps"]: self.write_rps[conc] = summary["rps"] overhead = time.monotonic() - wall - summary["duration_s"] if scenario == "read": overhead -= p.read_warmup_s self.run_overhead = max(1.0, overhead) errors = summary["requests_failed"] + summary["check_failed"] self.say( f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} " f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps " f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} " f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} " f"rss {row['rss_end_mib']} MiB err {errors} " f"~{fmt_left(self.eta())} left" + (f" [{' '.join(flags)}]" if flags else ""), ) self.progress(last=row) if "oom" in flags or "app_down" in flags: raise BenchError( f"{APP} is down after {name} ({' '.join(flags)}): " "matrix stopped", ) return end # ------------------------------------------------------------------ CLI def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--profile", choices=sorted(PROFILES), required=True) parser.add_argument("--label", required=True) # отладка стенда: переопределения пишутся в паспорт как overrides parser.add_argument("--sizes", help="через запятую, например 120000") parser.add_argument("--concurrency", help="через запятую") parser.add_argument("--repeats", type=int) parser.add_argument("--memory", help="лимит памяти, например 64m") return parser.parse_args(argv) def apply_overrides( profile: Profile, args: argparse.Namespace, ) -> tuple[Profile, dict[str, Any]]: overrides: dict[str, Any] = {} if args.sizes: overrides["sizes"] = tuple(int(x) for x in args.sizes.split(",")) if args.concurrency: overrides["concurrency"] = tuple( int(x) for x in args.concurrency.split(",") ) if args.repeats: overrides["repeats"] = args.repeats if args.memory: overrides["memory"] = args.memory return replace(profile, **overrides), overrides def main(argv: list[str]) -> int: args = parse_args(argv) if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label): print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True) return 2 label_dir = ROOT / "evidence" / "bench" / args.label relative = label_dir.relative_to(ROOT) if label_dir.exists(): print(f"label directory {relative} already exists", flush=True) return 2 profile, overrides = apply_overrides(PROFILES[args.profile], args) git = git_info() state = host_state() warnings = host_warnings(state) meta: dict[str, Any] = { "label": args.label, "status": "incomplete", "reason": "running", "started_at": datetime.now(UTC).isoformat(timespec="seconds"), "command": " ".join(["uv run python bench/run.py", *argv]), "git": git, "overrides": overrides, "host_state": {"start": state}, "host_warnings": warnings, # чужие контейнеры в той же VM: стенд их не трогает, но они шумят "other_containers": sorted( sh("docker", "ps", "--format", "{{.Names}}", check=False).split(), ), } stand_check = Stand(profile, label_dir) started = time.monotonic() def say(text: str) -> None: elapsed = int(time.monotonic() - started) stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed)) print(f"[{stamp}] {text}", flush=True) for warning in warnings: say(f"WARNING: {warning}") if overrides: say(f"WARNING: profile overrides {overrides}: debug run") if git["dirty"]: say("NOTE: working tree has uncommitted changes (dirty: true)") try: stand_check.check_free() image = stand_check.build_image() except BenchError as exc: print(f"bench failed: {exc}", flush=True) return 1 say(f"image {APP_IMAGE} built ({image[:19]}…), network {NETWORK}") meta.update(static_passport(profile)) results = Results(label_dir, meta) bench = Bench(profile, args.label, results) bench.started = started code = 0 def terminate(*_: object) -> None: raise KeyboardInterrupt signal.signal(signal.SIGTERM, terminate) try: bench.run() meta["status"] = "complete" meta.pop("reason", None) except KeyboardInterrupt: meta["reason"] = "interrupted" say("interrupted: partial results kept") code = 130 except BenchError as exc: meta["reason"] = str(exc).splitlines()[0] say(f"bench failed: {exc}") code = 1 finally: # повторный Ctrl+C не должен оборвать уборку стенда signal.signal(signal.SIGINT, signal.SIG_IGN) bench.stand.stop() shutil.rmtree(bench.stand.work_dir, ignore_errors=True) meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds") meta["host_state"]["end"] = host_state() results.write_meta() results.write_summary() bench.stage = "done" if code == 0 else meta.get("reason", "stopped") bench.progress() say( f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, " f"results in {relative}", ) return code if __name__ == "__main__": sys.exit(main(sys.argv[1:]))