"""Нагрузочный стенд: матрица размеров истории и конкуренции. Запуск — `make bench` / `make bench_quick`. Стенд — API, PostgreSQL и Valkey: проект compose `mtusi-bench` из `docker-compose.yml` с override `bench/bench.compose.yml`. Методика — design changes `load-bench-harness` и `bench-postgres`, инварианты — `.claude/rules/bench.md`. Только stdlib: приложение не импортируется, его нагружает k6 по HTTP; к PostgreSQL стенда оркестратор ходит только для обслуживания и измерений (VACUUM, размеры, планы запроса). """ import argparse import csv import json import os import platform import re import secrets import shutil import signal import statistics import subprocess import sys import threading import time import uuid from collections.abc import Callable from dataclasses import asdict, dataclass, replace from datetime import UTC, datetime from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] K6_IMAGE = ( "grafana/k6:2.3.0@sha256:" "9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34" ) APP_IMAGE = "mtusi-messenger:bench" PROJECT = "mtusi-bench" NETWORK = "mtusi-bench" APP = "mtusi-bench-api" DB = "mtusi-bench-postgres" CACHE = "mtusi-bench-valkey" K6 = "mtusi-bench-k6" # контейнеры стенда по ролям: лимиты, состояние, паспорт CONTAINERS = {"api": APP, "postgres": DB, "valkey": CACHE} COMPOSE_FILES = ( ROOT / "docker-compose.yml", ROOT / "bench" / "bench.compose.yml", ) # подстановки compose вместо .env рабочего стенда ENV_FILE = ROOT / "bench" / "bench.env" # что из окружения хоста нужно самому docker compose; остальное (в том # числе POSTGRES_* из shell) в стенд не попадает HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "DOCKER_HOST", "DOCKER_CONFIG") READY_CHECK = ( "import urllib.request; urllib.request.urlopen(" "'http://127.0.0.1:8080/health/ready', timeout=2)" ) SUCCESS_METRIC = "http_req_duration{expected_response:true}" # предупреждение о хосте: swap занят больше, МиБ SWAP_WARN_MIB = 1024 # предупреждение о VM: лимиты памяти стенда + этот запас больше памяти VM VM_RESERVE_GIB = 1.5 # psql в контейнере Postgres от пользователя образа, без пароля; SQL — stdin PSQL = ( 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1' ) # настройки Postgres в паспорте: от них зависят план и стоимость записи PG_SETTINGS = ( "shared_buffers", "effective_cache_size", "work_mem", "maintenance_work_mem", "max_connections", "synchronous_commit", "fsync", "wal_level", "checkpoint_timeout", "max_wal_size", "random_page_cost", "jit", "autovacuum", "plan_cache_mode", "shared_preload_libraries", ) # auto_explain на этапе плана; вне его log_min_duration = -1 (выключен) EXPLAIN_SETTINGS = { "auto_explain.log_min_duration": "0", "auto_explain.log_analyze": "on", "auto_explain.log_buffers": "on", "auto_explain.log_timing": "on", "auto_explain.log_format": "text", } # пауза после pg_reload_conf(): SIGHUP доходит до соединений пула API RELOAD_PAUSE_S = 1.0 # страницы этапа плана в порядке запросов explain.js EXPLAIN_PAGES = ("first", "middle", "last") INDEX_NODE_RE = re.compile( r"Index Only Scan(?: Backward)? using (\w+)" r"|Index Scan(?: Backward)? using (\w+)" r"|Bitmap Index Scan on (\w+)", ) RUN_FIELDS = [ "run", "scenario", "size", "concurrency", "repeat", "duration_s", "requests_ok", "requests_failed", "rps", "error_ratio", "p50_ms", "p95_ms", "p99_ms", "max_ms", "app_cpu_cores", "rss_start_mib", "rss_end_mib", "rss_max_mib", "mem_cgroup_end_mib", "store_messages_start", "store_messages_end", "generator_cpu_cores", "flags", "check_failed", # лаба 2: новые столбцы — только в конец "db_cpu_cores", ] PREFILL_FIELDS = [ "size", "chunk", "messages_before", "messages_after", "duration_s", "requests_ok", "requests_failed", "rps", "error_ratio", "p50_ms", "p95_ms", "p99_ms", "max_ms", "app_cpu_cores", "rss_end_mib", "mem_cgroup_end_mib", "generator_cpu_cores", "flags", "db_cpu_cores", ] @dataclass(frozen=True) class Profile: name: str sizes: tuple[int, ...] concurrency: tuple[int, ...] repeats: int read_warmup_s: int read_measure_s: int write_warmup: int write_measure: int # порция ~30 с при ~850 сообщ/с на Postgres: прогресс наполнения # печатается после порции, а спек требует строку не реже раза в минуту prefill_chunk: int = 25_000 prefill_vus: int = 32 text_length: int = 64 page_limit: int = 50 walk_limit: int = 100 cursor_pool: int = 1000 members: int = 10 seed: int = 20260930 # зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно # те страницы, с которых начнётся замер warmup_seed_shift: int = 1_000_003 generator_cpu_limit: float = 4.0 cpus: str = "1" memory: str = "2g" # лаба 2: лимиты хранилищ (swap = память) и срок сессии стенда; идут в # compose переменными BENCH_* (bench/bench.compose.yml). Решение 2 # design change bench-postgres db_cpus: str = "4" db_memory: str = "3g" cache_cpus: str = "2" cache_memory: str = "512m" session_ttl_seconds: int = 21600 PROFILES = { "full": Profile( name="full", # 500k вместо 1M лабы 1: прогон ~30 мин вместо ~45, а диапазон # 10k → 500k (×50) уже показывает, растёт ли страница с историей sizes=(10_000, 100_000, 500_000), concurrency=(1, 10, 50), repeats=3, read_warmup_s=5, read_measure_s=20, write_warmup=500, write_measure=5000, ), "quick": Profile( name="quick", sizes=(1_000, 5_000, 10_000), concurrency=(1, 4, 16), repeats=1, read_warmup_s=2, read_measure_s=5, write_warmup=100, write_measure=1000, ), } class BenchError(Exception): """Стенд не может продолжать: сообщение для человека.""" # ---------------------------------------------------------------- утилиты def sh(*args: str, check: bool = True) -> str: """Команда без shell; stdout без хвостовых пробелов. `LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод разбирается регулярками с точкой. """ proc = subprocess.run( args, capture_output=True, text=True, check=False, env={**os.environ, "LC_ALL": "C"}, ) if check and proc.returncode != 0: raise BenchError( f"{' '.join(args[:4])}…: exit {proc.returncode}\n" f"{(proc.stdout + proc.stderr)[-2000:]}", ) return proc.stdout.strip() def human(n: int) -> str: """10000 → 10k, 1000000 → 1M.""" if n >= 1_000_000 and n % 1_000_000 == 0: return f"{n // 1_000_000}M" if n >= 1000 and n % 1000 == 0: return f"{n // 1000}k" return str(n) def fmt_left(seconds: float) -> str: seconds = max(0, int(seconds)) hours, rest = divmod(seconds, 3600) minutes, secs = divmod(rest, 60) if hours: return f"{hours}h{minutes:02d}m" if minutes: return f"{minutes}m{secs:02d}s" return f"{secs}s" def mib(kib: float | None) -> float | None: return None if kib is None else round(kib / 1024, 1) def mib_bytes(value: int | None) -> float | None: return None if value is None else round(value / 2**20, 1) def size_bytes(text: str) -> int: """Лимит памяти docker (`512m`, `2g`) в байтах.""" match = re.fullmatch(r"(\d+(?:\.\d+)?)([kmg]?)b?", text.strip().lower()) if not match: raise BenchError(f"bad memory limit {text!r}: use 512m or 2g") power = {"": 0, "k": 1, "m": 2, "g": 3}[match.group(2)] return int(float(match.group(1)) * 1024**power) def fmt(value: float | None, digits: int = 0) -> str: return "—" if value is None else f"{value:.{digits}f}" def rounded(value: float | None, digits: int = 3) -> float | None: return None if value is None else round(value, digits) # ------------------------------------------------------------ хост и паспорт def host_state() -> dict[str, Any]: """Swap, свободная память и питание (macOS); на другой ОС — null.""" state: dict[str, Any] = { "swap_used_mib": None, "swap_total_mib": None, "memory_free_percent": None, "power": None, "battery_percent": None, } if platform.system() != "Darwin": return state swap = sh("sysctl", "-n", "vm.swapusage", check=False) for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")): match = re.search(rf"{key} = ([\d.,]+)M", swap) if match: state[name] = float(match.group(1).replace(",", ".")) pressure = sh("memory_pressure", check=False) match = re.search(r"free percentage: (\d+)%", pressure) if match: state["memory_free_percent"] = int(match.group(1)) batt = sh("pmset", "-g", "batt", check=False) match = re.search(r"drawing from '([^']+)'", batt) if match: state["power"] = match.group(1) match = re.search(r"(\d+)%;", batt) if match: state["battery_percent"] = int(match.group(1)) return state def host_warnings(state: dict[str, Any]) -> list[str]: warnings = [] if state["power"] == "Battery Power": warnings.append("host on battery power: CPU may be throttled") used = state["swap_used_mib"] if used is not None and used > SWAP_WARN_MIB: warnings.append( f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: " "reboot before a full run", ) return warnings def vm_warnings(profile: Profile) -> list[str]: """Лимиты памяти стенда не помещаются в VM docker с запасом. Тогда лимит перестаёт быть гарантией: контейнеры делят память VM, и метки разных дней несравнимы. """ total = sh("docker", "info", "--format", "{{.MemTotal}}", check=False) if not total.isdigit(): return [] limits = sum( size_bytes(value) for value in (profile.memory, profile.db_memory, profile.cache_memory) ) budget = int(total) - int(VM_RESERVE_GIB * 2**30) if limits <= budget: return [] return [ f"memory limits {limits / 2**30:.1f} GiB + {VM_RESERVE_GIB} GiB " f"reserve exceed docker VM memory {int(total) / 2**30:.1f} GiB: " "limits are not guaranteed", ] def git_info() -> dict[str, Any]: commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD") status = sh("git", "-C", str(ROOT), "status", "--porcelain") return {"commit": commit, "dirty": bool(status)} def static_passport(profile: Profile) -> dict[str, Any]: docker = json.loads(sh("docker", "info", "--format", "{{json .}}")) versions = sh( "docker", "run", "--rm", "--entrypoint", "python", APP_IMAGE, "-c", "import importlib.metadata as m, json, platform; print(json.dumps(" "{'python': platform.python_version(), **{p: m.version(p) for p in " "('fastapi', 'starlette', 'pydantic', 'uvicorn', 'sqlalchemy', " "'asyncpg', 'redis')}}))", ) k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version") host: dict[str, Any] = {"os": platform.system(), "machine": None} if platform.system() == "Darwin": host["os"] = f"macOS {platform.mac_ver()[0]}" host["machine"] = sh( "sysctl", "-n", "machdep.cpu.brand_string", check=False, ) memsize = sh("sysctl", "-n", "hw.memsize", check=False) host["memory_gib"] = ( round(int(memsize) / 2**30, 1) if memsize.isdigit() else None ) return { "app_image": { "tag": APP_IMAGE, "id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE), "versions": json.loads(versions), }, "generator": {"image": K6_IMAGE, "version": k6_version}, "docker": { "server": docker.get("ServerVersion"), "vm_os": docker.get("OperatingSystem"), "vm_cpus": docker.get("NCPU"), "vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1), "cgroup": docker.get("CgroupVersion"), "logging_driver": docker.get("LoggingDriver"), }, "host": host, "profile": asdict(profile), } # ------------------------------------------------------------------ стенд class Watcher: """Раз в секунду, пока идёт запуск k6: VmRSS приложения, живы ли оно, PostgreSQL и Valkey. Приложение или хранилище умерло (OOM, kill) — k6 останавливается: иначе каждый его запрос ждал бы таймаута, и наполнение тянулось бы часами. """ def __init__(self) -> None: self.samples: list[int] = [] self.app_died = False self.stand_died: list[str] = [] self._stop = threading.Event() self._thread = threading.Thread(target=self._loop, daemon=True) def _halt(self) -> None: sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False) def _loop(self) -> None: while not self._stop.wait(1.0): states = sh( "docker", "inspect", "-f", "{{.Name}} {{.State.Running}}", DB, CACHE, check=False, ) dead = [ name.lstrip("/") for name, running in ( line.split() for line in states.splitlines() if line ) if running != "true" ] if dead: self.stand_died = dead self._halt() return out = sh( "docker", "exec", APP, "grep", "^VmRSS:", "/proc/1/status", check=False, ) match = re.search(r"(\d+) kB", out) if match: self.samples.append(int(match.group(1))) continue running = sh( "docker", "inspect", "-f", "{{.State.Running}}", APP, check=False, ) if running != "true": self.app_died = True self._halt() return def __enter__(self) -> "Watcher": self._thread.start() return self def __exit__(self, *_: object) -> None: self._stop.set() self._thread.join() def compose_env(profile: Profile) -> dict[str, str]: """Окружение `docker compose` стенда, собранное с нуля. Пароли хранилищ — новые на каждый прогон: живут только в этом словаре и в контейнерах стенда, в результаты не пишутся. Лимиты и срок сессии — из профиля (`BENCH_*` в `bench/bench.compose.yml`). """ env = {key: os.environ[key] for key in HOST_ENV if key in os.environ} return { **env, "LC_ALL": "C", "POSTGRES_PASSWORD": secrets.token_urlsafe(24), "VALKEY_PASSWORD": secrets.token_urlsafe(24), "BENCH_APP_CPUS": profile.cpus, "BENCH_APP_MEMORY": profile.memory, "BENCH_DB_CPUS": profile.db_cpus, "BENCH_DB_MEMORY": profile.db_memory, "BENCH_CACHE_CPUS": profile.cache_cpus, "BENCH_CACHE_MEMORY": profile.cache_memory, "BENCH_SESSION_TTL_SECONDS": str(profile.session_ttl_seconds), } def leftovers() -> dict[str, list[str]]: """Контейнеры, volume и сеть стенда, оставшиеся от другого прогона. Ключ — подкоманда docker, которой их удалить. """ project = f"label=com.docker.compose.project={PROJECT}" names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split() containers = sh( "docker", "ps", "-a", "--filter", project, "--format", "{{.Names}}", ).split() networks = sh("docker", "network", "ls", "--format", "{{.Name}}").split() return { "rm -f": containers + ([K6] if K6 in names else []), "volume rm": sh( "docker", "volume", "ls", "-q", "--filter", project, ).split(), "network rm": [NETWORK] if NETWORK in networks else [], } class Stand: """Проект compose `mtusi-bench` и контейнер k6 на время одного размера.""" def __init__(self, profile: Profile, label_dir: Path) -> None: self.profile = profile self.label_dir = label_dir self.work_dir = label_dir / ".work" self.env = compose_env(profile) def compose(self, *args: str, check: bool = True) -> str: """docker compose стенда; без `-p mtusi-bench` не вызывается.""" files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))] proc = subprocess.run( [ "docker", "compose", "-p", PROJECT, "--env-file", str(ENV_FILE), *files, *args, ], capture_output=True, text=True, env=self.env, check=False, ) if check and proc.returncode != 0: raise BenchError( f"docker compose {' '.join(args[:3])}: exit " f"{proc.returncode}\n{(proc.stdout + proc.stderr)[-2000:]}", ) return proc.stdout.strip() def check_free(self) -> None: found = {cmd: names for cmd, names in leftovers().items() if names} if found: fix = "; ".join( f"docker {cmd} {' '.join(names)}" for cmd, names in found.items() ) raise BenchError( "bench stand leftovers exist: another bench is running or a " f"crashed one left them; remove with: {fix}", ) def build_image(self) -> str: sh( "docker", "build", "-q", "-t", APP_IMAGE, str(ROOT), ) return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE) def start(self, say: Callable[[str], None]) -> None: """Свежий стенд: пустая база на новом volume, миграции, API, k6.""" self.work_dir.mkdir(parents=True, exist_ok=True) self.compose("up", "-d", "--wait", "postgres", "valkey") say("postgres and valkey healthy") self.compose("run", "--rm", "migrate") say("migrations applied") self.compose("up", "-d", "--no-deps", "--no-build", "api") sh( "docker", "run", "-d", "--name", K6, "--network", NETWORK, "-v", f"{ROOT / 'bench' / 'k6'}:/scripts:ro", "-v", f"{self.label_dir}:/out", "-v", f"{self.work_dir}:/work", "--entrypoint", "sleep", K6_IMAGE, "infinity", ) self.wait_ready() def wait_ready(self, timeout: float = 60) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: probe = subprocess.run( ["docker", "exec", APP, "python", "-c", READY_CHECK], capture_output=True, check=False, ) if probe.returncode == 0: return state = self.state(APP) if not state["running"]: reason = "oom" if state["oom"] else "exited" raise BenchError(f"{APP} died during start ({reason})") time.sleep(0.25) raise BenchError(f"{APP} not ready in {timeout:.0f}s") def stop(self) -> bool: """k6, контейнеры, сеть и volume стенда; True — ничего не осталось. k6 — первым: подключённый к сети чужой контейнер не дал бы compose удалить сеть. """ subprocess.run( ["docker", "rm", "-f", K6], capture_output=True, check=False, ) self.compose("down", "-v", "--remove-orphans", check=False) return not any(leftovers().values()) def limits(self) -> dict[str, Any]: """Лимиты контейнеров стенда, прочитанные у запущенных контейнеров.""" limits = {} for role, name in CONTAINERS.items(): host = json.loads( sh("docker", "inspect", "-f", "{{json .HostConfig}}", name), ) config = json.loads( sh("docker", "inspect", "-f", "{{json .Config}}", name), ) limits[role] = { "nano_cpus": host.get("NanoCpus"), "memory": host.get("Memory"), "memory_swap": host.get("MemorySwap"), "published_ports": host.get("PortBindings") or {}, "healthcheck": (config.get("Healthcheck") or {}).get("Test"), "log_driver": (host.get("LogConfig") or {}).get("Type"), } return limits def state(self, name: str) -> dict[str, Any]: state = json.loads( sh("docker", "inspect", "-f", "{{json .State}}", name), ) restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", name) return { "running": state.get("Running"), "oom": state.get("OOMKilled"), "restarts": int(restarts or 0), } def down_storages(self) -> list[str]: """Хранилища, которые за прогон умирали, убивались или рестартовали.""" down = [] for name in (DB, CACHE): state = self.state(name) if not state["running"] or state["oom"] or state["restarts"]: down.append(name) return down def psql(self, sql: str, *, check: bool = True) -> str: """SQL в базе стенда через stdin: пользователь контейнера, без пароля.""" proc = subprocess.run( ["docker", "exec", "-i", DB, "sh", "-c", PSQL], input=sql, capture_output=True, text=True, check=False, ) if check and proc.returncode != 0: raise BenchError(f"psql: exit {proc.returncode}\n{proc.stderr}") return proc.stdout.strip() def maintain(self) -> float: """VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды. Статистика и состояние таблиц одинаковы у всех меток, а не зависят от того, когда сработал autovacuum (решение 4). """ started = time.monotonic() self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n") return time.monotonic() - started def storage(self) -> dict[str, Any]: """Объём таблицы сообщений и её индексов, число строк.""" row = self.psql( "SELECT pg_table_size('messages'), pg_indexes_size('messages'), " "count(*) FROM messages;\n", ) table, indexes, count = (int(value) for value in row.split("|")) return { "messages_rows": count, "messages_table_bytes": table, "messages_indexes_bytes": indexes, "db_bytes_per_message": ( round((table + indexes) / count) if count else None ), } def db_log(self) -> str: """Журнал Postgres стенда (stderr контейнера): только в памяти.""" proc = subprocess.run( ["docker", "logs", DB], capture_output=True, text=True, check=False, ) return proc.stdout + proc.stderr def passport(self) -> dict[str, Any]: """Хранилища и окружение API для паспорта; паролей здесь нет.""" settings = self.psql( "SELECT name, current_setting(name) FROM pg_settings " "WHERE name = ANY(string_to_array(" f"'{','.join(PG_SETTINGS)}', ',')) ORDER BY name;\n", ) info = sh( "docker", "exec", CACHE, "sh", "-c", 'exec valkey-cli -p "$VALKEY_PORT" INFO server', ) valkey_version = re.search(r"^valkey_version:(\S+)", info, re.M) memory = sh( "docker", "exec", CACHE, "sh", "-c", 'exec valkey-cli -p "$VALKEY_PORT" CONFIG GET "maxmemory*"', ).splitlines() env = json.loads( sh("docker", "inspect", "-f", "{{json .Config.Env}}", APP), ) app_env = dict( sorted( item.split("=", 1) for item in env if re.match(r"[A-Z0-9_]+__[A-Z0-9_]+=", item) and "PASSWORD" not in item.split("=", 1)[0] ), ) def image(name: str) -> dict[str, str]: return { "image": sh( "docker", "inspect", "-f", "{{.Config.Image}}", name, ), "id": sh("docker", "inspect", "-f", "{{.Image}}", name), } return { "postgres": { **image(DB), "version": self.psql("SHOW server_version;\n"), "settings": dict( line.split("|", 1) for line in settings.splitlines() ), }, "valkey": { **image(CACHE), "version": valkey_version.group(1) if valkey_version else None, **dict(zip(memory[::2], memory[1::2], strict=True)), }, "app_env": app_env, } def snapshot(self) -> dict[str, Any]: """CPU и память приложения, CPU PostgreSQL и генератора. Если приложение уже мертво (OOM), его поля — None: снимок нужен и для записи проваленного прогона. """ app = sh( "docker", "exec", APP, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; " "grep -E '^(VmRSS|VmHWM):' /proc/1/status", check=False, ) k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat") db = sh( "docker", "exec", DB, "cat", "/sys/fs/cgroup/cpu.stat", check=False, ) def number(pattern: str, text: str) -> int | None: match = re.search(pattern, text, re.MULTILINE) return int(match.group(1)) if match else None return { "app_usec": number(r"^usage_usec (\d+)$", app), "mem_bytes": number(r"^(\d+)$", app), "rss_kib": number(r"^VmRSS:\s+(\d+) kB", app), "hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app), "k6_usec": number(r"^usage_usec (\d+)$", k6), "db_usec": number(r"^usage_usec (\d+)$", db), } def k6( self, script: str, env: dict[str, Any], *, check: bool = True, ) -> subprocess.CompletedProcess[str]: common = { "TEXT_LENGTH": self.profile.text_length, "PAGE_LIMIT": self.profile.page_limit, } flags = [ item for key, value in {**common, **env}.items() for item in ("-e", f"{key}={value}") ] proc = subprocess.run( [ "docker", "exec", K6, "k6", "run", "--quiet", "--no-color", "--log-format", "raw", *flags, f"/scripts/{script}", ], capture_output=True, text=True, check=False, ) if check and proc.returncode != 0: raise BenchError(k6_failure(script, proc)) return proc def work_json(self, name: str) -> Any: return json.loads((self.work_dir / name).read_text(encoding="utf-8")) # -------------------------------------------------------------- измерение def parse_summary(path: Path) -> dict[str, Any]: """Итог k6: успехи, ошибки, RPS и перцентили успешных запросов. Нет файла (k6 остановлен сторожем до итога) — нулевой итог. """ if not path.exists(): return { "duration_s": 0, "requests_ok": 0, "requests_failed": 0, "check_failed": 0, "rps": None, "error_ratio": None, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, } data = json.loads(path.read_text(encoding="utf-8")) metrics = data["metrics"] failed = metrics.get("http_req_failed", {}).get("values", {}) ok = int(failed.get("fails", 0)) bad = int(failed.get("passes", 0)) checks = metrics.get("checks", {}).get("values", {}) success = metrics.get(SUCCESS_METRIC, {}).get("values", {}) duration = data["state"]["testRunDurationMs"] / 1000 return { "duration_s": round(duration, 3), "requests_ok": ok, "requests_failed": bad, "check_failed": int(checks.get("fails", 0)), "rps": round(ok / duration, 1) if duration else None, "error_ratio": round(bad / (ok + bad), 6) if ok + bad else None, "p50_ms": rounded(success.get("p(50)")), "p95_ms": rounded(success.get("p(95)")), "p99_ms": rounded(success.get("p(99)")), "max_ms": rounded(success.get("max")), } @dataclass class Measured: summary: dict[str, Any] before: dict[str, Any] after: dict[str, Any] rss_max_kib: int | None def cores(self, key: str) -> float | None: """Среднее число занятых ядер за прогон по счётчику cgroup.""" duration = self.summary["duration_s"] start, end = self.before[key], self.after[key] if not duration or start is None or end is None: return None return round((end - start) / 1e6 / duration, 3) @property def app_cpu(self) -> float | None: return self.cores("app_usec") @property def generator_cpu(self) -> float | None: return self.cores("k6_usec") @property def db_cpu(self) -> float | None: return self.cores("db_usec") def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str: tail = (proc.stdout + proc.stderr)[-3000:] return f"k6 {script} failed ({proc.returncode}):\n{tail}" def measure( stand: Stand, script: str, env: dict[str, Any], raw: Path, ) -> Measured: """Один запуск k6 со снимками до и после, опросом RSS и сторожем.""" before = stand.snapshot() with Watcher() as watcher: proc = stand.k6(script, env, check=False) # если приложение или хранилище умерло, k6 остановлен сторожем: прогон # пишется с пометкой, а не падает здесь died = watcher.app_died or watcher.stand_died if proc.returncode != 0 and not died: raise BenchError(k6_failure(script, proc)) after = stand.snapshot() values = [ v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v ] return Measured( summary=parse_summary(raw), before=before, after=after, rss_max_kib=max(values) if values else None, ) def run_flags( profile: Profile, measured: Measured, app: dict[str, Any], storages_down: list[str], ) -> list[str]: flags = [] if app["oom"]: flags.append("oom") if not app["running"] or app["restarts"]: flags.append("app_down") if storages_down: flags.append("stand_down") summary = measured.summary if summary["requests_failed"] or summary["check_failed"]: flags.append("errors") cpu = measured.generator_cpu if cpu is not None and cpu >= profile.generator_cpu_limit: flags.append("generator") return flags # ----------------------------------------------------------- план запроса def plan_entries(log: str, marker: str) -> list[str]: """Записи auto_explain между маркерами этапа плана. Запись начинается строкой `LOG: duration: … plan:` и продолжается строками с табуляцией; префикс времени и pid отбрасывается. """ lines = log.splitlines() bounds = [ next( (i for i, line in enumerate(lines) if f"{marker} {edge}" in line), None, ) for edge in ("start", "end") ] if None in bounds: return [] entries: list[list[str]] = [] current: list[str] | None = None for line in lines[bounds[0] + 1 : bounds[1]]: if line.startswith("\t"): if current is not None: current.append(line[1:]) continue current = None head = line.partition("LOG: ")[2].rstrip() if head.startswith("duration:") and head.endswith("plan:"): current = [head] entries.append(current) return ["\n".join(entry) for entry in entries] def is_history(entry: str) -> bool: """Запись — запрос страницы истории (читает таблицу messages). Остальные записи окна (проверка сессии с хэшем токена в параметрах, служебные запросы) в результаты не попадают. """ text = entry.partition("Query Parameters:")[0] return "Query Text:" in text and "FROM messages" in text def plan_parts(entry: str) -> dict[str, Any] | None: """Текст запроса, параметры и тело плана записи auto_explain.""" lines = entry.splitlines() starts = [i for i, line in enumerate(lines) if line.startswith("Query ")] if len(starts) != 2 or not lines[starts[1]].startswith("Query Param"): return None text = "\n".join(lines[starts[0] : starts[1]]) params = lines[starts[1]].partition(": ")[2] return { "text": text.removeprefix("Query Text: "), "params": [ value.replace("''", "'") for value in re.findall(r"\$\d+ = '((?:[^']|'')*)'", params) ], "plan": "\n".join(lines[starts[1] + 1 :]), } def plan_summary(plan: str, time_ms: float | None) -> dict[str, Any]: """Узел доступа к messages, буферы верхнего узла, строки, время.""" node = re.search( r"((?:Parallel )?(?:Seq Scan|Index Only Scan|Index Scan|" r"Bitmap Heap Scan)(?: Backward)?(?: using \w+)? on messages)", plan, ) buffers = re.search(r"Buffers: ([^\n]*)", plan) hit = re.search(r"shared hit=(\d+)", buffers.group(1)) if buffers else None read = re.search(r"read=(\d+)", buffers.group(1)) if buffers else None rows = re.search(r"\(actual time=[\d.]+\.\.[\d.]+ rows=([\d.]+)", plan) if time_ms is None: execution = re.search(r"Execution Time: ([\d.]+) ms", plan) time_ms = float(execution.group(1)) if execution else None return { "node": node.group(1) if node else None, "shared_hit": int(hit.group(1)) if hit else 0, "shared_read": int(read.group(1)) if read else 0, "rows": float(rows.group(1)) if rows else None, "time_ms": time_ms, # параметры в условиях плана — generic plan, литералы — custom "kind": "generic" if re.search(r"\$\d", plan) else "custom", } def entry_duration(entry: str) -> float | None: match = re.match(r"duration: ([\d.]+) ms", entry) return float(match.group(1)) if match else None def index_name(plan: str) -> str | None: match = INDEX_NODE_RE.search(plan) return ( next((name for name in match.groups() if name), None) if match else None ) def render_explain( size: int, git: dict[str, Any], positions: dict[str, int], entries: list[str], contrast: dict[str, str] | None, ) -> str: """Файл `explain-<размер>.txt`: планы приложения и пара psql.""" lines = [ f"# План запроса истории: размер {human(size)}", "", f"Commit {git['commit']}" + (" + незакоммиченные изменения" if git["dirty"] else ""), "Страница истории читаемого чата; позиция — сколько сообщений", "истории до страницы:", *(f"- {page}: после {after}" for page, after in positions.items()), "", "## Планы приложения", "", "auto_explain (ANALYZE, BUFFERS, TIMING): запрос, параметры и план,", "с которыми его выполнило приложение (prepared statement asyncpg).", "", ] for page, entry in zip(EXPLAIN_PAGES, entries, strict=True): lines += [f"### {page}", "", entry, ""] if contrast is None: lines += [ "## Средняя страница без индекса", "", "План приложения не использовал индекс: сравнения нет.", "", ] return "\n".join(lines) index = contrast["index"] lines += [ "## Средняя страница в psql: с индексом и без", "", "Тот же текст запроса и те же параметры, что у плана middle.", "psql планирует их заново (custom plan). DROP INDEX — внутри", "транзакции, затем ROLLBACK: индекс остаётся в базе.", "", f"### С индексом {index}", "", contrast["with_index"], "", f"### Без индекса {index}", "", contrast["without_index"], "", ] return "\n".join(lines) # ------------------------------------------------------------ результаты class Results: """Файлы метки: runs.csv, prefill.csv, meta.json, progress.json.""" def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None: self.dir = label_dir self.meta = meta self.runs: list[dict[str, Any]] = [] self.prefill: list[dict[str, Any]] = [] self.sizes: list[dict[str, Any]] = [] (label_dir / "raw").mkdir(parents=True) self.write_meta() def append( self, name: str, fields: list[str], row: dict[str, Any], ) -> None: path = self.dir / name new = not path.exists() with path.open("a", newline="", encoding="utf-8") as file: writer = csv.DictWriter( file, fieldnames=fields, lineterminator="\n", ) if new: writer.writeheader() writer.writerow({key: row.get(key) for key in fields}) file.flush() def add_run(self, row: dict[str, Any]) -> None: self.runs.append(row) self.append("runs.csv", RUN_FIELDS, row) def add_prefill(self, row: dict[str, Any]) -> None: self.prefill.append(row) self.append("prefill.csv", PREFILL_FIELDS, row) def write_meta(self) -> None: self.meta["sizes"] = self.sizes text = json.dumps(self.meta, ensure_ascii=False, indent=2) (self.dir / "meta.json").write_text(text + "\n", encoding="utf-8") def write_progress(self, progress: dict[str, Any]) -> None: progress["updated_at"] = datetime.now(UTC).isoformat( timespec="seconds", ) text = json.dumps(progress, ensure_ascii=False, indent=2) (self.dir / "progress.json").write_text(text + "\n", encoding="utf-8") def write_summary(self) -> None: text = build_summary(self.meta, self.runs, self.prefill, self.sizes) (self.dir / "summary.md").write_text(text, encoding="utf-8") def cell(values: list[float | None], digits: int = 1) -> str: """Медиана и min–max повторов; один повтор — просто значение.""" present = [v for v in values if v is not None] if not present: return "—" median = statistics.median(present) if len(present) == 1: return f"{median:.{digits}f}" return ( f"{median:.{digits}f} ({min(present):.{digits}f}–" f"{max(present):.{digits}f})" ) def build_summary( meta: dict[str, Any], runs: list[dict[str, Any]], prefill: list[dict[str, Any]], sizes: list[dict[str, Any]], ) -> str: profile = meta["profile"] lines = [ f"# Нагрузочный замер: метка `{meta['label']}`", "", f"- Статус: **{meta['status']}**" + (f" ({meta['reason']})" if meta.get("reason") else ""), f"- Commit: `{meta['git']['commit']}`" + (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""), f"- Профиль `{profile['name']}`: размеры " f"{', '.join(human(s) for s in profile['sizes'])}; клиенты " f"{', '.join(map(str, profile['concurrency']))}; повторов " f"{profile['repeats']}; чтение {profile['read_measure_s']} с после " f"{profile['read_warmup_s']} с прогрева; запись " f"{profile['write_measure']} после {profile['write_warmup']}", f"- Лимиты без swap: API {profile['cpus']} CPU / " f"{profile['memory']}, PostgreSQL {profile['db_cpus']} CPU / " f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / " f"{profile['cache_memory']}; значения — медиана (min–max) по " "повторам", "- Посторонние контейнеры в VM во время замера: " + (", ".join(meta.get("other_containers") or []) or "нет"), "", ] for scenario in ("read", "write"): rows = [r for r in runs if r["scenario"] == scenario] if not rows: continue lines += [ f"## Сценарий `{scenario}`", "", "| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | " "max, мс | CPU API, ядер | CPU БД, ядер | RSS конец, МиБ | " "Сообщений в начале | Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|---|", ] cells = sorted({(r["size"], r["concurrency"]) for r in rows}) for size, conc in cells: group = [ r for r in rows if r["size"] == size and r["concurrency"] == conc ] flags = sorted({f for r in group for f in r["flags"].split()}) lines.append( f"| {human(size)} | {conc} | " f"{cell([r['rps'] for r in group], 0)} | " f"{cell([r['p50_ms'] for r in group], 2)} | " f"{cell([r['p95_ms'] for r in group], 2)} | " f"{cell([r['p99_ms'] for r in group], 2)} | " f"{cell([r['max_ms'] for r in group], 1)} | " f"{cell([r['app_cpu_cores'] for r in group], 2)} | " f"{cell([r['db_cpu_cores'] for r in group], 2)} | " f"{cell([r['rss_end_mib'] for r in group], 1)} | " f"{cell([r['store_messages_start'] for r in group], 0)} | " f"{' '.join(flags) or '—'} |", ) lines.append("") if sizes: lines += [ "## Память и наполнение по размерам", "", "Объём `messages` — после `VACUUM (ANALYZE)`, до прогонов: в " "таблице ровно N сообщений читаемого чата.", "", "| Размер | RSS API до, МиБ | RSS API после, МиБ | " "Байт RSS на сообщение | Таблица, МиБ | Индексы, МиБ | " "Байт в БД на сообщение | Наполнение, сообщ/с | " "Курсоров в пуле | Пик RSS за размер, МиБ |", "|---|---|---|---|---|---|---|---|---|---|", ] for info in sizes: lines.append( f"| {human(info['size'])} | {info['rss_before_mib']} | " f"{info['rss_after_mib']} | {info['bytes_per_message']} | " f"{mib_bytes(info.get('messages_table_bytes'))} | " f"{mib_bytes(info.get('messages_indexes_bytes'))} | " f"{info.get('db_bytes_per_message', '—')} | " f"{info['prefill_rate']} | {info['cursors']} | " f"{info.get('rss_peak_mib', '—')} |", ) lines.append("") lines += [ "## Планы запроса истории", "", "Страница читаемого чата (`limit` профиля). `app` — план " "запроса приложения (auto_explain), `psql` — тот же запрос с " "теми же параметрами в psql, `no index` — без индекса в " "откатываемой транзакции. Буферы — верхнего узла плана, файлы " "— `explain-<размер>.txt`.", "", "| Размер | Страница | После сообщений | Источник | Узел | " "shared hit | read | Строк | Время, мс | План |", "|---|---|---|---|---|---|---|---|---|---|", ] for info in sizes: for plan in info.get("plans", []): lines.append( f"| {human(info['size'])} | {plan['page']} | " f"{plan['after']} | {plan['source']} | " f"{plan['node'] or '—'} | {plan['shared_hit']} | " f"{plan['shared_read']} | {fmt(plan['rows'])} | " f"{fmt(plan['time_ms'], 3)} | {plan['kind']} |", ) lines.append("") if prefill: largest = max(r["size"] for r in prefill) curve = [r for r in prefill if r["size"] == largest] lines += [ f"## Кривая наполнения {human(largest)}: запись при росте " f"хранилища, {profile['prefill_vus']} клиента", "", "| Порция | Сообщений после | RPS | p50, мс | p95, мс | " "p99, мс | max, мс | CPU API, ядер | CPU БД, ядер | RSS, МиБ | " "Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for r in curve: lines.append( f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | " f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | " f"{r['max_ms']} | {r['app_cpu_cores']} | " f"{r['db_cpu_cores']} | {r['rss_end_mib']} " f"| {r['flags'] or '—'} |", ) lines.append("") if meta.get("host_warnings"): lines += ["## Предупреждения о хосте", ""] lines += [f"- {w}" for w in meta["host_warnings"]] lines.append("") return "\n".join(lines) # ---------------------------------------------------------------- прогон class Bench: def __init__( self, profile: Profile, label: str, results: Results, ) -> None: self.profile = profile self.label = label self.results = results self.stand = Stand(profile, results.dir) self.started = time.monotonic() self.runs_total = ( len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats ) self.runs_done = 0 self.stage = "start" self.size: int | None = None # скорости стенда для оценки оставшегося времени self.prefill_rate = 1500.0 self.write_rps: dict[int, float] = {} self.run_overhead = 3.0 self.done_sizes = 0 self.prefilled = 0 # текущий размер наполнен и курсоры собраны self.prepared = False # ------------------------------------------------------------ вывод def say(self, text: str) -> None: elapsed = int(time.monotonic() - self.started) stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed)) print(f"[{stamp}] {text}", flush=True) def eta(self) -> float: p = self.profile left = 0.0 for index, size in enumerate(p.sizes): current = index == self.done_sizes if index < self.done_sizes or (current and self.prepared): continue filled = self.prefilled if current else 0 left += (size - filled) / self.prefill_rate + size / 100 / 500 # стенд, миграции, обслуживание БД и план left += 30 per_size = len(p.concurrency) * p.repeats reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes) done_in_size = self.runs_done - self.done_sizes * per_size * 2 reads_left -= min(done_in_size, per_size) writes_left -= max(0, done_in_size - per_size) read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead left += reads_left * read_time rps = ( statistics.mean(self.write_rps.values()) if self.write_rps else 1000 ) write_time = (p.write_warmup + p.write_measure) / rps left += writes_left * (write_time + self.run_overhead) return left def progress(self, last: dict[str, Any] | None = None) -> None: state = { "label": self.label, "profile": self.profile.name, "stage": self.stage, "size": self.size, "runs_done": self.runs_done, "runs_total": self.runs_total, "elapsed_s": round(time.monotonic() - self.started), "eta_s": round(self.eta()), "warnings": self.results.meta.get("host_warnings", []), } if last is not None: state["last_run"] = last self.results.write_progress(state) # --------------------------------------------------------- этапы def run(self) -> None: for index, size in enumerate(self.profile.sizes, start=1): self.size = size self.prefilled = 0 self.prepared = False self.run_size(index, size) self.done_sizes += 1 def run_size(self, index: int, size: int) -> None: stand = self.stand name = human(size) self.stage = f"size {name}: start" self.say(f"size {name} ({index}/{len(self.profile.sizes)}): start") stand.start(lambda text: self.say(f"size {name}: {text}")) limits = stand.limits() self.results.meta.setdefault("limits", limits) if limits != self.results.meta["limits"]: raise BenchError(f"limits changed between sizes: {limits}") if "stand" not in self.results.meta: self.results.meta["stand"] = stand.passport() self.results.write_meta() rss_start = stand.snapshot()["rss_kib"] self.say(f"size {name}: app started, RSS {mib(rss_start)} MiB") stand.k6("prepare.js", {"MEMBERS": self.profile.members}) self.progress() rss_before = stand.snapshot()["rss_kib"] prefill_seconds = self.run_prefill(size) after = stand.snapshot() rss_after = after["rss_kib"] self.stage = f"size {name}: maintenance" self.progress() seconds = stand.maintain() storage = stand.storage() if storage["messages_rows"] != size: raise BenchError( f"messages table has {storage['messages_rows']} rows after " f"prefill, expected {size}", ) self.say( f"size {name}: VACUUM (ANALYZE) and CHECKPOINT in {seconds:.1f}s; " f"messages {mib_bytes(storage['messages_table_bytes'])} MiB + " f"indexes {mib_bytes(storage['messages_indexes_bytes'])} MiB, " f"{storage['db_bytes_per_message']} B/msg", ) self.stage = f"size {name}: cursors" self.progress() stand.k6( "cursors.js", { "WALK_LIMIT": self.profile.walk_limit, "POOL_SIZE": self.profile.cursor_pool, }, ) pool = stand.work_json("cursors.json") if pool["walked"] != size: raise BenchError( f"read chat has {pool['walked']} messages, expected {size}", ) self.say(f"size {name}: cursors {len(pool['cursors'])}") self.stage = f"size {name}: query plan" self.progress() plans = self.explain(size, pool) info = { "size": size, "rss_before_mib": mib(rss_before), "rss_after_mib": mib(rss_after), "bytes_per_message": round( (rss_after - rss_before) * 1024 / size, ), "prefill_seconds": round(prefill_seconds, 1), "prefill_rate": round(size / prefill_seconds), "cursors": len(pool["cursors"]), "maintenance_seconds": round(seconds, 1), **storage, "plans": plans, } self.results.sizes.append(info) self.results.write_meta() self.prepared = True self.say( f"size {name}: prefill done, app RSS {info['rss_after_mib']} " f"MiB (+{info['bytes_per_message']} B/msg)", ) store = size for scenario in ("read", "write"): for conc in self.profile.concurrency: for repeat in range(1, self.profile.repeats + 1): store = self.run_one(scenario, size, conc, repeat, store) if scenario == "read": self.check_read_chat(size) info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"]) self.results.write_meta() if not stand.stop(): raise BenchError(f"stand of size {name} was not removed") def check_read_chat(self, size: int) -> None: """После блока чтения в читаемом чате по-прежнему ровно N. Новое сообщение встало бы в конец истории, поэтому достаточно дочитать хвост от последнего курсора пула. """ self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit}) tail = self.stand.work_json("tail.json") total = tail["from_after"] + tail["tail"] if total != size: raise BenchError( f"read chat has {total} messages after the read block, " f"expected {size}", ) self.say(f"size {human(size)}: read chat unchanged, {total} messages") def explain(self, size: int, pool: dict[str, Any]) -> list[dict[str, Any]]: """Этап плана: EXPLAIN (ANALYZE, BUFFERS) запросов истории приложения. auto_explain включается между маркерами, `explain.js` читает первую, среднюю и последнюю полную страницу; планы берутся из журнала Postgres. Затем средняя страница в psql с индексом и без него (`DROP INDEX` в откатываемой транзакции). Решение 5 design change `bench-postgres`. """ stand = self.stand marker = f"bench-explain {uuid.uuid4().hex}" enable = "".join( f"ALTER SYSTEM SET {key} = '{value}';\n" for key, value in EXPLAIN_SETTINGS.items() ) disable = "".join( f"ALTER SYSTEM RESET {key};\n" for key in EXPLAIN_SETTINGS ) stand.psql( f"DO $$ BEGIN RAISE LOG '{marker} start'; END $$;\n" f"{enable}SELECT pg_reload_conf();\n", ) time.sleep(RELOAD_PAUSE_S) try: stand.k6("explain.js", {}) finally: stand.psql( f"{disable}SELECT pg_reload_conf();\n" f"DO $$ BEGIN RAISE LOG '{marker} end'; END $$;\n", ) # соединения пула API должны получить выключение до прогонов time.sleep(RELOAD_PAUSE_S) entries = [ entry for entry in plan_entries(stand.db_log(), marker) if is_history(entry) ] if len(entries) != len(EXPLAIN_PAGES): raise BenchError( f"size {human(size)}: expected {len(EXPLAIN_PAGES)} history " f"query plans in the PostgreSQL log, found {len(entries)}", ) afters = pool["afters"] positions = dict( zip( EXPLAIN_PAGES, (0, afters[len(afters) // 2], afters[-1]), strict=True, ), ) plans = [] for page, entry in zip(EXPLAIN_PAGES, entries, strict=True): parts = plan_parts(entry) or {"plan": entry} plans.append( { "page": page, "after": positions[page], "source": "app", **plan_summary(parts["plan"], entry_duration(entry)), }, ) contrast = self.explain_without_index(entries[1]) if contrast is not None: for key, source in ( ("with_index", "psql"), ("without_index", "psql, no index"), ): plans.append( { "page": "middle", "after": positions["middle"], "source": source, **plan_summary(contrast[key], None), }, ) text = render_explain( size, self.results.meta["git"], positions, entries, contrast, ) path = self.results.dir / f"explain-{human(size)}.txt" path.write_text(text, encoding="utf-8") for plan in plans: self.say( f"size {human(size)}: plan {plan['page']} ({plan['source']}): " f"{plan['node']}, shared hit={plan['shared_hit']} " f"read={plan['shared_read']}, {plan['time_ms']} ms", ) return plans def explain_without_index(self, entry: str) -> dict[str, str] | None: """Тот же запрос и параметры в psql: с индексом и без него. `DROP INDEX` идёт внутри транзакции, которая откатывается: индекс остаётся, что проверяется по `pg_indexes`. Нагрузки в этот момент нет. План приложения без индекса — сравнения нет, None. """ parts = plan_parts(entry) index = index_name(parts["plan"]) if parts else None if parts is None or index is None: return None args = ", ".join( "'" + value.replace("'", "''") + "'" for value in parts["params"] ) out = self.stand.psql( "SET lock_timeout = '5s';\n" "BEGIN;\n" f"PREPARE page AS {parts['text']};\n" "\\echo @@with-index\n" f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE page({args});\n" f'DROP INDEX "{index}";\n' "\\echo @@without-index\n" f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE page({args});\n" "ROLLBACK;\n", ) with_index, _, without_index = out.partition("@@without-index") kept = self.stand.psql( "SELECT count(*) FROM pg_indexes WHERE tablename = 'messages' " f"AND indexname = '{index}';\n", ) if kept != "1": raise BenchError(f"index {index} is missing after the rollback") return { "index": index, "with_index": with_index.replace("@@with-index", "").strip(), "without_index": without_index.strip(), } def run_prefill(self, size: int) -> float: chunk_size = self.profile.prefill_chunk total_seconds = 0.0 for chunk, offset in enumerate(range(0, size, chunk_size), start=1): count = min(chunk_size, size - offset) name = f"prefill-{human(size)}-{chunk:02d}" self.stage = f"size {human(size)}: prefill" measured = measure( self.stand, "prefill.js", { "COUNT": count, "OFFSET": offset, "VUS": self.profile.prefill_vus, "NAME": name, }, self.results.dir / "raw" / f"{name}.json", ) summary = measured.summary down = self.stand.down_storages() flags = run_flags( self.profile, measured, self.stand.state(APP), down, ) self.results.add_prefill( { **summary, "size": size, "chunk": chunk, "messages_before": offset, "messages_after": offset + summary["requests_ok"], "app_cpu_cores": measured.app_cpu, "rss_end_mib": mib(measured.after["rss_kib"]), "mem_cgroup_end_mib": mib_bytes( measured.after["mem_bytes"], ), "generator_cpu_cores": measured.generator_cpu, "flags": " ".join(flags), "db_cpu_cores": measured.db_cpu, }, ) if down: raise BenchError( f"{', '.join(down)} went down during prefill chunk " f"{chunk}: matrix stopped", ) if summary["requests_ok"] != count: raise BenchError( f"prefill chunk {chunk}: {summary['requests_ok']} of " f"{count} messages confirmed ({' '.join(flags)})", ) total_seconds += summary["duration_s"] self.prefilled = offset + count self.prefill_rate = self.prefilled / total_seconds self.say( f"size {human(size)}: prefill {self.prefilled}/{size} " f"({self.prefilled * 100 // size}%), " f"{self.prefill_rate:.0f} msg/s, " f"~{fmt_left((size - self.prefilled) / self.prefill_rate)} " "left", ) self.progress() return total_seconds def run_one( self, scenario: str, size: int, conc: int, repeat: int, store: int, ) -> int: """Прогрев и измеряемый прогон; возвращает сообщений в хранилище.""" p = self.profile number = self.runs_done + 1 name = f"{scenario}-{human(size)}-c{conc}-r{repeat}" self.stage = f"run {number}/{self.runs_total} {name}" self.progress() wall = time.monotonic() if scenario == "read": warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"} self.stand.k6( "read.js", {**warm, "SEED": p.seed + p.warmup_seed_shift}, ) env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"} env["SEED"] = p.seed else: self.stand.k6( "write.js", {"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store}, ) warm_ok = parse_summary(self.stand.work_dir / "last.json") store += warm_ok["requests_ok"] env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store} measured = measure( self.stand, f"{scenario}.js", {**env, "NAME": name}, self.results.dir / "raw" / f"{name}.json", ) summary = measured.summary app = self.stand.state(APP) down = self.stand.down_storages() flags = run_flags(p, measured, app, down) end = store + (summary["requests_ok"] if scenario == "write" else 0) row = { **summary, "run": number, "scenario": scenario, "size": size, "concurrency": conc, "repeat": repeat, "app_cpu_cores": measured.app_cpu, "rss_start_mib": mib(measured.before["rss_kib"]), "rss_end_mib": mib(measured.after["rss_kib"]), "rss_max_mib": mib(measured.rss_max_kib), "mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]), "store_messages_start": store, "store_messages_end": end, "generator_cpu_cores": measured.generator_cpu, "flags": " ".join(flags), "db_cpu_cores": measured.db_cpu, } self.results.add_run(row) self.runs_done += 1 if scenario == "write" and summary["rps"]: self.write_rps[conc] = summary["rps"] overhead = time.monotonic() - wall - summary["duration_s"] if scenario == "read": overhead -= p.read_warmup_s self.run_overhead = max(1.0, overhead) errors = summary["requests_failed"] + summary["check_failed"] self.say( f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} " f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps " f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} " f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} " f"db {measured.db_cpu} rss {row['rss_end_mib']} MiB " f"err {errors} " f"~{fmt_left(self.eta())} left" + (f" [{' '.join(flags)}]" if flags else ""), ) self.progress(last=row) if "oom" in flags or "app_down" in flags: raise BenchError( f"{APP} is down after {name} ({' '.join(flags)}): " "matrix stopped", ) if down: raise BenchError( f"{', '.join(down)} went down during {name}: matrix stopped", ) return end # ------------------------------------------------------------------ CLI def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--profile", choices=sorted(PROFILES), required=True) parser.add_argument("--label", required=True) # отладка стенда: переопределения пишутся в паспорт как overrides parser.add_argument("--sizes", help="через запятую, например 120000") parser.add_argument("--concurrency", help="через запятую") parser.add_argument("--repeats", type=int) parser.add_argument("--memory", help="лимит памяти, например 64m") return parser.parse_args(argv) def apply_overrides( profile: Profile, args: argparse.Namespace, ) -> tuple[Profile, dict[str, Any]]: overrides: dict[str, Any] = {} if args.sizes: overrides["sizes"] = tuple(int(x) for x in args.sizes.split(",")) if args.concurrency: overrides["concurrency"] = tuple( int(x) for x in args.concurrency.split(",") ) if args.repeats: overrides["repeats"] = args.repeats if args.memory: overrides["memory"] = args.memory return replace(profile, **overrides), overrides def main(argv: list[str]) -> int: args = parse_args(argv) if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label): print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True) return 2 label_dir = ROOT / "evidence" / "bench" / args.label relative = label_dir.relative_to(ROOT) if label_dir.exists(): print(f"label directory {relative} already exists", flush=True) return 2 profile, overrides = apply_overrides(PROFILES[args.profile], args) git = git_info() state = host_state() try: warnings = host_warnings(state) + vm_warnings(profile) except BenchError as exc: print(f"bench failed: {exc}", flush=True) return 2 meta: dict[str, Any] = { "label": args.label, "status": "incomplete", "reason": "running", "started_at": datetime.now(UTC).isoformat(timespec="seconds"), "command": " ".join(["uv run python bench/run.py", *argv]), "git": git, "overrides": overrides, "host_state": {"start": state}, "host_warnings": warnings, # чужие контейнеры в той же VM: стенд их не трогает, но они шумят "other_containers": sorted( sh("docker", "ps", "--format", "{{.Names}}", check=False).split(), ), } stand_check = Stand(profile, label_dir) started = time.monotonic() def say(text: str) -> None: elapsed = int(time.monotonic() - started) stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed)) print(f"[{stamp}] {text}", flush=True) for warning in warnings: say(f"WARNING: {warning}") if overrides: say(f"WARNING: profile overrides {overrides}: debug run") if git["dirty"]: say("NOTE: working tree has uncommitted changes (dirty: true)") try: stand_check.check_free() image = stand_check.build_image() except BenchError as exc: print(f"bench failed: {exc}", flush=True) return 1 say(f"image {APP_IMAGE} built ({image[:19]}…), compose project {PROJECT}") meta.update(static_passport(profile)) results = Results(label_dir, meta) bench = Bench(profile, args.label, results) bench.started = started code = 0 def terminate(*_: object) -> None: raise KeyboardInterrupt signal.signal(signal.SIGTERM, terminate) try: bench.run() meta["status"] = "complete" meta.pop("reason", None) except KeyboardInterrupt: meta["reason"] = "interrupted" say("interrupted: partial results kept") code = 130 except BenchError as exc: meta["reason"] = str(exc).splitlines()[0] say(f"bench failed: {exc}") code = 1 finally: # повторный Ctrl+C не должен оборвать уборку стенда signal.signal(signal.SIGINT, signal.SIG_IGN) if not bench.stand.stop(): say(f"cleanup incomplete: {leftovers()}") code = code or 1 # fixture.json с токенами — только здесь, в результаты не попадает shutil.rmtree(bench.stand.work_dir, ignore_errors=True) meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds") meta["host_state"]["end"] = host_state() results.write_meta() results.write_summary() bench.stage = "done" if code == 0 else meta.get("reason", "stopped") bench.progress() say( f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, " f"results in {relative}", ) return code if __name__ == "__main__": sys.exit(main(sys.argv[1:]))