Files
mtusi2026lab2/bench/run.py
T
ricoromaandClaude Opus 5.5 e76eed789f feat: run load bench on PostgreSQL with query plans
- bench stand is a separate compose project mtusi-bench (API, Postgres,
  Valkey) with fixed limits, fresh volume per size and migrations
- k6 registers with a password, logs in with Basic and sends Bearer
- VACUUM (ANALYZE) and CHECKPOINT after prefill
- EXPLAIN (ANALYZE, BUFFERS) of the app's history query via auto_explain,
  plus the same query without the index in a rolled back transaction
- Postgres CPU, table and index size in results; storages in passport
- bench_compare checks storage limits, images and settings
- remove inproc.py and make bench_inproc

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 01:21:15 +03:00

1999 lines
73 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Нагрузочный стенд: матрица размеров истории и конкуренции.
Запуск — `make bench` / `make bench_quick`. Стенд — API, PostgreSQL и
Valkey: проект compose `mtusi-bench` из `docker-compose.yml` с override
`bench/bench.compose.yml`. Методика — design changes `load-bench-harness`
и `bench-postgres`, инварианты — `.claude/rules/bench.md`. Только stdlib:
приложение не импортируется, его нагружает k6 по HTTP; к PostgreSQL
стенда оркестратор ходит только для обслуживания и измерений (VACUUM,
размеры, планы запроса).
"""
import argparse
import csv
import json
import os
import platform
import re
import secrets
import shutil
import signal
import statistics
import subprocess
import sys
import threading
import time
import uuid
from collections.abc import Callable
from dataclasses import asdict, dataclass, replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
K6_IMAGE = (
"grafana/k6:2.3.0@sha256:"
"9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34"
)
APP_IMAGE = "mtusi-messenger:bench"
PROJECT = "mtusi-bench"
NETWORK = "mtusi-bench"
APP = "mtusi-bench-api"
DB = "mtusi-bench-postgres"
CACHE = "mtusi-bench-valkey"
K6 = "mtusi-bench-k6"
# контейнеры стенда по ролям: лимиты, состояние, паспорт
CONTAINERS = {"api": APP, "postgres": DB, "valkey": CACHE}
COMPOSE_FILES = (
ROOT / "docker-compose.yml",
ROOT / "bench" / "bench.compose.yml",
)
# подстановки compose вместо .env рабочего стенда
ENV_FILE = ROOT / "bench" / "bench.env"
# что из окружения хоста нужно самому docker compose; остальное (в том
# числе POSTGRES_* из shell) в стенд не попадает
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "DOCKER_HOST", "DOCKER_CONFIG")
READY_CHECK = (
"import urllib.request; urllib.request.urlopen("
"'http://127.0.0.1:8080/health/ready', timeout=2)"
)
SUCCESS_METRIC = "http_req_duration{expected_response:true}"
# предупреждение о хосте: swap занят больше, МиБ
SWAP_WARN_MIB = 1024
# предупреждение о VM: лимиты памяти стенда + этот запас больше памяти VM
VM_RESERVE_GIB = 1.5
# psql в контейнере Postgres от пользователя образа, без пароля; SQL — stdin
PSQL = (
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
)
# настройки Postgres в паспорте: от них зависят план и стоимость записи
PG_SETTINGS = (
"shared_buffers",
"effective_cache_size",
"work_mem",
"maintenance_work_mem",
"max_connections",
"synchronous_commit",
"fsync",
"wal_level",
"checkpoint_timeout",
"max_wal_size",
"random_page_cost",
"jit",
"autovacuum",
"plan_cache_mode",
"shared_preload_libraries",
)
# auto_explain на этапе плана; вне его log_min_duration = -1 (выключен)
EXPLAIN_SETTINGS = {
"auto_explain.log_min_duration": "0",
"auto_explain.log_analyze": "on",
"auto_explain.log_buffers": "on",
"auto_explain.log_timing": "on",
"auto_explain.log_format": "text",
}
# пауза после pg_reload_conf(): SIGHUP доходит до соединений пула API
RELOAD_PAUSE_S = 1.0
# страницы этапа плана в порядке запросов explain.js
EXPLAIN_PAGES = ("first", "middle", "last")
INDEX_NODE_RE = re.compile(
r"Index Only Scan(?: Backward)? using (\w+)"
r"|Index Scan(?: Backward)? using (\w+)"
r"|Bitmap Index Scan on (\w+)",
)
RUN_FIELDS = [
"run",
"scenario",
"size",
"concurrency",
"repeat",
"duration_s",
"requests_ok",
"requests_failed",
"rps",
"error_ratio",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"app_cpu_cores",
"rss_start_mib",
"rss_end_mib",
"rss_max_mib",
"mem_cgroup_end_mib",
"store_messages_start",
"store_messages_end",
"generator_cpu_cores",
"flags",
"check_failed",
# лаба 2: новые столбцы — только в конец
"db_cpu_cores",
]
PREFILL_FIELDS = [
"size",
"chunk",
"messages_before",
"messages_after",
"duration_s",
"requests_ok",
"requests_failed",
"rps",
"error_ratio",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"app_cpu_cores",
"rss_end_mib",
"mem_cgroup_end_mib",
"generator_cpu_cores",
"flags",
"db_cpu_cores",
]
@dataclass(frozen=True)
class Profile:
name: str
sizes: tuple[int, ...]
concurrency: tuple[int, ...]
repeats: int
read_warmup_s: int
read_measure_s: int
write_warmup: int
write_measure: int
prefill_chunk: int = 50_000
prefill_vus: int = 32
text_length: int = 64
page_limit: int = 50
walk_limit: int = 100
cursor_pool: int = 1000
members: int = 10
seed: int = 20260930
# зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно
# те страницы, с которых начнётся замер
warmup_seed_shift: int = 1_000_003
generator_cpu_limit: float = 4.0
cpus: str = "1"
memory: str = "2g"
# лаба 2: лимиты хранилищ (swap = память) и срок сессии стенда; идут в
# compose переменными BENCH_* (bench/bench.compose.yml). Решение 2
# design change bench-postgres
db_cpus: str = "4"
db_memory: str = "3g"
cache_cpus: str = "2"
cache_memory: str = "512m"
session_ttl_seconds: int = 21600
PROFILES = {
"full": Profile(
name="full",
sizes=(10_000, 100_000, 1_000_000),
concurrency=(1, 10, 50),
repeats=3,
read_warmup_s=5,
read_measure_s=20,
write_warmup=500,
write_measure=5000,
),
"quick": Profile(
name="quick",
sizes=(1_000, 5_000, 10_000),
concurrency=(1, 4, 16),
repeats=1,
read_warmup_s=2,
read_measure_s=5,
write_warmup=100,
write_measure=1000,
),
}
class BenchError(Exception):
"""Стенд не может продолжать: сообщение для человека."""
# ---------------------------------------------------------------- утилиты
def sh(*args: str, check: bool = True) -> str:
"""Команда без shell; stdout без хвостовых пробелов.
`LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод
разбирается регулярками с точкой.
"""
proc = subprocess.run(
args,
capture_output=True,
text=True,
check=False,
env={**os.environ, "LC_ALL": "C"},
)
if check and proc.returncode != 0:
raise BenchError(
f"{' '.join(args[:4])}…: exit {proc.returncode}\n"
f"{(proc.stdout + proc.stderr)[-2000:]}",
)
return proc.stdout.strip()
def human(n: int) -> str:
"""10000 → 10k, 1000000 → 1M."""
if n >= 1_000_000 and n % 1_000_000 == 0:
return f"{n // 1_000_000}M"
if n >= 1000 and n % 1000 == 0:
return f"{n // 1000}k"
return str(n)
def fmt_left(seconds: float) -> str:
seconds = max(0, int(seconds))
hours, rest = divmod(seconds, 3600)
minutes, secs = divmod(rest, 60)
if hours:
return f"{hours}h{minutes:02d}m"
if minutes:
return f"{minutes}m{secs:02d}s"
return f"{secs}s"
def mib(kib: float | None) -> float | None:
return None if kib is None else round(kib / 1024, 1)
def mib_bytes(value: int | None) -> float | None:
return None if value is None else round(value / 2**20, 1)
def size_bytes(text: str) -> int:
"""Лимит памяти docker (`512m`, `2g`) в байтах."""
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kmg]?)b?", text.strip().lower())
if not match:
raise BenchError(f"bad memory limit {text!r}: use 512m or 2g")
power = {"": 0, "k": 1, "m": 2, "g": 3}[match.group(2)]
return int(float(match.group(1)) * 1024**power)
def fmt(value: float | None, digits: int = 0) -> str:
return "—" if value is None else f"{value:.{digits}f}"
def rounded(value: float | None, digits: int = 3) -> float | None:
return None if value is None else round(value, digits)
# ------------------------------------------------------------ хост и паспорт
def host_state() -> dict[str, Any]:
"""Swap, свободная память и питание (macOS); на другой ОС — null."""
state: dict[str, Any] = {
"swap_used_mib": None,
"swap_total_mib": None,
"memory_free_percent": None,
"power": None,
"battery_percent": None,
}
if platform.system() != "Darwin":
return state
swap = sh("sysctl", "-n", "vm.swapusage", check=False)
for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")):
match = re.search(rf"{key} = ([\d.,]+)M", swap)
if match:
state[name] = float(match.group(1).replace(",", "."))
pressure = sh("memory_pressure", check=False)
match = re.search(r"free percentage: (\d+)%", pressure)
if match:
state["memory_free_percent"] = int(match.group(1))
batt = sh("pmset", "-g", "batt", check=False)
match = re.search(r"drawing from '([^']+)'", batt)
if match:
state["power"] = match.group(1)
match = re.search(r"(\d+)%;", batt)
if match:
state["battery_percent"] = int(match.group(1))
return state
def host_warnings(state: dict[str, Any]) -> list[str]:
warnings = []
if state["power"] == "Battery Power":
warnings.append("host on battery power: CPU may be throttled")
used = state["swap_used_mib"]
if used is not None and used > SWAP_WARN_MIB:
warnings.append(
f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: "
"reboot before a full run",
)
return warnings
def vm_warnings(profile: Profile) -> list[str]:
"""Лимиты памяти стенда не помещаются в VM docker с запасом.
Тогда лимит перестаёт быть гарантией: контейнеры делят память VM, и
метки разных дней несравнимы.
"""
total = sh("docker", "info", "--format", "{{.MemTotal}}", check=False)
if not total.isdigit():
return []
limits = sum(
size_bytes(value)
for value in (profile.memory, profile.db_memory, profile.cache_memory)
)
budget = int(total) - int(VM_RESERVE_GIB * 2**30)
if limits <= budget:
return []
return [
f"memory limits {limits / 2**30:.1f} GiB + {VM_RESERVE_GIB} GiB "
f"reserve exceed docker VM memory {int(total) / 2**30:.1f} GiB: "
"limits are not guaranteed",
]
def git_info() -> dict[str, Any]:
commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD")
status = sh("git", "-C", str(ROOT), "status", "--porcelain")
return {"commit": commit, "dirty": bool(status)}
def static_passport(profile: Profile) -> dict[str, Any]:
docker = json.loads(sh("docker", "info", "--format", "{{json .}}"))
versions = sh(
"docker",
"run",
"--rm",
"--entrypoint",
"python",
APP_IMAGE,
"-c",
"import importlib.metadata as m, json, platform; print(json.dumps("
"{'python': platform.python_version(), **{p: m.version(p) for p in "
"('fastapi', 'starlette', 'pydantic', 'uvicorn', 'sqlalchemy', "
"'asyncpg', 'redis')}}))",
)
k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version")
host: dict[str, Any] = {"os": platform.system(), "machine": None}
if platform.system() == "Darwin":
host["os"] = f"macOS {platform.mac_ver()[0]}"
host["machine"] = sh(
"sysctl",
"-n",
"machdep.cpu.brand_string",
check=False,
)
memsize = sh("sysctl", "-n", "hw.memsize", check=False)
host["memory_gib"] = (
round(int(memsize) / 2**30, 1) if memsize.isdigit() else None
)
return {
"app_image": {
"tag": APP_IMAGE,
"id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE),
"versions": json.loads(versions),
},
"generator": {"image": K6_IMAGE, "version": k6_version},
"docker": {
"server": docker.get("ServerVersion"),
"vm_os": docker.get("OperatingSystem"),
"vm_cpus": docker.get("NCPU"),
"vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1),
"cgroup": docker.get("CgroupVersion"),
"logging_driver": docker.get("LoggingDriver"),
},
"host": host,
"profile": asdict(profile),
}
# ------------------------------------------------------------------ стенд
class Watcher:
"""Раз в секунду, пока идёт запуск k6: VmRSS приложения, живы ли оно,
PostgreSQL и Valkey.
Приложение или хранилище умерло (OOM, kill) — k6 останавливается:
иначе каждый его запрос ждал бы таймаута, и наполнение тянулось бы
часами.
"""
def __init__(self) -> None:
self.samples: list[int] = []
self.app_died = False
self.stand_died: list[str] = []
self._stop = threading.Event()
self._thread = threading.Thread(target=self._loop, daemon=True)
def _halt(self) -> None:
sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False)
def _loop(self) -> None:
while not self._stop.wait(1.0):
states = sh(
"docker",
"inspect",
"-f",
"{{.Name}} {{.State.Running}}",
DB,
CACHE,
check=False,
)
dead = [
name.lstrip("/")
for name, running in (
line.split() for line in states.splitlines() if line
)
if running != "true"
]
if dead:
self.stand_died = dead
self._halt()
return
out = sh(
"docker",
"exec",
APP,
"grep",
"^VmRSS:",
"/proc/1/status",
check=False,
)
match = re.search(r"(\d+) kB", out)
if match:
self.samples.append(int(match.group(1)))
continue
running = sh(
"docker",
"inspect",
"-f",
"{{.State.Running}}",
APP,
check=False,
)
if running != "true":
self.app_died = True
self._halt()
return
def __enter__(self) -> "Watcher":
self._thread.start()
return self
def __exit__(self, *_: object) -> None:
self._stop.set()
self._thread.join()
def compose_env(profile: Profile) -> dict[str, str]:
"""Окружение `docker compose` стенда, собранное с нуля.
Пароли хранилищ — новые на каждый прогон: живут только в этом словаре
и в контейнерах стенда, в результаты не пишутся. Лимиты и срок сессии
— из профиля (`BENCH_*` в `bench/bench.compose.yml`).
"""
env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
return {
**env,
"LC_ALL": "C",
"POSTGRES_PASSWORD": secrets.token_urlsafe(24),
"VALKEY_PASSWORD": secrets.token_urlsafe(24),
"BENCH_APP_CPUS": profile.cpus,
"BENCH_APP_MEMORY": profile.memory,
"BENCH_DB_CPUS": profile.db_cpus,
"BENCH_DB_MEMORY": profile.db_memory,
"BENCH_CACHE_CPUS": profile.cache_cpus,
"BENCH_CACHE_MEMORY": profile.cache_memory,
"BENCH_SESSION_TTL_SECONDS": str(profile.session_ttl_seconds),
}
def leftovers() -> dict[str, list[str]]:
"""Контейнеры, volume и сеть стенда, оставшиеся от другого прогона.
Ключ — подкоманда docker, которой их удалить.
"""
project = f"label=com.docker.compose.project={PROJECT}"
names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split()
containers = sh(
"docker",
"ps",
"-a",
"--filter",
project,
"--format",
"{{.Names}}",
).split()
networks = sh("docker", "network", "ls", "--format", "{{.Name}}").split()
return {
"rm -f": containers + ([K6] if K6 in names else []),
"volume rm": sh(
"docker",
"volume",
"ls",
"-q",
"--filter",
project,
).split(),
"network rm": [NETWORK] if NETWORK in networks else [],
}
class Stand:
"""Проект compose `mtusi-bench` и контейнер k6 на время одного размера."""
def __init__(self, profile: Profile, label_dir: Path) -> None:
self.profile = profile
self.label_dir = label_dir
self.work_dir = label_dir / ".work"
self.env = compose_env(profile)
def compose(self, *args: str, check: bool = True) -> str:
"""docker compose стенда; без `-p mtusi-bench` не вызывается."""
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
proc = subprocess.run(
[
"docker",
"compose",
"-p",
PROJECT,
"--env-file",
str(ENV_FILE),
*files,
*args,
],
capture_output=True,
text=True,
env=self.env,
check=False,
)
if check and proc.returncode != 0:
raise BenchError(
f"docker compose {' '.join(args[:3])}: exit "
f"{proc.returncode}\n{(proc.stdout + proc.stderr)[-2000:]}",
)
return proc.stdout.strip()
def check_free(self) -> None:
found = {cmd: names for cmd, names in leftovers().items() if names}
if found:
fix = "; ".join(
f"docker {cmd} {' '.join(names)}"
for cmd, names in found.items()
)
raise BenchError(
"bench stand leftovers exist: another bench is running or a "
f"crashed one left them; remove with: {fix}",
)
def build_image(self) -> str:
sh(
"docker",
"build",
"-q",
"-t",
APP_IMAGE,
str(ROOT),
)
return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE)
def start(self, say: Callable[[str], None]) -> None:
"""Свежий стенд: пустая база на новом volume, миграции, API, k6."""
self.work_dir.mkdir(parents=True, exist_ok=True)
self.compose("up", "-d", "--wait", "postgres", "valkey")
say("postgres and valkey healthy")
self.compose("run", "--rm", "migrate")
say("migrations applied")
self.compose("up", "-d", "--no-deps", "--no-build", "api")
sh(
"docker",
"run",
"-d",
"--name",
K6,
"--network",
NETWORK,
"-v",
f"{ROOT / 'bench' / 'k6'}:/scripts:ro",
"-v",
f"{self.label_dir}:/out",
"-v",
f"{self.work_dir}:/work",
"--entrypoint",
"sleep",
K6_IMAGE,
"infinity",
)
self.wait_ready()
def wait_ready(self, timeout: float = 60) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
probe = subprocess.run(
["docker", "exec", APP, "python", "-c", READY_CHECK],
capture_output=True,
check=False,
)
if probe.returncode == 0:
return
state = self.state(APP)
if not state["running"]:
reason = "oom" if state["oom"] else "exited"
raise BenchError(f"{APP} died during start ({reason})")
time.sleep(0.25)
raise BenchError(f"{APP} not ready in {timeout:.0f}s")
def stop(self) -> bool:
"""k6, контейнеры, сеть и volume стенда; True — ничего не осталось.
k6 — первым: подключённый к сети чужой контейнер не дал бы compose
удалить сеть.
"""
subprocess.run(
["docker", "rm", "-f", K6],
capture_output=True,
check=False,
)
self.compose("down", "-v", "--remove-orphans", check=False)
return not any(leftovers().values())
def limits(self) -> dict[str, Any]:
"""Лимиты контейнеров стенда, прочитанные у запущенных контейнеров."""
limits = {}
for role, name in CONTAINERS.items():
host = json.loads(
sh("docker", "inspect", "-f", "{{json .HostConfig}}", name),
)
config = json.loads(
sh("docker", "inspect", "-f", "{{json .Config}}", name),
)
limits[role] = {
"nano_cpus": host.get("NanoCpus"),
"memory": host.get("Memory"),
"memory_swap": host.get("MemorySwap"),
"published_ports": host.get("PortBindings") or {},
"healthcheck": (config.get("Healthcheck") or {}).get("Test"),
"log_driver": (host.get("LogConfig") or {}).get("Type"),
}
return limits
def state(self, name: str) -> dict[str, Any]:
state = json.loads(
sh("docker", "inspect", "-f", "{{json .State}}", name),
)
restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", name)
return {
"running": state.get("Running"),
"oom": state.get("OOMKilled"),
"restarts": int(restarts or 0),
}
def down_storages(self) -> list[str]:
"""Хранилища, которые за прогон умирали, убивались или рестартовали."""
down = []
for name in (DB, CACHE):
state = self.state(name)
if not state["running"] or state["oom"] or state["restarts"]:
down.append(name)
return down
def psql(self, sql: str, *, check: bool = True) -> str:
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля."""
proc = subprocess.run(
["docker", "exec", "-i", DB, "sh", "-c", PSQL],
input=sql,
capture_output=True,
text=True,
check=False,
)
if check and proc.returncode != 0:
raise BenchError(f"psql: exit {proc.returncode}\n{proc.stderr}")
return proc.stdout.strip()
def maintain(self) -> float:
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды.
Статистика и состояние таблиц одинаковы у всех меток, а не
зависят от того, когда сработал autovacuum (решение 4).
"""
started = time.monotonic()
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
return time.monotonic() - started
def storage(self) -> dict[str, Any]:
"""Объём таблицы сообщений и её индексов, число строк."""
row = self.psql(
"SELECT pg_table_size('messages'), pg_indexes_size('messages'), "
"count(*) FROM messages;\n",
)
table, indexes, count = (int(value) for value in row.split("|"))
return {
"messages_rows": count,
"messages_table_bytes": table,
"messages_indexes_bytes": indexes,
"db_bytes_per_message": (
round((table + indexes) / count) if count else None
),
}
def db_log(self) -> str:
"""Журнал Postgres стенда (stderr контейнера): только в памяти."""
proc = subprocess.run(
["docker", "logs", DB],
capture_output=True,
text=True,
check=False,
)
return proc.stdout + proc.stderr
def passport(self) -> dict[str, Any]:
"""Хранилища и окружение API для паспорта; паролей здесь нет."""
settings = self.psql(
"SELECT name, current_setting(name) FROM pg_settings "
"WHERE name = ANY(string_to_array("
f"'{','.join(PG_SETTINGS)}', ',')) ORDER BY name;\n",
)
info = sh(
"docker",
"exec",
CACHE,
"sh",
"-c",
'exec valkey-cli -p "$VALKEY_PORT" INFO server',
)
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.M)
memory = sh(
"docker",
"exec",
CACHE,
"sh",
"-c",
'exec valkey-cli -p "$VALKEY_PORT" CONFIG GET "maxmemory*"',
).splitlines()
env = json.loads(
sh("docker", "inspect", "-f", "{{json .Config.Env}}", APP),
)
app_env = dict(
sorted(
item.split("=", 1)
for item in env
if re.match(r"[A-Z0-9_]+__[A-Z0-9_]+=", item)
and "PASSWORD" not in item.split("=", 1)[0]
),
)
def image(name: str) -> dict[str, str]:
return {
"image": sh(
"docker",
"inspect",
"-f",
"{{.Config.Image}}",
name,
),
"id": sh("docker", "inspect", "-f", "{{.Image}}", name),
}
return {
"postgres": {
**image(DB),
"version": self.psql("SHOW server_version;\n"),
"settings": dict(
line.split("|", 1) for line in settings.splitlines()
),
},
"valkey": {
**image(CACHE),
"version": valkey_version.group(1) if valkey_version else None,
**dict(zip(memory[::2], memory[1::2], strict=True)),
},
"app_env": app_env,
}
def snapshot(self) -> dict[str, Any]:
"""CPU и память приложения, CPU PostgreSQL и генератора.
Если приложение уже мертво (OOM), его поля — None: снимок нужен и
для записи проваленного прогона.
"""
app = sh(
"docker",
"exec",
APP,
"sh",
"-c",
"cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; "
"grep -E '^(VmRSS|VmHWM):' /proc/1/status",
check=False,
)
k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat")
db = sh(
"docker",
"exec",
DB,
"cat",
"/sys/fs/cgroup/cpu.stat",
check=False,
)
def number(pattern: str, text: str) -> int | None:
match = re.search(pattern, text, re.MULTILINE)
return int(match.group(1)) if match else None
return {
"app_usec": number(r"^usage_usec (\d+)$", app),
"mem_bytes": number(r"^(\d+)$", app),
"rss_kib": number(r"^VmRSS:\s+(\d+) kB", app),
"hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app),
"k6_usec": number(r"^usage_usec (\d+)$", k6),
"db_usec": number(r"^usage_usec (\d+)$", db),
}
def k6(
self,
script: str,
env: dict[str, Any],
*,
check: bool = True,
) -> subprocess.CompletedProcess[str]:
common = {
"TEXT_LENGTH": self.profile.text_length,
"PAGE_LIMIT": self.profile.page_limit,
}
flags = [
item
for key, value in {**common, **env}.items()
for item in ("-e", f"{key}={value}")
]
proc = subprocess.run(
[
"docker",
"exec",
K6,
"k6",
"run",
"--quiet",
"--no-color",
"--log-format",
"raw",
*flags,
f"/scripts/{script}",
],
capture_output=True,
text=True,
check=False,
)
if check and proc.returncode != 0:
raise BenchError(k6_failure(script, proc))
return proc
def work_json(self, name: str) -> Any:
return json.loads((self.work_dir / name).read_text(encoding="utf-8"))
# -------------------------------------------------------------- измерение
def parse_summary(path: Path) -> dict[str, Any]:
"""Итог k6: успехи, ошибки, RPS и перцентили успешных запросов.
Нет файла (k6 остановлен сторожем до итога) — нулевой итог.
"""
if not path.exists():
return {
"duration_s": 0,
"requests_ok": 0,
"requests_failed": 0,
"check_failed": 0,
"rps": None,
"error_ratio": None,
"p50_ms": None,
"p95_ms": None,
"p99_ms": None,
"max_ms": None,
}
data = json.loads(path.read_text(encoding="utf-8"))
metrics = data["metrics"]
failed = metrics.get("http_req_failed", {}).get("values", {})
ok = int(failed.get("fails", 0))
bad = int(failed.get("passes", 0))
checks = metrics.get("checks", {}).get("values", {})
success = metrics.get(SUCCESS_METRIC, {}).get("values", {})
duration = data["state"]["testRunDurationMs"] / 1000
return {
"duration_s": round(duration, 3),
"requests_ok": ok,
"requests_failed": bad,
"check_failed": int(checks.get("fails", 0)),
"rps": round(ok / duration, 1) if duration else None,
"error_ratio": round(bad / (ok + bad), 6) if ok + bad else None,
"p50_ms": rounded(success.get("p(50)")),
"p95_ms": rounded(success.get("p(95)")),
"p99_ms": rounded(success.get("p(99)")),
"max_ms": rounded(success.get("max")),
}
@dataclass
class Measured:
summary: dict[str, Any]
before: dict[str, Any]
after: dict[str, Any]
rss_max_kib: int | None
def cores(self, key: str) -> float | None:
"""Среднее число занятых ядер за прогон по счётчику cgroup."""
duration = self.summary["duration_s"]
start, end = self.before[key], self.after[key]
if not duration or start is None or end is None:
return None
return round((end - start) / 1e6 / duration, 3)
@property
def app_cpu(self) -> float | None:
return self.cores("app_usec")
@property
def generator_cpu(self) -> float | None:
return self.cores("k6_usec")
@property
def db_cpu(self) -> float | None:
return self.cores("db_usec")
def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str:
tail = (proc.stdout + proc.stderr)[-3000:]
return f"k6 {script} failed ({proc.returncode}):\n{tail}"
def measure(
stand: Stand,
script: str,
env: dict[str, Any],
raw: Path,
) -> Measured:
"""Один запуск k6 со снимками до и после, опросом RSS и сторожем."""
before = stand.snapshot()
with Watcher() as watcher:
proc = stand.k6(script, env, check=False)
# если приложение или хранилище умерло, k6 остановлен сторожем: прогон
# пишется с пометкой, а не падает здесь
died = watcher.app_died or watcher.stand_died
if proc.returncode != 0 and not died:
raise BenchError(k6_failure(script, proc))
after = stand.snapshot()
values = [
v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v
]
return Measured(
summary=parse_summary(raw),
before=before,
after=after,
rss_max_kib=max(values) if values else None,
)
def run_flags(
profile: Profile,
measured: Measured,
app: dict[str, Any],
storages_down: list[str],
) -> list[str]:
flags = []
if app["oom"]:
flags.append("oom")
if not app["running"] or app["restarts"]:
flags.append("app_down")
if storages_down:
flags.append("stand_down")
summary = measured.summary
if summary["requests_failed"] or summary["check_failed"]:
flags.append("errors")
cpu = measured.generator_cpu
if cpu is not None and cpu >= profile.generator_cpu_limit:
flags.append("generator")
return flags
# ----------------------------------------------------------- план запроса
def plan_entries(log: str, marker: str) -> list[str]:
"""Записи auto_explain между маркерами этапа плана.
Запись начинается строкой `LOG: duration: … plan:` и продолжается
строками с табуляцией; префикс времени и pid отбрасывается.
"""
lines = log.splitlines()
bounds = [
next(
(i for i, line in enumerate(lines) if f"{marker} {edge}" in line),
None,
)
for edge in ("start", "end")
]
if None in bounds:
return []
entries: list[list[str]] = []
current: list[str] | None = None
for line in lines[bounds[0] + 1 : bounds[1]]:
if line.startswith("\t"):
if current is not None:
current.append(line[1:])
continue
current = None
head = line.partition("LOG: ")[2].rstrip()
if head.startswith("duration:") and head.endswith("plan:"):
current = [head]
entries.append(current)
return ["\n".join(entry) for entry in entries]
def is_history(entry: str) -> bool:
"""Запись — запрос страницы истории (читает таблицу messages).
Остальные записи окна (проверка сессии с хэшем токена в параметрах,
служебные запросы) в результаты не попадают.
"""
text = entry.partition("Query Parameters:")[0]
return "Query Text:" in text and "FROM messages" in text
def plan_parts(entry: str) -> dict[str, Any] | None:
"""Текст запроса, параметры и тело плана записи auto_explain."""
lines = entry.splitlines()
starts = [i for i, line in enumerate(lines) if line.startswith("Query ")]
if len(starts) != 2 or not lines[starts[1]].startswith("Query Param"):
return None
text = "\n".join(lines[starts[0] : starts[1]])
params = lines[starts[1]].partition(": ")[2]
return {
"text": text.removeprefix("Query Text: "),
"params": [
value.replace("''", "'")
for value in re.findall(r"\$\d+ = '((?:[^']|'')*)'", params)
],
"plan": "\n".join(lines[starts[1] + 1 :]),
}
def plan_summary(plan: str, time_ms: float | None) -> dict[str, Any]:
"""Узел доступа к messages, буферы верхнего узла, строки, время."""
node = re.search(
r"((?:Parallel )?(?:Seq Scan|Index Only Scan|Index Scan|"
r"Bitmap Heap Scan)(?: Backward)?(?: using \w+)? on messages)",
plan,
)
buffers = re.search(r"Buffers: ([^\n]*)", plan)
hit = re.search(r"shared hit=(\d+)", buffers.group(1)) if buffers else None
read = re.search(r"read=(\d+)", buffers.group(1)) if buffers else None
rows = re.search(r"\(actual time=[\d.]+\.\.[\d.]+ rows=([\d.]+)", plan)
if time_ms is None:
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
time_ms = float(execution.group(1)) if execution else None
return {
"node": node.group(1) if node else None,
"shared_hit": int(hit.group(1)) if hit else 0,
"shared_read": int(read.group(1)) if read else 0,
"rows": float(rows.group(1)) if rows else None,
"time_ms": time_ms,
# параметры в условиях плана — generic plan, литералы — custom
"kind": "generic" if re.search(r"\$\d", plan) else "custom",
}
def entry_duration(entry: str) -> float | None:
match = re.match(r"duration: ([\d.]+) ms", entry)
return float(match.group(1)) if match else None
def index_name(plan: str) -> str | None:
match = INDEX_NODE_RE.search(plan)
return (
next((name for name in match.groups() if name), None)
if match
else None
)
def render_explain(
size: int,
git: dict[str, Any],
positions: dict[str, int],
entries: list[str],
contrast: dict[str, str] | None,
) -> str:
"""Файл `explain-<размер>.txt`: планы приложения и пара psql."""
lines = [
f"# План запроса истории: размер {human(size)}",
"",
f"Commit {git['commit']}"
+ (" + незакоммиченные изменения" if git["dirty"] else ""),
"Страница истории читаемого чата; позиция — сколько сообщений",
"истории до страницы:",
*(f"- {page}: после {after}" for page, after in positions.items()),
"",
"## Планы приложения",
"",
"auto_explain (ANALYZE, BUFFERS, TIMING): запрос, параметры и план,",
"с которыми его выполнило приложение (prepared statement asyncpg).",
"",
]
for page, entry in zip(EXPLAIN_PAGES, entries, strict=True):
lines += [f"### {page}", "", entry, ""]
if contrast is None:
lines += [
"## Средняя страница без индекса",
"",
"План приложения не использовал индекс: сравнения нет.",
"",
]
return "\n".join(lines)
index = contrast["index"]
lines += [
"## Средняя страница в psql: с индексом и без",
"",
"Тот же текст запроса и те же параметры, что у плана middle.",
"psql планирует их заново (custom plan). DROP INDEX — внутри",
"транзакции, затем ROLLBACK: индекс остаётся в базе.",
"",
f"### С индексом {index}",
"",
contrast["with_index"],
"",
f"### Без индекса {index}",
"",
contrast["without_index"],
"",
]
return "\n".join(lines)
# ------------------------------------------------------------ результаты
class Results:
"""Файлы метки: runs.csv, prefill.csv, meta.json, progress.json."""
def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None:
self.dir = label_dir
self.meta = meta
self.runs: list[dict[str, Any]] = []
self.prefill: list[dict[str, Any]] = []
self.sizes: list[dict[str, Any]] = []
(label_dir / "raw").mkdir(parents=True)
self.write_meta()
def append(
self,
name: str,
fields: list[str],
row: dict[str, Any],
) -> None:
path = self.dir / name
new = not path.exists()
with path.open("a", newline="", encoding="utf-8") as file:
writer = csv.DictWriter(
file,
fieldnames=fields,
lineterminator="\n",
)
if new:
writer.writeheader()
writer.writerow({key: row.get(key) for key in fields})
file.flush()
def add_run(self, row: dict[str, Any]) -> None:
self.runs.append(row)
self.append("runs.csv", RUN_FIELDS, row)
def add_prefill(self, row: dict[str, Any]) -> None:
self.prefill.append(row)
self.append("prefill.csv", PREFILL_FIELDS, row)
def write_meta(self) -> None:
self.meta["sizes"] = self.sizes
text = json.dumps(self.meta, ensure_ascii=False, indent=2)
(self.dir / "meta.json").write_text(text + "\n", encoding="utf-8")
def write_progress(self, progress: dict[str, Any]) -> None:
progress["updated_at"] = datetime.now(UTC).isoformat(
timespec="seconds",
)
text = json.dumps(progress, ensure_ascii=False, indent=2)
(self.dir / "progress.json").write_text(text + "\n", encoding="utf-8")
def write_summary(self) -> None:
text = build_summary(self.meta, self.runs, self.prefill, self.sizes)
(self.dir / "summary.md").write_text(text, encoding="utf-8")
def cell(values: list[float | None], digits: int = 1) -> str:
"""Медиана и min–max повторов; один повтор — просто значение."""
present = [v for v in values if v is not None]
if not present:
return "—"
median = statistics.median(present)
if len(present) == 1:
return f"{median:.{digits}f}"
return (
f"{median:.{digits}f} ({min(present):.{digits}f}–"
f"{max(present):.{digits}f})"
)
def build_summary(
meta: dict[str, Any],
runs: list[dict[str, Any]],
prefill: list[dict[str, Any]],
sizes: list[dict[str, Any]],
) -> str:
profile = meta["profile"]
lines = [
f"# Нагрузочный замер: метка `{meta['label']}`",
"",
f"- Статус: **{meta['status']}**"
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
f"- Commit: `{meta['git']['commit']}`"
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
f"- Профиль `{profile['name']}`: размеры "
f"{', '.join(human(s) for s in profile['sizes'])}; клиенты "
f"{', '.join(map(str, profile['concurrency']))}; повторов "
f"{profile['repeats']}; чтение {profile['read_measure_s']} с после "
f"{profile['read_warmup_s']} с прогрева; запись "
f"{profile['write_measure']} после {profile['write_warmup']}",
f"- Лимиты без swap: API {profile['cpus']} CPU / "
f"{profile['memory']}, PostgreSQL {profile['db_cpus']} CPU / "
f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / "
f"{profile['cache_memory']}; значения — медиана (min–max) по "
"повторам",
"- Посторонние контейнеры в VM во время замера: "
+ (", ".join(meta.get("other_containers") or []) or "нет"),
"",
]
for scenario in ("read", "write"):
rows = [r for r in runs if r["scenario"] == scenario]
if not rows:
continue
lines += [
f"## Сценарий `{scenario}`",
"",
"| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | "
"max, мс | CPU API, ядер | CPU БД, ядер | RSS конец, МиБ | "
"Сообщений в начале | Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|---|",
]
cells = sorted({(r["size"], r["concurrency"]) for r in rows})
for size, conc in cells:
group = [
r
for r in rows
if r["size"] == size and r["concurrency"] == conc
]
flags = sorted({f for r in group for f in r["flags"].split()})
lines.append(
f"| {human(size)} | {conc} | "
f"{cell([r['rps'] for r in group], 0)} | "
f"{cell([r['p50_ms'] for r in group], 2)} | "
f"{cell([r['p95_ms'] for r in group], 2)} | "
f"{cell([r['p99_ms'] for r in group], 2)} | "
f"{cell([r['max_ms'] for r in group], 1)} | "
f"{cell([r['app_cpu_cores'] for r in group], 2)} | "
f"{cell([r['db_cpu_cores'] for r in group], 2)} | "
f"{cell([r['rss_end_mib'] for r in group], 1)} | "
f"{cell([r['store_messages_start'] for r in group], 0)} | "
f"{' '.join(flags) or '—'} |",
)
lines.append("")
if sizes:
lines += [
"## Память и наполнение по размерам",
"",
"Объём `messages` — после `VACUUM (ANALYZE)`, до прогонов: в "
"таблице ровно N сообщений читаемого чата.",
"",
"| Размер | RSS API до, МиБ | RSS API после, МиБ | "
"Байт RSS на сообщение | Таблица, МиБ | Индексы, МиБ | "
"Байт в БД на сообщение | Наполнение, сообщ/с | "
"Курсоров в пуле | Пик RSS за размер, МиБ |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for info in sizes:
lines.append(
f"| {human(info['size'])} | {info['rss_before_mib']} | "
f"{info['rss_after_mib']} | {info['bytes_per_message']} | "
f"{mib_bytes(info.get('messages_table_bytes'))} | "
f"{mib_bytes(info.get('messages_indexes_bytes'))} | "
f"{info.get('db_bytes_per_message', '—')} | "
f"{info['prefill_rate']} | {info['cursors']} | "
f"{info.get('rss_peak_mib', '—')} |",
)
lines.append("")
lines += [
"## Планы запроса истории",
"",
"Страница читаемого чата (`limit` профиля). `app` — план "
"запроса приложения (auto_explain), `psql` — тот же запрос с "
"теми же параметрами в psql, `no index` — без индекса в "
"откатываемой транзакции. Буферы — верхнего узла плана, файлы "
"— `explain-<размер>.txt`.",
"",
"| Размер | Страница | После сообщений | Источник | Узел | "
"shared hit | read | Строк | Время, мс | План |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for info in sizes:
for plan in info.get("plans", []):
lines.append(
f"| {human(info['size'])} | {plan['page']} | "
f"{plan['after']} | {plan['source']} | "
f"{plan['node'] or '—'} | {plan['shared_hit']} | "
f"{plan['shared_read']} | {fmt(plan['rows'])} | "
f"{fmt(plan['time_ms'], 3)} | {plan['kind']} |",
)
lines.append("")
if prefill:
largest = max(r["size"] for r in prefill)
curve = [r for r in prefill if r["size"] == largest]
lines += [
f"## Кривая наполнения {human(largest)}: запись при росте "
f"хранилища, {profile['prefill_vus']} клиента",
"",
"| Порция | Сообщений после | RPS | p50, мс | p95, мс | "
"p99, мс | max, мс | CPU API, ядер | CPU БД, ядер | RSS, МиБ | "
"Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|",
]
for r in curve:
lines.append(
f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | "
f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | "
f"{r['max_ms']} | {r['app_cpu_cores']} | "
f"{r['db_cpu_cores']} | {r['rss_end_mib']} "
f"| {r['flags'] or '—'} |",
)
lines.append("")
if meta.get("host_warnings"):
lines += ["## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
lines.append("")
return "\n".join(lines)
# ---------------------------------------------------------------- прогон
class Bench:
def __init__(
self,
profile: Profile,
label: str,
results: Results,
) -> None:
self.profile = profile
self.label = label
self.results = results
self.stand = Stand(profile, results.dir)
self.started = time.monotonic()
self.runs_total = (
len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats
)
self.runs_done = 0
self.stage = "start"
self.size: int | None = None
# скорости стенда для оценки оставшегося времени
self.prefill_rate = 1500.0
self.write_rps: dict[int, float] = {}
self.run_overhead = 3.0
self.done_sizes = 0
self.prefilled = 0
# текущий размер наполнен и курсоры собраны
self.prepared = False
# ------------------------------------------------------------ вывод
def say(self, text: str) -> None:
elapsed = int(time.monotonic() - self.started)
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
print(f"[{stamp}] {text}", flush=True)
def eta(self) -> float:
p = self.profile
left = 0.0
for index, size in enumerate(p.sizes):
current = index == self.done_sizes
if index < self.done_sizes or (current and self.prepared):
continue
filled = self.prefilled if current else 0
left += (size - filled) / self.prefill_rate + size / 100 / 500
# стенд, миграции, обслуживание БД и план
left += 30
per_size = len(p.concurrency) * p.repeats
reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes)
done_in_size = self.runs_done - self.done_sizes * per_size * 2
reads_left -= min(done_in_size, per_size)
writes_left -= max(0, done_in_size - per_size)
read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead
left += reads_left * read_time
rps = (
statistics.mean(self.write_rps.values())
if self.write_rps
else 1000
)
write_time = (p.write_warmup + p.write_measure) / rps
left += writes_left * (write_time + self.run_overhead)
return left
def progress(self, last: dict[str, Any] | None = None) -> None:
state = {
"label": self.label,
"profile": self.profile.name,
"stage": self.stage,
"size": self.size,
"runs_done": self.runs_done,
"runs_total": self.runs_total,
"elapsed_s": round(time.monotonic() - self.started),
"eta_s": round(self.eta()),
"warnings": self.results.meta.get("host_warnings", []),
}
if last is not None:
state["last_run"] = last
self.results.write_progress(state)
# --------------------------------------------------------- этапы
def run(self) -> None:
for index, size in enumerate(self.profile.sizes, start=1):
self.size = size
self.prefilled = 0
self.prepared = False
self.run_size(index, size)
self.done_sizes += 1
def run_size(self, index: int, size: int) -> None:
stand = self.stand
name = human(size)
self.stage = f"size {name}: start"
self.say(f"size {name} ({index}/{len(self.profile.sizes)}): start")
stand.start(lambda text: self.say(f"size {name}: {text}"))
limits = stand.limits()
self.results.meta.setdefault("limits", limits)
if limits != self.results.meta["limits"]:
raise BenchError(f"limits changed between sizes: {limits}")
if "stand" not in self.results.meta:
self.results.meta["stand"] = stand.passport()
self.results.write_meta()
rss_start = stand.snapshot()["rss_kib"]
self.say(f"size {name}: app started, RSS {mib(rss_start)} MiB")
stand.k6("prepare.js", {"MEMBERS": self.profile.members})
self.progress()
rss_before = stand.snapshot()["rss_kib"]
prefill_seconds = self.run_prefill(size)
after = stand.snapshot()
rss_after = after["rss_kib"]
self.stage = f"size {name}: maintenance"
self.progress()
seconds = stand.maintain()
storage = stand.storage()
if storage["messages_rows"] != size:
raise BenchError(
f"messages table has {storage['messages_rows']} rows after "
f"prefill, expected {size}",
)
self.say(
f"size {name}: VACUUM (ANALYZE) and CHECKPOINT in {seconds:.1f}s; "
f"messages {mib_bytes(storage['messages_table_bytes'])} MiB + "
f"indexes {mib_bytes(storage['messages_indexes_bytes'])} MiB, "
f"{storage['db_bytes_per_message']} B/msg",
)
self.stage = f"size {name}: cursors"
self.progress()
stand.k6(
"cursors.js",
{
"WALK_LIMIT": self.profile.walk_limit,
"POOL_SIZE": self.profile.cursor_pool,
},
)
pool = stand.work_json("cursors.json")
if pool["walked"] != size:
raise BenchError(
f"read chat has {pool['walked']} messages, expected {size}",
)
self.say(f"size {name}: cursors {len(pool['cursors'])}")
self.stage = f"size {name}: query plan"
self.progress()
plans = self.explain(size, pool)
info = {
"size": size,
"rss_before_mib": mib(rss_before),
"rss_after_mib": mib(rss_after),
"bytes_per_message": round(
(rss_after - rss_before) * 1024 / size,
),
"prefill_seconds": round(prefill_seconds, 1),
"prefill_rate": round(size / prefill_seconds),
"cursors": len(pool["cursors"]),
"maintenance_seconds": round(seconds, 1),
**storage,
"plans": plans,
}
self.results.sizes.append(info)
self.results.write_meta()
self.prepared = True
self.say(
f"size {name}: prefill done, app RSS {info['rss_after_mib']} "
f"MiB (+{info['bytes_per_message']} B/msg)",
)
store = size
for scenario in ("read", "write"):
for conc in self.profile.concurrency:
for repeat in range(1, self.profile.repeats + 1):
store = self.run_one(scenario, size, conc, repeat, store)
if scenario == "read":
self.check_read_chat(size)
info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"])
self.results.write_meta()
if not stand.stop():
raise BenchError(f"stand of size {name} was not removed")
def check_read_chat(self, size: int) -> None:
"""После блока чтения в читаемом чате по-прежнему ровно N.
Новое сообщение встало бы в конец истории, поэтому достаточно
дочитать хвост от последнего курсора пула.
"""
self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit})
tail = self.stand.work_json("tail.json")
total = tail["from_after"] + tail["tail"]
if total != size:
raise BenchError(
f"read chat has {total} messages after the read block, "
f"expected {size}",
)
self.say(f"size {human(size)}: read chat unchanged, {total} messages")
def explain(self, size: int, pool: dict[str, Any]) -> list[dict[str, Any]]:
"""Этап плана: EXPLAIN (ANALYZE, BUFFERS) запросов истории приложения.
auto_explain включается между маркерами, `explain.js` читает первую,
среднюю и последнюю полную страницу; планы берутся из журнала
Postgres. Затем средняя страница в psql с индексом и без него
(`DROP INDEX` в откатываемой транзакции). Решение 5 design change
`bench-postgres`.
"""
stand = self.stand
marker = f"bench-explain {uuid.uuid4().hex}"
enable = "".join(
f"ALTER SYSTEM SET {key} = '{value}';\n"
for key, value in EXPLAIN_SETTINGS.items()
)
disable = "".join(
f"ALTER SYSTEM RESET {key};\n" for key in EXPLAIN_SETTINGS
)
stand.psql(
f"DO $$ BEGIN RAISE LOG '{marker} start'; END $$;\n"
f"{enable}SELECT pg_reload_conf();\n",
)
time.sleep(RELOAD_PAUSE_S)
try:
stand.k6("explain.js", {})
finally:
stand.psql(
f"{disable}SELECT pg_reload_conf();\n"
f"DO $$ BEGIN RAISE LOG '{marker} end'; END $$;\n",
)
# соединения пула API должны получить выключение до прогонов
time.sleep(RELOAD_PAUSE_S)
entries = [
entry
for entry in plan_entries(stand.db_log(), marker)
if is_history(entry)
]
if len(entries) != len(EXPLAIN_PAGES):
raise BenchError(
f"size {human(size)}: expected {len(EXPLAIN_PAGES)} history "
f"query plans in the PostgreSQL log, found {len(entries)}",
)
afters = pool["afters"]
positions = dict(
zip(
EXPLAIN_PAGES,
(0, afters[len(afters) // 2], afters[-1]),
strict=True,
),
)
plans = []
for page, entry in zip(EXPLAIN_PAGES, entries, strict=True):
parts = plan_parts(entry) or {"plan": entry}
plans.append(
{
"page": page,
"after": positions[page],
"source": "app",
**plan_summary(parts["plan"], entry_duration(entry)),
},
)
contrast = self.explain_without_index(entries[1])
if contrast is not None:
for key, source in (
("with_index", "psql"),
("without_index", "psql, no index"),
):
plans.append(
{
"page": "middle",
"after": positions["middle"],
"source": source,
**plan_summary(contrast[key], None),
},
)
text = render_explain(
size,
self.results.meta["git"],
positions,
entries,
contrast,
)
path = self.results.dir / f"explain-{human(size)}.txt"
path.write_text(text, encoding="utf-8")
for plan in plans:
self.say(
f"size {human(size)}: plan {plan['page']} ({plan['source']}): "
f"{plan['node']}, shared hit={plan['shared_hit']} "
f"read={plan['shared_read']}, {plan['time_ms']} ms",
)
return plans
def explain_without_index(self, entry: str) -> dict[str, str] | None:
"""Тот же запрос и параметры в psql: с индексом и без него.
`DROP INDEX` идёт внутри транзакции, которая откатывается: индекс
остаётся, что проверяется по `pg_indexes`. Нагрузки в этот момент
нет. План приложения без индекса — сравнения нет, None.
"""
parts = plan_parts(entry)
index = index_name(parts["plan"]) if parts else None
if parts is None or index is None:
return None
args = ", ".join(
"'" + value.replace("'", "''") + "'" for value in parts["params"]
)
out = self.stand.psql(
"SET lock_timeout = '5s';\n"
"BEGIN;\n"
f"PREPARE page AS {parts['text']};\n"
"\\echo @@with-index\n"
f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE page({args});\n"
f'DROP INDEX "{index}";\n'
"\\echo @@without-index\n"
f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE page({args});\n"
"ROLLBACK;\n",
)
with_index, _, without_index = out.partition("@@without-index")
kept = self.stand.psql(
"SELECT count(*) FROM pg_indexes WHERE tablename = 'messages' "
f"AND indexname = '{index}';\n",
)
if kept != "1":
raise BenchError(f"index {index} is missing after the rollback")
return {
"index": index,
"with_index": with_index.replace("@@with-index", "").strip(),
"without_index": without_index.strip(),
}
def run_prefill(self, size: int) -> float:
chunk_size = self.profile.prefill_chunk
total_seconds = 0.0
for chunk, offset in enumerate(range(0, size, chunk_size), start=1):
count = min(chunk_size, size - offset)
name = f"prefill-{human(size)}-{chunk:02d}"
self.stage = f"size {human(size)}: prefill"
measured = measure(
self.stand,
"prefill.js",
{
"COUNT": count,
"OFFSET": offset,
"VUS": self.profile.prefill_vus,
"NAME": name,
},
self.results.dir / "raw" / f"{name}.json",
)
summary = measured.summary
down = self.stand.down_storages()
flags = run_flags(
self.profile,
measured,
self.stand.state(APP),
down,
)
self.results.add_prefill(
{
**summary,
"size": size,
"chunk": chunk,
"messages_before": offset,
"messages_after": offset + summary["requests_ok"],
"app_cpu_cores": measured.app_cpu,
"rss_end_mib": mib(measured.after["rss_kib"]),
"mem_cgroup_end_mib": mib_bytes(
measured.after["mem_bytes"],
),
"generator_cpu_cores": measured.generator_cpu,
"flags": " ".join(flags),
"db_cpu_cores": measured.db_cpu,
},
)
if down:
raise BenchError(
f"{', '.join(down)} went down during prefill chunk "
f"{chunk}: matrix stopped",
)
if summary["requests_ok"] != count:
raise BenchError(
f"prefill chunk {chunk}: {summary['requests_ok']} of "
f"{count} messages confirmed ({' '.join(flags)})",
)
total_seconds += summary["duration_s"]
self.prefilled = offset + count
self.prefill_rate = self.prefilled / total_seconds
self.say(
f"size {human(size)}: prefill {self.prefilled}/{size} "
f"({self.prefilled * 100 // size}%), "
f"{self.prefill_rate:.0f} msg/s, "
f"~{fmt_left((size - self.prefilled) / self.prefill_rate)} "
"left",
)
self.progress()
return total_seconds
def run_one(
self,
scenario: str,
size: int,
conc: int,
repeat: int,
store: int,
) -> int:
"""Прогрев и измеряемый прогон; возвращает сообщений в хранилище."""
p = self.profile
number = self.runs_done + 1
name = f"{scenario}-{human(size)}-c{conc}-r{repeat}"
self.stage = f"run {number}/{self.runs_total} {name}"
self.progress()
wall = time.monotonic()
if scenario == "read":
warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"}
self.stand.k6(
"read.js",
{**warm, "SEED": p.seed + p.warmup_seed_shift},
)
env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"}
env["SEED"] = p.seed
else:
self.stand.k6(
"write.js",
{"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store},
)
warm_ok = parse_summary(self.stand.work_dir / "last.json")
store += warm_ok["requests_ok"]
env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store}
measured = measure(
self.stand,
f"{scenario}.js",
{**env, "NAME": name},
self.results.dir / "raw" / f"{name}.json",
)
summary = measured.summary
app = self.stand.state(APP)
down = self.stand.down_storages()
flags = run_flags(p, measured, app, down)
end = store + (summary["requests_ok"] if scenario == "write" else 0)
row = {
**summary,
"run": number,
"scenario": scenario,
"size": size,
"concurrency": conc,
"repeat": repeat,
"app_cpu_cores": measured.app_cpu,
"rss_start_mib": mib(measured.before["rss_kib"]),
"rss_end_mib": mib(measured.after["rss_kib"]),
"rss_max_mib": mib(measured.rss_max_kib),
"mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]),
"store_messages_start": store,
"store_messages_end": end,
"generator_cpu_cores": measured.generator_cpu,
"flags": " ".join(flags),
"db_cpu_cores": measured.db_cpu,
}
self.results.add_run(row)
self.runs_done += 1
if scenario == "write" and summary["rps"]:
self.write_rps[conc] = summary["rps"]
overhead = time.monotonic() - wall - summary["duration_s"]
if scenario == "read":
overhead -= p.read_warmup_s
self.run_overhead = max(1.0, overhead)
errors = summary["requests_failed"] + summary["check_failed"]
self.say(
f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} "
f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps "
f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} "
f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} "
f"db {measured.db_cpu} rss {row['rss_end_mib']} MiB "
f"err {errors} "
f"~{fmt_left(self.eta())} left"
+ (f" [{' '.join(flags)}]" if flags else ""),
)
self.progress(last=row)
if "oom" in flags or "app_down" in flags:
raise BenchError(
f"{APP} is down after {name} ({' '.join(flags)}): "
"matrix stopped",
)
if down:
raise BenchError(
f"{', '.join(down)} went down during {name}: matrix stopped",
)
return end
# ------------------------------------------------------------------ CLI
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--profile", choices=sorted(PROFILES), required=True)
parser.add_argument("--label", required=True)
# отладка стенда: переопределения пишутся в паспорт как overrides
parser.add_argument("--sizes", help="через запятую, например 120000")
parser.add_argument("--concurrency", help="через запятую")
parser.add_argument("--repeats", type=int)
parser.add_argument("--memory", help="лимит памяти, например 64m")
return parser.parse_args(argv)
def apply_overrides(
profile: Profile,
args: argparse.Namespace,
) -> tuple[Profile, dict[str, Any]]:
overrides: dict[str, Any] = {}
if args.sizes:
overrides["sizes"] = tuple(int(x) for x in args.sizes.split(","))
if args.concurrency:
overrides["concurrency"] = tuple(
int(x) for x in args.concurrency.split(",")
)
if args.repeats:
overrides["repeats"] = args.repeats
if args.memory:
overrides["memory"] = args.memory
return replace(profile, **overrides), overrides
def main(argv: list[str]) -> int:
args = parse_args(argv)
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label):
print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True)
return 2
label_dir = ROOT / "evidence" / "bench" / args.label
relative = label_dir.relative_to(ROOT)
if label_dir.exists():
print(f"label directory {relative} already exists", flush=True)
return 2
profile, overrides = apply_overrides(PROFILES[args.profile], args)
git = git_info()
state = host_state()
try:
warnings = host_warnings(state) + vm_warnings(profile)
except BenchError as exc:
print(f"bench failed: {exc}", flush=True)
return 2
meta: dict[str, Any] = {
"label": args.label,
"status": "incomplete",
"reason": "running",
"started_at": datetime.now(UTC).isoformat(timespec="seconds"),
"command": " ".join(["uv run python bench/run.py", *argv]),
"git": git,
"overrides": overrides,
"host_state": {"start": state},
"host_warnings": warnings,
# чужие контейнеры в той же VM: стенд их не трогает, но они шумят
"other_containers": sorted(
sh("docker", "ps", "--format", "{{.Names}}", check=False).split(),
),
}
stand_check = Stand(profile, label_dir)
started = time.monotonic()
def say(text: str) -> None:
elapsed = int(time.monotonic() - started)
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
print(f"[{stamp}] {text}", flush=True)
for warning in warnings:
say(f"WARNING: {warning}")
if overrides:
say(f"WARNING: profile overrides {overrides}: debug run")
if git["dirty"]:
say("NOTE: working tree has uncommitted changes (dirty: true)")
try:
stand_check.check_free()
image = stand_check.build_image()
except BenchError as exc:
print(f"bench failed: {exc}", flush=True)
return 1
say(f"image {APP_IMAGE} built ({image[:19]}…), compose project {PROJECT}")
meta.update(static_passport(profile))
results = Results(label_dir, meta)
bench = Bench(profile, args.label, results)
bench.started = started
code = 0
def terminate(*_: object) -> None:
raise KeyboardInterrupt
signal.signal(signal.SIGTERM, terminate)
try:
bench.run()
meta["status"] = "complete"
meta.pop("reason", None)
except KeyboardInterrupt:
meta["reason"] = "interrupted"
say("interrupted: partial results kept")
code = 130
except BenchError as exc:
meta["reason"] = str(exc).splitlines()[0]
say(f"bench failed: {exc}")
code = 1
finally:
# повторный Ctrl+C не должен оборвать уборку стенда
signal.signal(signal.SIGINT, signal.SIG_IGN)
if not bench.stand.stop():
say(f"cleanup incomplete: {leftovers()}")
code = code or 1
# fixture.json с токенами — только здесь, в результаты не попадает
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds")
meta["host_state"]["end"] = host_state()
results.write_meta()
results.write_summary()
bench.stage = "done" if code == 0 else meta.get("reason", "stopped")
bench.progress()
say(
f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, "
f"results in {relative}",
)
return code
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))