"""Нагрузочный стенд: матрица размеров истории и конкуренции. Запуск — `make bench` / `make bench_quick`. Стенд — API, PostgreSQL и Valkey: проект compose `mtusi-bench` из `docker-compose.yml` с override `bench/bench.compose.yml`. Методика — design changes `load-bench-harness` и `bench-postgres`, инварианты — `.claude/rules/bench.md`. Только stdlib: приложение не импортируется, его нагружает k6 по HTTP; к PostgreSQL стенда оркестратор ходит только для обслуживания и измерений (VACUUM, размеры, планы запроса). """ import argparse import csv import json import os import platform import re import secrets import shutil import signal import statistics import subprocess import sys import threading import time import uuid from collections.abc import Callable from dataclasses import asdict, dataclass, replace from datetime import UTC, datetime from pathlib import Path from typing import Any ROOT = Path(__file__).resolve().parents[1] K6_IMAGE = ( "grafana/k6:2.3.0@sha256:" "9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34" ) APP_IMAGE = "mtusi-messenger:bench" PROJECT = "mtusi-bench" NETWORK = "mtusi-bench" APP = "mtusi-bench-api" DB = "mtusi-bench-postgres" CACHE = "mtusi-bench-valkey" K6 = "mtusi-bench-k6" # второе хранилище этапа восстановления: контейнер, volume и имя в сети # стенда (по нему API переключается на восстановленную базу) RESTORE_DB = "mtusi-bench-postgres-restore" RESTORE_VOLUME = "mtusi-bench_restore" RESTORE_HOST = "postgres-restore" # таблицы данных приложения: сверка числа строк после восстановления DATA_TABLES = ("users", "chats", "chat_members", "sessions", "messages") # контейнеры стенда по ролям: лимиты, состояние, паспорт CONTAINERS = {"api": APP, "postgres": DB, "valkey": CACHE} COMPOSE_FILES = ( ROOT / "docker-compose.yml", ROOT / "bench" / "bench.compose.yml", ) # подстановки compose вместо .env рабочего стенда ENV_FILE = ROOT / "bench" / "bench.env" # что из окружения хоста нужно самому docker compose; остальное (в том # числе POSTGRES_* из shell) в стенд не попадает HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "DOCKER_HOST", "DOCKER_CONFIG") READY_CHECK = ( "import urllib.request; urllib.request.urlopen(" "'http://127.0.0.1:8080/health/ready', timeout=2)" ) SUCCESS_METRIC = "http_req_duration{expected_response:true}" # предупреждение о хосте: swap занят больше, МиБ SWAP_WARN_MIB = 1024 # предупреждение о VM: лимиты памяти стенда + этот запас больше памяти VM VM_RESERVE_GIB = 1.5 # предупреждение о хосте: реальное время прогона больше монотонного, с SLEEP_WARN_S = 5 # psql в контейнере Postgres от пользователя образа, без пароля; SQL — stdin PSQL = ( 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1' ) # настройки Postgres в паспорте: от них зависят план и стоимость записи PG_SETTINGS = ( "shared_buffers", "effective_cache_size", "work_mem", "maintenance_work_mem", "max_connections", "synchronous_commit", "fsync", "wal_level", "checkpoint_timeout", "max_wal_size", "random_page_cost", "jit", "autovacuum", "plan_cache_mode", "shared_preload_libraries", ) # auto_explain на этапе плана; вне его log_min_duration = -1 (выключен) EXPLAIN_SETTINGS = { "auto_explain.log_min_duration": "0", "auto_explain.log_analyze": "on", "auto_explain.log_buffers": "on", "auto_explain.log_timing": "on", "auto_explain.log_format": "text", } # пауза после pg_reload_conf(): SIGHUP доходит до соединений пула API RELOAD_PAUSE_S = 1.0 # индекс, который использовал план приложения INDEX_NODE_RE = re.compile( r"Index Only Scan(?: Backward)? using (\w+)" r"|Index Scan(?: Backward)? using (\w+)" r"|Bitmap Index Scan on (\w+)", ) # условие курсора в запросе приложения: сравнение кортежей OR_CURSOR_RE = re.compile( r"\(messages\.created_at, messages\.id\) > " r"\((\$\d+::[A-Z][A-Z ]*[A-Z]), (\$\d+::UUID)\)", ) # длина текста сообщения по контракту (contracts/openapi.json, text # maxLength): проба записи проверяет, что такое сообщение вставляется MAX_TEXT_LENGTH = 4000 # сообщение максимальной длины из несжимаемых символов CJK (~12 КБ UTF-8, # md5 номера символа — детерминировано): вставка в DO, итог — NOTICE LONG_TEXT_PROBE = f"""DO $$ BEGIN INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT gen_random_uuid(), chat_id, sender_id, (SELECT string_agg( chr(19968 + ('x' || substr(md5(i::text), 1, 4))::bit(16)::int % 20000), '') FROM generate_series(1, {MAX_TEXT_LENGTH}) i), now() + interval '1 hour' FROM probe_src LIMIT 1; RAISE NOTICE 'long-text: ok'; EXCEPTION WHEN others THEN RAISE NOTICE 'long-text: %', SQLERRM; END $$; """ RUN_FIELDS = [ "run", "scenario", "size", "concurrency", "repeat", "duration_s", "requests_ok", "requests_failed", "rps", "error_ratio", "p50_ms", "p95_ms", "p99_ms", "max_ms", "app_cpu_cores", "rss_start_mib", "rss_end_mib", "rss_max_mib", "mem_cgroup_end_mib", "store_messages_start", "store_messages_end", "generator_cpu_cores", "flags", "check_failed", # лаба 2: новые столбцы — только в конец "db_cpu_cores", ] PREFILL_FIELDS = [ "size", "chunk", "messages_before", "messages_after", "duration_s", "requests_ok", "requests_failed", "rps", "error_ratio", "p50_ms", "p95_ms", "p99_ms", "max_ms", "app_cpu_cores", "rss_end_mib", "mem_cgroup_end_mib", "generator_cpu_cores", "flags", "db_cpu_cores", ] @dataclass(frozen=True) class Profile: name: str sizes: tuple[int, ...] concurrency: tuple[int, ...] repeats: int read_warmup_s: int read_measure_s: int write_warmup: int write_measure: int # порция ~30 с при ~850 сообщ/с на Postgres: прогресс наполнения # печатается после порции, а спек требует строку не реже раза в минуту prefill_chunk: int = 25_000 prefill_vus: int = 32 text_length: int = 64 page_limit: int = 50 walk_limit: int = 100 cursor_pool: int = 1000 members: int = 10 seed: int = 20260930 # зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно # те страницы, с которых начнётся замер warmup_seed_shift: int = 1_000_003 generator_cpu_limit: float = 4.0 cpus: str = "1" memory: str = "2g" # лаба 2: лимиты хранилищ (swap = память) и срок сессии стенда; идут в # compose переменными BENCH_* (bench/bench.compose.yml). Решение 2 # design change bench-postgres db_cpus: str = "4" db_memory: str = "3g" cache_cpus: str = "2" cache_memory: str = "512m" session_ttl_seconds: int = 21600 # трек 2: фоновые чаты уровнями (число чатов, сообщений в чате), то же # на каждом размере; варианты индекса истории (имя, столбцы) для # планов и стоимости записи; пробы записи — строк и повторов. # Решения 1, 4, 5 design change track2-query-plans background: tuple[tuple[int, int], ...] = () index_variants: tuple[tuple[str, str], ...] = ( ("chat_only", "(chat_id)"), ("covering", "(chat_id, created_at, id) INCLUDE (sender_id, text)"), ) write_probe_rows: int = 20_000 write_probe_repeats: int = 3 # трек 2, часть 2: backup и восстановление в новое хранилище в конце # размера; pg_restore -j, запись через HTTP во время backup. Решение 1 # design change track2-backup-restore restore: bool = False restore_jobs: int = 4 restore_write_load: int = 5_000 restore_write_vus: int = 10 @property def background_total(self) -> int: return sum(count * size for count, size in self.background) PROFILES = { "full": Profile( name="full", # 500k вместо 1M лабы 1: прогон ~30 мин вместо ~45, а диапазон # 10k → 500k (×50) уже показывает, растёт ли страница с историей sizes=(10_000, 100_000, 500_000), concurrency=(1, 10, 50), repeats=3, read_warmup_s=5, read_measure_s=20, write_warmup=500, write_measure=5000, # 100 000 фоновых сообщений: в таблице ≥110 000 на любом размере background=((10, 5_000), (100, 400), (1_000, 10)), ), "quick": Profile( name="quick", sizes=(1_000, 5_000, 10_000), concurrency=(1, 4, 16), repeats=1, read_warmup_s=2, read_measure_s=5, write_warmup=100, write_measure=1000, background=((2, 1_000), (10, 100), (50, 10)), write_probe_rows=5_000, ), # backup и восстановление на данных полной матрицы, без прогонов "restore": Profile( name="restore", sizes=(10_000, 100_000, 500_000), concurrency=(), repeats=0, read_warmup_s=0, read_measure_s=0, write_warmup=0, write_measure=0, background=((10, 5_000), (100, 400), (1_000, 10)), restore=True, ), "restore_quick": Profile( name="restore_quick", sizes=(1_000, 5_000, 10_000), concurrency=(), repeats=0, read_warmup_s=0, read_measure_s=0, write_warmup=0, write_measure=0, background=((2, 1_000), (10, 100), (50, 10)), write_probe_rows=5_000, restore=True, restore_write_load=1_000, ), } class BenchError(Exception): """Стенд не может продолжать: сообщение для человека.""" # ---------------------------------------------------------------- утилиты def sh(*args: str, check: bool = True) -> str: """Команда без shell; stdout без хвостовых пробелов. `LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод разбирается регулярками с точкой. """ proc = subprocess.run( args, capture_output=True, text=True, check=False, env={**os.environ, "LC_ALL": "C"}, ) if check and proc.returncode != 0: raise BenchError( f"{' '.join(args[:4])}…: exit {proc.returncode}\n" f"{(proc.stdout + proc.stderr)[-2000:]}", ) return proc.stdout.strip() def human(n: int) -> str: """10000 → 10k, 1000000 → 1M.""" if n >= 1_000_000 and n % 1_000_000 == 0: return f"{n // 1_000_000}M" if n >= 1000 and n % 1000 == 0: return f"{n // 1000}k" return str(n) def fmt_left(seconds: float) -> str: seconds = max(0, int(seconds)) hours, rest = divmod(seconds, 3600) minutes, secs = divmod(rest, 60) if hours: return f"{hours}h{minutes:02d}m" if minutes: return f"{minutes}m{secs:02d}s" return f"{secs}s" def mib(kib: float | None) -> float | None: return None if kib is None else round(kib / 1024, 1) def mib_bytes(value: int | None) -> float | None: return None if value is None else round(value / 2**20, 1) def size_bytes(text: str) -> int: """Лимит памяти docker (`512m`, `2g`) в байтах.""" match = re.fullmatch(r"(\d+(?:\.\d+)?)([kmg]?)b?", text.strip().lower()) if not match: raise BenchError(f"bad memory limit {text!r}: use 512m or 2g") power = {"": 0, "k": 1, "m": 2, "g": 3}[match.group(2)] return int(float(match.group(1)) * 1024**power) def fmt(value: float | None, digits: int = 0) -> str: return "—" if value is None else f"{value:.{digits}f}" def rounded(value: float | None, digits: int = 3) -> float | None: return None if value is None else round(value, digits) def chat_sizes(profile: Profile, size: int) -> list[int]: """Размеры чатов наполнения: читаемый, затем фоновые по уровням.""" return [size] + [ chat_size for count, chat_size in profile.background for _ in range(count) ] def fill_schedule(sizes: list[int]) -> list[int]: """Номер чата для каждого сообщения наполнения. Чат с s сообщениями получает позиции (k + 0,5) / s, k = 0…s−1; все позиции сортируются, при равенстве — по номеру чата. Сообщения каждого чата равномерно размазаны по наполнению, порядок детерминирован (решение 2 design change `track2-query-plans`). """ slots = sorted( ((k + 0.5) / size, chat) for chat, size in enumerate(sizes) for k in range(size) ) return [chat for _, chat in slots] def or_cursor(text: str) -> str: """Запрос приложения с условием курсора через OR вместо кортежей. Остальное — список столбцов, ORDER BY, LIMIT — как у приложения. Нет шаблона — приложение изменило запрос, тихой копии SQL не делаем. """ new, count = OR_CURSOR_RE.subn( r"(messages.created_at > \1 OR " r"(messages.created_at = \1 AND messages.id > \2))", text, ) if count != 1: raise BenchError( "history query has no (created_at, id) row comparison: the app " "query changed, update OR_CURSOR_RE", ) return new # ------------------------------------------------------------ хост и паспорт def host_state() -> dict[str, Any]: """Swap, свободная память и питание (macOS); на другой ОС — null.""" state: dict[str, Any] = { "swap_used_mib": None, "swap_total_mib": None, "memory_free_percent": None, "power": None, "battery_percent": None, } if platform.system() != "Darwin": return state swap = sh("sysctl", "-n", "vm.swapusage", check=False) for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")): match = re.search(rf"{key} = ([\d.,]+)M", swap) if match: state[name] = float(match.group(1).replace(",", ".")) pressure = sh("memory_pressure", check=False) match = re.search(r"free percentage: (\d+)%", pressure) if match: state["memory_free_percent"] = int(match.group(1)) batt = sh("pmset", "-g", "batt", check=False) match = re.search(r"drawing from '([^']+)'", batt) if match: state["power"] = match.group(1) match = re.search(r"(\d+)%;", batt) if match: state["battery_percent"] = int(match.group(1)) return state def host_warnings(state: dict[str, Any]) -> list[str]: warnings = [] if state["power"] == "Battery Power": warnings.append("host on battery power: CPU may be throttled") used = state["swap_used_mib"] if used is not None and used > SWAP_WARN_MIB: warnings.append( f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: " "reboot before a full run", ) return warnings def vm_warnings(profile: Profile) -> list[str]: """Лимиты памяти стенда не помещаются в VM docker с запасом. Тогда лимит перестаёт быть гарантией: контейнеры делят память VM, и метки разных дней несравнимы. """ total = sh("docker", "info", "--format", "{{.MemTotal}}", check=False) if not total.isdigit(): return [] limits = sum( size_bytes(value) for value in (profile.memory, profile.db_memory, profile.cache_memory) ) budget = int(total) - int(VM_RESERVE_GIB * 2**30) if limits <= budget: return [] return [ f"memory limits {limits / 2**30:.1f} GiB + {VM_RESERVE_GIB} GiB " f"reserve exceed docker VM memory {int(total) / 2**30:.1f} GiB: " "limits are not guaranteed", ] def git_info() -> dict[str, Any]: commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD") status = sh("git", "-C", str(ROOT), "status", "--porcelain") return {"commit": commit, "dirty": bool(status)} def static_passport(profile: Profile) -> dict[str, Any]: docker = json.loads(sh("docker", "info", "--format", "{{json .}}")) versions = sh( "docker", "run", "--rm", "--entrypoint", "python", APP_IMAGE, "-c", "import importlib.metadata as m, json, platform; print(json.dumps(" "{'python': platform.python_version(), **{p: m.version(p) for p in " "('fastapi', 'starlette', 'pydantic', 'uvicorn', 'sqlalchemy', " "'asyncpg', 'redis')}}))", ) k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version") host: dict[str, Any] = {"os": platform.system(), "machine": None} if platform.system() == "Darwin": host["os"] = f"macOS {platform.mac_ver()[0]}" host["machine"] = sh( "sysctl", "-n", "machdep.cpu.brand_string", check=False, ) memsize = sh("sysctl", "-n", "hw.memsize", check=False) host["memory_gib"] = ( round(int(memsize) / 2**30, 1) if memsize.isdigit() else None ) return { "app_image": { "tag": APP_IMAGE, "id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE), "versions": json.loads(versions), }, "generator": {"image": K6_IMAGE, "version": k6_version}, "docker": { "server": docker.get("ServerVersion"), "vm_os": docker.get("OperatingSystem"), "vm_cpus": docker.get("NCPU"), "vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1), "cgroup": docker.get("CgroupVersion"), "logging_driver": docker.get("LoggingDriver"), }, "host": host, "profile": asdict(profile), } # ------------------------------------------------------------------ стенд class Watcher: """Раз в секунду, пока идёт запуск k6: VmRSS приложения, живы ли оно, PostgreSQL и Valkey. Приложение или хранилище умерло (OOM, kill) — k6 останавливается: иначе каждый его запрос ждал бы таймаута, и наполнение тянулось бы часами. """ def __init__(self) -> None: self.samples: list[int] = [] self.app_died = False self.stand_died: list[str] = [] self._stop = threading.Event() self._thread = threading.Thread(target=self._loop, daemon=True) def _halt(self) -> None: sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False) def _loop(self) -> None: while not self._stop.wait(1.0): states = sh( "docker", "inspect", "-f", "{{.Name}} {{.State.Running}}", DB, CACHE, check=False, ) dead = [ name.lstrip("/") for name, running in ( line.split() for line in states.splitlines() if line ) if running != "true" ] if dead: self.stand_died = dead self._halt() return out = sh( "docker", "exec", APP, "grep", "^VmRSS:", "/proc/1/status", check=False, ) match = re.search(r"(\d+) kB", out) if match: self.samples.append(int(match.group(1))) continue running = sh( "docker", "inspect", "-f", "{{.State.Running}}", APP, check=False, ) if running != "true": self.app_died = True self._halt() return def __enter__(self) -> "Watcher": self._thread.start() return self def __exit__(self, *_: object) -> None: self._stop.set() self._thread.join() def compose_env(profile: Profile) -> dict[str, str]: """Окружение `docker compose` стенда, собранное с нуля. Пароли хранилищ — новые на каждый прогон: живут только в этом словаре и в контейнерах стенда, в результаты не пишутся. Лимиты и срок сессии — из профиля (`BENCH_*` в `bench/bench.compose.yml`). """ env = {key: os.environ[key] for key in HOST_ENV if key in os.environ} return { **env, "LC_ALL": "C", "POSTGRES_PASSWORD": secrets.token_urlsafe(24), "VALKEY_PASSWORD": secrets.token_urlsafe(24), "BENCH_APP_CPUS": profile.cpus, "BENCH_APP_MEMORY": profile.memory, "BENCH_DB_CPUS": profile.db_cpus, "BENCH_DB_MEMORY": profile.db_memory, "BENCH_CACHE_CPUS": profile.cache_cpus, "BENCH_CACHE_MEMORY": profile.cache_memory, "BENCH_SESSION_TTL_SECONDS": str(profile.session_ttl_seconds), } def leftovers() -> dict[str, list[str]]: """Контейнеры, volume и сеть стенда, оставшиеся от другого прогона. Ключ — подкоманда docker, которой их удалить. """ project = f"label=com.docker.compose.project={PROJECT}" names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split() containers = sh( "docker", "ps", "-a", "--filter", project, "--format", "{{.Names}}", ).split() networks = sh("docker", "network", "ls", "--format", "{{.Name}}").split() return { "rm -f": containers + ([K6] if K6 in names else []), "volume rm": sh( "docker", "volume", "ls", "-q", "--filter", project, ).split(), "network rm": [NETWORK] if NETWORK in networks else [], } class Stand: """Проект compose `mtusi-bench` и контейнер k6 на время одного размера.""" def __init__(self, profile: Profile, label_dir: Path) -> None: self.profile = profile self.label_dir = label_dir self.work_dir = label_dir / ".work" self.env = compose_env(profile) def compose(self, *args: str, check: bool = True) -> str: """docker compose стенда; без `-p mtusi-bench` не вызывается.""" files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))] proc = subprocess.run( [ "docker", "compose", "-p", PROJECT, "--env-file", str(ENV_FILE), *files, *args, ], capture_output=True, text=True, env=self.env, check=False, ) if check and proc.returncode != 0: raise BenchError( f"docker compose {' '.join(args[:3])}: exit " f"{proc.returncode}\n{(proc.stdout + proc.stderr)[-2000:]}", ) return proc.stdout.strip() def check_free(self) -> None: found = {cmd: names for cmd, names in leftovers().items() if names} if found: fix = "; ".join( f"docker {cmd} {' '.join(names)}" for cmd, names in found.items() ) raise BenchError( "bench stand leftovers exist: another bench is running or a " f"crashed one left them; remove with: {fix}", ) def build_image(self) -> str: sh( "docker", "build", "-q", "-t", APP_IMAGE, str(ROOT), ) return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE) def start(self, say: Callable[[str], None]) -> None: """Свежий стенд: пустая база на новом volume, миграции, API, k6.""" self.work_dir.mkdir(parents=True, exist_ok=True) self.compose("up", "-d", "--wait", "postgres", "valkey") say("postgres and valkey healthy") self.compose("run", "--rm", "migrate") say("migrations applied") self.compose("up", "-d", "--no-deps", "--no-build", "api") sh( "docker", "run", "-d", "--name", K6, "--network", NETWORK, "-v", f"{ROOT / 'bench' / 'k6'}:/scripts:ro", "-v", f"{self.label_dir}:/out", "-v", f"{self.work_dir}:/work", "--entrypoint", "sleep", K6_IMAGE, "infinity", ) self.wait_ready() def wait_ready(self, timeout: float = 60) -> None: deadline = time.monotonic() + timeout while time.monotonic() < deadline: probe = subprocess.run( ["docker", "exec", APP, "python", "-c", READY_CHECK], capture_output=True, check=False, ) if probe.returncode == 0: return state = self.state(APP) if not state["running"]: reason = "oom" if state["oom"] else "exited" raise BenchError(f"{APP} died during start ({reason})") time.sleep(0.25) raise BenchError(f"{APP} not ready in {timeout:.0f}s") def stop(self) -> bool: """k6, контейнеры, сеть и volume стенда; True — ничего не осталось. k6 и хранилище восстановления — первыми: подключённый к сети контейнер не из compose не дал бы compose удалить сеть. Volume восстановления compose не знает, он удаляется отдельно. """ subprocess.run( ["docker", "rm", "-f", K6, RESTORE_DB], capture_output=True, check=False, ) self.compose("down", "-v", "--remove-orphans", check=False) subprocess.run( ["docker", "volume", "rm", RESTORE_VOLUME], capture_output=True, check=False, ) return not any(leftovers().values()) def start_restore(self) -> None: """Новое хранилище: контейнер PostgreSQL на новом volume. Образ — тот же, что у исходного контейнера (пин без копии в коде), лимиты — как у Postgres стенда, пароль — `--env` без значения из окружения процесса. Метка проекта — чтобы `leftovers()` видел остатки (решение 3 design change `track2-backup-restore`). """ image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB) env = dict( item.split("=", 1) for item in json.loads( sh("docker", "inspect", "-f", "{{json .Config.Env}}", DB), ) ) label = f"com.docker.compose.project={PROJECT}" sh("docker", "volume", "create", "--label", label, RESTORE_VOLUME) proc = subprocess.run( [ "docker", "run", "-d", "--name", RESTORE_DB, "--label", label, "--network", NETWORK, "--network-alias", RESTORE_HOST, "--env", "POSTGRES_PASSWORD", "-e", f"POSTGRES_USER={env['POSTGRES_USER']}", "-e", f"POSTGRES_DB={env['POSTGRES_DB']}", "--cpus", self.profile.db_cpus, "--memory", self.profile.db_memory, "--memory-swap", self.profile.db_memory, "-v", f"{RESTORE_VOLUME}:/var/lib/postgresql", image, ], capture_output=True, text=True, env=self.env, check=False, ) if proc.returncode != 0: raise BenchError(f"restore storage did not start: {proc.stderr}") # по TCP, как healthcheck compose: при инициализации образ # поднимает временный сервер только на сокете deadline = time.monotonic() + 60 while time.monotonic() < deadline: ready = subprocess.run( [ "docker", "exec", RESTORE_DB, "sh", "-c", 'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" ' '-d "$POSTGRES_DB"', ], capture_output=True, check=False, ) if ready.returncode == 0: return time.sleep(0.25) raise BenchError(f"{RESTORE_DB} not ready in 60s") def limits(self) -> dict[str, Any]: """Лимиты контейнеров стенда, прочитанные у запущенных контейнеров.""" limits = {} for role, name in CONTAINERS.items(): host = json.loads( sh("docker", "inspect", "-f", "{{json .HostConfig}}", name), ) config = json.loads( sh("docker", "inspect", "-f", "{{json .Config}}", name), ) limits[role] = { "nano_cpus": host.get("NanoCpus"), "memory": host.get("Memory"), "memory_swap": host.get("MemorySwap"), "published_ports": host.get("PortBindings") or {}, "healthcheck": (config.get("Healthcheck") or {}).get("Test"), "log_driver": (host.get("LogConfig") or {}).get("Type"), } return limits def state(self, name: str) -> dict[str, Any]: state = json.loads( sh("docker", "inspect", "-f", "{{json .State}}", name), ) restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", name) return { "running": state.get("Running"), "oom": state.get("OOMKilled"), "restarts": int(restarts or 0), } def down_storages(self) -> list[str]: """Хранилища, которые за прогон умирали, убивались или рестартовали.""" down = [] for name in (DB, CACHE): state = self.state(name) if not state["running"] or state["oom"] or state["restarts"]: down.append(name) return down def psql( self, sql: str, *, check: bool = True, container: str = DB, ) -> str: """SQL в базе стенда через stdin: пользователь контейнера, без пароля. `container` — исходное хранилище или хранилище восстановления. """ proc = subprocess.run( ["docker", "exec", "-i", container, "sh", "-c", PSQL], input=sql, capture_output=True, text=True, check=False, ) if check and proc.returncode != 0: raise BenchError(f"psql: exit {proc.returncode}\n{proc.stderr}") return proc.stdout.strip() def maintain(self) -> float: """VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды. Статистика и состояние таблиц одинаковы у всех меток, а не зависят от того, когда сработал autovacuum (решение 4). """ started = time.monotonic() self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n") return time.monotonic() - started def storage(self) -> dict[str, Any]: """Объём таблицы сообщений и её индексов, число строк.""" row = self.psql( "SELECT pg_table_size('messages'), pg_indexes_size('messages'), " "count(*) FROM messages;\n", ) table, indexes, count = (int(value) for value in row.split("|")) return { "messages_rows": count, "messages_table_bytes": table, "messages_indexes_bytes": indexes, "db_bytes_per_message": ( round((table + indexes) / count) if count else None ), } def chat_counts(self) -> dict[str, int]: """Число сообщений по чатам (проверка распределения наполнения).""" rows = self.psql( "SELECT chat_id, count(*) FROM messages GROUP BY chat_id;\n", ) return { chat: int(count) for chat, count in (line.split("|") for line in rows.splitlines()) } def indexes(self, container: str = DB) -> list[str]: """Определения индексов messages: до и после этапов с вариантами.""" return self.psql( "SELECT indexdef FROM pg_indexes WHERE tablename = 'messages' " "ORDER BY indexname;\n", container=container, ).splitlines() def message_count(self) -> int: return int(self.psql("SELECT count(*) FROM messages;\n")) def write_probe( self, app_index: str, columns: str | None, ) -> dict[str, Any]: """Одна проба записи при индексе истории `columns` (None — без него). Всё в транзакции, которая откатывается: индекс приложения удаляется, конфигурация строится заново, `CHECKPOINT`, вставка порции во все чаты в пропорции наполнения и сообщение максимальной длины (решение 5 design change `track2-query-plans`). """ p = self.profile create = ( f"CREATE INDEX bench_probe ON messages {columns};\n" if columns else "" ) size = "pg_relation_size('bench_probe')" if columns else "0" proc = subprocess.run( ["docker", "exec", "-i", DB, "sh", "-c", PSQL], input=( "SET lock_timeout = '5s';\n" "BEGIN;\n" "CREATE TEMP TABLE probe_src ON COMMIT DROP AS " "SELECT chat_id, sender_id, " "row_number() OVER (ORDER BY created_at, id) AS g " "FROM (SELECT chat_id, sender_id, created_at, id FROM messages " f"ORDER BY created_at, id LIMIT {p.write_probe_rows}) s;\n" f'DROP INDEX "{app_index}";\n' f"{create}" f"SELECT {size}, (SELECT count(*) FROM messages), " "(SELECT count(*) FROM probe_src);\n" "CHECKPOINT;\n" "EXPLAIN (ANALYZE, BUFFERS, WAL) INSERT INTO messages " "(id, chat_id, sender_id, text, created_at) " "SELECT gen_random_uuid(), chat_id, sender_id, " f"rpad('probe-' || g, {p.text_length}, 'x'), " "now() + g * interval '1 microsecond' FROM probe_src;\n" f"{LONG_TEXT_PROBE}" "ROLLBACK;\n" ), capture_output=True, text=True, check=False, ) if proc.returncode != 0: raise BenchError( f"write probe: exit {proc.returncode}\n{proc.stderr}", ) head, _, plan = proc.stdout.strip().partition("\n") index_bytes, rows_before, rows = (int(v) for v in head.split("|")) wal = re.search(r"WAL: ([^\n]*)", plan) def wal_field(name: str) -> int: # EXPLAIN не печатает нулевые поля WAL match = ( re.search(rf"\b{name}=(\d+)", wal.group(1)) if wal else None ) return int(match.group(1)) if match else 0 execution = re.search(r"Execution Time: ([\d.]+) ms", plan) long_text = re.search(r"long-text: ([^\n]*)", proc.stderr) if execution is None or long_text is None: raise BenchError(f"write probe: unexpected output\n{proc.stdout}") return { "index_bytes": index_bytes, "rows_before": rows_before, "rows": rows, "time_ms": float(execution.group(1)), "wal_records": wal_field("records"), "wal_fpi": wal_field("fpi"), "wal_bytes": wal_field("bytes"), "long_text": long_text.group(1).strip(), } def db_log(self) -> str: """Журнал Postgres стенда (stderr контейнера): только в памяти.""" proc = subprocess.run( ["docker", "logs", DB], capture_output=True, text=True, check=False, ) return proc.stdout + proc.stderr def passport(self) -> dict[str, Any]: """Хранилища и окружение API для паспорта; паролей здесь нет.""" settings = self.psql( "SELECT name, current_setting(name) FROM pg_settings " "WHERE name = ANY(string_to_array(" f"'{','.join(PG_SETTINGS)}', ',')) ORDER BY name;\n", ) info = sh( "docker", "exec", CACHE, "sh", "-c", 'exec valkey-cli -p "$VALKEY_PORT" INFO server', ) valkey_version = re.search(r"^valkey_version:(\S+)", info, re.M) memory = sh( "docker", "exec", CACHE, "sh", "-c", 'exec valkey-cli -p "$VALKEY_PORT" CONFIG GET "maxmemory*"', ).splitlines() env = json.loads( sh("docker", "inspect", "-f", "{{json .Config.Env}}", APP), ) app_env = dict( sorted( item.split("=", 1) for item in env if re.match(r"[A-Z0-9_]+__[A-Z0-9_]+=", item) and "PASSWORD" not in item.split("=", 1)[0] ), ) def image(name: str) -> dict[str, str]: return { "image": sh( "docker", "inspect", "-f", "{{.Config.Image}}", name, ), "id": sh("docker", "inspect", "-f", "{{.Image}}", name), } return { "postgres": { **image(DB), "version": self.psql("SHOW server_version;\n"), "settings": dict( line.split("|", 1) for line in settings.splitlines() ), }, "valkey": { **image(CACHE), "version": valkey_version.group(1) if valkey_version else None, **dict(zip(memory[::2], memory[1::2], strict=True)), }, "app_env": app_env, } def snapshot(self) -> dict[str, Any]: """CPU и память приложения, CPU PostgreSQL и генератора. Если приложение уже мертво (OOM), его поля — None: снимок нужен и для записи проваленного прогона. """ app = sh( "docker", "exec", APP, "sh", "-c", "cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; " "grep -E '^(VmRSS|VmHWM):' /proc/1/status", check=False, ) k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat") db = sh( "docker", "exec", DB, "cat", "/sys/fs/cgroup/cpu.stat", check=False, ) def number(pattern: str, text: str) -> int | None: match = re.search(pattern, text, re.MULTILINE) return int(match.group(1)) if match else None return { "app_usec": number(r"^usage_usec (\d+)$", app), "mem_bytes": number(r"^(\d+)$", app), "rss_kib": number(r"^VmRSS:\s+(\d+) kB", app), "hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app), "k6_usec": number(r"^usage_usec (\d+)$", k6), "db_usec": number(r"^usage_usec (\d+)$", db), } def k6_args(self, script: str, env: dict[str, Any]) -> list[str]: """Команда `k6 run` сценария в контейнере k6 стенда.""" common = { "TEXT_LENGTH": self.profile.text_length, "PAGE_LIMIT": self.profile.page_limit, } flags = [ item for key, value in {**common, **env}.items() for item in ("-e", f"{key}={value}") ] return [ "docker", "exec", K6, "k6", "run", "--quiet", "--no-color", "--log-format", "raw", *flags, f"/scripts/{script}", ] def k6( self, script: str, env: dict[str, Any], *, check: bool = True, ) -> subprocess.CompletedProcess[str]: proc = subprocess.run( self.k6_args(script, env), capture_output=True, text=True, check=False, ) if check and proc.returncode != 0: raise BenchError(k6_failure(script, proc)) return proc def k6_start(self, script: str, env: dict[str, Any]) -> subprocess.Popen: """Сценарий k6 в фоне: запись во время backup.""" return subprocess.Popen( self.k6_args(script, env), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) def pg_dump(self, path: Path) -> None: """Логический backup исходной базы (формат custom) в файл хоста.""" with path.open("wb") as file: proc = subprocess.run( [ "docker", "exec", DB, "sh", "-c", 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc', ], stdout=file, stderr=subprocess.PIPE, check=False, ) if proc.returncode != 0: raise BenchError(f"pg_dump: exit {proc.returncode}\n{proc.stderr}") def pg_restore(self, path: Path) -> tuple[float, float]: """Backup в хранилище восстановления: копия файла и pg_restore -j. Возвращает секунды копирования и восстановления. """ started = time.monotonic() sh("docker", "cp", str(path), f"{RESTORE_DB}:/tmp/backup.dump") copied = time.monotonic() proc = subprocess.run( [ "docker", "exec", RESTORE_DB, "sh", "-c", 'exec pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" ' f"-j {self.profile.restore_jobs} --no-owner --no-privileges " "--exit-on-error /tmp/backup.dump", ], capture_output=True, text=True, check=False, ) if proc.returncode != 0: raise BenchError( f"pg_restore: exit {proc.returncode}\n{proc.stderr}", ) return copied - started, time.monotonic() - copied def switch_api(self, host: str) -> None: """API заново, с адресом базы `host`: `POSTGRES_HOST` перекрывает значение из `bench/bench.env` для `x-app-env`.""" env = self.env self.env = {**env, "POSTGRES_HOST": host} try: self.compose("up", "-d", "--no-deps", "--force-recreate", "api") finally: self.env = env self.wait_ready() def table_counts(self, container: str = DB) -> dict[str, int]: """Число строк таблиц данных (сверка восстановления).""" sql = " UNION ALL ".join( f"SELECT '{table}', count(*) FROM {table}" for table in DATA_TABLES ) rows = self.psql(f"{sql};\n", container=container) return { table: int(count) for table, count in (line.split("|") for line in rows.splitlines()) } def message_rows(self, container: str = DB) -> dict[str, tuple[str, str]]: """Сообщения базы: id → (chat_id, md5 всех полей).""" rows = self.psql( "SELECT id, chat_id, md5(concat_ws('|', chat_id, sender_id, " "text, created_at)) FROM messages;\n", container=container, ) return { message: (chat, digest) for message, chat, digest in ( line.split("|") for line in rows.splitlines() ) } def work_json(self, name: str) -> Any: return json.loads((self.work_dir / name).read_text(encoding="utf-8")) # -------------------------------------------------------------- измерение def parse_summary(path: Path) -> dict[str, Any]: """Итог k6: успехи, ошибки, RPS и перцентили успешных запросов. Нет файла (k6 остановлен сторожем до итога) — нулевой итог. """ if not path.exists(): return { "duration_s": 0, "requests_ok": 0, "requests_failed": 0, "check_failed": 0, "rps": None, "error_ratio": None, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, } data = json.loads(path.read_text(encoding="utf-8")) metrics = data["metrics"] failed = metrics.get("http_req_failed", {}).get("values", {}) ok = int(failed.get("fails", 0)) bad = int(failed.get("passes", 0)) checks = metrics.get("checks", {}).get("values", {}) success = metrics.get(SUCCESS_METRIC, {}).get("values", {}) duration = data["state"]["testRunDurationMs"] / 1000 return { "duration_s": round(duration, 3), "requests_ok": ok, "requests_failed": bad, "check_failed": int(checks.get("fails", 0)), "rps": round(ok / duration, 1) if duration else None, "error_ratio": round(bad / (ok + bad), 6) if ok + bad else None, "p50_ms": rounded(success.get("p(50)")), "p95_ms": rounded(success.get("p(95)")), "p99_ms": rounded(success.get("p(99)")), "max_ms": rounded(success.get("max")), } @dataclass class Measured: summary: dict[str, Any] before: dict[str, Any] after: dict[str, Any] rss_max_kib: int | None def cores(self, key: str) -> float | None: """Среднее число занятых ядер за прогон по счётчику cgroup.""" duration = self.summary["duration_s"] start, end = self.before[key], self.after[key] if not duration or start is None or end is None: return None return round((end - start) / 1e6 / duration, 3) @property def app_cpu(self) -> float | None: return self.cores("app_usec") @property def generator_cpu(self) -> float | None: return self.cores("k6_usec") @property def db_cpu(self) -> float | None: return self.cores("db_usec") def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str: tail = (proc.stdout + proc.stderr)[-3000:] return f"k6 {script} failed ({proc.returncode}):\n{tail}" def measure( stand: Stand, script: str, env: dict[str, Any], raw: Path, ) -> Measured: """Один запуск k6 со снимками до и после, опросом RSS и сторожем.""" before = stand.snapshot() with Watcher() as watcher: proc = stand.k6(script, env, check=False) # если приложение или хранилище умерло, k6 остановлен сторожем: прогон # пишется с пометкой, а не падает здесь died = watcher.app_died or watcher.stand_died if proc.returncode != 0 and not died: raise BenchError(k6_failure(script, proc)) after = stand.snapshot() values = [ v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v ] return Measured( summary=parse_summary(raw), before=before, after=after, rss_max_kib=max(values) if values else None, ) def run_flags( profile: Profile, measured: Measured, app: dict[str, Any], storages_down: list[str], ) -> list[str]: flags = [] if app["oom"]: flags.append("oom") if not app["running"] or app["restarts"]: flags.append("app_down") if storages_down: flags.append("stand_down") summary = measured.summary if summary["requests_failed"] or summary["check_failed"]: flags.append("errors") cpu = measured.generator_cpu if cpu is not None and cpu >= profile.generator_cpu_limit: flags.append("generator") return flags # ----------------------------------------------------------- план запроса def plan_entries(log: str, marker: str) -> list[str]: """Записи auto_explain между маркерами этапа плана. Запись начинается строкой `LOG: duration: … plan:` и продолжается строками с табуляцией; префикс времени и pid отбрасывается. """ lines = log.splitlines() bounds = [ next( (i for i, line in enumerate(lines) if f"{marker} {edge}" in line), None, ) for edge in ("start", "end") ] if None in bounds: return [] entries: list[list[str]] = [] current: list[str] | None = None for line in lines[bounds[0] + 1 : bounds[1]]: if line.startswith("\t"): if current is not None: current.append(line[1:]) continue current = None head = line.partition("LOG: ")[2].rstrip() if head.startswith("duration:") and head.endswith("plan:"): current = [head] entries.append(current) return ["\n".join(entry) for entry in entries] def is_history(entry: str) -> bool: """Запись — запрос страницы истории (читает таблицу messages). Остальные записи окна (проверка сессии с хэшем токена в параметрах, служебные запросы) в результаты не попадают. """ text = entry.partition("Query Parameters:")[0] return "Query Text:" in text and "FROM messages" in text def plan_parts(entry: str) -> dict[str, Any] | None: """Текст запроса, параметры и тело плана записи auto_explain.""" lines = entry.splitlines() starts = [i for i, line in enumerate(lines) if line.startswith("Query ")] if len(starts) != 2 or not lines[starts[1]].startswith("Query Param"): return None text = "\n".join(lines[starts[0] : starts[1]]) params = lines[starts[1]].partition(": ")[2] return { "text": text.removeprefix("Query Text: "), "params": [ value.replace("''", "'") for value in re.findall(r"\$\d+ = '((?:[^']|'')*)'", params) ], "plan": "\n".join(lines[starts[1] + 1 :]), } def plan_summary(plan: str, time_ms: float | None) -> dict[str, Any]: """Узел доступа к messages, буферы верхнего узла, строки, время.""" node = re.search( r"((?:Parallel )?(?:Seq Scan|Index Only Scan|Index Scan|" r"Bitmap Heap Scan)(?: Backward)?(?: using \w+)? on messages)", plan, ) buffers = re.search(r"Buffers: ([^\n]*)", plan) hit = re.search(r"shared hit=(\d+)", buffers.group(1)) if buffers else None read = re.search(r"read=(\d+)", buffers.group(1)) if buffers else None rows = re.search(r"\(actual time=[\d.]+\.\.[\d.]+ rows=([\d.]+)", plan) if time_ms is None: execution = re.search(r"Execution Time: ([\d.]+) ms", plan) time_ms = float(execution.group(1)) if execution else None return { "node": node.group(1) if node else None, "shared_hit": int(hit.group(1)) if hit else 0, "shared_read": int(read.group(1)) if read else 0, "rows": float(rows.group(1)) if rows else None, "time_ms": time_ms, # параметры в условиях плана — generic plan, литералы — custom "kind": "generic" if re.search(r"\$\d", plan) else "custom", } def entry_duration(entry: str) -> float | None: match = re.match(r"duration: ([\d.]+) ms", entry) return float(match.group(1)) if match else None def index_name(plan: str) -> str | None: match = INDEX_NODE_RE.search(plan) return ( next((name for name in match.groups() if name), None) if match else None ) def render_explain( size: int, git: dict[str, Any], profile: Profile, pages: list[dict[str, Any]], entries: list[str], compared: list[tuple[dict[str, Any], dict[str, str]]], ) -> str: """Файл `explain-<размер>.txt`: планы приложения и варианты psql.""" tiers = ", ".join( f"{count} × {human(chat_size)}" for count, chat_size in profile.background ) lines = [ f"# План запроса истории: размер {human(size)}", "", f"Commit {git['commit']}" + (" + незакоммиченные изменения" if git["dirty"] else ""), f"Читаемый чат — {size} сообщений; фоновые чаты: {tiers or 'нет'}.", "Страница — `limit` профиля; «после» — сколько сообщений чата до", "страницы.", "", "## Планы приложения", "", "auto_explain (ANALYZE, BUFFERS, TIMING): запрос, параметры и план,", "с которыми его выполнило приложение (prepared statement asyncpg).", "", ] for page, entry in zip(pages, entries, strict=True): lines += [ f"### {page['chat']}: {page['page']}, после {page['after']}", "", entry, "", ] variants = ", ".join( f"`{name}` {columns}" for name, columns in profile.index_variants ) lines += [ "## Страницы сравнения в psql: варианты", "", "Тот же текст запроса и те же параметры, что у плана приложения этой", "страницы, в одной транзакции, которая откатывается: индексы базы", "после неё те же. Каждый вариант выполнен дважды, здесь — второе", "выполнение. `psql` — индекс приложения, custom plan; `generic` — он", "же с force_generic_plan; `or-cursor` — условие курсора через OR;", "`no-index` — без индекса приложения; варианты индекса вместо него:", f"{variants}.", "", ] for page, plans in compared: lines += [ f"### {page['chat']}: {page['page']}, после {page['after']}", "", ] if not plans: lines += [ "План приложения не использовал индекс: вариантов нет.", "", ] for name, plan in plans.items(): lines += [f"#### {name}", "", plan, ""] return "\n".join(lines) # ------------------------------------------------------------ результаты class Results: """Файлы метки: runs.csv, prefill.csv, meta.json, progress.json.""" def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None: self.dir = label_dir self.meta = meta self.runs: list[dict[str, Any]] = [] self.prefill: list[dict[str, Any]] = [] self.sizes: list[dict[str, Any]] = [] (label_dir / "raw").mkdir(parents=True) self.write_meta() def append( self, name: str, fields: list[str], row: dict[str, Any], ) -> None: path = self.dir / name new = not path.exists() with path.open("a", newline="", encoding="utf-8") as file: writer = csv.DictWriter( file, fieldnames=fields, lineterminator="\n", ) if new: writer.writeheader() writer.writerow({key: row.get(key) for key in fields}) file.flush() def add_run(self, row: dict[str, Any]) -> None: self.runs.append(row) self.append("runs.csv", RUN_FIELDS, row) def add_prefill(self, row: dict[str, Any]) -> None: self.prefill.append(row) self.append("prefill.csv", PREFILL_FIELDS, row) def write_meta(self) -> None: self.meta["sizes"] = self.sizes text = json.dumps(self.meta, ensure_ascii=False, indent=2) (self.dir / "meta.json").write_text(text + "\n", encoding="utf-8") def write_progress(self, progress: dict[str, Any]) -> None: progress["updated_at"] = datetime.now(UTC).isoformat( timespec="seconds", ) text = json.dumps(progress, ensure_ascii=False, indent=2) (self.dir / "progress.json").write_text(text + "\n", encoding="utf-8") def write_summary(self) -> None: text = build_summary(self.meta, self.runs, self.prefill, self.sizes) (self.dir / "summary.md").write_text(text, encoding="utf-8") def cell(values: list[float | None], digits: int = 1) -> str: """Медиана и min–max повторов; один повтор — просто значение.""" present = [v for v in values if v is not None] if not present: return "—" median = statistics.median(present) if len(present) == 1: return f"{median:.{digits}f}" return ( f"{median:.{digits}f} ({min(present):.{digits}f}–" f"{max(present):.{digits}f})" ) def build_summary( meta: dict[str, Any], runs: list[dict[str, Any]], prefill: list[dict[str, Any]], sizes: list[dict[str, Any]], ) -> str: profile = meta["profile"] lines = [ f"# Нагрузочный замер: метка `{meta['label']}`", "", f"- Статус: **{meta['status']}**" + (f" ({meta['reason']})" if meta.get("reason") else ""), f"- Commit: `{meta['git']['commit']}`" + (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""), f"- Профиль `{profile['name']}`: размеры " f"{', '.join(human(s) for s in profile['sizes'])}; клиенты " f"{', '.join(map(str, profile['concurrency']))}; повторов " f"{profile['repeats']}; чтение {profile['read_measure_s']} с после " f"{profile['read_warmup_s']} с прогрева; запись " f"{profile['write_measure']} после {profile['write_warmup']}", f"- Лимиты без swap: API {profile['cpus']} CPU / " f"{profile['memory']}, PostgreSQL {profile['db_cpus']} CPU / " f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / " f"{profile['cache_memory']}; значения — медиана (min–max) по " "повторам", "- Посторонние контейнеры в VM во время замера: " + (", ".join(meta.get("other_containers") or []) or "нет"), "", ] for scenario in ("read", "write"): rows = [r for r in runs if r["scenario"] == scenario] if not rows: continue lines += [ f"## Сценарий `{scenario}`", "", "| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | " "max, мс | CPU API, ядер | CPU БД, ядер | RSS конец, МиБ | " "Сообщений в начале | Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|---|", ] cells = sorted({(r["size"], r["concurrency"]) for r in rows}) for size, conc in cells: group = [ r for r in rows if r["size"] == size and r["concurrency"] == conc ] flags = sorted({f for r in group for f in r["flags"].split()}) lines.append( f"| {human(size)} | {conc} | " f"{cell([r['rps'] for r in group], 0)} | " f"{cell([r['p50_ms'] for r in group], 2)} | " f"{cell([r['p95_ms'] for r in group], 2)} | " f"{cell([r['p99_ms'] for r in group], 2)} | " f"{cell([r['max_ms'] for r in group], 1)} | " f"{cell([r['app_cpu_cores'] for r in group], 2)} | " f"{cell([r['db_cpu_cores'] for r in group], 2)} | " f"{cell([r['rss_end_mib'] for r in group], 1)} | " f"{cell([r['store_messages_start'] for r in group], 0)} | " f"{' '.join(flags) or '—'} |", ) lines.append("") if sizes: lines += [ "## Память и наполнение по размерам", "", "Объём `messages` — после `VACUUM (ANALYZE)`, до прогонов: в " "таблице N сообщений читаемого чата и сообщения фоновых чатов " "(`Сообщений`).", "", "| Размер | Сообщений | RSS API до, МиБ | RSS API после, МиБ | " "Байт RSS на сообщение | Таблица, МиБ | Индексы, МиБ | " "Байт в БД на сообщение | Наполнение, сообщ/с | " "Курсоров в пуле | Пик RSS за размер, МиБ |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for info in sizes: lines.append( f"| {human(info['size'])} | " f"{info.get('messages_total', info['size'])} | " f"{info['rss_before_mib']} | " f"{info['rss_after_mib']} | {info['bytes_per_message']} | " f"{mib_bytes(info.get('messages_table_bytes'))} | " f"{mib_bytes(info.get('messages_indexes_bytes'))} | " f"{info.get('db_bytes_per_message', '—')} | " f"{info['prefill_rate']} | {info['cursors']} | " f"{info.get('rss_peak_mib', '—')} |", ) lines.append("") lines += [ "## Планы запроса истории", "", "Страница (`limit` профиля) читаемого чата и первого чата " "каждого фонового уровня. `app` — план запроса приложения " "(auto_explain). Остальное — тот же запрос с теми же " "параметрами в psql, в откатываемой транзакции: `psql` — " "индекс приложения, `generic` — он же с generic plan, " "`or-cursor` — условие курсора через OR, `no-index` — без " "индекса, дальше — варианты индекса профиля. Буферы — верхнего " "узла плана, файлы — `explain-<размер>.txt`.", "", "| Размер | Чат | Страница | После сообщений | Вариант | Узел | " "shared hit | read | Строк | Время, мс | План |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for info in sizes: for plan in info.get("plans", []): lines.append( f"| {human(info['size'])} | {plan.get('chat', 'read')} | " f"{plan['page']} | {plan['after']} | {plan['source']} | " f"{plan['node'] or '—'} | {plan['shared_hit']} | " f"{plan['shared_read']} | {fmt(plan['rows'])} | " f"{fmt(plan['time_ms'], 3)} | {plan['kind']} |", ) lines.append("") costs = [ (info, row) for info in sizes for row in info.get("write_cost", []) ] if costs: lines += [ "## Стоимость записи по вариантам индекса", "", "Вставка порции сообщений во все чаты в пропорции наполнения " "одной командой, в транзакции, которая откатывается; индекс " "истории построен заново, перед вставкой `CHECKPOINT`. " "Значения — медиана (min–max) повторов. «4000 символов» — " "вставка сообщения максимальной длины из несжимаемого текста.", "", "| Размер | Вариант | Строк вставки | Индекс, МиБ | " "Байт индекса на сообщение | мкс на сообщение | " "WAL байт на сообщение | FPI на 1000 | 4000 символов |", "|---|---|---|---|---|---|---|---|---|", ] for info, row in costs: lines.append( f"| {human(info['size'])} | {row['variant']} | " f"{row['rows']} | {mib_bytes(row['index_bytes'])} | " f"{row['index_bytes_per_message']} | " f"{cell(row['insert_us_per_message'], 2)} | " f"{cell(row['wal_bytes_per_message'], 0)} | " f"{cell(row['fpi_per_1000'], 1)} | {row['long_text']} |", ) lines.append("") restores = [info for info in sizes if info.get("restore")] if restores: lines += [ "## Backup и восстановление", "", "`pg_dump -Fc` исходной базы во время записи через HTTP, " "затем активность после backup (пользователь, сессия, чат, " "сообщение), `pg_restore -j` в новый контейнер PostgreSQL " "на новом volume, `ANALYZE`, сверка обеих баз (число строк, " "md5 каждого сообщения, индексы, ревизия схемы), " "переключение API и проверки через HTTP до и после " "`FLUSHALL`. RTO — от запуска нового хранилища до " "готовности API, без сверки. Сверка прошла на каждом " "размере: иначе команда завершилась бы ошибкой.", "", "| Размер | До backup | Исходная в конце | Восстановлено | " "Потеряно | Записей во время backup (в backup) | Backup, МиБ | " "Байт backup на сообщение | dump, с | restore, с | " "ANALYZE, с | API, с | RTO, с | Сверка, с | Чат после " "backup: GET / POST до FLUSHALL, GET после | Сессия после " "backup |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|" "---|---|", ] for info in restores: r = info["restore"] lines.append( f"| {human(info['size'])} | {r['messages_before_backup']} " f"| {r['messages_source_end']} | {r['messages_restored']} " f"| {r['messages_lost']} | {r['load_written']} " f"({r['load_in_backup']}) | " f"{mib_bytes(r['backup_bytes'])} | " f"{r['backup_bytes_per_message']} | {r['dump_seconds']} | " f"{r['restore_seconds']} | {r['analyze_seconds']} | " f"{r['api_ready_seconds']} | {r['rto_seconds']} | " f"{r['verify_seconds']} | {r['stale_chat_get']} / " f"{r['stale_chat_post']}, {r['flushed_chat_get']} | " f"{r['late_session']} |", ) lines.append("") if prefill: largest = max(r["size"] for r in prefill) curve = [r for r in prefill if r["size"] == largest] lines += [ f"## Кривая наполнения {human(largest)}: запись при росте " f"хранилища, {profile['prefill_vus']} клиента", "", "| Порция | Сообщений после | RPS | p50, мс | p95, мс | " "p99, мс | max, мс | CPU API, ядер | CPU БД, ядер | RSS, МиБ | " "Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for r in curve: lines.append( f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | " f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | " f"{r['max_ms']} | {r['app_cpu_cores']} | " f"{r['db_cpu_cores']} | {r['rss_end_mib']} " f"| {r['flags'] or '—'} |", ) lines.append("") if meta.get("host_warnings"): lines += ["## Предупреждения о хосте", ""] lines += [f"- {w}" for w in meta["host_warnings"]] lines.append("") return "\n".join(lines) # ---------------------------------------------------------------- прогон class Bench: def __init__( self, profile: Profile, label: str, results: Results, ) -> None: self.profile = profile self.label = label self.results = results self.stand = Stand(profile, results.dir) self.started = time.monotonic() self.runs_total = ( len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats ) self.runs_done = 0 self.stage = "start" self.size: int | None = None # скорости стенда для оценки оставшегося времени self.prefill_rate = 1500.0 self.write_rps: dict[int, float] = {} self.run_overhead = 3.0 self.done_sizes = 0 self.prefilled = 0 # текущий размер наполнен и курсоры собраны self.prepared = False # индекс плана приложения: от него варианты и пробы записи self.app_index: str | None = None # ------------------------------------------------------------ вывод def say(self, text: str) -> None: elapsed = int(time.monotonic() - self.started) stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed)) print(f"[{stamp}] {text}", flush=True) def eta(self) -> float: p = self.profile left = 0.0 for index, size in enumerate(p.sizes): current = index == self.done_sizes if index < self.done_sizes: continue # пробы записи в конце размера: сборка индексов и вставки; # backup, восстановление и сверка left += 60 + (30 if p.restore else 0) if current and self.prepared: continue filled = self.prefilled if current else 0 total = size + p.background_total left += (total - filled) / self.prefill_rate + size / 100 / 500 # стенд, миграции, обслуживание БД, план и его варианты left += 60 per_size = len(p.concurrency) * p.repeats reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes) done_in_size = self.runs_done - self.done_sizes * per_size * 2 reads_left -= min(done_in_size, per_size) writes_left -= max(0, done_in_size - per_size) read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead left += reads_left * read_time rps = ( statistics.mean(self.write_rps.values()) if self.write_rps else 1000 ) write_time = (p.write_warmup + p.write_measure) / rps left += writes_left * (write_time + self.run_overhead) return left def progress(self, last: dict[str, Any] | None = None) -> None: state = { "label": self.label, "profile": self.profile.name, "stage": self.stage, "size": self.size, "runs_done": self.runs_done, "runs_total": self.runs_total, "elapsed_s": round(time.monotonic() - self.started), "eta_s": round(self.eta()), "warnings": self.results.meta.get("host_warnings", []), } if last is not None: state["last_run"] = last self.results.write_progress(state) # --------------------------------------------------------- этапы def run(self) -> None: for index, size in enumerate(self.profile.sizes, start=1): self.size = size self.prefilled = 0 self.prepared = False # индекс плана приложения ищется заново на каждом размере self.app_index = None self.run_size(index, size) self.done_sizes += 1 def run_size(self, index: int, size: int) -> None: stand = self.stand name = human(size) self.stage = f"size {name}: start" self.say(f"size {name} ({index}/{len(self.profile.sizes)}): start") stand.start(lambda text: self.say(f"size {name}: {text}")) limits = stand.limits() self.results.meta.setdefault("limits", limits) if limits != self.results.meta["limits"]: raise BenchError(f"limits changed between sizes: {limits}") if "stand" not in self.results.meta: self.results.meta["stand"] = stand.passport() self.results.write_meta() rss_start = stand.snapshot()["rss_kib"] self.say(f"size {name}: app started, RSS {mib(rss_start)} MiB") stand.k6( "prepare.js", { "MEMBERS": self.profile.members, "BACKGROUND": ",".join( f"{count}x{chat_size}" for count, chat_size in self.profile.background ), }, ) fixture = stand.work_json("fixture.json") sizes = chat_sizes(self.profile, size) if [size] + [c["size"] for c in fixture["bgChats"]] != sizes: raise BenchError( f"size {name}: background chats differ from profile", ) total = sum(sizes) (stand.work_dir / "schedule.json").write_text( json.dumps(fill_schedule(sizes)), encoding="utf-8", ) self.say( f"size {name}: {len(fixture['bgChats'])} background chats, " f"{total} messages to prefill", ) self.progress() rss_before = stand.snapshot()["rss_kib"] prefill_seconds = self.run_prefill(size, total) after = stand.snapshot() rss_after = after["rss_kib"] self.stage = f"size {name}: maintenance" self.progress() seconds = stand.maintain() storage = stand.storage() if storage["messages_rows"] != total: raise BenchError( f"messages table has {storage['messages_rows']} rows after " f"prefill, expected {total}", ) self.check_distribution(size, fixture) self.say( f"size {name}: VACUUM (ANALYZE) and CHECKPOINT in {seconds:.1f}s; " f"messages {mib_bytes(storage['messages_table_bytes'])} MiB + " f"indexes {mib_bytes(storage['messages_indexes_bytes'])} MiB, " f"{storage['db_bytes_per_message']} B/msg", ) self.stage = f"size {name}: cursors" self.progress() stand.k6( "cursors.js", { "WALK_LIMIT": self.profile.walk_limit, "POOL_SIZE": self.profile.cursor_pool, }, ) pool = stand.work_json("cursors.json") if pool["walked"] != size: raise BenchError( f"read chat has {pool['walked']} messages, expected {size}", ) self.say(f"size {name}: cursors {len(pool['cursors'])}") self.stage = f"size {name}: query plan" self.progress() plans = self.explain(size, pool, fixture) info = { "size": size, "messages_total": total, "rss_before_mib": mib(rss_before), "rss_after_mib": mib(rss_after), "bytes_per_message": round( (rss_after - rss_before) * 1024 / total, ), "prefill_seconds": round(prefill_seconds, 1), "prefill_rate": round(total / prefill_seconds), "cursors": len(pool["cursors"]), "maintenance_seconds": round(seconds, 1), **storage, "plans": plans, } self.results.sizes.append(info) self.results.write_meta() self.prepared = True self.say( f"size {name}: prefill done, app RSS {info['rss_after_mib']} " f"MiB (+{info['bytes_per_message']} B/msg)", ) store = total for scenario in ("read", "write"): for conc in self.profile.concurrency: for repeat in range(1, self.profile.repeats + 1): store = self.run_one(scenario, size, conc, repeat, store) if scenario == "read": self.check_read_chat(size) info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"]) self.stage = f"size {name}: write cost" self.progress() info["write_cost"] = self.write_cost(size) self.results.write_meta() if self.profile.restore: self.stage = f"size {name}: backup and restore" self.progress() info["restore"] = self.backup_restore(size, fixture, store) self.results.write_meta() if not stand.stop(): raise BenchError(f"stand of size {name} was not removed") def check_read_chat(self, size: int) -> None: """После блока чтения в читаемом чате по-прежнему ровно N. Новое сообщение встало бы в конец истории, поэтому достаточно дочитать хвост от последнего курсора пула. """ self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit}) tail = self.stand.work_json("tail.json") total = tail["from_after"] + tail["tail"] if total != size: raise BenchError( f"read chat has {total} messages after the read block, " f"expected {size}", ) self.say(f"size {human(size)}: read chat unchanged, {total} messages") def check_distribution(self, size: int, fixture: dict[str, Any]) -> None: """После наполнения у каждого чата ровно его размер по профилю.""" expected = { fixture["readChatId"]: size, fixture["writeChatId"]: 0, **{chat["id"]: chat["size"] for chat in fixture["bgChats"]}, } counts = self.stand.chat_counts() wrong = [ f"{counts.get(chat, 0)} instead of {want}" for chat, want in expected.items() if counts.get(chat, 0) != want ] wrong += [ f"unknown chat with {counts[c]}" for c in counts.keys() - expected.keys() ] if wrong: raise BenchError( f"size {human(size)}: messages per chat differ from the " f"profile in {len(wrong)} chats: {', '.join(wrong[:5])}", ) sizes = sorted(expected.values()) self.say( f"size {human(size)}: {len(expected) - 1} chats with " f"{sizes[1]}–{sizes[-1]} messages, distribution matches profile", ) def plan_pages( self, size: int, pool: dict[str, Any], fixture: dict[str, Any], ) -> list[dict[str, Any]]: """Страницы этапа плана: читаемый чат и первый чат каждого уровня. Читаемый — из пула курсоров, фоновые — проходом `plan_cursors.js` до включения auto_explain. Первая, средняя и последняя полная страница; у чата без полных страниц после первой — только первая. Страница сравнения — средняя, иначе первая (решение 3 design change `track2-query-plans`). """ p = self.profile representatives = [] offset = 0 for count, _ in p.background: representatives.append(fixture["bgChats"][offset]) offset += count walks: dict[str, Any] = {} if representatives: (self.stand.work_dir / "plan_chats.json").write_text( json.dumps(representatives), encoding="utf-8", ) self.stand.k6("plan_cursors.js", {"WALK_LIMIT": p.walk_limit}) walks = { walk["id"]: walk for walk in self.stand.work_json("plan_walks.json") } chats = [ ( f"read {human(size)}", fixture["readChatId"], size, [ {"after": after, "cursor": cursor} for after, cursor in zip( pool["afters"], pool["cursors"], strict=True, ) ], ), ] for chat in representatives: walk = walks[chat["id"]] if walk["walked"] != chat["size"]: raise BenchError( f"size {human(size)}: background chat has " f"{walk['walked']} messages, expected {chat['size']}", ) chats.append( ( f"bg {human(chat['size'])}", chat["id"], chat["size"], [ position for position in walk["positions"] if chat["size"] - position["after"] >= p.page_limit ], ), ) pages = [] for label, chat_id, chat_size, positions in chats: common = { "chat": label, "chat_id": chat_id, "chat_size": chat_size, } pages.append( { **common, "page": "first", "after": 0, "cursor": None, "items": min(p.page_limit, chat_size), "compare": not positions, }, ) if not positions: continue for page, position in ( ("middle", positions[len(positions) // 2]), ("last", positions[-1]), ): pages.append( { **common, "page": page, "after": position["after"], "cursor": position["cursor"], "items": p.page_limit, "compare": page == "middle", }, ) return pages def explain( self, size: int, pool: dict[str, Any], fixture: dict[str, Any], ) -> list[dict[str, Any]]: """Этап плана: EXPLAIN (ANALYZE, BUFFERS) запросов истории приложения. auto_explain включается между маркерами, `explain.js` читает страницы из `plan_pages.json`; планы берутся из журнала Postgres. Затем страница сравнения каждого чата в psql: варианты индекса и формы запроса в откатываемой транзакции. Решение 5 design change `bench-postgres`, решения 3 и 4 design change `track2-query-plans`. """ stand = self.stand pages = self.plan_pages(size, pool, fixture) (stand.work_dir / "plan_pages.json").write_text( json.dumps(pages), encoding="utf-8", ) indexes = stand.indexes() marker = f"bench-explain {uuid.uuid4().hex}" enable = "".join( f"ALTER SYSTEM SET {key} = '{value}';\n" for key, value in EXPLAIN_SETTINGS.items() ) disable = "".join( f"ALTER SYSTEM RESET {key};\n" for key in EXPLAIN_SETTINGS ) stand.psql( f"DO $$ BEGIN RAISE LOG '{marker} start'; END $$;\n" f"{enable}SELECT pg_reload_conf();\n", ) time.sleep(RELOAD_PAUSE_S) try: stand.k6("explain.js", {}) finally: stand.psql( f"{disable}SELECT pg_reload_conf();\n" f"DO $$ BEGIN RAISE LOG '{marker} end'; END $$;\n", ) # соединения пула API должны получить выключение до прогонов time.sleep(RELOAD_PAUSE_S) entries = [ entry for entry in plan_entries(stand.db_log(), marker) if is_history(entry) ] if len(entries) != len(pages): raise BenchError( f"size {human(size)}: expected {len(pages)} history " f"query plans in the PostgreSQL log, found {len(entries)}", ) plans = [] compared = [] for page, entry in zip(pages, entries, strict=True): parts = plan_parts(entry) or {"plan": entry} where = { "chat": page["chat"], "chat_size": page["chat_size"], "page": page["page"], "after": page["after"], } plans.append( { **where, "source": "app", **plan_summary(parts["plan"], entry_duration(entry)), }, ) if self.app_index is None: self.app_index = index_name(parts["plan"]) if not page["compare"]: continue variants = self.explain_variants(entry, page["cursor"] is not None) compared.append((page, variants)) for variant, plan in variants.items(): summary = plan_summary(plan, None) plans.append({**where, "source": variant, **summary}) self.say( f"size {human(size)}: {page['chat']} {page['page']} " f"{variant}: {summary['node']}, " f"hit={summary['shared_hit']} read={summary['shared_read']}" f", {summary['time_ms']} ms", ) if stand.indexes() != indexes: raise BenchError( f"size {human(size)}: messages indexes changed by the plan " "stage", ) text = render_explain( size, self.results.meta["git"], self.profile, pages, entries, compared, ) path = self.results.dir / f"explain-{human(size)}.txt" path.write_text(text, encoding="utf-8") return plans def explain_variants(self, entry: str, has_cursor: bool) -> dict[str, str]: """Тот же запрос и параметры в psql: варианты индекса и запроса. Всё в транзакции, которая откатывается; нагрузки в этот момент нет. Каждый вариант выполняется дважды, в результат идёт второе выполнение: только что построенный индекс ещё не в shared_buffers. План приложения без индекса — вариантов нет (решение 4 design change `track2-query-plans`). """ parts = plan_parts(entry) index = index_name(parts["plan"]) if parts else None if parts is None or index is None: return {} args = ", ".join( "'" + value.replace("'", "''") + "'" for value in parts["params"] ) def twice(name: str, statement: str) -> str: run = f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE {statement}({args});\n" return f"\\echo @@warmup\n{run}\\echo @@{name}\n{run}" script = ( "SET lock_timeout = '5s';\n" "SET plan_cache_mode = force_custom_plan;\n" "BEGIN;\n" f"PREPARE page AS {parts['text']};\n" + twice("psql", "page") + "SET LOCAL plan_cache_mode = force_generic_plan;\n" + twice("generic", "page") + "SET LOCAL plan_cache_mode = force_custom_plan;\n" ) if has_cursor: script += f"PREPARE page_or AS {or_cursor(parts['text'])};\n" script += twice("or-cursor", "page_or") script += f'DROP INDEX "{index}";\n' + twice("no-index", "page") for name, columns in self.profile.index_variants: script += ( f"CREATE INDEX bench_variant ON messages {columns};\n" + twice(name, "page") + "DROP INDEX bench_variant;\n" ) script += "ROLLBACK;\n" sections: dict[str, list[str]] = {} current: list[str] = [] for line in self.stand.psql(script).splitlines(): if line.startswith("@@"): current = sections.setdefault(line[2:], []) current.clear() continue current.append(line) sections.pop("warmup", None) return { name: "\n".join(lines).strip() for name, lines in sections.items() } def write_cost(self, size: int) -> list[dict[str, Any]]: """Стоимость записи по вариантам индекса истории. Конфигурации `app` (индекс приложения, построенный заново), варианты профиля и `none`; повторы со сдвигом порядка, `VACUUM` после каждой пробы. Медиана и разброс — в сводке (решение 5 design change `track2-query-plans`). """ stand = self.stand p = self.profile if self.app_index is None: raise BenchError( f"size {human(size)}: the app history plan used no index", ) definition = stand.psql( f"SELECT pg_get_indexdef('{self.app_index}'::regclass);\n", ) columns = definition.partition(" USING btree ")[2] if not columns: raise BenchError(f"unexpected index definition: {definition}") configs: list[tuple[str, str | None]] = [ ("app", columns), *p.index_variants, ("none", None), ] before = (stand.indexes(), stand.message_count()) samples: dict[str, list[dict[str, Any]]] = {n: [] for n, _ in configs} for repeat in range(p.write_probe_repeats): shift = repeat % len(configs) for name, config in configs[shift:] + configs[:shift]: samples[name].append(stand.write_probe(self.app_index, config)) stand.psql("VACUUM messages;\n") if (stand.indexes(), stand.message_count()) != before: raise BenchError( f"size {human(size)}: messages indexes or rows changed by the " "write probes", ) result = [] for name, config in configs: runs = samples[name] index_bytes = int( statistics.median(r["index_bytes"] for r in runs), ) row = { "variant": name, "columns": config, "rows": runs[0]["rows"], "messages": runs[0]["rows_before"], "index_bytes": index_bytes, "index_bytes_per_message": round( index_bytes / runs[0]["rows_before"], 1, ), "insert_us_per_message": [ round(r["time_ms"] * 1000 / r["rows"], 2) for r in runs ], "wal_bytes_per_message": [ round(r["wal_bytes"] / r["rows"], 1) for r in runs ], "fpi_per_1000": [ round(r["wal_fpi"] * 1000 / r["rows"], 1) for r in runs ], "long_text": runs[0]["long_text"], } result.append(row) self.say( f"size {human(size)}: write cost {name}: " f"{cell(row['insert_us_per_message'], 2)} us/msg, " f"{cell(row['wal_bytes_per_message'], 0)} WAL B/msg, index " f"{row['index_bytes_per_message']} B/msg, 4000 chars: " f"{row['long_text']}", ) return result def backup_restore( self, size: int, fixture: dict[str, Any], store: int, ) -> dict[str, Any]: """Backup под записью, восстановление в новое хранилище, сверка. Порядок и проверки — решения 2–5 design change `track2-backup-restore`. Расхождение — BenchError с размером; хранилище восстановления убирает `Stand.stop()`. """ stand = self.stand p = self.profile name = human(size) rows_before = stand.message_rows() counts_before = stand.table_counts() # запись через HTTP во время backup: dump начинается, когда она идёт load = stand.k6_start( "write.js", { "VUS": p.restore_write_vus, "COUNT": p.restore_write_load, "OFFSET": store, }, ) dump = stand.work_dir / "backup.dump" try: deadline = time.monotonic() + 30 while stand.message_count() <= counts_before["messages"]: if load.poll() is not None or time.monotonic() > deadline: raise BenchError( f"size {name}: write load did not start before backup", ) time.sleep(0.05) started = time.monotonic() stand.pg_dump(dump) dump_seconds = time.monotonic() - started at_dump_end = stand.message_count() finally: out, err = load.communicate() if load.returncode != 0: raise BenchError( k6_failure( "write.js", subprocess.CompletedProcess([], load.returncode, out, err), ), ) written = parse_summary(stand.work_dir / "last.json") # backup не должен мешать записи: k6 выходит с 0 и при ошибках if ( written["requests_failed"] or written["requests_ok"] != p.restore_write_load ): raise BenchError( f"size {name}: writes during backup: " f"{written['requests_ok']} of {p.restore_write_load} ok, " f"{written['requests_failed']} failed", ) dump_bytes = dump.stat().st_size counts_loaded = stand.table_counts() self.say( f"size {name}: backup {mib_bytes(dump_bytes)} MiB in " f"{dump_seconds:.2f}s during {written['requests_ok']} writes " f"({written['requests_failed']} failed)", ) stand.k6("restore_check.js", {"PHASE": "source"}) source_check = stand.work_json("check-source.json") stand.k6("after_backup.js", {}) after = stand.work_json("after.json") rows_source = stand.message_rows() # восстановление: RTO — от запуска нового хранилища до готовности API started = time.monotonic() stand.start_restore() storage_seconds = time.monotonic() - started copy_seconds, restore_seconds = stand.pg_restore(dump) mark = time.monotonic() stand.psql("ANALYZE;\n", container=RESTORE_DB) analyze_seconds = time.monotonic() - mark db_seconds = time.monotonic() - started mark = time.monotonic() rows_restored = stand.message_rows(container=RESTORE_DB) counts_restored = stand.table_counts(container=RESTORE_DB) problems = [] mismatched = sum( 1 for message, row in rows_restored.items() if rows_source.get(message) != row ) if mismatched: problems.append(f"{mismatched} restored messages differ") missing_before = len(rows_before.keys() - rows_restored.keys()) if missing_before: problems.append( f"{missing_before} messages from before backup lost", ) lost = rows_source.keys() - rows_restored.keys() allowed = {fixture["writeChatId"], after["chatId"]} foreign = sum(1 for m in lost if rows_source[m][0] not in allowed) if foreign: problems.append(f"{foreign} lost messages outside written chats") for table in DATA_TABLES[:-1]: if counts_restored[table] != counts_loaded[table]: problems.append( f"{table}: {counts_restored[table]} restored, " f"{counts_loaded[table]} at backup", ) if stand.indexes(container=RESTORE_DB) != stand.indexes(): problems.append("messages indexes differ") version = "SELECT version_num FROM alembic_version;\n" if stand.psql(version, container=RESTORE_DB) != stand.psql(version): problems.append("alembic revision differs") verify_seconds = time.monotonic() - mark mark = time.monotonic() stand.switch_api(RESTORE_HOST) api_seconds = time.monotonic() - mark stand.k6("restore_check.js", {"PHASE": "stale"}) stale = stand.work_json("check-stale.json") sh( "docker", "exec", CACHE, "sh", "-c", 'exec valkey-cli -p "$VALKEY_PORT" FLUSHALL', ) stand.k6("restore_check.js", {"PHASE": "restored"}) restored = stand.work_json("check-restored.json") for phase, check in ( ("source", source_check), ("stale", stale), ("restored", restored), ): if check["reader_me"] != 200: problems.append( f"{phase}: reader session {check['reader_me']}", ) if stale["late_me"] != 401: problems.append(f"session after backup: {stale['late_me']}") if restored["after_chat_get"] != 404: problems.append( f"chat after backup after FLUSHALL: {restored['after_chat_get']}", ) for phase, check in (("stale", stale), ("restored", restored)): differ = [ f"{a['chat']} {a['page']}" for a, b in zip( check["pages"], source_check["pages"], strict=True, ) if a["status"] != 200 or a["items"] != b["items"] ] if differ: problems.append(f"{phase}: pages differ: {', '.join(differ)}") if problems: raise BenchError( f"size {name}: restore check failed: {'; '.join(problems)}", ) result = { "messages_before_backup": counts_before["messages"], "messages_at_dump_end": at_dump_end, "messages_source_end": len(rows_source), "messages_restored": len(rows_restored), "messages_lost": len(lost), "lost_after_backup_chat": sum( 1 for m in lost if rows_source[m][0] == after["chatId"] ), "load_written": written["requests_ok"], "load_failed": written["requests_failed"], "load_in_backup": len(rows_restored) - counts_before["messages"], "tables_restored": counts_restored, "backup_bytes": dump_bytes, "backup_bytes_per_message": round( dump_bytes / len(rows_restored), 1, ), "dump_seconds": round(dump_seconds, 2), "storage_seconds": round(storage_seconds, 2), "copy_seconds": round(copy_seconds, 2), "restore_seconds": round(restore_seconds, 2), "analyze_seconds": round(analyze_seconds, 2), "verify_seconds": round(verify_seconds, 2), "api_ready_seconds": round(api_seconds, 2), "rto_seconds": round(db_seconds + api_seconds, 2), "pages_checked": len(source_check["pages"]), "stale_chat_get": stale["after_chat_get"], "stale_chat_post": stale["after_chat_post"], "flushed_chat_get": restored["after_chat_get"], "late_session": stale["late_me"], } self.say( f"size {name}: restored {len(rows_restored)} of " f"{len(rows_source)} messages ({len(lost)} lost after the " f"snapshot), restore {restore_seconds:.2f}s, ANALYZE " f"{analyze_seconds:.2f}s, API {api_seconds:.2f}s, RTO " f"{result['rto_seconds']}s; stale chat GET " f"{result['stale_chat_get']} POST {result['stale_chat_post']}, " f"after FLUSHALL {result['flushed_chat_get']}", ) return result def run_prefill(self, size: int, total: int) -> float: """Наполнение всех чатов размера по расписанию, порциями.""" chunk_size = self.profile.prefill_chunk total_seconds = 0.0 for chunk, offset in enumerate(range(0, total, chunk_size), start=1): count = min(chunk_size, total - offset) name = f"prefill-{human(size)}-{chunk:02d}" self.stage = f"size {human(size)}: prefill" measured = measure( self.stand, "prefill.js", { "COUNT": count, "OFFSET": offset, "VUS": self.profile.prefill_vus, "NAME": name, }, self.results.dir / "raw" / f"{name}.json", ) summary = measured.summary down = self.stand.down_storages() flags = run_flags( self.profile, measured, self.stand.state(APP), down, ) self.results.add_prefill( { **summary, "size": size, "chunk": chunk, "messages_before": offset, "messages_after": offset + summary["requests_ok"], "app_cpu_cores": measured.app_cpu, "rss_end_mib": mib(measured.after["rss_kib"]), "mem_cgroup_end_mib": mib_bytes( measured.after["mem_bytes"], ), "generator_cpu_cores": measured.generator_cpu, "flags": " ".join(flags), "db_cpu_cores": measured.db_cpu, }, ) if down: raise BenchError( f"{', '.join(down)} went down during prefill chunk " f"{chunk}: matrix stopped", ) if summary["requests_ok"] != count: raise BenchError( f"prefill chunk {chunk}: {summary['requests_ok']} of " f"{count} messages confirmed ({' '.join(flags)})", ) total_seconds += summary["duration_s"] self.prefilled = offset + count self.prefill_rate = self.prefilled / total_seconds self.say( f"size {human(size)}: prefill {self.prefilled}/{total} " f"({self.prefilled * 100 // total}%), " f"{self.prefill_rate:.0f} msg/s, " f"~{fmt_left((total - self.prefilled) / self.prefill_rate)} " "left", ) self.progress() return total_seconds def run_one( self, scenario: str, size: int, conc: int, repeat: int, store: int, ) -> int: """Прогрев и измеряемый прогон; возвращает сообщений в хранилище.""" p = self.profile number = self.runs_done + 1 name = f"{scenario}-{human(size)}-c{conc}-r{repeat}" self.stage = f"run {number}/{self.runs_total} {name}" self.progress() wall = time.monotonic() if scenario == "read": warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"} self.stand.k6( "read.js", {**warm, "SEED": p.seed + p.warmup_seed_shift}, ) env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"} env["SEED"] = p.seed else: self.stand.k6( "write.js", {"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store}, ) warm_ok = parse_summary(self.stand.work_dir / "last.json") store += warm_ok["requests_ok"] env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store} measured = measure( self.stand, f"{scenario}.js", {**env, "NAME": name}, self.results.dir / "raw" / f"{name}.json", ) summary = measured.summary app = self.stand.state(APP) down = self.stand.down_storages() flags = run_flags(p, measured, app, down) end = store + (summary["requests_ok"] if scenario == "write" else 0) row = { **summary, "run": number, "scenario": scenario, "size": size, "concurrency": conc, "repeat": repeat, "app_cpu_cores": measured.app_cpu, "rss_start_mib": mib(measured.before["rss_kib"]), "rss_end_mib": mib(measured.after["rss_kib"]), "rss_max_mib": mib(measured.rss_max_kib), "mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]), "store_messages_start": store, "store_messages_end": end, "generator_cpu_cores": measured.generator_cpu, "flags": " ".join(flags), "db_cpu_cores": measured.db_cpu, } self.results.add_run(row) self.runs_done += 1 if scenario == "write" and summary["rps"]: self.write_rps[conc] = summary["rps"] overhead = time.monotonic() - wall - summary["duration_s"] if scenario == "read": overhead -= p.read_warmup_s self.run_overhead = max(1.0, overhead) errors = summary["requests_failed"] + summary["check_failed"] self.say( f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} " f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps " f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} " f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} " f"db {measured.db_cpu} rss {row['rss_end_mib']} MiB " f"err {errors} " f"~{fmt_left(self.eta())} left" + (f" [{' '.join(flags)}]" if flags else ""), ) self.progress(last=row) if "oom" in flags or "app_down" in flags: raise BenchError( f"{APP} is down after {name} ({' '.join(flags)}): " "matrix stopped", ) if down: raise BenchError( f"{', '.join(down)} went down during {name}: matrix stopped", ) return end # ------------------------------------------------------------------ CLI def parse_args(argv: list[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--profile", choices=sorted(PROFILES), required=True) parser.add_argument("--label", required=True) # отладка стенда: переопределения пишутся в паспорт как overrides parser.add_argument("--sizes", help="через запятую, например 120000") parser.add_argument("--concurrency", help="через запятую") parser.add_argument("--repeats", type=int) parser.add_argument("--memory", help="лимит памяти, например 64m") return parser.parse_args(argv) def apply_overrides( profile: Profile, args: argparse.Namespace, ) -> tuple[Profile, dict[str, Any]]: overrides: dict[str, Any] = {} if args.sizes: overrides["sizes"] = tuple(int(x) for x in args.sizes.split(",")) if args.concurrency: overrides["concurrency"] = tuple( int(x) for x in args.concurrency.split(",") ) if args.repeats: overrides["repeats"] = args.repeats if args.memory: overrides["memory"] = args.memory return replace(profile, **overrides), overrides def main(argv: list[str]) -> int: args = parse_args(argv) if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label): print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True) return 2 label_dir = ROOT / "evidence" / "bench" / args.label relative = label_dir.relative_to(ROOT) if label_dir.exists(): print(f"label directory {relative} already exists", flush=True) return 2 profile, overrides = apply_overrides(PROFILES[args.profile], args) git = git_info() state = host_state() try: warnings = host_warnings(state) + vm_warnings(profile) except BenchError as exc: print(f"bench failed: {exc}", flush=True) return 2 meta: dict[str, Any] = { "label": args.label, "status": "incomplete", "reason": "running", "started_at": datetime.now(UTC).isoformat(timespec="seconds"), "command": " ".join(["uv run python bench/run.py", *argv]), "git": git, "overrides": overrides, "host_state": {"start": state}, "host_warnings": warnings, # чужие контейнеры в той же VM: стенд их не трогает, но они шумят "other_containers": sorted( sh("docker", "ps", "--format", "{{.Names}}", check=False).split(), ), } stand_check = Stand(profile, label_dir) started = time.monotonic() # на macOS монотонные часы во сне стоят, а реальные идут: разница — # сколько хост проспал посреди прогона (решение 9 design change # track2-backup-restore) started_wall = time.time() def say(text: str) -> None: elapsed = int(time.monotonic() - started) stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed)) print(f"[{stamp}] {text}", flush=True) for warning in warnings: say(f"WARNING: {warning}") if overrides: say(f"WARNING: profile overrides {overrides}: debug run") if git["dirty"]: say("NOTE: working tree has uncommitted changes (dirty: true)") try: stand_check.check_free() image = stand_check.build_image() except BenchError as exc: print(f"bench failed: {exc}", flush=True) return 1 say(f"image {APP_IMAGE} built ({image[:19]}…), compose project {PROJECT}") meta.update(static_passport(profile)) results = Results(label_dir, meta) bench = Bench(profile, args.label, results) bench.started = started code = 0 def terminate(*_: object) -> None: raise KeyboardInterrupt signal.signal(signal.SIGTERM, terminate) try: bench.run() meta["status"] = "complete" meta.pop("reason", None) except KeyboardInterrupt: meta["reason"] = "interrupted" say("interrupted: partial results kept") code = 130 except BenchError as exc: meta["reason"] = str(exc).splitlines()[0] say(f"bench failed: {exc}") code = 1 finally: # повторный Ctrl+C не должен оборвать уборку стенда signal.signal(signal.SIGINT, signal.SIG_IGN) if not bench.stand.stop(): say(f"cleanup incomplete: {leftovers()}") code = code or 1 # fixture.json с токенами — только здесь, в результаты не попадает shutil.rmtree(bench.stand.work_dir, ignore_errors=True) meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds") meta["host_state"]["end"] = host_state() slept = (time.time() - started_wall) - (time.monotonic() - started) meta["host_state"]["slept_seconds"] = round(max(0.0, slept)) if slept > SLEEP_WARN_S: warning = f"host slept {slept:.0f}s during the run: rerun" meta["host_warnings"].append(warning) say(f"WARNING: {warning}") results.write_meta() results.write_summary() bench.stage = "done" if code == 0 else meta.get("reason", "stopped") bench.progress() say( f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, " f"results in {relative}", ) return code if __name__ == "__main__": sys.exit(main(sys.argv[1:]))