- carry the lab 2 solution over and plan lab 3 (PLAN.md, OpenSpec change instance-id-idempotency) - rename the compose project to mtusi-lab3 and the image to :lab3 - add X-Instance-Id to every response and instance_id to every log record ([app] instance_id, hostname by default) - require Idempotency-Key on POST messages: keys table in Postgres, key and message in one transaction, replay returns the same 201, another body gets 409, TTL from [idempotency] ttl_seconds - send the key from student-tests, k6 scenarios and the skill driver; add idempotency, X-Instance-Id, key race and fault tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2992 lines
117 KiB
Python
2992 lines
117 KiB
Python
"""Нагрузочный стенд: матрица размеров истории и конкуренции.
|
||
|
||
Запуск — `make bench` / `make bench_quick`. Стенд — API, PostgreSQL и
|
||
Valkey: проект compose `mtusi-bench` из `docker-compose.yml` с override
|
||
`bench/bench.compose.yml`. Методика — design changes `load-bench-harness`
|
||
и `bench-postgres`, инварианты — `.claude/rules/bench.md`. Только stdlib:
|
||
приложение не импортируется, его нагружает k6 по HTTP; к PostgreSQL
|
||
стенда оркестратор ходит только для обслуживания и измерений (VACUUM,
|
||
размеры, планы запроса).
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import secrets
|
||
import shutil
|
||
import signal
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable
|
||
from dataclasses import asdict, dataclass, replace
|
||
from datetime import UTC, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
K6_IMAGE = (
|
||
"grafana/k6:2.3.0@sha256:"
|
||
"9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34"
|
||
)
|
||
APP_IMAGE = "mtusi-messenger:bench"
|
||
PROJECT = "mtusi-bench"
|
||
NETWORK = "mtusi-bench"
|
||
APP = "mtusi-bench-api"
|
||
DB = "mtusi-bench-postgres"
|
||
CACHE = "mtusi-bench-valkey"
|
||
K6 = "mtusi-bench-k6"
|
||
# второе хранилище этапа восстановления: контейнер, volume и имя в сети
|
||
# стенда (по нему API переключается на восстановленную базу)
|
||
RESTORE_DB = "mtusi-bench-postgres-restore"
|
||
RESTORE_VOLUME = "mtusi-bench_restore"
|
||
RESTORE_HOST = "postgres-restore"
|
||
# таблицы данных приложения: сверка числа строк после восстановления
|
||
DATA_TABLES = ("users", "chats", "chat_members", "sessions", "messages")
|
||
# контейнеры стенда по ролям: лимиты, состояние, паспорт
|
||
CONTAINERS = {"api": APP, "postgres": DB, "valkey": CACHE}
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "bench" / "bench.compose.yml",
|
||
)
|
||
# подстановки compose вместо .env рабочего стенда
|
||
ENV_FILE = ROOT / "bench" / "bench.env"
|
||
# что из окружения хоста нужно самому docker compose; остальное (в том
|
||
# числе POSTGRES_* из shell) в стенд не попадает
|
||
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "DOCKER_HOST", "DOCKER_CONFIG")
|
||
READY_CHECK = (
|
||
"import urllib.request; urllib.request.urlopen("
|
||
"'http://127.0.0.1:8080/health/ready', timeout=2)"
|
||
)
|
||
SUCCESS_METRIC = "http_req_duration{expected_response:true}"
|
||
# предупреждение о хосте: swap занят больше, МиБ
|
||
SWAP_WARN_MIB = 1024
|
||
# предупреждение о VM: лимиты памяти стенда + этот запас больше памяти VM
|
||
VM_RESERVE_GIB = 1.5
|
||
# предупреждение о хосте: реальное время прогона больше монотонного, с
|
||
SLEEP_WARN_S = 5
|
||
# psql в контейнере Postgres от пользователя образа, без пароля; SQL — stdin
|
||
PSQL = (
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
|
||
)
|
||
# настройки Postgres в паспорте: от них зависят план и стоимость записи
|
||
PG_SETTINGS = (
|
||
"shared_buffers",
|
||
"effective_cache_size",
|
||
"work_mem",
|
||
"maintenance_work_mem",
|
||
"max_connections",
|
||
"synchronous_commit",
|
||
"fsync",
|
||
"wal_level",
|
||
"checkpoint_timeout",
|
||
"max_wal_size",
|
||
"random_page_cost",
|
||
"jit",
|
||
"autovacuum",
|
||
"plan_cache_mode",
|
||
"shared_preload_libraries",
|
||
)
|
||
# auto_explain на этапе плана; вне его log_min_duration = -1 (выключен)
|
||
EXPLAIN_SETTINGS = {
|
||
"auto_explain.log_min_duration": "0",
|
||
"auto_explain.log_analyze": "on",
|
||
"auto_explain.log_buffers": "on",
|
||
"auto_explain.log_timing": "on",
|
||
"auto_explain.log_format": "text",
|
||
}
|
||
# пауза после pg_reload_conf(): SIGHUP доходит до соединений пула API
|
||
RELOAD_PAUSE_S = 1.0
|
||
# индекс, который использовал план приложения
|
||
INDEX_NODE_RE = re.compile(
|
||
r"Index Only Scan(?: Backward)? using (\w+)"
|
||
r"|Index Scan(?: Backward)? using (\w+)"
|
||
r"|Bitmap Index Scan on (\w+)",
|
||
)
|
||
# условие курсора в запросе приложения: сравнение кортежей
|
||
OR_CURSOR_RE = re.compile(
|
||
r"\(messages\.created_at, messages\.id\) > "
|
||
r"\((\$\d+::[A-Z][A-Z ]*[A-Z]), (\$\d+::UUID)\)",
|
||
)
|
||
# длина текста сообщения по контракту (contracts/openapi.json, text
|
||
# maxLength): проба записи проверяет, что такое сообщение вставляется
|
||
MAX_TEXT_LENGTH = 4000
|
||
# сообщение максимальной длины из несжимаемых символов CJK (~12 КБ UTF-8,
|
||
# md5 номера символа — детерминировано): вставка в DO, итог — NOTICE
|
||
LONG_TEXT_PROBE = f"""DO $$
|
||
BEGIN
|
||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||
SELECT gen_random_uuid(), chat_id, sender_id,
|
||
(SELECT string_agg(
|
||
chr(19968 + ('x' || substr(md5(i::text), 1, 4))::bit(16)::int % 20000),
|
||
'')
|
||
FROM generate_series(1, {MAX_TEXT_LENGTH}) i),
|
||
now() + interval '1 hour'
|
||
FROM probe_src LIMIT 1;
|
||
RAISE NOTICE 'long-text: ok';
|
||
EXCEPTION WHEN others THEN
|
||
RAISE NOTICE 'long-text: %', SQLERRM;
|
||
END $$;
|
||
"""
|
||
|
||
RUN_FIELDS = [
|
||
"run",
|
||
"scenario",
|
||
"size",
|
||
"concurrency",
|
||
"repeat",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_start_mib",
|
||
"rss_end_mib",
|
||
"rss_max_mib",
|
||
"mem_cgroup_end_mib",
|
||
"store_messages_start",
|
||
"store_messages_end",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"check_failed",
|
||
# лаба 2: новые столбцы — только в конец
|
||
"db_cpu_cores",
|
||
]
|
||
PREFILL_FIELDS = [
|
||
"size",
|
||
"chunk",
|
||
"messages_before",
|
||
"messages_after",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_end_mib",
|
||
"mem_cgroup_end_mib",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"db_cpu_cores",
|
||
]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Profile:
|
||
name: str
|
||
sizes: tuple[int, ...]
|
||
concurrency: tuple[int, ...]
|
||
repeats: int
|
||
read_warmup_s: int
|
||
read_measure_s: int
|
||
write_warmup: int
|
||
write_measure: int
|
||
# порция ~30 с при ~850 сообщ/с на Postgres: прогресс наполнения
|
||
# печатается после порции, а спек требует строку не реже раза в минуту
|
||
prefill_chunk: int = 25_000
|
||
prefill_vus: int = 32
|
||
text_length: int = 64
|
||
page_limit: int = 50
|
||
walk_limit: int = 100
|
||
cursor_pool: int = 1000
|
||
members: int = 10
|
||
seed: int = 20260930
|
||
# зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно
|
||
# те страницы, с которых начнётся замер
|
||
warmup_seed_shift: int = 1_000_003
|
||
generator_cpu_limit: float = 4.0
|
||
cpus: str = "1"
|
||
memory: str = "2g"
|
||
# лаба 2: лимиты хранилищ (swap = память) и срок сессии стенда; идут в
|
||
# compose переменными BENCH_* (bench/bench.compose.yml). Решение 2
|
||
# design change bench-postgres
|
||
db_cpus: str = "4"
|
||
db_memory: str = "3g"
|
||
cache_cpus: str = "2"
|
||
cache_memory: str = "512m"
|
||
session_ttl_seconds: int = 21600
|
||
# трек 2: фоновые чаты уровнями (число чатов, сообщений в чате), то же
|
||
# на каждом размере; варианты индекса истории (имя, столбцы) для
|
||
# планов и стоимости записи; пробы записи — строк и повторов.
|
||
# Решения 1, 4, 5 design change track2-query-plans
|
||
background: tuple[tuple[int, int], ...] = ()
|
||
index_variants: tuple[tuple[str, str], ...] = (
|
||
("chat_only", "(chat_id)"),
|
||
("covering", "(chat_id, created_at, id) INCLUDE (sender_id, text)"),
|
||
)
|
||
write_probe_rows: int = 20_000
|
||
write_probe_repeats: int = 3
|
||
# трек 2, часть 2: backup и восстановление в новое хранилище в конце
|
||
# размера; pg_restore -j, запись через HTTP во время backup. Решение 1
|
||
# design change track2-backup-restore
|
||
restore: bool = False
|
||
restore_jobs: int = 4
|
||
restore_write_load: int = 5_000
|
||
restore_write_vus: int = 10
|
||
|
||
@property
|
||
def background_total(self) -> int:
|
||
return sum(count * size for count, size in self.background)
|
||
|
||
|
||
PROFILES = {
|
||
"full": Profile(
|
||
name="full",
|
||
# 500k вместо 1M лабы 1: прогон ~30 мин вместо ~45, а диапазон
|
||
# 10k → 500k (×50) уже показывает, растёт ли страница с историей
|
||
sizes=(10_000, 100_000, 500_000),
|
||
concurrency=(1, 10, 50),
|
||
repeats=3,
|
||
read_warmup_s=5,
|
||
read_measure_s=20,
|
||
write_warmup=500,
|
||
write_measure=5000,
|
||
# 100 000 фоновых сообщений: в таблице ≥110 000 на любом размере
|
||
background=((10, 5_000), (100, 400), (1_000, 10)),
|
||
),
|
||
"quick": Profile(
|
||
name="quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(1, 4, 16),
|
||
repeats=1,
|
||
read_warmup_s=2,
|
||
read_measure_s=5,
|
||
write_warmup=100,
|
||
write_measure=1000,
|
||
background=((2, 1_000), (10, 100), (50, 10)),
|
||
write_probe_rows=5_000,
|
||
),
|
||
# backup и восстановление на данных полной матрицы, без прогонов
|
||
"restore": Profile(
|
||
name="restore",
|
||
sizes=(10_000, 100_000, 500_000),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
background=((10, 5_000), (100, 400), (1_000, 10)),
|
||
restore=True,
|
||
),
|
||
"restore_quick": Profile(
|
||
name="restore_quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
background=((2, 1_000), (10, 100), (50, 10)),
|
||
write_probe_rows=5_000,
|
||
restore=True,
|
||
restore_write_load=1_000,
|
||
),
|
||
}
|
||
|
||
|
||
class BenchError(Exception):
|
||
"""Стенд не может продолжать: сообщение для человека."""
|
||
|
||
|
||
# ---------------------------------------------------------------- утилиты
|
||
|
||
|
||
def sh(*args: str, check: bool = True) -> str:
|
||
"""Команда без shell; stdout без хвостовых пробелов.
|
||
|
||
`LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод
|
||
разбирается регулярками с точкой.
|
||
"""
|
||
proc = subprocess.run(
|
||
args,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
env={**os.environ, "LC_ALL": "C"},
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"{' '.join(args[:4])}…: exit {proc.returncode}\n"
|
||
f"{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
|
||
def human(n: int) -> str:
|
||
"""10000 → 10k, 1000000 → 1M."""
|
||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||
return f"{n // 1_000_000}M"
|
||
if n >= 1000 and n % 1000 == 0:
|
||
return f"{n // 1000}k"
|
||
return str(n)
|
||
|
||
|
||
def fmt_left(seconds: float) -> str:
|
||
seconds = max(0, int(seconds))
|
||
hours, rest = divmod(seconds, 3600)
|
||
minutes, secs = divmod(rest, 60)
|
||
if hours:
|
||
return f"{hours}h{minutes:02d}m"
|
||
if minutes:
|
||
return f"{minutes}m{secs:02d}s"
|
||
return f"{secs}s"
|
||
|
||
|
||
def mib(kib: float | None) -> float | None:
|
||
return None if kib is None else round(kib / 1024, 1)
|
||
|
||
|
||
def mib_bytes(value: int | None) -> float | None:
|
||
return None if value is None else round(value / 2**20, 1)
|
||
|
||
|
||
def size_bytes(text: str) -> int:
|
||
"""Лимит памяти docker (`512m`, `2g`) в байтах."""
|
||
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kmg]?)b?", text.strip().lower())
|
||
if not match:
|
||
raise BenchError(f"bad memory limit {text!r}: use 512m or 2g")
|
||
power = {"": 0, "k": 1, "m": 2, "g": 3}[match.group(2)]
|
||
return int(float(match.group(1)) * 1024**power)
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 0) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 3) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def chat_sizes(profile: Profile, size: int) -> list[int]:
|
||
"""Размеры чатов наполнения: читаемый, затем фоновые по уровням."""
|
||
return [size] + [
|
||
chat_size
|
||
for count, chat_size in profile.background
|
||
for _ in range(count)
|
||
]
|
||
|
||
|
||
def fill_schedule(sizes: list[int]) -> list[int]:
|
||
"""Номер чата для каждого сообщения наполнения.
|
||
|
||
Чат с s сообщениями получает позиции (k + 0,5) / s, k = 0…s−1; все
|
||
позиции сортируются, при равенстве — по номеру чата. Сообщения
|
||
каждого чата равномерно размазаны по наполнению, порядок
|
||
детерминирован (решение 2 design change `track2-query-plans`).
|
||
"""
|
||
slots = sorted(
|
||
((k + 0.5) / size, chat)
|
||
for chat, size in enumerate(sizes)
|
||
for k in range(size)
|
||
)
|
||
return [chat for _, chat in slots]
|
||
|
||
|
||
def or_cursor(text: str) -> str:
|
||
"""Запрос приложения с условием курсора через OR вместо кортежей.
|
||
|
||
Остальное — список столбцов, ORDER BY, LIMIT — как у приложения. Нет
|
||
шаблона — приложение изменило запрос, тихой копии SQL не делаем.
|
||
"""
|
||
new, count = OR_CURSOR_RE.subn(
|
||
r"(messages.created_at > \1 OR "
|
||
r"(messages.created_at = \1 AND messages.id > \2))",
|
||
text,
|
||
)
|
||
if count != 1:
|
||
raise BenchError(
|
||
"history query has no (created_at, id) row comparison: the app "
|
||
"query changed, update OR_CURSOR_RE",
|
||
)
|
||
return new
|
||
|
||
|
||
# ------------------------------------------------------------ хост и паспорт
|
||
|
||
|
||
def host_state() -> dict[str, Any]:
|
||
"""Swap, свободная память и питание (macOS); на другой ОС — null."""
|
||
state: dict[str, Any] = {
|
||
"swap_used_mib": None,
|
||
"swap_total_mib": None,
|
||
"memory_free_percent": None,
|
||
"power": None,
|
||
"battery_percent": None,
|
||
}
|
||
if platform.system() != "Darwin":
|
||
return state
|
||
swap = sh("sysctl", "-n", "vm.swapusage", check=False)
|
||
for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")):
|
||
match = re.search(rf"{key} = ([\d.,]+)M", swap)
|
||
if match:
|
||
state[name] = float(match.group(1).replace(",", "."))
|
||
pressure = sh("memory_pressure", check=False)
|
||
match = re.search(r"free percentage: (\d+)%", pressure)
|
||
if match:
|
||
state["memory_free_percent"] = int(match.group(1))
|
||
batt = sh("pmset", "-g", "batt", check=False)
|
||
match = re.search(r"drawing from '([^']+)'", batt)
|
||
if match:
|
||
state["power"] = match.group(1)
|
||
match = re.search(r"(\d+)%;", batt)
|
||
if match:
|
||
state["battery_percent"] = int(match.group(1))
|
||
return state
|
||
|
||
|
||
def host_warnings(state: dict[str, Any]) -> list[str]:
|
||
warnings = []
|
||
if state["power"] == "Battery Power":
|
||
warnings.append("host on battery power: CPU may be throttled")
|
||
used = state["swap_used_mib"]
|
||
if used is not None and used > SWAP_WARN_MIB:
|
||
warnings.append(
|
||
f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: "
|
||
"reboot before a full run",
|
||
)
|
||
return warnings
|
||
|
||
|
||
def vm_warnings(profile: Profile) -> list[str]:
|
||
"""Лимиты памяти стенда не помещаются в VM docker с запасом.
|
||
|
||
Тогда лимит перестаёт быть гарантией: контейнеры делят память VM, и
|
||
метки разных дней несравнимы.
|
||
"""
|
||
total = sh("docker", "info", "--format", "{{.MemTotal}}", check=False)
|
||
if not total.isdigit():
|
||
return []
|
||
limits = sum(
|
||
size_bytes(value)
|
||
for value in (profile.memory, profile.db_memory, profile.cache_memory)
|
||
)
|
||
budget = int(total) - int(VM_RESERVE_GIB * 2**30)
|
||
if limits <= budget:
|
||
return []
|
||
return [
|
||
f"memory limits {limits / 2**30:.1f} GiB + {VM_RESERVE_GIB} GiB "
|
||
f"reserve exceed docker VM memory {int(total) / 2**30:.1f} GiB: "
|
||
"limits are not guaranteed",
|
||
]
|
||
|
||
|
||
def git_info() -> dict[str, Any]:
|
||
commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD")
|
||
status = sh("git", "-C", str(ROOT), "status", "--porcelain")
|
||
return {"commit": commit, "dirty": bool(status)}
|
||
|
||
|
||
def static_passport(profile: Profile) -> dict[str, Any]:
|
||
docker = json.loads(sh("docker", "info", "--format", "{{json .}}"))
|
||
versions = sh(
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
"-c",
|
||
"import importlib.metadata as m, json, platform; print(json.dumps("
|
||
"{'python': platform.python_version(), **{p: m.version(p) for p in "
|
||
"('fastapi', 'starlette', 'pydantic', 'uvicorn', 'sqlalchemy', "
|
||
"'asyncpg', 'redis')}}))",
|
||
)
|
||
k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version")
|
||
host: dict[str, Any] = {"os": platform.system(), "machine": None}
|
||
if platform.system() == "Darwin":
|
||
host["os"] = f"macOS {platform.mac_ver()[0]}"
|
||
host["machine"] = sh(
|
||
"sysctl",
|
||
"-n",
|
||
"machdep.cpu.brand_string",
|
||
check=False,
|
||
)
|
||
memsize = sh("sysctl", "-n", "hw.memsize", check=False)
|
||
host["memory_gib"] = (
|
||
round(int(memsize) / 2**30, 1) if memsize.isdigit() else None
|
||
)
|
||
return {
|
||
"app_image": {
|
||
"tag": APP_IMAGE,
|
||
"id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE),
|
||
"versions": json.loads(versions),
|
||
},
|
||
"generator": {"image": K6_IMAGE, "version": k6_version},
|
||
"docker": {
|
||
"server": docker.get("ServerVersion"),
|
||
"vm_os": docker.get("OperatingSystem"),
|
||
"vm_cpus": docker.get("NCPU"),
|
||
"vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1),
|
||
"cgroup": docker.get("CgroupVersion"),
|
||
"logging_driver": docker.get("LoggingDriver"),
|
||
},
|
||
"host": host,
|
||
"profile": asdict(profile),
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------------------ стенд
|
||
|
||
|
||
class Watcher:
|
||
"""Раз в секунду, пока идёт запуск k6: VmRSS приложения, живы ли оно,
|
||
PostgreSQL и Valkey.
|
||
|
||
Приложение или хранилище умерло (OOM, kill) — k6 останавливается:
|
||
иначе каждый его запрос ждал бы таймаута, и наполнение тянулось бы
|
||
часами.
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self.samples: list[int] = []
|
||
self.app_died = False
|
||
self.stand_died: list[str] = []
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(target=self._loop, daemon=True)
|
||
|
||
def _halt(self) -> None:
|
||
sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False)
|
||
|
||
def _loop(self) -> None:
|
||
while not self._stop.wait(1.0):
|
||
states = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Name}} {{.State.Running}}",
|
||
DB,
|
||
CACHE,
|
||
check=False,
|
||
)
|
||
dead = [
|
||
name.lstrip("/")
|
||
for name, running in (
|
||
line.split() for line in states.splitlines() if line
|
||
)
|
||
if running != "true"
|
||
]
|
||
if dead:
|
||
self.stand_died = dead
|
||
self._halt()
|
||
return
|
||
out = sh(
|
||
"docker",
|
||
"exec",
|
||
APP,
|
||
"grep",
|
||
"^VmRSS:",
|
||
"/proc/1/status",
|
||
check=False,
|
||
)
|
||
match = re.search(r"(\d+) kB", out)
|
||
if match:
|
||
self.samples.append(int(match.group(1)))
|
||
continue
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
APP,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
self.app_died = True
|
||
self._halt()
|
||
return
|
||
|
||
def __enter__(self) -> "Watcher":
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *_: object) -> None:
|
||
self._stop.set()
|
||
self._thread.join()
|
||
|
||
|
||
def compose_env(profile: Profile) -> dict[str, str]:
|
||
"""Окружение `docker compose` стенда, собранное с нуля.
|
||
|
||
Пароли хранилищ — новые на каждый прогон: живут только в этом словаре
|
||
и в контейнерах стенда, в результаты не пишутся. Лимиты и срок сессии
|
||
— из профиля (`BENCH_*` в `bench/bench.compose.yml`).
|
||
"""
|
||
env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
|
||
return {
|
||
**env,
|
||
"LC_ALL": "C",
|
||
"POSTGRES_PASSWORD": secrets.token_urlsafe(24),
|
||
"VALKEY_PASSWORD": secrets.token_urlsafe(24),
|
||
"BENCH_APP_CPUS": profile.cpus,
|
||
"BENCH_APP_MEMORY": profile.memory,
|
||
"BENCH_DB_CPUS": profile.db_cpus,
|
||
"BENCH_DB_MEMORY": profile.db_memory,
|
||
"BENCH_CACHE_CPUS": profile.cache_cpus,
|
||
"BENCH_CACHE_MEMORY": profile.cache_memory,
|
||
"BENCH_SESSION_TTL_SECONDS": str(profile.session_ttl_seconds),
|
||
}
|
||
|
||
|
||
def leftovers() -> dict[str, list[str]]:
|
||
"""Контейнеры, volume и сеть стенда, оставшиеся от другого прогона.
|
||
|
||
Ключ — подкоманда docker, которой их удалить.
|
||
"""
|
||
project = f"label=com.docker.compose.project={PROJECT}"
|
||
names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split()
|
||
containers = sh(
|
||
"docker",
|
||
"ps",
|
||
"-a",
|
||
"--filter",
|
||
project,
|
||
"--format",
|
||
"{{.Names}}",
|
||
).split()
|
||
networks = sh("docker", "network", "ls", "--format", "{{.Name}}").split()
|
||
return {
|
||
"rm -f": containers + ([K6] if K6 in names else []),
|
||
"volume rm": sh(
|
||
"docker",
|
||
"volume",
|
||
"ls",
|
||
"-q",
|
||
"--filter",
|
||
project,
|
||
).split(),
|
||
"network rm": [NETWORK] if NETWORK in networks else [],
|
||
}
|
||
|
||
|
||
class Stand:
|
||
"""Проект compose `mtusi-bench` и контейнер k6 на время одного размера."""
|
||
|
||
def __init__(self, profile: Profile, label_dir: Path) -> None:
|
||
self.profile = profile
|
||
self.label_dir = label_dir
|
||
self.work_dir = label_dir / ".work"
|
||
self.env = compose_env(profile)
|
||
|
||
def compose(self, *args: str, check: bool = True) -> str:
|
||
"""docker compose стенда; без `-p mtusi-bench` не вызывается."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
PROJECT,
|
||
"--env-file",
|
||
str(ENV_FILE),
|
||
*files,
|
||
*args,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"docker compose {' '.join(args[:3])}: exit "
|
||
f"{proc.returncode}\n{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
def check_free(self) -> None:
|
||
found = {cmd: names for cmd, names in leftovers().items() if names}
|
||
if found:
|
||
fix = "; ".join(
|
||
f"docker {cmd} {' '.join(names)}"
|
||
for cmd, names in found.items()
|
||
)
|
||
raise BenchError(
|
||
"bench stand leftovers exist: another bench is running or a "
|
||
f"crashed one left them; remove with: {fix}",
|
||
)
|
||
|
||
def build_image(self) -> str:
|
||
sh(
|
||
"docker",
|
||
"build",
|
||
"-q",
|
||
"-t",
|
||
APP_IMAGE,
|
||
str(ROOT),
|
||
)
|
||
return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE)
|
||
|
||
def start(self, say: Callable[[str], None]) -> None:
|
||
"""Свежий стенд: пустая база на новом volume, миграции, API, k6."""
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
self.compose("up", "-d", "--wait", "postgres", "valkey")
|
||
say("postgres and valkey healthy")
|
||
self.compose("run", "--rm", "migrate")
|
||
say("migrations applied")
|
||
self.compose("up", "-d", "--no-deps", "--no-build", "api")
|
||
sh(
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
K6,
|
||
"--network",
|
||
NETWORK,
|
||
"-v",
|
||
f"{ROOT / 'bench' / 'k6'}:/scripts:ro",
|
||
"-v",
|
||
f"{self.label_dir}:/out",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"sleep",
|
||
K6_IMAGE,
|
||
"infinity",
|
||
)
|
||
self.wait_ready()
|
||
|
||
def wait_ready(self, timeout: float = 60) -> None:
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
probe = subprocess.run(
|
||
["docker", "exec", APP, "python", "-c", READY_CHECK],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if probe.returncode == 0:
|
||
return
|
||
state = self.state(APP)
|
||
if not state["running"]:
|
||
reason = "oom" if state["oom"] else "exited"
|
||
raise BenchError(f"{APP} died during start ({reason})")
|
||
time.sleep(0.25)
|
||
raise BenchError(f"{APP} not ready in {timeout:.0f}s")
|
||
|
||
def stop(self) -> bool:
|
||
"""k6, контейнеры, сеть и volume стенда; True — ничего не осталось.
|
||
|
||
k6 и хранилище восстановления — первыми: подключённый к сети
|
||
контейнер не из compose не дал бы compose удалить сеть. Volume
|
||
восстановления compose не знает, он удаляется отдельно.
|
||
"""
|
||
subprocess.run(
|
||
["docker", "rm", "-f", K6, RESTORE_DB],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
self.compose("down", "-v", "--remove-orphans", check=False)
|
||
subprocess.run(
|
||
["docker", "volume", "rm", RESTORE_VOLUME],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
return not any(leftovers().values())
|
||
|
||
def start_restore(self) -> None:
|
||
"""Новое хранилище: контейнер PostgreSQL на новом volume.
|
||
|
||
Образ — тот же, что у исходного контейнера (пин без копии в коде),
|
||
лимиты — как у Postgres стенда, пароль — `--env` без значения из
|
||
окружения процесса. Метка проекта — чтобы `leftovers()` видел
|
||
остатки (решение 3 design change `track2-backup-restore`).
|
||
"""
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
env = dict(
|
||
item.split("=", 1)
|
||
for item in json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", DB),
|
||
)
|
||
)
|
||
label = f"com.docker.compose.project={PROJECT}"
|
||
sh("docker", "volume", "create", "--label", label, RESTORE_VOLUME)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
RESTORE_DB,
|
||
"--label",
|
||
label,
|
||
"--network",
|
||
NETWORK,
|
||
"--network-alias",
|
||
RESTORE_HOST,
|
||
"--env",
|
||
"POSTGRES_PASSWORD",
|
||
"-e",
|
||
f"POSTGRES_USER={env['POSTGRES_USER']}",
|
||
"-e",
|
||
f"POSTGRES_DB={env['POSTGRES_DB']}",
|
||
"--cpus",
|
||
self.profile.db_cpus,
|
||
"--memory",
|
||
self.profile.db_memory,
|
||
"--memory-swap",
|
||
self.profile.db_memory,
|
||
"-v",
|
||
f"{RESTORE_VOLUME}:/var/lib/postgresql",
|
||
image,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"restore storage did not start: {proc.stderr}")
|
||
# по TCP, как healthcheck compose: при инициализации образ
|
||
# поднимает временный сервер только на сокете
|
||
deadline = time.monotonic() + 60
|
||
while time.monotonic() < deadline:
|
||
ready = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
RESTORE_DB,
|
||
"sh",
|
||
"-c",
|
||
'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" '
|
||
'-d "$POSTGRES_DB"',
|
||
],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if ready.returncode == 0:
|
||
return
|
||
time.sleep(0.25)
|
||
raise BenchError(f"{RESTORE_DB} not ready in 60s")
|
||
|
||
def limits(self) -> dict[str, Any]:
|
||
"""Лимиты контейнеров стенда, прочитанные у запущенных контейнеров."""
|
||
limits = {}
|
||
for role, name in CONTAINERS.items():
|
||
host = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .HostConfig}}", name),
|
||
)
|
||
config = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config}}", name),
|
||
)
|
||
limits[role] = {
|
||
"nano_cpus": host.get("NanoCpus"),
|
||
"memory": host.get("Memory"),
|
||
"memory_swap": host.get("MemorySwap"),
|
||
"published_ports": host.get("PortBindings") or {},
|
||
"healthcheck": (config.get("Healthcheck") or {}).get("Test"),
|
||
"log_driver": (host.get("LogConfig") or {}).get("Type"),
|
||
}
|
||
return limits
|
||
|
||
def state(self, name: str) -> dict[str, Any]:
|
||
state = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .State}}", name),
|
||
)
|
||
restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", name)
|
||
return {
|
||
"running": state.get("Running"),
|
||
"oom": state.get("OOMKilled"),
|
||
"restarts": int(restarts or 0),
|
||
}
|
||
|
||
def down_storages(self) -> list[str]:
|
||
"""Хранилища, которые за прогон умирали, убивались или рестартовали."""
|
||
down = []
|
||
for name in (DB, CACHE):
|
||
state = self.state(name)
|
||
if not state["running"] or state["oom"] or state["restarts"]:
|
||
down.append(name)
|
||
return down
|
||
|
||
def psql(
|
||
self,
|
||
sql: str,
|
||
*,
|
||
check: bool = True,
|
||
container: str = DB,
|
||
) -> str:
|
||
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля.
|
||
|
||
`container` — исходное хранилище или хранилище восстановления.
|
||
"""
|
||
proc = subprocess.run(
|
||
["docker", "exec", "-i", container, "sh", "-c", PSQL],
|
||
input=sql,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(f"psql: exit {proc.returncode}\n{proc.stderr}")
|
||
return proc.stdout.strip()
|
||
|
||
def maintain(self) -> float:
|
||
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды.
|
||
|
||
Статистика и состояние таблиц одинаковы у всех меток, а не
|
||
зависят от того, когда сработал autovacuum (решение 4).
|
||
"""
|
||
started = time.monotonic()
|
||
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
|
||
return time.monotonic() - started
|
||
|
||
def storage(self) -> dict[str, Any]:
|
||
"""Объём таблицы сообщений и её индексов, число строк."""
|
||
row = self.psql(
|
||
"SELECT pg_table_size('messages'), pg_indexes_size('messages'), "
|
||
"count(*) FROM messages;\n",
|
||
)
|
||
table, indexes, count = (int(value) for value in row.split("|"))
|
||
return {
|
||
"messages_rows": count,
|
||
"messages_table_bytes": table,
|
||
"messages_indexes_bytes": indexes,
|
||
"db_bytes_per_message": (
|
||
round((table + indexes) / count) if count else None
|
||
),
|
||
}
|
||
|
||
def chat_counts(self) -> dict[str, int]:
|
||
"""Число сообщений по чатам (проверка распределения наполнения)."""
|
||
rows = self.psql(
|
||
"SELECT chat_id, count(*) FROM messages GROUP BY chat_id;\n",
|
||
)
|
||
return {
|
||
chat: int(count)
|
||
for chat, count in (line.split("|") for line in rows.splitlines())
|
||
}
|
||
|
||
def indexes(self, container: str = DB) -> list[str]:
|
||
"""Определения индексов messages: до и после этапов с вариантами."""
|
||
return self.psql(
|
||
"SELECT indexdef FROM pg_indexes WHERE tablename = 'messages' "
|
||
"ORDER BY indexname;\n",
|
||
container=container,
|
||
).splitlines()
|
||
|
||
def message_count(self) -> int:
|
||
return int(self.psql("SELECT count(*) FROM messages;\n"))
|
||
|
||
def write_probe(
|
||
self,
|
||
app_index: str,
|
||
columns: str | None,
|
||
) -> dict[str, Any]:
|
||
"""Одна проба записи при индексе истории `columns` (None — без него).
|
||
|
||
Всё в транзакции, которая откатывается: индекс приложения
|
||
удаляется, конфигурация строится заново, `CHECKPOINT`, вставка
|
||
порции во все чаты в пропорции наполнения и сообщение максимальной
|
||
длины (решение 5 design change `track2-query-plans`).
|
||
"""
|
||
p = self.profile
|
||
create = (
|
||
f"CREATE INDEX bench_probe ON messages {columns};\n"
|
||
if columns
|
||
else ""
|
||
)
|
||
size = "pg_relation_size('bench_probe')" if columns else "0"
|
||
proc = subprocess.run(
|
||
["docker", "exec", "-i", DB, "sh", "-c", PSQL],
|
||
input=(
|
||
"SET lock_timeout = '5s';\n"
|
||
"BEGIN;\n"
|
||
"CREATE TEMP TABLE probe_src ON COMMIT DROP AS "
|
||
"SELECT chat_id, sender_id, "
|
||
"row_number() OVER (ORDER BY created_at, id) AS g "
|
||
"FROM (SELECT chat_id, sender_id, created_at, id FROM messages "
|
||
f"ORDER BY created_at, id LIMIT {p.write_probe_rows}) s;\n"
|
||
f'DROP INDEX "{app_index}";\n'
|
||
f"{create}"
|
||
f"SELECT {size}, (SELECT count(*) FROM messages), "
|
||
"(SELECT count(*) FROM probe_src);\n"
|
||
"CHECKPOINT;\n"
|
||
"EXPLAIN (ANALYZE, BUFFERS, WAL) INSERT INTO messages "
|
||
"(id, chat_id, sender_id, text, created_at) "
|
||
"SELECT gen_random_uuid(), chat_id, sender_id, "
|
||
f"rpad('probe-' || g, {p.text_length}, 'x'), "
|
||
"now() + g * interval '1 microsecond' FROM probe_src;\n"
|
||
f"{LONG_TEXT_PROBE}"
|
||
"ROLLBACK;\n"
|
||
),
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"write probe: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
head, _, plan = proc.stdout.strip().partition("\n")
|
||
index_bytes, rows_before, rows = (int(v) for v in head.split("|"))
|
||
wal = re.search(r"WAL: ([^\n]*)", plan)
|
||
|
||
def wal_field(name: str) -> int:
|
||
# EXPLAIN не печатает нулевые поля WAL
|
||
match = (
|
||
re.search(rf"\b{name}=(\d+)", wal.group(1)) if wal else None
|
||
)
|
||
return int(match.group(1)) if match else 0
|
||
|
||
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
|
||
long_text = re.search(r"long-text: ([^\n]*)", proc.stderr)
|
||
if execution is None or long_text is None:
|
||
raise BenchError(f"write probe: unexpected output\n{proc.stdout}")
|
||
return {
|
||
"index_bytes": index_bytes,
|
||
"rows_before": rows_before,
|
||
"rows": rows,
|
||
"time_ms": float(execution.group(1)),
|
||
"wal_records": wal_field("records"),
|
||
"wal_fpi": wal_field("fpi"),
|
||
"wal_bytes": wal_field("bytes"),
|
||
"long_text": long_text.group(1).strip(),
|
||
}
|
||
|
||
def db_log(self) -> str:
|
||
"""Журнал Postgres стенда (stderr контейнера): только в памяти."""
|
||
proc = subprocess.run(
|
||
["docker", "logs", DB],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def passport(self) -> dict[str, Any]:
|
||
"""Хранилища и окружение API для паспорта; паролей здесь нет."""
|
||
settings = self.psql(
|
||
"SELECT name, current_setting(name) FROM pg_settings "
|
||
"WHERE name = ANY(string_to_array("
|
||
f"'{','.join(PG_SETTINGS)}', ',')) ORDER BY name;\n",
|
||
)
|
||
info = sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" INFO server',
|
||
)
|
||
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.M)
|
||
memory = sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" CONFIG GET "maxmemory*"',
|
||
).splitlines()
|
||
env = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", APP),
|
||
)
|
||
app_env = dict(
|
||
sorted(
|
||
item.split("=", 1)
|
||
for item in env
|
||
if re.match(r"[A-Z0-9_]+__[A-Z0-9_]+=", item)
|
||
and "PASSWORD" not in item.split("=", 1)[0]
|
||
),
|
||
)
|
||
|
||
def image(name: str) -> dict[str, str]:
|
||
return {
|
||
"image": sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Config.Image}}",
|
||
name,
|
||
),
|
||
"id": sh("docker", "inspect", "-f", "{{.Image}}", name),
|
||
}
|
||
|
||
return {
|
||
"postgres": {
|
||
**image(DB),
|
||
"version": self.psql("SHOW server_version;\n"),
|
||
"settings": dict(
|
||
line.split("|", 1) for line in settings.splitlines()
|
||
),
|
||
},
|
||
"valkey": {
|
||
**image(CACHE),
|
||
"version": valkey_version.group(1) if valkey_version else None,
|
||
**dict(zip(memory[::2], memory[1::2], strict=True)),
|
||
},
|
||
"app_env": app_env,
|
||
}
|
||
|
||
def snapshot(self) -> dict[str, Any]:
|
||
"""CPU и память приложения, CPU PostgreSQL и генератора.
|
||
|
||
Если приложение уже мертво (OOM), его поля — None: снимок нужен и
|
||
для записи проваленного прогона.
|
||
"""
|
||
app = sh(
|
||
"docker",
|
||
"exec",
|
||
APP,
|
||
"sh",
|
||
"-c",
|
||
"cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; "
|
||
"grep -E '^(VmRSS|VmHWM):' /proc/1/status",
|
||
check=False,
|
||
)
|
||
k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat")
|
||
db = sh(
|
||
"docker",
|
||
"exec",
|
||
DB,
|
||
"cat",
|
||
"/sys/fs/cgroup/cpu.stat",
|
||
check=False,
|
||
)
|
||
|
||
def number(pattern: str, text: str) -> int | None:
|
||
match = re.search(pattern, text, re.MULTILINE)
|
||
return int(match.group(1)) if match else None
|
||
|
||
return {
|
||
"app_usec": number(r"^usage_usec (\d+)$", app),
|
||
"mem_bytes": number(r"^(\d+)$", app),
|
||
"rss_kib": number(r"^VmRSS:\s+(\d+) kB", app),
|
||
"hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app),
|
||
"k6_usec": number(r"^usage_usec (\d+)$", k6),
|
||
"db_usec": number(r"^usage_usec (\d+)$", db),
|
||
}
|
||
|
||
def k6_args(self, script: str, env: dict[str, Any]) -> list[str]:
|
||
"""Команда `k6 run` сценария в контейнере k6 стенда."""
|
||
common = {
|
||
"TEXT_LENGTH": self.profile.text_length,
|
||
"PAGE_LIMIT": self.profile.page_limit,
|
||
}
|
||
flags = [
|
||
item
|
||
for key, value in {**common, **env}.items()
|
||
for item in ("-e", f"{key}={value}")
|
||
]
|
||
return [
|
||
"docker",
|
||
"exec",
|
||
K6,
|
||
"k6",
|
||
"run",
|
||
"--quiet",
|
||
"--no-color",
|
||
"--log-format",
|
||
"raw",
|
||
*flags,
|
||
f"/scripts/{script}",
|
||
]
|
||
|
||
def k6(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
*,
|
||
check: bool = True,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
proc = subprocess.run(
|
||
self.k6_args(script, env),
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(k6_failure(script, proc))
|
||
return proc
|
||
|
||
def k6_start(self, script: str, env: dict[str, Any]) -> subprocess.Popen:
|
||
"""Сценарий k6 в фоне: запись во время backup."""
|
||
return subprocess.Popen(
|
||
self.k6_args(script, env),
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
)
|
||
|
||
def pg_dump(self, path: Path) -> None:
|
||
"""Логический backup исходной базы (формат custom) в файл хоста."""
|
||
with path.open("wb") as file:
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
DB,
|
||
"sh",
|
||
"-c",
|
||
'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc',
|
||
],
|
||
stdout=file,
|
||
stderr=subprocess.PIPE,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"pg_dump: exit {proc.returncode}\n{proc.stderr}")
|
||
|
||
def pg_restore(self, path: Path) -> tuple[float, float]:
|
||
"""Backup в хранилище восстановления: копия файла и pg_restore -j.
|
||
|
||
Возвращает секунды копирования и восстановления.
|
||
"""
|
||
started = time.monotonic()
|
||
sh("docker", "cp", str(path), f"{RESTORE_DB}:/tmp/backup.dump")
|
||
copied = time.monotonic()
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
RESTORE_DB,
|
||
"sh",
|
||
"-c",
|
||
'exec pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
||
f"-j {self.profile.restore_jobs} --no-owner --no-privileges "
|
||
"--exit-on-error /tmp/backup.dump",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"pg_restore: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
return copied - started, time.monotonic() - copied
|
||
|
||
def switch_api(self, host: str) -> None:
|
||
"""API заново, с адресом базы `host`: `POSTGRES_HOST` перекрывает
|
||
значение из `bench/bench.env` для `x-app-env`."""
|
||
env = self.env
|
||
self.env = {**env, "POSTGRES_HOST": host}
|
||
try:
|
||
self.compose("up", "-d", "--no-deps", "--force-recreate", "api")
|
||
finally:
|
||
self.env = env
|
||
self.wait_ready()
|
||
|
||
def table_counts(self, container: str = DB) -> dict[str, int]:
|
||
"""Число строк таблиц данных (сверка восстановления)."""
|
||
sql = " UNION ALL ".join(
|
||
f"SELECT '{table}', count(*) FROM {table}" for table in DATA_TABLES
|
||
)
|
||
rows = self.psql(f"{sql};\n", container=container)
|
||
return {
|
||
table: int(count)
|
||
for table, count in (line.split("|") for line in rows.splitlines())
|
||
}
|
||
|
||
def message_rows(self, container: str = DB) -> dict[str, tuple[str, str]]:
|
||
"""Сообщения базы: id → (chat_id, md5 всех полей)."""
|
||
rows = self.psql(
|
||
"SELECT id, chat_id, md5(concat_ws('|', chat_id, sender_id, "
|
||
"text, created_at)) FROM messages;\n",
|
||
container=container,
|
||
)
|
||
return {
|
||
message: (chat, digest)
|
||
for message, chat, digest in (
|
||
line.split("|") for line in rows.splitlines()
|
||
)
|
||
}
|
||
|
||
def work_json(self, name: str) -> Any:
|
||
return json.loads((self.work_dir / name).read_text(encoding="utf-8"))
|
||
|
||
|
||
# -------------------------------------------------------------- измерение
|
||
|
||
|
||
def parse_summary(path: Path) -> dict[str, Any]:
|
||
"""Итог k6: успехи, ошибки, RPS и перцентили успешных запросов.
|
||
|
||
Нет файла (k6 остановлен сторожем до итога) — нулевой итог.
|
||
"""
|
||
if not path.exists():
|
||
return {
|
||
"duration_s": 0,
|
||
"requests_ok": 0,
|
||
"requests_failed": 0,
|
||
"check_failed": 0,
|
||
"rps": None,
|
||
"error_ratio": None,
|
||
"p50_ms": None,
|
||
"p95_ms": None,
|
||
"p99_ms": None,
|
||
"max_ms": None,
|
||
}
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
metrics = data["metrics"]
|
||
failed = metrics.get("http_req_failed", {}).get("values", {})
|
||
ok = int(failed.get("fails", 0))
|
||
bad = int(failed.get("passes", 0))
|
||
checks = metrics.get("checks", {}).get("values", {})
|
||
success = metrics.get(SUCCESS_METRIC, {}).get("values", {})
|
||
duration = data["state"]["testRunDurationMs"] / 1000
|
||
return {
|
||
"duration_s": round(duration, 3),
|
||
"requests_ok": ok,
|
||
"requests_failed": bad,
|
||
"check_failed": int(checks.get("fails", 0)),
|
||
"rps": round(ok / duration, 1) if duration else None,
|
||
"error_ratio": round(bad / (ok + bad), 6) if ok + bad else None,
|
||
"p50_ms": rounded(success.get("p(50)")),
|
||
"p95_ms": rounded(success.get("p(95)")),
|
||
"p99_ms": rounded(success.get("p(99)")),
|
||
"max_ms": rounded(success.get("max")),
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class Measured:
|
||
summary: dict[str, Any]
|
||
before: dict[str, Any]
|
||
after: dict[str, Any]
|
||
rss_max_kib: int | None
|
||
|
||
def cores(self, key: str) -> float | None:
|
||
"""Среднее число занятых ядер за прогон по счётчику cgroup."""
|
||
duration = self.summary["duration_s"]
|
||
start, end = self.before[key], self.after[key]
|
||
if not duration or start is None or end is None:
|
||
return None
|
||
return round((end - start) / 1e6 / duration, 3)
|
||
|
||
@property
|
||
def app_cpu(self) -> float | None:
|
||
return self.cores("app_usec")
|
||
|
||
@property
|
||
def generator_cpu(self) -> float | None:
|
||
return self.cores("k6_usec")
|
||
|
||
@property
|
||
def db_cpu(self) -> float | None:
|
||
return self.cores("db_usec")
|
||
|
||
|
||
def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str:
|
||
tail = (proc.stdout + proc.stderr)[-3000:]
|
||
return f"k6 {script} failed ({proc.returncode}):\n{tail}"
|
||
|
||
|
||
def measure(
|
||
stand: Stand,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
raw: Path,
|
||
) -> Measured:
|
||
"""Один запуск k6 со снимками до и после, опросом RSS и сторожем."""
|
||
before = stand.snapshot()
|
||
with Watcher() as watcher:
|
||
proc = stand.k6(script, env, check=False)
|
||
# если приложение или хранилище умерло, k6 остановлен сторожем: прогон
|
||
# пишется с пометкой, а не падает здесь
|
||
died = watcher.app_died or watcher.stand_died
|
||
if proc.returncode != 0 and not died:
|
||
raise BenchError(k6_failure(script, proc))
|
||
after = stand.snapshot()
|
||
values = [
|
||
v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v
|
||
]
|
||
return Measured(
|
||
summary=parse_summary(raw),
|
||
before=before,
|
||
after=after,
|
||
rss_max_kib=max(values) if values else None,
|
||
)
|
||
|
||
|
||
def run_flags(
|
||
profile: Profile,
|
||
measured: Measured,
|
||
app: dict[str, Any],
|
||
storages_down: list[str],
|
||
) -> list[str]:
|
||
flags = []
|
||
if app["oom"]:
|
||
flags.append("oom")
|
||
if not app["running"] or app["restarts"]:
|
||
flags.append("app_down")
|
||
if storages_down:
|
||
flags.append("stand_down")
|
||
summary = measured.summary
|
||
if summary["requests_failed"] or summary["check_failed"]:
|
||
flags.append("errors")
|
||
cpu = measured.generator_cpu
|
||
if cpu is not None and cpu >= profile.generator_cpu_limit:
|
||
flags.append("generator")
|
||
return flags
|
||
|
||
|
||
# ----------------------------------------------------------- план запроса
|
||
|
||
|
||
def plan_entries(log: str, marker: str) -> list[str]:
|
||
"""Записи auto_explain между маркерами этапа плана.
|
||
|
||
Запись начинается строкой `LOG: duration: … plan:` и продолжается
|
||
строками с табуляцией; префикс времени и pid отбрасывается.
|
||
"""
|
||
lines = log.splitlines()
|
||
bounds = [
|
||
next(
|
||
(i for i, line in enumerate(lines) if f"{marker} {edge}" in line),
|
||
None,
|
||
)
|
||
for edge in ("start", "end")
|
||
]
|
||
if None in bounds:
|
||
return []
|
||
entries: list[list[str]] = []
|
||
current: list[str] | None = None
|
||
for line in lines[bounds[0] + 1 : bounds[1]]:
|
||
if line.startswith("\t"):
|
||
if current is not None:
|
||
current.append(line[1:])
|
||
continue
|
||
current = None
|
||
head = line.partition("LOG: ")[2].rstrip()
|
||
if head.startswith("duration:") and head.endswith("plan:"):
|
||
current = [head]
|
||
entries.append(current)
|
||
return ["\n".join(entry) for entry in entries]
|
||
|
||
|
||
def is_history(entry: str) -> bool:
|
||
"""Запись — запрос страницы истории (читает таблицу messages).
|
||
|
||
Остальные записи окна (проверка сессии с хэшем токена в параметрах,
|
||
служебные запросы) в результаты не попадают.
|
||
"""
|
||
text = entry.partition("Query Parameters:")[0]
|
||
return "Query Text:" in text and "FROM messages" in text
|
||
|
||
|
||
def plan_parts(entry: str) -> dict[str, Any] | None:
|
||
"""Текст запроса, параметры и тело плана записи auto_explain."""
|
||
lines = entry.splitlines()
|
||
starts = [i for i, line in enumerate(lines) if line.startswith("Query ")]
|
||
if len(starts) != 2 or not lines[starts[1]].startswith("Query Param"):
|
||
return None
|
||
text = "\n".join(lines[starts[0] : starts[1]])
|
||
params = lines[starts[1]].partition(": ")[2]
|
||
return {
|
||
"text": text.removeprefix("Query Text: "),
|
||
"params": [
|
||
value.replace("''", "'")
|
||
for value in re.findall(r"\$\d+ = '((?:[^']|'')*)'", params)
|
||
],
|
||
"plan": "\n".join(lines[starts[1] + 1 :]),
|
||
}
|
||
|
||
|
||
def plan_summary(plan: str, time_ms: float | None) -> dict[str, Any]:
|
||
"""Узел доступа к messages, буферы верхнего узла, строки, время."""
|
||
node = re.search(
|
||
r"((?:Parallel )?(?:Seq Scan|Index Only Scan|Index Scan|"
|
||
r"Bitmap Heap Scan)(?: Backward)?(?: using \w+)? on messages)",
|
||
plan,
|
||
)
|
||
buffers = re.search(r"Buffers: ([^\n]*)", plan)
|
||
hit = re.search(r"shared hit=(\d+)", buffers.group(1)) if buffers else None
|
||
read = re.search(r"read=(\d+)", buffers.group(1)) if buffers else None
|
||
rows = re.search(r"\(actual time=[\d.]+\.\.[\d.]+ rows=([\d.]+)", plan)
|
||
if time_ms is None:
|
||
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
|
||
time_ms = float(execution.group(1)) if execution else None
|
||
return {
|
||
"node": node.group(1) if node else None,
|
||
"shared_hit": int(hit.group(1)) if hit else 0,
|
||
"shared_read": int(read.group(1)) if read else 0,
|
||
"rows": float(rows.group(1)) if rows else None,
|
||
"time_ms": time_ms,
|
||
# параметры в условиях плана — generic plan, литералы — custom
|
||
"kind": "generic" if re.search(r"\$\d", plan) else "custom",
|
||
}
|
||
|
||
|
||
def entry_duration(entry: str) -> float | None:
|
||
match = re.match(r"duration: ([\d.]+) ms", entry)
|
||
return float(match.group(1)) if match else None
|
||
|
||
|
||
def index_name(plan: str) -> str | None:
|
||
match = INDEX_NODE_RE.search(plan)
|
||
return (
|
||
next((name for name in match.groups() if name), None)
|
||
if match
|
||
else None
|
||
)
|
||
|
||
|
||
def render_explain(
|
||
size: int,
|
||
git: dict[str, Any],
|
||
profile: Profile,
|
||
pages: list[dict[str, Any]],
|
||
entries: list[str],
|
||
compared: list[tuple[dict[str, Any], dict[str, str]]],
|
||
) -> str:
|
||
"""Файл `explain-<размер>.txt`: планы приложения и варианты psql."""
|
||
tiers = ", ".join(
|
||
f"{count} × {human(chat_size)}"
|
||
for count, chat_size in profile.background
|
||
)
|
||
lines = [
|
||
f"# План запроса истории: размер {human(size)}",
|
||
"",
|
||
f"Commit {git['commit']}"
|
||
+ (" + незакоммиченные изменения" if git["dirty"] else ""),
|
||
f"Читаемый чат — {size} сообщений; фоновые чаты: {tiers or 'нет'}.",
|
||
"Страница — `limit` профиля; «после» — сколько сообщений чата до",
|
||
"страницы.",
|
||
"",
|
||
"## Планы приложения",
|
||
"",
|
||
"auto_explain (ANALYZE, BUFFERS, TIMING): запрос, параметры и план,",
|
||
"с которыми его выполнило приложение (prepared statement asyncpg).",
|
||
"",
|
||
]
|
||
for page, entry in zip(pages, entries, strict=True):
|
||
lines += [
|
||
f"### {page['chat']}: {page['page']}, после {page['after']}",
|
||
"",
|
||
entry,
|
||
"",
|
||
]
|
||
variants = ", ".join(
|
||
f"`{name}` {columns}" for name, columns in profile.index_variants
|
||
)
|
||
lines += [
|
||
"## Страницы сравнения в psql: варианты",
|
||
"",
|
||
"Тот же текст запроса и те же параметры, что у плана приложения этой",
|
||
"страницы, в одной транзакции, которая откатывается: индексы базы",
|
||
"после неё те же. Каждый вариант выполнен дважды, здесь — второе",
|
||
"выполнение. `psql` — индекс приложения, custom plan; `generic` — он",
|
||
"же с force_generic_plan; `or-cursor` — условие курсора через OR;",
|
||
"`no-index` — без индекса приложения; варианты индекса вместо него:",
|
||
f"{variants}.",
|
||
"",
|
||
]
|
||
for page, plans in compared:
|
||
lines += [
|
||
f"### {page['chat']}: {page['page']}, после {page['after']}",
|
||
"",
|
||
]
|
||
if not plans:
|
||
lines += [
|
||
"План приложения не использовал индекс: вариантов нет.",
|
||
"",
|
||
]
|
||
for name, plan in plans.items():
|
||
lines += [f"#### {name}", "", plan, ""]
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ------------------------------------------------------------ результаты
|
||
|
||
|
||
class Results:
|
||
"""Файлы метки: runs.csv, prefill.csv, meta.json, progress.json."""
|
||
|
||
def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None:
|
||
self.dir = label_dir
|
||
self.meta = meta
|
||
self.runs: list[dict[str, Any]] = []
|
||
self.prefill: list[dict[str, Any]] = []
|
||
self.sizes: list[dict[str, Any]] = []
|
||
(label_dir / "raw").mkdir(parents=True)
|
||
self.write_meta()
|
||
|
||
def append(
|
||
self,
|
||
name: str,
|
||
fields: list[str],
|
||
row: dict[str, Any],
|
||
) -> None:
|
||
path = self.dir / name
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=fields,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
writer.writerow({key: row.get(key) for key in fields})
|
||
file.flush()
|
||
|
||
def add_run(self, row: dict[str, Any]) -> None:
|
||
self.runs.append(row)
|
||
self.append("runs.csv", RUN_FIELDS, row)
|
||
|
||
def add_prefill(self, row: dict[str, Any]) -> None:
|
||
self.prefill.append(row)
|
||
self.append("prefill.csv", PREFILL_FIELDS, row)
|
||
|
||
def write_meta(self) -> None:
|
||
self.meta["sizes"] = self.sizes
|
||
text = json.dumps(self.meta, ensure_ascii=False, indent=2)
|
||
(self.dir / "meta.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_progress(self, progress: dict[str, Any]) -> None:
|
||
progress["updated_at"] = datetime.now(UTC).isoformat(
|
||
timespec="seconds",
|
||
)
|
||
text = json.dumps(progress, ensure_ascii=False, indent=2)
|
||
(self.dir / "progress.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_summary(self) -> None:
|
||
text = build_summary(self.meta, self.runs, self.prefill, self.sizes)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
|
||
|
||
def cell(values: list[float | None], digits: int = 1) -> str:
|
||
"""Медиана и min–max повторов; один повтор — просто значение."""
|
||
present = [v for v in values if v is not None]
|
||
if not present:
|
||
return "—"
|
||
median = statistics.median(present)
|
||
if len(present) == 1:
|
||
return f"{median:.{digits}f}"
|
||
return (
|
||
f"{median:.{digits}f} ({min(present):.{digits}f}–"
|
||
f"{max(present):.{digits}f})"
|
||
)
|
||
|
||
|
||
def build_summary(
|
||
meta: dict[str, Any],
|
||
runs: list[dict[str, Any]],
|
||
prefill: list[dict[str, Any]],
|
||
sizes: list[dict[str, Any]],
|
||
) -> str:
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Нагрузочный замер: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{meta['status']}**"
|
||
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
|
||
f"- Commit: `{meta['git']['commit']}`"
|
||
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
|
||
f"- Профиль `{profile['name']}`: размеры "
|
||
f"{', '.join(human(s) for s in profile['sizes'])}; клиенты "
|
||
f"{', '.join(map(str, profile['concurrency']))}; повторов "
|
||
f"{profile['repeats']}; чтение {profile['read_measure_s']} с после "
|
||
f"{profile['read_warmup_s']} с прогрева; запись "
|
||
f"{profile['write_measure']} после {profile['write_warmup']}",
|
||
f"- Лимиты без swap: API {profile['cpus']} CPU / "
|
||
f"{profile['memory']}, PostgreSQL {profile['db_cpus']} CPU / "
|
||
f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / "
|
||
f"{profile['cache_memory']}; значения — медиана (min–max) по "
|
||
"повторам",
|
||
"- Посторонние контейнеры в VM во время замера: "
|
||
+ (", ".join(meta.get("other_containers") or []) or "нет"),
|
||
"",
|
||
]
|
||
for scenario in ("read", "write"):
|
||
rows = [r for r in runs if r["scenario"] == scenario]
|
||
if not rows:
|
||
continue
|
||
lines += [
|
||
f"## Сценарий `{scenario}`",
|
||
"",
|
||
"| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | "
|
||
"max, мс | CPU API, ядер | CPU БД, ядер | RSS конец, МиБ | "
|
||
"Сообщений в начале | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
cells = sorted({(r["size"], r["concurrency"]) for r in rows})
|
||
for size, conc in cells:
|
||
group = [
|
||
r
|
||
for r in rows
|
||
if r["size"] == size and r["concurrency"] == conc
|
||
]
|
||
flags = sorted({f for r in group for f in r["flags"].split()})
|
||
lines.append(
|
||
f"| {human(size)} | {conc} | "
|
||
f"{cell([r['rps'] for r in group], 0)} | "
|
||
f"{cell([r['p50_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p95_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p99_ms'] for r in group], 2)} | "
|
||
f"{cell([r['max_ms'] for r in group], 1)} | "
|
||
f"{cell([r['app_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['db_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['rss_end_mib'] for r in group], 1)} | "
|
||
f"{cell([r['store_messages_start'] for r in group], 0)} | "
|
||
f"{' '.join(flags) or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if sizes:
|
||
lines += [
|
||
"## Память и наполнение по размерам",
|
||
"",
|
||
"Объём `messages` — после `VACUUM (ANALYZE)`, до прогонов: в "
|
||
"таблице N сообщений читаемого чата и сообщения фоновых чатов "
|
||
"(`Сообщений`).",
|
||
"",
|
||
"| Размер | Сообщений | RSS API до, МиБ | RSS API после, МиБ | "
|
||
"Байт RSS на сообщение | Таблица, МиБ | Индексы, МиБ | "
|
||
"Байт в БД на сообщение | Наполнение, сообщ/с | "
|
||
"Курсоров в пуле | Пик RSS за размер, МиБ |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
lines.append(
|
||
f"| {human(info['size'])} | "
|
||
f"{info.get('messages_total', info['size'])} | "
|
||
f"{info['rss_before_mib']} | "
|
||
f"{info['rss_after_mib']} | {info['bytes_per_message']} | "
|
||
f"{mib_bytes(info.get('messages_table_bytes'))} | "
|
||
f"{mib_bytes(info.get('messages_indexes_bytes'))} | "
|
||
f"{info.get('db_bytes_per_message', '—')} | "
|
||
f"{info['prefill_rate']} | {info['cursors']} | "
|
||
f"{info.get('rss_peak_mib', '—')} |",
|
||
)
|
||
lines.append("")
|
||
lines += [
|
||
"## Планы запроса истории",
|
||
"",
|
||
"Страница (`limit` профиля) читаемого чата и первого чата "
|
||
"каждого фонового уровня. `app` — план запроса приложения "
|
||
"(auto_explain). Остальное — тот же запрос с теми же "
|
||
"параметрами в psql, в откатываемой транзакции: `psql` — "
|
||
"индекс приложения, `generic` — он же с generic plan, "
|
||
"`or-cursor` — условие курсора через OR, `no-index` — без "
|
||
"индекса, дальше — варианты индекса профиля. Буферы — верхнего "
|
||
"узла плана, файлы — `explain-<размер>.txt`.",
|
||
"",
|
||
"| Размер | Чат | Страница | После сообщений | Вариант | Узел | "
|
||
"shared hit | read | Строк | Время, мс | План |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
for plan in info.get("plans", []):
|
||
lines.append(
|
||
f"| {human(info['size'])} | {plan.get('chat', 'read')} | "
|
||
f"{plan['page']} | {plan['after']} | {plan['source']} | "
|
||
f"{plan['node'] or '—'} | {plan['shared_hit']} | "
|
||
f"{plan['shared_read']} | {fmt(plan['rows'])} | "
|
||
f"{fmt(plan['time_ms'], 3)} | {plan['kind']} |",
|
||
)
|
||
lines.append("")
|
||
costs = [
|
||
(info, row) for info in sizes for row in info.get("write_cost", [])
|
||
]
|
||
if costs:
|
||
lines += [
|
||
"## Стоимость записи по вариантам индекса",
|
||
"",
|
||
"Вставка порции сообщений во все чаты в пропорции наполнения "
|
||
"одной командой, в транзакции, которая откатывается; индекс "
|
||
"истории построен заново, перед вставкой `CHECKPOINT`. "
|
||
"Значения — медиана (min–max) повторов. «4000 символов» — "
|
||
"вставка сообщения максимальной длины из несжимаемого текста.",
|
||
"",
|
||
"| Размер | Вариант | Строк вставки | Индекс, МиБ | "
|
||
"Байт индекса на сообщение | мкс на сообщение | "
|
||
"WAL байт на сообщение | FPI на 1000 | 4000 символов |",
|
||
"|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info, row in costs:
|
||
lines.append(
|
||
f"| {human(info['size'])} | {row['variant']} | "
|
||
f"{row['rows']} | {mib_bytes(row['index_bytes'])} | "
|
||
f"{row['index_bytes_per_message']} | "
|
||
f"{cell(row['insert_us_per_message'], 2)} | "
|
||
f"{cell(row['wal_bytes_per_message'], 0)} | "
|
||
f"{cell(row['fpi_per_1000'], 1)} | {row['long_text']} |",
|
||
)
|
||
lines.append("")
|
||
restores = [info for info in sizes if info.get("restore")]
|
||
if restores:
|
||
lines += [
|
||
"## Backup и восстановление",
|
||
"",
|
||
"`pg_dump -Fc` исходной базы во время записи через HTTP, "
|
||
"затем активность после backup (пользователь, сессия, чат, "
|
||
"сообщение), `pg_restore -j` в новый контейнер PostgreSQL "
|
||
"на новом volume, `ANALYZE`, сверка обеих баз (число строк, "
|
||
"md5 каждого сообщения, индексы, ревизия схемы), "
|
||
"переключение API и проверки через HTTP до и после "
|
||
"`FLUSHALL`. RTO — от запуска нового хранилища до "
|
||
"готовности API, без сверки. Сверка прошла на каждом "
|
||
"размере: иначе команда завершилась бы ошибкой.",
|
||
"",
|
||
"| Размер | До backup | Исходная в конце | Восстановлено | "
|
||
"Потеряно | Записей во время backup (в backup) | Backup, МиБ | "
|
||
"Байт backup на сообщение | dump, с | restore, с | "
|
||
"ANALYZE, с | API, с | RTO, с | Сверка, с | Чат после "
|
||
"backup: GET / POST до FLUSHALL, GET после | Сессия после "
|
||
"backup |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"
|
||
"---|---|",
|
||
]
|
||
for info in restores:
|
||
r = info["restore"]
|
||
lines.append(
|
||
f"| {human(info['size'])} | {r['messages_before_backup']} "
|
||
f"| {r['messages_source_end']} | {r['messages_restored']} "
|
||
f"| {r['messages_lost']} | {r['load_written']} "
|
||
f"({r['load_in_backup']}) | "
|
||
f"{mib_bytes(r['backup_bytes'])} | "
|
||
f"{r['backup_bytes_per_message']} | {r['dump_seconds']} | "
|
||
f"{r['restore_seconds']} | {r['analyze_seconds']} | "
|
||
f"{r['api_ready_seconds']} | {r['rto_seconds']} | "
|
||
f"{r['verify_seconds']} | {r['stale_chat_get']} / "
|
||
f"{r['stale_chat_post']}, {r['flushed_chat_get']} | "
|
||
f"{r['late_session']} |",
|
||
)
|
||
lines.append("")
|
||
if prefill:
|
||
largest = max(r["size"] for r in prefill)
|
||
curve = [r for r in prefill if r["size"] == largest]
|
||
lines += [
|
||
f"## Кривая наполнения {human(largest)}: запись при росте "
|
||
f"хранилища, {profile['prefill_vus']} клиента",
|
||
"",
|
||
"| Порция | Сообщений после | RPS | p50, мс | p95, мс | "
|
||
"p99, мс | max, мс | CPU API, ядер | CPU БД, ядер | RSS, МиБ | "
|
||
"Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in curve:
|
||
lines.append(
|
||
f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | "
|
||
f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | "
|
||
f"{r['max_ms']} | {r['app_cpu_cores']} | "
|
||
f"{r['db_cpu_cores']} | {r['rss_end_mib']} "
|
||
f"| {r['flags'] or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if meta.get("host_warnings"):
|
||
lines += ["## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ---------------------------------------------------------------- прогон
|
||
|
||
|
||
class Bench:
|
||
def __init__(
|
||
self,
|
||
profile: Profile,
|
||
label: str,
|
||
results: Results,
|
||
) -> None:
|
||
self.profile = profile
|
||
self.label = label
|
||
self.results = results
|
||
self.stand = Stand(profile, results.dir)
|
||
self.started = time.monotonic()
|
||
self.runs_total = (
|
||
len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats
|
||
)
|
||
self.runs_done = 0
|
||
self.stage = "start"
|
||
self.size: int | None = None
|
||
# скорости стенда для оценки оставшегося времени
|
||
self.prefill_rate = 1500.0
|
||
self.write_rps: dict[int, float] = {}
|
||
self.run_overhead = 3.0
|
||
self.done_sizes = 0
|
||
self.prefilled = 0
|
||
# текущий размер наполнен и курсоры собраны
|
||
self.prepared = False
|
||
# индекс плана приложения: от него варианты и пробы записи
|
||
self.app_index: str | None = None
|
||
|
||
# ------------------------------------------------------------ вывод
|
||
|
||
def say(self, text: str) -> None:
|
||
elapsed = int(time.monotonic() - self.started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
def eta(self) -> float:
|
||
p = self.profile
|
||
left = 0.0
|
||
for index, size in enumerate(p.sizes):
|
||
current = index == self.done_sizes
|
||
if index < self.done_sizes:
|
||
continue
|
||
# пробы записи в конце размера: сборка индексов и вставки;
|
||
# backup, восстановление и сверка
|
||
left += 60 + (30 if p.restore else 0)
|
||
if current and self.prepared:
|
||
continue
|
||
filled = self.prefilled if current else 0
|
||
total = size + p.background_total
|
||
left += (total - filled) / self.prefill_rate + size / 100 / 500
|
||
# стенд, миграции, обслуживание БД, план и его варианты
|
||
left += 60
|
||
per_size = len(p.concurrency) * p.repeats
|
||
reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes)
|
||
done_in_size = self.runs_done - self.done_sizes * per_size * 2
|
||
reads_left -= min(done_in_size, per_size)
|
||
writes_left -= max(0, done_in_size - per_size)
|
||
read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead
|
||
left += reads_left * read_time
|
||
rps = (
|
||
statistics.mean(self.write_rps.values())
|
||
if self.write_rps
|
||
else 1000
|
||
)
|
||
write_time = (p.write_warmup + p.write_measure) / rps
|
||
left += writes_left * (write_time + self.run_overhead)
|
||
return left
|
||
|
||
def progress(self, last: dict[str, Any] | None = None) -> None:
|
||
state = {
|
||
"label": self.label,
|
||
"profile": self.profile.name,
|
||
"stage": self.stage,
|
||
"size": self.size,
|
||
"runs_done": self.runs_done,
|
||
"runs_total": self.runs_total,
|
||
"elapsed_s": round(time.monotonic() - self.started),
|
||
"eta_s": round(self.eta()),
|
||
"warnings": self.results.meta.get("host_warnings", []),
|
||
}
|
||
if last is not None:
|
||
state["last_run"] = last
|
||
self.results.write_progress(state)
|
||
|
||
# --------------------------------------------------------- этапы
|
||
|
||
def run(self) -> None:
|
||
for index, size in enumerate(self.profile.sizes, start=1):
|
||
self.size = size
|
||
self.prefilled = 0
|
||
self.prepared = False
|
||
# индекс плана приложения ищется заново на каждом размере
|
||
self.app_index = None
|
||
self.run_size(index, size)
|
||
self.done_sizes += 1
|
||
|
||
def run_size(self, index: int, size: int) -> None:
|
||
stand = self.stand
|
||
name = human(size)
|
||
self.stage = f"size {name}: start"
|
||
self.say(f"size {name} ({index}/{len(self.profile.sizes)}): start")
|
||
stand.start(lambda text: self.say(f"size {name}: {text}"))
|
||
limits = stand.limits()
|
||
self.results.meta.setdefault("limits", limits)
|
||
if limits != self.results.meta["limits"]:
|
||
raise BenchError(f"limits changed between sizes: {limits}")
|
||
if "stand" not in self.results.meta:
|
||
self.results.meta["stand"] = stand.passport()
|
||
self.results.write_meta()
|
||
rss_start = stand.snapshot()["rss_kib"]
|
||
self.say(f"size {name}: app started, RSS {mib(rss_start)} MiB")
|
||
stand.k6(
|
||
"prepare.js",
|
||
{
|
||
"MEMBERS": self.profile.members,
|
||
"BACKGROUND": ",".join(
|
||
f"{count}x{chat_size}"
|
||
for count, chat_size in self.profile.background
|
||
),
|
||
},
|
||
)
|
||
fixture = stand.work_json("fixture.json")
|
||
sizes = chat_sizes(self.profile, size)
|
||
if [size] + [c["size"] for c in fixture["bgChats"]] != sizes:
|
||
raise BenchError(
|
||
f"size {name}: background chats differ from profile",
|
||
)
|
||
total = sum(sizes)
|
||
(stand.work_dir / "schedule.json").write_text(
|
||
json.dumps(fill_schedule(sizes)),
|
||
encoding="utf-8",
|
||
)
|
||
self.say(
|
||
f"size {name}: {len(fixture['bgChats'])} background chats, "
|
||
f"{total} messages to prefill",
|
||
)
|
||
self.progress()
|
||
rss_before = stand.snapshot()["rss_kib"]
|
||
prefill_seconds = self.run_prefill(size, total)
|
||
after = stand.snapshot()
|
||
rss_after = after["rss_kib"]
|
||
self.stage = f"size {name}: maintenance"
|
||
self.progress()
|
||
seconds = stand.maintain()
|
||
storage = stand.storage()
|
||
if storage["messages_rows"] != total:
|
||
raise BenchError(
|
||
f"messages table has {storage['messages_rows']} rows after "
|
||
f"prefill, expected {total}",
|
||
)
|
||
self.check_distribution(size, fixture)
|
||
self.say(
|
||
f"size {name}: VACUUM (ANALYZE) and CHECKPOINT in {seconds:.1f}s; "
|
||
f"messages {mib_bytes(storage['messages_table_bytes'])} MiB + "
|
||
f"indexes {mib_bytes(storage['messages_indexes_bytes'])} MiB, "
|
||
f"{storage['db_bytes_per_message']} B/msg",
|
||
)
|
||
self.stage = f"size {name}: cursors"
|
||
self.progress()
|
||
stand.k6(
|
||
"cursors.js",
|
||
{
|
||
"WALK_LIMIT": self.profile.walk_limit,
|
||
"POOL_SIZE": self.profile.cursor_pool,
|
||
},
|
||
)
|
||
pool = stand.work_json("cursors.json")
|
||
if pool["walked"] != size:
|
||
raise BenchError(
|
||
f"read chat has {pool['walked']} messages, expected {size}",
|
||
)
|
||
self.say(f"size {name}: cursors {len(pool['cursors'])}")
|
||
self.stage = f"size {name}: query plan"
|
||
self.progress()
|
||
plans = self.explain(size, pool, fixture)
|
||
info = {
|
||
"size": size,
|
||
"messages_total": total,
|
||
"rss_before_mib": mib(rss_before),
|
||
"rss_after_mib": mib(rss_after),
|
||
"bytes_per_message": round(
|
||
(rss_after - rss_before) * 1024 / total,
|
||
),
|
||
"prefill_seconds": round(prefill_seconds, 1),
|
||
"prefill_rate": round(total / prefill_seconds),
|
||
"cursors": len(pool["cursors"]),
|
||
"maintenance_seconds": round(seconds, 1),
|
||
**storage,
|
||
"plans": plans,
|
||
}
|
||
self.results.sizes.append(info)
|
||
self.results.write_meta()
|
||
self.prepared = True
|
||
self.say(
|
||
f"size {name}: prefill done, app RSS {info['rss_after_mib']} "
|
||
f"MiB (+{info['bytes_per_message']} B/msg)",
|
||
)
|
||
store = total
|
||
for scenario in ("read", "write"):
|
||
for conc in self.profile.concurrency:
|
||
for repeat in range(1, self.profile.repeats + 1):
|
||
store = self.run_one(scenario, size, conc, repeat, store)
|
||
if scenario == "read":
|
||
self.check_read_chat(size)
|
||
info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"])
|
||
self.stage = f"size {name}: write cost"
|
||
self.progress()
|
||
info["write_cost"] = self.write_cost(size)
|
||
self.results.write_meta()
|
||
if self.profile.restore:
|
||
self.stage = f"size {name}: backup and restore"
|
||
self.progress()
|
||
info["restore"] = self.backup_restore(size, fixture, store)
|
||
self.results.write_meta()
|
||
if not stand.stop():
|
||
raise BenchError(f"stand of size {name} was not removed")
|
||
|
||
def check_read_chat(self, size: int) -> None:
|
||
"""После блока чтения в читаемом чате по-прежнему ровно N.
|
||
|
||
Новое сообщение встало бы в конец истории, поэтому достаточно
|
||
дочитать хвост от последнего курсора пула.
|
||
"""
|
||
self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit})
|
||
tail = self.stand.work_json("tail.json")
|
||
total = tail["from_after"] + tail["tail"]
|
||
if total != size:
|
||
raise BenchError(
|
||
f"read chat has {total} messages after the read block, "
|
||
f"expected {size}",
|
||
)
|
||
self.say(f"size {human(size)}: read chat unchanged, {total} messages")
|
||
|
||
def check_distribution(self, size: int, fixture: dict[str, Any]) -> None:
|
||
"""После наполнения у каждого чата ровно его размер по профилю."""
|
||
expected = {
|
||
fixture["readChatId"]: size,
|
||
fixture["writeChatId"]: 0,
|
||
**{chat["id"]: chat["size"] for chat in fixture["bgChats"]},
|
||
}
|
||
counts = self.stand.chat_counts()
|
||
wrong = [
|
||
f"{counts.get(chat, 0)} instead of {want}"
|
||
for chat, want in expected.items()
|
||
if counts.get(chat, 0) != want
|
||
]
|
||
wrong += [
|
||
f"unknown chat with {counts[c]}"
|
||
for c in counts.keys() - expected.keys()
|
||
]
|
||
if wrong:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages per chat differ from the "
|
||
f"profile in {len(wrong)} chats: {', '.join(wrong[:5])}",
|
||
)
|
||
sizes = sorted(expected.values())
|
||
self.say(
|
||
f"size {human(size)}: {len(expected) - 1} chats with "
|
||
f"{sizes[1]}–{sizes[-1]} messages, distribution matches profile",
|
||
)
|
||
|
||
def plan_pages(
|
||
self,
|
||
size: int,
|
||
pool: dict[str, Any],
|
||
fixture: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Страницы этапа плана: читаемый чат и первый чат каждого уровня.
|
||
|
||
Читаемый — из пула курсоров, фоновые — проходом `plan_cursors.js`
|
||
до включения auto_explain. Первая, средняя и последняя полная
|
||
страница; у чата без полных страниц после первой — только первая.
|
||
Страница сравнения — средняя, иначе первая (решение 3 design
|
||
change `track2-query-plans`).
|
||
"""
|
||
p = self.profile
|
||
representatives = []
|
||
offset = 0
|
||
for count, _ in p.background:
|
||
representatives.append(fixture["bgChats"][offset])
|
||
offset += count
|
||
walks: dict[str, Any] = {}
|
||
if representatives:
|
||
(self.stand.work_dir / "plan_chats.json").write_text(
|
||
json.dumps(representatives),
|
||
encoding="utf-8",
|
||
)
|
||
self.stand.k6("plan_cursors.js", {"WALK_LIMIT": p.walk_limit})
|
||
walks = {
|
||
walk["id"]: walk
|
||
for walk in self.stand.work_json("plan_walks.json")
|
||
}
|
||
chats = [
|
||
(
|
||
f"read {human(size)}",
|
||
fixture["readChatId"],
|
||
size,
|
||
[
|
||
{"after": after, "cursor": cursor}
|
||
for after, cursor in zip(
|
||
pool["afters"],
|
||
pool["cursors"],
|
||
strict=True,
|
||
)
|
||
],
|
||
),
|
||
]
|
||
for chat in representatives:
|
||
walk = walks[chat["id"]]
|
||
if walk["walked"] != chat["size"]:
|
||
raise BenchError(
|
||
f"size {human(size)}: background chat has "
|
||
f"{walk['walked']} messages, expected {chat['size']}",
|
||
)
|
||
chats.append(
|
||
(
|
||
f"bg {human(chat['size'])}",
|
||
chat["id"],
|
||
chat["size"],
|
||
[
|
||
position
|
||
for position in walk["positions"]
|
||
if chat["size"] - position["after"] >= p.page_limit
|
||
],
|
||
),
|
||
)
|
||
pages = []
|
||
for label, chat_id, chat_size, positions in chats:
|
||
common = {
|
||
"chat": label,
|
||
"chat_id": chat_id,
|
||
"chat_size": chat_size,
|
||
}
|
||
pages.append(
|
||
{
|
||
**common,
|
||
"page": "first",
|
||
"after": 0,
|
||
"cursor": None,
|
||
"items": min(p.page_limit, chat_size),
|
||
"compare": not positions,
|
||
},
|
||
)
|
||
if not positions:
|
||
continue
|
||
for page, position in (
|
||
("middle", positions[len(positions) // 2]),
|
||
("last", positions[-1]),
|
||
):
|
||
pages.append(
|
||
{
|
||
**common,
|
||
"page": page,
|
||
"after": position["after"],
|
||
"cursor": position["cursor"],
|
||
"items": p.page_limit,
|
||
"compare": page == "middle",
|
||
},
|
||
)
|
||
return pages
|
||
|
||
def explain(
|
||
self,
|
||
size: int,
|
||
pool: dict[str, Any],
|
||
fixture: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Этап плана: EXPLAIN (ANALYZE, BUFFERS) запросов истории приложения.
|
||
|
||
auto_explain включается между маркерами, `explain.js` читает
|
||
страницы из `plan_pages.json`; планы берутся из журнала Postgres.
|
||
Затем страница сравнения каждого чата в psql: варианты индекса и
|
||
формы запроса в откатываемой транзакции. Решение 5 design change
|
||
`bench-postgres`, решения 3 и 4 design change `track2-query-plans`.
|
||
"""
|
||
stand = self.stand
|
||
pages = self.plan_pages(size, pool, fixture)
|
||
(stand.work_dir / "plan_pages.json").write_text(
|
||
json.dumps(pages),
|
||
encoding="utf-8",
|
||
)
|
||
indexes = stand.indexes()
|
||
marker = f"bench-explain {uuid.uuid4().hex}"
|
||
enable = "".join(
|
||
f"ALTER SYSTEM SET {key} = '{value}';\n"
|
||
for key, value in EXPLAIN_SETTINGS.items()
|
||
)
|
||
disable = "".join(
|
||
f"ALTER SYSTEM RESET {key};\n" for key in EXPLAIN_SETTINGS
|
||
)
|
||
stand.psql(
|
||
f"DO $$ BEGIN RAISE LOG '{marker} start'; END $$;\n"
|
||
f"{enable}SELECT pg_reload_conf();\n",
|
||
)
|
||
time.sleep(RELOAD_PAUSE_S)
|
||
try:
|
||
stand.k6("explain.js", {})
|
||
finally:
|
||
stand.psql(
|
||
f"{disable}SELECT pg_reload_conf();\n"
|
||
f"DO $$ BEGIN RAISE LOG '{marker} end'; END $$;\n",
|
||
)
|
||
# соединения пула API должны получить выключение до прогонов
|
||
time.sleep(RELOAD_PAUSE_S)
|
||
entries = [
|
||
entry
|
||
for entry in plan_entries(stand.db_log(), marker)
|
||
if is_history(entry)
|
||
]
|
||
if len(entries) != len(pages):
|
||
raise BenchError(
|
||
f"size {human(size)}: expected {len(pages)} history "
|
||
f"query plans in the PostgreSQL log, found {len(entries)}",
|
||
)
|
||
plans = []
|
||
compared = []
|
||
for page, entry in zip(pages, entries, strict=True):
|
||
parts = plan_parts(entry) or {"plan": entry}
|
||
where = {
|
||
"chat": page["chat"],
|
||
"chat_size": page["chat_size"],
|
||
"page": page["page"],
|
||
"after": page["after"],
|
||
}
|
||
plans.append(
|
||
{
|
||
**where,
|
||
"source": "app",
|
||
**plan_summary(parts["plan"], entry_duration(entry)),
|
||
},
|
||
)
|
||
if self.app_index is None:
|
||
self.app_index = index_name(parts["plan"])
|
||
if not page["compare"]:
|
||
continue
|
||
variants = self.explain_variants(entry, page["cursor"] is not None)
|
||
compared.append((page, variants))
|
||
for variant, plan in variants.items():
|
||
summary = plan_summary(plan, None)
|
||
plans.append({**where, "source": variant, **summary})
|
||
self.say(
|
||
f"size {human(size)}: {page['chat']} {page['page']} "
|
||
f"{variant}: {summary['node']}, "
|
||
f"hit={summary['shared_hit']} read={summary['shared_read']}"
|
||
f", {summary['time_ms']} ms",
|
||
)
|
||
if stand.indexes() != indexes:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages indexes changed by the plan "
|
||
"stage",
|
||
)
|
||
text = render_explain(
|
||
size,
|
||
self.results.meta["git"],
|
||
self.profile,
|
||
pages,
|
||
entries,
|
||
compared,
|
||
)
|
||
path = self.results.dir / f"explain-{human(size)}.txt"
|
||
path.write_text(text, encoding="utf-8")
|
||
return plans
|
||
|
||
def explain_variants(self, entry: str, has_cursor: bool) -> dict[str, str]:
|
||
"""Тот же запрос и параметры в psql: варианты индекса и запроса.
|
||
|
||
Всё в транзакции, которая откатывается; нагрузки в этот момент нет.
|
||
Каждый вариант выполняется дважды, в результат идёт второе
|
||
выполнение: только что построенный индекс ещё не в shared_buffers.
|
||
План приложения без индекса — вариантов нет (решение 4 design
|
||
change `track2-query-plans`).
|
||
"""
|
||
parts = plan_parts(entry)
|
||
index = index_name(parts["plan"]) if parts else None
|
||
if parts is None or index is None:
|
||
return {}
|
||
args = ", ".join(
|
||
"'" + value.replace("'", "''") + "'" for value in parts["params"]
|
||
)
|
||
|
||
def twice(name: str, statement: str) -> str:
|
||
run = f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE {statement}({args});\n"
|
||
return f"\\echo @@warmup\n{run}\\echo @@{name}\n{run}"
|
||
|
||
script = (
|
||
"SET lock_timeout = '5s';\n"
|
||
"SET plan_cache_mode = force_custom_plan;\n"
|
||
"BEGIN;\n"
|
||
f"PREPARE page AS {parts['text']};\n"
|
||
+ twice("psql", "page")
|
||
+ "SET LOCAL plan_cache_mode = force_generic_plan;\n"
|
||
+ twice("generic", "page")
|
||
+ "SET LOCAL plan_cache_mode = force_custom_plan;\n"
|
||
)
|
||
if has_cursor:
|
||
script += f"PREPARE page_or AS {or_cursor(parts['text'])};\n"
|
||
script += twice("or-cursor", "page_or")
|
||
script += f'DROP INDEX "{index}";\n' + twice("no-index", "page")
|
||
for name, columns in self.profile.index_variants:
|
||
script += (
|
||
f"CREATE INDEX bench_variant ON messages {columns};\n"
|
||
+ twice(name, "page")
|
||
+ "DROP INDEX bench_variant;\n"
|
||
)
|
||
script += "ROLLBACK;\n"
|
||
sections: dict[str, list[str]] = {}
|
||
current: list[str] = []
|
||
for line in self.stand.psql(script).splitlines():
|
||
if line.startswith("@@"):
|
||
current = sections.setdefault(line[2:], [])
|
||
current.clear()
|
||
continue
|
||
current.append(line)
|
||
sections.pop("warmup", None)
|
||
return {
|
||
name: "\n".join(lines).strip() for name, lines in sections.items()
|
||
}
|
||
|
||
def write_cost(self, size: int) -> list[dict[str, Any]]:
|
||
"""Стоимость записи по вариантам индекса истории.
|
||
|
||
Конфигурации `app` (индекс приложения, построенный заново), варианты
|
||
профиля и `none`; повторы со сдвигом порядка, `VACUUM` после каждой
|
||
пробы. Медиана и разброс — в сводке (решение 5 design change
|
||
`track2-query-plans`).
|
||
"""
|
||
stand = self.stand
|
||
p = self.profile
|
||
if self.app_index is None:
|
||
raise BenchError(
|
||
f"size {human(size)}: the app history plan used no index",
|
||
)
|
||
definition = stand.psql(
|
||
f"SELECT pg_get_indexdef('{self.app_index}'::regclass);\n",
|
||
)
|
||
columns = definition.partition(" USING btree ")[2]
|
||
if not columns:
|
||
raise BenchError(f"unexpected index definition: {definition}")
|
||
configs: list[tuple[str, str | None]] = [
|
||
("app", columns),
|
||
*p.index_variants,
|
||
("none", None),
|
||
]
|
||
before = (stand.indexes(), stand.message_count())
|
||
samples: dict[str, list[dict[str, Any]]] = {n: [] for n, _ in configs}
|
||
for repeat in range(p.write_probe_repeats):
|
||
shift = repeat % len(configs)
|
||
for name, config in configs[shift:] + configs[:shift]:
|
||
samples[name].append(stand.write_probe(self.app_index, config))
|
||
stand.psql("VACUUM messages;\n")
|
||
if (stand.indexes(), stand.message_count()) != before:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages indexes or rows changed by the "
|
||
"write probes",
|
||
)
|
||
result = []
|
||
for name, config in configs:
|
||
runs = samples[name]
|
||
index_bytes = int(
|
||
statistics.median(r["index_bytes"] for r in runs),
|
||
)
|
||
row = {
|
||
"variant": name,
|
||
"columns": config,
|
||
"rows": runs[0]["rows"],
|
||
"messages": runs[0]["rows_before"],
|
||
"index_bytes": index_bytes,
|
||
"index_bytes_per_message": round(
|
||
index_bytes / runs[0]["rows_before"],
|
||
1,
|
||
),
|
||
"insert_us_per_message": [
|
||
round(r["time_ms"] * 1000 / r["rows"], 2) for r in runs
|
||
],
|
||
"wal_bytes_per_message": [
|
||
round(r["wal_bytes"] / r["rows"], 1) for r in runs
|
||
],
|
||
"fpi_per_1000": [
|
||
round(r["wal_fpi"] * 1000 / r["rows"], 1) for r in runs
|
||
],
|
||
"long_text": runs[0]["long_text"],
|
||
}
|
||
result.append(row)
|
||
self.say(
|
||
f"size {human(size)}: write cost {name}: "
|
||
f"{cell(row['insert_us_per_message'], 2)} us/msg, "
|
||
f"{cell(row['wal_bytes_per_message'], 0)} WAL B/msg, index "
|
||
f"{row['index_bytes_per_message']} B/msg, 4000 chars: "
|
||
f"{row['long_text']}",
|
||
)
|
||
return result
|
||
|
||
def backup_restore(
|
||
self,
|
||
size: int,
|
||
fixture: dict[str, Any],
|
||
store: int,
|
||
) -> dict[str, Any]:
|
||
"""Backup под записью, восстановление в новое хранилище, сверка.
|
||
|
||
Порядок и проверки — решения 2–5 design change
|
||
`track2-backup-restore`. Расхождение — BenchError с размером;
|
||
хранилище восстановления убирает `Stand.stop()`.
|
||
"""
|
||
stand = self.stand
|
||
p = self.profile
|
||
name = human(size)
|
||
rows_before = stand.message_rows()
|
||
counts_before = stand.table_counts()
|
||
# запись через HTTP во время backup: dump начинается, когда она идёт
|
||
load = stand.k6_start(
|
||
"write.js",
|
||
{
|
||
"VUS": p.restore_write_vus,
|
||
"COUNT": p.restore_write_load,
|
||
"OFFSET": store,
|
||
},
|
||
)
|
||
dump = stand.work_dir / "backup.dump"
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while stand.message_count() <= counts_before["messages"]:
|
||
if load.poll() is not None or time.monotonic() > deadline:
|
||
raise BenchError(
|
||
f"size {name}: write load did not start before backup",
|
||
)
|
||
time.sleep(0.05)
|
||
started = time.monotonic()
|
||
stand.pg_dump(dump)
|
||
dump_seconds = time.monotonic() - started
|
||
at_dump_end = stand.message_count()
|
||
finally:
|
||
out, err = load.communicate()
|
||
if load.returncode != 0:
|
||
raise BenchError(
|
||
k6_failure(
|
||
"write.js",
|
||
subprocess.CompletedProcess([], load.returncode, out, err),
|
||
),
|
||
)
|
||
written = parse_summary(stand.work_dir / "last.json")
|
||
# backup не должен мешать записи: k6 выходит с 0 и при ошибках
|
||
if (
|
||
written["requests_failed"]
|
||
or written["requests_ok"] != p.restore_write_load
|
||
):
|
||
raise BenchError(
|
||
f"size {name}: writes during backup: "
|
||
f"{written['requests_ok']} of {p.restore_write_load} ok, "
|
||
f"{written['requests_failed']} failed",
|
||
)
|
||
dump_bytes = dump.stat().st_size
|
||
counts_loaded = stand.table_counts()
|
||
self.say(
|
||
f"size {name}: backup {mib_bytes(dump_bytes)} MiB in "
|
||
f"{dump_seconds:.2f}s during {written['requests_ok']} writes "
|
||
f"({written['requests_failed']} failed)",
|
||
)
|
||
stand.k6("restore_check.js", {"PHASE": "source"})
|
||
source_check = stand.work_json("check-source.json")
|
||
stand.k6("after_backup.js", {})
|
||
after = stand.work_json("after.json")
|
||
rows_source = stand.message_rows()
|
||
|
||
# восстановление: RTO — от запуска нового хранилища до готовности API
|
||
started = time.monotonic()
|
||
stand.start_restore()
|
||
storage_seconds = time.monotonic() - started
|
||
copy_seconds, restore_seconds = stand.pg_restore(dump)
|
||
mark = time.monotonic()
|
||
stand.psql("ANALYZE;\n", container=RESTORE_DB)
|
||
analyze_seconds = time.monotonic() - mark
|
||
db_seconds = time.monotonic() - started
|
||
|
||
mark = time.monotonic()
|
||
rows_restored = stand.message_rows(container=RESTORE_DB)
|
||
counts_restored = stand.table_counts(container=RESTORE_DB)
|
||
problems = []
|
||
mismatched = sum(
|
||
1
|
||
for message, row in rows_restored.items()
|
||
if rows_source.get(message) != row
|
||
)
|
||
if mismatched:
|
||
problems.append(f"{mismatched} restored messages differ")
|
||
missing_before = len(rows_before.keys() - rows_restored.keys())
|
||
if missing_before:
|
||
problems.append(
|
||
f"{missing_before} messages from before backup lost",
|
||
)
|
||
lost = rows_source.keys() - rows_restored.keys()
|
||
allowed = {fixture["writeChatId"], after["chatId"]}
|
||
foreign = sum(1 for m in lost if rows_source[m][0] not in allowed)
|
||
if foreign:
|
||
problems.append(f"{foreign} lost messages outside written chats")
|
||
for table in DATA_TABLES[:-1]:
|
||
if counts_restored[table] != counts_loaded[table]:
|
||
problems.append(
|
||
f"{table}: {counts_restored[table]} restored, "
|
||
f"{counts_loaded[table]} at backup",
|
||
)
|
||
if stand.indexes(container=RESTORE_DB) != stand.indexes():
|
||
problems.append("messages indexes differ")
|
||
version = "SELECT version_num FROM alembic_version;\n"
|
||
if stand.psql(version, container=RESTORE_DB) != stand.psql(version):
|
||
problems.append("alembic revision differs")
|
||
verify_seconds = time.monotonic() - mark
|
||
|
||
mark = time.monotonic()
|
||
stand.switch_api(RESTORE_HOST)
|
||
api_seconds = time.monotonic() - mark
|
||
stand.k6("restore_check.js", {"PHASE": "stale"})
|
||
stale = stand.work_json("check-stale.json")
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" FLUSHALL',
|
||
)
|
||
stand.k6("restore_check.js", {"PHASE": "restored"})
|
||
restored = stand.work_json("check-restored.json")
|
||
for phase, check in (
|
||
("source", source_check),
|
||
("stale", stale),
|
||
("restored", restored),
|
||
):
|
||
if check["reader_me"] != 200:
|
||
problems.append(
|
||
f"{phase}: reader session {check['reader_me']}",
|
||
)
|
||
if stale["late_me"] != 401:
|
||
problems.append(f"session after backup: {stale['late_me']}")
|
||
if restored["after_chat_get"] != 404:
|
||
problems.append(
|
||
f"chat after backup after FLUSHALL: {restored['after_chat_get']}",
|
||
)
|
||
for phase, check in (("stale", stale), ("restored", restored)):
|
||
differ = [
|
||
f"{a['chat']} {a['page']}"
|
||
for a, b in zip(
|
||
check["pages"],
|
||
source_check["pages"],
|
||
strict=True,
|
||
)
|
||
if a["status"] != 200 or a["items"] != b["items"]
|
||
]
|
||
if differ:
|
||
problems.append(f"{phase}: pages differ: {', '.join(differ)}")
|
||
if problems:
|
||
raise BenchError(
|
||
f"size {name}: restore check failed: {'; '.join(problems)}",
|
||
)
|
||
result = {
|
||
"messages_before_backup": counts_before["messages"],
|
||
"messages_at_dump_end": at_dump_end,
|
||
"messages_source_end": len(rows_source),
|
||
"messages_restored": len(rows_restored),
|
||
"messages_lost": len(lost),
|
||
"lost_after_backup_chat": sum(
|
||
1 for m in lost if rows_source[m][0] == after["chatId"]
|
||
),
|
||
"load_written": written["requests_ok"],
|
||
"load_failed": written["requests_failed"],
|
||
"load_in_backup": len(rows_restored) - counts_before["messages"],
|
||
"tables_restored": counts_restored,
|
||
"backup_bytes": dump_bytes,
|
||
"backup_bytes_per_message": round(
|
||
dump_bytes / len(rows_restored),
|
||
1,
|
||
),
|
||
"dump_seconds": round(dump_seconds, 2),
|
||
"storage_seconds": round(storage_seconds, 2),
|
||
"copy_seconds": round(copy_seconds, 2),
|
||
"restore_seconds": round(restore_seconds, 2),
|
||
"analyze_seconds": round(analyze_seconds, 2),
|
||
"verify_seconds": round(verify_seconds, 2),
|
||
"api_ready_seconds": round(api_seconds, 2),
|
||
"rto_seconds": round(db_seconds + api_seconds, 2),
|
||
"pages_checked": len(source_check["pages"]),
|
||
"stale_chat_get": stale["after_chat_get"],
|
||
"stale_chat_post": stale["after_chat_post"],
|
||
"flushed_chat_get": restored["after_chat_get"],
|
||
"late_session": stale["late_me"],
|
||
}
|
||
self.say(
|
||
f"size {name}: restored {len(rows_restored)} of "
|
||
f"{len(rows_source)} messages ({len(lost)} lost after the "
|
||
f"snapshot), restore {restore_seconds:.2f}s, ANALYZE "
|
||
f"{analyze_seconds:.2f}s, API {api_seconds:.2f}s, RTO "
|
||
f"{result['rto_seconds']}s; stale chat GET "
|
||
f"{result['stale_chat_get']} POST {result['stale_chat_post']}, "
|
||
f"after FLUSHALL {result['flushed_chat_get']}",
|
||
)
|
||
return result
|
||
|
||
def run_prefill(self, size: int, total: int) -> float:
|
||
"""Наполнение всех чатов размера по расписанию, порциями."""
|
||
chunk_size = self.profile.prefill_chunk
|
||
total_seconds = 0.0
|
||
for chunk, offset in enumerate(range(0, total, chunk_size), start=1):
|
||
count = min(chunk_size, total - offset)
|
||
name = f"prefill-{human(size)}-{chunk:02d}"
|
||
self.stage = f"size {human(size)}: prefill"
|
||
measured = measure(
|
||
self.stand,
|
||
"prefill.js",
|
||
{
|
||
"COUNT": count,
|
||
"OFFSET": offset,
|
||
"VUS": self.profile.prefill_vus,
|
||
"NAME": name,
|
||
},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
down = self.stand.down_storages()
|
||
flags = run_flags(
|
||
self.profile,
|
||
measured,
|
||
self.stand.state(APP),
|
||
down,
|
||
)
|
||
self.results.add_prefill(
|
||
{
|
||
**summary,
|
||
"size": size,
|
||
"chunk": chunk,
|
||
"messages_before": offset,
|
||
"messages_after": offset + summary["requests_ok"],
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"mem_cgroup_end_mib": mib_bytes(
|
||
measured.after["mem_bytes"],
|
||
),
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
"db_cpu_cores": measured.db_cpu,
|
||
},
|
||
)
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during prefill chunk "
|
||
f"{chunk}: matrix stopped",
|
||
)
|
||
if summary["requests_ok"] != count:
|
||
raise BenchError(
|
||
f"prefill chunk {chunk}: {summary['requests_ok']} of "
|
||
f"{count} messages confirmed ({' '.join(flags)})",
|
||
)
|
||
total_seconds += summary["duration_s"]
|
||
self.prefilled = offset + count
|
||
self.prefill_rate = self.prefilled / total_seconds
|
||
self.say(
|
||
f"size {human(size)}: prefill {self.prefilled}/{total} "
|
||
f"({self.prefilled * 100 // total}%), "
|
||
f"{self.prefill_rate:.0f} msg/s, "
|
||
f"~{fmt_left((total - self.prefilled) / self.prefill_rate)} "
|
||
"left",
|
||
)
|
||
self.progress()
|
||
return total_seconds
|
||
|
||
def run_one(
|
||
self,
|
||
scenario: str,
|
||
size: int,
|
||
conc: int,
|
||
repeat: int,
|
||
store: int,
|
||
) -> int:
|
||
"""Прогрев и измеряемый прогон; возвращает сообщений в хранилище."""
|
||
p = self.profile
|
||
number = self.runs_done + 1
|
||
name = f"{scenario}-{human(size)}-c{conc}-r{repeat}"
|
||
self.stage = f"run {number}/{self.runs_total} {name}"
|
||
self.progress()
|
||
wall = time.monotonic()
|
||
if scenario == "read":
|
||
warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"}
|
||
self.stand.k6(
|
||
"read.js",
|
||
{**warm, "SEED": p.seed + p.warmup_seed_shift},
|
||
)
|
||
env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"}
|
||
env["SEED"] = p.seed
|
||
else:
|
||
self.stand.k6(
|
||
"write.js",
|
||
{"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store},
|
||
)
|
||
warm_ok = parse_summary(self.stand.work_dir / "last.json")
|
||
store += warm_ok["requests_ok"]
|
||
env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store}
|
||
measured = measure(
|
||
self.stand,
|
||
f"{scenario}.js",
|
||
{**env, "NAME": name},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
app = self.stand.state(APP)
|
||
down = self.stand.down_storages()
|
||
flags = run_flags(p, measured, app, down)
|
||
end = store + (summary["requests_ok"] if scenario == "write" else 0)
|
||
row = {
|
||
**summary,
|
||
"run": number,
|
||
"scenario": scenario,
|
||
"size": size,
|
||
"concurrency": conc,
|
||
"repeat": repeat,
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_start_mib": mib(measured.before["rss_kib"]),
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"rss_max_mib": mib(measured.rss_max_kib),
|
||
"mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]),
|
||
"store_messages_start": store,
|
||
"store_messages_end": end,
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
"db_cpu_cores": measured.db_cpu,
|
||
}
|
||
self.results.add_run(row)
|
||
self.runs_done += 1
|
||
if scenario == "write" and summary["rps"]:
|
||
self.write_rps[conc] = summary["rps"]
|
||
overhead = time.monotonic() - wall - summary["duration_s"]
|
||
if scenario == "read":
|
||
overhead -= p.read_warmup_s
|
||
self.run_overhead = max(1.0, overhead)
|
||
errors = summary["requests_failed"] + summary["check_failed"]
|
||
self.say(
|
||
f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} "
|
||
f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps "
|
||
f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} "
|
||
f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} "
|
||
f"db {measured.db_cpu} rss {row['rss_end_mib']} MiB "
|
||
f"err {errors} "
|
||
f"~{fmt_left(self.eta())} left"
|
||
+ (f" [{' '.join(flags)}]" if flags else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if "oom" in flags or "app_down" in flags:
|
||
raise BenchError(
|
||
f"{APP} is down after {name} ({' '.join(flags)}): "
|
||
"matrix stopped",
|
||
)
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during {name}: matrix stopped",
|
||
)
|
||
return end
|
||
|
||
|
||
# ------------------------------------------------------------------ CLI
|
||
|
||
|
||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--profile", choices=sorted(PROFILES), required=True)
|
||
parser.add_argument("--label", required=True)
|
||
# отладка стенда: переопределения пишутся в паспорт как overrides
|
||
parser.add_argument("--sizes", help="через запятую, например 120000")
|
||
parser.add_argument("--concurrency", help="через запятую")
|
||
parser.add_argument("--repeats", type=int)
|
||
parser.add_argument("--memory", help="лимит памяти, например 64m")
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def apply_overrides(
|
||
profile: Profile,
|
||
args: argparse.Namespace,
|
||
) -> tuple[Profile, dict[str, Any]]:
|
||
overrides: dict[str, Any] = {}
|
||
if args.sizes:
|
||
overrides["sizes"] = tuple(int(x) for x in args.sizes.split(","))
|
||
if args.concurrency:
|
||
overrides["concurrency"] = tuple(
|
||
int(x) for x in args.concurrency.split(",")
|
||
)
|
||
if args.repeats:
|
||
overrides["repeats"] = args.repeats
|
||
if args.memory:
|
||
overrides["memory"] = args.memory
|
||
return replace(profile, **overrides), overrides
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
args = parse_args(argv)
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label):
|
||
print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True)
|
||
return 2
|
||
label_dir = ROOT / "evidence" / "bench" / args.label
|
||
relative = label_dir.relative_to(ROOT)
|
||
if label_dir.exists():
|
||
print(f"label directory {relative} already exists", flush=True)
|
||
return 2
|
||
profile, overrides = apply_overrides(PROFILES[args.profile], args)
|
||
git = git_info()
|
||
state = host_state()
|
||
try:
|
||
warnings = host_warnings(state) + vm_warnings(profile)
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 2
|
||
meta: dict[str, Any] = {
|
||
"label": args.label,
|
||
"status": "incomplete",
|
||
"reason": "running",
|
||
"started_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"command": " ".join(["uv run python bench/run.py", *argv]),
|
||
"git": git,
|
||
"overrides": overrides,
|
||
"host_state": {"start": state},
|
||
"host_warnings": warnings,
|
||
# чужие контейнеры в той же VM: стенд их не трогает, но они шумят
|
||
"other_containers": sorted(
|
||
sh("docker", "ps", "--format", "{{.Names}}", check=False).split(),
|
||
),
|
||
}
|
||
stand_check = Stand(profile, label_dir)
|
||
started = time.monotonic()
|
||
# на macOS монотонные часы во сне стоят, а реальные идут: разница —
|
||
# сколько хост проспал посреди прогона (решение 9 design change
|
||
# track2-backup-restore)
|
||
started_wall = time.time()
|
||
|
||
def say(text: str) -> None:
|
||
elapsed = int(time.monotonic() - started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
for warning in warnings:
|
||
say(f"WARNING: {warning}")
|
||
if overrides:
|
||
say(f"WARNING: profile overrides {overrides}: debug run")
|
||
if git["dirty"]:
|
||
say("NOTE: working tree has uncommitted changes (dirty: true)")
|
||
try:
|
||
stand_check.check_free()
|
||
image = stand_check.build_image()
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 1
|
||
say(f"image {APP_IMAGE} built ({image[:19]}…), compose project {PROJECT}")
|
||
meta.update(static_passport(profile))
|
||
results = Results(label_dir, meta)
|
||
bench = Bench(profile, args.label, results)
|
||
bench.started = started
|
||
code = 0
|
||
|
||
def terminate(*_: object) -> None:
|
||
raise KeyboardInterrupt
|
||
|
||
signal.signal(signal.SIGTERM, terminate)
|
||
try:
|
||
bench.run()
|
||
meta["status"] = "complete"
|
||
meta.pop("reason", None)
|
||
except KeyboardInterrupt:
|
||
meta["reason"] = "interrupted"
|
||
say("interrupted: partial results kept")
|
||
code = 130
|
||
except BenchError as exc:
|
||
meta["reason"] = str(exc).splitlines()[0]
|
||
say(f"bench failed: {exc}")
|
||
code = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку стенда
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if not bench.stand.stop():
|
||
say(f"cleanup incomplete: {leftovers()}")
|
||
code = code or 1
|
||
# fixture.json с токенами — только здесь, в результаты не попадает
|
||
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
|
||
meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds")
|
||
meta["host_state"]["end"] = host_state()
|
||
slept = (time.time() - started_wall) - (time.monotonic() - started)
|
||
meta["host_state"]["slept_seconds"] = round(max(0.0, slept))
|
||
if slept > SLEEP_WARN_S:
|
||
warning = f"host slept {slept:.0f}s during the run: rerun"
|
||
meta["host_warnings"].append(warning)
|
||
say(f"WARNING: {warning}")
|
||
results.write_meta()
|
||
results.write_summary()
|
||
bench.stage = "done" if code == 0 else meta.get("reason", "stopped")
|
||
bench.progress()
|
||
say(
|
||
f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, "
|
||
f"results in {relative}",
|
||
)
|
||
return code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|