Files
mtusi2026lab3/bench/common.py
T

868 lines
32 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Общее для профилей стенда: docker, стенд compose, паспорт, хост.
Импортируют `run.py`, `compare.py` и `plots.py`; сам по себе не
запускается. Только стандартная библиотека.
Все вызовы compose — с `-p vlad-bench`, обоими `-f` и `--env-file
bench/bench.env`: рабочий стенд и его `.env` не затрагиваются. Пароли
хранилищ генерируются на прогон и никуда не пишутся. Свои контейнеры —
`vlad-bench-*`; чужие стенд не останавливает и не удаляет.
"""
import dataclasses
import datetime
import fcntl
import json
import os
import platform
import re
import secrets
import shutil
import subprocess
import sys
import tempfile
import time
from contextlib import contextmanager
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BENCH = ROOT / "bench"
EVIDENCE = ROOT / "evidence"
IMAGE_REPO = "mtusi-messenger-vladimirceva"
# тег, на который смотрит сервис api стенда (BENCH_IMAGE); профиль rollout
# переставляет его со старого образа на новый
SERVICE_TAG = f"{IMAGE_REPO}:bench"
PROJECT = "vlad-bench"
NET = "vlad-bench-net"
DB = "vlad-bench-postgres"
CACHE = "vlad-bench-valkey"
LB = "vlad-bench-lb"
GEN = "vlad-bench-gen"
SAMPLER = "vlad-bench-sampler"
NAMED = (DB, CACHE, LB, GEN, SAMPLER)
COMPOSE_FILES = (ROOT / "compose.yaml", BENCH / "bench.compose.yml")
ENV_FILE = BENCH / "bench.env"
LOCK = BENCH / ".bench.lock"
# что из окружения хоста нужно самому docker compose; остальное (в том
# числе POSTGRES_* из shell) в стенд не попадает
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "LANG", "DOCKER_HOST", "DOCKER_CONFIG")
LB_URL = "http://haproxy:8080"
LABEL_RE = re.compile(r"[A-Za-z0-9._-]{1,48}")
# фикстура генератора: токены пользователей стенда живут только в его контейнере
FIXTURE = "/tmp/bench-fixture.json"
# psql в контейнере PostgreSQL от пользователя образа, без пароля; SQL — stdin
PSQL = 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
# статистика HAProxy изнутри его контейнера (curl в образе нет, bash есть)
LB_STATS_CMD = (
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
)
PG_SETTINGS = (
"shared_buffers",
"max_connections",
"synchronous_commit",
"fsync",
"wal_level",
"checkpoint_timeout",
"max_wal_size",
"jit",
"password_encryption",
)
class StandError(Exception):
pass
def say(text: str) -> None:
stamp = datetime.datetime.now().strftime("%H:%M:%S")
print(f"[{stamp}] {text}", flush=True)
def utc_now() -> str:
return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(
cmd: list[str],
*,
check: bool = True,
timeout: float | None = None,
env: dict[str, str] | None = None,
stdin: str | None = None,
):
done = subprocess.run(
cmd,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=timeout,
check=False,
cwd=ROOT,
env=env,
input=stdin,
)
if check and done.returncode != 0:
tail = (done.stderr or done.stdout).strip().splitlines()[-15:]
raise StandError(
f"{' '.join(cmd[:4])}... exit {done.returncode}\n" + "\n".join(tail)
)
return done
def docker(*args: str, check: bool = True, timeout: float | None = None):
return sh(["docker", *args], check=check, timeout=timeout)
@contextmanager
def bench_lock():
"""Один прогон стенда за раз: второй — код 2 (общие имена контейнеров)."""
LOCK.parent.mkdir(parents=True, exist_ok=True)
with LOCK.open("w") as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise SystemExit(
"another bench run holds bench/.bench.lock: wait for it to finish"
) from None
yield
# --- хост и паспорт ------------------------------------------------------------------
def cpu_model() -> str:
try:
if sys.platform == "darwin":
return sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
if sys.platform.startswith("linux"):
for line in Path("/proc/cpuinfo").read_text().splitlines():
if line.lower().startswith(("model name", "hardware")):
return line.split(":", 1)[1].strip()
except (OSError, StandError):
pass
return platform.processor() or "unknown"
def ram_gb() -> float | None:
try:
if sys.platform == "darwin":
return round(int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30, 1)
if sys.platform.startswith("linux"):
for line in Path("/proc/meminfo").read_text().splitlines():
if line.startswith("MemTotal:"):
return round(int(line.split()[1]) / 2**20, 1)
except (OSError, StandError, ValueError):
pass
return None
def swapouts() -> int | None:
"""Счётчик выгрузок страниц macOS (`vm_stat`): прирост за прогон — помеха."""
if sys.platform != "darwin":
return None
out = sh(["vm_stat"], check=False).stdout
match = re.search(r"Swapouts:\s+(\d+)", out)
return int(match.group(1)) if match else None
def host_state() -> dict:
"""Питание, swap и давление на память (macOS)."""
state: dict = {"swap_used_mib": None, "power": None, "memory_pressure": None}
if sys.platform != "darwin":
return state
swap = sh(["sysctl", "-n", "vm.swapusage"], check=False).stdout
match = re.search(r"used = ([\d.,]+)M", swap)
if match:
state["swap_used_mib"] = float(match.group(1).replace(",", "."))
batt = sh(["pmset", "-g", "batt"], check=False).stdout
match = re.search(r"drawing from '([^']+)'", batt)
if match:
state["power"] = match.group(1)
pressure = sh(["memory_pressure", "-Q"], check=False).stdout
match = re.search(r"free percentage: (\d+)%", pressure)
if match:
state["memory_free_percent"] = int(match.group(1))
state["swapouts"] = swapouts()
return state
def host_info() -> dict:
info = {
"platform": platform.platform(),
"machine": platform.machine(),
"cpu_model": cpu_model(),
"cpu_count": os.cpu_count(),
"ram_gb": ram_gb(),
**host_state(),
}
if hasattr(os, "getloadavg"):
info["loadavg_1m"] = round(os.getloadavg()[0], 2)
return info
def docker_info() -> dict:
raw = json.loads(docker("info", "--format", "{{json .}}").stdout)
keys = (
"ServerVersion",
"OperatingSystem",
"KernelVersion",
"Architecture",
"NCPU",
"MemTotal",
"CgroupVersion",
"CgroupDriver",
)
info = {key: raw.get(key) for key in keys}
compose = docker("compose", "version", "--short", check=False).stdout.strip()
info["compose_version"] = compose or None
names = docker("ps", "--format", "{{.Names}}").stdout.split()
info["other_running_containers"] = sorted(
name
for name in names
if not name.startswith(f"{PROJECT}-") and name not in NAMED
)
return info
def host_warnings(host: dict, dock: dict) -> list[str]:
"""Хост не в форме: метку переснять (bench/AGENTS.md)."""
warnings = []
if host.get("power") == "Battery Power":
warnings.append("host on battery power: CPU may be throttled")
free = host.get("memory_free_percent")
if free is not None and free < 20:
warnings.append(f"memory pressure: only {free}% free")
if dock["other_running_containers"]:
warnings.append(
f"{len(dock['other_running_containers'])} other container(s) running: "
"they share CPU with the stand (make stop on the working stand)",
)
return warnings
def budget_check(cpus: float, memory_gib: float, dock: dict) -> dict:
"""Сумма лимитов стенда и ВМ Docker: не помещается — отказ до стенда."""
vm_cpus = dock["NCPU"]
vm_gib = dock["MemTotal"] / 2**30
budget = {
"limits_cpus": round(cpus, 2),
"limits_memory_gib": round(memory_gib, 2),
"vm_cpus": vm_cpus,
"vm_memory_gib": round(vm_gib, 2),
}
if cpus > vm_cpus:
raise SystemExit(
f"limits need {cpus:.2f} CPU, docker VM has {vm_cpus}: raise CPUs in "
"Docker Desktop (Settings → Resources) or use a smaller series",
)
if memory_gib + 1.5 > vm_gib:
raise SystemExit(
f"limits need {memory_gib:.1f} GiB + 1.5 GiB reserve, docker VM has "
f"{vm_gib:.1f} GiB: raise memory in Docker Desktop",
)
return budget
def base_image() -> str:
"""Базовый образ из Dockerfile: генератор работает на нём же."""
for line in (ROOT / "Dockerfile").read_text(encoding="utf-8").splitlines():
if line.startswith("FROM "):
return line.split()[1]
raise StandError("no FROM line in Dockerfile")
# Выполняется внутри образа: версии, хеш кода и пул приложения — того кода,
# который реально измеряется
IMAGE_PROBE = """
import hashlib, importlib.metadata as m, json, pathlib, platform
from src.config import Database, Settings
root = pathlib.Path.cwd()
digest = hashlib.sha256()
for path in sorted([*(root / "src").rglob("*.py"), *(root / "migrations").rglob("*.py")]):
digest.update(path.relative_to(root).as_posix().encode())
digest.update(b"\\0")
digest.update(path.read_bytes())
digest.update(b"\\0")
packages = ("fastapi", "starlette", "uvicorn", "pydantic", "sqlalchemy", "asyncpg",
"alembic", "redis", "argon2-cffi")
db = Database()
app = Settings()
print(json.dumps({
"src_sha256": digest.hexdigest(),
"versions": {"python": platform.python_version(), **{p: m.version(p) for p in packages}},
"app_defaults": {
"pool_size": db.pool_size, "pool_overflow": db.pool_overflow,
"pool_timeout": db.pool_timeout, "operation_timeout": db.operation_timeout,
"statement_timeout_ms": db.statement_timeout_ms,
"kdf_concurrency": app.kdf_concurrency, "keep_alive": app.keep_alive,
"shutdown_timeout": app.shutdown_timeout,
},
}))
"""
def build_image(tag: str, context: Path | None = None) -> None:
sh(["docker", "build", "-q", "-t", tag, str(context or ROOT)], timeout=1800)
def image_info(image: str) -> dict:
raw = json.loads(docker("image", "inspect", image).stdout)[0]
probe = docker("run", "--rm", "--entrypoint", "python", image, "-c", IMAGE_PROBE)
data = json.loads(probe.stdout.strip().splitlines()[-1])
return {"tag": image, "id": raw["Id"], "created": raw["Created"], **data}
def code_state(image: dict) -> dict:
"""Что измеряется: хеш кода из образа и, если есть git, commit."""
state = {"src_sha256": image["src_sha256"], "git_commit": None, "git_dirty": None}
if (ROOT / ".git").exists():
head = sh(["git", "rev-parse", "HEAD"], check=False)
if head.returncode == 0:
state["git_commit"] = head.stdout.strip()
status = sh(["git", "status", "--porcelain"], check=False).stdout
state["git_dirty"] = bool(status.strip())
return state
def probe_migration_context(workdir: Path) -> tuple[Path, str]:
"""Контекст сборки «нового» образа профиля rollout: файлы рабочего
дерева, которые копирует Dockerfile, плюс expand-ревизия
`9001_rollout_probe` — новая колонка `messages.rollout_probe` (NULL).
Ревизия существует только в этом образе и в базе стенда: так замена
под нагрузкой проходит с настоящей миграцией схемы, а рабочее дерево
и миграции репозитория не меняются. Возвращает каталог и ревизию head.
"""
context = workdir / "rollout-context"
for name in ("Dockerfile", "requirements.txt", "alembic.ini"):
target = context / name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(ROOT / name, target)
for folder in ("src", "migrations"):
shutil.copytree(
ROOT / folder,
context / folder,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "AGENTS.md"),
)
revisions, parents = set(), set()
for path in (ROOT / "migrations" / "versions").glob("*.py"):
text = path.read_text(encoding="utf-8")
rev = re.search(r'^revision: str = "([^"]+)"', text, re.MULTILINE)
down = re.search(r'^down_revision: [^=]+= "([^"]+)"', text, re.MULTILINE)
if rev:
revisions.add(rev.group(1))
if down:
parents.add(down.group(1))
heads = revisions - parents
if len(heads) != 1:
raise StandError(f"cannot find a single migration head: {sorted(heads)}")
head = heads.pop()
(context / "migrations" / "versions" / "9001_rollout_probe.py").write_text(
PROBE_MIGRATION.replace("@HEAD@", head), encoding="utf-8"
)
return context, head
PROBE_MIGRATION = '''"""rollout probe: nullable column for the bench rollout profile
Revision ID: 9001
Revises: @HEAD@
Только в образе профиля rollout стенда bench/ (bench/common.py):
expand-миграция под нагрузкой. Колонка NULL, старый код её не замечает.
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "9001"
down_revision: str | Sequence[str] | None = "@HEAD@"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# ADD COLUMN берёт ACCESS EXCLUSIVE: очередь за ним ограничена
op.execute("SET LOCAL lock_timeout = '2s'")
op.add_column("messages", sa.Column("rollout_probe", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("messages", "rollout_probe")
'''
# --- стенд ------------------------------------------------------------------------
@dataclasses.dataclass(frozen=True)
class Limits:
"""Лимиты контейнеров стенда и число реплик. Часть методики."""
api_replicas: int
api_cpus: float
api_memory: str
db_cpus: float = 2.0
db_memory: str = "1g"
db_max_connections: int = 100
cache_cpus: float = 0.5
cache_memory: str = "128m"
lb_cpus: float = 1.0
lb_memory: str = "256m"
gen_cpus: float = 2.0
gen_memory: str = "1g"
sampler_cpus: float = 0.25
sampler_memory: str = "256m"
def total_cpus(self, replicas: int | None = None) -> float:
replicas = self.api_replicas if replicas is None else replicas
return (
replicas * self.api_cpus
+ self.db_cpus
+ self.cache_cpus
+ self.lb_cpus
+ self.gen_cpus
+ self.sampler_cpus
)
def total_memory_gib(self, replicas: int | None = None) -> float:
replicas = self.api_replicas if replicas is None else replicas
return (
replicas * gib(self.api_memory)
+ gib(self.db_memory)
+ gib(self.cache_memory)
+ gib(self.lb_memory)
+ gib(self.gen_memory)
+ gib(self.sampler_memory)
)
def gib(value: str) -> float:
number, unit = float(value[:-1]), value[-1].lower()
return number / 1024 if unit == "m" else number
def leftovers() -> list[str]:
"""Команды уборки остатков другого (прерванного) прогона."""
label = f"label=com.docker.compose.project={PROJECT}"
names = set(docker("ps", "-a", "--format", "{{.Names}}").stdout.split())
project = docker("ps", "-aq", "--filter", label).stdout.split()
volumes = docker("volume", "ls", "-q", "--filter", label).stdout.split()
networks = docker("network", "ls", "--format", "{{.Name}}").stdout.split()
fixes = []
containers = sorted(set(project) | (names & set(NAMED)))
if containers:
fixes.append(f"docker rm -f {' '.join(containers)}")
if volumes:
fixes.append(f"docker volume rm {' '.join(volumes)}")
if NET in networks:
fixes.append(f"docker network rm {NET}")
return fixes
def lb_slots_from_csv(text: str) -> list[dict]:
"""Слоты backend api со статистикой HAProxy (адрес — без порта)."""
lines = [line for line in text.splitlines() if line.startswith(("# ", "api,"))]
if not lines or not lines[0].startswith("# "):
return []
header = lines[0][2:].split(",")
slots = []
for line in lines[1:]:
row = dict(zip(header, line.split(","), strict=False))
if row.get("svname") in ("BACKEND", "FRONTEND"):
continue
addr = row.get("addr", "").rsplit(":", 1)[0]
slots.append(
{"slot": row["svname"], "addr": addr, "status": row.get("status", "")}
)
return slots
class Stand:
"""Проект compose `vlad-bench`: хранилища, миграции, реплики, LB,
контейнеры генератора и сэмплера."""
def __init__(self, limits: Limits, *, session_ttl: int, workdir: Path) -> None:
self.limits = limits
self.workdir = workdir
host_env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
self.env = {
**host_env,
"POSTGRES_PASSWORD": secrets.token_hex(16),
"VALKEY_PASSWORD": secrets.token_hex(16),
"BENCH_IMAGE": SERVICE_TAG,
"BENCH_API_REPLICAS": str(limits.api_replicas),
"BENCH_API_CPUS": str(limits.api_cpus),
"BENCH_API_MEMORY": limits.api_memory,
"BENCH_DB_CPUS": str(limits.db_cpus),
"BENCH_DB_MEMORY": limits.db_memory,
"BENCH_DB_MAX_CONNECTIONS": str(limits.db_max_connections),
"BENCH_CACHE_CPUS": str(limits.cache_cpus),
"BENCH_CACHE_MEMORY": limits.cache_memory,
"BENCH_LB_CPUS": str(limits.lb_cpus),
"BENCH_LB_MEMORY": limits.lb_memory,
"BENCH_SESSION_TTL": str(session_ttl),
}
# --- compose и контейнеры ---
def compose_args(self) -> list[str]:
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
return ["docker", "compose", "-p", PROJECT, "--env-file", str(ENV_FILE), *files]
def compose(self, *args: str, check: bool = True, timeout: float | None = 600):
"""docker compose стенда; без `-p vlad-bench` не вызывается."""
return sh(
[*self.compose_args(), *args], check=check, timeout=timeout, env=self.env
)
def script_env(self) -> dict[str, str]:
"""Окружение скриптов репозитория (rolling-update.sh): тот же
проект compose переменными, без `-p` в самом скрипте."""
return {
**self.env,
"COMPOSE_PROJECT_NAME": PROJECT,
"COMPOSE_FILE": os.pathsep.join(str(path) for path in COMPOSE_FILES),
"COMPOSE_ENV_FILES": str(ENV_FILE),
}
def start(self) -> None:
"""Свежий стенд: пустая база на новом volume, миграции, реплики, LB,
генератор. Готов, когда LB видит `UP` все реплики."""
self.compose("up", "-d", "--wait", "postgres", "valkey")
self.compose("run", "--rm", "migrate")
self.compose("up", "-d", "--no-deps", "--no-build", "api")
self.compose("up", "-d", "--no-deps", "--no-build", "haproxy")
self.wait_lb_up(self.limits.api_replicas)
docker(
"run",
"-d",
"--name",
GEN,
"--network",
NET,
"--cpus",
str(self.limits.gen_cpus),
"--memory",
self.limits.gen_memory,
"--memory-swap",
self.limits.gen_memory,
"--entrypoint",
"python",
base_image(),
"-c",
"import time; time.sleep(86400)",
)
# docker cp, а не bind-mount: не зависит от общих каталогов Docker Desktop
docker("cp", str(BENCH / "loadgen.py"), f"{GEN}:/tmp/loadgen.py")
ready = self.gen("ready", "--timeout", "60")
if not ready["ready"]:
raise StandError("API is not ready through the LB:\n" + self.app_tail())
def stop(self) -> None:
"""Генератор и сэмплер первыми: подключённые к сети чужие для compose
контейнеры не дали бы удалить сеть. Затем стенд с volume."""
docker("rm", "-f", GEN, SAMPLER, check=False)
self.compose("down", "-v", "--remove-orphans", check=False)
def api_ids(self, *, all_states: bool = False) -> list[str]:
args = (
["ps", "-a", "-q", "--no-trunc", "api"]
if all_states
else [
"ps",
"-q",
"--no-trunc",
"api",
]
)
return self.compose(*args).stdout.split()
def address(self, container: str) -> str:
return docker(
"inspect",
"-f",
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
container,
).stdout.strip()
def lb_slots(self) -> list[dict]:
out = docker("exec", LB, "bash", "-c", LB_STATS_CMD, check=False)
return lb_slots_from_csv(out.stdout) if out.returncode == 0 else []
def wait_lb_up(self, replicas: int, timeout: float = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
up = [slot for slot in self.lb_slots() if slot["status"] == "UP"]
if len(up) >= replicas:
return
time.sleep(0.5)
raise StandError(
f"LB does not see {replicas} replicas UP in {timeout:.0f}s:\n"
+ self.app_tail()
)
def wait_slot(self, address: str, status: str, timeout: float = 30) -> float | None:
"""Ждать состояния слота с адресом; секунды или None."""
started = time.monotonic()
while time.monotonic() - started < timeout:
for slot in self.lb_slots():
if slot["addr"] == address and slot["status"].startswith(status):
return time.monotonic() - started
time.sleep(0.25)
return None
def app_tail(self, lines: int = 30) -> str:
done = self.compose("logs", "--tail", str(lines), "api", "haproxy", check=False)
return done.stdout + done.stderr
def gen(self, *args: str, timeout: float = 3600) -> dict:
done = docker(
"exec",
GEN,
"python",
"/tmp/loadgen.py",
"--base-url",
LB_URL,
*args,
timeout=timeout,
)
return json.loads(done.stdout.strip().splitlines()[-1])
def gen_popen(self, *args: str) -> subprocess.Popen:
"""Генератор в фоне: оркестратор действует на стенд, пока идёт нагрузка."""
return subprocess.Popen(
[
"docker",
"exec",
GEN,
"python",
"/tmp/loadgen.py",
"--base-url",
LB_URL,
*args,
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
def gen_file(self, path: str) -> str:
return docker("exec", GEN, "cat", path).stdout
def psql(self, sql: str) -> str:
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля."""
done = sh(
["docker", "exec", "-i", DB, "sh", "-c", PSQL], stdin=sql, check=False
)
if done.returncode != 0:
raise StandError(f"psql: exit {done.returncode}\n{done.stderr}")
return done.stdout.strip()
def maintain(self) -> float:
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды."""
started = time.monotonic()
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
return time.monotonic() - started
# --- сэмплер ---
def start_sampler(self, mode: str, containers: list[str] | None = None) -> None:
"""Сэмплер на образе приложения с `--cgroupns host` (bench/sampler.py)."""
args = [
"run",
"-d",
"--name",
SAMPLER,
"--cgroupns",
"host",
"--cpus",
str(self.limits.sampler_cpus),
"--memory",
self.limits.sampler_memory,
"--memory-swap",
self.limits.sampler_memory,
]
if mode == "list":
# пароль — только в окружении контейнера сэмплера
args += [
"--network",
NET,
"-e",
"PGUSER=bench",
"-e",
"PGDATABASE=bench",
"-e",
f"PGPASSWORD={self.env['POSTGRES_PASSWORD']}",
]
else:
args += ["--network", "none"]
docker(
*args,
"--entrypoint",
"python",
SERVICE_TAG,
"-c",
"import time; time.sleep(86400)",
)
docker("cp", str(BENCH / "sampler.py"), f"{SAMPLER}:/tmp/sampler.py")
command = [
"exec",
"-d",
SAMPLER,
"python",
"/tmp/sampler.py",
"--mode",
mode,
"--out",
"/tmp/samples.jsonl",
]
for container in containers or []:
command += ["--container", container]
docker(*command)
def sampler_lines(self) -> list[dict]:
done = docker("exec", SAMPLER, "cat", "/tmp/samples.jsonl", check=False)
return [json.loads(line) for line in done.stdout.splitlines() if line.strip()]
def stop_sampler(self) -> None:
docker("rm", "-f", SAMPLER, check=False)
# --- паспорт ---
def passport(self) -> dict:
"""Хранилища, лимиты и окружение API для паспорта; паролей здесь нет."""
settings = self.psql(
"SELECT name, current_setting(name) FROM pg_settings "
f"WHERE name = ANY(string_to_array('{','.join(PG_SETTINGS)}', ',')) "
"ORDER BY name;\n",
)
info = docker("exec", CACHE, "valkey-cli", "INFO", "server", check=False).stdout
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.MULTILINE)
api = self.api_ids()
env = json.loads(docker("inspect", "-f", "{{json .Config.Env}}", api[0]).stdout)
app_env = dict(
sorted(
item.split("=", 1)
for item in env
if re.match(
r"(DB|CACHE|SESSION|LOG|SHUTDOWN|KDF|IDEMPOTENCY|APP)_[A-Z_]*=",
item,
)
and "PASSWORD" not in item.split("=", 1)[0]
),
)
limits = {}
for role, name in (
("api", api[0]),
("postgres", DB),
("valkey", CACHE),
("haproxy", LB),
):
host = json.loads(
docker("inspect", "-f", "{{json .HostConfig}}", name).stdout
)
limits[role] = {
"nano_cpus": host.get("NanoCpus"),
"memory": host.get("Memory"),
"memory_swap": host.get("MemorySwap"),
"published_ports": host.get("PortBindings") or {},
}
def image(name: str) -> str:
return docker("inspect", "-f", "{{.Config.Image}}", name).stdout.strip()
return {
"postgres": {
"image": image(DB),
"version": self.psql("SHOW server_version;\n"),
"settings": dict(line.split("|", 1) for line in settings.splitlines()),
},
"valkey": {
"image": image(CACHE),
"version": valkey_version.group(1) if valkey_version else None,
},
"haproxy": {"image": image(LB)},
"limits": limits,
"api_replicas": len(api),
"app_env": app_env,
}
def new_label_dir(kind: str, label: str) -> Path:
if not LABEL_RE.fullmatch(label):
raise SystemExit(f"label must match {LABEL_RE.pattern}")
folder = EVIDENCE / kind / label
if folder.exists():
raise SystemExit(
f"{folder.relative_to(ROOT)} exists: labels are not overwritten"
)
folder.mkdir(parents=True)
return folder
def quick_label(kind: str) -> str:
stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
return f"quick-{kind}-{stamp}" if kind else f"quick-{stamp}"
def write_json(path: Path, data: dict) -> None:
path.write_text(
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
def preflight(limits_cpus: float, limits_memory_gib: float) -> dict:
"""Хост, Docker, бюджет лимитов, остатки прошлых прогонов."""
fixes = leftovers()
if fixes:
raise SystemExit(
"leftovers of an interrupted bench run; remove them first:\n "
+ "\n ".join(fixes)
)
host = host_info()
dock = docker_info()
budget = budget_check(limits_cpus, limits_memory_gib, dock)
return {
"host": host,
"docker": dock,
"budget": budget,
"warnings": host_warnings(host, dock),
}
def finish_host_state(
meta: dict, started_monotonic: float, started_wall: float
) -> None:
"""Сон ноутбука и выгрузка страниц за прогон — в паспорт и предупреждения."""
elapsed_wall = time.time() - started_wall
elapsed_mono = time.monotonic() - started_monotonic
end = host_state()
state = {
"slept_seconds": round(max(0.0, elapsed_wall - elapsed_mono), 1),
"swapouts_start": meta["host"].get("swapouts"),
"swapouts_end": end.get("swapouts"),
"power_end": end.get("power"),
}
if state["swapouts_start"] is not None and state["swapouts_end"] is not None:
state["swapouts_delta"] = state["swapouts_end"] - state["swapouts_start"]
if state["swapouts_delta"] > 0:
meta["host_warnings"].append(
f"host swapped out {state['swapouts_delta']} pages during the run"
)
if state["slept_seconds"] > 5:
meta["host_warnings"].append(
f"host slept {state['slept_seconds']}s during the run: rerun"
)
meta["host_state_end"] = state
def temp_workdir() -> Path:
return Path(tempfile.mkdtemp(prefix="vlad-bench-"))