868 lines
32 KiB
Python
868 lines
32 KiB
Python
"""Общее для профилей стенда: docker, стенд compose, паспорт, хост.
|
||
|
||
Импортируют `run.py`, `compare.py` и `plots.py`; сам по себе не
|
||
запускается. Только стандартная библиотека.
|
||
|
||
Все вызовы compose — с `-p vlad-bench`, обоими `-f` и `--env-file
|
||
bench/bench.env`: рабочий стенд и его `.env` не затрагиваются. Пароли
|
||
хранилищ генерируются на прогон и никуда не пишутся. Свои контейнеры —
|
||
`vlad-bench-*`; чужие стенд не останавливает и не удаляет.
|
||
"""
|
||
|
||
import dataclasses
|
||
import datetime
|
||
import fcntl
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import secrets
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from contextlib import contextmanager
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BENCH = ROOT / "bench"
|
||
EVIDENCE = ROOT / "evidence"
|
||
IMAGE_REPO = "mtusi-messenger-vladimirceva"
|
||
# тег, на который смотрит сервис api стенда (BENCH_IMAGE); профиль rollout
|
||
# переставляет его со старого образа на новый
|
||
SERVICE_TAG = f"{IMAGE_REPO}:bench"
|
||
PROJECT = "vlad-bench"
|
||
NET = "vlad-bench-net"
|
||
DB = "vlad-bench-postgres"
|
||
CACHE = "vlad-bench-valkey"
|
||
LB = "vlad-bench-lb"
|
||
GEN = "vlad-bench-gen"
|
||
SAMPLER = "vlad-bench-sampler"
|
||
NAMED = (DB, CACHE, LB, GEN, SAMPLER)
|
||
COMPOSE_FILES = (ROOT / "compose.yaml", BENCH / "bench.compose.yml")
|
||
ENV_FILE = BENCH / "bench.env"
|
||
LOCK = BENCH / ".bench.lock"
|
||
# что из окружения хоста нужно самому docker compose; остальное (в том
|
||
# числе POSTGRES_* из shell) в стенд не попадает
|
||
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "LANG", "DOCKER_HOST", "DOCKER_CONFIG")
|
||
LB_URL = "http://haproxy:8080"
|
||
LABEL_RE = re.compile(r"[A-Za-z0-9._-]{1,48}")
|
||
# фикстура генератора: токены пользователей стенда живут только в его контейнере
|
||
FIXTURE = "/tmp/bench-fixture.json"
|
||
# psql в контейнере PostgreSQL от пользователя образа, без пароля; SQL — stdin
|
||
PSQL = 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
|
||
# статистика HAProxy изнутри его контейнера (curl в образе нет, bash есть)
|
||
LB_STATS_CMD = (
|
||
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
|
||
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
|
||
)
|
||
PG_SETTINGS = (
|
||
"shared_buffers",
|
||
"max_connections",
|
||
"synchronous_commit",
|
||
"fsync",
|
||
"wal_level",
|
||
"checkpoint_timeout",
|
||
"max_wal_size",
|
||
"jit",
|
||
"password_encryption",
|
||
)
|
||
|
||
|
||
class StandError(Exception):
|
||
pass
|
||
|
||
|
||
def say(text: str) -> None:
|
||
stamp = datetime.datetime.now().strftime("%H:%M:%S")
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
|
||
def utc_now() -> str:
|
||
return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
|
||
def sh(
|
||
cmd: list[str],
|
||
*,
|
||
check: bool = True,
|
||
timeout: float | None = None,
|
||
env: dict[str, str] | None = None,
|
||
stdin: str | None = None,
|
||
):
|
||
done = subprocess.run(
|
||
cmd,
|
||
capture_output=True,
|
||
text=True,
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
timeout=timeout,
|
||
check=False,
|
||
cwd=ROOT,
|
||
env=env,
|
||
input=stdin,
|
||
)
|
||
if check and done.returncode != 0:
|
||
tail = (done.stderr or done.stdout).strip().splitlines()[-15:]
|
||
raise StandError(
|
||
f"{' '.join(cmd[:4])}... exit {done.returncode}\n" + "\n".join(tail)
|
||
)
|
||
return done
|
||
|
||
|
||
def docker(*args: str, check: bool = True, timeout: float | None = None):
|
||
return sh(["docker", *args], check=check, timeout=timeout)
|
||
|
||
|
||
@contextmanager
|
||
def bench_lock():
|
||
"""Один прогон стенда за раз: второй — код 2 (общие имена контейнеров)."""
|
||
LOCK.parent.mkdir(parents=True, exist_ok=True)
|
||
with LOCK.open("w") as handle:
|
||
try:
|
||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except BlockingIOError:
|
||
raise SystemExit(
|
||
"another bench run holds bench/.bench.lock: wait for it to finish"
|
||
) from None
|
||
yield
|
||
|
||
|
||
# --- хост и паспорт ------------------------------------------------------------------
|
||
|
||
|
||
def cpu_model() -> str:
|
||
try:
|
||
if sys.platform == "darwin":
|
||
return sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
|
||
if sys.platform.startswith("linux"):
|
||
for line in Path("/proc/cpuinfo").read_text().splitlines():
|
||
if line.lower().startswith(("model name", "hardware")):
|
||
return line.split(":", 1)[1].strip()
|
||
except (OSError, StandError):
|
||
pass
|
||
return platform.processor() or "unknown"
|
||
|
||
|
||
def ram_gb() -> float | None:
|
||
try:
|
||
if sys.platform == "darwin":
|
||
return round(int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30, 1)
|
||
if sys.platform.startswith("linux"):
|
||
for line in Path("/proc/meminfo").read_text().splitlines():
|
||
if line.startswith("MemTotal:"):
|
||
return round(int(line.split()[1]) / 2**20, 1)
|
||
except (OSError, StandError, ValueError):
|
||
pass
|
||
return None
|
||
|
||
|
||
def swapouts() -> int | None:
|
||
"""Счётчик выгрузок страниц macOS (`vm_stat`): прирост за прогон — помеха."""
|
||
if sys.platform != "darwin":
|
||
return None
|
||
out = sh(["vm_stat"], check=False).stdout
|
||
match = re.search(r"Swapouts:\s+(\d+)", out)
|
||
return int(match.group(1)) if match else None
|
||
|
||
|
||
def host_state() -> dict:
|
||
"""Питание, swap и давление на память (macOS)."""
|
||
state: dict = {"swap_used_mib": None, "power": None, "memory_pressure": None}
|
||
if sys.platform != "darwin":
|
||
return state
|
||
swap = sh(["sysctl", "-n", "vm.swapusage"], check=False).stdout
|
||
match = re.search(r"used = ([\d.,]+)M", swap)
|
||
if match:
|
||
state["swap_used_mib"] = float(match.group(1).replace(",", "."))
|
||
batt = sh(["pmset", "-g", "batt"], check=False).stdout
|
||
match = re.search(r"drawing from '([^']+)'", batt)
|
||
if match:
|
||
state["power"] = match.group(1)
|
||
pressure = sh(["memory_pressure", "-Q"], check=False).stdout
|
||
match = re.search(r"free percentage: (\d+)%", pressure)
|
||
if match:
|
||
state["memory_free_percent"] = int(match.group(1))
|
||
state["swapouts"] = swapouts()
|
||
return state
|
||
|
||
|
||
def host_info() -> dict:
|
||
info = {
|
||
"platform": platform.platform(),
|
||
"machine": platform.machine(),
|
||
"cpu_model": cpu_model(),
|
||
"cpu_count": os.cpu_count(),
|
||
"ram_gb": ram_gb(),
|
||
**host_state(),
|
||
}
|
||
if hasattr(os, "getloadavg"):
|
||
info["loadavg_1m"] = round(os.getloadavg()[0], 2)
|
||
return info
|
||
|
||
|
||
def docker_info() -> dict:
|
||
raw = json.loads(docker("info", "--format", "{{json .}}").stdout)
|
||
keys = (
|
||
"ServerVersion",
|
||
"OperatingSystem",
|
||
"KernelVersion",
|
||
"Architecture",
|
||
"NCPU",
|
||
"MemTotal",
|
||
"CgroupVersion",
|
||
"CgroupDriver",
|
||
)
|
||
info = {key: raw.get(key) for key in keys}
|
||
compose = docker("compose", "version", "--short", check=False).stdout.strip()
|
||
info["compose_version"] = compose or None
|
||
names = docker("ps", "--format", "{{.Names}}").stdout.split()
|
||
info["other_running_containers"] = sorted(
|
||
name
|
||
for name in names
|
||
if not name.startswith(f"{PROJECT}-") and name not in NAMED
|
||
)
|
||
return info
|
||
|
||
|
||
def host_warnings(host: dict, dock: dict) -> list[str]:
|
||
"""Хост не в форме: метку переснять (bench/AGENTS.md)."""
|
||
warnings = []
|
||
if host.get("power") == "Battery Power":
|
||
warnings.append("host on battery power: CPU may be throttled")
|
||
free = host.get("memory_free_percent")
|
||
if free is not None and free < 20:
|
||
warnings.append(f"memory pressure: only {free}% free")
|
||
if dock["other_running_containers"]:
|
||
warnings.append(
|
||
f"{len(dock['other_running_containers'])} other container(s) running: "
|
||
"they share CPU with the stand (make stop on the working stand)",
|
||
)
|
||
return warnings
|
||
|
||
|
||
def budget_check(cpus: float, memory_gib: float, dock: dict) -> dict:
|
||
"""Сумма лимитов стенда и ВМ Docker: не помещается — отказ до стенда."""
|
||
vm_cpus = dock["NCPU"]
|
||
vm_gib = dock["MemTotal"] / 2**30
|
||
budget = {
|
||
"limits_cpus": round(cpus, 2),
|
||
"limits_memory_gib": round(memory_gib, 2),
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_gib, 2),
|
||
}
|
||
if cpus > vm_cpus:
|
||
raise SystemExit(
|
||
f"limits need {cpus:.2f} CPU, docker VM has {vm_cpus}: raise CPUs in "
|
||
"Docker Desktop (Settings → Resources) or use a smaller series",
|
||
)
|
||
if memory_gib + 1.5 > vm_gib:
|
||
raise SystemExit(
|
||
f"limits need {memory_gib:.1f} GiB + 1.5 GiB reserve, docker VM has "
|
||
f"{vm_gib:.1f} GiB: raise memory in Docker Desktop",
|
||
)
|
||
return budget
|
||
|
||
|
||
def base_image() -> str:
|
||
"""Базовый образ из Dockerfile: генератор работает на нём же."""
|
||
for line in (ROOT / "Dockerfile").read_text(encoding="utf-8").splitlines():
|
||
if line.startswith("FROM "):
|
||
return line.split()[1]
|
||
raise StandError("no FROM line in Dockerfile")
|
||
|
||
|
||
# Выполняется внутри образа: версии, хеш кода и пул приложения — того кода,
|
||
# который реально измеряется
|
||
IMAGE_PROBE = """
|
||
import hashlib, importlib.metadata as m, json, pathlib, platform
|
||
from src.config import Database, Settings
|
||
|
||
root = pathlib.Path.cwd()
|
||
digest = hashlib.sha256()
|
||
for path in sorted([*(root / "src").rglob("*.py"), *(root / "migrations").rglob("*.py")]):
|
||
digest.update(path.relative_to(root).as_posix().encode())
|
||
digest.update(b"\\0")
|
||
digest.update(path.read_bytes())
|
||
digest.update(b"\\0")
|
||
packages = ("fastapi", "starlette", "uvicorn", "pydantic", "sqlalchemy", "asyncpg",
|
||
"alembic", "redis", "argon2-cffi")
|
||
db = Database()
|
||
app = Settings()
|
||
print(json.dumps({
|
||
"src_sha256": digest.hexdigest(),
|
||
"versions": {"python": platform.python_version(), **{p: m.version(p) for p in packages}},
|
||
"app_defaults": {
|
||
"pool_size": db.pool_size, "pool_overflow": db.pool_overflow,
|
||
"pool_timeout": db.pool_timeout, "operation_timeout": db.operation_timeout,
|
||
"statement_timeout_ms": db.statement_timeout_ms,
|
||
"kdf_concurrency": app.kdf_concurrency, "keep_alive": app.keep_alive,
|
||
"shutdown_timeout": app.shutdown_timeout,
|
||
},
|
||
}))
|
||
"""
|
||
|
||
|
||
def build_image(tag: str, context: Path | None = None) -> None:
|
||
sh(["docker", "build", "-q", "-t", tag, str(context or ROOT)], timeout=1800)
|
||
|
||
|
||
def image_info(image: str) -> dict:
|
||
raw = json.loads(docker("image", "inspect", image).stdout)[0]
|
||
probe = docker("run", "--rm", "--entrypoint", "python", image, "-c", IMAGE_PROBE)
|
||
data = json.loads(probe.stdout.strip().splitlines()[-1])
|
||
return {"tag": image, "id": raw["Id"], "created": raw["Created"], **data}
|
||
|
||
|
||
def code_state(image: dict) -> dict:
|
||
"""Что измеряется: хеш кода из образа и, если есть git, commit."""
|
||
state = {"src_sha256": image["src_sha256"], "git_commit": None, "git_dirty": None}
|
||
if (ROOT / ".git").exists():
|
||
head = sh(["git", "rev-parse", "HEAD"], check=False)
|
||
if head.returncode == 0:
|
||
state["git_commit"] = head.stdout.strip()
|
||
status = sh(["git", "status", "--porcelain"], check=False).stdout
|
||
state["git_dirty"] = bool(status.strip())
|
||
return state
|
||
|
||
|
||
def probe_migration_context(workdir: Path) -> tuple[Path, str]:
|
||
"""Контекст сборки «нового» образа профиля rollout: файлы рабочего
|
||
дерева, которые копирует Dockerfile, плюс expand-ревизия
|
||
`9001_rollout_probe` — новая колонка `messages.rollout_probe` (NULL).
|
||
|
||
Ревизия существует только в этом образе и в базе стенда: так замена
|
||
под нагрузкой проходит с настоящей миграцией схемы, а рабочее дерево
|
||
и миграции репозитория не меняются. Возвращает каталог и ревизию head.
|
||
"""
|
||
context = workdir / "rollout-context"
|
||
for name in ("Dockerfile", "requirements.txt", "alembic.ini"):
|
||
target = context / name
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(ROOT / name, target)
|
||
for folder in ("src", "migrations"):
|
||
shutil.copytree(
|
||
ROOT / folder,
|
||
context / folder,
|
||
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "AGENTS.md"),
|
||
)
|
||
revisions, parents = set(), set()
|
||
for path in (ROOT / "migrations" / "versions").glob("*.py"):
|
||
text = path.read_text(encoding="utf-8")
|
||
rev = re.search(r'^revision: str = "([^"]+)"', text, re.MULTILINE)
|
||
down = re.search(r'^down_revision: [^=]+= "([^"]+)"', text, re.MULTILINE)
|
||
if rev:
|
||
revisions.add(rev.group(1))
|
||
if down:
|
||
parents.add(down.group(1))
|
||
heads = revisions - parents
|
||
if len(heads) != 1:
|
||
raise StandError(f"cannot find a single migration head: {sorted(heads)}")
|
||
head = heads.pop()
|
||
(context / "migrations" / "versions" / "9001_rollout_probe.py").write_text(
|
||
PROBE_MIGRATION.replace("@HEAD@", head), encoding="utf-8"
|
||
)
|
||
return context, head
|
||
|
||
|
||
PROBE_MIGRATION = '''"""rollout probe: nullable column for the bench rollout profile
|
||
|
||
Revision ID: 9001
|
||
Revises: @HEAD@
|
||
|
||
Только в образе профиля rollout стенда bench/ (bench/common.py):
|
||
expand-миграция под нагрузкой. Колонка NULL, старый код её не замечает.
|
||
"""
|
||
|
||
from collections.abc import Sequence
|
||
|
||
import sqlalchemy as sa
|
||
from alembic import op
|
||
|
||
revision: str = "9001"
|
||
down_revision: str | Sequence[str] | None = "@HEAD@"
|
||
branch_labels: str | Sequence[str] | None = None
|
||
depends_on: str | Sequence[str] | None = None
|
||
|
||
|
||
def upgrade() -> None:
|
||
# ADD COLUMN берёт ACCESS EXCLUSIVE: очередь за ним ограничена
|
||
op.execute("SET LOCAL lock_timeout = '2s'")
|
||
op.add_column("messages", sa.Column("rollout_probe", sa.Text(), nullable=True))
|
||
|
||
|
||
def downgrade() -> None:
|
||
op.drop_column("messages", "rollout_probe")
|
||
'''
|
||
|
||
|
||
# --- стенд ------------------------------------------------------------------------
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class Limits:
|
||
"""Лимиты контейнеров стенда и число реплик. Часть методики."""
|
||
|
||
api_replicas: int
|
||
api_cpus: float
|
||
api_memory: str
|
||
db_cpus: float = 2.0
|
||
db_memory: str = "1g"
|
||
db_max_connections: int = 100
|
||
cache_cpus: float = 0.5
|
||
cache_memory: str = "128m"
|
||
lb_cpus: float = 1.0
|
||
lb_memory: str = "256m"
|
||
gen_cpus: float = 2.0
|
||
gen_memory: str = "1g"
|
||
sampler_cpus: float = 0.25
|
||
sampler_memory: str = "256m"
|
||
|
||
def total_cpus(self, replicas: int | None = None) -> float:
|
||
replicas = self.api_replicas if replicas is None else replicas
|
||
return (
|
||
replicas * self.api_cpus
|
||
+ self.db_cpus
|
||
+ self.cache_cpus
|
||
+ self.lb_cpus
|
||
+ self.gen_cpus
|
||
+ self.sampler_cpus
|
||
)
|
||
|
||
def total_memory_gib(self, replicas: int | None = None) -> float:
|
||
replicas = self.api_replicas if replicas is None else replicas
|
||
return (
|
||
replicas * gib(self.api_memory)
|
||
+ gib(self.db_memory)
|
||
+ gib(self.cache_memory)
|
||
+ gib(self.lb_memory)
|
||
+ gib(self.gen_memory)
|
||
+ gib(self.sampler_memory)
|
||
)
|
||
|
||
|
||
def gib(value: str) -> float:
|
||
number, unit = float(value[:-1]), value[-1].lower()
|
||
return number / 1024 if unit == "m" else number
|
||
|
||
|
||
def leftovers() -> list[str]:
|
||
"""Команды уборки остатков другого (прерванного) прогона."""
|
||
label = f"label=com.docker.compose.project={PROJECT}"
|
||
names = set(docker("ps", "-a", "--format", "{{.Names}}").stdout.split())
|
||
project = docker("ps", "-aq", "--filter", label).stdout.split()
|
||
volumes = docker("volume", "ls", "-q", "--filter", label).stdout.split()
|
||
networks = docker("network", "ls", "--format", "{{.Name}}").stdout.split()
|
||
fixes = []
|
||
containers = sorted(set(project) | (names & set(NAMED)))
|
||
if containers:
|
||
fixes.append(f"docker rm -f {' '.join(containers)}")
|
||
if volumes:
|
||
fixes.append(f"docker volume rm {' '.join(volumes)}")
|
||
if NET in networks:
|
||
fixes.append(f"docker network rm {NET}")
|
||
return fixes
|
||
|
||
|
||
def lb_slots_from_csv(text: str) -> list[dict]:
|
||
"""Слоты backend api со статистикой HAProxy (адрес — без порта)."""
|
||
lines = [line for line in text.splitlines() if line.startswith(("# ", "api,"))]
|
||
if not lines or not lines[0].startswith("# "):
|
||
return []
|
||
header = lines[0][2:].split(",")
|
||
slots = []
|
||
for line in lines[1:]:
|
||
row = dict(zip(header, line.split(","), strict=False))
|
||
if row.get("svname") in ("BACKEND", "FRONTEND"):
|
||
continue
|
||
addr = row.get("addr", "").rsplit(":", 1)[0]
|
||
slots.append(
|
||
{"slot": row["svname"], "addr": addr, "status": row.get("status", "")}
|
||
)
|
||
return slots
|
||
|
||
|
||
class Stand:
|
||
"""Проект compose `vlad-bench`: хранилища, миграции, реплики, LB,
|
||
контейнеры генератора и сэмплера."""
|
||
|
||
def __init__(self, limits: Limits, *, session_ttl: int, workdir: Path) -> None:
|
||
self.limits = limits
|
||
self.workdir = workdir
|
||
host_env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
|
||
self.env = {
|
||
**host_env,
|
||
"POSTGRES_PASSWORD": secrets.token_hex(16),
|
||
"VALKEY_PASSWORD": secrets.token_hex(16),
|
||
"BENCH_IMAGE": SERVICE_TAG,
|
||
"BENCH_API_REPLICAS": str(limits.api_replicas),
|
||
"BENCH_API_CPUS": str(limits.api_cpus),
|
||
"BENCH_API_MEMORY": limits.api_memory,
|
||
"BENCH_DB_CPUS": str(limits.db_cpus),
|
||
"BENCH_DB_MEMORY": limits.db_memory,
|
||
"BENCH_DB_MAX_CONNECTIONS": str(limits.db_max_connections),
|
||
"BENCH_CACHE_CPUS": str(limits.cache_cpus),
|
||
"BENCH_CACHE_MEMORY": limits.cache_memory,
|
||
"BENCH_LB_CPUS": str(limits.lb_cpus),
|
||
"BENCH_LB_MEMORY": limits.lb_memory,
|
||
"BENCH_SESSION_TTL": str(session_ttl),
|
||
}
|
||
|
||
# --- compose и контейнеры ---
|
||
|
||
def compose_args(self) -> list[str]:
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return ["docker", "compose", "-p", PROJECT, "--env-file", str(ENV_FILE), *files]
|
||
|
||
def compose(self, *args: str, check: bool = True, timeout: float | None = 600):
|
||
"""docker compose стенда; без `-p vlad-bench` не вызывается."""
|
||
return sh(
|
||
[*self.compose_args(), *args], check=check, timeout=timeout, env=self.env
|
||
)
|
||
|
||
def script_env(self) -> dict[str, str]:
|
||
"""Окружение скриптов репозитория (rolling-update.sh): тот же
|
||
проект compose переменными, без `-p` в самом скрипте."""
|
||
return {
|
||
**self.env,
|
||
"COMPOSE_PROJECT_NAME": PROJECT,
|
||
"COMPOSE_FILE": os.pathsep.join(str(path) for path in COMPOSE_FILES),
|
||
"COMPOSE_ENV_FILES": str(ENV_FILE),
|
||
}
|
||
|
||
def start(self) -> None:
|
||
"""Свежий стенд: пустая база на новом volume, миграции, реплики, LB,
|
||
генератор. Готов, когда LB видит `UP` все реплики."""
|
||
self.compose("up", "-d", "--wait", "postgres", "valkey")
|
||
self.compose("run", "--rm", "migrate")
|
||
self.compose("up", "-d", "--no-deps", "--no-build", "api")
|
||
self.compose("up", "-d", "--no-deps", "--no-build", "haproxy")
|
||
self.wait_lb_up(self.limits.api_replicas)
|
||
docker(
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
GEN,
|
||
"--network",
|
||
NET,
|
||
"--cpus",
|
||
str(self.limits.gen_cpus),
|
||
"--memory",
|
||
self.limits.gen_memory,
|
||
"--memory-swap",
|
||
self.limits.gen_memory,
|
||
"--entrypoint",
|
||
"python",
|
||
base_image(),
|
||
"-c",
|
||
"import time; time.sleep(86400)",
|
||
)
|
||
# docker cp, а не bind-mount: не зависит от общих каталогов Docker Desktop
|
||
docker("cp", str(BENCH / "loadgen.py"), f"{GEN}:/tmp/loadgen.py")
|
||
ready = self.gen("ready", "--timeout", "60")
|
||
if not ready["ready"]:
|
||
raise StandError("API is not ready through the LB:\n" + self.app_tail())
|
||
|
||
def stop(self) -> None:
|
||
"""Генератор и сэмплер первыми: подключённые к сети чужие для compose
|
||
контейнеры не дали бы удалить сеть. Затем стенд с volume."""
|
||
docker("rm", "-f", GEN, SAMPLER, check=False)
|
||
self.compose("down", "-v", "--remove-orphans", check=False)
|
||
|
||
def api_ids(self, *, all_states: bool = False) -> list[str]:
|
||
args = (
|
||
["ps", "-a", "-q", "--no-trunc", "api"]
|
||
if all_states
|
||
else [
|
||
"ps",
|
||
"-q",
|
||
"--no-trunc",
|
||
"api",
|
||
]
|
||
)
|
||
return self.compose(*args).stdout.split()
|
||
|
||
def address(self, container: str) -> str:
|
||
return docker(
|
||
"inspect",
|
||
"-f",
|
||
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
|
||
container,
|
||
).stdout.strip()
|
||
|
||
def lb_slots(self) -> list[dict]:
|
||
out = docker("exec", LB, "bash", "-c", LB_STATS_CMD, check=False)
|
||
return lb_slots_from_csv(out.stdout) if out.returncode == 0 else []
|
||
|
||
def wait_lb_up(self, replicas: int, timeout: float = 90) -> None:
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
up = [slot for slot in self.lb_slots() if slot["status"] == "UP"]
|
||
if len(up) >= replicas:
|
||
return
|
||
time.sleep(0.5)
|
||
raise StandError(
|
||
f"LB does not see {replicas} replicas UP in {timeout:.0f}s:\n"
|
||
+ self.app_tail()
|
||
)
|
||
|
||
def wait_slot(self, address: str, status: str, timeout: float = 30) -> float | None:
|
||
"""Ждать состояния слота с адресом; секунды или None."""
|
||
started = time.monotonic()
|
||
while time.monotonic() - started < timeout:
|
||
for slot in self.lb_slots():
|
||
if slot["addr"] == address and slot["status"].startswith(status):
|
||
return time.monotonic() - started
|
||
time.sleep(0.25)
|
||
return None
|
||
|
||
def app_tail(self, lines: int = 30) -> str:
|
||
done = self.compose("logs", "--tail", str(lines), "api", "haproxy", check=False)
|
||
return done.stdout + done.stderr
|
||
|
||
def gen(self, *args: str, timeout: float = 3600) -> dict:
|
||
done = docker(
|
||
"exec",
|
||
GEN,
|
||
"python",
|
||
"/tmp/loadgen.py",
|
||
"--base-url",
|
||
LB_URL,
|
||
*args,
|
||
timeout=timeout,
|
||
)
|
||
return json.loads(done.stdout.strip().splitlines()[-1])
|
||
|
||
def gen_popen(self, *args: str) -> subprocess.Popen:
|
||
"""Генератор в фоне: оркестратор действует на стенд, пока идёт нагрузка."""
|
||
return subprocess.Popen(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
GEN,
|
||
"python",
|
||
"/tmp/loadgen.py",
|
||
"--base-url",
|
||
LB_URL,
|
||
*args,
|
||
],
|
||
cwd=ROOT,
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
)
|
||
|
||
def gen_file(self, path: str) -> str:
|
||
return docker("exec", GEN, "cat", path).stdout
|
||
|
||
def psql(self, sql: str) -> str:
|
||
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля."""
|
||
done = sh(
|
||
["docker", "exec", "-i", DB, "sh", "-c", PSQL], stdin=sql, check=False
|
||
)
|
||
if done.returncode != 0:
|
||
raise StandError(f"psql: exit {done.returncode}\n{done.stderr}")
|
||
return done.stdout.strip()
|
||
|
||
def maintain(self) -> float:
|
||
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды."""
|
||
started = time.monotonic()
|
||
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
|
||
return time.monotonic() - started
|
||
|
||
# --- сэмплер ---
|
||
|
||
def start_sampler(self, mode: str, containers: list[str] | None = None) -> None:
|
||
"""Сэмплер на образе приложения с `--cgroupns host` (bench/sampler.py)."""
|
||
args = [
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--cgroupns",
|
||
"host",
|
||
"--cpus",
|
||
str(self.limits.sampler_cpus),
|
||
"--memory",
|
||
self.limits.sampler_memory,
|
||
"--memory-swap",
|
||
self.limits.sampler_memory,
|
||
]
|
||
if mode == "list":
|
||
# пароль — только в окружении контейнера сэмплера
|
||
args += [
|
||
"--network",
|
||
NET,
|
||
"-e",
|
||
"PGUSER=bench",
|
||
"-e",
|
||
"PGDATABASE=bench",
|
||
"-e",
|
||
f"PGPASSWORD={self.env['POSTGRES_PASSWORD']}",
|
||
]
|
||
else:
|
||
args += ["--network", "none"]
|
||
docker(
|
||
*args,
|
||
"--entrypoint",
|
||
"python",
|
||
SERVICE_TAG,
|
||
"-c",
|
||
"import time; time.sleep(86400)",
|
||
)
|
||
docker("cp", str(BENCH / "sampler.py"), f"{SAMPLER}:/tmp/sampler.py")
|
||
command = [
|
||
"exec",
|
||
"-d",
|
||
SAMPLER,
|
||
"python",
|
||
"/tmp/sampler.py",
|
||
"--mode",
|
||
mode,
|
||
"--out",
|
||
"/tmp/samples.jsonl",
|
||
]
|
||
for container in containers or []:
|
||
command += ["--container", container]
|
||
docker(*command)
|
||
|
||
def sampler_lines(self) -> list[dict]:
|
||
done = docker("exec", SAMPLER, "cat", "/tmp/samples.jsonl", check=False)
|
||
return [json.loads(line) for line in done.stdout.splitlines() if line.strip()]
|
||
|
||
def stop_sampler(self) -> None:
|
||
docker("rm", "-f", SAMPLER, check=False)
|
||
|
||
# --- паспорт ---
|
||
|
||
def passport(self) -> dict:
|
||
"""Хранилища, лимиты и окружение API для паспорта; паролей здесь нет."""
|
||
settings = self.psql(
|
||
"SELECT name, current_setting(name) FROM pg_settings "
|
||
f"WHERE name = ANY(string_to_array('{','.join(PG_SETTINGS)}', ',')) "
|
||
"ORDER BY name;\n",
|
||
)
|
||
info = docker("exec", CACHE, "valkey-cli", "INFO", "server", check=False).stdout
|
||
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.MULTILINE)
|
||
api = self.api_ids()
|
||
env = json.loads(docker("inspect", "-f", "{{json .Config.Env}}", api[0]).stdout)
|
||
app_env = dict(
|
||
sorted(
|
||
item.split("=", 1)
|
||
for item in env
|
||
if re.match(
|
||
r"(DB|CACHE|SESSION|LOG|SHUTDOWN|KDF|IDEMPOTENCY|APP)_[A-Z_]*=",
|
||
item,
|
||
)
|
||
and "PASSWORD" not in item.split("=", 1)[0]
|
||
),
|
||
)
|
||
limits = {}
|
||
for role, name in (
|
||
("api", api[0]),
|
||
("postgres", DB),
|
||
("valkey", CACHE),
|
||
("haproxy", LB),
|
||
):
|
||
host = json.loads(
|
||
docker("inspect", "-f", "{{json .HostConfig}}", name).stdout
|
||
)
|
||
limits[role] = {
|
||
"nano_cpus": host.get("NanoCpus"),
|
||
"memory": host.get("Memory"),
|
||
"memory_swap": host.get("MemorySwap"),
|
||
"published_ports": host.get("PortBindings") or {},
|
||
}
|
||
|
||
def image(name: str) -> str:
|
||
return docker("inspect", "-f", "{{.Config.Image}}", name).stdout.strip()
|
||
|
||
return {
|
||
"postgres": {
|
||
"image": image(DB),
|
||
"version": self.psql("SHOW server_version;\n"),
|
||
"settings": dict(line.split("|", 1) for line in settings.splitlines()),
|
||
},
|
||
"valkey": {
|
||
"image": image(CACHE),
|
||
"version": valkey_version.group(1) if valkey_version else None,
|
||
},
|
||
"haproxy": {"image": image(LB)},
|
||
"limits": limits,
|
||
"api_replicas": len(api),
|
||
"app_env": app_env,
|
||
}
|
||
|
||
|
||
def new_label_dir(kind: str, label: str) -> Path:
|
||
if not LABEL_RE.fullmatch(label):
|
||
raise SystemExit(f"label must match {LABEL_RE.pattern}")
|
||
folder = EVIDENCE / kind / label
|
||
if folder.exists():
|
||
raise SystemExit(
|
||
f"{folder.relative_to(ROOT)} exists: labels are not overwritten"
|
||
)
|
||
folder.mkdir(parents=True)
|
||
return folder
|
||
|
||
|
||
def quick_label(kind: str) -> str:
|
||
stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
|
||
return f"quick-{kind}-{stamp}" if kind else f"quick-{stamp}"
|
||
|
||
|
||
def write_json(path: Path, data: dict) -> None:
|
||
path.write_text(
|
||
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
|
||
|
||
def preflight(limits_cpus: float, limits_memory_gib: float) -> dict:
|
||
"""Хост, Docker, бюджет лимитов, остатки прошлых прогонов."""
|
||
fixes = leftovers()
|
||
if fixes:
|
||
raise SystemExit(
|
||
"leftovers of an interrupted bench run; remove them first:\n "
|
||
+ "\n ".join(fixes)
|
||
)
|
||
host = host_info()
|
||
dock = docker_info()
|
||
budget = budget_check(limits_cpus, limits_memory_gib, dock)
|
||
return {
|
||
"host": host,
|
||
"docker": dock,
|
||
"budget": budget,
|
||
"warnings": host_warnings(host, dock),
|
||
}
|
||
|
||
|
||
def finish_host_state(
|
||
meta: dict, started_monotonic: float, started_wall: float
|
||
) -> None:
|
||
"""Сон ноутбука и выгрузка страниц за прогон — в паспорт и предупреждения."""
|
||
elapsed_wall = time.time() - started_wall
|
||
elapsed_mono = time.monotonic() - started_monotonic
|
||
end = host_state()
|
||
state = {
|
||
"slept_seconds": round(max(0.0, elapsed_wall - elapsed_mono), 1),
|
||
"swapouts_start": meta["host"].get("swapouts"),
|
||
"swapouts_end": end.get("swapouts"),
|
||
"power_end": end.get("power"),
|
||
}
|
||
if state["swapouts_start"] is not None and state["swapouts_end"] is not None:
|
||
state["swapouts_delta"] = state["swapouts_end"] - state["swapouts_start"]
|
||
if state["swapouts_delta"] > 0:
|
||
meta["host_warnings"].append(
|
||
f"host swapped out {state['swapouts_delta']} pages during the run"
|
||
)
|
||
if state["slept_seconds"] > 5:
|
||
meta["host_warnings"].append(
|
||
f"host slept {state['slept_seconds']}s during the run: rerun"
|
||
)
|
||
meta["host_state_end"] = state
|
||
|
||
|
||
def temp_workdir() -> Path:
|
||
return Path(tempfile.mkdtemp(prefix="vlad-bench-"))
|