Files
mtusi2026lab4/bench/common.py
T

1057 lines
40 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Общее для профилей стенда: docker, стенд compose, паспорт, хост.
Импортируют `run.py`, `compare.py` и `plots.py`; сам по себе не
запускается. Только стандартная библиотека.
Все вызовы compose — с `-p vlad-bench`, обоими `-f` и `--env-file
bench/bench.env`: рабочий стенд и его `.env` не затрагиваются. Пароли
хранилищ и ключи S3 генерируются на прогон и никуда не пишутся. Свои
контейнеры — `vlad-bench-*`; чужие стенд не останавливает и не удаляет.
S3 и уборщик (лаба 4) поднимают только профили `uploads` и `reconcile`
(`Stand.start(with_s3=True)`, разовые проходы `cleaner`); прочие профили
их не запускают.
"""
import dataclasses
import datetime
import fcntl
import json
import os
import platform
import re
import secrets
import shutil
import subprocess
import sys
import tempfile
import time
from contextlib import contextmanager
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BENCH = ROOT / "bench"
EVIDENCE = ROOT / "evidence"
IMAGE_REPO = "mtusi-messenger-vladimirceva"
# тег, на который смотрит сервис api стенда (BENCH_IMAGE); профиль rollout
# переставляет его со старого образа на новый
SERVICE_TAG = f"{IMAGE_REPO}:bench"
PROJECT = "vlad-bench"
NET = "vlad-bench-net"
DB = "vlad-bench-postgres"
CACHE = "vlad-bench-valkey"
LB = "vlad-bench-lb"
GEN = "vlad-bench-gen"
SAMPLER = "vlad-bench-sampler"
S3 = "vlad-bench-s3"
# загрузчик объектов профиля reconcile (bench/fill_objects.py)
FILL = "vlad-bench-fill"
# разовые проходы уборщика профиля reconcile: vlad-bench-reconcile-<n>
RECONCILE_PREFIX = "vlad-bench-reconcile-"
NAMED = (DB, CACHE, LB, GEN, SAMPLER, S3, FILL)
S3_VOLUME = f"{PROJECT}_s3data"
# секреты S3 стенда: compose требует их при разборе файла у всех профилей
S3_SECRETS = (
"S3_ROOT_ACCESS_KEY",
"S3_ROOT_SECRET_KEY",
"S3_JWT_SIGNING_KEY",
"S3_API_ACCESS_KEY",
"S3_API_SECRET_KEY",
"S3_CLEANER_ACCESS_KEY",
"S3_CLEANER_SECRET_KEY",
)
COMPOSE_FILES = (ROOT / "compose.yaml", BENCH / "bench.compose.yml")
ENV_FILE = BENCH / "bench.env"
LOCK = BENCH / ".bench.lock"
# что из окружения хоста нужно самому docker compose; остальное (в том
# числе POSTGRES_* из shell) в стенд не попадает
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "LANG", "DOCKER_HOST", "DOCKER_CONFIG")
LB_URL = "http://haproxy:8080"
LABEL_RE = re.compile(r"[A-Za-z0-9._-]{1,48}")
# фикстура генератора: токены пользователей стенда живут только в его контейнере
FIXTURE = "/tmp/bench-fixture.json"
# psql в контейнере PostgreSQL от пользователя образа, без пароля; SQL — stdin
PSQL = 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
# статистика HAProxy изнутри его контейнера (curl в образе нет, bash есть)
LB_STATS_CMD = (
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
)
PG_SETTINGS = (
"shared_buffers",
"max_connections",
"synchronous_commit",
"fsync",
"wal_level",
"checkpoint_timeout",
"max_wal_size",
"jit",
"password_encryption",
)
class StandError(Exception):
pass
def say(text: str) -> None:
stamp = datetime.datetime.now().strftime("%H:%M:%S")
print(f"[{stamp}] {text}", flush=True)
def utc_now() -> str:
return datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(
cmd: list[str],
*,
check: bool = True,
timeout: float | None = None,
env: dict[str, str] | None = None,
stdin: str | None = None,
):
done = subprocess.run(
cmd,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=timeout,
check=False,
cwd=ROOT,
env=env,
input=stdin,
)
if check and done.returncode != 0:
tail = (done.stderr or done.stdout).strip().splitlines()[-15:]
raise StandError(
f"{' '.join(cmd[:4])}... exit {done.returncode}\n" + "\n".join(tail)
)
return done
def docker(*args: str, check: bool = True, timeout: float | None = None):
return sh(["docker", *args], check=check, timeout=timeout)
@contextmanager
def bench_lock():
"""Один прогон стенда за раз: второй — код 2 (общие имена контейнеров)."""
LOCK.parent.mkdir(parents=True, exist_ok=True)
with LOCK.open("w") as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise SystemExit(
"another bench run holds bench/.bench.lock: wait for it to finish"
) from None
yield
# --- хост и паспорт ------------------------------------------------------------------
def cpu_model() -> str:
try:
if sys.platform == "darwin":
return sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
if sys.platform.startswith("linux"):
for line in Path("/proc/cpuinfo").read_text().splitlines():
if line.lower().startswith(("model name", "hardware")):
return line.split(":", 1)[1].strip()
except (OSError, StandError):
pass
return platform.processor() or "unknown"
def ram_gb() -> float | None:
try:
if sys.platform == "darwin":
return round(int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30, 1)
if sys.platform.startswith("linux"):
for line in Path("/proc/meminfo").read_text().splitlines():
if line.startswith("MemTotal:"):
return round(int(line.split()[1]) / 2**20, 1)
except (OSError, StandError, ValueError):
pass
return None
def swapouts() -> int | None:
"""Счётчик выгрузок страниц macOS (`vm_stat`): прирост за прогон — помеха."""
if sys.platform != "darwin":
return None
out = sh(["vm_stat"], check=False).stdout
match = re.search(r"Swapouts:\s+(\d+)", out)
return int(match.group(1)) if match else None
def host_state() -> dict:
"""Питание, swap и давление на память (macOS)."""
state: dict = {"swap_used_mib": None, "power": None, "memory_pressure": None}
if sys.platform != "darwin":
return state
swap = sh(["sysctl", "-n", "vm.swapusage"], check=False).stdout
match = re.search(r"used = ([\d.,]+)M", swap)
if match:
state["swap_used_mib"] = float(match.group(1).replace(",", "."))
batt = sh(["pmset", "-g", "batt"], check=False).stdout
match = re.search(r"drawing from '([^']+)'", batt)
if match:
state["power"] = match.group(1)
pressure = sh(["memory_pressure", "-Q"], check=False).stdout
match = re.search(r"free percentage: (\d+)%", pressure)
if match:
state["memory_free_percent"] = int(match.group(1))
state["swapouts"] = swapouts()
return state
def host_info() -> dict:
info = {
"platform": platform.platform(),
"machine": platform.machine(),
"cpu_model": cpu_model(),
"cpu_count": os.cpu_count(),
"ram_gb": ram_gb(),
**host_state(),
}
if hasattr(os, "getloadavg"):
info["loadavg_1m"] = round(os.getloadavg()[0], 2)
return info
def docker_info() -> dict:
raw = json.loads(docker("info", "--format", "{{json .}}").stdout)
keys = (
"ServerVersion",
"OperatingSystem",
"KernelVersion",
"Architecture",
"NCPU",
"MemTotal",
"CgroupVersion",
"CgroupDriver",
)
info = {key: raw.get(key) for key in keys}
compose = docker("compose", "version", "--short", check=False).stdout.strip()
info["compose_version"] = compose or None
names = docker("ps", "--format", "{{.Names}}").stdout.split()
info["other_running_containers"] = sorted(
name
for name in names
if not name.startswith(f"{PROJECT}-") and name not in NAMED
)
return info
def host_warnings(host: dict, dock: dict) -> list[str]:
"""Хост не в форме: метку переснять (bench/AGENTS.md)."""
warnings = []
if host.get("power") == "Battery Power":
warnings.append("host on battery power: CPU may be throttled")
free = host.get("memory_free_percent")
if free is not None and free < 20:
warnings.append(f"memory pressure: only {free}% free")
if dock["other_running_containers"]:
warnings.append(
f"{len(dock['other_running_containers'])} other container(s) running: "
"they share CPU with the stand (make stop on the working stand)",
)
return warnings
def budget_check(cpus: float, memory_gib: float, dock: dict) -> dict:
"""Сумма лимитов стенда и ВМ Docker: не помещается — отказ до стенда."""
vm_cpus = dock["NCPU"]
vm_gib = dock["MemTotal"] / 2**30
budget = {
"limits_cpus": round(cpus, 2),
"limits_memory_gib": round(memory_gib, 2),
"vm_cpus": vm_cpus,
"vm_memory_gib": round(vm_gib, 2),
}
if cpus > vm_cpus:
raise SystemExit(
f"limits need {cpus:.2f} CPU, docker VM has {vm_cpus}: raise CPUs in "
"Docker Desktop (Settings → Resources) or use a smaller series",
)
if memory_gib + 1.5 > vm_gib:
raise SystemExit(
f"limits need {memory_gib:.1f} GiB + 1.5 GiB reserve, docker VM has "
f"{vm_gib:.1f} GiB: raise memory in Docker Desktop",
)
return budget
def base_image() -> str:
"""Базовый образ из Dockerfile: генератор работает на нём же."""
for line in (ROOT / "Dockerfile").read_text(encoding="utf-8").splitlines():
if line.startswith("FROM "):
return line.split()[1]
raise StandError("no FROM line in Dockerfile")
# Выполняется внутри образа: версии, хеш кода и пул приложения — того кода,
# который реально измеряется
IMAGE_PROBE = """
import dataclasses, hashlib, importlib.metadata as m, json, os, pathlib, platform
from src.config import Database, Settings
root = pathlib.Path.cwd()
digest = hashlib.sha256()
for path in sorted([*(root / "src").rglob("*.py"), *(root / "migrations").rglob("*.py")]):
digest.update(path.relative_to(root).as_posix().encode())
digest.update(b"\\0")
digest.update(path.read_bytes())
digest.update(b"\\0")
packages = ("fastapi", "starlette", "uvicorn", "pydantic", "sqlalchemy", "asyncpg",
"alembic", "redis", "argon2-cffi", "aiobotocore", "botocore", "aiohttp")
db = Database()
app = Settings()
print(json.dumps({
"src_sha256": digest.hexdigest(),
"versions": {"python": platform.python_version(), **{p: m.version(p) for p in packages}},
"app_defaults": {
"pool_size": db.pool_size, "pool_overflow": db.pool_overflow,
"pool_timeout": db.pool_timeout, "operation_timeout": db.operation_timeout,
"statement_timeout_ms": db.statement_timeout_ms,
"kdf_concurrency": app.kdf_concurrency, "keep_alive": app.keep_alive,
"shutdown_timeout": app.shutdown_timeout,
"s3_timeout": app.s3.timeout, "s3_max_connections": app.s3.max_connections,
"uploads": dataclasses.asdict(app.uploads),
"cleaner": dataclasses.asdict(app.cleaner),
"malloc_mmap_threshold": os.environ.get("MALLOC_MMAP_THRESHOLD_"),
},
}))
"""
def build_image(tag: str, context: Path | None = None) -> None:
sh(["docker", "build", "-q", "-t", tag, str(context or ROOT)], timeout=1800)
def image_info(image: str) -> dict:
raw = json.loads(docker("image", "inspect", image).stdout)[0]
probe = docker("run", "--rm", "--entrypoint", "python", image, "-c", IMAGE_PROBE)
data = json.loads(probe.stdout.strip().splitlines()[-1])
return {"tag": image, "id": raw["Id"], "created": raw["Created"], **data}
def code_state(image: dict) -> dict:
"""Что измеряется: хеш кода из образа и, если есть git, commit."""
state = {"src_sha256": image["src_sha256"], "git_commit": None, "git_dirty": None}
if (ROOT / ".git").exists():
head = sh(["git", "rev-parse", "HEAD"], check=False)
if head.returncode == 0:
state["git_commit"] = head.stdout.strip()
status = sh(["git", "status", "--porcelain"], check=False).stdout
state["git_dirty"] = bool(status.strip())
return state
def probe_migration_context(workdir: Path) -> tuple[Path, str]:
"""Контекст сборки «нового» образа профиля rollout: файлы рабочего
дерева, которые копирует Dockerfile, плюс expand-ревизия
`9001_rollout_probe` — новая колонка `messages.rollout_probe` (NULL).
Ревизия существует только в этом образе и в базе стенда: так замена
под нагрузкой проходит с настоящей миграцией схемы, а рабочее дерево
и миграции репозитория не меняются. Возвращает каталог и ревизию head.
"""
context = workdir / "rollout-context"
for name in ("Dockerfile", "requirements.txt", "alembic.ini"):
target = context / name
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(ROOT / name, target)
for folder in ("src", "migrations"):
shutil.copytree(
ROOT / folder,
context / folder,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "AGENTS.md"),
)
revisions, parents = set(), set()
for path in (ROOT / "migrations" / "versions").glob("*.py"):
text = path.read_text(encoding="utf-8")
rev = re.search(r'^revision: str = "([^"]+)"', text, re.MULTILINE)
down = re.search(r'^down_revision: [^=]+= "([^"]+)"', text, re.MULTILINE)
if rev:
revisions.add(rev.group(1))
if down:
parents.add(down.group(1))
heads = revisions - parents
if len(heads) != 1:
raise StandError(f"cannot find a single migration head: {sorted(heads)}")
head = heads.pop()
(context / "migrations" / "versions" / "9001_rollout_probe.py").write_text(
PROBE_MIGRATION.replace("@HEAD@", head), encoding="utf-8"
)
return context, head
PROBE_MIGRATION = '''"""rollout probe: nullable column for the bench rollout profile
Revision ID: 9001
Revises: @HEAD@
Только в образе профиля rollout стенда bench/ (bench/common.py):
expand-миграция под нагрузкой. Колонка NULL, старый код её не замечает.
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "9001"
down_revision: str | Sequence[str] | None = "@HEAD@"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
# ADD COLUMN берёт ACCESS EXCLUSIVE: очередь за ним ограничена
op.execute("SET LOCAL lock_timeout = '2s'")
op.add_column("messages", sa.Column("rollout_probe", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("messages", "rollout_probe")
'''
# --- стенд ------------------------------------------------------------------------
@dataclasses.dataclass(frozen=True)
class Limits:
"""Лимиты контейнеров стенда и число реплик. Часть методики."""
api_replicas: int
api_cpus: float
api_memory: str
db_cpus: float = 2.0
db_memory: str = "1g"
db_max_connections: int = 100
cache_cpus: float = 0.5
cache_memory: str = "128m"
lb_cpus: float = 1.0
lb_memory: str = "256m"
gen_cpus: float = 2.0
gen_memory: str = "1g"
sampler_cpus: float = 0.25
sampler_memory: str = "256m"
# лаба 4: S3 и уборщик — лимиты рабочего стенда; в бюджет входят только
# у профилей, которые их запускают (with_s3)
with_s3: bool = False
s3_cpus: float = 1.0
s3_memory: str = "512m"
cleaner_cpus: float = 0.25
cleaner_memory: str = "256m"
def total_cpus(self, replicas: int | None = None) -> float:
replicas = self.api_replicas if replicas is None else replicas
return (
replicas * self.api_cpus
+ self.db_cpus
+ self.cache_cpus
+ self.lb_cpus
+ self.gen_cpus
+ self.sampler_cpus
+ (self.s3_cpus + self.cleaner_cpus if self.with_s3 else 0.0)
)
def total_memory_gib(self, replicas: int | None = None) -> float:
replicas = self.api_replicas if replicas is None else replicas
return (
replicas * gib(self.api_memory)
+ gib(self.db_memory)
+ gib(self.cache_memory)
+ gib(self.lb_memory)
+ gib(self.gen_memory)
+ gib(self.sampler_memory)
+ (gib(self.s3_memory) + gib(self.cleaner_memory) if self.with_s3 else 0.0)
)
def gib(value: str) -> float:
number, unit = float(value[:-1]), value[-1].lower()
return number / 1024 if unit == "m" else number
def bench_env(name: str) -> str:
"""Несекретная подстановка стенда из `bench/bench.env` (имя базы,
bucket): её же видит compose."""
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
if line.startswith(f"{name}="):
return line.split("=", 1)[1].strip()
raise StandError(f"{name} is not set in {ENV_FILE.relative_to(ROOT)}")
def leftovers() -> list[str]:
"""Команды уборки остатков другого (прерванного) прогона."""
label = f"label=com.docker.compose.project={PROJECT}"
names = set(docker("ps", "-a", "--format", "{{.Names}}").stdout.split())
project = docker("ps", "-aq", "--filter", label).stdout.split()
volumes = docker("volume", "ls", "-q", "--filter", label).stdout.split()
networks = docker("network", "ls", "--format", "{{.Name}}").stdout.split()
fixes = []
containers = sorted(set(project) | (names & set(NAMED)))
if containers:
fixes.append(f"docker rm -f {' '.join(containers)}")
if volumes:
fixes.append(f"docker volume rm {' '.join(volumes)}")
if NET in networks:
fixes.append(f"docker network rm {NET}")
return fixes
def lb_slots_from_csv(text: str) -> list[dict]:
"""Слоты backend api со статистикой HAProxy (адрес — без порта)."""
lines = [line for line in text.splitlines() if line.startswith(("# ", "api,"))]
if not lines or not lines[0].startswith("# "):
return []
header = lines[0][2:].split(",")
slots = []
for line in lines[1:]:
row = dict(zip(header, line.split(","), strict=False))
if row.get("svname") in ("BACKEND", "FRONTEND"):
continue
addr = row.get("addr", "").rsplit(":", 1)[0]
slots.append(
{"slot": row["svname"], "addr": addr, "status": row.get("status", "")}
)
return slots
class Stand:
"""Проект compose `vlad-bench`: хранилища, миграции, реплики, LB,
контейнеры генератора и сэмплера; с лабы 4 — S3 и уборщик."""
def __init__(self, limits: Limits, *, session_ttl: int, workdir: Path) -> None:
self.limits = limits
self.workdir = workdir
# override варианта профиля (окружение реплик uploads «без лимита»):
# третий -f у compose и скрипта замены
self.extra_files: list[Path] = []
host_env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
self.env = {
**host_env,
"POSTGRES_PASSWORD": secrets.token_hex(16),
"VALKEY_PASSWORD": secrets.token_hex(16),
**{name: secrets.token_hex(32) for name in S3_SECRETS},
"BENCH_S3_CPUS": str(limits.s3_cpus),
"BENCH_S3_MEMORY": limits.s3_memory,
"BENCH_CLEANER_CPUS": str(limits.cleaner_cpus),
"BENCH_CLEANER_MEMORY": limits.cleaner_memory,
"BENCH_IMAGE": SERVICE_TAG,
"BENCH_API_REPLICAS": str(limits.api_replicas),
"BENCH_API_CPUS": str(limits.api_cpus),
"BENCH_API_MEMORY": limits.api_memory,
"BENCH_DB_CPUS": str(limits.db_cpus),
"BENCH_DB_MEMORY": limits.db_memory,
"BENCH_DB_MAX_CONNECTIONS": str(limits.db_max_connections),
"BENCH_CACHE_CPUS": str(limits.cache_cpus),
"BENCH_CACHE_MEMORY": limits.cache_memory,
"BENCH_LB_CPUS": str(limits.lb_cpus),
"BENCH_LB_MEMORY": limits.lb_memory,
"BENCH_SESSION_TTL": str(session_ttl),
}
# --- compose и контейнеры ---
def files(self) -> list[Path]:
return [*COMPOSE_FILES, *self.extra_files]
def compose_args(self) -> list[str]:
files = [arg for path in self.files() for arg in ("-f", str(path))]
return ["docker", "compose", "-p", PROJECT, "--env-file", str(ENV_FILE), *files]
def compose(
self,
*args: str,
check: bool = True,
timeout: float | None = 600,
env: dict[str, str] | None = None,
):
"""docker compose стенда; без `-p vlad-bench` не вызывается. `env` —
поправки окружения вызова (лимит CPU разового прохода уборщика)."""
return sh(
[*self.compose_args(), *args],
check=check,
timeout=timeout,
env={**self.env, **(env or {})},
)
def script_env(self) -> dict[str, str]:
"""Окружение скриптов репозитория (rolling-update.sh): тот же
проект compose переменными, без `-p` в самом скрипте."""
return {
**self.env,
"COMPOSE_PROJECT_NAME": PROJECT,
"COMPOSE_FILE": os.pathsep.join(str(path) for path in self.files()),
"COMPOSE_ENV_FILES": str(ENV_FILE),
}
def start_s3(self) -> None:
"""S3 до healthy и bootstrap (bucket, пользователи API и уборщика,
проверки приватности); код bootstrap — код шага."""
self.compose("up", "-d", "--wait", "s3")
self.compose(
"up", "--no-deps", "--exit-code-from", "s3-bootstrap", "s3-bootstrap"
)
def reset_s3(self) -> None:
"""Новый S3 на пустом volume: прежний убивается (volume всё равно
удаляется), bootstrap — заново. К bucket и объектам оркестратор не
обращается."""
self.compose("rm", "-f", "-s", "-v", "s3-bootstrap", check=False)
self.compose("kill", "s3", check=False)
self.compose("rm", "-f", "-v", "s3", check=False)
docker("volume", "rm", "-f", S3_VOLUME, check=False)
self.start_s3()
def start(self, *, with_s3: bool = False) -> None:
"""Свежий стенд: пустая база на новом volume, миграции, реплики, LB,
генератор. Готов, когда LB видит `UP` все реплики. `with_s3` —
профили лабы 4: S3 и bootstrap до реплик, как у рабочего стенда."""
self.compose("up", "-d", "--wait", "postgres", "valkey")
if with_s3:
self.start_s3()
self.compose("run", "--rm", "migrate")
self.compose("up", "-d", "--no-deps", "--no-build", "api")
self.compose("up", "-d", "--no-deps", "--no-build", "haproxy")
self.wait_lb_up(self.limits.api_replicas)
docker(
"run",
"-d",
"--name",
GEN,
"--network",
NET,
"--cpus",
str(self.limits.gen_cpus),
"--memory",
self.limits.gen_memory,
"--memory-swap",
self.limits.gen_memory,
"--entrypoint",
"python",
base_image(),
"-c",
"import time; time.sleep(86400)",
)
# docker cp, а не bind-mount: не зависит от общих каталогов Docker Desktop
docker("cp", str(BENCH / "loadgen.py"), f"{GEN}:/tmp/loadgen.py")
ready = self.gen("ready", "--timeout", "60")
if not ready["ready"]:
raise StandError("API is not ready through the LB:\n" + self.app_tail())
def stop(self) -> None:
"""Генератор, сэмплер и загрузчик первыми: подключённые к сети чужие
для compose контейнеры не дали бы удалить сеть. Затем разовые
проходы уборщика и стенд с volume."""
docker("rm", "-f", GEN, SAMPLER, FILL, check=False)
passes = docker(
"ps", "-aq", "--filter", f"name=^{RECONCILE_PREFIX}", check=False
).stdout.split()
if passes:
docker("rm", "-f", *passes, check=False)
self.compose("down", "-v", "--remove-orphans", check=False)
def api_ids(self, *, all_states: bool = False) -> list[str]:
args = (
["ps", "-a", "-q", "--no-trunc", "api"]
if all_states
else [
"ps",
"-q",
"--no-trunc",
"api",
]
)
return self.compose(*args).stdout.split()
def address(self, container: str) -> str:
return docker(
"inspect",
"-f",
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
container,
).stdout.strip()
def lb_slots(self) -> list[dict]:
out = docker("exec", LB, "bash", "-c", LB_STATS_CMD, check=False)
return lb_slots_from_csv(out.stdout) if out.returncode == 0 else []
def wait_lb_up(self, replicas: int, timeout: float = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
up = [slot for slot in self.lb_slots() if slot["status"] == "UP"]
if len(up) >= replicas:
return
time.sleep(0.5)
raise StandError(
f"LB does not see {replicas} replicas UP in {timeout:.0f}s:\n"
+ self.app_tail()
)
def wait_slot(self, address: str, status: str, timeout: float = 30) -> float | None:
"""Ждать состояния слота с адресом; секунды или None."""
started = time.monotonic()
while time.monotonic() - started < timeout:
for slot in self.lb_slots():
if slot["addr"] == address and slot["status"].startswith(status):
return time.monotonic() - started
time.sleep(0.25)
return None
def app_tail(self, lines: int = 30) -> str:
done = self.compose("logs", "--tail", str(lines), "api", "haproxy", check=False)
return done.stdout + done.stderr
def gen(self, *args: str, timeout: float = 3600) -> dict:
done = docker(
"exec",
GEN,
"python",
"/tmp/loadgen.py",
"--base-url",
LB_URL,
*args,
timeout=timeout,
)
return json.loads(done.stdout.strip().splitlines()[-1])
def gen_popen(self, *args: str) -> subprocess.Popen:
"""Генератор в фоне: оркестратор действует на стенд, пока идёт нагрузка."""
return subprocess.Popen(
[
"docker",
"exec",
GEN,
"python",
"/tmp/loadgen.py",
"--base-url",
LB_URL,
*args,
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
def gen_file(self, path: str) -> str:
return docker("exec", GEN, "cat", path).stdout
def psql(self, sql: str) -> str:
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля."""
done = sh(
["docker", "exec", "-i", DB, "sh", "-c", PSQL], stdin=sql, check=False
)
if done.returncode != 0:
raise StandError(f"psql: exit {done.returncode}\n{done.stderr}")
return done.stdout.strip()
def maintain(self) -> float:
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды."""
started = time.monotonic()
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
return time.monotonic() - started
# --- сэмплер ---
def start_sampler(
self, mode: str, containers: list[str] | None = None, interval: float = 1.0
) -> None:
"""Сэмплер на образе приложения с `--cgroupns host` (bench/sampler.py)."""
args = [
"run",
"-d",
"--name",
SAMPLER,
"--cgroupns",
"host",
"--cpus",
str(self.limits.sampler_cpus),
"--memory",
self.limits.sampler_memory,
"--memory-swap",
self.limits.sampler_memory,
]
if mode == "list":
# пароль — только в окружении контейнера сэмплера
args += [
"--network",
NET,
"-e",
"PGUSER=bench",
"-e",
"PGDATABASE=bench",
"-e",
f"PGPASSWORD={self.env['POSTGRES_PASSWORD']}",
]
else:
args += ["--network", "none"]
docker(
*args,
"--entrypoint",
"python",
SERVICE_TAG,
"-c",
"import time; time.sleep(86400)",
)
docker("cp", str(BENCH / "sampler.py"), f"{SAMPLER}:/tmp/sampler.py")
command = [
"exec",
"-d",
SAMPLER,
"python",
"/tmp/sampler.py",
"--mode",
mode,
"--interval",
f"{interval:g}",
"--out",
"/tmp/samples.jsonl",
]
for container in containers or []:
command += ["--container", container]
docker(*command)
def sampler_lines(self, path: str = "/tmp/samples.jsonl") -> list[dict]:
done = docker("exec", SAMPLER, "cat", path, check=False)
return [json.loads(line) for line in done.stdout.splitlines() if line.strip()]
def stop_sampler(self) -> None:
docker("rm", "-f", SAMPLER, check=False)
def start_peak_sampler(self, targets: dict[str, str], interval: float) -> None:
"""Сэмплер прогона профиля uploads в режиме `peak`: `--user 0` и
cgroupfs на запись (сброс `memory.peak` через свой дескриптор),
`--network none`, без паролей. Готов, когда записан итог сброса.
`targets` — роль → полный ID контейнера.
"""
docker(
"run",
"-d",
"--name",
SAMPLER,
"--user",
"0",
"--cgroupns",
"host",
"--network",
"none",
# проверка образа приложения — лишний процесс в квоте сэмплера
"--no-healthcheck",
"--cpus",
str(self.limits.sampler_cpus),
"--memory",
self.limits.sampler_memory,
"--memory-swap",
self.limits.sampler_memory,
"-v",
"/sys/fs/cgroup:/sys/fs/cgroup:rw",
"--entrypoint",
"python",
SERVICE_TAG,
"-c",
"import time; time.sleep(86400)",
)
docker("cp", str(BENCH / "sampler.py"), f"{SAMPLER}:/tmp/sampler.py")
command = [
"exec",
"-d",
SAMPLER,
"python",
"/tmp/sampler.py",
"--mode",
"peak",
"--interval",
f"{interval:g}",
"--out",
"/tmp/peak.jsonl",
"--stop-file",
"/tmp/peak.stop",
]
for role, container in targets.items():
command += ["--container", f"{role}={container}"]
docker(*command)
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
lines = self.sampler_lines("/tmp/peak.jsonl")
if lines and "reset" in lines[0]:
return
time.sleep(0.2)
raise StandError("peak sampler did not reset memory.peak in 10 s")
def stop_peak_sampler(self) -> list[dict]:
"""Пик через тот же дескриптор и строка `end`; затем контейнер
удаляется. Строки — `sampler.py`, режим `peak`."""
docker("exec", SAMPLER, "touch", "/tmp/peak.stop", check=False)
deadline = time.monotonic() + 10
lines: list[dict] = []
while time.monotonic() < deadline:
lines = self.sampler_lines("/tmp/peak.jsonl")
if lines and "end" in lines[-1]:
break
time.sleep(0.2)
self.stop_sampler()
return lines
# --- паспорт ---
def passport(self) -> dict:
"""Хранилища, лимиты и окружение API для паспорта; паролей здесь нет."""
settings = self.psql(
"SELECT name, current_setting(name) FROM pg_settings "
f"WHERE name = ANY(string_to_array('{','.join(PG_SETTINGS)}', ',')) "
"ORDER BY name;\n",
)
info = docker("exec", CACHE, "valkey-cli", "INFO", "server", check=False).stdout
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.MULTILINE)
api = self.api_ids()
env = json.loads(docker("inspect", "-f", "{{json .Config.Env}}", api[0]).stdout)
app_env = dict(
sorted(
item.split("=", 1)
for item in env
if re.match(
r"(DB|CACHE|SESSION|LOG|SHUTDOWN|KDF|IDEMPOTENCY|APP|S3|UPLOAD"
r"|MALLOC)_[A-Z_]*=",
item,
)
and not any(
mark in item.split("=", 1)[0]
for mark in ("PASSWORD", "SECRET", "ACCESS_KEY")
)
),
)
# буфер приёма сокетов реплики (sysctls compose.yaml): тела загрузок
# копятся в нём до чтения
rmem = docker(
"exec", api[0], "cat", "/proc/sys/net/ipv4/tcp_rmem", check=False
).stdout.split()
s3_running = (
docker(
"inspect", "-f", "{{.State.Running}}", S3, check=False
).stdout.strip()
== "true"
)
roles = [("api", api[0]), ("postgres", DB), ("valkey", CACHE), ("haproxy", LB)]
if s3_running:
roles.append(("s3", S3))
limits = {}
for role, name in roles:
host = json.loads(
docker("inspect", "-f", "{{json .HostConfig}}", name).stdout
)
limits[role] = {
"nano_cpus": host.get("NanoCpus"),
"memory": host.get("Memory"),
"memory_swap": host.get("MemorySwap"),
"published_ports": host.get("PortBindings") or {},
}
def image(name: str) -> str:
return docker("inspect", "-f", "{{.Config.Image}}", name).stdout.strip()
passport = {
"postgres": {
"image": image(DB),
"version": self.psql("SHOW server_version;\n"),
"settings": dict(line.split("|", 1) for line in settings.splitlines()),
},
"valkey": {
"image": image(CACHE),
"version": valkey_version.group(1) if valkey_version else None,
},
"haproxy": {"image": image(LB)},
"limits": limits,
"api_replicas": len(api),
"app_env": app_env,
"api_tcp_rmem": " ".join(rmem) or None,
}
if s3_running:
command = json.loads(
docker("inspect", "-f", "{{json .Config.Cmd}}", S3).stdout
)
passport["s3"] = {"image": image(S3), "command": command}
return passport
def new_label_dir(kind: str, label: str) -> Path:
if not LABEL_RE.fullmatch(label):
raise SystemExit(f"label must match {LABEL_RE.pattern}")
folder = EVIDENCE / kind / label
if folder.exists():
raise SystemExit(
f"{folder.relative_to(ROOT)} exists: labels are not overwritten"
)
folder.mkdir(parents=True)
return folder
def quick_label(kind: str) -> str:
stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
return f"quick-{kind}-{stamp}" if kind else f"quick-{stamp}"
def write_json(path: Path, data: dict) -> None:
path.write_text(
json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
def preflight(limits_cpus: float, limits_memory_gib: float) -> dict:
"""Хост, Docker, бюджет лимитов, остатки прошлых прогонов."""
fixes = leftovers()
if fixes:
raise SystemExit(
"leftovers of an interrupted bench run; remove them first:\n "
+ "\n ".join(fixes)
)
host = host_info()
dock = docker_info()
budget = budget_check(limits_cpus, limits_memory_gib, dock)
return {
"host": host,
"docker": dock,
"budget": budget,
"warnings": host_warnings(host, dock),
}
def finish_host_state(
meta: dict, started_monotonic: float, started_wall: float
) -> None:
"""Сон ноутбука и выгрузка страниц за прогон — в паспорт и предупреждения."""
elapsed_wall = time.time() - started_wall
elapsed_mono = time.monotonic() - started_monotonic
end = host_state()
state = {
"slept_seconds": round(max(0.0, elapsed_wall - elapsed_mono), 1),
"swapouts_start": meta["host"].get("swapouts"),
"swapouts_end": end.get("swapouts"),
"power_end": end.get("power"),
}
if state["swapouts_start"] is not None and state["swapouts_end"] is not None:
state["swapouts_delta"] = state["swapouts_end"] - state["swapouts_start"]
if state["swapouts_delta"] > 0:
meta["host_warnings"].append(
f"host swapped out {state['swapouts_delta']} pages during the run"
)
if state["slept_seconds"] > 5:
meta["host_warnings"].append(
f"host slept {state['slept_seconds']}s during the run: rerun"
)
meta["host_state_end"] = state
def temp_workdir() -> Path:
return Path(tempfile.mkdtemp(prefix="vlad-bench-"))