Copy the committed lab 1 sources as-is: src/, student-tests/, bench/, Dockerfile, docker-compose.yml, pyproject.toml, uv.lock, .python-version, config_example.toml, AGENTS.md, CLAUDE.md, .claude/, openspec/, .graphifyignore and KNOWN_ISSUES.md. Lab 1 evidence stays in the lab 1 repository. Merge the new skeleton files with own sections: Makefile (lab 2 targets plus own run, docker, bench and common targets), .gitignore, .dockerignore and .env.example. Rename the image to mtusi-messenger:lab2 and point AGENTS.md at lab 2. The application is still the in-memory lab 1 implementation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1274 lines
44 KiB
Python
1274 lines
44 KiB
Python
"""Нагрузочный стенд трека 1: матрица размеров истории и конкуренции.
|
||
|
||
Запуск — `make bench` / `make bench_quick`. Методика — design change
|
||
`load-bench-harness`, инварианты — `.claude/rules/bench.md`. Только
|
||
stdlib: приложение не импортируется, всё через docker CLI и k6.
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import shutil
|
||
import signal
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
from dataclasses import asdict, dataclass, replace
|
||
from datetime import UTC, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
K6_IMAGE = (
|
||
"grafana/k6:2.3.0@sha256:"
|
||
"9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34"
|
||
)
|
||
APP_IMAGE = "mtusi-messenger:bench"
|
||
NETWORK = "mtusi-bench"
|
||
APP = "mtusi-bench-api"
|
||
K6 = "mtusi-bench-k6"
|
||
READY_CHECK = (
|
||
"import urllib.request; urllib.request.urlopen("
|
||
"'http://127.0.0.1:8080/health/ready', timeout=2)"
|
||
)
|
||
SUCCESS_METRIC = "http_req_duration{expected_response:true}"
|
||
# предупреждение о хосте: swap занят больше, МиБ
|
||
SWAP_WARN_MIB = 1024
|
||
|
||
RUN_FIELDS = [
|
||
"run",
|
||
"scenario",
|
||
"size",
|
||
"concurrency",
|
||
"repeat",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_start_mib",
|
||
"rss_end_mib",
|
||
"rss_max_mib",
|
||
"mem_cgroup_end_mib",
|
||
"store_messages_start",
|
||
"store_messages_end",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"check_failed",
|
||
]
|
||
PREFILL_FIELDS = [
|
||
"size",
|
||
"chunk",
|
||
"messages_before",
|
||
"messages_after",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_end_mib",
|
||
"mem_cgroup_end_mib",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Profile:
|
||
name: str
|
||
sizes: tuple[int, ...]
|
||
concurrency: tuple[int, ...]
|
||
repeats: int
|
||
read_warmup_s: int
|
||
read_measure_s: int
|
||
write_warmup: int
|
||
write_measure: int
|
||
prefill_chunk: int = 50_000
|
||
prefill_vus: int = 32
|
||
text_length: int = 64
|
||
page_limit: int = 50
|
||
walk_limit: int = 100
|
||
cursor_pool: int = 1000
|
||
members: int = 10
|
||
seed: int = 20260930
|
||
# зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно
|
||
# те страницы, с которых начнётся замер
|
||
warmup_seed_shift: int = 1_000_003
|
||
generator_cpu_limit: float = 4.0
|
||
cpus: str = "1"
|
||
memory: str = "2g"
|
||
|
||
|
||
PROFILES = {
|
||
"full": Profile(
|
||
name="full",
|
||
sizes=(10_000, 100_000, 1_000_000),
|
||
concurrency=(1, 10, 50),
|
||
repeats=3,
|
||
read_warmup_s=5,
|
||
read_measure_s=20,
|
||
write_warmup=500,
|
||
write_measure=5000,
|
||
),
|
||
"quick": Profile(
|
||
name="quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(1, 4, 16),
|
||
repeats=1,
|
||
read_warmup_s=2,
|
||
read_measure_s=5,
|
||
write_warmup=100,
|
||
write_measure=1000,
|
||
),
|
||
}
|
||
|
||
|
||
class BenchError(Exception):
|
||
"""Стенд не может продолжать: сообщение для человека."""
|
||
|
||
|
||
# ---------------------------------------------------------------- утилиты
|
||
|
||
|
||
def sh(*args: str, check: bool = True) -> str:
|
||
"""Команда без shell; stdout без хвостовых пробелов.
|
||
|
||
`LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод
|
||
разбирается регулярками с точкой.
|
||
"""
|
||
proc = subprocess.run(
|
||
args,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
env={**os.environ, "LC_ALL": "C"},
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"{' '.join(args[:4])}…: exit {proc.returncode}\n"
|
||
f"{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
|
||
def human(n: int) -> str:
|
||
"""10000 → 10k, 1000000 → 1M."""
|
||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||
return f"{n // 1_000_000}M"
|
||
if n >= 1000 and n % 1000 == 0:
|
||
return f"{n // 1000}k"
|
||
return str(n)
|
||
|
||
|
||
def fmt_left(seconds: float) -> str:
|
||
seconds = max(0, int(seconds))
|
||
hours, rest = divmod(seconds, 3600)
|
||
minutes, secs = divmod(rest, 60)
|
||
if hours:
|
||
return f"{hours}h{minutes:02d}m"
|
||
if minutes:
|
||
return f"{minutes}m{secs:02d}s"
|
||
return f"{secs}s"
|
||
|
||
|
||
def mib(kib: float | None) -> float | None:
|
||
return None if kib is None else round(kib / 1024, 1)
|
||
|
||
|
||
def mib_bytes(value: int | None) -> float | None:
|
||
return None if value is None else round(value / 2**20, 1)
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 0) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 3) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
# ------------------------------------------------------------ хост и паспорт
|
||
|
||
|
||
def host_state() -> dict[str, Any]:
|
||
"""Swap, свободная память и питание (macOS); на другой ОС — null."""
|
||
state: dict[str, Any] = {
|
||
"swap_used_mib": None,
|
||
"swap_total_mib": None,
|
||
"memory_free_percent": None,
|
||
"power": None,
|
||
"battery_percent": None,
|
||
}
|
||
if platform.system() != "Darwin":
|
||
return state
|
||
swap = sh("sysctl", "-n", "vm.swapusage", check=False)
|
||
for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")):
|
||
match = re.search(rf"{key} = ([\d.,]+)M", swap)
|
||
if match:
|
||
state[name] = float(match.group(1).replace(",", "."))
|
||
pressure = sh("memory_pressure", check=False)
|
||
match = re.search(r"free percentage: (\d+)%", pressure)
|
||
if match:
|
||
state["memory_free_percent"] = int(match.group(1))
|
||
batt = sh("pmset", "-g", "batt", check=False)
|
||
match = re.search(r"drawing from '([^']+)'", batt)
|
||
if match:
|
||
state["power"] = match.group(1)
|
||
match = re.search(r"(\d+)%;", batt)
|
||
if match:
|
||
state["battery_percent"] = int(match.group(1))
|
||
return state
|
||
|
||
|
||
def host_warnings(state: dict[str, Any]) -> list[str]:
|
||
warnings = []
|
||
if state["power"] == "Battery Power":
|
||
warnings.append("host on battery power: CPU may be throttled")
|
||
used = state["swap_used_mib"]
|
||
if used is not None and used > SWAP_WARN_MIB:
|
||
warnings.append(
|
||
f"swap used {used:.0f} MiB > {SWAP_WARN_MIB} MiB: "
|
||
"reboot before a full run",
|
||
)
|
||
return warnings
|
||
|
||
|
||
def git_info() -> dict[str, Any]:
|
||
commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD")
|
||
status = sh("git", "-C", str(ROOT), "status", "--porcelain")
|
||
return {"commit": commit, "dirty": bool(status)}
|
||
|
||
|
||
def static_passport(profile: Profile) -> dict[str, Any]:
|
||
docker = json.loads(sh("docker", "info", "--format", "{{json .}}"))
|
||
versions = sh(
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
"-c",
|
||
"import importlib.metadata as m, json, platform; print(json.dumps("
|
||
"{'python': platform.python_version(), **{p: m.version(p) for p in "
|
||
"('fastapi', 'starlette', 'pydantic', 'uvicorn')}}))",
|
||
)
|
||
k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version")
|
||
host: dict[str, Any] = {"os": platform.system(), "machine": None}
|
||
if platform.system() == "Darwin":
|
||
host["os"] = f"macOS {platform.mac_ver()[0]}"
|
||
host["machine"] = sh(
|
||
"sysctl",
|
||
"-n",
|
||
"machdep.cpu.brand_string",
|
||
check=False,
|
||
)
|
||
memsize = sh("sysctl", "-n", "hw.memsize", check=False)
|
||
host["memory_gib"] = (
|
||
round(int(memsize) / 2**30, 1) if memsize.isdigit() else None
|
||
)
|
||
return {
|
||
"app_image": {
|
||
"tag": APP_IMAGE,
|
||
"id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE),
|
||
"versions": json.loads(versions),
|
||
},
|
||
"generator": {"image": K6_IMAGE, "version": k6_version},
|
||
"docker": {
|
||
"server": docker.get("ServerVersion"),
|
||
"vm_os": docker.get("OperatingSystem"),
|
||
"vm_cpus": docker.get("NCPU"),
|
||
"vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1),
|
||
"cgroup": docker.get("CgroupVersion"),
|
||
"logging_driver": docker.get("LoggingDriver"),
|
||
},
|
||
"host": host,
|
||
"profile": asdict(profile),
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------------------ стенд
|
||
|
||
|
||
class Watcher:
|
||
"""Раз в секунду, пока идёт запуск k6: VmRSS приложения и жив ли оно.
|
||
|
||
Приложение умерло (OOM) — k6 останавливается: иначе каждый его запрос
|
||
ждал бы таймаута, и наполнение тянулось бы часами.
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self.samples: list[int] = []
|
||
self.app_died = False
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(target=self._loop, daemon=True)
|
||
|
||
def _loop(self) -> None:
|
||
while not self._stop.wait(1.0):
|
||
out = sh(
|
||
"docker",
|
||
"exec",
|
||
APP,
|
||
"grep",
|
||
"^VmRSS:",
|
||
"/proc/1/status",
|
||
check=False,
|
||
)
|
||
match = re.search(r"(\d+) kB", out)
|
||
if match:
|
||
self.samples.append(int(match.group(1)))
|
||
continue
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
APP,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
self.app_died = True
|
||
sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False)
|
||
return
|
||
|
||
def __enter__(self) -> "Watcher":
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *_: object) -> None:
|
||
self._stop.set()
|
||
self._thread.join()
|
||
|
||
|
||
class Stand:
|
||
"""Сеть, контейнер приложения и контейнер k6 на время одного размера."""
|
||
|
||
def __init__(self, profile: Profile, label_dir: Path) -> None:
|
||
self.profile = profile
|
||
self.label_dir = label_dir
|
||
self.work_dir = label_dir / ".work"
|
||
|
||
def check_free(self) -> None:
|
||
names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split()
|
||
busy = [name for name in (APP, K6) if name in names]
|
||
if busy:
|
||
raise BenchError(
|
||
f"containers {', '.join(busy)} already exist: another bench "
|
||
"is running or a crashed one left them; remove with "
|
||
f"docker rm -f {' '.join(busy)}",
|
||
)
|
||
|
||
def build_image(self) -> str:
|
||
sh(
|
||
"docker",
|
||
"build",
|
||
"-q",
|
||
"-t",
|
||
APP_IMAGE,
|
||
str(ROOT),
|
||
)
|
||
return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE)
|
||
|
||
def start(self) -> None:
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
sh("docker", "network", "create", NETWORK)
|
||
sh(
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
APP,
|
||
"--network",
|
||
NETWORK,
|
||
f"--cpus={self.profile.cpus}",
|
||
f"--memory={self.profile.memory}",
|
||
f"--memory-swap={self.profile.memory}",
|
||
"--no-healthcheck",
|
||
APP_IMAGE,
|
||
)
|
||
sh(
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
K6,
|
||
"--network",
|
||
NETWORK,
|
||
"-v",
|
||
f"{ROOT / 'bench' / 'k6'}:/scripts:ro",
|
||
"-v",
|
||
f"{self.label_dir}:/out",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"sleep",
|
||
K6_IMAGE,
|
||
"infinity",
|
||
)
|
||
self.wait_ready()
|
||
|
||
def wait_ready(self, timeout: float = 60) -> None:
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
probe = subprocess.run(
|
||
["docker", "exec", APP, "python", "-c", READY_CHECK],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if probe.returncode == 0:
|
||
return
|
||
state = self.app_state()
|
||
if not state["running"]:
|
||
reason = "oom" if state["oom"] else "exited"
|
||
raise BenchError(f"{APP} died during start ({reason})")
|
||
time.sleep(0.25)
|
||
raise BenchError(f"{APP} not ready in {timeout:.0f}s")
|
||
|
||
def stop(self) -> None:
|
||
subprocess.run(
|
||
["docker", "rm", "-f", APP, K6],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
subprocess.run(
|
||
["docker", "network", "rm", NETWORK],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
def limits(self) -> dict[str, Any]:
|
||
host = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .HostConfig}}", APP),
|
||
)
|
||
config = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config}}", APP),
|
||
)
|
||
return {
|
||
"nano_cpus": host.get("NanoCpus"),
|
||
"memory": host.get("Memory"),
|
||
"memory_swap": host.get("MemorySwap"),
|
||
"published_ports": host.get("PortBindings") or {},
|
||
"healthcheck": (config.get("Healthcheck") or {}).get("Test"),
|
||
"log_driver": (host.get("LogConfig") or {}).get("Type"),
|
||
}
|
||
|
||
def app_state(self) -> dict[str, Any]:
|
||
state = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .State}}", APP),
|
||
)
|
||
restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", APP)
|
||
return {
|
||
"running": state.get("Running"),
|
||
"oom": state.get("OOMKilled"),
|
||
"restarts": int(restarts or 0),
|
||
}
|
||
|
||
def snapshot(self) -> dict[str, Any]:
|
||
"""CPU и память приложения, CPU генератора.
|
||
|
||
Если приложение уже мертво (OOM), его поля — None: снимок нужен и
|
||
для записи проваленного прогона.
|
||
"""
|
||
app = sh(
|
||
"docker",
|
||
"exec",
|
||
APP,
|
||
"sh",
|
||
"-c",
|
||
"cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; "
|
||
"grep -E '^(VmRSS|VmHWM):' /proc/1/status",
|
||
check=False,
|
||
)
|
||
k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat")
|
||
|
||
def number(pattern: str, text: str) -> int | None:
|
||
match = re.search(pattern, text, re.MULTILINE)
|
||
return int(match.group(1)) if match else None
|
||
|
||
return {
|
||
"app_usec": number(r"^usage_usec (\d+)$", app),
|
||
"mem_bytes": number(r"^(\d+)$", app),
|
||
"rss_kib": number(r"^VmRSS:\s+(\d+) kB", app),
|
||
"hwm_kib": number(r"^VmHWM:\s+(\d+) kB", app),
|
||
"k6_usec": number(r"^usage_usec (\d+)$", k6),
|
||
}
|
||
|
||
def k6(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
*,
|
||
check: bool = True,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
common = {
|
||
"TEXT_LENGTH": self.profile.text_length,
|
||
"PAGE_LIMIT": self.profile.page_limit,
|
||
}
|
||
flags = [
|
||
item
|
||
for key, value in {**common, **env}.items()
|
||
for item in ("-e", f"{key}={value}")
|
||
]
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
K6,
|
||
"k6",
|
||
"run",
|
||
"--quiet",
|
||
"--no-color",
|
||
"--log-format",
|
||
"raw",
|
||
*flags,
|
||
f"/scripts/{script}",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(k6_failure(script, proc))
|
||
return proc
|
||
|
||
def work_json(self, name: str) -> Any:
|
||
return json.loads((self.work_dir / name).read_text(encoding="utf-8"))
|
||
|
||
|
||
# -------------------------------------------------------------- измерение
|
||
|
||
|
||
def parse_summary(path: Path) -> dict[str, Any]:
|
||
"""Итог k6: успехи, ошибки, RPS и перцентили успешных запросов.
|
||
|
||
Нет файла (k6 остановлен сторожем до итога) — нулевой итог.
|
||
"""
|
||
if not path.exists():
|
||
return {
|
||
"duration_s": 0,
|
||
"requests_ok": 0,
|
||
"requests_failed": 0,
|
||
"check_failed": 0,
|
||
"rps": None,
|
||
"error_ratio": None,
|
||
"p50_ms": None,
|
||
"p95_ms": None,
|
||
"p99_ms": None,
|
||
"max_ms": None,
|
||
}
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
metrics = data["metrics"]
|
||
failed = metrics.get("http_req_failed", {}).get("values", {})
|
||
ok = int(failed.get("fails", 0))
|
||
bad = int(failed.get("passes", 0))
|
||
checks = metrics.get("checks", {}).get("values", {})
|
||
success = metrics.get(SUCCESS_METRIC, {}).get("values", {})
|
||
duration = data["state"]["testRunDurationMs"] / 1000
|
||
return {
|
||
"duration_s": round(duration, 3),
|
||
"requests_ok": ok,
|
||
"requests_failed": bad,
|
||
"check_failed": int(checks.get("fails", 0)),
|
||
"rps": round(ok / duration, 1) if duration else None,
|
||
"error_ratio": round(bad / (ok + bad), 6) if ok + bad else None,
|
||
"p50_ms": rounded(success.get("p(50)")),
|
||
"p95_ms": rounded(success.get("p(95)")),
|
||
"p99_ms": rounded(success.get("p(99)")),
|
||
"max_ms": rounded(success.get("max")),
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class Measured:
|
||
summary: dict[str, Any]
|
||
before: dict[str, Any]
|
||
after: dict[str, Any]
|
||
rss_max_kib: int | None
|
||
|
||
def cores(self, key: str) -> float | None:
|
||
"""Среднее число занятых ядер за прогон по счётчику cgroup."""
|
||
duration = self.summary["duration_s"]
|
||
start, end = self.before[key], self.after[key]
|
||
if not duration or start is None or end is None:
|
||
return None
|
||
return round((end - start) / 1e6 / duration, 3)
|
||
|
||
@property
|
||
def app_cpu(self) -> float | None:
|
||
return self.cores("app_usec")
|
||
|
||
@property
|
||
def generator_cpu(self) -> float | None:
|
||
return self.cores("k6_usec")
|
||
|
||
|
||
def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str:
|
||
tail = (proc.stdout + proc.stderr)[-3000:]
|
||
return f"k6 {script} failed ({proc.returncode}):\n{tail}"
|
||
|
||
|
||
def measure(
|
||
stand: Stand,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
raw: Path,
|
||
) -> Measured:
|
||
"""Один запуск k6 со снимками до и после, опросом RSS и сторожем."""
|
||
before = stand.snapshot()
|
||
with Watcher() as watcher:
|
||
proc = stand.k6(script, env, check=False)
|
||
# если приложение умерло, k6 остановлен сторожем: прогон пишется с
|
||
# пометкой, а не падает здесь
|
||
if proc.returncode != 0 and not watcher.app_died:
|
||
raise BenchError(k6_failure(script, proc))
|
||
after = stand.snapshot()
|
||
values = [
|
||
v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v
|
||
]
|
||
return Measured(
|
||
summary=parse_summary(raw),
|
||
before=before,
|
||
after=after,
|
||
rss_max_kib=max(values) if values else None,
|
||
)
|
||
|
||
|
||
def run_flags(
|
||
profile: Profile,
|
||
measured: Measured,
|
||
app: dict[str, Any],
|
||
) -> list[str]:
|
||
flags = []
|
||
if app["oom"]:
|
||
flags.append("oom")
|
||
if not app["running"] or app["restarts"]:
|
||
flags.append("app_down")
|
||
summary = measured.summary
|
||
if summary["requests_failed"] or summary["check_failed"]:
|
||
flags.append("errors")
|
||
cpu = measured.generator_cpu
|
||
if cpu is not None and cpu >= profile.generator_cpu_limit:
|
||
flags.append("generator")
|
||
return flags
|
||
|
||
|
||
# ------------------------------------------------------------ результаты
|
||
|
||
|
||
class Results:
|
||
"""Файлы метки: runs.csv, prefill.csv, meta.json, progress.json."""
|
||
|
||
def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None:
|
||
self.dir = label_dir
|
||
self.meta = meta
|
||
self.runs: list[dict[str, Any]] = []
|
||
self.prefill: list[dict[str, Any]] = []
|
||
self.sizes: list[dict[str, Any]] = []
|
||
(label_dir / "raw").mkdir(parents=True)
|
||
self.write_meta()
|
||
|
||
def append(
|
||
self,
|
||
name: str,
|
||
fields: list[str],
|
||
row: dict[str, Any],
|
||
) -> None:
|
||
path = self.dir / name
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=fields,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
writer.writerow({key: row.get(key) for key in fields})
|
||
file.flush()
|
||
|
||
def add_run(self, row: dict[str, Any]) -> None:
|
||
self.runs.append(row)
|
||
self.append("runs.csv", RUN_FIELDS, row)
|
||
|
||
def add_prefill(self, row: dict[str, Any]) -> None:
|
||
self.prefill.append(row)
|
||
self.append("prefill.csv", PREFILL_FIELDS, row)
|
||
|
||
def write_meta(self) -> None:
|
||
self.meta["sizes"] = self.sizes
|
||
text = json.dumps(self.meta, ensure_ascii=False, indent=2)
|
||
(self.dir / "meta.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_progress(self, progress: dict[str, Any]) -> None:
|
||
progress["updated_at"] = datetime.now(UTC).isoformat(
|
||
timespec="seconds",
|
||
)
|
||
text = json.dumps(progress, ensure_ascii=False, indent=2)
|
||
(self.dir / "progress.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_summary(self) -> None:
|
||
text = build_summary(self.meta, self.runs, self.prefill, self.sizes)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
|
||
|
||
def cell(values: list[float | None], digits: int = 1) -> str:
|
||
"""Медиана и min–max повторов; один повтор — просто значение."""
|
||
present = [v for v in values if v is not None]
|
||
if not present:
|
||
return "—"
|
||
median = statistics.median(present)
|
||
if len(present) == 1:
|
||
return f"{median:.{digits}f}"
|
||
return (
|
||
f"{median:.{digits}f} ({min(present):.{digits}f}–"
|
||
f"{max(present):.{digits}f})"
|
||
)
|
||
|
||
|
||
def build_summary(
|
||
meta: dict[str, Any],
|
||
runs: list[dict[str, Any]],
|
||
prefill: list[dict[str, Any]],
|
||
sizes: list[dict[str, Any]],
|
||
) -> str:
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Нагрузочный замер: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{meta['status']}**"
|
||
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
|
||
f"- Commit: `{meta['git']['commit']}`"
|
||
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
|
||
f"- Профиль `{profile['name']}`: размеры "
|
||
f"{', '.join(human(s) for s in profile['sizes'])}; клиенты "
|
||
f"{', '.join(map(str, profile['concurrency']))}; повторов "
|
||
f"{profile['repeats']}; чтение {profile['read_measure_s']} с после "
|
||
f"{profile['read_warmup_s']} с прогрева; запись "
|
||
f"{profile['write_measure']} после {profile['write_warmup']}",
|
||
f"- Лимиты приложения: {profile['cpus']} CPU, {profile['memory']} "
|
||
"без swap; значения — медиана (min–max) по повторам",
|
||
"- Посторонние контейнеры в VM во время замера: "
|
||
+ (", ".join(meta.get("other_containers") or []) or "нет"),
|
||
"",
|
||
]
|
||
for scenario in ("read", "write"):
|
||
rows = [r for r in runs if r["scenario"] == scenario]
|
||
if not rows:
|
||
continue
|
||
lines += [
|
||
f"## Сценарий `{scenario}`",
|
||
"",
|
||
"| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | "
|
||
"max, мс | CPU, ядер | RSS конец, МиБ | Сообщений в начале | "
|
||
"Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
cells = sorted({(r["size"], r["concurrency"]) for r in rows})
|
||
for size, conc in cells:
|
||
group = [
|
||
r
|
||
for r in rows
|
||
if r["size"] == size and r["concurrency"] == conc
|
||
]
|
||
flags = sorted({f for r in group for f in r["flags"].split()})
|
||
lines.append(
|
||
f"| {human(size)} | {conc} | "
|
||
f"{cell([r['rps'] for r in group], 0)} | "
|
||
f"{cell([r['p50_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p95_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p99_ms'] for r in group], 2)} | "
|
||
f"{cell([r['max_ms'] for r in group], 1)} | "
|
||
f"{cell([r['app_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['rss_end_mib'] for r in group], 1)} | "
|
||
f"{cell([r['store_messages_start'] for r in group], 0)} | "
|
||
f"{' '.join(flags) or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if sizes:
|
||
lines += [
|
||
"## Память и наполнение по размерам",
|
||
"",
|
||
"| Размер | RSS до, МиБ | RSS после, МиБ | Байт на сообщение | "
|
||
"Наполнение, сообщ/с | Курсоров в пуле | Пик RSS за размер, МиБ |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
lines.append(
|
||
f"| {human(info['size'])} | {info['rss_before_mib']} | "
|
||
f"{info['rss_after_mib']} | {info['bytes_per_message']} | "
|
||
f"{info['prefill_rate']} | {info['cursors']} | "
|
||
f"{info.get('rss_peak_mib', '—')} |",
|
||
)
|
||
lines.append("")
|
||
if prefill:
|
||
largest = max(r["size"] for r in prefill)
|
||
curve = [r for r in prefill if r["size"] == largest]
|
||
lines += [
|
||
f"## Кривая наполнения {human(largest)}: запись при росте "
|
||
f"хранилища, {profile['prefill_vus']} клиента",
|
||
"",
|
||
"| Порция | Сообщений после | RPS | p50, мс | p95, мс | "
|
||
"p99, мс | max, мс | CPU, ядер | RSS, МиБ | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in curve:
|
||
lines.append(
|
||
f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | "
|
||
f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | "
|
||
f"{r['max_ms']} | {r['app_cpu_cores']} | {r['rss_end_mib']} "
|
||
f"| {r['flags'] or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if meta.get("host_warnings"):
|
||
lines += ["## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ---------------------------------------------------------------- прогон
|
||
|
||
|
||
class Bench:
|
||
def __init__(
|
||
self,
|
||
profile: Profile,
|
||
label: str,
|
||
results: Results,
|
||
) -> None:
|
||
self.profile = profile
|
||
self.label = label
|
||
self.results = results
|
||
self.stand = Stand(profile, results.dir)
|
||
self.started = time.monotonic()
|
||
self.runs_total = (
|
||
len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats
|
||
)
|
||
self.runs_done = 0
|
||
self.stage = "start"
|
||
self.size: int | None = None
|
||
# скорости стенда для оценки оставшегося времени
|
||
self.prefill_rate = 1500.0
|
||
self.write_rps: dict[int, float] = {}
|
||
self.run_overhead = 3.0
|
||
self.done_sizes = 0
|
||
self.prefilled = 0
|
||
# текущий размер наполнен и курсоры собраны
|
||
self.prepared = False
|
||
|
||
# ------------------------------------------------------------ вывод
|
||
|
||
def say(self, text: str) -> None:
|
||
elapsed = int(time.monotonic() - self.started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
def eta(self) -> float:
|
||
p = self.profile
|
||
left = 0.0
|
||
for index, size in enumerate(p.sizes):
|
||
current = index == self.done_sizes
|
||
if index < self.done_sizes or (current and self.prepared):
|
||
continue
|
||
filled = self.prefilled if current else 0
|
||
left += (size - filled) / self.prefill_rate + size / 100 / 500
|
||
left += 15
|
||
per_size = len(p.concurrency) * p.repeats
|
||
reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes)
|
||
done_in_size = self.runs_done - self.done_sizes * per_size * 2
|
||
reads_left -= min(done_in_size, per_size)
|
||
writes_left -= max(0, done_in_size - per_size)
|
||
read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead
|
||
left += reads_left * read_time
|
||
rps = (
|
||
statistics.mean(self.write_rps.values())
|
||
if self.write_rps
|
||
else 1000
|
||
)
|
||
write_time = (p.write_warmup + p.write_measure) / rps
|
||
left += writes_left * (write_time + self.run_overhead)
|
||
return left
|
||
|
||
def progress(self, last: dict[str, Any] | None = None) -> None:
|
||
state = {
|
||
"label": self.label,
|
||
"profile": self.profile.name,
|
||
"stage": self.stage,
|
||
"size": self.size,
|
||
"runs_done": self.runs_done,
|
||
"runs_total": self.runs_total,
|
||
"elapsed_s": round(time.monotonic() - self.started),
|
||
"eta_s": round(self.eta()),
|
||
"warnings": self.results.meta.get("host_warnings", []),
|
||
}
|
||
if last is not None:
|
||
state["last_run"] = last
|
||
self.results.write_progress(state)
|
||
|
||
# --------------------------------------------------------- этапы
|
||
|
||
def run(self) -> None:
|
||
for index, size in enumerate(self.profile.sizes, start=1):
|
||
self.size = size
|
||
self.prefilled = 0
|
||
self.prepared = False
|
||
self.run_size(index, size)
|
||
self.done_sizes += 1
|
||
|
||
def run_size(self, index: int, size: int) -> None:
|
||
stand = self.stand
|
||
self.stage = f"size {human(size)}: start"
|
||
stand.start()
|
||
limits = stand.limits()
|
||
self.results.meta.setdefault("limits", limits)
|
||
if limits != self.results.meta["limits"]:
|
||
raise BenchError(f"limits changed between sizes: {limits}")
|
||
rss_start = stand.snapshot()["rss_kib"]
|
||
self.say(
|
||
f"size {human(size)} ({index}/{len(self.profile.sizes)}): "
|
||
f"app started, RSS {mib(rss_start)} MiB",
|
||
)
|
||
stand.k6("prepare.js", {"MEMBERS": self.profile.members})
|
||
self.progress()
|
||
rss_before = stand.snapshot()["rss_kib"]
|
||
prefill_seconds = self.run_prefill(size)
|
||
after = stand.snapshot()
|
||
rss_after = after["rss_kib"]
|
||
stand.k6(
|
||
"cursors.js",
|
||
{
|
||
"WALK_LIMIT": self.profile.walk_limit,
|
||
"POOL_SIZE": self.profile.cursor_pool,
|
||
},
|
||
)
|
||
pool = stand.work_json("cursors.json")
|
||
if pool["walked"] != size:
|
||
raise BenchError(
|
||
f"read chat has {pool['walked']} messages, expected {size}",
|
||
)
|
||
info = {
|
||
"size": size,
|
||
"rss_before_mib": mib(rss_before),
|
||
"rss_after_mib": mib(rss_after),
|
||
"bytes_per_message": round(
|
||
(rss_after - rss_before) * 1024 / size,
|
||
),
|
||
"prefill_seconds": round(prefill_seconds, 1),
|
||
"prefill_rate": round(size / prefill_seconds),
|
||
"cursors": len(pool["cursors"]),
|
||
}
|
||
self.results.sizes.append(info)
|
||
self.results.write_meta()
|
||
self.prepared = True
|
||
self.say(
|
||
f"size {human(size)}: prefill done, RSS {info['rss_after_mib']} "
|
||
f"MiB (+{info['bytes_per_message']} B/msg); "
|
||
f"cursors {info['cursors']}",
|
||
)
|
||
store = size
|
||
for scenario in ("read", "write"):
|
||
for conc in self.profile.concurrency:
|
||
for repeat in range(1, self.profile.repeats + 1):
|
||
store = self.run_one(scenario, size, conc, repeat, store)
|
||
if scenario == "read":
|
||
self.check_read_chat(size)
|
||
info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"])
|
||
self.results.write_meta()
|
||
stand.stop()
|
||
|
||
def check_read_chat(self, size: int) -> None:
|
||
"""После блока чтения в читаемом чате по-прежнему ровно N.
|
||
|
||
Новое сообщение встало бы в конец истории, поэтому достаточно
|
||
дочитать хвост от последнего курсора пула.
|
||
"""
|
||
self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit})
|
||
tail = self.stand.work_json("tail.json")
|
||
total = tail["from_after"] + tail["tail"]
|
||
if total != size:
|
||
raise BenchError(
|
||
f"read chat has {total} messages after the read block, "
|
||
f"expected {size}",
|
||
)
|
||
self.say(f"size {human(size)}: read chat unchanged, {total} messages")
|
||
|
||
def run_prefill(self, size: int) -> float:
|
||
chunk_size = self.profile.prefill_chunk
|
||
total_seconds = 0.0
|
||
for chunk, offset in enumerate(range(0, size, chunk_size), start=1):
|
||
count = min(chunk_size, size - offset)
|
||
name = f"prefill-{human(size)}-{chunk:02d}"
|
||
self.stage = f"size {human(size)}: prefill"
|
||
measured = measure(
|
||
self.stand,
|
||
"prefill.js",
|
||
{
|
||
"COUNT": count,
|
||
"OFFSET": offset,
|
||
"VUS": self.profile.prefill_vus,
|
||
"NAME": name,
|
||
},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
flags = run_flags(self.profile, measured, self.stand.app_state())
|
||
self.results.add_prefill(
|
||
{
|
||
**summary,
|
||
"size": size,
|
||
"chunk": chunk,
|
||
"messages_before": offset,
|
||
"messages_after": offset + summary["requests_ok"],
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"mem_cgroup_end_mib": mib_bytes(
|
||
measured.after["mem_bytes"],
|
||
),
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
},
|
||
)
|
||
if summary["requests_ok"] != count:
|
||
raise BenchError(
|
||
f"prefill chunk {chunk}: {summary['requests_ok']} of "
|
||
f"{count} messages confirmed ({' '.join(flags)})",
|
||
)
|
||
total_seconds += summary["duration_s"]
|
||
self.prefilled = offset + count
|
||
self.prefill_rate = self.prefilled / total_seconds
|
||
self.say(
|
||
f"size {human(size)}: prefill {self.prefilled}/{size} "
|
||
f"({self.prefilled * 100 // size}%), "
|
||
f"{self.prefill_rate:.0f} msg/s, "
|
||
f"~{fmt_left((size - self.prefilled) / self.prefill_rate)} "
|
||
"left",
|
||
)
|
||
self.progress()
|
||
return total_seconds
|
||
|
||
def run_one(
|
||
self,
|
||
scenario: str,
|
||
size: int,
|
||
conc: int,
|
||
repeat: int,
|
||
store: int,
|
||
) -> int:
|
||
"""Прогрев и измеряемый прогон; возвращает сообщений в хранилище."""
|
||
p = self.profile
|
||
number = self.runs_done + 1
|
||
name = f"{scenario}-{human(size)}-c{conc}-r{repeat}"
|
||
self.stage = f"run {number}/{self.runs_total} {name}"
|
||
self.progress()
|
||
wall = time.monotonic()
|
||
if scenario == "read":
|
||
warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"}
|
||
self.stand.k6(
|
||
"read.js",
|
||
{**warm, "SEED": p.seed + p.warmup_seed_shift},
|
||
)
|
||
env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"}
|
||
env["SEED"] = p.seed
|
||
else:
|
||
self.stand.k6(
|
||
"write.js",
|
||
{"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store},
|
||
)
|
||
warm_ok = parse_summary(self.stand.work_dir / "last.json")
|
||
store += warm_ok["requests_ok"]
|
||
env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store}
|
||
measured = measure(
|
||
self.stand,
|
||
f"{scenario}.js",
|
||
{**env, "NAME": name},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
app = self.stand.app_state()
|
||
flags = run_flags(p, measured, app)
|
||
end = store + (summary["requests_ok"] if scenario == "write" else 0)
|
||
row = {
|
||
**summary,
|
||
"run": number,
|
||
"scenario": scenario,
|
||
"size": size,
|
||
"concurrency": conc,
|
||
"repeat": repeat,
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_start_mib": mib(measured.before["rss_kib"]),
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"rss_max_mib": mib(measured.rss_max_kib),
|
||
"mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]),
|
||
"store_messages_start": store,
|
||
"store_messages_end": end,
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
}
|
||
self.results.add_run(row)
|
||
self.runs_done += 1
|
||
if scenario == "write" and summary["rps"]:
|
||
self.write_rps[conc] = summary["rps"]
|
||
overhead = time.monotonic() - wall - summary["duration_s"]
|
||
if scenario == "read":
|
||
overhead -= p.read_warmup_s
|
||
self.run_overhead = max(1.0, overhead)
|
||
errors = summary["requests_failed"] + summary["check_failed"]
|
||
self.say(
|
||
f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} "
|
||
f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps "
|
||
f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} "
|
||
f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} "
|
||
f"rss {row['rss_end_mib']} MiB err {errors} "
|
||
f"~{fmt_left(self.eta())} left"
|
||
+ (f" [{' '.join(flags)}]" if flags else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if "oom" in flags or "app_down" in flags:
|
||
raise BenchError(
|
||
f"{APP} is down after {name} ({' '.join(flags)}): "
|
||
"matrix stopped",
|
||
)
|
||
return end
|
||
|
||
|
||
# ------------------------------------------------------------------ CLI
|
||
|
||
|
||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--profile", choices=sorted(PROFILES), required=True)
|
||
parser.add_argument("--label", required=True)
|
||
# отладка стенда: переопределения пишутся в паспорт как overrides
|
||
parser.add_argument("--sizes", help="через запятую, например 120000")
|
||
parser.add_argument("--concurrency", help="через запятую")
|
||
parser.add_argument("--repeats", type=int)
|
||
parser.add_argument("--memory", help="лимит памяти, например 64m")
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def apply_overrides(
|
||
profile: Profile,
|
||
args: argparse.Namespace,
|
||
) -> tuple[Profile, dict[str, Any]]:
|
||
overrides: dict[str, Any] = {}
|
||
if args.sizes:
|
||
overrides["sizes"] = tuple(int(x) for x in args.sizes.split(","))
|
||
if args.concurrency:
|
||
overrides["concurrency"] = tuple(
|
||
int(x) for x in args.concurrency.split(",")
|
||
)
|
||
if args.repeats:
|
||
overrides["repeats"] = args.repeats
|
||
if args.memory:
|
||
overrides["memory"] = args.memory
|
||
return replace(profile, **overrides), overrides
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
args = parse_args(argv)
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label):
|
||
print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True)
|
||
return 2
|
||
label_dir = ROOT / "evidence" / "bench" / args.label
|
||
relative = label_dir.relative_to(ROOT)
|
||
if label_dir.exists():
|
||
print(f"label directory {relative} already exists", flush=True)
|
||
return 2
|
||
profile, overrides = apply_overrides(PROFILES[args.profile], args)
|
||
git = git_info()
|
||
state = host_state()
|
||
warnings = host_warnings(state)
|
||
meta: dict[str, Any] = {
|
||
"label": args.label,
|
||
"status": "incomplete",
|
||
"reason": "running",
|
||
"started_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"command": " ".join(["uv run python bench/run.py", *argv]),
|
||
"git": git,
|
||
"overrides": overrides,
|
||
"host_state": {"start": state},
|
||
"host_warnings": warnings,
|
||
# чужие контейнеры в той же VM: стенд их не трогает, но они шумят
|
||
"other_containers": sorted(
|
||
sh("docker", "ps", "--format", "{{.Names}}", check=False).split(),
|
||
),
|
||
}
|
||
stand_check = Stand(profile, label_dir)
|
||
started = time.monotonic()
|
||
|
||
def say(text: str) -> None:
|
||
elapsed = int(time.monotonic() - started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
for warning in warnings:
|
||
say(f"WARNING: {warning}")
|
||
if overrides:
|
||
say(f"WARNING: profile overrides {overrides}: debug run")
|
||
if git["dirty"]:
|
||
say("NOTE: working tree has uncommitted changes (dirty: true)")
|
||
try:
|
||
stand_check.check_free()
|
||
image = stand_check.build_image()
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 1
|
||
say(f"image {APP_IMAGE} built ({image[:19]}…), network {NETWORK}")
|
||
meta.update(static_passport(profile))
|
||
results = Results(label_dir, meta)
|
||
bench = Bench(profile, args.label, results)
|
||
bench.started = started
|
||
code = 0
|
||
|
||
def terminate(*_: object) -> None:
|
||
raise KeyboardInterrupt
|
||
|
||
signal.signal(signal.SIGTERM, terminate)
|
||
try:
|
||
bench.run()
|
||
meta["status"] = "complete"
|
||
meta.pop("reason", None)
|
||
except KeyboardInterrupt:
|
||
meta["reason"] = "interrupted"
|
||
say("interrupted: partial results kept")
|
||
code = 130
|
||
except BenchError as exc:
|
||
meta["reason"] = str(exc).splitlines()[0]
|
||
say(f"bench failed: {exc}")
|
||
code = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку стенда
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
bench.stand.stop()
|
||
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
|
||
meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds")
|
||
meta["host_state"]["end"] = host_state()
|
||
results.write_meta()
|
||
results.write_summary()
|
||
bench.stage = "done" if code == 0 else meta.get("reason", "stopped")
|
||
bench.progress()
|
||
say(
|
||
f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, "
|
||
f"results in {relative}",
|
||
)
|
||
return code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|