Files
mtusi2026lab4/bench/scaling.py
T
ricoromaandClaude Opus 5.5 5d5d8f9d2a chore: carry lab 3 over
- carry the lab 3 solution over from mtusi2026lab3-safronov
  (solution/bvt2301-safronov, ce89eba): src, migrations, student-tests,
  bench, openspec, .claude, infra/haproxy and infra/valkey, own scripts,
  Dockerfile, compose, pyproject and uv lock
- merge Makefile, .gitignore and .dockerignore (the lab 3 versions only
  add lines to the skeleton, which is the same in lab 4) and
  .env.example (lab 4 skeleton names listed in the header)
- rename the compose project to mtusi-lab4 and the image to :lab4 so
  the stand does not reuse the lab 3 containers and volumes
- update AGENTS.md for lab 4: current lab, origin commit, contract and
  test files of the new skeleton

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 17:32:33 +03:00

1777 lines
65 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Трек 1 «Цена масштабирования»: чистые функции серий без docker.
Лестница и поиск максимальной устойчивой интенсивности (R*), разбор итога
k6 ступени, строк сэмплера и PostgreSQL, агрегаты окна замера, вердикт
SLO и брака генератора, признаки узкого места, строки таблиц и сводка
серии. Методика — design change `scaling-bench` (решения 3, 6–9), то,
что стенд обязан делать, — спек `load-benchmark`. Только stdlib:
вызывает `bench/run.py`, самопроверка —
`uv run python bench/scaling.py --selftest`.
"""
import csv
import math
import random
import statistics
import sys
from collections.abc import Iterable
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# новые столбцы runs.csv: только в конец (решение 8)
SCALING_FIELDS = [
"series",
"cell",
"attempt",
"config",
"replicas",
"replica_cpus",
"replica_memory",
"db_max_connections",
"step",
"step_kind",
"target_rps",
"sent",
"missing",
"dropped_iterations",
"slo_error_ratio",
"slo_pass",
"fail_reason",
"read_p95_ms",
"write_p95_ms",
"me_p95_ms",
"read_errors",
"write_errors",
"me_errors",
"short_pages",
"vus_max",
"generator_throttled_ratio",
"generator_mem_peak_mib",
"app_cpu_replica_max",
"app_cpu_second_max",
"app_throttled_ratio",
"app_throttled_ms",
"app_mem_peak_mib",
"app_us_per_request",
"db_throttled_ratio",
"db_mem_peak_mib",
"db_us_per_request",
"lb_cpu_cores",
"lb_throttled_ratio",
"cache_cpu_cores",
"vm_busy_cores",
"vm_cpu_pressure",
"db_commits_per_s",
"db_rollbacks_per_s",
"db_wal_mib_per_s",
"db_wal_fsyncs_per_s",
"db_sessions_new",
"db_conn_max",
"db_conn_replica_max",
"db_aas_cpu",
"db_aas_wal",
"db_aas_lock",
"db_aas_lwlock",
"db_aas_io",
"db_aas_other",
"db_idle_in_tx_mean",
"pool_busy_share",
"db_too_many_clients",
"lb_qtime_ms",
"lb_ctime_ms",
"lb_rtime_ms",
"lb_ttime_ms",
"lb_scur_max",
"lb_qcur_max",
"lb_retries",
"lb_redispatches",
"lb_econ",
"lb_eresp",
"lb_chkdown",
"lb_5xx",
"instances",
"instance_share_min",
"instance_share_max",
"host_swapouts",
"signs",
# найдено в apply: коды неуспешных ответов окна (H3, 500 или 503) и
# уровень ступени лестницы; target_rps — сумма интенсивностей
# сценариев, может отличаться от уровня на ±1
"error_codes",
"level",
]
SEARCH_FIELDS = [
"series",
"cell",
"attempt",
"config",
"replicas",
"replica_cpus",
"replica_memory",
"db_max_connections",
"repeat",
"round",
"order",
"started_at",
"finished_at",
"status",
"r_star_rps",
"r_star_bound",
"first_fail_rps",
"first_fail_reason",
"steps",
"resolution",
"fixed_rps_1",
"p50_fixed_1_ms",
"p95_fixed_1_ms",
"p99_fixed_1_ms",
"pass_fixed_1",
"fixed_rps_2",
"p50_fixed_2_ms",
"p95_fixed_2_ms",
"p99_fixed_2_ms",
"pass_fixed_2",
"app_us_per_request_fixed_2",
"app_cpu_second_max",
"app_throttled_ratio_fixed_2",
"writes_ok",
"writes_sent",
"write_chat_messages",
"read_chat_messages",
"host_power",
"host_memory_free_percent",
"host_swapouts",
"other_containers",
"flags",
]
SAMPLE_FIELDS = [
"cell",
"attempt",
"step",
"t_s",
"role",
"instance",
"cpu_cores",
"throttled_ms",
"pressure_ms",
"mem_mib",
]
# пороги признаков узкого места (решение 9) и брака генератора (решение 6):
# заданы до прогонов
THRESHOLDS: dict[str, float] = {
# CPU реплики в ядрах: один процесс больше одного ядра не берёт (H1)
"app_cpu_cores": 0.9,
# доля периодов CFS с троттлингом
"throttled_ratio": 0.10,
# CPU PostgreSQL и LB — доля лимита
"db_cpu_share": 0.9,
"lb_cpu_share": 0.9,
# доля ожиданий класса от всех активных соединений
"wait_share": 0.30,
# признаки ожиданий — только при среднем ≥ 1 активного соединения
# (найдено в apply: без порога одно ожидание на почти пустой базе)
"db_active_min": 1.0,
"pool_busy_share": 0.5,
# хост: занятые ядра ВМ — доля её CPU, PSI CPU — доля времени
"vm_busy_share": 0.9,
"vm_cpu_pressure": 0.20,
# предельный CPU реплик на запрос (наклон по ступеням, прошедшим SLO)
# против 1 × 1 серии роста (marginal_us, найдено в apply)
"host_cost": 1.2,
# генератор: троттлинг и недосланные при CPU k6 не меньше доли лимита
"generator_throttled": 0.10,
"generator_missing_cpu_share": 0.7,
# точка насыщения: реплика добавила меньше 10 %
"k_sat_gain": 1.1,
}
# признаки общего ресурса: узкое место «реплики» называется без них
SHARED_SIGNS = ("db_cpu", "db_wal", "db_lock", "pool", "lb", "host")
SIGN_ORDER = ("app", *SHARED_SIGNS, "host_cost", "generator")
MIX_NAMES = ("read", "write", "me")
# коды неуспешных ответов окна, которые k6 выделяет подметриками
ERROR_STATUSES = (0, 400, 401, 403, 404, 409, 422, 429, 500, 502, 503, 504)
TOO_MANY_CLIENTS = "sorry, too many clients already"
def num(value: Any) -> float | None:
"""Число из CSV или итога; пусто — None."""
if value is None or value == "":
return None
return float(value)
def rounded(value: float | None, digits: int = 3) -> float | None:
return None if value is None else round(value, digits)
def median(values: Iterable[float | None]) -> float | None:
present = [v for v in values if v is not None]
return statistics.median(present) if present else None
def spread(values: Iterable[float | None], digits: int = 0) -> str:
"""Медиана и min–max повторов, как `cell()` матрицы."""
present = [v for v in values if v is not None]
if not present:
return "—"
mid = statistics.median(present)
if len(present) == 1:
return f"{mid:.{digits}f}"
return (
f"{mid:.{digits}f} ({min(present):.{digits}f}–"
f"{max(present):.{digits}f})"
)
# ----------------------------------------------------- конфигурации
def config_key(
replicas: int,
cpus: str,
_memory: str,
max_conn: int,
db_cpus: str = "",
) -> str:
"""Имя конфигурации: `4x1`, у предела соединений не 100 — `4x1-c50`,
у своего лимита CPU PostgreSQL (серия `db`) — `4x1-pg0.5`.
Память реплики в имя не входит: в сериях она задана числом CPU."""
key = f"{replicas}x{cpus}"
if max_conn != 100:
key = f"{key}-c{max_conn}"
return f"{key}-pg{db_cpus}" if db_cpus else key
def unpack_config(config: Any) -> tuple[int, str, str, int, str]:
"""Конфигурация серии: (реплик, CPU и память реплики, max_connections
[, CPU PostgreSQL]). Пятый элемент — только у серии `db`; без него
лимит PostgreSQL — профиля."""
replicas, cpus, memory, max_conn, *rest = config
return (
int(replicas),
str(cpus),
str(memory),
int(max_conn),
str(rest[0]) if rest else "",
)
def plan_cells(
configs: list[tuple[Any, ...]],
repeats: int,
seed: int,
) -> list[dict[str, Any]]:
"""Порядок ячеек кругами: в круге r — перестановка конфигураций
генератором `Random(seed + r)`. Если круг начался с конфигурации, на
которой кончился прежний, она меняется местами со второй (решение 4).
"""
plan: list[dict[str, Any]] = []
previous = None
for round_no in range(1, repeats + 1):
order = list(range(len(configs)))
random.Random(seed + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for index in order:
replicas, cpus, memory, max_conn, db_cpus = unpack_config(
configs[index],
)
key = config_key(replicas, cpus, memory, max_conn, db_cpus)
cell = {
"cell": f"{key}-r{round_no}",
"config": key,
"replicas": replicas,
"replica_cpus": cpus,
"replica_memory": memory,
"db_max_connections": max_conn,
"repeat": round_no,
"round": round_no,
"order": len(plan) + 1,
}
if db_cpus:
cell["db_cpus"] = db_cpus
plan.append(cell)
previous = order[-1]
return plan
# ----------------------------------------------------- лестница и поиск
def ladder(base: int, factor: float, count: int = 48) -> list[int]:
"""Ступеньки `round(base × factor^i)` по возрастанию, без повторов."""
values: list[int] = []
for i in range(count):
value = round(base * factor**i)
if not values or value > values[-1]:
values.append(value)
return values
def start_level(
replicas: int,
per_replica: int,
passed_fixed: int | None,
steps: list[int],
) -> int:
"""Начало подъёма L0: наименьшая ступенька ≥ max(per_replica × k,
наибольший пройденный фиксированный уровень) и строго больше него."""
floor = max(per_replica * replicas, passed_fixed or 0)
for value in steps:
if value >= floor and (passed_fixed is None or value > passed_fixed):
return value
raise ValueError(f"ladder ends below {floor}")
@dataclass
class Search:
"""Автомат ступеней ячейки: фиксированные уровни, подъём или спуск по
лестнице, деления по среднему геометрическому (решение 3).
`next_step()` — следующий уровень и вид ступени или None; итог ступени
— `record()`: `pass`, `fail` или `generator` (брак)."""
replicas: int
fixed: tuple[int, ...]
base: int = 100
factor: float = 1.25
per_replica: int = 300
bisect_steps: int = 2
max_steps: int = 12
steps: list[int] = field(default_factory=list)
outcomes: list[tuple[int, str, str]] = field(default_factory=list)
fixed_left: list[int] = field(default_factory=list)
passed_fixed: int | None = None
mode: str | None = None
current: int | None = None
passed: int | None = None
failed: int | None = None
bisects: int = 0
search_steps: int = 0
done: bool = False
status: str = "complete"
flags: list[str] = field(default_factory=list)
def __post_init__(self) -> None:
self.steps = ladder(self.base, self.factor)
self.fixed_left = list(self.fixed)
def next_step(self) -> tuple[int, str] | None:
if self.done:
return None
if self.fixed_left:
return self.fixed_left[0], "fixed"
if self.mode is None:
self._begin()
if self.done or self.current is None or self.mode is None:
return None
return self.current, self.mode
def _below(self, level: int) -> int | None:
lower = [v for v in self.steps if v < level]
return lower[-1] if lower else None
def _above(self, level: int) -> int:
return next(v for v in self.steps if v > level)
def _begin(self) -> None:
passed = [lvl for lvl, kind, out in self.outcomes if out == "pass"]
self.passed_fixed = max(passed) if passed else None
if self.passed_fixed is None:
# ни один фиксированный уровень не пройден: спуск от
# наименьшего из них до основания лестницы
self.failed = min(self.fixed) if self.fixed else None
self.mode = "down"
self.current = self._below(self.failed or self.steps[0] + 1)
if self.current is None:
self._floor()
return
self.mode = "up"
self.current = start_level(
self.replicas,
self.per_replica,
self.passed_fixed,
self.steps,
)
def _floor(self) -> None:
self.passed = 0
self.flags.append("floor")
self.done = True
def _bisect(self) -> None:
self.mode = "bisect"
if (
self.passed is None
or self.failed is None
or self.bisects >= self.bisect_steps
):
self.done = True
return
middle = round(math.sqrt(self.passed * self.failed))
if middle <= self.passed or middle >= self.failed:
self.done = True
return
self.current = middle
def record(self, level: int, kind: str, outcome: str) -> None:
self.outcomes.append((level, kind, outcome))
if outcome == "generator":
self.status = "generator_bound"
self.flags.append("generator")
passes = [lvl for lvl, _, out in self.outcomes if out == "pass"]
self.passed = max(passes) if passes else 0
self.done = True
return
if kind == "fixed":
self.fixed_left.remove(level)
return
self.search_steps += 1
ok = outcome == "pass"
if self.mode == "up":
if ok:
self.passed = level
self.current = self._above(level)
else:
self.failed = level
if self.passed is None:
self._descend(level)
else:
self._bisect()
elif self.mode == "down":
if ok:
self.passed = level
self._bisect()
else:
self.failed = level
self._descend(level)
else:
if ok:
self.passed = level
else:
self.failed = level
self.bisects += 1
self._bisect()
if not self.done and self.search_steps >= self.max_steps:
self.flags.append("search_limit")
self.done = True
def _descend(self, level: int) -> None:
self.mode = "down"
below = self._below(level)
if self.passed_fixed is not None and (
below is None or below <= self.passed_fixed
):
# спуск дошёл до пройденного фиксированного уровня: P = он
self.passed = self.passed_fixed
self._bisect()
return
if below is None:
self._floor()
return
self.current = below
@property
def r_star(self) -> int:
return self.passed or 0
@property
def first_fail(self) -> int | None:
"""Ближайший нарушивший уровень выше R*."""
above = [
lvl
for lvl, _, out in self.outcomes
if out == "fail" and lvl > self.r_star
]
return min(above) if above else None
@property
def resolution(self) -> float | None:
fail = self.first_fail
if not self.r_star or fail is None:
return None
return fail / self.r_star
# ------------------------------------------------------- итог k6 ступени
def metric(
metrics: dict[str, Any],
name: str,
key: str,
default: float | None = None,
) -> float | None:
value = metrics.get(name, {}).get("values", {}).get(key)
return default if value is None else float(value)
def targets(level: int, mix: list[tuple[str, float]]) -> dict[str, int]:
"""Интенсивность сценариев: `round(уровень × доля)` с половиной вверх,
как `Math.round` в `mixed.js` (у `round` Python — к чётному)."""
return {name: math.floor(level * share + 0.5) for name, share in mix}
def parse_k6(
data: dict[str, Any],
*,
level: int,
mix: list[tuple[str, float]],
measure_s: float,
instances: list[str],
) -> dict[str, Any]:
"""Окно замера ступени из итога k6 (`mixed.js`).
Недосланные — `max(0, запланированные − отправленные − 1)` на
сценарий: допуск ±1 на границе окна (решение 3). Неуспешные — с
короткими страницами. Распределение по `X-Instance-Id` — по счётчику
полностью успешных ответов окна."""
m = data["metrics"]
hold = "window:hold"
plan = targets(level, mix)
sent = errors = missing = 0
row: dict[str, Any] = {}
for name, rate in plan.items():
sent_x = int(
metric(m, f"http_reqs{{{hold},scenario:{name}}}", "count", 0),
)
bad_x = int(
metric(
m,
f"http_req_failed{{{hold},scenario:{name}}}",
"passes",
0,
),
)
planned_x = rate * measure_s
missing += max(0, round(planned_x - sent_x - 1))
sent += sent_x
errors += bad_x
row[f"{name}_errors"] = bad_x
row[f"{name}_p95_ms"] = rounded(
metric(
m,
f"http_req_duration{{{hold},expected_response:true,"
f"scenario:{name}}}",
"p(95)",
),
)
row[f"{name}_sent_total"] = int(
metric(m, f"http_reqs{{scenario:{name}}}", "count", 0),
)
row[f"{name}_failed_total"] = int(
metric(m, f"http_req_failed{{scenario:{name}}}", "passes", 0),
)
short = int(metric(m, f"short_pages{{{hold}}}", "count", 0))
planned = sum(rate * measure_s for rate in plan.values())
failed = errors + short
ok = sent - failed
duration = f"http_req_duration{{{hold},expected_response:true}}"
counts = {
ident: int(
metric(m, f"instance_ok{{{hold},instance:{ident}}}", "count", 0),
)
for ident in [*instances, "-"]
}
codes = {
status: int(
metric(m, f"http_reqs{{{hold},status:{status}}}", "count", 0),
)
for status in ERROR_STATUSES
}
total_ok = sum(counts.values())
shares = [counts[i] / total_ok for i in instances] if total_ok else []
row.update(
{
"target_rps": sum(plan.values()),
"planned": planned,
"sent": sent,
"missing": missing,
"requests_ok": ok,
"requests_failed": failed,
"short_pages": short,
"duration_s": measure_s,
"rps": round(ok / measure_s, 1) if measure_s else None,
"error_ratio": round(failed / sent, 6) if sent else None,
"slo_error_ratio": (
round((failed + missing) / planned, 6) if planned else None
),
"p50_ms": rounded(metric(m, duration, "p(50)")),
"p95_ms": rounded(metric(m, duration, "p(95)")),
"p99_ms": rounded(metric(m, duration, "p(99)")),
"max_ms": rounded(metric(m, duration, "max")),
"check_failed": int(metric(m, f"checks{{{hold}}}", "fails", 0)),
"dropped_iterations": int(
metric(m, "dropped_iterations", "count", 0),
),
"concurrency": int(metric(m, "vus", "max", 0)),
"vus_max": int(metric(m, "vus_max", "max", 0)),
"instances": " ".join(
f"{ident}:{count}" for ident, count in counts.items() if count
),
"instances_total": total_ok,
"instance_share_min": rounded(min(shares)) if shares else None,
"instance_share_max": rounded(max(shares)) if shares else None,
"error_codes": " ".join(
f"{status}:{count}" for status, count in codes.items() if count
),
"t0_ms": (data.get("setup_data") or {}).get("t0"),
"run_ms": data["state"]["testRunDurationMs"],
},
)
return row
# ------------------------------------------------------------- сэмплер
@dataclass
class Samples:
"""Отсчёты сэмплера (часы ВМ, нс): контейнеры, ВМ, LB, PostgreSQL."""
cg: dict[tuple[str, str], list[tuple[int, ...]]] = field(
default_factory=dict,
)
vm: list[tuple[int, int, int, int]] = field(default_factory=list)
lb: dict[int, list[dict[str, str]]] = field(default_factory=dict)
act: dict[int, list[tuple[str, str, str, str, int]]] = field(
default_factory=dict,
)
db: list[tuple[int, ...]] = field(default_factory=list)
wal: list[tuple[int, ...]] = field(default_factory=list)
io: list[tuple[int, ...]] = field(default_factory=list)
missing: list[str] = field(default_factory=list)
def parse_samples(text: str, pg_text: str = "") -> Samples:
"""Строки `samples.log` (`cg`, `vm`, `lb`) и `pg-samples.log`
(`act`, `db`, `wal`, `io`) — формат `bench/sampler.sh`."""
out = Samples()
header: list[str] = []
for line in text.splitlines():
parts = line.split(" ")
if len(parts) < 3:
continue
kind = parts[0]
try:
ns = int(parts[1])
except ValueError:
continue
if kind == "cg":
role, ident = parts[2], parts[3] if len(parts) > 3 else "?"
if len(parts) < 10 or parts[4] == "missing":
out.missing.append(f"{role}:{ident}")
continue
values = tuple(int(v or 0) for v in parts[4:10])
out.cg.setdefault((role, ident), []).append((ns, *values))
elif kind == "vm" and len(parts) >= 5:
out.vm.append((ns, int(parts[2]), int(parts[3]), int(parts[4])))
elif kind == "lb":
rest = line.split(" ", 2)[2]
if rest == "missing":
out.missing.append("lb")
elif rest.startswith("# "):
header = rest[2:].split(",")
elif header:
row = dict(zip(header, rest.split(","), strict=False))
out.lb.setdefault(ns, []).append(row)
for line in pg_text.splitlines():
parts = line.split("|")
if len(parts) < 7 or not parts[1].isdigit():
continue
kind, ns = parts[0], int(parts[1]) * 1_000_000
if kind == "act":
out.act.setdefault(ns, []).append(
(parts[2], parts[3], parts[4], parts[5], int(parts[6] or 0)),
)
elif kind in ("db", "wal", "io"):
values = tuple(int(v) if v else 0 for v in parts[2:7])
getattr(out, kind).append((ns, *values))
return out
def inside(
points: list[tuple[int, ...]],
w0: int,
w1: int,
) -> list[tuple[int, ...]]:
return [p for p in points if w0 <= p[0] <= w1]
def rate(points: list[tuple[int, ...]], index: int) -> float | None:
"""Разность счётчика между первым и последним отсчётом на Δt, в
секунду."""
if len(points) < 2:
return None
seconds = (points[-1][0] - points[0][0]) / 1e9
if seconds <= 0:
return None
return (points[-1][index] - points[0][index]) / seconds
def delta(points: list[tuple[int, ...]], index: int) -> int | None:
if len(points) < 2:
return None
return points[-1][index] - points[0][index]
@dataclass
class Container:
"""Агрегаты окна одного контейнера (решение 8)."""
cpu: float | None
seconds: list[tuple[float, float, float, float, float]]
periods: int
throttled: int
throttled_ms: float
mem_peak_mib: float | None
def container(points: list[tuple[int, ...]], w0: int) -> Container:
"""`points` — (нс, usage_usec, nr_periods, nr_throttled,
throttled_usec, memory, psi_usec), уже внутри окна."""
seconds = []
for prev, cur in zip(points, points[1:], strict=False):
dt = (cur[0] - prev[0]) / 1e9
if dt <= 0:
continue
seconds.append(
(
round((cur[0] - w0) / 1e9, 2),
(cur[1] - prev[1]) / 1e6 / dt,
(cur[4] - prev[4]) / 1000 / dt,
(cur[6] - prev[6]) / 1000 / dt,
cur[5] / 2**20,
),
)
cpu = rate(points, 1)
return Container(
cpu=None if cpu is None else cpu / 1e6,
seconds=seconds,
periods=delta(points, 2) or 0,
throttled=delta(points, 3) or 0,
throttled_ms=(delta(points, 4) or 0) / 1000,
mem_peak_mib=(max(p[5] for p in points) / 2**20 if points else None),
)
def wait_class(state: str, wtype: str, event: str) -> str | None:
"""Класс ожидания активного соединения (решение 8); не активное —
None."""
if state != "active":
return None
if wtype == "CPU":
return "cpu"
if "wal" in event.lower():
return "wal"
return {"Lock": "lock", "LWLock": "lwlock", "IO": "io"}.get(wtype, "other")
def analyze_samples(
samples: Samples,
w0: int,
w1: int,
*,
addresses: dict[str, str],
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
"""Агрегаты окна `[w0, w1]` (нс часов ВМ) и посекундные строки.
`addresses` — адрес реплики в сети стенда → короткий ID."""
row: dict[str, Any] = {}
per_second: list[dict[str, Any]] = []
roles: dict[str, list[tuple[str, Container]]] = {}
for (role, ident), points in sorted(samples.cg.items()):
agg = container(inside(points, w0, w1), w0)
roles.setdefault(role, []).append((ident, agg))
for t_s, cpu, thr, psi, mem in agg.seconds:
per_second.append(
{
"t_s": t_s,
"role": role,
"instance": ident,
"cpu_cores": round(cpu, 3),
"throttled_ms": round(thr, 1),
"pressure_ms": round(psi, 1),
"mem_mib": round(mem, 1),
},
)
def ratio(items: list[tuple[str, Container]]) -> float | None:
periods = sum(c.periods for _, c in items)
if not periods:
return 0.0 if items else None
return round(sum(c.throttled for _, c in items) / periods, 4)
def total_cpu(items: list[tuple[str, Container]]) -> float | None:
values = [c.cpu for _, c in items]
if not values or None in values:
return None
return round(sum(v for v in values if v is not None), 3)
def peak(items: list[tuple[str, Container]]) -> float | None:
values = [c.mem_peak_mib for _, c in items]
if not values or None in values:
return None
return round(sum(v for v in values if v is not None), 1)
apps = roles.get("api", [])
row["app_cpu_cores"] = total_cpu(apps)
replica_cpus = [c.cpu for _, c in apps if c.cpu is not None]
row["app_cpu_replica_max"] = rounded(max(replica_cpus, default=None))
seconds = [s[1] for _, c in apps for s in c.seconds]
row["app_cpu_second_max"] = rounded(max(seconds, default=None))
row["app_throttled_ratio"] = ratio(apps)
row["app_throttled_ms"] = round(sum(c.throttled_ms for _, c in apps), 1)
row["app_mem_peak_mib"] = peak(apps)
for role, prefix in (
("postgres", "db"),
("haproxy", "lb"),
("valkey", "cache"),
("k6", "generator"),
):
items = roles.get(role, [])
row[f"{prefix}_cpu_cores"] = total_cpu(items)
row[f"{prefix}_throttled_ratio"] = ratio(items)
row[f"{prefix}_mem_peak_mib"] = peak(items)
vm = inside(samples.vm, w0, w1)
busy = rate(vm, 1)
row["vm_busy_cores"] = None if busy is None else round(busy / 100, 3)
psi = rate(vm, 3)
row["vm_cpu_pressure"] = None if psi is None else round(psi / 1e6, 4)
for prev, cur in zip(vm, vm[1:], strict=False):
dt = (cur[0] - prev[0]) / 1e9
if dt > 0:
per_second.append(
{
"t_s": round((cur[0] - w0) / 1e9, 2),
"role": "vm",
"instance": "",
"cpu_cores": round((cur[1] - prev[1]) / 100 / dt, 3),
"throttled_ms": None,
"pressure_ms": round((cur[3] - prev[3]) / 1000 / dt, 1),
"mem_mib": None,
},
)
row.update(analyze_pg(samples, w0, w1, addresses))
row.update(analyze_lb(samples, w0, w1))
return row, per_second
def analyze_pg(
samples: Samples,
w0: int,
w1: int,
addresses: dict[str, str],
) -> dict[str, Any]:
"""PostgreSQL окна: commit/s, WAL, соединения, классы ожидания, пул."""
db = inside(samples.db, w0, w1)
wal = inside(samples.wal, w0, w1)
io = inside(samples.io, w0, w1)
commits = rate(db, 1)
rollbacks = rate(db, 2)
wal_bytes = rate(wal, 3)
fsyncs = rate(io, 2)
ticks = sorted(
{p[0] for p in db} | {t for t in samples.act if w0 <= t <= w1},
)
classes = dict.fromkeys(("cpu", "wal", "lock", "lwlock", "io"), 0)
classes["other"] = 0
idle_in_tx = 0
busy = 0
conn_max = 0
replica_max = 0
for tick in ticks:
rows = samples.act.get(tick, [])
total = 0
per_addr: dict[str, int] = {}
idle: dict[str, int] = {}
for addr, state, wtype, event, count in rows:
total += count
per_addr[addr] = per_addr.get(addr, 0) + count
if state == "idle":
idle[addr] = idle.get(addr, 0) + count
if state == "idle in transaction":
idle_in_tx += count
kind = wait_class(state, wtype, event)
if kind is not None:
classes[kind] += count
conn_max = max(conn_max, total)
replica_max = max(
[replica_max, *(per_addr.get(a, 0) for a in addresses)],
)
if addresses and any(idle.get(a, 0) == 0 for a in addresses):
busy += 1
n = len(ticks)
return {
"db_commits_per_s": rounded(commits, 1),
"db_rollbacks_per_s": rounded(rollbacks, 1),
"db_wal_mib_per_s": (
None if wal_bytes is None else round(wal_bytes / 2**20, 3)
),
"db_wal_fsyncs_per_s": rounded(fsyncs, 1),
"db_sessions_new": delta(db, 3),
"db_conn_max": conn_max if n else None,
"db_conn_replica_max": replica_max if n else None,
**{
f"db_aas_{name}": round(count / n, 3) if n else None
for name, count in classes.items()
},
"db_idle_in_tx_mean": round(idle_in_tx / n, 3) if n else None,
"pool_busy_share": round(busy / n, 3) if n else None,
}
def analyze_lb(samples: Samples, w0: int, w1: int) -> dict[str, Any]:
"""HAProxy окна: строка BACKEND `api` и сумма слотов."""
ticks = sorted(t for t in samples.lb if w0 <= t <= w1)
def backend(tick: int) -> dict[str, str]:
return next(
(r for r in samples.lb[tick] if r.get("svname") == "BACKEND"),
{},
)
def value(row: dict[str, str], key: str) -> int:
text = row.get(key) or "0"
return int(text) if text.isdigit() else 0
def slots(tick: int, key: str) -> int:
return sum(
value(r, key)
for r in samples.lb[tick]
if r.get("svname") not in ("BACKEND", "FRONTEND")
)
if len(ticks) < 2:
return {}
first, last = backend(ticks[0]), backend(ticks[-1])
return {
"lb_qtime_ms": value(last, "qtime"),
"lb_ctime_ms": value(last, "ctime"),
"lb_rtime_ms": value(last, "rtime"),
"lb_ttime_ms": value(last, "ttime"),
"lb_scur_max": max(value(backend(t), "scur") for t in ticks),
"lb_qcur_max": max(value(backend(t), "qcur") for t in ticks),
"lb_retries": value(last, "wretr") - value(first, "wretr"),
"lb_redispatches": value(last, "wredis") - value(first, "wredis"),
"lb_econ": value(last, "econ") - value(first, "econ"),
"lb_eresp": value(last, "eresp") - value(first, "eresp"),
"lb_chkdown": slots(ticks[-1], "chkdown") - slots(ticks[0], "chkdown"),
"lb_5xx": value(last, "hrsp_5xx") - value(first, "hrsp_5xx"),
}
def count_too_many(log: str) -> int:
return sum(1 for line in log.splitlines() if TOO_MANY_CLIENTS in line)
# ------------------------------------------------- вердикт и признаки
@dataclass(frozen=True)
class Limits:
"""Лимиты конфигурации, против которых считаются признаки."""
replicas: int
db_cpus: float
lb_cpus: float
generator_cpus: float
generator_cpu_limit: float
vm_cpus: float
slo_p95_ms: float = 200
slo_error_ratio: float = 0.01
def generator_bound(row: dict[str, Any], limits: Limits) -> bool:
"""Брак генератора (решение 6; третий признак — замена из apply:
`vus` k6 — выборка раз в секунду, «свободные VU» по нему не видны)."""
cpu = row.get("generator_cpu_cores")
if cpu is not None and cpu >= limits.generator_cpu_limit:
return True
if (row.get("generator_throttled_ratio") or 0) >= THRESHOLDS[
"generator_throttled"
]:
return True
return bool(
row.get("missing")
and cpu is not None
and limits.generator_cpus
and cpu
>= THRESHOLDS["generator_missing_cpu_share"] * limits.generator_cpus,
)
def verdict(
row: dict[str, Any],
limits: Limits,
*,
aborted: bool,
) -> tuple[str, str]:
"""Итог ступени: `pass`, `fail` или `generator` и причина."""
if generator_bound(row, limits):
return "generator", "generator"
p95 = row.get("p95_ms")
errors = row.get("slo_error_ratio")
slow = p95 is None or p95 > limits.slo_p95_ms
bad = errors is None or errors > limits.slo_error_ratio
if aborted:
return "fail", "abort"
if slow and bad:
return "fail", "p95+errors"
if slow:
return "fail", "p95"
if bad:
return "fail", "errors"
return "pass", ""
def signs(row: dict[str, Any], limits: Limits) -> list[str]:
"""Сработавшие признаки ступени (решение 9), кроме `host_cost`."""
t = THRESHOLDS
found = []
def val(key: str) -> float:
value = row.get(key)
return 0.0 if value is None else float(value)
per_replica = val("app_cpu_cores") / max(1, limits.replicas)
if (
per_replica >= t["app_cpu_cores"]
or val("app_throttled_ratio") >= t["throttled_ratio"]
):
found.append("app")
if (
val("db_cpu_cores") >= t["db_cpu_share"] * limits.db_cpus
or val("db_throttled_ratio") >= t["throttled_ratio"]
):
found.append("db_cpu")
active = sum(
val(f"db_aas_{name}")
for name in ("cpu", "wal", "lock", "lwlock", "io", "other")
)
# доля ожиданий осмысленна, только когда база занята: при 0,1
# активного соединения одно ожидание WAL в отсчёте — уже 100 %
# (найдено в apply, самопроверка)
busy = active >= t["db_active_min"]
if busy and val("db_aas_wal") >= t["wait_share"] * active:
found.append("db_wal")
if (
busy
and val("db_aas_lock") + val("db_aas_lwlock")
>= t["wait_share"] * active
):
found.append("db_lock")
if (
val("pool_busy_share") >= t["pool_busy_share"]
and per_replica < t["app_cpu_cores"]
) or val("db_too_many_clients") > 0:
found.append("pool")
if (
val("lb_cpu_cores") >= t["lb_cpu_share"] * limits.lb_cpus
or val("lb_throttled_ratio") >= t["throttled_ratio"]
or val("lb_econ") > 0
or val("lb_eresp") > 0
):
found.append("lb")
if (
val("vm_busy_cores") >= t["vm_busy_share"] * limits.vm_cpus
or val("vm_cpu_pressure") >= t["vm_cpu_pressure"]
):
found.append("host")
if generator_bound(row, limits):
found.append("generator")
return found
def per_request(cores: float | None, rps: float | None) -> float | None:
"""CPU на успешный запрос, мкс."""
if cores is None or not rps:
return None
return round(cores * 1e6 / rps, 1)
# ------------------------------------------------------- таблицы метки
def read_csv(path: Path) -> list[dict[str, str]]:
if not path.is_file():
return []
with path.open(encoding="utf-8", newline="") as file:
return list(csv.DictReader(file))
def chosen_steps(
runs: list[dict[str, str]],
searches: list[dict[str, str]],
) -> dict[str, list[dict[str, str]]]:
"""Ступени попыток из `searches.csv`: прерванные попытки в сводку не
входят (решение 5)."""
attempts = {s["cell"]: s["attempt"] for s in searches}
chosen: dict[str, list[dict[str, str]]] = {}
for run in runs:
if (
run.get("cell") in attempts
and run["attempt"] == attempts[run["cell"]]
):
chosen.setdefault(run["cell"], []).append(run)
return chosen
def step_at(
steps: list[dict[str, str]],
level: float | None,
) -> dict[str, str] | None:
"""Последняя ступень ячейки с целевым уровнем `level`."""
if level is None:
return None
found = [s for s in steps if num(s.get("level")) == level]
return found[-1] if found else None
def fail_step(
steps: list[dict[str, str]],
r_star: float,
) -> dict[str, str] | None:
"""Ближайшая нарушившая ступень выше R*."""
above = [
s
for s in steps
if s.get("slo_pass") == "False" and (num(s["level"]) or 0) > r_star
]
return min(above, key=lambda s: num(s["level"]) or 0) if above else None
def majority(lists: list[list[str]]) -> list[str]:
"""Признаки, сработавшие хотя бы в половине повторов (то же, что
порог на медиане метрики по повторам)."""
if not lists:
return []
need = len(lists) / 2
found = [
name
for name in SIGN_ORDER
if sum(name in items for items in lists) >= need
]
return found
def config_stats(
meta: dict[str, Any],
runs: list[dict[str, str]],
searches: list[dict[str, str]],
) -> list[dict[str, Any]]:
"""Сводка по конфигурациям серии в порядке профиля."""
chosen = chosen_steps(runs, searches)
profile = meta["profile"]
fixed = profile.get("fixed_rps") or []
order = [
config_key(*unpack_config(config))
for config in profile.get("configs", [])
]
db_limits = {
config_key(*parts): parts[4] or str(profile.get("db_cpus", ""))
for parts in map(unpack_config, profile.get("configs", []))
}
result = []
for key in order:
cells = [s for s in searches if s["config"] == key]
if not cells:
continue
ok = [s for s in cells if s["status"] == "complete"]
r_stars = [num(s["r_star_rps"]) for s in ok]
at_r: list[list[str]] = []
at_f: list[list[str]] = []
reasons = []
db_at_r = []
db_thr_at_r: list[float | None] = []
# PostgreSQL на R* и первой нарушившей ступени (change
# scaling-improvement, решение 7): уровень, CPU на запрос, новые
# сессии за окно, соединения
pg_r: dict[str, list[float | None]] = {
"db_us": [],
"sessions": [],
"conn": [],
}
pg_f: dict[str, list[float | None]] = {
"level": [],
"db_us": [],
"sessions": [],
"conn": [],
}
app_at_r = []
app_sec = []
marginals: list[float | None] = []
fixed_rows: dict[int, list[dict[str, str]]] = {
lvl: [] for lvl in fixed
}
for s in cells:
steps = chosen.get(s["cell"], [])
for lvl in fixed:
row = next(
(
x
for x in steps
if x.get("step_kind") == "fixed"
and num(x["level"]) == lvl
),
None,
)
if row is not None:
fixed_rows[lvl].append(row)
if s["status"] != "complete":
continue
r_star = num(s["r_star_rps"]) or 0
r_row = step_at(steps, r_star)
f_row = fail_step(steps, r_star)
if r_row is not None:
at_r.append((r_row.get("signs") or "").split())
db_at_r.append(num(r_row.get("db_cpu_cores")))
db_thr_at_r.append(num(r_row.get("db_throttled_ratio")))
app_at_r.append(num(r_row.get("app_cpu_cores")))
pg_r["db_us"].append(num(r_row.get("db_us_per_request")))
pg_r["sessions"].append(num(r_row.get("db_sessions_new")))
pg_r["conn"].append(num(r_row.get("db_conn_max")))
if f_row is not None:
at_f.append((f_row.get("signs") or "").split())
reasons.append(f_row.get("fail_reason") or "")
pg_f["level"].append(num(f_row.get("level")))
pg_f["db_us"].append(num(f_row.get("db_us_per_request")))
pg_f["sessions"].append(num(f_row.get("db_sessions_new")))
pg_f["conn"].append(num(f_row.get("db_conn_max")))
app_sec.extend(num(x.get("app_cpu_second_max")) for x in steps)
marginals.append(marginal_us(steps))
replicas = int(cells[0]["replicas"])
result.append(
{
"config": key,
"replicas": replicas,
"replica_cpus": cells[0]["replica_cpus"],
"db_max_connections": int(cells[0]["db_max_connections"]),
"cells": len(cells),
"complete": len(ok),
"generator_bound": [
s["cell"]
for s in cells
if s["status"] == "generator_bound"
],
"r_stars": [v for v in r_stars if v is not None],
"r_star": median(r_stars),
"fixed": {
lvl: {
"p50": median(num(x["p50_ms"]) for x in rows),
"p95": median(num(x["p95_ms"]) for x in rows),
"p99": median(num(x["p99_ms"]) for x in rows),
"p99_all": [num(x["p99_ms"]) for x in rows],
"passed": sum(x["slo_pass"] == "True" for x in rows),
"app_us": median(
num(x["app_us_per_request"]) for x in rows
),
"throttled": median(
num(x["app_throttled_ratio"]) for x in rows
),
"throttled_all": [
num(x["app_throttled_ratio"]) for x in rows
],
"too_many": sum(
int(num(x["db_too_many_clients"]) or 0)
for x in rows
),
"codes": " ".join(
x.get("error_codes") or "" for x in rows
).split(),
}
for lvl, rows in fixed_rows.items()
},
"fail_reasons": reasons,
"signs_r": majority(at_r),
"signs_f": majority(at_f),
"db_cpu_r": median(db_at_r),
"db_throttled_r": median(db_thr_at_r),
"db_us_r": median(pg_r["db_us"]),
"sessions_r": median(pg_r["sessions"]),
"conn_r": median(pg_r["conn"]),
"level_f": median(pg_f["level"]),
"db_us_f": median(pg_f["db_us"]),
"sessions_f": median(pg_f["sessions"]),
"conn_f": median(pg_f["conn"]),
"db_cpus": db_limits.get(key, ""),
"app_cpu_r": median(app_at_r),
"app_cpu_second_max": max(
(v for v in app_sec if v is not None),
default=None,
),
"app_us_marginal": median(marginals),
},
)
base = next(
(
c
for c in result
if c["replicas"] == 1
and float(c["replica_cpus"]) == 1
and c["db_max_connections"] == 100
),
None,
)
for c in result:
c["host_cost"] = None
if base is not None:
apply_host_cost(c, base["app_us_marginal"])
return result
def apply_host_cost(c: dict[str, Any], reference: float | None) -> None:
"""Признак `host_cost`: предельный CPU реплик на запрос конфигурации
против 1 × 1 (найдено в apply: CPU на запрос на 500/с у k реплик
включает постоянные расходы каждой реплики — проверки LB, простой
event loop — и растёт с k без всякого хоста; наклон их исключает)."""
own = c.get("app_us_marginal")
if not own or not reference:
return
c["host_cost"] = round(own / reference, 3)
if own >= THRESHOLDS["host_cost"] * reference:
for key in ("signs_r", "signs_f"):
if "host_cost" not in c[key]:
c[key] = [*c[key], "host_cost"]
def marginal_us(steps: list[dict[str, str]]) -> float | None:
"""Предельный CPU реплик на успешный запрос, мкс: наклон прямой
`app_cpu_cores` от `rps` по ступеням ячейки, прошедшим SLO
(наименьшие квадраты); меньше двух уровней — None."""
points = [
(num(s.get("rps")), num(s.get("app_cpu_cores")))
for s in steps
if s.get("slo_pass") == "True"
]
points = [(x, y) for x, y in points if x and y is not None]
if len({round(x) for x, _ in points}) < 2:
return None
mean_x = statistics.mean(x for x, _ in points)
mean_y = statistics.mean(y for _, y in points)
var = sum((x - mean_x) ** 2 for x, _ in points)
cov = sum((x - mean_x) * (y - mean_y) for x, y in points)
return round(cov / var * 1e6, 1) if var else None
def saturation(
stats: list[dict[str, Any]],
reference: float | None = None,
) -> dict[str, Any]:
"""Точка насыщения серии роста (решение 9): наименьшее k ≤ 5, при
котором медиана R*(k + 1) < 1,1 × R*(k); эффективность —
R*(k) / (k × R*(1 × 1))."""
by_k = {
c["replicas"]: c
for c in stats
if float(c["replica_cpus"]) == 1 and c["db_max_connections"] == 100
}
base = (
reference
if reference is not None
else (by_k[1]["r_star"] if 1 in by_k else None)
)
efficiency = {
k: (
round(c["r_star"] / (k * base), 3)
if base and c["r_star"] is not None
else None
)
for k, c in sorted(by_k.items())
}
k_sat = None
for k in sorted(by_k):
nxt = by_k.get(k + 1)
if k > 5 or nxt is None:
continue
cur_r, nxt_r = by_k[k]["r_star"], nxt["r_star"]
if (
cur_r
and nxt_r is not None
and nxt_r < THRESHOLDS["k_sat_gain"] * cur_r
):
k_sat = k
break
return {"efficiency": efficiency, "k_sat": k_sat, "by_k": by_k}
def fmt(value: Any, digits: int = 0) -> str:
if value is None or value == "":
return "—"
if isinstance(value, str):
return value
return f"{value:.{digits}f}"
def render_summary(
meta: dict[str, Any],
runs: list[dict[str, str]],
searches: list[dict[str, str]],
) -> str:
"""`summary.md` серии: методика, R* по конфигурациям, задержки на
фиксированных уровнях, признаки на R* и первой нарушившей, брак."""
profile = meta["profile"]
stats = config_stats(meta, runs, searches)
fixed = profile.get("fixed_rps") or []
mix = ", ".join(f"{n} {s:g}" for n, s in profile.get("mix", []))
lines = [
f"# Серия трека 1 `{profile.get('series')}`: метка `{meta['label']}`",
"",
f"- Статус: **{meta['status']}**"
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
f"- Commit: `{meta['git']['commit']}`"
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
f"- Профиль `{profile['name']}`: смесь {mix}; читаемый чат "
f"{profile['sizes'][0]} сообщений, чатов записи "
f"{profile['write_chats']}; фиксированные уровни "
f"{', '.join(map(str, fixed))} запросов/с; поиск от "
f"{profile['start_rps_per_replica']} × реплик по лестнице "
f"{profile['ladder_base']} × {profile['step_factor']}^i, делений "
f"{profile['bisect_steps']}; прогрев {profile['first_warmup_s']} с "
f"(первая ступень и после паузы) / {profile['step_warmup_s']} с, "
f"окно {profile['step_measure_s']} с; SLO p95 ≤ "
f"{profile['slo_p95_ms']:g} мс, ошибок с недосланными ≤ "
f"{profile['slo_error_ratio'] * 100:g} %; повторов "
f"{profile['repeats']}, порядок кругами (зерно "
f"{profile['order_seed']})",
f"- Лимиты без swap: PostgreSQL {profile['db_cpus']} CPU / "
f"{profile['db_memory']} (у конфигураций `-pg<CPU>` — свой лимит "
f"CPU), HAProxy {profile['lb_cpus']} / "
f"{profile['lb_memory']}, Valkey {profile['cache_cpus']} / "
f"{profile['cache_memory']}, k6 {profile['generator_cpus']} / "
f"{profile['generator_memory']}, сэмплер {profile['sampler_cpus']} "
f"/ {profile['sampler_memory']}",
"- Значения — медиана (min–max) по повторам; R* — наибольший "
"уровень поиска, прошедший SLO; признаки — сработавшие хотя бы в "
"половине повторов (`THRESHOLDS` в `bench/scaling.py`)",
"",
]
budget = meta.get("budget") or {}
if budget:
lines += [
"## Бюджет конфигураций",
"",
"| Конфигурация | Σ CPU стенда и генератора | Σ памяти, ГиБ | "
"CPU ВМ | Память ВМ, ГиБ |",
"|---|---|---|---|---|",
]
for key, item in budget.items():
lines.append(
f"| {key} | {item['cpus']:g} | {item['memory_gib']:g} | "
f"{item['vm_cpus']} | {item['vm_memory_gib']} |",
)
lines.append("")
lines += [
"## R* по конфигурациям",
"",
"| Конфигурация | Ячеек | R*, запросов/с | Причина первого нарушения "
"| CPU реплик на R*, ядер | Посекундный max CPU реплики | CPU "
"PostgreSQL на R*, ядер | Признаки на R* | Признаки на первой "
"нарушившей | Предельный CPU реплик на запрос, мкс | host_cost |",
"|---|---|---|---|---|---|---|---|---|---|---|",
]
for c in stats:
reasons = sorted(set(c["fail_reasons"]))
lines.append(
f"| {c['config']} | {c['complete']}/{c['cells']} | "
f"{spread(c['r_stars'])} | {', '.join(reasons) or '—'} | "
f"{fmt(c['app_cpu_r'], 2)} | {fmt(c['app_cpu_second_max'], 2)} "
f"| {fmt(c['db_cpu_r'], 2)} | {' '.join(c['signs_r']) or '—'} | "
f"{' '.join(c['signs_f']) or '—'} | "
f"{fmt(c['app_us_marginal'], 0)} | {fmt(c['host_cost'], 2)} |",
)
lines.append("")
if any("-pg" in c["config"] for c in stats):
# серия db: насыщение PostgreSQL — CPU у лимита и троттлинг CFS
lines += [
"## PostgreSQL на R*",
"",
"| Конфигурация | Лимит CPU PostgreSQL | CPU PostgreSQL на R*, "
"ядер | Доля лимита | Троттлинг PostgreSQL на R*, доля "
"периодов |",
"|---|---|---|---|---|",
]
for c in stats:
limit = num(c.get("db_cpus"))
share = (
c["db_cpu_r"] / limit
if limit and c["db_cpu_r"] is not None
else None
)
lines.append(
f"| {c['config']} | {c.get('db_cpus') or '—'} | "
f"{fmt(c['db_cpu_r'], 2)} | {fmt(share, 2)} | "
f"{fmt(c.get('db_throttled_r'), 3)} |",
)
lines.append("")
for lvl in fixed:
lines += [
f"## Фиксированный уровень {lvl} запросов/с",
"",
"| Конфигурация | Прошли SLO | p50, мс | p95, мс | p99, мс | "
"CPU реплик на запрос, мкс | Троттлинг реплик, доля периодов "
"| Отказов «too many clients» | Коды ошибок |",
"|---|---|---|---|---|---|---|---|---|",
]
for c in stats:
f = c["fixed"].get(lvl) or {}
codes: dict[str, int] = {}
for item in f.get("codes", []):
status, _, count = item.partition(":")
codes[status] = codes.get(status, 0) + int(count or 0)
lines.append(
f"| {c['config']} | {f.get('passed', 0)}/{c['cells']} | "
f"{fmt(f.get('p50'), 1)} | {fmt(f.get('p95'), 1)} | "
f"{spread(f.get('p99_all', []), 1)} | "
f"{fmt(f.get('app_us'), 0)} | "
f"{spread(f.get('throttled_all', []), 4)} | "
f"{f.get('too_many', 0)} | "
+ (" ".join(f"{k}:{v}" for k, v in codes.items()) or "—")
+ " |",
)
lines.append("")
sat = saturation(stats)
if sat["efficiency"] and len(sat["by_k"]) > 2:
lines += [
"## Рост числа реплик",
"",
"| Реплик | R*, запросов/с | R* / (k × R*(1 × 1)) |",
"|---|---|---|",
]
for k, c in sorted(sat["by_k"].items()):
lines.append(
f"| {k} | {spread(c['r_stars'])} | "
f"{fmt(sat['efficiency'].get(k), 2)} |",
)
lines += [
"",
"Точка насыщения k_sat (медиана R*(k + 1) < "
f"{THRESHOLDS['k_sat_gain']} × R*(k)): "
+ (str(sat["k_sat"]) if sat["k_sat"] else "нет в пределах серии"),
"",
]
bound = [cell for c in stats for cell in c["generator_bound"]]
lines += ["## Брак генератора", ""]
lines.append(
"Ячейки с R* — нижней границей: " + ", ".join(bound)
if bound
else "Нет: ни одна ступень не упёрлась в генератор.",
)
lines += [
"",
"## Ячейки",
"",
"| № | Ячейка | Попытка | Статус | R* | Первая нарушившая | "
"Разрешение | Ступеней | Записей 201 / отправлено / в БД | "
"Выгрузок в swap | Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|",
]
for s in sorted(searches, key=lambda x: int(x["order"])):
lines.append(
f"| {s['order']} | {s['cell']} | {s['attempt']} | {s['status']} "
f"| {s['r_star_rps']}{' (нижняя граница)' if s['r_star_bound'] == 'lower' else ''} "
f"| {s['first_fail_rps'] or '—'} {s['first_fail_reason'] or ''} | "
f"{s['resolution'] or '—'} | {s['steps']} | {s['writes_ok']} / "
f"{s['writes_sent']} / {s['write_chat_messages']} | "
f"{s['host_swapouts'] or 0} | {s['flags'] or '—'} |",
)
lines.append("")
if meta.get("host_warnings"):
lines += ["## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
lines.append("")
return "\n".join(lines)
# ------------------------------------------------------- самопроверка
def selftest() -> int:
"""Синтетические исходы поиска и строки сэмплера (задача 1.5)."""
failures: list[str] = []
def expect(name: str, got: Any, want: Any) -> None:
if got != want:
failures.append(f"{name}: got {got!r}, want {want!r}")
steps = ladder(100, 1.25)
expect("ladder head", steps[:6], [100, 125, 156, 195, 244, 305])
for k, per, fixed, want in (
(1, 300, 500, 596),
(2, 300, 500, 745),
(4, 300, 500, 1455),
(6, 300, 500, 1819),
):
expect(f"L0 k={k}", start_level(k, per, fixed, steps), want)
def drive(search: Search, ceiling: float) -> Search:
for _ in range(40):
step = search.next_step()
if step is None:
break
level, kind = step
search.record(level, kind, "pass" if level <= ceiling else "fail")
return search
# подъём: потолок 2 500 при k = 6
s = drive(Search(replicas=6, fixed=(250, 500)), 2500)
fail = s.first_fail
expect("up status", s.status, "complete")
expect("up r*<=ceiling", s.r_star <= 2500, True)
expect("up resolution", fail is not None and fail / s.r_star <= 1.06, True)
expect(
"up kinds",
{k for _, k, _ in s.outcomes},
{"fixed", "up", "bisect"},
)
# спуск от первой нарушенной: потолок 520 при k = 4 (L0 = 1455)
s = drive(Search(replicas=4, fixed=(250, 500)), 520)
expect("down r*", s.r_star >= 500 and s.r_star <= 520, True)
expect("down resolution", (s.first_fail or 0) / s.r_star <= 1.06, True)
# оба фиксированных уровня провалены: спуск от 250
s = drive(Search(replicas=1, fixed=(250, 500)), 130)
expect("low r*", s.r_star, 125)
expect("low first", s.outcomes[2][:2], (244, "down"))
s = drive(Search(replicas=1, fixed=(250, 500)), 50)
expect("floor", (s.r_star, s.flags), (0, ["floor"]))
# брак генератора: R* — наибольший пройденный уровень
s = Search(replicas=2, fixed=(250, 500))
for outcome in ("pass", "pass", "pass", "generator"):
level, kind = s.next_step() or (0, "")
s.record(level, kind, outcome)
expect(
"generator",
(s.status, s.r_star, s.done),
("generator_bound", 745, True),
)
# порядок кругами
plan = plan_cells(
[(1, "4", "2g", 100), (2, "2", "1g", 100), (4, "1", "512m", 100)],
3,
20261011,
)
for r in (1, 2, 3):
expect(
f"round {r}",
sorted(p["config"] for p in plan if p["round"] == r),
["1x4", "2x2", "4x1"],
)
for a, b in zip(plan, plan[1:], strict=False):
if a["config"] == b["config"]:
failures.append(f"repeats adjacent: {a['cell']} {b['cell']}")
# сэмплер: известные разности
ns = 10**18
lines = []
for i in range(3):
t = ns + i * 10**9
lines.append(
f"cg {t} api aaaaaaaaaaaa {i * 900_000} {i * 10} {i * 2} "
f"{i * 30_000} {100 * 2**20} {i * 5000}",
)
lines.append(
f"cg {t} postgres bbbbbbbbbbbb {i * 500_000} {i * 10} 0 0 "
f"{200 * 2**20} 0",
)
lines.append(f"vm {t} {i * 400} {i * 1200} {i * 100_000}")
lines.append(
f"lb {t} # pxname,svname,qcur,scur,econ,eresp,wretr,wredis,"
"chkdown,hrsp_5xx,qtime,ctime,rtime,ttime",
)
lines.append(f"lb {t} api,api-1,0,{3 + i},0,0,0,0,0,0,0,0,0,0")
lines.append(f"lb {t} api,BACKEND,0,{3 + i},0,0,{i},0,0,{i},0,1,5,6")
pg = []
for i in range(3):
ms = (ns + i * 10**9) // 1_000_000
pg.append(f"db|{ms}|{i * 100}|0|{i}|0|5")
pg.append(f"act|{ms}|10.0.0.5|active|CPU|-|2")
pg.append(f"act|{ms}|10.0.0.5|idle|Client|ClientRead|1")
pg.append(f"act|{ms}|10.0.0.6|active|LWLock|WALWrite|3")
pg.append(f"wal|{ms}|0|0|{i * 2**20}||")
pg.append(f"io|{ms}|0|{i * 7}|||")
parsed = parse_samples("\n".join(lines), "\n".join(pg))
row, seconds = analyze_samples(
parsed,
ns,
ns + 2 * 10**9,
addresses={"10.0.0.5": "aaaaaaaaaaaa", "10.0.0.6": "cccccccccccc"},
)
expect("app cpu", row["app_cpu_cores"], 0.9)
expect("app throttled", row["app_throttled_ratio"], 0.2)
expect("app throttled ms", row["app_throttled_ms"], 60.0)
expect("db cpu", row["db_cpu_cores"], 0.5)
expect("vm busy", row["vm_busy_cores"], 4.0)
expect("vm psi", row["vm_cpu_pressure"], 0.1)
expect("commits", row["db_commits_per_s"], 100.0)
expect("wal", row["db_wal_mib_per_s"], 1.0)
expect("fsyncs", row["db_wal_fsyncs_per_s"], 7.0)
expect("aas cpu", row["db_aas_cpu"], 2.0)
expect("aas wal", row["db_aas_wal"], 3.0)
expect("pool busy", row["pool_busy_share"], 1.0)
expect("conn replica", row["db_conn_replica_max"], 3)
expect("lb retries", row["lb_retries"], 2)
expect("lb scur", row["lb_scur_max"], 5)
expect(
"seconds",
len([s for s in seconds if s["role"] == "api"]),
2,
)
limits = Limits(
replicas=1,
db_cpus=2,
lb_cpus=1,
generator_cpus=2,
generator_cpu_limit=1.8,
vm_cpus=12,
)
expect("signs", signs(row, limits), ["app", "db_wal"])
expect(
"signs pool",
signs(
{**row, "app_cpu_cores": 0.5, "app_throttled_ratio": 0.0},
limits,
),
["db_wal", "pool"],
)
expect(
"generator cpu",
generator_bound({"generator_cpu_cores": 1.85}, limits),
True,
)
expect(
"generator missing",
generator_bound(
{"generator_cpu_cores": 1.5, "missing": 3},
limits,
),
True,
)
expect(
"verdict pass",
verdict({"p95_ms": 50, "slo_error_ratio": 0.0}, limits, aborted=False),
("pass", ""),
)
expect(
"verdict p95",
verdict(
{"p95_ms": 250, "slo_error_ratio": 0.0},
limits,
aborted=False,
),
("fail", "p95"),
)
if failures:
print("selftest failed:\n " + "\n ".join(failures), flush=True)
return 1
print("selftest ok", flush=True)
return 0
if __name__ == "__main__":
if sys.argv[1:] == ["--selftest"]:
sys.exit(selftest())
print("usage: uv run python bench/scaling.py --selftest", flush=True)
sys.exit(2)