"""Трек 1 «Цена масштабирования»: чистые функции серий без docker. Лестница и поиск максимальной устойчивой интенсивности (R*), разбор итога k6 ступени, строк сэмплера и PostgreSQL, агрегаты окна замера, вердикт SLO и брака генератора, признаки узкого места, строки таблиц и сводка серии. Методика — design change `scaling-bench` (решения 3, 6–9), то, что стенд обязан делать, — спек `load-benchmark`. Только stdlib: вызывает `bench/run.py`, самопроверка — `uv run python bench/scaling.py --selftest`. """ import csv import math import random import statistics import sys from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path from typing import Any # новые столбцы runs.csv: только в конец (решение 8) SCALING_FIELDS = [ "series", "cell", "attempt", "config", "replicas", "replica_cpus", "replica_memory", "db_max_connections", "step", "step_kind", "target_rps", "sent", "missing", "dropped_iterations", "slo_error_ratio", "slo_pass", "fail_reason", "read_p95_ms", "write_p95_ms", "me_p95_ms", "read_errors", "write_errors", "me_errors", "short_pages", "vus_max", "generator_throttled_ratio", "generator_mem_peak_mib", "app_cpu_replica_max", "app_cpu_second_max", "app_throttled_ratio", "app_throttled_ms", "app_mem_peak_mib", "app_us_per_request", "db_throttled_ratio", "db_mem_peak_mib", "db_us_per_request", "lb_cpu_cores", "lb_throttled_ratio", "cache_cpu_cores", "vm_busy_cores", "vm_cpu_pressure", "db_commits_per_s", "db_rollbacks_per_s", "db_wal_mib_per_s", "db_wal_fsyncs_per_s", "db_sessions_new", "db_conn_max", "db_conn_replica_max", "db_aas_cpu", "db_aas_wal", "db_aas_lock", "db_aas_lwlock", "db_aas_io", "db_aas_other", "db_idle_in_tx_mean", "pool_busy_share", "db_too_many_clients", "lb_qtime_ms", "lb_ctime_ms", "lb_rtime_ms", "lb_ttime_ms", "lb_scur_max", "lb_qcur_max", "lb_retries", "lb_redispatches", "lb_econ", "lb_eresp", "lb_chkdown", "lb_5xx", "instances", "instance_share_min", "instance_share_max", "host_swapouts", "signs", # найдено в apply: коды неуспешных ответов окна (H3, 500 или 503) и # уровень ступени лестницы; target_rps — сумма интенсивностей # сценариев, может отличаться от уровня на ±1 "error_codes", "level", ] SEARCH_FIELDS = [ "series", "cell", "attempt", "config", "replicas", "replica_cpus", "replica_memory", "db_max_connections", "repeat", "round", "order", "started_at", "finished_at", "status", "r_star_rps", "r_star_bound", "first_fail_rps", "first_fail_reason", "steps", "resolution", "fixed_rps_1", "p50_fixed_1_ms", "p95_fixed_1_ms", "p99_fixed_1_ms", "pass_fixed_1", "fixed_rps_2", "p50_fixed_2_ms", "p95_fixed_2_ms", "p99_fixed_2_ms", "pass_fixed_2", "app_us_per_request_fixed_2", "app_cpu_second_max", "app_throttled_ratio_fixed_2", "writes_ok", "writes_sent", "write_chat_messages", "read_chat_messages", "host_power", "host_memory_free_percent", "host_swapouts", "other_containers", "flags", ] SAMPLE_FIELDS = [ "cell", "attempt", "step", "t_s", "role", "instance", "cpu_cores", "throttled_ms", "pressure_ms", "mem_mib", ] # пороги признаков узкого места (решение 9) и брака генератора (решение 6): # заданы до прогонов THRESHOLDS: dict[str, float] = { # CPU реплики в ядрах: один процесс больше одного ядра не берёт (H1) "app_cpu_cores": 0.9, # доля периодов CFS с троттлингом "throttled_ratio": 0.10, # CPU PostgreSQL и LB — доля лимита "db_cpu_share": 0.9, "lb_cpu_share": 0.9, # доля ожиданий класса от всех активных соединений "wait_share": 0.30, # признаки ожиданий — только при среднем ≥ 1 активного соединения # (найдено в apply: без порога одно ожидание на почти пустой базе) "db_active_min": 1.0, "pool_busy_share": 0.5, # хост: занятые ядра ВМ — доля её CPU, PSI CPU — доля времени "vm_busy_share": 0.9, "vm_cpu_pressure": 0.20, # предельный CPU реплик на запрос (наклон по ступеням, прошедшим SLO) # против 1 × 1 серии роста (marginal_us, найдено в apply) "host_cost": 1.2, # генератор: троттлинг и недосланные при CPU k6 не меньше доли лимита "generator_throttled": 0.10, "generator_missing_cpu_share": 0.7, # точка насыщения: реплика добавила меньше 10 % "k_sat_gain": 1.1, } # признаки общего ресурса: узкое место «реплики» называется без них SHARED_SIGNS = ("db_cpu", "db_wal", "db_lock", "pool", "lb", "host") SIGN_ORDER = ("app", *SHARED_SIGNS, "host_cost", "generator") MIX_NAMES = ("read", "write", "me") # коды неуспешных ответов окна, которые k6 выделяет подметриками ERROR_STATUSES = (0, 400, 401, 403, 404, 409, 422, 429, 500, 502, 503, 504) TOO_MANY_CLIENTS = "sorry, too many clients already" def num(value: Any) -> float | None: """Число из CSV или итога; пусто — None.""" if value is None or value == "": return None return float(value) def rounded(value: float | None, digits: int = 3) -> float | None: return None if value is None else round(value, digits) def median(values: Iterable[float | None]) -> float | None: present = [v for v in values if v is not None] return statistics.median(present) if present else None def spread(values: Iterable[float | None], digits: int = 0) -> str: """Медиана и min–max повторов, как `cell()` матрицы.""" present = [v for v in values if v is not None] if not present: return "—" mid = statistics.median(present) if len(present) == 1: return f"{mid:.{digits}f}" return ( f"{mid:.{digits}f} ({min(present):.{digits}f}–" f"{max(present):.{digits}f})" ) # ----------------------------------------------------- конфигурации def config_key( replicas: int, cpus: str, _memory: str, max_conn: int, db_cpus: str = "", ) -> str: """Имя конфигурации: `4x1`, у предела соединений не 100 — `4x1-c50`, у своего лимита CPU PostgreSQL (серия `db`) — `4x1-pg0.5`. Память реплики в имя не входит: в сериях она задана числом CPU.""" key = f"{replicas}x{cpus}" if max_conn != 100: key = f"{key}-c{max_conn}" return f"{key}-pg{db_cpus}" if db_cpus else key def unpack_config(config: Any) -> tuple[int, str, str, int, str]: """Конфигурация серии: (реплик, CPU и память реплики, max_connections [, CPU PostgreSQL]). Пятый элемент — только у серии `db`; без него лимит PostgreSQL — профиля.""" replicas, cpus, memory, max_conn, *rest = config return ( int(replicas), str(cpus), str(memory), int(max_conn), str(rest[0]) if rest else "", ) def plan_cells( configs: list[tuple[Any, ...]], repeats: int, seed: int, ) -> list[dict[str, Any]]: """Порядок ячеек кругами: в круге r — перестановка конфигураций генератором `Random(seed + r)`. Если круг начался с конфигурации, на которой кончился прежний, она меняется местами со второй (решение 4). """ plan: list[dict[str, Any]] = [] previous = None for round_no in range(1, repeats + 1): order = list(range(len(configs))) random.Random(seed + round_no).shuffle(order) if len(order) > 1 and order[0] == previous: order[0], order[1] = order[1], order[0] for index in order: replicas, cpus, memory, max_conn, db_cpus = unpack_config( configs[index], ) key = config_key(replicas, cpus, memory, max_conn, db_cpus) cell = { "cell": f"{key}-r{round_no}", "config": key, "replicas": replicas, "replica_cpus": cpus, "replica_memory": memory, "db_max_connections": max_conn, "repeat": round_no, "round": round_no, "order": len(plan) + 1, } if db_cpus: cell["db_cpus"] = db_cpus plan.append(cell) previous = order[-1] return plan # ----------------------------------------------------- лестница и поиск def ladder(base: int, factor: float, count: int = 48) -> list[int]: """Ступеньки `round(base × factor^i)` по возрастанию, без повторов.""" values: list[int] = [] for i in range(count): value = round(base * factor**i) if not values or value > values[-1]: values.append(value) return values def start_level( replicas: int, per_replica: int, passed_fixed: int | None, steps: list[int], ) -> int: """Начало подъёма L0: наименьшая ступенька ≥ max(per_replica × k, наибольший пройденный фиксированный уровень) и строго больше него.""" floor = max(per_replica * replicas, passed_fixed or 0) for value in steps: if value >= floor and (passed_fixed is None or value > passed_fixed): return value raise ValueError(f"ladder ends below {floor}") @dataclass class Search: """Автомат ступеней ячейки: фиксированные уровни, подъём или спуск по лестнице, деления по среднему геометрическому (решение 3). `next_step()` — следующий уровень и вид ступени или None; итог ступени — `record()`: `pass`, `fail` или `generator` (брак).""" replicas: int fixed: tuple[int, ...] base: int = 100 factor: float = 1.25 per_replica: int = 300 bisect_steps: int = 2 max_steps: int = 12 steps: list[int] = field(default_factory=list) outcomes: list[tuple[int, str, str]] = field(default_factory=list) fixed_left: list[int] = field(default_factory=list) passed_fixed: int | None = None mode: str | None = None current: int | None = None passed: int | None = None failed: int | None = None bisects: int = 0 search_steps: int = 0 done: bool = False status: str = "complete" flags: list[str] = field(default_factory=list) def __post_init__(self) -> None: self.steps = ladder(self.base, self.factor) self.fixed_left = list(self.fixed) def next_step(self) -> tuple[int, str] | None: if self.done: return None if self.fixed_left: return self.fixed_left[0], "fixed" if self.mode is None: self._begin() if self.done or self.current is None or self.mode is None: return None return self.current, self.mode def _below(self, level: int) -> int | None: lower = [v for v in self.steps if v < level] return lower[-1] if lower else None def _above(self, level: int) -> int: return next(v for v in self.steps if v > level) def _begin(self) -> None: passed = [lvl for lvl, kind, out in self.outcomes if out == "pass"] self.passed_fixed = max(passed) if passed else None if self.passed_fixed is None: # ни один фиксированный уровень не пройден: спуск от # наименьшего из них до основания лестницы self.failed = min(self.fixed) if self.fixed else None self.mode = "down" self.current = self._below(self.failed or self.steps[0] + 1) if self.current is None: self._floor() return self.mode = "up" self.current = start_level( self.replicas, self.per_replica, self.passed_fixed, self.steps, ) def _floor(self) -> None: self.passed = 0 self.flags.append("floor") self.done = True def _bisect(self) -> None: self.mode = "bisect" if ( self.passed is None or self.failed is None or self.bisects >= self.bisect_steps ): self.done = True return middle = round(math.sqrt(self.passed * self.failed)) if middle <= self.passed or middle >= self.failed: self.done = True return self.current = middle def record(self, level: int, kind: str, outcome: str) -> None: self.outcomes.append((level, kind, outcome)) if outcome == "generator": self.status = "generator_bound" self.flags.append("generator") passes = [lvl for lvl, _, out in self.outcomes if out == "pass"] self.passed = max(passes) if passes else 0 self.done = True return if kind == "fixed": self.fixed_left.remove(level) return self.search_steps += 1 ok = outcome == "pass" if self.mode == "up": if ok: self.passed = level self.current = self._above(level) else: self.failed = level if self.passed is None: self._descend(level) else: self._bisect() elif self.mode == "down": if ok: self.passed = level self._bisect() else: self.failed = level self._descend(level) else: if ok: self.passed = level else: self.failed = level self.bisects += 1 self._bisect() if not self.done and self.search_steps >= self.max_steps: self.flags.append("search_limit") self.done = True def _descend(self, level: int) -> None: self.mode = "down" below = self._below(level) if self.passed_fixed is not None and ( below is None or below <= self.passed_fixed ): # спуск дошёл до пройденного фиксированного уровня: P = он self.passed = self.passed_fixed self._bisect() return if below is None: self._floor() return self.current = below @property def r_star(self) -> int: return self.passed or 0 @property def first_fail(self) -> int | None: """Ближайший нарушивший уровень выше R*.""" above = [ lvl for lvl, _, out in self.outcomes if out == "fail" and lvl > self.r_star ] return min(above) if above else None @property def resolution(self) -> float | None: fail = self.first_fail if not self.r_star or fail is None: return None return fail / self.r_star # ------------------------------------------------------- итог k6 ступени def metric( metrics: dict[str, Any], name: str, key: str, default: float | None = None, ) -> float | None: value = metrics.get(name, {}).get("values", {}).get(key) return default if value is None else float(value) def targets(level: int, mix: list[tuple[str, float]]) -> dict[str, int]: """Интенсивность сценариев: `round(уровень × доля)` с половиной вверх, как `Math.round` в `mixed.js` (у `round` Python — к чётному).""" return {name: math.floor(level * share + 0.5) for name, share in mix} def parse_k6( data: dict[str, Any], *, level: int, mix: list[tuple[str, float]], measure_s: float, instances: list[str], ) -> dict[str, Any]: """Окно замера ступени из итога k6 (`mixed.js`). Недосланные — `max(0, запланированные − отправленные − 1)` на сценарий: допуск ±1 на границе окна (решение 3). Неуспешные — с короткими страницами. Распределение по `X-Instance-Id` — по счётчику полностью успешных ответов окна.""" m = data["metrics"] hold = "window:hold" plan = targets(level, mix) sent = errors = missing = 0 row: dict[str, Any] = {} for name, rate in plan.items(): sent_x = int( metric(m, f"http_reqs{{{hold},scenario:{name}}}", "count", 0), ) bad_x = int( metric( m, f"http_req_failed{{{hold},scenario:{name}}}", "passes", 0, ), ) planned_x = rate * measure_s missing += max(0, round(planned_x - sent_x - 1)) sent += sent_x errors += bad_x row[f"{name}_errors"] = bad_x row[f"{name}_p95_ms"] = rounded( metric( m, f"http_req_duration{{{hold},expected_response:true," f"scenario:{name}}}", "p(95)", ), ) row[f"{name}_sent_total"] = int( metric(m, f"http_reqs{{scenario:{name}}}", "count", 0), ) row[f"{name}_failed_total"] = int( metric(m, f"http_req_failed{{scenario:{name}}}", "passes", 0), ) short = int(metric(m, f"short_pages{{{hold}}}", "count", 0)) planned = sum(rate * measure_s for rate in plan.values()) failed = errors + short ok = sent - failed duration = f"http_req_duration{{{hold},expected_response:true}}" counts = { ident: int( metric(m, f"instance_ok{{{hold},instance:{ident}}}", "count", 0), ) for ident in [*instances, "-"] } codes = { status: int( metric(m, f"http_reqs{{{hold},status:{status}}}", "count", 0), ) for status in ERROR_STATUSES } total_ok = sum(counts.values()) shares = [counts[i] / total_ok for i in instances] if total_ok else [] row.update( { "target_rps": sum(plan.values()), "planned": planned, "sent": sent, "missing": missing, "requests_ok": ok, "requests_failed": failed, "short_pages": short, "duration_s": measure_s, "rps": round(ok / measure_s, 1) if measure_s else None, "error_ratio": round(failed / sent, 6) if sent else None, "slo_error_ratio": ( round((failed + missing) / planned, 6) if planned else None ), "p50_ms": rounded(metric(m, duration, "p(50)")), "p95_ms": rounded(metric(m, duration, "p(95)")), "p99_ms": rounded(metric(m, duration, "p(99)")), "max_ms": rounded(metric(m, duration, "max")), "check_failed": int(metric(m, f"checks{{{hold}}}", "fails", 0)), "dropped_iterations": int( metric(m, "dropped_iterations", "count", 0), ), "concurrency": int(metric(m, "vus", "max", 0)), "vus_max": int(metric(m, "vus_max", "max", 0)), "instances": " ".join( f"{ident}:{count}" for ident, count in counts.items() if count ), "instances_total": total_ok, "instance_share_min": rounded(min(shares)) if shares else None, "instance_share_max": rounded(max(shares)) if shares else None, "error_codes": " ".join( f"{status}:{count}" for status, count in codes.items() if count ), "t0_ms": (data.get("setup_data") or {}).get("t0"), "run_ms": data["state"]["testRunDurationMs"], }, ) return row # ------------------------------------------------------------- сэмплер @dataclass class Samples: """Отсчёты сэмплера (часы ВМ, нс): контейнеры, ВМ, LB, PostgreSQL.""" cg: dict[tuple[str, str], list[tuple[int, ...]]] = field( default_factory=dict, ) vm: list[tuple[int, int, int, int]] = field(default_factory=list) lb: dict[int, list[dict[str, str]]] = field(default_factory=dict) act: dict[int, list[tuple[str, str, str, str, int]]] = field( default_factory=dict, ) db: list[tuple[int, ...]] = field(default_factory=list) wal: list[tuple[int, ...]] = field(default_factory=list) io: list[tuple[int, ...]] = field(default_factory=list) missing: list[str] = field(default_factory=list) def parse_samples(text: str, pg_text: str = "") -> Samples: """Строки `samples.log` (`cg`, `vm`, `lb`) и `pg-samples.log` (`act`, `db`, `wal`, `io`) — формат `bench/sampler.sh`.""" out = Samples() header: list[str] = [] for line in text.splitlines(): parts = line.split(" ") if len(parts) < 3: continue kind = parts[0] try: ns = int(parts[1]) except ValueError: continue if kind == "cg": role, ident = parts[2], parts[3] if len(parts) > 3 else "?" if len(parts) < 10 or parts[4] == "missing": out.missing.append(f"{role}:{ident}") continue values = tuple(int(v or 0) for v in parts[4:10]) out.cg.setdefault((role, ident), []).append((ns, *values)) elif kind == "vm" and len(parts) >= 5: out.vm.append((ns, int(parts[2]), int(parts[3]), int(parts[4]))) elif kind == "lb": rest = line.split(" ", 2)[2] if rest == "missing": out.missing.append("lb") elif rest.startswith("# "): header = rest[2:].split(",") elif header: row = dict(zip(header, rest.split(","), strict=False)) out.lb.setdefault(ns, []).append(row) for line in pg_text.splitlines(): parts = line.split("|") if len(parts) < 7 or not parts[1].isdigit(): continue kind, ns = parts[0], int(parts[1]) * 1_000_000 if kind == "act": out.act.setdefault(ns, []).append( (parts[2], parts[3], parts[4], parts[5], int(parts[6] or 0)), ) elif kind in ("db", "wal", "io"): values = tuple(int(v) if v else 0 for v in parts[2:7]) getattr(out, kind).append((ns, *values)) return out def inside( points: list[tuple[int, ...]], w0: int, w1: int, ) -> list[tuple[int, ...]]: return [p for p in points if w0 <= p[0] <= w1] def rate(points: list[tuple[int, ...]], index: int) -> float | None: """Разность счётчика между первым и последним отсчётом на Δt, в секунду.""" if len(points) < 2: return None seconds = (points[-1][0] - points[0][0]) / 1e9 if seconds <= 0: return None return (points[-1][index] - points[0][index]) / seconds def delta(points: list[tuple[int, ...]], index: int) -> int | None: if len(points) < 2: return None return points[-1][index] - points[0][index] @dataclass class Container: """Агрегаты окна одного контейнера (решение 8).""" cpu: float | None seconds: list[tuple[float, float, float, float, float]] periods: int throttled: int throttled_ms: float mem_peak_mib: float | None def container(points: list[tuple[int, ...]], w0: int) -> Container: """`points` — (нс, usage_usec, nr_periods, nr_throttled, throttled_usec, memory, psi_usec), уже внутри окна.""" seconds = [] for prev, cur in zip(points, points[1:], strict=False): dt = (cur[0] - prev[0]) / 1e9 if dt <= 0: continue seconds.append( ( round((cur[0] - w0) / 1e9, 2), (cur[1] - prev[1]) / 1e6 / dt, (cur[4] - prev[4]) / 1000 / dt, (cur[6] - prev[6]) / 1000 / dt, cur[5] / 2**20, ), ) cpu = rate(points, 1) return Container( cpu=None if cpu is None else cpu / 1e6, seconds=seconds, periods=delta(points, 2) or 0, throttled=delta(points, 3) or 0, throttled_ms=(delta(points, 4) or 0) / 1000, mem_peak_mib=(max(p[5] for p in points) / 2**20 if points else None), ) def wait_class(state: str, wtype: str, event: str) -> str | None: """Класс ожидания активного соединения (решение 8); не активное — None.""" if state != "active": return None if wtype == "CPU": return "cpu" if "wal" in event.lower(): return "wal" return {"Lock": "lock", "LWLock": "lwlock", "IO": "io"}.get(wtype, "other") def analyze_samples( samples: Samples, w0: int, w1: int, *, addresses: dict[str, str], ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """Агрегаты окна `[w0, w1]` (нс часов ВМ) и посекундные строки. `addresses` — адрес реплики в сети стенда → короткий ID.""" row: dict[str, Any] = {} per_second: list[dict[str, Any]] = [] roles: dict[str, list[tuple[str, Container]]] = {} for (role, ident), points in sorted(samples.cg.items()): agg = container(inside(points, w0, w1), w0) roles.setdefault(role, []).append((ident, agg)) for t_s, cpu, thr, psi, mem in agg.seconds: per_second.append( { "t_s": t_s, "role": role, "instance": ident, "cpu_cores": round(cpu, 3), "throttled_ms": round(thr, 1), "pressure_ms": round(psi, 1), "mem_mib": round(mem, 1), }, ) def ratio(items: list[tuple[str, Container]]) -> float | None: periods = sum(c.periods for _, c in items) if not periods: return 0.0 if items else None return round(sum(c.throttled for _, c in items) / periods, 4) def total_cpu(items: list[tuple[str, Container]]) -> float | None: values = [c.cpu for _, c in items] if not values or None in values: return None return round(sum(v for v in values if v is not None), 3) def peak(items: list[tuple[str, Container]]) -> float | None: values = [c.mem_peak_mib for _, c in items] if not values or None in values: return None return round(sum(v for v in values if v is not None), 1) apps = roles.get("api", []) row["app_cpu_cores"] = total_cpu(apps) replica_cpus = [c.cpu for _, c in apps if c.cpu is not None] row["app_cpu_replica_max"] = rounded(max(replica_cpus, default=None)) seconds = [s[1] for _, c in apps for s in c.seconds] row["app_cpu_second_max"] = rounded(max(seconds, default=None)) row["app_throttled_ratio"] = ratio(apps) row["app_throttled_ms"] = round(sum(c.throttled_ms for _, c in apps), 1) row["app_mem_peak_mib"] = peak(apps) for role, prefix in ( ("postgres", "db"), ("haproxy", "lb"), ("valkey", "cache"), ("k6", "generator"), ): items = roles.get(role, []) row[f"{prefix}_cpu_cores"] = total_cpu(items) row[f"{prefix}_throttled_ratio"] = ratio(items) row[f"{prefix}_mem_peak_mib"] = peak(items) vm = inside(samples.vm, w0, w1) busy = rate(vm, 1) row["vm_busy_cores"] = None if busy is None else round(busy / 100, 3) psi = rate(vm, 3) row["vm_cpu_pressure"] = None if psi is None else round(psi / 1e6, 4) for prev, cur in zip(vm, vm[1:], strict=False): dt = (cur[0] - prev[0]) / 1e9 if dt > 0: per_second.append( { "t_s": round((cur[0] - w0) / 1e9, 2), "role": "vm", "instance": "", "cpu_cores": round((cur[1] - prev[1]) / 100 / dt, 3), "throttled_ms": None, "pressure_ms": round((cur[3] - prev[3]) / 1000 / dt, 1), "mem_mib": None, }, ) row.update(analyze_pg(samples, w0, w1, addresses)) row.update(analyze_lb(samples, w0, w1)) return row, per_second def analyze_pg( samples: Samples, w0: int, w1: int, addresses: dict[str, str], ) -> dict[str, Any]: """PostgreSQL окна: commit/s, WAL, соединения, классы ожидания, пул.""" db = inside(samples.db, w0, w1) wal = inside(samples.wal, w0, w1) io = inside(samples.io, w0, w1) commits = rate(db, 1) rollbacks = rate(db, 2) wal_bytes = rate(wal, 3) fsyncs = rate(io, 2) ticks = sorted( {p[0] for p in db} | {t for t in samples.act if w0 <= t <= w1}, ) classes = dict.fromkeys(("cpu", "wal", "lock", "lwlock", "io"), 0) classes["other"] = 0 idle_in_tx = 0 busy = 0 conn_max = 0 replica_max = 0 for tick in ticks: rows = samples.act.get(tick, []) total = 0 per_addr: dict[str, int] = {} idle: dict[str, int] = {} for addr, state, wtype, event, count in rows: total += count per_addr[addr] = per_addr.get(addr, 0) + count if state == "idle": idle[addr] = idle.get(addr, 0) + count if state == "idle in transaction": idle_in_tx += count kind = wait_class(state, wtype, event) if kind is not None: classes[kind] += count conn_max = max(conn_max, total) replica_max = max( [replica_max, *(per_addr.get(a, 0) for a in addresses)], ) if addresses and any(idle.get(a, 0) == 0 for a in addresses): busy += 1 n = len(ticks) return { "db_commits_per_s": rounded(commits, 1), "db_rollbacks_per_s": rounded(rollbacks, 1), "db_wal_mib_per_s": ( None if wal_bytes is None else round(wal_bytes / 2**20, 3) ), "db_wal_fsyncs_per_s": rounded(fsyncs, 1), "db_sessions_new": delta(db, 3), "db_conn_max": conn_max if n else None, "db_conn_replica_max": replica_max if n else None, **{ f"db_aas_{name}": round(count / n, 3) if n else None for name, count in classes.items() }, "db_idle_in_tx_mean": round(idle_in_tx / n, 3) if n else None, "pool_busy_share": round(busy / n, 3) if n else None, } def analyze_lb(samples: Samples, w0: int, w1: int) -> dict[str, Any]: """HAProxy окна: строка BACKEND `api` и сумма слотов.""" ticks = sorted(t for t in samples.lb if w0 <= t <= w1) def backend(tick: int) -> dict[str, str]: return next( (r for r in samples.lb[tick] if r.get("svname") == "BACKEND"), {}, ) def value(row: dict[str, str], key: str) -> int: text = row.get(key) or "0" return int(text) if text.isdigit() else 0 def slots(tick: int, key: str) -> int: return sum( value(r, key) for r in samples.lb[tick] if r.get("svname") not in ("BACKEND", "FRONTEND") ) if len(ticks) < 2: return {} first, last = backend(ticks[0]), backend(ticks[-1]) return { "lb_qtime_ms": value(last, "qtime"), "lb_ctime_ms": value(last, "ctime"), "lb_rtime_ms": value(last, "rtime"), "lb_ttime_ms": value(last, "ttime"), "lb_scur_max": max(value(backend(t), "scur") for t in ticks), "lb_qcur_max": max(value(backend(t), "qcur") for t in ticks), "lb_retries": value(last, "wretr") - value(first, "wretr"), "lb_redispatches": value(last, "wredis") - value(first, "wredis"), "lb_econ": value(last, "econ") - value(first, "econ"), "lb_eresp": value(last, "eresp") - value(first, "eresp"), "lb_chkdown": slots(ticks[-1], "chkdown") - slots(ticks[0], "chkdown"), "lb_5xx": value(last, "hrsp_5xx") - value(first, "hrsp_5xx"), } def count_too_many(log: str) -> int: return sum(1 for line in log.splitlines() if TOO_MANY_CLIENTS in line) # ------------------------------------------------- вердикт и признаки @dataclass(frozen=True) class Limits: """Лимиты конфигурации, против которых считаются признаки.""" replicas: int db_cpus: float lb_cpus: float generator_cpus: float generator_cpu_limit: float vm_cpus: float slo_p95_ms: float = 200 slo_error_ratio: float = 0.01 def generator_bound(row: dict[str, Any], limits: Limits) -> bool: """Брак генератора (решение 6; третий признак — замена из apply: `vus` k6 — выборка раз в секунду, «свободные VU» по нему не видны).""" cpu = row.get("generator_cpu_cores") if cpu is not None and cpu >= limits.generator_cpu_limit: return True if (row.get("generator_throttled_ratio") or 0) >= THRESHOLDS[ "generator_throttled" ]: return True return bool( row.get("missing") and cpu is not None and limits.generator_cpus and cpu >= THRESHOLDS["generator_missing_cpu_share"] * limits.generator_cpus, ) def verdict( row: dict[str, Any], limits: Limits, *, aborted: bool, ) -> tuple[str, str]: """Итог ступени: `pass`, `fail` или `generator` и причина.""" if generator_bound(row, limits): return "generator", "generator" p95 = row.get("p95_ms") errors = row.get("slo_error_ratio") slow = p95 is None or p95 > limits.slo_p95_ms bad = errors is None or errors > limits.slo_error_ratio if aborted: return "fail", "abort" if slow and bad: return "fail", "p95+errors" if slow: return "fail", "p95" if bad: return "fail", "errors" return "pass", "" def signs(row: dict[str, Any], limits: Limits) -> list[str]: """Сработавшие признаки ступени (решение 9), кроме `host_cost`.""" t = THRESHOLDS found = [] def val(key: str) -> float: value = row.get(key) return 0.0 if value is None else float(value) per_replica = val("app_cpu_cores") / max(1, limits.replicas) if ( per_replica >= t["app_cpu_cores"] or val("app_throttled_ratio") >= t["throttled_ratio"] ): found.append("app") if ( val("db_cpu_cores") >= t["db_cpu_share"] * limits.db_cpus or val("db_throttled_ratio") >= t["throttled_ratio"] ): found.append("db_cpu") active = sum( val(f"db_aas_{name}") for name in ("cpu", "wal", "lock", "lwlock", "io", "other") ) # доля ожиданий осмысленна, только когда база занята: при 0,1 # активного соединения одно ожидание WAL в отсчёте — уже 100 % # (найдено в apply, самопроверка) busy = active >= t["db_active_min"] if busy and val("db_aas_wal") >= t["wait_share"] * active: found.append("db_wal") if ( busy and val("db_aas_lock") + val("db_aas_lwlock") >= t["wait_share"] * active ): found.append("db_lock") if ( val("pool_busy_share") >= t["pool_busy_share"] and per_replica < t["app_cpu_cores"] ) or val("db_too_many_clients") > 0: found.append("pool") if ( val("lb_cpu_cores") >= t["lb_cpu_share"] * limits.lb_cpus or val("lb_throttled_ratio") >= t["throttled_ratio"] or val("lb_econ") > 0 or val("lb_eresp") > 0 ): found.append("lb") if ( val("vm_busy_cores") >= t["vm_busy_share"] * limits.vm_cpus or val("vm_cpu_pressure") >= t["vm_cpu_pressure"] ): found.append("host") if generator_bound(row, limits): found.append("generator") return found def per_request(cores: float | None, rps: float | None) -> float | None: """CPU на успешный запрос, мкс.""" if cores is None or not rps: return None return round(cores * 1e6 / rps, 1) # ------------------------------------------------------- таблицы метки def read_csv(path: Path) -> list[dict[str, str]]: if not path.is_file(): return [] with path.open(encoding="utf-8", newline="") as file: return list(csv.DictReader(file)) def chosen_steps( runs: list[dict[str, str]], searches: list[dict[str, str]], ) -> dict[str, list[dict[str, str]]]: """Ступени попыток из `searches.csv`: прерванные попытки в сводку не входят (решение 5).""" attempts = {s["cell"]: s["attempt"] for s in searches} chosen: dict[str, list[dict[str, str]]] = {} for run in runs: if ( run.get("cell") in attempts and run["attempt"] == attempts[run["cell"]] ): chosen.setdefault(run["cell"], []).append(run) return chosen def step_at( steps: list[dict[str, str]], level: float | None, ) -> dict[str, str] | None: """Последняя ступень ячейки с целевым уровнем `level`.""" if level is None: return None found = [s for s in steps if num(s.get("level")) == level] return found[-1] if found else None def fail_step( steps: list[dict[str, str]], r_star: float, ) -> dict[str, str] | None: """Ближайшая нарушившая ступень выше R*.""" above = [ s for s in steps if s.get("slo_pass") == "False" and (num(s["level"]) or 0) > r_star ] return min(above, key=lambda s: num(s["level"]) or 0) if above else None def majority(lists: list[list[str]]) -> list[str]: """Признаки, сработавшие хотя бы в половине повторов (то же, что порог на медиане метрики по повторам).""" if not lists: return [] need = len(lists) / 2 found = [ name for name in SIGN_ORDER if sum(name in items for items in lists) >= need ] return found def config_stats( meta: dict[str, Any], runs: list[dict[str, str]], searches: list[dict[str, str]], ) -> list[dict[str, Any]]: """Сводка по конфигурациям серии в порядке профиля.""" chosen = chosen_steps(runs, searches) profile = meta["profile"] fixed = profile.get("fixed_rps") or [] order = [ config_key(*unpack_config(config)) for config in profile.get("configs", []) ] db_limits = { config_key(*parts): parts[4] or str(profile.get("db_cpus", "")) for parts in map(unpack_config, profile.get("configs", [])) } result = [] for key in order: cells = [s for s in searches if s["config"] == key] if not cells: continue ok = [s for s in cells if s["status"] == "complete"] r_stars = [num(s["r_star_rps"]) for s in ok] at_r: list[list[str]] = [] at_f: list[list[str]] = [] reasons = [] db_at_r = [] db_thr_at_r: list[float | None] = [] # PostgreSQL на R* и первой нарушившей ступени (change # scaling-improvement, решение 7): уровень, CPU на запрос, новые # сессии за окно, соединения pg_r: dict[str, list[float | None]] = { "db_us": [], "sessions": [], "conn": [], } pg_f: dict[str, list[float | None]] = { "level": [], "db_us": [], "sessions": [], "conn": [], } app_at_r = [] app_sec = [] marginals: list[float | None] = [] fixed_rows: dict[int, list[dict[str, str]]] = { lvl: [] for lvl in fixed } for s in cells: steps = chosen.get(s["cell"], []) for lvl in fixed: row = next( ( x for x in steps if x.get("step_kind") == "fixed" and num(x["level"]) == lvl ), None, ) if row is not None: fixed_rows[lvl].append(row) if s["status"] != "complete": continue r_star = num(s["r_star_rps"]) or 0 r_row = step_at(steps, r_star) f_row = fail_step(steps, r_star) if r_row is not None: at_r.append((r_row.get("signs") or "").split()) db_at_r.append(num(r_row.get("db_cpu_cores"))) db_thr_at_r.append(num(r_row.get("db_throttled_ratio"))) app_at_r.append(num(r_row.get("app_cpu_cores"))) pg_r["db_us"].append(num(r_row.get("db_us_per_request"))) pg_r["sessions"].append(num(r_row.get("db_sessions_new"))) pg_r["conn"].append(num(r_row.get("db_conn_max"))) if f_row is not None: at_f.append((f_row.get("signs") or "").split()) reasons.append(f_row.get("fail_reason") or "") pg_f["level"].append(num(f_row.get("level"))) pg_f["db_us"].append(num(f_row.get("db_us_per_request"))) pg_f["sessions"].append(num(f_row.get("db_sessions_new"))) pg_f["conn"].append(num(f_row.get("db_conn_max"))) app_sec.extend(num(x.get("app_cpu_second_max")) for x in steps) marginals.append(marginal_us(steps)) replicas = int(cells[0]["replicas"]) result.append( { "config": key, "replicas": replicas, "replica_cpus": cells[0]["replica_cpus"], "db_max_connections": int(cells[0]["db_max_connections"]), "cells": len(cells), "complete": len(ok), "generator_bound": [ s["cell"] for s in cells if s["status"] == "generator_bound" ], "r_stars": [v for v in r_stars if v is not None], "r_star": median(r_stars), "fixed": { lvl: { "p50": median(num(x["p50_ms"]) for x in rows), "p95": median(num(x["p95_ms"]) for x in rows), "p99": median(num(x["p99_ms"]) for x in rows), "p99_all": [num(x["p99_ms"]) for x in rows], "passed": sum(x["slo_pass"] == "True" for x in rows), "app_us": median( num(x["app_us_per_request"]) for x in rows ), "throttled": median( num(x["app_throttled_ratio"]) for x in rows ), "throttled_all": [ num(x["app_throttled_ratio"]) for x in rows ], "too_many": sum( int(num(x["db_too_many_clients"]) or 0) for x in rows ), "codes": " ".join( x.get("error_codes") or "" for x in rows ).split(), } for lvl, rows in fixed_rows.items() }, "fail_reasons": reasons, "signs_r": majority(at_r), "signs_f": majority(at_f), "db_cpu_r": median(db_at_r), "db_throttled_r": median(db_thr_at_r), "db_us_r": median(pg_r["db_us"]), "sessions_r": median(pg_r["sessions"]), "conn_r": median(pg_r["conn"]), "level_f": median(pg_f["level"]), "db_us_f": median(pg_f["db_us"]), "sessions_f": median(pg_f["sessions"]), "conn_f": median(pg_f["conn"]), "db_cpus": db_limits.get(key, ""), "app_cpu_r": median(app_at_r), "app_cpu_second_max": max( (v for v in app_sec if v is not None), default=None, ), "app_us_marginal": median(marginals), }, ) base = next( ( c for c in result if c["replicas"] == 1 and float(c["replica_cpus"]) == 1 and c["db_max_connections"] == 100 ), None, ) for c in result: c["host_cost"] = None if base is not None: apply_host_cost(c, base["app_us_marginal"]) return result def apply_host_cost(c: dict[str, Any], reference: float | None) -> None: """Признак `host_cost`: предельный CPU реплик на запрос конфигурации против 1 × 1 (найдено в apply: CPU на запрос на 500/с у k реплик включает постоянные расходы каждой реплики — проверки LB, простой event loop — и растёт с k без всякого хоста; наклон их исключает).""" own = c.get("app_us_marginal") if not own or not reference: return c["host_cost"] = round(own / reference, 3) if own >= THRESHOLDS["host_cost"] * reference: for key in ("signs_r", "signs_f"): if "host_cost" not in c[key]: c[key] = [*c[key], "host_cost"] def marginal_us(steps: list[dict[str, str]]) -> float | None: """Предельный CPU реплик на успешный запрос, мкс: наклон прямой `app_cpu_cores` от `rps` по ступеням ячейки, прошедшим SLO (наименьшие квадраты); меньше двух уровней — None.""" points = [ (num(s.get("rps")), num(s.get("app_cpu_cores"))) for s in steps if s.get("slo_pass") == "True" ] points = [(x, y) for x, y in points if x and y is not None] if len({round(x) for x, _ in points}) < 2: return None mean_x = statistics.mean(x for x, _ in points) mean_y = statistics.mean(y for _, y in points) var = sum((x - mean_x) ** 2 for x, _ in points) cov = sum((x - mean_x) * (y - mean_y) for x, y in points) return round(cov / var * 1e6, 1) if var else None def saturation( stats: list[dict[str, Any]], reference: float | None = None, ) -> dict[str, Any]: """Точка насыщения серии роста (решение 9): наименьшее k ≤ 5, при котором медиана R*(k + 1) < 1,1 × R*(k); эффективность — R*(k) / (k × R*(1 × 1)).""" by_k = { c["replicas"]: c for c in stats if float(c["replica_cpus"]) == 1 and c["db_max_connections"] == 100 } base = ( reference if reference is not None else (by_k[1]["r_star"] if 1 in by_k else None) ) efficiency = { k: ( round(c["r_star"] / (k * base), 3) if base and c["r_star"] is not None else None ) for k, c in sorted(by_k.items()) } k_sat = None for k in sorted(by_k): nxt = by_k.get(k + 1) if k > 5 or nxt is None: continue cur_r, nxt_r = by_k[k]["r_star"], nxt["r_star"] if ( cur_r and nxt_r is not None and nxt_r < THRESHOLDS["k_sat_gain"] * cur_r ): k_sat = k break return {"efficiency": efficiency, "k_sat": k_sat, "by_k": by_k} def fmt(value: Any, digits: int = 0) -> str: if value is None or value == "": return "—" if isinstance(value, str): return value return f"{value:.{digits}f}" def render_summary( meta: dict[str, Any], runs: list[dict[str, str]], searches: list[dict[str, str]], ) -> str: """`summary.md` серии: методика, R* по конфигурациям, задержки на фиксированных уровнях, признаки на R* и первой нарушившей, брак.""" profile = meta["profile"] stats = config_stats(meta, runs, searches) fixed = profile.get("fixed_rps") or [] mix = ", ".join(f"{n} {s:g}" for n, s in profile.get("mix", [])) lines = [ f"# Серия трека 1 `{profile.get('series')}`: метка `{meta['label']}`", "", f"- Статус: **{meta['status']}**" + (f" ({meta['reason']})" if meta.get("reason") else ""), f"- Commit: `{meta['git']['commit']}`" + (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""), f"- Профиль `{profile['name']}`: смесь {mix}; читаемый чат " f"{profile['sizes'][0]} сообщений, чатов записи " f"{profile['write_chats']}; фиксированные уровни " f"{', '.join(map(str, fixed))} запросов/с; поиск от " f"{profile['start_rps_per_replica']} × реплик по лестнице " f"{profile['ladder_base']} × {profile['step_factor']}^i, делений " f"{profile['bisect_steps']}; прогрев {profile['first_warmup_s']} с " f"(первая ступень и после паузы) / {profile['step_warmup_s']} с, " f"окно {profile['step_measure_s']} с; SLO p95 ≤ " f"{profile['slo_p95_ms']:g} мс, ошибок с недосланными ≤ " f"{profile['slo_error_ratio'] * 100:g} %; повторов " f"{profile['repeats']}, порядок кругами (зерно " f"{profile['order_seed']})", f"- Лимиты без swap: PostgreSQL {profile['db_cpus']} CPU / " f"{profile['db_memory']} (у конфигураций `-pg` — свой лимит " f"CPU), HAProxy {profile['lb_cpus']} / " f"{profile['lb_memory']}, Valkey {profile['cache_cpus']} / " f"{profile['cache_memory']}, k6 {profile['generator_cpus']} / " f"{profile['generator_memory']}, сэмплер {profile['sampler_cpus']} " f"/ {profile['sampler_memory']}", "- Значения — медиана (min–max) по повторам; R* — наибольший " "уровень поиска, прошедший SLO; признаки — сработавшие хотя бы в " "половине повторов (`THRESHOLDS` в `bench/scaling.py`)", "", ] budget = meta.get("budget") or {} if budget: lines += [ "## Бюджет конфигураций", "", "| Конфигурация | Σ CPU стенда и генератора | Σ памяти, ГиБ | " "CPU ВМ | Память ВМ, ГиБ |", "|---|---|---|---|---|", ] for key, item in budget.items(): lines.append( f"| {key} | {item['cpus']:g} | {item['memory_gib']:g} | " f"{item['vm_cpus']} | {item['vm_memory_gib']} |", ) lines.append("") lines += [ "## R* по конфигурациям", "", "| Конфигурация | Ячеек | R*, запросов/с | Причина первого нарушения " "| CPU реплик на R*, ядер | Посекундный max CPU реплики | CPU " "PostgreSQL на R*, ядер | Признаки на R* | Признаки на первой " "нарушившей | Предельный CPU реплик на запрос, мкс | host_cost |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for c in stats: reasons = sorted(set(c["fail_reasons"])) lines.append( f"| {c['config']} | {c['complete']}/{c['cells']} | " f"{spread(c['r_stars'])} | {', '.join(reasons) or '—'} | " f"{fmt(c['app_cpu_r'], 2)} | {fmt(c['app_cpu_second_max'], 2)} " f"| {fmt(c['db_cpu_r'], 2)} | {' '.join(c['signs_r']) or '—'} | " f"{' '.join(c['signs_f']) or '—'} | " f"{fmt(c['app_us_marginal'], 0)} | {fmt(c['host_cost'], 2)} |", ) lines.append("") if any("-pg" in c["config"] for c in stats): # серия db: насыщение PostgreSQL — CPU у лимита и троттлинг CFS lines += [ "## PostgreSQL на R*", "", "| Конфигурация | Лимит CPU PostgreSQL | CPU PostgreSQL на R*, " "ядер | Доля лимита | Троттлинг PostgreSQL на R*, доля " "периодов |", "|---|---|---|---|---|", ] for c in stats: limit = num(c.get("db_cpus")) share = ( c["db_cpu_r"] / limit if limit and c["db_cpu_r"] is not None else None ) lines.append( f"| {c['config']} | {c.get('db_cpus') or '—'} | " f"{fmt(c['db_cpu_r'], 2)} | {fmt(share, 2)} | " f"{fmt(c.get('db_throttled_r'), 3)} |", ) lines.append("") for lvl in fixed: lines += [ f"## Фиксированный уровень {lvl} запросов/с", "", "| Конфигурация | Прошли SLO | p50, мс | p95, мс | p99, мс | " "CPU реплик на запрос, мкс | Троттлинг реплик, доля периодов " "| Отказов «too many clients» | Коды ошибок |", "|---|---|---|---|---|---|---|---|---|", ] for c in stats: f = c["fixed"].get(lvl) or {} codes: dict[str, int] = {} for item in f.get("codes", []): status, _, count = item.partition(":") codes[status] = codes.get(status, 0) + int(count or 0) lines.append( f"| {c['config']} | {f.get('passed', 0)}/{c['cells']} | " f"{fmt(f.get('p50'), 1)} | {fmt(f.get('p95'), 1)} | " f"{spread(f.get('p99_all', []), 1)} | " f"{fmt(f.get('app_us'), 0)} | " f"{spread(f.get('throttled_all', []), 4)} | " f"{f.get('too_many', 0)} | " + (" ".join(f"{k}:{v}" for k, v in codes.items()) or "—") + " |", ) lines.append("") sat = saturation(stats) if sat["efficiency"] and len(sat["by_k"]) > 2: lines += [ "## Рост числа реплик", "", "| Реплик | R*, запросов/с | R* / (k × R*(1 × 1)) |", "|---|---|---|", ] for k, c in sorted(sat["by_k"].items()): lines.append( f"| {k} | {spread(c['r_stars'])} | " f"{fmt(sat['efficiency'].get(k), 2)} |", ) lines += [ "", "Точка насыщения k_sat (медиана R*(k + 1) < " f"{THRESHOLDS['k_sat_gain']} × R*(k)): " + (str(sat["k_sat"]) if sat["k_sat"] else "нет в пределах серии"), "", ] bound = [cell for c in stats for cell in c["generator_bound"]] lines += ["## Брак генератора", ""] lines.append( "Ячейки с R* — нижней границей: " + ", ".join(bound) if bound else "Нет: ни одна ступень не упёрлась в генератор.", ) lines += [ "", "## Ячейки", "", "| № | Ячейка | Попытка | Статус | R* | Первая нарушившая | " "Разрешение | Ступеней | Записей 201 / отправлено / в БД | " "Выгрузок в swap | Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|", ] for s in sorted(searches, key=lambda x: int(x["order"])): lines.append( f"| {s['order']} | {s['cell']} | {s['attempt']} | {s['status']} " f"| {s['r_star_rps']}{' (нижняя граница)' if s['r_star_bound'] == 'lower' else ''} " f"| {s['first_fail_rps'] or '—'} {s['first_fail_reason'] or ''} | " f"{s['resolution'] or '—'} | {s['steps']} | {s['writes_ok']} / " f"{s['writes_sent']} / {s['write_chat_messages']} | " f"{s['host_swapouts'] or 0} | {s['flags'] or '—'} |", ) lines.append("") if meta.get("host_warnings"): lines += ["## Предупреждения о хосте", ""] lines += [f"- {w}" for w in meta["host_warnings"]] lines.append("") return "\n".join(lines) # ------------------------------------------------------- самопроверка def selftest() -> int: """Синтетические исходы поиска и строки сэмплера (задача 1.5).""" failures: list[str] = [] def expect(name: str, got: Any, want: Any) -> None: if got != want: failures.append(f"{name}: got {got!r}, want {want!r}") steps = ladder(100, 1.25) expect("ladder head", steps[:6], [100, 125, 156, 195, 244, 305]) for k, per, fixed, want in ( (1, 300, 500, 596), (2, 300, 500, 745), (4, 300, 500, 1455), (6, 300, 500, 1819), ): expect(f"L0 k={k}", start_level(k, per, fixed, steps), want) def drive(search: Search, ceiling: float) -> Search: for _ in range(40): step = search.next_step() if step is None: break level, kind = step search.record(level, kind, "pass" if level <= ceiling else "fail") return search # подъём: потолок 2 500 при k = 6 s = drive(Search(replicas=6, fixed=(250, 500)), 2500) fail = s.first_fail expect("up status", s.status, "complete") expect("up r*<=ceiling", s.r_star <= 2500, True) expect("up resolution", fail is not None and fail / s.r_star <= 1.06, True) expect( "up kinds", {k for _, k, _ in s.outcomes}, {"fixed", "up", "bisect"}, ) # спуск от первой нарушенной: потолок 520 при k = 4 (L0 = 1455) s = drive(Search(replicas=4, fixed=(250, 500)), 520) expect("down r*", s.r_star >= 500 and s.r_star <= 520, True) expect("down resolution", (s.first_fail or 0) / s.r_star <= 1.06, True) # оба фиксированных уровня провалены: спуск от 250 s = drive(Search(replicas=1, fixed=(250, 500)), 130) expect("low r*", s.r_star, 125) expect("low first", s.outcomes[2][:2], (244, "down")) s = drive(Search(replicas=1, fixed=(250, 500)), 50) expect("floor", (s.r_star, s.flags), (0, ["floor"])) # брак генератора: R* — наибольший пройденный уровень s = Search(replicas=2, fixed=(250, 500)) for outcome in ("pass", "pass", "pass", "generator"): level, kind = s.next_step() or (0, "") s.record(level, kind, outcome) expect( "generator", (s.status, s.r_star, s.done), ("generator_bound", 745, True), ) # порядок кругами plan = plan_cells( [(1, "4", "2g", 100), (2, "2", "1g", 100), (4, "1", "512m", 100)], 3, 20261011, ) for r in (1, 2, 3): expect( f"round {r}", sorted(p["config"] for p in plan if p["round"] == r), ["1x4", "2x2", "4x1"], ) for a, b in zip(plan, plan[1:], strict=False): if a["config"] == b["config"]: failures.append(f"repeats adjacent: {a['cell']} {b['cell']}") # сэмплер: известные разности ns = 10**18 lines = [] for i in range(3): t = ns + i * 10**9 lines.append( f"cg {t} api aaaaaaaaaaaa {i * 900_000} {i * 10} {i * 2} " f"{i * 30_000} {100 * 2**20} {i * 5000}", ) lines.append( f"cg {t} postgres bbbbbbbbbbbb {i * 500_000} {i * 10} 0 0 " f"{200 * 2**20} 0", ) lines.append(f"vm {t} {i * 400} {i * 1200} {i * 100_000}") lines.append( f"lb {t} # pxname,svname,qcur,scur,econ,eresp,wretr,wredis," "chkdown,hrsp_5xx,qtime,ctime,rtime,ttime", ) lines.append(f"lb {t} api,api-1,0,{3 + i},0,0,0,0,0,0,0,0,0,0") lines.append(f"lb {t} api,BACKEND,0,{3 + i},0,0,{i},0,0,{i},0,1,5,6") pg = [] for i in range(3): ms = (ns + i * 10**9) // 1_000_000 pg.append(f"db|{ms}|{i * 100}|0|{i}|0|5") pg.append(f"act|{ms}|10.0.0.5|active|CPU|-|2") pg.append(f"act|{ms}|10.0.0.5|idle|Client|ClientRead|1") pg.append(f"act|{ms}|10.0.0.6|active|LWLock|WALWrite|3") pg.append(f"wal|{ms}|0|0|{i * 2**20}||") pg.append(f"io|{ms}|0|{i * 7}|||") parsed = parse_samples("\n".join(lines), "\n".join(pg)) row, seconds = analyze_samples( parsed, ns, ns + 2 * 10**9, addresses={"10.0.0.5": "aaaaaaaaaaaa", "10.0.0.6": "cccccccccccc"}, ) expect("app cpu", row["app_cpu_cores"], 0.9) expect("app throttled", row["app_throttled_ratio"], 0.2) expect("app throttled ms", row["app_throttled_ms"], 60.0) expect("db cpu", row["db_cpu_cores"], 0.5) expect("vm busy", row["vm_busy_cores"], 4.0) expect("vm psi", row["vm_cpu_pressure"], 0.1) expect("commits", row["db_commits_per_s"], 100.0) expect("wal", row["db_wal_mib_per_s"], 1.0) expect("fsyncs", row["db_wal_fsyncs_per_s"], 7.0) expect("aas cpu", row["db_aas_cpu"], 2.0) expect("aas wal", row["db_aas_wal"], 3.0) expect("pool busy", row["pool_busy_share"], 1.0) expect("conn replica", row["db_conn_replica_max"], 3) expect("lb retries", row["lb_retries"], 2) expect("lb scur", row["lb_scur_max"], 5) expect( "seconds", len([s for s in seconds if s["role"] == "api"]), 2, ) limits = Limits( replicas=1, db_cpus=2, lb_cpus=1, generator_cpus=2, generator_cpu_limit=1.8, vm_cpus=12, ) expect("signs", signs(row, limits), ["app", "db_wal"]) expect( "signs pool", signs( {**row, "app_cpu_cores": 0.5, "app_throttled_ratio": 0.0}, limits, ), ["db_wal", "pool"], ) expect( "generator cpu", generator_bound({"generator_cpu_cores": 1.85}, limits), True, ) expect( "generator missing", generator_bound( {"generator_cpu_cores": 1.5, "missing": 3}, limits, ), True, ) expect( "verdict pass", verdict({"p95_ms": 50, "slo_error_ratio": 0.0}, limits, aborted=False), ("pass", ""), ) expect( "verdict p95", verdict( {"p95_ms": 250, "slo_error_ratio": 0.0}, limits, aborted=False, ), ("fail", "p95"), ) if failures: print("selftest failed:\n " + "\n ".join(failures), flush=True) return 1 print("selftest ok", flush=True) return 0 if __name__ == "__main__": if sys.argv[1:] == ["--selftest"]: sys.exit(selftest()) print("usage: uv run python bench/scaling.py --selftest", flush=True) sys.exit(2)