Files
mtusi2026lab6/bench/overload.py
T
ricoromaandClaude Opus 5.5 9cc6c0c7a6 chore: port lab 5 solution into the lab 6 skeleton
Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e.
Contracts, public tests and evidence of lab 5 are not carried over.
The compose project, image and CI prefix are renamed to lab6; the first
broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the
OpenSpec changes of lab 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 04:46:56 +03:00

2617 lines
90 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль очереди без docker (design change `track1-queue-overload`, решения
5–8, 11, 12): план кругами, разбор строк k6, сэмплера и журналов, строка ячейки и
посекундная шкала, пометки, вердикты H1–H9, правило рекомендации, сводка и
графики. Только stdlib; `--selftest`. Имя не `queue.py`: каталог скрипта первый в
`sys.path`, и `queue` затенил бы модуль stdlib.
"""
from __future__ import annotations
import csv
import json
import random
import sys
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any
import plots
import reconcile
WINDOWS = ("before", "over", "after")
CONTROL_WINDOWS = ("before", "over", "recovery", "after")
CONTROL_KINDS = {"w": "write", "r": "read", "d": "download"}
CONTROL_STATS = ("n", "errors", "s5xx", "p50", "p95", "p99")
ROLES = (
"worker",
"s3",
"postgres",
"api",
"relay",
"rabbitmq",
"cleaner",
"haproxy",
)
# условие H6, записанное до метки: состав не меняется
H6_FLAGS = ("lost", "stuck", "duplicate", "sha_mismatch")
# пометки, при которых метка не зелёная: система потеряла работу (решение 11) или
# в базе есть вложение, на которое никто не отвечал (verify S4)
FATAL_FLAGS = (*H6_FLAGS, "unexpected")
DEFAULT_SETTING = (2, 2)
# `oom` и `die` одного завершения: на пробе ~50 мс, запас на нагруженную ВМ
OOM_DIE_WINDOW_S = 2.0
MIB = 2**20
CONFIRMED = "подтверждено"
REFUTED = "нет"
NO_DATA = "нет данных"
BACKLOG_RETRY_AFTER = "5"
# запас превышения предела: загрузки в полёте двух реплик (решение 3)
IN_FLIGHT = 40
LEASE_S = 70
# окно CPU S3 блока accept: до срабатывания предела (гипотеза H9)
ACCEPT_S3_WINDOW_S = 30
CELL_FIELDS = [
"cell",
"config",
"block",
"class",
"concurrency",
"prefetch",
"repeat",
"order",
"rate_low",
"rate_high",
"over_s",
"limit",
"action",
"accepted",
"s202",
"s429",
"s5xx",
"s_other",
"s_conn",
"dropped",
"accepted_before",
"accepted_over",
"accepted_after",
"retry_after",
"busy_429",
"backlog_429",
"first_busy_s",
"first_backlog_s",
"upload_before_p50_ms",
"upload_over_p50_ms",
"upload_over_p95_ms",
"ready",
"failed",
"lost",
"stuck",
"duplicate",
"sha_checked",
"sha_bad",
"ready_s_over",
"ready_s_recovery",
"over_end",
"zero_at",
"action_at",
"recovery_s",
"peak_unfinished",
"peak_outbox",
"peak_waiting",
"peak_published",
"peak_processing",
"peak_broker_ready",
"peak_broker_unacked",
"peak_prefetched",
"peak_oldest_s",
"staging_ready_end",
"queue_left",
"ready_before_p50_s",
"ready_before_p95_s",
"ready_before_p99_s",
"ready_over_p50_s",
"ready_over_p95_s",
"ready_over_p99_s",
"ready_after_p50_s",
"ready_after_p95_s",
"ready_after_p99_s",
"ready_before_list",
"ready_after_list",
"queue_wait_p50_ms",
"queue_wait_p95_ms",
"duration_p50_ms",
"render_p50_ms",
"worker_hwm_mib",
"render_peak_p50_mib",
"render_peak_max_mib",
"memory_peak_mib",
"anon_max_mib",
"oom_kill",
"worker_restarts",
*[f"cpu_{role}_over" for role in ROLES],
*[f"cpu_{role}_recovery" for role in ROLES],
*[f"throttle_{role}_over" for role in ("worker", "s3")],
"cpu_s3_accept",
"cpu_worker_per_job_s",
"broker_memory_mib",
"broker_refused",
"attempts_over_one",
"skipped",
"relay_published",
"relay_republished",
"relay_recovered",
"relay_backlog_full",
"failed_by",
"second_jobs",
"second_share",
*[
f"ctrl_{kind}_{window}_{stat}"
for kind in CONTROL_KINDS.values()
for window in CONTROL_WINDOWS
for stat in CONTROL_STATS
],
"swapouts",
"lb_down",
"flags",
# новые столбцы — только в конец (verify change track1-queue-overload)
*[f"queue_wait_{w}_{q}_ms" for w in WINDOWS for q in ("p50", "p95")],
*[f"peak_oldest_{w}_s" for w in CONTROL_WINDOWS],
"second_jobs_all",
"second_share_all",
"second_share_after",
*[f"cpu_{role}_{w}" for role in ROLES for w in ("before", "after")],
*[
f"throttle_{role}_{w}"
for role in ROLES
for w in CONTROL_WINDOWS
if not (role in ("worker", "s3") and w == "over")
],
"relay_restarts",
"relay_die",
"unexpected_rows",
"worker_die_oom",
"worker_die_command",
"worker_die_other",
"worker_exit_codes",
"worker_oom_events",
"lb_up_min",
]
TIMELINE_FIELDS = [
"cell",
"t",
"window",
"outbox",
"waiting",
"published",
"processing",
"unfinished",
"broker_ready",
"broker_unacked",
"prefetched",
"oldest_s",
"staging_ready",
"accepted",
"ready",
]
# ------------------------------------------------------------------ план
def config_key(
block: str,
klass: str,
concurrency: int,
prefetch: int,
) -> str:
setting = f"{klass}-c{concurrency}p{prefetch}"
return setting if block == "matrix" else f"{block}-{setting}"
def plan_cells(
configs: Iterable[Mapping[str, Any]],
repeats: int,
seed: int,
) -> list[dict[str, Any]]:
"""Круги: на круге — перестановка конфигураций от зерна; круг не начинается
с той, на которой кончился прошлый (как `scaling.plan_cells`)."""
items = list(configs)
plan: list[dict[str, Any]] = []
previous = None
for round_no in range(1, repeats + 1):
order = list(range(len(items)))
random.Random(seed + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for index in order:
item = dict(items[index])
key = config_key(
item["block"],
item["class"],
item["concurrency"],
item["prefetch"],
)
plan.append(
{
**item,
"cell": f"{key}-r{round_no}",
"config": key,
"repeat": round_no,
"order": len(plan) + 1,
},
)
previous = order[-1] if order else None
return plan
# --------------------------------------------------------- разбор источников
@dataclass(frozen=True)
class Upload:
window: str
start_ms: int
duration_ms: int
status: int
retry_after: str
instance: str
attachment_id: str
@property
def answered(self) -> float:
return (self.start_ms + self.duration_ms) / 1000
@dataclass(frozen=True)
class JobTick:
t: float
outbox: int
waiting: int
published: int
processing: int
oldest_s: float
staging_ready: int
@property
def unfinished(self) -> int:
return self.outbox + self.waiting + self.published + self.processing
@dataclass(frozen=True)
class Meta:
http: int
status: str
error_code: str
variants: int
updated_at: float | None
@dataclass
class Sources:
"""Всё снятое за ячейку; сюда же — готовые разборы журналов."""
uploads: list[Upload]
t0: float
metas: dict[str, Meta]
sha_checks: list[bool]
jobs: list[JobTick]
mq: list[tuple[float, int, int]]
mq_missing: int
cg: dict[str, list[list[tuple[int, ...]]]]
mp: dict[str, list[list[tuple[int, ...]]]]
control: list[Any]
worker_records: list[dict[str, Any]]
second_records: list[dict[str, Any]]
relay_records: list[dict[str, Any]]
api_records: list[dict[str, Any]]
sql: dict[str, Any]
hwm_kib: int | None = None
restarts: int = 0
dropped: int = 0
action_at: float | None = None
swapouts: int | None = None
lb_down: bool = False
storage_down: bool = False
extra: dict[str, Any] = field(default_factory=dict)
docker_events: list[DockerEvent] = field(default_factory=list)
worker_name: str = "worker"
relay_name: str = "relay"
relay_restarts: int = 0
lb_up_min: int | None = None
# все вложения стенда и разогрев: сверка в обратную сторону (verify S4)
db_ids: set[str] | None = None
warm_ids: set[str] = field(default_factory=set)
def parse_uploads(text: str) -> tuple[float | None, list[Upload]]:
"""Строки `q-t0,<мс>` и `q,<окно>,<мс>,<мс>,<статус>,<RA>,<реплика>,<id>`."""
t0 = None
out = []
for raw in text.splitlines():
line = raw.strip()
if line.startswith("q-t0,"):
t0 = int(line.split(",")[1]) / 1000
elif line.startswith("q,"):
parts = line.split(",")
if len(parts) == 8:
out.append(
Upload(
parts[1],
int(parts[2]),
int(parts[3]),
int(parts[4]),
parts[5],
parts[6],
parts[7],
),
)
return t0, out
def parse_sweep(text: str) -> tuple[dict[str, Meta], list[bool]]:
metas: dict[str, Meta] = {}
checks: list[bool] = []
for raw in text.splitlines():
parts = raw.strip().split(",")
if parts[0] == "qs" and len(parts) == 7:
updated = (
None if parts[6] == "-" else reconcile.timestamp(parts[6])
)
metas[parts[1]] = Meta(
int(parts[2]),
parts[3],
parts[4],
int(parts[5]),
updated,
)
elif parts[0] == "qv" and len(parts) == 4:
checks.append(parts[3] == "1")
return metas, checks
def parse_jobs(text: str) -> list[JobTick]:
out = []
for line in text.splitlines():
parts = line.strip().split("|")
if len(parts) == 8 and parts[0] == "jobs":
out.append(
JobTick(
int(parts[1]) / 1000,
int(parts[2]),
int(parts[3]),
int(parts[4]),
int(parts[5]),
float(parts[6]),
int(parts[7]),
),
)
return out
def parse_mq(text: str) -> tuple[list[tuple[float, int, int]], int]:
"""Строки `mq`: значения и число `missing` (брокер не ответил)."""
out = []
missing = 0
for line in text.splitlines():
parts = line.split(" ")
if parts[0] != "mq":
continue
if len(parts) == 3 and parts[2] == "missing":
missing += 1
elif len(parts) == 4:
out.append(
(
int(parts[1]) / 1e9,
int(float(parts[2])),
int(float(parts[3])),
),
)
return out, missing
def parse_mp(text: str) -> dict[str, list[tuple[int, ...]]]:
"""Строки `mp <нс> <id12> <memory.peak> <anon> <oom_kill>` по контейнерам."""
out: dict[str, list[tuple[int, ...]]] = {}
for line in text.splitlines():
parts = line.split(" ")
if parts[0] == "mp" and len(parts) == 6:
out.setdefault(parts[2], []).append(
(int(parts[1]), *(int(v) for v in parts[3:6])),
)
return out
def split_lives(
points: list[tuple[int, ...]],
key: int = 1,
) -> list[list[tuple[int, ...]]]:
"""Жизни контейнера: перезапуск сбрасывает cgroup того же контейнера, и
счётчик `points[i][key]` падает (решение 10)."""
lives: list[list[tuple[int, ...]]] = []
for point in points:
if lives and point[key] >= lives[-1][-1][key]:
lives[-1].append(point)
else:
lives.append([point])
return lives
def lives_delta(
lives: Iterable[list[tuple[int, ...]]],
w0: float,
w1: float,
key: int = 1,
) -> float | None:
"""Разность счётчика за окно [w0, w1] (секунды) — сумма разностей внутри
жизней, на краях окна — линейно между соседними отсчётами: тик сэмплера не
совпадает с краем, и разность только по внутренним отсчётам занижала бы
короткое окно на ~10 %. Окно не задевает ни одну жизнь — `None`."""
def value_at(life: list[tuple[int, ...]], ns: float) -> float:
for prev, cur in zip(life, life[1:], strict=False):
if prev[0] <= ns <= cur[0]:
share = (ns - prev[0]) / ((cur[0] - prev[0]) or 1)
return prev[key] + (cur[key] - prev[key]) * share
return float(life[-1][key])
total = 0.0
seen = False
for life in lives:
if len(life) < 2:
continue
a, b = max(w0 * 1e9, life[0][0]), min(w1 * 1e9, life[-1][0])
if b > a:
total += value_at(life, b) - value_at(life, a)
seen = True
return total if seen else None
def role_cores(
lives_by_container: Iterable[list[list[tuple[int, ...]]]],
w0: float,
w1: float,
) -> float | None:
"""Ядер роли за окно: сумма по контейнерам, `usage_usec` — столбец 1."""
if w1 <= w0:
return None
total = None
for lives in lives_by_container:
delta = lives_delta(lives, w0, w1)
if delta is not None:
total = (total or 0.0) + delta
return None if total is None else round(total / 1e6 / (w1 - w0), 4)
def throttle_share(
lives: Iterable[list[tuple[int, ...]]],
w0: float,
w1: float,
) -> float | None:
"""Доля периодов с троттлингом: столбцы 2 (`nr_periods`), 3 (`nr_throttled`)."""
lives = list(lives)
periods = lives_delta(lives, w0, w1, key=2)
throttled = lives_delta(lives, w0, w1, key=3)
if not periods or throttled is None:
return None
return round(throttled / periods, 4)
def memory_by_lives(
lives: Iterable[list[tuple[int, ...]]],
) -> tuple[float | None, float | None, int]:
"""`memory.peak` и `anon` — наибольшее по жизням, `oom_kill` — сумма: у
каждой жизни считается наибольшее значение."""
peak = anon = None
kills = 0
for life in lives:
peak = max([p[1] for p in life] + ([peak] if peak is not None else []))
anon = max([p[2] for p in life] + ([anon] if anon is not None else []))
kills += max(p[3] for p in life)
return (
None if peak is None else round(peak / MIB, 1),
None if anon is None else round(anon / MIB, 1),
kills,
)
def lb_up_count(rows: Iterable[Mapping[str, str]]) -> int:
"""Реплик в балансировке в строках статистики одного тика LB: только слоты-
серверы, `BACKEND` и `FRONTEND` — не реплики. `UP 1/2` — проверка уже не
прошла, но реплика ещё получает запросы (verify C1)."""
return sum(
1
for row in rows
if row.get("svname") not in ("BACKEND", "FRONTEND")
and (row.get("status") or "").split(" ")[0] == "UP"
)
def lb_up_min(lb: Mapping[int, list[dict[str, str]]]) -> int | None:
"""Наименьшее число реплик в балансировке за все тики; тиков нет — `None`."""
return min((lb_up_count(rows) for rows in lb.values()), default=None)
def lb_down(lb: Mapping[int, list[dict[str, str]]], replicas: int) -> bool:
"""Реплика выходила из балансировки хотя бы на одном тике."""
least = lb_up_min(lb)
return least is not None and least < replicas
@dataclass(frozen=True)
class DockerEvent:
"""Событие `docker events` контейнера (часы ВМ)."""
t: float
action: str
name: str
exit_code: str
signal: str
def parse_docker_events(text: str) -> list[DockerEvent]:
"""Строки `<нс>|<действие>|<имя>|<код выхода>|<сигнал>`."""
out = []
for line in text.splitlines():
parts = line.strip().split("|")
if len(parts) == 5 and parts[0].isdigit():
out.append(
DockerEvent(int(parts[0]) / 1e9, *parts[1:]),
)
return sorted(out, key=lambda event: event.t)
def die_causes(events: Iterable[DockerEvent], name: str) -> dict[str, Any]:
"""Причины завершений контейнера (verify S5). Проба на Docker 29.9: ядро —
`oom`, затем `die` с кодом 137 в те же десятки мс; `docker kill` — `kill` с
сигналом, затем `die`; завершение процесса обработки — `oom` без `die`.
Поэтому `command` — был `kill` с прошлого `die`, `oom` — код 137 рядом с
`oom`, `other` — всё остальное, по кодам выхода."""
own = [event for event in events if event.name == name]
ooms = [event.t for event in own if event.action == "oom"]
causes: dict[str, Any] = {
"oom": 0,
"command": 0,
"other": 0,
"die": 0,
"codes": {},
"oom_events": len(ooms),
}
killed = False
for event in own:
if event.action == "kill":
killed = True
elif event.action == "die":
causes["die"] += 1
if killed:
causes["command"] += 1
elif event.exit_code == "137" and any(
abs(event.t - moment) <= OOM_DIE_WINDOW_S for moment in ooms
):
causes["oom"] += 1
else:
causes["other"] += 1
code = event.exit_code or "-"
causes["codes"][code] = causes["codes"].get(code, 0) + 1
killed = False
return causes
def events(
records: Iterable[Mapping[str, Any]],
name: str,
) -> list[Mapping[str, Any]]:
return [item for item in records if item.get("event") == name]
def rejected_by_reason(
records: Iterable[Mapping[str, Any]],
reason: str,
) -> list[float]:
"""Моменты отказов «Rejected attachment» с этой причиной, по времени."""
return sorted(
reconcile.timestamp(item["timestamp"])
for item in events(records, "Rejected attachment")
if item.get("reason") == reason and item.get("timestamp")
)
def rounded(value: float | None, digits: int = 2) -> float | None:
return None if value is None else round(value, digits)
def percentile(values: list[float], q: float, digits: int = 2) -> float | None:
return rounded(reconcile.percentile(values, q), digits)
# ---------------------------------------------------------------- ячейка
def control_columns(
requests: list[Any],
bounds: Mapping[str, tuple[float, float]],
) -> dict[str, Any]:
out: dict[str, Any] = {}
for kind, name in CONTROL_KINDS.items():
for window, (w0, w1) in bounds.items():
stats = reconcile.window_stats(requests, kind, w0, w1)
prefix = f"ctrl_{name}_{window}"
out[f"{prefix}_n"] = stats["responses"]
out[f"{prefix}_errors"] = stats["errors"]
out[f"{prefix}_s5xx"] = stats["s5xx"]
out[f"{prefix}_p50"] = rounded(stats["p50_ms"], 1)
out[f"{prefix}_p95"] = rounded(stats["p95_ms"], 1)
out[f"{prefix}_p99"] = rounded(stats["p99_ms"], 1)
return out
def cell_row(cell: Mapping[str, Any], src: Sources) -> dict[str, Any]:
"""Строка `cells.csv`. Окна — от начала `q-t0`: `before`, `over`, затем
`recovery` до нуля незавершённых."""
before_s = cell["before_s"]
over_s = cell["over_s"]
over_start = src.t0 + before_s
over_end = over_start + over_s
answers = src.uploads
in_over = [u for u in answers if u.window == "over"]
# последняя загрузка окна: её ответ — конец приёма
load_end = max((u.answered for u in in_over), default=over_end)
zero_at = next(
(t.t for t in src.jobs if t.t >= load_end and t.unfinished == 0),
None,
)
recovery_end = zero_at if zero_at is not None else load_end
accepted = [u for u in answers if u.status == 202]
statuses = [u.status for u in answers]
metas = src.metas
ready = [m for m in metas.values() if m.status == "ready"]
failed = [m for m in metas.values() if m.status == "failed"]
lost = sum(
1
for u in accepted
if u.attachment_id not in metas or metas[u.attachment_id].http != 200
)
stuck = sum(
1
for u in accepted
if u.attachment_id in metas
and metas[u.attachment_id].http == 200
and metas[u.attachment_id].status not in ("ready", "failed")
)
ready_at = [m.updated_at for m in ready if m.updated_at is not None]
def per_second(w0: float, w1: float) -> float | None:
if w1 <= w0:
return None
return round(sum(1 for t in ready_at if w0 <= t < w1) / (w1 - w0), 3)
waits: dict[str, list[float]] = {window: [] for window in WINDOWS}
for upload in accepted:
meta = metas.get(upload.attachment_id)
if meta and meta.status == "ready" and meta.updated_at is not None:
waits[upload.window].append(meta.updated_at - upload.answered)
jobs_over = [t for t in src.jobs if src.t0 <= t.t <= recovery_end]
peak = {
name: max((getattr(t, name) for t in jobs_over), default=None)
for name in (
"unfinished",
"outbox",
"waiting",
"published",
"processing",
"oldest_s",
)
}
mq_inside = [m for m in src.mq if src.t0 <= m[0] <= recovery_end]
# неподтверждённые против взятых: ближайший по времени тик заданий
prefetched = 0
for moment, _, unacked in mq_inside:
nearest = min(src.jobs, key=lambda t: abs(t.t - moment), default=None)
if nearest is not None:
prefetched = max(prefetched, unacked - nearest.processing)
worker_ready = events(src.worker_records, "Attachment became ready")
second_ready = events(src.second_records, "Attachment became ready")
both_ready = [*worker_ready, *second_ready]
started_jobs = events(
[*src.worker_records, *src.second_records],
"Started job attempt",
)
queue_waits = [
float(item["queue_wait_ms"])
for item in started_jobs
if item.get("queue_wait_ms") is not None
]
durations = [float(i["duration_ms"]) for i in both_ready]
renders = [float(i["render_ms"]) for i in both_ready if i.get("render_ms")]
render_peaks = [
float(i["render_peak_kib"]) / 1024
for i in both_ready
if i.get("render_peak_kib")
]
skipped: dict[str, int] = {}
for item in events(src.worker_records, "Skipped job message"):
reason = str(item.get("reason"))
skipped[reason] = skipped.get(reason, 0) + 1
relay_passes = events(src.relay_records, "Relay pass")
busy = rejected_by_reason(src.api_records, "busy")
backlog = rejected_by_reason(src.api_records, "backlog")
retry_values = sorted(
{u.retry_after for u in answers if u.status == 429},
)
worker_lives = src.cg.get("worker", [])
# только основной контейнер worker: лимит памяти у второго свой
peak_mem, anon_mem, kills = memory_by_lives(src.mp.get("worker", []))
# процесс 1, завершённый ядром, уносит cgroup раньше следующего тика: его
# счётчик не прочитать, поэтому — завершения, подтверждённые событиями Docker
causes = die_causes(src.docker_events, src.worker_name)
relay_causes = die_causes(src.docker_events, src.relay_name)
kills += causes["oom"]
per_job = None
delta = lives_delta(
[life for lives in worker_lives for life in lives],
src.t0,
recovery_end,
)
# CPU считается до конца восстановления, значит, и задания — до него: окно
# after в этот счёт не входит (verify W2)
ready_inside = sum(1 for moment in ready_at if moment <= recovery_end)
if delta is not None and ready_inside:
per_job = round(delta / 1e6 / ready_inside, 3)
after_start = src.extra.get("after_start")
after_end = src.extra.get("after_end")
bounds = {
"before": (src.t0, over_start),
"over": (over_start, over_end),
"recovery": (over_end, recovery_end),
"after": (after_start or recovery_end, after_end or recovery_end),
}
# queue lag по окнам: ожидание задания — по окну его прихода, возраст
# старейшего — по окну тика (решение 7, verify W1)
arrival = {u.attachment_id: u.window for u in accepted}
waits_by_window: dict[str, list[float]] = {w: [] for w in WINDOWS}
for item in started_jobs:
window = arrival.get(str(item.get("attachment_id")))
if window and item.get("queue_wait_ms") is not None:
waits_by_window[window].append(float(item["queue_wait_ms"]))
unexpected = None
if src.db_ids is not None:
unexpected = len(
src.db_ids - {u.attachment_id for u in accepted} - src.warm_ids,
)
row: dict[str, Any] = {
key: cell.get(key)
for key in (
"cell",
"config",
"block",
"class",
"concurrency",
"prefetch",
"repeat",
"order",
"rate_low",
"rate_high",
"over_s",
"limit",
"action",
)
}
row.update(
{
"accepted": len(accepted),
"s202": statuses.count(202),
"s429": statuses.count(429),
"s5xx": sum(1 for s in statuses if 500 <= s < 600),
"s_other": sum(
1
for s in statuses
if s not in (0, 202, 429) and not 500 <= s < 600
),
"s_conn": statuses.count(0),
"dropped": src.dropped,
"accepted_before": sum(
1 for u in accepted if u.window == "before"
),
"accepted_over": sum(1 for u in accepted if u.window == "over"),
"accepted_after": sum(1 for u in accepted if u.window == "after"),
"retry_after": " ".join(retry_values),
"busy_429": len(busy),
"backlog_429": len(backlog),
"first_busy_s": rounded(busy[0] - over_start) if busy else None,
"first_backlog_s": (
rounded(backlog[0] - over_start) if backlog else None
),
"upload_before_p50_ms": percentile(
[u.duration_ms for u in accepted if u.window == "before"],
50,
1,
),
"upload_over_p50_ms": percentile(
[u.duration_ms for u in accepted if u.window == "over"],
50,
1,
),
"upload_over_p95_ms": percentile(
[u.duration_ms for u in accepted if u.window == "over"],
95,
1,
),
"ready": len(ready),
"failed": len(failed),
"lost": lost,
"stuck": stuck,
"duplicate": src.sql.get("extra_jobs", 0)
+ src.sql.get("bad_variants", 0),
"sha_checked": len(src.sha_checks),
"sha_bad": src.sha_checks.count(False),
"ready_s_over": per_second(over_start, over_end),
"ready_s_recovery": per_second(over_end, recovery_end),
"over_end": round(over_end, 3),
"zero_at": rounded(zero_at, 3),
"action_at": rounded(src.action_at, 3),
"recovery_s": (
rounded(zero_at - load_end) if zero_at is not None else None
),
"peak_unfinished": peak["unfinished"],
"peak_outbox": peak["outbox"],
"peak_waiting": peak["waiting"],
"peak_published": peak["published"],
"peak_processing": peak["processing"],
"peak_broker_ready": max((m[1] for m in mq_inside), default=None),
"peak_broker_unacked": max(
(m[2] for m in mq_inside),
default=None,
),
"peak_prefetched": prefetched,
"peak_oldest_s": peak["oldest_s"],
"staging_ready_end": (
src.jobs[-1].staging_ready if src.jobs else None
),
"queue_left": sum(src.mq[-1][1:]) if src.mq else None,
"ready_before_list": " ".join(f"{v:.2f}" for v in waits["before"]),
"ready_after_list": " ".join(f"{v:.2f}" for v in waits["after"]),
"queue_wait_p50_ms": percentile(queue_waits, 50, 0),
"queue_wait_p95_ms": percentile(queue_waits, 95, 0),
"duration_p50_ms": percentile(durations, 50, 0),
"render_p50_ms": percentile(renders, 50, 0),
"worker_hwm_mib": (
round(src.hwm_kib / 1024, 1) if src.hwm_kib else None
),
"render_peak_p50_mib": percentile(render_peaks, 50, 1),
"render_peak_max_mib": rounded(
max(render_peaks, default=None),
1,
),
"memory_peak_mib": peak_mem,
"anon_max_mib": anon_mem,
"oom_kill": kills,
"worker_restarts": src.restarts,
"cpu_s3_accept": role_cores(
src.cg.get("s3", []),
over_start,
min(over_end, over_start + ACCEPT_S3_WINDOW_S),
),
"cpu_worker_per_job_s": per_job,
"broker_memory_mib": src.extra.get("broker_memory_mib"),
"broker_refused": src.extra.get("broker_refused", 0),
"attempts_over_one": src.sql.get("attempts_over_one"),
"skipped": json.dumps(skipped, sort_keys=True),
"relay_published": sum(
i.get("published", 0) for i in relay_passes
),
"relay_republished": sum(
i.get("republished", 0) for i in relay_passes
),
"relay_recovered": sum(
i.get("recovered", 0) for i in relay_passes
),
"relay_backlog_full": len(
events(src.relay_records, "Relay backlog is full"),
),
"failed_by": json.dumps(src.sql.get("failed", {}), sort_keys=True),
"second_jobs": len(second_ready) if src.second_records else None,
"swapouts": src.swapouts,
"lb_down": int(src.lb_down),
"relay_restarts": src.relay_restarts,
"relay_die": relay_causes["die"],
"unexpected_rows": unexpected,
"worker_die_oom": causes["oom"],
"worker_die_command": causes["command"],
"worker_die_other": causes["other"],
"worker_exit_codes": json.dumps(causes["codes"], sort_keys=True),
"worker_oom_events": causes["oom_events"],
"lb_up_min": src.lb_up_min,
},
)
for window in WINDOWS:
for q in (50, 95, 99):
row[f"ready_{window}_p{q}_s"] = percentile(waits[window], q)
for q in (50, 95):
row[f"queue_wait_{window}_p{q}_ms"] = percentile(
waits_by_window[window],
q,
0,
)
for window, (w0, w1) in bounds.items():
row[f"peak_oldest_{window}_s"] = max(
(t.oldest_s for t in src.jobs if w0 <= t.t < w1),
default=None,
)
for role in ROLES:
lives = src.cg.get(role, [])
flat = [life for each in lives for life in each]
row[f"cpu_{role}_over"] = role_cores(lives, over_start, over_end)
row[f"cpu_{role}_recovery"] = role_cores(lives, over_end, recovery_end)
for window in ("before", "after"):
row[f"cpu_{role}_{window}"] = role_cores(lives, *bounds[window])
for window, (w0, w1) in bounds.items():
row[f"throttle_{role}_{window}"] = (
throttle_share(flat, w0, w1) if w1 > w0 else None
)
action_at = src.extra.get("second_started_at")
if action_at is not None and src.second_records:
def done_at(item: Mapping[str, Any]) -> float:
return reconcile.timestamp(item["timestamp"])
def share(items: list[Mapping[str, Any]], total: int) -> float | None:
return round(len(items) / total, 3) if total else None
# столбец порога H3 — только разбор накопленного: окно after делится
# поровну при любом prefetch (design, «Уточнения apply», п. 2)
backlog_all = [
i for i in both_ready if action_at <= done_at(i) <= recovery_end
]
backlog_second = [
i for i in second_ready if action_at <= done_at(i) <= recovery_end
]
row["second_jobs"] = len(backlog_second)
row["second_share"] = share(backlog_second, len(backlog_all))
# буквальное прочтение порога и само окно after: числами (verify W3)
since_all = [i for i in both_ready if done_at(i) >= action_at]
since_second = [i for i in second_ready if done_at(i) >= action_at]
row["second_jobs_all"] = len(since_second)
row["second_share_all"] = share(since_second, len(since_all))
row["second_share_after"] = share(
[i for i in second_ready if done_at(i) > recovery_end],
sum(1 for i in both_ready if done_at(i) > recovery_end),
)
row.update(control_columns(src.control, bounds))
row["flags"] = " ".join(
cell_flags(row, zero_at is None, src.storage_down),
)
return row
def cell_flags(
row: Mapping[str, Any],
not_recovered: bool,
storage_down: bool,
) -> list[str]:
"""Пометки решения 11; `oom` — только подтверждённое завершение ядром, а любой
перезапуск — `worker_restart`, его причина — столбцы `worker_die_*`."""
flags = []
if num(row.get("oom_kill")) or num(row.get("worker_oom_events")):
flags.append("oom")
if num(row.get("worker_restarts")):
flags.append("worker_restart")
if num(row.get("worker_die_other")):
flags.append("worker_crash")
if (num(row.get("unexpected_rows")) or 0) > (
(num(row.get("s5xx")) or 0) + (num(row.get("s_conn")) or 0)
):
# строка после 5xx или обрыва законна: на неё клиент не получил 202
flags.append("unexpected")
if num(row.get("lost")):
flags.append("lost")
if num(row.get("stuck")):
flags.append("stuck")
if num(row.get("duplicate")):
flags.append("duplicate")
if num(row.get("sha_bad")):
flags.append("sha_mismatch")
if not_recovered:
flags.append("not_recovered")
if num(row.get("failed")):
flags.append("failed")
if num(row.get("dropped")):
flags.append("dropped")
if num(row.get("lb_down")):
flags.append("lb_down")
if num(row.get("swapouts")):
flags.append("host_swap")
if storage_down:
flags.append("storage_down")
return flags
def timeline_rows(
cell: Mapping[str, Any],
src: Sources,
) -> list[dict[str, Any]]:
"""Посекундная шкала от `q-t0`: окно, очередь в трёх местах, принято и
готово за секунду."""
over_start = src.t0 + cell["before_s"]
over_end = over_start + cell["over_s"]
after_start = src.extra.get("after_start")
accepted: dict[int, int] = {}
for upload in src.uploads:
if upload.status == 202:
second = int(upload.answered - src.t0)
accepted[second] = accepted.get(second, 0) + 1
ready: dict[int, int] = {}
for meta in src.metas.values():
if meta.status == "ready" and meta.updated_at is not None:
second = int(meta.updated_at - src.t0)
ready[second] = ready.get(second, 0) + 1
mq = {int(m[0] - src.t0): m for m in src.mq}
rows = []
for tick in src.jobs:
if tick.t < src.t0:
continue
second = int(tick.t - src.t0)
if tick.t < over_start:
window = "before"
elif tick.t < over_end:
window = "over"
elif after_start is None or tick.t < after_start:
window = "recovery"
else:
window = "after"
broker = mq.get(second)
rows.append(
{
"cell": cell["cell"],
"t": second,
"window": window,
"outbox": tick.outbox,
"waiting": tick.waiting,
"published": tick.published,
"processing": tick.processing,
"unfinished": tick.unfinished,
"broker_ready": broker[1] if broker else None,
"broker_unacked": broker[2] if broker else None,
"prefetched": (
max(0, broker[2] - tick.processing) if broker else None
),
"oldest_s": tick.oldest_s,
"staging_ready": tick.staging_ready,
"accepted": accepted.get(second, 0),
"ready": ready.get(second, 0),
},
)
return rows
# --------------------------------------------------------------- вердикты
def read_csv(path: Path) -> list[dict[str, Any]]:
if not path.exists():
return []
with path.open(encoding="utf-8", newline="") as file:
return list(csv.DictReader(file))
def num(value: Any) -> float | None:
return reconcile.num(value)
def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None:
return reconcile.med(row.get(key) for row in rows)
def select(
rows: Iterable[Mapping[str, Any]],
*,
block: str = "matrix",
klass: str | None = None,
setting: tuple[int, int] | None = None,
) -> list[Mapping[str, Any]]:
found = []
for row in rows:
if row.get("block") != block:
continue
if klass is not None and row.get("class") != klass:
continue
if setting is not None and (
num(row.get("concurrency")) != setting[0]
or num(row.get("prefetch")) != setting[1]
):
continue
found.append(row)
return found
def flagged(row: Mapping[str, Any], flag: str) -> bool:
return flag in (row.get("flags") or "").split()
def classes_of(rows: Iterable[Mapping[str, Any]]) -> list[str]:
return sorted({r["class"] for r in rows if r.get("block") == "matrix"})
def ratio(a: float | None, b: float | None) -> float | None:
return None if a is None or not b else a / b
class Check:
"""Условия гипотезы: `нет данных`, если нечем проверить условие; `нет` — если
условие нарушено, и причина названа."""
def __init__(self) -> None:
self.failed: list[str] = []
self.missing: list[str] = []
self.passed = 0
def within(
self,
name: str,
value: float | None,
low: float | None,
high: float | None,
) -> None:
if value is None:
self.missing.append(name)
elif (low is not None and value < low) or (
high is not None and value > high
):
self.failed.append(f"{name} = {value:.3g}")
else:
self.passed += 1
def true(self, name: str, value: bool | None) -> None:
if value is None:
self.missing.append(name)
elif not value:
self.failed.append(name)
else:
self.passed += 1
def verdict(self) -> tuple[str, str]:
if self.failed:
return REFUTED, "; ".join(self.failed[:6])
if self.missing:
return NO_DATA, "нет: " + ", ".join(self.missing[:6])
if not self.passed:
return NO_DATA, "нет ячеек"
return CONFIRMED, f"условий: {self.passed}"
def h1(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for klass in classes_of(rows):
base = select(rows, klass=klass, setting=(2, 2))
wide = select(rows, klass=klass, setting=(4, 4))
check.within(
f"{klass}: (4,4)/(2,2) throughput",
ratio(
med(wide, "ready_s_recovery"),
med(base, "ready_s_recovery"),
),
None,
1.05,
)
for row in select(rows):
if (num(row.get("concurrency")) or 0) >= 2:
check.within(
f"{row['cell']}: CPU worker",
num(row.get("cpu_worker_over")),
0.85,
None,
)
for klass in ("m", "lp"):
base = select(rows, klass=klass, setting=(2, 2))
small = select(rows, klass=klass, setting=(1, 1))
check.within(
f"{klass}: (1,1)/(2,2) throughput",
ratio(
med(small, "ready_s_recovery"),
med(base, "ready_s_recovery"),
),
None,
1.0,
)
return check
def h2(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
limit = 1024.0
for klass in classes_of(rows):
base = select(rows, klass=klass, setting=(2, 2))
wide = select(rows, klass=klass, setting=(4, 4))
check.within(
f"{klass}: duration (4,4)/(2,2)",
ratio(med(wide, "duration_p50_ms"), med(base, "duration_p50_ms")),
1.5,
None,
)
for row in base:
check.within(
f"{row['cell']}: memory.peak",
num(row.get("memory_peak_mib")),
None,
limit - 0.001,
)
check.true(f"{row['cell']}: без oom", not flagged(row, "oom"))
wide_lp = select(rows, klass="lp", setting=(4, 4))
for row in wide_lp:
check.within(
f"{row['cell']}: memory.peak",
num(row.get("memory_peak_mib")),
0.95 * limit,
None,
)
check.true(
"lp (4,4): oom хотя бы в одном повторе",
any(flagged(r, "oom") for r in wide_lp) if wide_lp else None,
)
wide_lj = select(rows, klass="lj", setting=(4, 4))
for row in wide_lj:
check.true(f"{row['cell']}: без oom", not flagged(row, "oom"))
check.within(
f"{row['cell']}: memory.peak",
num(row.get("memory_peak_mib")),
None,
500,
)
if not wide_lj:
check.missing.append("lj (4,4)")
return check
def h3(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for klass in ("s", "lp"):
base = select(rows, klass=klass, setting=(2, 2))
big = select(rows, klass=klass, setting=(2, 256))
check.within(
f"{klass}: throughput (2,256)/(2,2)",
ratio(med(big, "ready_s_recovery"), med(base, "ready_s_recovery")),
0.85,
1.15,
)
check.within(
f"{klass}: p95 (2,256)/(2,2)",
ratio(med(big, "ready_over_p95_s"), med(base, "ready_over_p95_s")),
0.8,
1.25,
)
for row in big:
check.within(
f"{row['cell']}: broker_ready / пик",
ratio(
num(row.get("peak_broker_ready")),
num(row.get("peak_unfinished")),
),
None,
0.10,
)
for row in base:
check.within(
f"{row['cell']}: broker_ready / пик",
ratio(
num(row.get("peak_broker_ready")),
num(row.get("peak_unfinished")),
),
0.70,
None,
)
p2 = select(rows, block="w2-p2")
p256 = select(rows, block="w2-p256")
check.within(
"доля второго при prefetch 2",
med(p2, "second_share"),
0.30,
None,
)
check.within(
"доля второго при prefetch 256",
med(p256, "second_share"),
None,
0.05,
)
check.within(
"восстановление p256 / p2",
ratio(med(p256, "recovery_s"), med(p2, "recovery_s")),
1.4,
None,
)
return check
def expected_wait(row: Mapping[str, Any]) -> float | None:
return ratio(num(row.get("peak_unfinished")), num(row.get("ready_s_over")))
def h4(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for row in select(rows):
if flagged(row, "oom"):
continue
check.within(
f"{row['cell']}: p95 / (пик / throughput)",
ratio(num(row.get("ready_over_p95_s")), expected_wait(row)),
0.8,
1.25,
)
return check
def h5(rows: list[Mapping[str, Any]], limit: int) -> Check:
check = Check()
on = select(rows, block="gate-on")
off = select(rows, block="gate-off")
if not on or not off:
check.missing.append("блоки gate-on и gate-off")
return check
for row in on:
check.within(
f"{row['cell']}: пик",
num(row.get("peak_unfinished")),
None,
limit + IN_FLIGHT,
)
check.true(f"{row['cell']}: 429 есть", (num(row.get("s429")) or 0) > 0)
check.true(
f"{row['cell']}: Retry-After 5",
row.get("retry_after") == BACKLOG_RETRY_AFTER,
)
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
check.true(
f"{row['cell']}: 429 = записи backlog",
num(row.get("s429")) == num(row.get("backlog_429")),
)
check.within(
f"{row['cell']}: p95 / (предел / throughput)",
ratio(
num(row.get("ready_over_p95_s")),
ratio(float(limit), num(row.get("ready_s_over"))),
),
0.8,
1.25,
)
for row in off:
check.within(
f"{row['cell']}: пик без предела",
num(row.get("peak_unfinished")),
1.5 * limit,
None,
)
check.within(
"p95 on / off",
ratio(med(on, "ready_over_p95_s"), med(off, "ready_over_p95_s")),
None,
0.75,
)
check.within(
"восстановление on / off",
ratio(med(on, "recovery_s"), med(off, "recovery_s")),
None,
0.75,
)
a = med(on, "upload_before_p50_ms")
b = med(off, "upload_before_p50_ms")
check.within(
"загрузка before p50 on − off, мс",
None if a is None or b is None else a - b,
None,
3,
)
return check
def pooled(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]:
values: list[float] = []
for row in rows:
values += [float(v) for v in (row.get(key) or "").split()]
return values
def h6(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for row in rows:
for flag in H6_FLAGS:
check.true(f"{row['cell']}: нет {flag}", not flagged(row, flag))
if num(row.get("failed")):
only_exhausted = all(
key.startswith("retry_exhausted")
for key in json.loads(row.get("failed_by") or "{}")
)
check.true(
f"{row['cell']}: failed только при oom и retry_exhausted",
flagged(row, "oom") and only_exhausted,
)
if flagged(row, "oom") or row.get("block") not in (
"matrix",
"gate-on",
"gate-off",
):
continue
bound = expected_wait(row)
check.within(
f"{row['cell']}: восстановление / (пик / throughput)",
ratio(num(row.get("recovery_s")), bound),
None,
1.3,
)
by_config: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
by_config.setdefault(row["config"], []).append(row)
for config, items in by_config.items():
before = reconcile.percentile(pooled(items, "ready_before_list"), 95)
after = reconcile.percentile(pooled(items, "ready_after_list"), 95)
check.within(
f"{config}: p95 after / before",
ratio(after, before),
None,
1.5,
)
return check
def h7(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for row in rows:
if row.get("block") == "accept":
continue
for window in CONTROL_WINDOWS:
for kind, limit in (
("write", 200),
("read", 200),
("download", 500),
):
value = num(row.get(f"ctrl_{kind}_{window}_p95"))
if value is not None:
check.within(
f"{row['cell']}: {kind} {window} p95",
value,
None,
limit,
)
check.true(
f"{row['cell']}: {kind} {window} 5xx нет",
not num(row.get(f"ctrl_{kind}_{window}_s5xx")),
)
check.true(
f"{row['cell']}: все реплики в LB",
not num(row.get("lb_down")),
)
return check
def h8(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
kills = select(rows, block="kill-worker")
restarts = select(rows, block="restart-broker")
if not kills:
check.missing.append("блок kill-worker")
if not restarts:
check.missing.append("блок restart-broker")
for row in kills:
check.true(
f"{row['cell']}: всё ready",
num(row.get("ready")) == num(row.get("accepted")),
)
check.within(
f"{row['cell']}: заданий с повтором",
num(row.get("attempts_over_one")),
None,
2,
)
end = num(row.get("over_end"))
action = num(row.get("action_at"))
wait = expected_wait(row)
zero = num(row.get("zero_at"))
if None in (end, action, wait, zero):
check.missing.append(f"{row['cell']}: моменты")
continue
check.within(
f"{row['cell']}: разбор до границы, с",
zero - max(end + 1.3 * wait, action + LEASE_S + 15),
None,
0,
)
for row in restarts:
check.true(
f"{row['cell']}: всё ready",
num(row.get("ready")) == num(row.get("accepted")),
)
check.within(
f"{row['cell']}: заданий с повтором",
num(row.get("attempts_over_one")),
None,
0,
)
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
wait = expected_wait(row)
recovery = num(row.get("recovery_s"))
check.within(
f"{row['cell']}: восстановление, с",
None
if wait is None or recovery is None
else recovery - (1.3 * wait + 15),
None,
0,
)
return check
def h9(rows: list[Mapping[str, Any]], limit: int) -> Check:
check = Check()
cells = select(rows, block="accept")
if not cells:
check.missing.append("блок accept")
for row in cells:
busy = num(row.get("first_busy_s"))
backlog = num(row.get("first_backlog_s"))
check.true(
f"{row['cell']}: первый busy раньше backlog",
None if busy is None or backlog is None else busy < backlog,
)
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
check.within(
f"{row['cell']}: CPU S3 первые 30 с",
num(row.get("cpu_s3_accept")),
0.85,
None,
)
check.within(
f"{row['cell']}: принято/с",
ratio(num(row.get("accepted_over")), num(row.get("over_s"))),
6,
None,
)
check.within(
f"{row['cell']}: пик",
num(row.get("peak_unfinished")),
None,
limit + IN_FLIGHT,
)
check.within(
f"{row['cell']}: готовых/с в окне",
num(row.get("ready_s_over")),
None,
1.5,
)
check.within(
f"{row['cell']}: готовых/с при восстановлении",
num(row.get("ready_s_recovery")),
2.4,
None,
)
for window in CONTROL_WINDOWS:
for kind, bound in (
("write", 200),
("read", 200),
("download", 1000),
):
value = num(row.get(f"ctrl_{kind}_{window}_p95"))
if value is not None:
check.within(
f"{row['cell']}: {kind} {window} p95",
value,
None,
bound,
)
check.true(
f"{row['cell']}: {kind} {window} 5xx нет",
not num(row.get(f"ctrl_{kind}_{window}_s5xx")),
)
return check
def verdicts(
rows: list[Mapping[str, Any]],
limit: int,
) -> dict[str, tuple[str, str]]:
return {
"H1": h1(rows).verdict(),
"H2": h2(rows).verdict(),
"H3": h3(rows).verdict(),
"H4": h4(rows).verdict(),
"H5": h5(rows, limit).verdict(),
"H6": h6(rows).verdict(),
"H7": h7(rows).verdict(),
"H8": h8(rows).verdict(),
"H9": h9(rows, limit).verdict(),
}
def settings_of(rows: Iterable[Mapping[str, Any]]) -> list[tuple[int, int]]:
return sorted(
{
(int(num(r["concurrency"])), int(num(r["prefetch"])))
for r in rows
if r.get("block") == "matrix"
},
)
def recommend(rows: list[Mapping[str, Any]]) -> dict[str, Any]:
"""Правило решения 12: кандидат — без `oom` и `failed` на всех классах, с
throughput не ниже 90 % лучшей на каждом классе; из кандидатов — наименьшая
конкуренция; `prefetch` равен конкуренции. Настройка, не снятая на части
классов, не кандидат: условие «на всех классах» для неё не проверено, она
названа в `partial` с этими классами (verify S6). В сравнении лучшей она
участвует: её снятые числа настоящие."""
settings = settings_of(rows)
candidates = []
reasons: dict[str, str] = {}
partial: dict[str, list[str]] = {}
for setting in settings:
problem = None
key = f"{setting[0]},{setting[1]}"
missing = [
klass
for klass in classes_of(rows)
if not select(rows, klass=klass, setting=setting)
]
if missing:
partial[key] = missing
for klass in classes_of(rows):
cells = select(rows, klass=klass, setting=setting)
if not cells:
continue
if any(flagged(r, "oom") or flagged(r, "failed") for r in cells):
problem = f"{klass}: oom или failed"
break
best = max(
(
value
for other in settings
if (
value := med(
select(rows, klass=klass, setting=other),
"ready_s_recovery",
)
)
is not None
),
default=None,
)
own = med(cells, "ready_s_recovery")
if best is None or own is None or own < 0.9 * best:
problem = f"{klass}: throughput ниже 90 % лучшей"
break
if problem:
reasons[key] = problem
elif not missing:
candidates.append(setting)
if not candidates:
return {
"setting": None,
"candidates": [],
"rejected": reasons,
"partial": partial,
}
concurrency = min(setting[0] for setting in candidates)
return {
"setting": (concurrency, concurrency),
"candidates": candidates,
"rejected": reasons,
"partial": partial,
}
# ----------------------------------------------------------------- сводка
def fmt(value: Any, digits: int = 1) -> str:
number = num(value)
return "—" if number is None else f"{number:.{digits}f}"
def spread(
rows: Iterable[Mapping[str, Any]],
key: str,
digits: int = 1,
) -> str:
return reconcile.spread((row.get(key) for row in rows), digits)
def config_table(rows: list[Mapping[str, Any]]) -> list[str]:
head = (
"| Конфигурация | Ячеек | Принято | 429 | Готовых/с: окно | восстановление | "
"Пик незаверш. | Пик брокера | До `ready` over p50 / p95 / p99, с | "
"Восст., с | `memory.peak`, МиБ | VmHWM, МиБ | Процесс p50 / макс., МиБ | "
"CPU worker, S3 | Троттлинг worker | OOM | Перезапуски | Пометки | "
"Queue lag: возраст старейшего, пик before / over / after, с | "
"Ожидание в очереди p50 / p95, мс: before; over; after | "
"Завершения worker: ядро / команда / прочие | Реплик в LB, минимум |"
)
lines = [head, "|" + "---|" * 22]
by_config: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
by_config.setdefault(row["config"], []).append(row)
for config in sorted(by_config):
items = by_config[config]
flags = sorted(
{f for r in items for f in (r.get("flags") or "").split()},
)
lines.append(
f"| `{config}` | {len(items)} | {spread(items, 'accepted', 0)} | "
f"{spread(items, 's429', 0)} | {spread(items, 'ready_s_over', 2)} | "
f"{spread(items, 'ready_s_recovery', 2)} | "
f"{spread(items, 'peak_unfinished', 0)} | "
f"{spread(items, 'peak_broker_ready', 0)} | "
f"{fmt(med(items, 'ready_over_p50_s'))} / "
f"{fmt(med(items, 'ready_over_p95_s'))} / "
f"{fmt(med(items, 'ready_over_p99_s'))} | "
f"{spread(items, 'recovery_s', 1)} | "
f"{spread(items, 'memory_peak_mib', 0)} | "
f"{fmt(med(items, 'worker_hwm_mib'), 0)} | "
f"{fmt(med(items, 'render_peak_p50_mib'), 0)} / "
f"{fmt(med(items, 'render_peak_max_mib'), 0)} | "
f"{fmt(med(items, 'cpu_worker_over'), 2)}, "
f"{fmt(med(items, 'cpu_s3_over'), 2)} | "
f"{fmt(med(items, 'throttle_worker_over'), 2)} | "
f"{sum(int(num(r.get('oom_kill')) or 0) for r in items)} | "
f"{sum(int(num(r.get('worker_restarts')) or 0) for r in items)} | "
f"{' '.join(flags) or '—'} | "
+ " / ".join(
fmt(med(items, f"peak_oldest_{w}_s")) for w in WINDOWS
)
+ " | "
+ "; ".join(
f"{fmt(med(items, f'queue_wait_{w}_p50_ms'), 0)} / "
f"{fmt(med(items, f'queue_wait_{w}_p95_ms'), 0)}"
for w in WINDOWS
)
+ " | "
+ " / ".join(
str(
sum(
int(num(r.get(f"worker_die_{k}")) or 0) for r in items
),
)
for k in ("oom", "command", "other")
)
+ " | "
+ fmt(
min(
(
value
for r in items
if (value := num(r.get("lb_up_min"))) is not None
),
default=None,
),
0,
)
+ " |",
)
return lines
def control_table(rows: list[Mapping[str, Any]]) -> list[str]:
lines = [
"| Конфигурация | Окно | Запись p50 / p95 / p99, мс | Чтение p50 / p95 / p99, "
"мс | Скачивание p50 / p95 / p99, мс | Ответов | 5xx |",
"|" + "---|" * 7,
]
by_config: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
by_config.setdefault(row["config"], []).append(row)
for config in sorted(by_config):
items = by_config[config]
for window in CONTROL_WINDOWS:
parts = []
total = errors = 0.0
for kind in CONTROL_KINDS.values():
prefix = f"ctrl_{kind}_{window}"
parts.append(
" / ".join(
fmt(med(items, f"{prefix}_{stat}"))
for stat in ("p50", "p95", "p99")
),
)
total += sum(num(r.get(f"{prefix}_n")) or 0 for r in items)
errors += sum(num(r.get(f"{prefix}_s5xx")) or 0 for r in items)
lines.append(
f"| `{config}` | {window} | {parts[0]} | {parts[1]} | {parts[2]} "
f"| {int(total)} | {int(errors)} |",
)
return lines
def block_table(rows: list[Mapping[str, Any]]) -> list[str]:
lines = [
"| Ячейка | Принято | 429 busy / backlog | Первый busy / backlog, с | Пик | "
"Готовых/с окно / восст. | Восст., с | Повторов | Второй worker: задания, доля "
"| Пометки | Второй: доля после запуска всего / в окне after | "
"Relay: перезапуски / завершения |",
"|" + "---|" * 12,
]
for row in sorted(rows, key=lambda r: r["cell"]):
if row.get("block") == "matrix":
continue
lines.append(
f"| `{row['cell']}` | {fmt(row.get('accepted'), 0)} | "
f"{fmt(row.get('busy_429'), 0)} / {fmt(row.get('backlog_429'), 0)} | "
f"{fmt(row.get('first_busy_s'))} / "
f"{fmt(row.get('first_backlog_s'))} | "
f"{fmt(row.get('peak_unfinished'), 0)} | "
f"{fmt(row.get('ready_s_over'), 2)} / "
f"{fmt(row.get('ready_s_recovery'), 2)} | "
f"{fmt(row.get('recovery_s'))} | "
f"{fmt(row.get('attempts_over_one'), 0)} | "
f"{fmt(row.get('second_jobs'), 0)}, "
f"{fmt(row.get('second_share'), 2)} | "
f"{row.get('flags') or '—'} | "
f"{fmt(row.get('second_share_all'), 3)} / "
f"{fmt(row.get('second_share_after'), 3)} | "
f"{fmt(row.get('relay_restarts'), 0)} / "
f"{fmt(row.get('relay_die'), 0)} |",
)
return lines
def render_summary(
meta: Mapping[str, Any],
rows: list[Mapping[str, Any]],
) -> str:
info = meta.get("queue", {})
limit = int(info.get("default_limit") or 250)
plan_total = len(info.get("plan", []))
lines = [
f"# Очередь под перегрузкой: метка {meta.get('label')}",
"",
f"commit `{meta.get('git', {}).get('commit')}`, "
f"dirty {meta.get('git', {}).get('dirty')}, статус "
f"{meta.get('status')}, ячеек {len(rows)} из {plan_total}.",
"",
]
fatal = [
f"`{row['cell']}`: {row['flags']}"
for row in rows
if any(flagged(row, flag) for flag in FATAL_FLAGS)
]
if fatal:
lines += [
"**Потеряна работа или лишние вложения:** " + "; ".join(fatal),
"",
]
lines += ["## Гипотезы", "", "| № | Вердикт | Детали |", "|---|---|---|"]
for name, (verdict, detail) in verdicts(rows, limit).items():
lines.append(f"| {name} | {verdict} | {detail} |")
advice = recommend(rows)
setting = advice["setting"]
lines += ["", "## Рекомендация", ""]
if setting is None:
lines.append("Кандидатов нет: правило не выбрало настройку.")
else:
keep = setting == DEFAULT_SETTING
lines.append(
f"Конкуренция {setting[0]}, prefetch {setting[1]}: "
+ (
"значения по умолчанию оставить."
if keep
else "отличается от (2, 2), решение — по правилу в отчёте."
),
)
for key, reason in advice["rejected"].items():
lines.append(f"- ({key.replace(',', ', ')}) не кандидат: {reason}")
for key, missing in advice["partial"].items():
lines.append(
f"- ({key.replace(',', ', ')}) снята не на всех классах (нет: "
f"{', '.join(missing)}): без данных не кандидат",
)
lines += [
"",
"## Ячейки по конфигурациям",
"",
"Значения — медиана (min–max) повторов.",
"",
*config_table(rows),
"",
"## Блоки",
"",
*block_table(rows),
"",
"## Контрольный поток",
"",
*control_table(rows),
"",
]
return "\n".join(lines)
def setting_chart(
rows: list[Mapping[str, Any]],
key: str,
title: str,
unit: str,
) -> str:
"""Столбики: класс → настройки, медиана повторов."""
classes = classes_of(rows)
settings = settings_of(rows)
colors = ["#2a78d6", "#d98a00", "#3a9a5b", "#b0569c"]
width, height = 860, 420
svg = plots.Svg(width, height, title)
left, right, top, bottom = 70, 20, 50, 60
values = [
med(select(rows, klass=klass, setting=setting), key)
for klass in classes
for setting in settings
]
highest = max((v for v in values if v is not None), default=0.0) or 1.0
y_scale = (height - top - bottom) / (highest * 1.1)
group = (width - left - right) / max(1, len(classes))
bar = group / (len(settings) + 1)
base = height - bottom
svg.line(left, base, width - right, base, color=plots.AXIS)
svg.text(left, top - 14, unit, size=12)
for ci, klass in enumerate(classes):
x0 = left + ci * group
svg.text(x0 + group / 2 - 8, base + 22, klass, size=13)
for si, setting in enumerate(settings):
value = med(select(rows, klass=klass, setting=setting), key)
if value is None:
continue
bar_h = value * y_scale
svg.bar(
x0 + si * bar + bar / 2,
base - bar_h,
bar * 0.9,
bar_h,
colors[si % len(colors)],
f"{klass} ({setting[0]}, {setting[1]}): {value:.2f} {unit}",
top=True,
)
for si, setting in enumerate(settings):
x = left + si * 130
svg.bar(
x,
height - 28,
14,
14,
colors[si % len(colors)],
"",
top=False,
)
svg.text(x + 20, height - 16, f"({setting[0]}, {setting[1]})", size=12)
return svg.render()
def gate_chart(timeline: list[Mapping[str, Any]]) -> str:
"""Незавершённые посекундно: блоки предела, первый повтор."""
width, height = 860, 420
svg = plots.Svg(width, height, "Незавершённые задания: предел и без него")
left, right, top, bottom = 70, 20, 50, 50
series: dict[str, list[tuple[float, float]]] = {}
for row in timeline:
cell = row.get("cell", "")
if cell.startswith(("gate-on", "gate-off")) and cell.endswith("-r1"):
t = num(row.get("t"))
value = num(row.get("unfinished"))
if t is not None and value is not None:
series.setdefault(cell, []).append((t, value))
if not series:
svg.text(left, top + 30, "нет данных", size=13)
return svg.render()
max_t = max(p[0] for points in series.values() for p in points) or 1.0
max_v = max(p[1] for points in series.values() for p in points) or 1.0
colors = ["#2a78d6", "#d98a00"]
def sx(t: float) -> float:
return left + t / max_t * (width - left - right)
def sy(v: float) -> float:
return height - bottom - v / (max_v * 1.1) * (height - top - bottom)
svg.line(
left,
height - bottom,
width - right,
height - bottom,
color=plots.AXIS,
)
svg.text(
left,
top - 14,
f"до {max_v:.0f} заданий, до {max_t:.0f} с",
size=12,
)
for index, (cell, points) in enumerate(sorted(series.items())):
svg.polyline(
[(sx(t), sy(v)) for t, v in points],
colors[index % len(colors)],
)
svg.text(left + index * 260, height - 18, cell, size=12)
return svg.render()
def render_plots(
rows: list[Mapping[str, Any]],
timeline: list[Mapping[str, Any]],
) -> dict[str, str]:
return {
"throughput-by-setting.svg": setting_chart(
rows,
"ready_s_recovery",
"Throughput по настройкам и классам",
"готовых/с",
),
"p95-by-setting.svg": setting_chart(
rows,
"ready_over_p95_s",
"p95 времени до ready, окно over",
"с",
),
"memory-by-setting.svg": setting_chart(
rows,
"memory_peak_mib",
"memory.peak worker",
"МиБ",
),
"backlog-gate.svg": gate_chart(timeline),
}
# -------------------------------------------------------------- самопроверка
def base_row(**values: Any) -> dict[str, Any]:
row = {
"cell": "s-c2p2-r1",
"config": "s-c2p2",
"block": "matrix",
"class": "s",
"concurrency": "2",
"prefetch": "2",
"repeat": "1",
"accepted": "100",
"ready": "100",
"failed": "0",
"s429": "0",
"s5xx": "0",
"busy_429": "0",
"backlog_429": "0",
"ready_s_over": "3",
"ready_s_recovery": "3",
"peak_unfinished": "60",
"peak_broker_ready": "50",
"ready_over_p95_s": "20",
"recovery_s": "20",
"memory_peak_mib": "200",
"duration_p50_ms": "300",
"cpu_worker_over": "0.95",
"failed_by": "{}",
"ready_before_list": "1.0 1.1",
"ready_after_list": "1.0 1.2",
"attempts_over_one": "0",
"over_end": "100",
"zero_at": "120",
"action_at": "90",
"second_share": "",
"flags": "",
"lb_down": "0",
"retry_after": "",
}
for key, value in values.items():
row[key] = value
return row
def matrix_fixture() -> list[dict[str, Any]]:
rows = []
for klass in ("s", "m", "lj", "lp"):
rows.append(
base_row(
cell=f"{klass}-c2p2-r1",
config=f"{klass}-c2p2",
**{"class": klass},
),
)
wide = {
"duration_p50_ms": "600",
"memory_peak_mib": "400",
}
flags = ""
if klass == "lp":
wide["memory_peak_mib"] = "1024"
wide["oom_kill"] = "3"
flags = "oom"
rows.append(
base_row(
cell=f"{klass}-c4p4-r1",
config=f"{klass}-c4p4",
concurrency="4",
prefetch="4",
flags=flags,
**{"class": klass, **wide},
),
)
for klass in ("m", "lp"):
rows.append(
base_row(
cell=f"{klass}-c1p1-r1",
config=f"{klass}-c1p1",
concurrency="1",
prefetch="1",
ready_s_recovery="2.6",
cpu_worker_over="0.7",
**{"class": klass},
),
)
for klass in ("s", "lp"):
rows.append(
base_row(
cell=f"{klass}-c2p256-r1",
config=f"{klass}-c2p256",
prefetch="256",
peak_broker_ready="0",
**{"class": klass},
),
)
return rows
def gate_fixture() -> list[dict[str, Any]]:
on = base_row(
cell="gate-on-s-c2p2-r1",
config="gate-on-s-c2p2",
block="gate-on",
peak_unfinished="255",
s429="180",
backlog_429="180",
retry_after="5",
ready_over_p95_s="85",
recovery_s="85",
upload_before_p50_ms="15",
)
off = base_row(
cell="gate-off-s-c2p2-r1",
config="gate-off-s-c2p2",
block="gate-off",
peak_unfinished="430",
ready_over_p95_s="145",
recovery_s="145",
upload_before_p50_ms="14",
)
return [on, off]
def selftest() -> None:
# план: без повтора подряд на границе кругов, все ячейки
configs = [
{"block": "matrix", "class": k, "concurrency": 2, "prefetch": 2}
for k in ("s", "m", "lj")
]
plan = plan_cells(configs, 3, 7)
assert len(plan) == 9
for prev, cur in zip(plan, plan[1:], strict=False):
assert prev["config"] != cur["config"]
assert plan[0]["order"] == 1 and plan[-1]["order"] == 9
# разбор строк
t0, ups = parse_uploads(
"q-t0,1000000\n"
"q,before,1000000,10,202,-,a,id1\n"
"q,over,1010000,20,429,5,a,-\nбред\n",
)
assert t0 == 1000.0 and len(ups) == 2 and ups[1].retry_after == "5"
metas, checks = parse_sweep(
"qs,id1,200,ready,-,3,2026-11-01T10:00:00.123456Z\n"
"qs,id2,404,-,-,0,-\nqv,id1,original,1\nqv,id1,crop,0\n",
)
assert abs(metas["id1"].updated_at % 1 - 0.123456) < 1e-5
assert metas["id2"].updated_at is None and checks == [True, False]
jobs = parse_jobs("jobs|1000000|1|2|3|4|5.5|6\nсбой\n")
assert jobs[0].unfinished == 10 and jobs[0].oldest_s == 5.5
mq, missing = parse_mq("mq 1000000000 5 2\nmq 2000000000 missing\n")
assert mq == [(1.0, 5, 2)] and missing == 1
# жизни: перезапуск сбрасывает счётчики
points = [
(10**9, 100, 10, 1),
(2 * 10**9, 200, 20, 2),
(3 * 10**9, 50, 5, 0),
(4 * 10**9, 150, 15, 1),
]
lives = split_lives(points)
assert len(lives) == 2
assert lives_delta(lives, 0, 5) == 200
assert role_cores([lives], 1, 4) == round(200 / 1e6 / 3, 4)
mp = [
(10**9, 500 * MIB, 400 * MIB, 2),
(2 * 10**9, 700 * MIB, 600 * MIB, 3),
(3 * 10**9, 100 * MIB, 90 * MIB, 0),
(4 * 10**9, 300 * MIB, 200 * MIB, 1),
]
assert memory_by_lives(split_lives(mp, key=1)) == (700.0, 600.0, 4)
# LB: BACKEND и FRONTEND — не реплики, UP 1/2 — ещё в балансировке
both = [
{"svname": "api-1", "status": "UP"},
{"svname": "api-2", "status": "UP"},
{"svname": "api-3", "status": "MAINT"},
{"svname": "BACKEND", "status": "UP"},
]
one = [
{"svname": "api-1", "status": "UP"},
{"svname": "api-2", "status": "DOWN"},
{"svname": "BACKEND", "status": "UP"},
]
assert lb_up_count(both) == 2
assert lb_up_count(one) == 1, "реплика вышла, BACKEND посчитан как UP"
assert lb_up_min({1: both, 2: one}) == 1
assert lb_up_min({}) is None
assert not lb_down({1: both}, 2), "обе UP — реплика не выходила"
assert lb_down({1: both, 2: one}, 2), "одна UP и BACKEND UP — вышла"
assert not lb_down({}, 2)
assert not lb_down(
{1: [{"svname": "api-1", "status": "UP 1/2"}, *one[:1]]},
2,
), "UP 1/2 — ещё в балансировке"
# причины завершений: ядро, команда, прочее; процесс обработки — без die
happened = parse_docker_events(
"1000000000|oom|w|| \n"
"1000000000|oom|w||\n"
"9000000000|oom|w||\n"
"9050000000|die|w|137|\n"
"20000000000|kill|w||9\n"
"20050000000|die|w|137|\n"
"30000000000|die|w|3|\n"
"40000000000|die|w|137|\n"
"50000000000|die|other|137|\nбред\n",
)
found = die_causes(happened, "w")
assert found["oom"] == 1 and found["command"] == 1, found
assert found["other"] == 2 and found["die"] == 4, found
assert found["codes"] == {"3": 1, "137": 1}, found
assert found["oom_events"] == 3, found
# ячейка на границах окон: потерянное, зависшее, OOM, перезапуск
base = {
"cell": "s-c2p2-r1",
"config": "s-c2p2",
"block": "matrix",
"class": "s",
"concurrency": 2,
"prefetch": 2,
"repeat": 1,
"order": 1,
"before_s": 10,
"over_s": 20,
}
t = 1000.0
uploads_seen = [
Upload("before", int(t * 1000), 10, 202, "-", "a", "b1"),
Upload("over", int((t + 10) * 1000), 10, 202, "-", "a", "o1"),
Upload("over", int((t + 11) * 1000), 10, 202, "-", "a", "o2"),
Upload("over", int((t + 12) * 1000), 10, 202, "-", "a", "o3"),
Upload("over", int((t + 13) * 1000), 10, 429, "5", "a", "-"),
]
src = Sources(
uploads=uploads_seen,
t0=t,
metas={
"b1": Meta(200, "ready", "-", 3, t + 1),
# ровно на начале окна over: входит
"o1": Meta(200, "ready", "-", 3, t + 10),
"o2": Meta(200, "queued", "-", 0, None),
},
sha_checks=[True],
jobs=[
JobTick(t + 5, 0, 0, 0, 1, 0.1, 0),
JobTick(t + 20, 1, 1, 1, 2, 9.0, 1),
JobTick(t + 40, 0, 0, 0, 0, 0.0, 0),
],
mq=[(t + 20, 3, 2), (t + 40, 0, 0)],
mq_missing=0,
cg={
"worker": [
split_lives(
[(int((t + i) * 1e9), i * 10**6, i, 0) for i in range(41)],
),
],
},
mp={"worker": split_lives(mp)},
control=[],
worker_records=[],
second_records=[],
relay_records=[],
api_records=[],
sql={"extra_jobs": 0, "bad_variants": 0, "attempts_over_one": 1},
hwm_kib=2048,
restarts=1,
docker_events=[
DockerEvent(t + 15, "oom", "worker", "", ""),
DockerEvent(t + 15.05, "die", "worker", "137", ""),
DockerEvent(t + 16, "die", "relay", "1", ""),
],
relay_restarts=1,
lb_up_min=2,
db_ids={"b1", "o1", "o2", "o3", "warm"},
warm_ids={"warm"},
)
row = cell_row(base, src)
assert row["lost"] == 1 and row["stuck"] == 1
# счётчик cgroup 4 и завершение процесса 1, подтверждённое событиями
assert row["ready"] == 2 and row["oom_kill"] == 5
assert row["worker_die_oom"] == 1 and row["worker_die_other"] == 0
assert row["relay_restarts"] == 1 and row["relay_die"] == 1
assert row["unexpected_rows"] == 0 and "unexpected" not in row["flags"]
assert row["lb_up_min"] == 2
assert "lost" in row["flags"] and "stuck" in row["flags"]
# перезапуск не от ядра — не oom; лишняя строка без 5xx и обрыва — пометка
plain = replace(
src,
mp={},
docker_events=[DockerEvent(t + 15, "die", "worker", "3", "")],
db_ids={"b1", "o1", "o2", "o3", "warm", "чужое"},
)
crash = cell_row(base, plain)
assert crash["oom_kill"] == 0 and crash["worker_die_other"] == 1, crash
assert crash["worker_exit_codes"] == '{"3": 1}'
flags = crash["flags"].split()
assert "oom" not in flags and "worker_restart" in flags, flags
assert "worker_crash" in flags and "unexpected" in flags, flags
assert crash["unexpected_rows"] == 1
explained = replace(
plain,
uploads=[
*uploads_seen,
Upload("over", int((t + 14) * 1000), 10, 503, "-", "a", "-"),
],
)
assert "unexpected" not in cell_row(base, explained)["flags"].split()
# CPU до конца восстановления делится на задания до него: не на окно after
zero_end = replace(
src,
metas={
"b1": Meta(200, "ready", "-", 3, t + 1),
"o1": Meta(200, "ready", "-", 3, t + 20),
"o2": Meta(200, "ready", "-", 3, t + 30),
"o3": Meta(200, "ready", "-", 3, t + 39),
"a1": Meta(200, "ready", "-", 3, t + 45),
"a2": Meta(200, "ready", "-", 3, t + 46),
},
)
# от t + 0 до нуля незавершённых в t + 40: 40 с CPU по 1 ядру на 4 задания
assert cell_row(base, zero_end)["cpu_worker_per_job_s"] == 10.0
# queue lag по окнам: ожидание — по окну прихода, разогрева в счёте нет
laggy = replace(
src,
worker_records=[
{
"event": "Started job attempt",
"attachment_id": ident,
"queue_wait_ms": wait,
}
for ident, wait in (
("b1", 100),
("o1", 1000),
("o2", 3000),
("warm", 9000),
)
],
)
lag = cell_row(base, laggy)
assert lag["queue_wait_before_p50_ms"] == 100
assert lag["queue_wait_over_p50_ms"] == 1000
assert lag["queue_wait_over_p95_ms"] == 3000
assert lag["queue_wait_after_p50_ms"] is None
assert lag["queue_wait_p95_ms"] == 9000, "столбец на ячейку прежний"
assert lag["peak_oldest_before_s"] == 0.1
assert lag["peak_oldest_over_s"] == 9.0
assert set(CELL_FIELDS) >= set(lag), set(lag) - set(CELL_FIELDS)
assert len(CELL_FIELDS) == len(set(CELL_FIELDS))
# доля второго: порог H3 — только разбор, окно after — отдельным числом
done = [
{
"event": "Attachment became ready",
"timestamp": stamp,
"duration_ms": 300,
}
for stamp in (
"2026-11-01T10:00:10Z",
"2026-11-01T10:00:20Z",
"2026-11-01T10:00:50Z",
"2026-11-01T10:00:50Z",
)
]
other_done = done[3:]
done = done[:3]
second = reconcile.timestamp("2026-11-01T10:00:00Z")
shared = cell_row(
base,
replace(
src,
t0=second,
uploads=[
Upload(
"over",
int((second + 25) * 1000),
10,
202,
"-",
"a",
"o1",
),
],
jobs=[
JobTick(second + 28, 0, 0, 0, 1, 1.0, 0),
JobTick(second + 30, 0, 0, 0, 0, 0.0, 0),
],
worker_records=done,
second_records=other_done,
extra={"second_started_at": second + 5},
),
)
assert shared["second_jobs"] == 0 and shared["second_share"] == 0
assert shared["second_jobs_all"] == 1
assert shared["second_share_all"] == 0.25
assert shared["second_share_after"] == 0.5
# край окна — между тиками: CPU не занижен
assert row["cpu_worker_over"] == 1.0, row["cpu_worker_over"]
assert "oom" in row["flags"] and "worker_restart" in row["flags"]
assert row["ready_s_over"] == round(1 / 20, 3)
assert row["peak_unfinished"] == 5 and row["peak_prefetched"] == 0
assert row["recovery_s"] is not None and row["memory_peak_mib"] == 700.0
assert row["accepted_over"] == 3 and row["s429"] == 1
timeline = timeline_rows(base, src)
assert [r["window"] for r in timeline] == ["before", "over", "recovery"]
# вердикты: подтверждено, нет, нет данных
rows = matrix_fixture() + gate_fixture()
w2 = [
base_row(
cell="w2-p2-r1",
config="w2-p2",
block="w2-p2",
second_share="0.4",
recovery_s="20",
),
base_row(
cell="w2-p256-r1",
config="w2-p256",
block="w2-p256",
prefetch="256",
second_share="0",
recovery_s="30",
),
]
restarts = [
base_row(
cell="kill-r1",
config="kill",
block="kill-worker",
attempts_over_one="2",
),
base_row(cell="brk-r1", config="brk", block="restart-broker"),
]
accept = [
base_row(
cell="acc-r1",
config="acc",
block="accept",
first_busy_s="10",
first_backlog_s="35",
cpu_s3_accept="0.97",
accepted_over="300",
over_s="45",
peak_unfinished="280",
ready_s_over="0.6",
ready_s_recovery="2.8",
ctrl_download_over_p95="340",
ready_over_p95_s="93",
recovery_s="100",
),
]
everything = rows + w2 + restarts + accept
assert h1(everything).verdict()[0] == CONFIRMED, h1(everything).verdict()
assert h2(everything).verdict()[0] == CONFIRMED, h2(everything).verdict()
assert h3(everything).verdict()[0] == CONFIRMED, h3(everything).verdict()
assert h4(rows[:1]).verdict()[0] == CONFIRMED
assert h5(everything, 250).verdict()[0] == CONFIRMED, h5(
everything,
250,
).verdict()
assert h6(rows[:1]).verdict()[0] == CONFIRMED, h6(rows[:1]).verdict()
assert h7(everything).verdict()[0] == CONFIRMED
assert h8(everything).verdict()[0] == CONFIRMED, h8(everything).verdict()
assert h9(everything, 250).verdict()[0] == CONFIRMED, h9(
everything,
250,
).verdict()
empty: list[dict[str, Any]] = []
for name, (verdict, _) in verdicts(empty, 250).items():
assert verdict == NO_DATA, (name, verdict)
def broken(**values: Any) -> list[dict[str, Any]]:
return [base_row(**{**r, **values}) for r in everything]
assert h1(broken(cpu_worker_over="0.5")).verdict()[0] == REFUTED
assert (
h2(broken(duration_p50_ms="300", memory_peak_mib="1100")).verdict()[0]
== REFUTED
)
assert h3(broken(peak_broker_ready="60")).verdict()[0] == REFUTED
assert h4(broken(ready_over_p95_s="60")[:1]).verdict()[0] == REFUTED
assert h5(broken(retry_after="1"), 250).verdict()[0] == REFUTED
assert h6(broken(flags="lost")).verdict()[0] == REFUTED
assert h7(broken(ctrl_write_over_p95="500")).verdict()[0] == REFUTED
assert h7(broken(ctrl_read_over_s5xx="1")).verdict()[0] == REFUTED
assert h8(broken(attempts_over_one="5")).verdict()[0] == REFUTED
assert (
h9(broken(first_busy_s="50", first_backlog_s="10"), 250).verdict()[0]
== REFUTED
)
# рекомендация: (2, 2) и другая
assert recommend(matrix_fixture())["setting"] == (2, 2)
assert recommend(empty)["setting"] is None
# настройка, не снятая на части классов, не кандидат, даже если на снятых
# классах она лучшая: данных об остальных нет
partly = [
base_row(**{**r, "ready_s_recovery": "3"})
if r["concurrency"] == "1"
else r
for r in matrix_fixture()
]
advice = recommend(partly)
assert advice["setting"] == (2, 2), advice
assert advice["partial"]["1,1"] == ["lj", "s"], advice
assert (1, 1) not in advice["candidates"], advice
assert advice["partial"]["2,256"] == ["lj", "m"], advice
text = render_summary(
{"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}},
partly,
)
assert "(1, 1) снята не на всех классах (нет: lj, s)" in text
# снятая на всех классах и лучшая по правилу — другая рекомендация
other = [
*partly,
*[
base_row(
cell=f"{klass}-c1p1-r1",
config=f"{klass}-c1p1",
concurrency="1",
prefetch="1",
**{"class": klass},
)
for klass in ("s", "lj")
],
]
advice = recommend(other)
assert advice["setting"] == (1, 1), advice
assert "1,1" not in advice["partial"], advice
# сводка и графики собираются
text = render_summary(
{"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}},
everything,
)
assert "H9" in text and "Рекомендация" in text
pictures = render_plots(everything, [])
assert set(pictures) == {
"throughput-by-setting.svg",
"p95-by-setting.svg",
"memory-by-setting.svg",
"backlog-gate.svg",
}
print("overload selftest ok")
if __name__ == "__main__":
if "--selftest" in sys.argv:
selftest()
else:
print(__doc__)