Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e. Contracts, public tests and evidence of lab 5 are not carried over. The compose project, image and CI prefix are renamed to lab6; the first broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the OpenSpec changes of lab 6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2617 lines
90 KiB
Python
2617 lines
90 KiB
Python
"""Профиль очереди без docker (design change `track1-queue-overload`, решения
|
||
5–8, 11, 12): план кругами, разбор строк k6, сэмплера и журналов, строка ячейки и
|
||
посекундная шкала, пометки, вердикты H1–H9, правило рекомендации, сводка и
|
||
графики. Только stdlib; `--selftest`. Имя не `queue.py`: каталог скрипта первый в
|
||
`sys.path`, и `queue` затенил бы модуль stdlib.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import json
|
||
import random
|
||
import sys
|
||
from collections.abc import Iterable, Mapping
|
||
from dataclasses import dataclass, field, replace
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import plots
|
||
import reconcile
|
||
|
||
WINDOWS = ("before", "over", "after")
|
||
CONTROL_WINDOWS = ("before", "over", "recovery", "after")
|
||
CONTROL_KINDS = {"w": "write", "r": "read", "d": "download"}
|
||
CONTROL_STATS = ("n", "errors", "s5xx", "p50", "p95", "p99")
|
||
ROLES = (
|
||
"worker",
|
||
"s3",
|
||
"postgres",
|
||
"api",
|
||
"relay",
|
||
"rabbitmq",
|
||
"cleaner",
|
||
"haproxy",
|
||
)
|
||
# условие H6, записанное до метки: состав не меняется
|
||
H6_FLAGS = ("lost", "stuck", "duplicate", "sha_mismatch")
|
||
# пометки, при которых метка не зелёная: система потеряла работу (решение 11) или
|
||
# в базе есть вложение, на которое никто не отвечал (verify S4)
|
||
FATAL_FLAGS = (*H6_FLAGS, "unexpected")
|
||
DEFAULT_SETTING = (2, 2)
|
||
# `oom` и `die` одного завершения: на пробе ~50 мс, запас на нагруженную ВМ
|
||
OOM_DIE_WINDOW_S = 2.0
|
||
MIB = 2**20
|
||
CONFIRMED = "подтверждено"
|
||
REFUTED = "нет"
|
||
NO_DATA = "нет данных"
|
||
BACKLOG_RETRY_AFTER = "5"
|
||
# запас превышения предела: загрузки в полёте двух реплик (решение 3)
|
||
IN_FLIGHT = 40
|
||
LEASE_S = 70
|
||
# окно CPU S3 блока accept: до срабатывания предела (гипотеза H9)
|
||
ACCEPT_S3_WINDOW_S = 30
|
||
|
||
CELL_FIELDS = [
|
||
"cell",
|
||
"config",
|
||
"block",
|
||
"class",
|
||
"concurrency",
|
||
"prefetch",
|
||
"repeat",
|
||
"order",
|
||
"rate_low",
|
||
"rate_high",
|
||
"over_s",
|
||
"limit",
|
||
"action",
|
||
"accepted",
|
||
"s202",
|
||
"s429",
|
||
"s5xx",
|
||
"s_other",
|
||
"s_conn",
|
||
"dropped",
|
||
"accepted_before",
|
||
"accepted_over",
|
||
"accepted_after",
|
||
"retry_after",
|
||
"busy_429",
|
||
"backlog_429",
|
||
"first_busy_s",
|
||
"first_backlog_s",
|
||
"upload_before_p50_ms",
|
||
"upload_over_p50_ms",
|
||
"upload_over_p95_ms",
|
||
"ready",
|
||
"failed",
|
||
"lost",
|
||
"stuck",
|
||
"duplicate",
|
||
"sha_checked",
|
||
"sha_bad",
|
||
"ready_s_over",
|
||
"ready_s_recovery",
|
||
"over_end",
|
||
"zero_at",
|
||
"action_at",
|
||
"recovery_s",
|
||
"peak_unfinished",
|
||
"peak_outbox",
|
||
"peak_waiting",
|
||
"peak_published",
|
||
"peak_processing",
|
||
"peak_broker_ready",
|
||
"peak_broker_unacked",
|
||
"peak_prefetched",
|
||
"peak_oldest_s",
|
||
"staging_ready_end",
|
||
"queue_left",
|
||
"ready_before_p50_s",
|
||
"ready_before_p95_s",
|
||
"ready_before_p99_s",
|
||
"ready_over_p50_s",
|
||
"ready_over_p95_s",
|
||
"ready_over_p99_s",
|
||
"ready_after_p50_s",
|
||
"ready_after_p95_s",
|
||
"ready_after_p99_s",
|
||
"ready_before_list",
|
||
"ready_after_list",
|
||
"queue_wait_p50_ms",
|
||
"queue_wait_p95_ms",
|
||
"duration_p50_ms",
|
||
"render_p50_ms",
|
||
"worker_hwm_mib",
|
||
"render_peak_p50_mib",
|
||
"render_peak_max_mib",
|
||
"memory_peak_mib",
|
||
"anon_max_mib",
|
||
"oom_kill",
|
||
"worker_restarts",
|
||
*[f"cpu_{role}_over" for role in ROLES],
|
||
*[f"cpu_{role}_recovery" for role in ROLES],
|
||
*[f"throttle_{role}_over" for role in ("worker", "s3")],
|
||
"cpu_s3_accept",
|
||
"cpu_worker_per_job_s",
|
||
"broker_memory_mib",
|
||
"broker_refused",
|
||
"attempts_over_one",
|
||
"skipped",
|
||
"relay_published",
|
||
"relay_republished",
|
||
"relay_recovered",
|
||
"relay_backlog_full",
|
||
"failed_by",
|
||
"second_jobs",
|
||
"second_share",
|
||
*[
|
||
f"ctrl_{kind}_{window}_{stat}"
|
||
for kind in CONTROL_KINDS.values()
|
||
for window in CONTROL_WINDOWS
|
||
for stat in CONTROL_STATS
|
||
],
|
||
"swapouts",
|
||
"lb_down",
|
||
"flags",
|
||
# новые столбцы — только в конец (verify change track1-queue-overload)
|
||
*[f"queue_wait_{w}_{q}_ms" for w in WINDOWS for q in ("p50", "p95")],
|
||
*[f"peak_oldest_{w}_s" for w in CONTROL_WINDOWS],
|
||
"second_jobs_all",
|
||
"second_share_all",
|
||
"second_share_after",
|
||
*[f"cpu_{role}_{w}" for role in ROLES for w in ("before", "after")],
|
||
*[
|
||
f"throttle_{role}_{w}"
|
||
for role in ROLES
|
||
for w in CONTROL_WINDOWS
|
||
if not (role in ("worker", "s3") and w == "over")
|
||
],
|
||
"relay_restarts",
|
||
"relay_die",
|
||
"unexpected_rows",
|
||
"worker_die_oom",
|
||
"worker_die_command",
|
||
"worker_die_other",
|
||
"worker_exit_codes",
|
||
"worker_oom_events",
|
||
"lb_up_min",
|
||
]
|
||
TIMELINE_FIELDS = [
|
||
"cell",
|
||
"t",
|
||
"window",
|
||
"outbox",
|
||
"waiting",
|
||
"published",
|
||
"processing",
|
||
"unfinished",
|
||
"broker_ready",
|
||
"broker_unacked",
|
||
"prefetched",
|
||
"oldest_s",
|
||
"staging_ready",
|
||
"accepted",
|
||
"ready",
|
||
]
|
||
|
||
|
||
# ------------------------------------------------------------------ план
|
||
|
||
|
||
def config_key(
|
||
block: str,
|
||
klass: str,
|
||
concurrency: int,
|
||
prefetch: int,
|
||
) -> str:
|
||
setting = f"{klass}-c{concurrency}p{prefetch}"
|
||
return setting if block == "matrix" else f"{block}-{setting}"
|
||
|
||
|
||
def plan_cells(
|
||
configs: Iterable[Mapping[str, Any]],
|
||
repeats: int,
|
||
seed: int,
|
||
) -> list[dict[str, Any]]:
|
||
"""Круги: на круге — перестановка конфигураций от зерна; круг не начинается
|
||
с той, на которой кончился прошлый (как `scaling.plan_cells`)."""
|
||
items = list(configs)
|
||
plan: list[dict[str, Any]] = []
|
||
previous = None
|
||
for round_no in range(1, repeats + 1):
|
||
order = list(range(len(items)))
|
||
random.Random(seed + round_no).shuffle(order)
|
||
if len(order) > 1 and order[0] == previous:
|
||
order[0], order[1] = order[1], order[0]
|
||
for index in order:
|
||
item = dict(items[index])
|
||
key = config_key(
|
||
item["block"],
|
||
item["class"],
|
||
item["concurrency"],
|
||
item["prefetch"],
|
||
)
|
||
plan.append(
|
||
{
|
||
**item,
|
||
"cell": f"{key}-r{round_no}",
|
||
"config": key,
|
||
"repeat": round_no,
|
||
"order": len(plan) + 1,
|
||
},
|
||
)
|
||
previous = order[-1] if order else None
|
||
return plan
|
||
|
||
|
||
# --------------------------------------------------------- разбор источников
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Upload:
|
||
window: str
|
||
start_ms: int
|
||
duration_ms: int
|
||
status: int
|
||
retry_after: str
|
||
instance: str
|
||
attachment_id: str
|
||
|
||
@property
|
||
def answered(self) -> float:
|
||
return (self.start_ms + self.duration_ms) / 1000
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class JobTick:
|
||
t: float
|
||
outbox: int
|
||
waiting: int
|
||
published: int
|
||
processing: int
|
||
oldest_s: float
|
||
staging_ready: int
|
||
|
||
@property
|
||
def unfinished(self) -> int:
|
||
return self.outbox + self.waiting + self.published + self.processing
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Meta:
|
||
http: int
|
||
status: str
|
||
error_code: str
|
||
variants: int
|
||
updated_at: float | None
|
||
|
||
|
||
@dataclass
|
||
class Sources:
|
||
"""Всё снятое за ячейку; сюда же — готовые разборы журналов."""
|
||
|
||
uploads: list[Upload]
|
||
t0: float
|
||
metas: dict[str, Meta]
|
||
sha_checks: list[bool]
|
||
jobs: list[JobTick]
|
||
mq: list[tuple[float, int, int]]
|
||
mq_missing: int
|
||
cg: dict[str, list[list[tuple[int, ...]]]]
|
||
mp: dict[str, list[list[tuple[int, ...]]]]
|
||
control: list[Any]
|
||
worker_records: list[dict[str, Any]]
|
||
second_records: list[dict[str, Any]]
|
||
relay_records: list[dict[str, Any]]
|
||
api_records: list[dict[str, Any]]
|
||
sql: dict[str, Any]
|
||
hwm_kib: int | None = None
|
||
restarts: int = 0
|
||
dropped: int = 0
|
||
action_at: float | None = None
|
||
swapouts: int | None = None
|
||
lb_down: bool = False
|
||
storage_down: bool = False
|
||
extra: dict[str, Any] = field(default_factory=dict)
|
||
docker_events: list[DockerEvent] = field(default_factory=list)
|
||
worker_name: str = "worker"
|
||
relay_name: str = "relay"
|
||
relay_restarts: int = 0
|
||
lb_up_min: int | None = None
|
||
# все вложения стенда и разогрев: сверка в обратную сторону (verify S4)
|
||
db_ids: set[str] | None = None
|
||
warm_ids: set[str] = field(default_factory=set)
|
||
|
||
|
||
def parse_uploads(text: str) -> tuple[float | None, list[Upload]]:
|
||
"""Строки `q-t0,<мс>` и `q,<окно>,<мс>,<мс>,<статус>,<RA>,<реплика>,<id>`."""
|
||
t0 = None
|
||
out = []
|
||
for raw in text.splitlines():
|
||
line = raw.strip()
|
||
if line.startswith("q-t0,"):
|
||
t0 = int(line.split(",")[1]) / 1000
|
||
elif line.startswith("q,"):
|
||
parts = line.split(",")
|
||
if len(parts) == 8:
|
||
out.append(
|
||
Upload(
|
||
parts[1],
|
||
int(parts[2]),
|
||
int(parts[3]),
|
||
int(parts[4]),
|
||
parts[5],
|
||
parts[6],
|
||
parts[7],
|
||
),
|
||
)
|
||
return t0, out
|
||
|
||
|
||
def parse_sweep(text: str) -> tuple[dict[str, Meta], list[bool]]:
|
||
metas: dict[str, Meta] = {}
|
||
checks: list[bool] = []
|
||
for raw in text.splitlines():
|
||
parts = raw.strip().split(",")
|
||
if parts[0] == "qs" and len(parts) == 7:
|
||
updated = (
|
||
None if parts[6] == "-" else reconcile.timestamp(parts[6])
|
||
)
|
||
metas[parts[1]] = Meta(
|
||
int(parts[2]),
|
||
parts[3],
|
||
parts[4],
|
||
int(parts[5]),
|
||
updated,
|
||
)
|
||
elif parts[0] == "qv" and len(parts) == 4:
|
||
checks.append(parts[3] == "1")
|
||
return metas, checks
|
||
|
||
|
||
def parse_jobs(text: str) -> list[JobTick]:
|
||
out = []
|
||
for line in text.splitlines():
|
||
parts = line.strip().split("|")
|
||
if len(parts) == 8 and parts[0] == "jobs":
|
||
out.append(
|
||
JobTick(
|
||
int(parts[1]) / 1000,
|
||
int(parts[2]),
|
||
int(parts[3]),
|
||
int(parts[4]),
|
||
int(parts[5]),
|
||
float(parts[6]),
|
||
int(parts[7]),
|
||
),
|
||
)
|
||
return out
|
||
|
||
|
||
def parse_mq(text: str) -> tuple[list[tuple[float, int, int]], int]:
|
||
"""Строки `mq`: значения и число `missing` (брокер не ответил)."""
|
||
out = []
|
||
missing = 0
|
||
for line in text.splitlines():
|
||
parts = line.split(" ")
|
||
if parts[0] != "mq":
|
||
continue
|
||
if len(parts) == 3 and parts[2] == "missing":
|
||
missing += 1
|
||
elif len(parts) == 4:
|
||
out.append(
|
||
(
|
||
int(parts[1]) / 1e9,
|
||
int(float(parts[2])),
|
||
int(float(parts[3])),
|
||
),
|
||
)
|
||
return out, missing
|
||
|
||
|
||
def parse_mp(text: str) -> dict[str, list[tuple[int, ...]]]:
|
||
"""Строки `mp <нс> <id12> <memory.peak> <anon> <oom_kill>` по контейнерам."""
|
||
out: dict[str, list[tuple[int, ...]]] = {}
|
||
for line in text.splitlines():
|
||
parts = line.split(" ")
|
||
if parts[0] == "mp" and len(parts) == 6:
|
||
out.setdefault(parts[2], []).append(
|
||
(int(parts[1]), *(int(v) for v in parts[3:6])),
|
||
)
|
||
return out
|
||
|
||
|
||
def split_lives(
|
||
points: list[tuple[int, ...]],
|
||
key: int = 1,
|
||
) -> list[list[tuple[int, ...]]]:
|
||
"""Жизни контейнера: перезапуск сбрасывает cgroup того же контейнера, и
|
||
счётчик `points[i][key]` падает (решение 10)."""
|
||
lives: list[list[tuple[int, ...]]] = []
|
||
for point in points:
|
||
if lives and point[key] >= lives[-1][-1][key]:
|
||
lives[-1].append(point)
|
||
else:
|
||
lives.append([point])
|
||
return lives
|
||
|
||
|
||
def lives_delta(
|
||
lives: Iterable[list[tuple[int, ...]]],
|
||
w0: float,
|
||
w1: float,
|
||
key: int = 1,
|
||
) -> float | None:
|
||
"""Разность счётчика за окно [w0, w1] (секунды) — сумма разностей внутри
|
||
жизней, на краях окна — линейно между соседними отсчётами: тик сэмплера не
|
||
совпадает с краем, и разность только по внутренним отсчётам занижала бы
|
||
короткое окно на ~10 %. Окно не задевает ни одну жизнь — `None`."""
|
||
|
||
def value_at(life: list[tuple[int, ...]], ns: float) -> float:
|
||
for prev, cur in zip(life, life[1:], strict=False):
|
||
if prev[0] <= ns <= cur[0]:
|
||
share = (ns - prev[0]) / ((cur[0] - prev[0]) or 1)
|
||
return prev[key] + (cur[key] - prev[key]) * share
|
||
return float(life[-1][key])
|
||
|
||
total = 0.0
|
||
seen = False
|
||
for life in lives:
|
||
if len(life) < 2:
|
||
continue
|
||
a, b = max(w0 * 1e9, life[0][0]), min(w1 * 1e9, life[-1][0])
|
||
if b > a:
|
||
total += value_at(life, b) - value_at(life, a)
|
||
seen = True
|
||
return total if seen else None
|
||
|
||
|
||
def role_cores(
|
||
lives_by_container: Iterable[list[list[tuple[int, ...]]]],
|
||
w0: float,
|
||
w1: float,
|
||
) -> float | None:
|
||
"""Ядер роли за окно: сумма по контейнерам, `usage_usec` — столбец 1."""
|
||
if w1 <= w0:
|
||
return None
|
||
total = None
|
||
for lives in lives_by_container:
|
||
delta = lives_delta(lives, w0, w1)
|
||
if delta is not None:
|
||
total = (total or 0.0) + delta
|
||
return None if total is None else round(total / 1e6 / (w1 - w0), 4)
|
||
|
||
|
||
def throttle_share(
|
||
lives: Iterable[list[tuple[int, ...]]],
|
||
w0: float,
|
||
w1: float,
|
||
) -> float | None:
|
||
"""Доля периодов с троттлингом: столбцы 2 (`nr_periods`), 3 (`nr_throttled`)."""
|
||
lives = list(lives)
|
||
periods = lives_delta(lives, w0, w1, key=2)
|
||
throttled = lives_delta(lives, w0, w1, key=3)
|
||
if not periods or throttled is None:
|
||
return None
|
||
return round(throttled / periods, 4)
|
||
|
||
|
||
def memory_by_lives(
|
||
lives: Iterable[list[tuple[int, ...]]],
|
||
) -> tuple[float | None, float | None, int]:
|
||
"""`memory.peak` и `anon` — наибольшее по жизням, `oom_kill` — сумма: у
|
||
каждой жизни считается наибольшее значение."""
|
||
peak = anon = None
|
||
kills = 0
|
||
for life in lives:
|
||
peak = max([p[1] for p in life] + ([peak] if peak is not None else []))
|
||
anon = max([p[2] for p in life] + ([anon] if anon is not None else []))
|
||
kills += max(p[3] for p in life)
|
||
return (
|
||
None if peak is None else round(peak / MIB, 1),
|
||
None if anon is None else round(anon / MIB, 1),
|
||
kills,
|
||
)
|
||
|
||
|
||
def lb_up_count(rows: Iterable[Mapping[str, str]]) -> int:
|
||
"""Реплик в балансировке в строках статистики одного тика LB: только слоты-
|
||
серверы, `BACKEND` и `FRONTEND` — не реплики. `UP 1/2` — проверка уже не
|
||
прошла, но реплика ещё получает запросы (verify C1)."""
|
||
return sum(
|
||
1
|
||
for row in rows
|
||
if row.get("svname") not in ("BACKEND", "FRONTEND")
|
||
and (row.get("status") or "").split(" ")[0] == "UP"
|
||
)
|
||
|
||
|
||
def lb_up_min(lb: Mapping[int, list[dict[str, str]]]) -> int | None:
|
||
"""Наименьшее число реплик в балансировке за все тики; тиков нет — `None`."""
|
||
return min((lb_up_count(rows) for rows in lb.values()), default=None)
|
||
|
||
|
||
def lb_down(lb: Mapping[int, list[dict[str, str]]], replicas: int) -> bool:
|
||
"""Реплика выходила из балансировки хотя бы на одном тике."""
|
||
least = lb_up_min(lb)
|
||
return least is not None and least < replicas
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class DockerEvent:
|
||
"""Событие `docker events` контейнера (часы ВМ)."""
|
||
|
||
t: float
|
||
action: str
|
||
name: str
|
||
exit_code: str
|
||
signal: str
|
||
|
||
|
||
def parse_docker_events(text: str) -> list[DockerEvent]:
|
||
"""Строки `<нс>|<действие>|<имя>|<код выхода>|<сигнал>`."""
|
||
out = []
|
||
for line in text.splitlines():
|
||
parts = line.strip().split("|")
|
||
if len(parts) == 5 and parts[0].isdigit():
|
||
out.append(
|
||
DockerEvent(int(parts[0]) / 1e9, *parts[1:]),
|
||
)
|
||
return sorted(out, key=lambda event: event.t)
|
||
|
||
|
||
def die_causes(events: Iterable[DockerEvent], name: str) -> dict[str, Any]:
|
||
"""Причины завершений контейнера (verify S5). Проба на Docker 29.9: ядро —
|
||
`oom`, затем `die` с кодом 137 в те же десятки мс; `docker kill` — `kill` с
|
||
сигналом, затем `die`; завершение процесса обработки — `oom` без `die`.
|
||
Поэтому `command` — был `kill` с прошлого `die`, `oom` — код 137 рядом с
|
||
`oom`, `other` — всё остальное, по кодам выхода."""
|
||
own = [event for event in events if event.name == name]
|
||
ooms = [event.t for event in own if event.action == "oom"]
|
||
causes: dict[str, Any] = {
|
||
"oom": 0,
|
||
"command": 0,
|
||
"other": 0,
|
||
"die": 0,
|
||
"codes": {},
|
||
"oom_events": len(ooms),
|
||
}
|
||
killed = False
|
||
for event in own:
|
||
if event.action == "kill":
|
||
killed = True
|
||
elif event.action == "die":
|
||
causes["die"] += 1
|
||
if killed:
|
||
causes["command"] += 1
|
||
elif event.exit_code == "137" and any(
|
||
abs(event.t - moment) <= OOM_DIE_WINDOW_S for moment in ooms
|
||
):
|
||
causes["oom"] += 1
|
||
else:
|
||
causes["other"] += 1
|
||
code = event.exit_code or "-"
|
||
causes["codes"][code] = causes["codes"].get(code, 0) + 1
|
||
killed = False
|
||
return causes
|
||
|
||
|
||
def events(
|
||
records: Iterable[Mapping[str, Any]],
|
||
name: str,
|
||
) -> list[Mapping[str, Any]]:
|
||
return [item for item in records if item.get("event") == name]
|
||
|
||
|
||
def rejected_by_reason(
|
||
records: Iterable[Mapping[str, Any]],
|
||
reason: str,
|
||
) -> list[float]:
|
||
"""Моменты отказов «Rejected attachment» с этой причиной, по времени."""
|
||
return sorted(
|
||
reconcile.timestamp(item["timestamp"])
|
||
for item in events(records, "Rejected attachment")
|
||
if item.get("reason") == reason and item.get("timestamp")
|
||
)
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 2) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def percentile(values: list[float], q: float, digits: int = 2) -> float | None:
|
||
return rounded(reconcile.percentile(values, q), digits)
|
||
|
||
|
||
# ---------------------------------------------------------------- ячейка
|
||
|
||
|
||
def control_columns(
|
||
requests: list[Any],
|
||
bounds: Mapping[str, tuple[float, float]],
|
||
) -> dict[str, Any]:
|
||
out: dict[str, Any] = {}
|
||
for kind, name in CONTROL_KINDS.items():
|
||
for window, (w0, w1) in bounds.items():
|
||
stats = reconcile.window_stats(requests, kind, w0, w1)
|
||
prefix = f"ctrl_{name}_{window}"
|
||
out[f"{prefix}_n"] = stats["responses"]
|
||
out[f"{prefix}_errors"] = stats["errors"]
|
||
out[f"{prefix}_s5xx"] = stats["s5xx"]
|
||
out[f"{prefix}_p50"] = rounded(stats["p50_ms"], 1)
|
||
out[f"{prefix}_p95"] = rounded(stats["p95_ms"], 1)
|
||
out[f"{prefix}_p99"] = rounded(stats["p99_ms"], 1)
|
||
return out
|
||
|
||
|
||
def cell_row(cell: Mapping[str, Any], src: Sources) -> dict[str, Any]:
|
||
"""Строка `cells.csv`. Окна — от начала `q-t0`: `before`, `over`, затем
|
||
`recovery` до нуля незавершённых."""
|
||
before_s = cell["before_s"]
|
||
over_s = cell["over_s"]
|
||
over_start = src.t0 + before_s
|
||
over_end = over_start + over_s
|
||
answers = src.uploads
|
||
in_over = [u for u in answers if u.window == "over"]
|
||
# последняя загрузка окна: её ответ — конец приёма
|
||
load_end = max((u.answered for u in in_over), default=over_end)
|
||
zero_at = next(
|
||
(t.t for t in src.jobs if t.t >= load_end and t.unfinished == 0),
|
||
None,
|
||
)
|
||
recovery_end = zero_at if zero_at is not None else load_end
|
||
accepted = [u for u in answers if u.status == 202]
|
||
statuses = [u.status for u in answers]
|
||
metas = src.metas
|
||
ready = [m for m in metas.values() if m.status == "ready"]
|
||
failed = [m for m in metas.values() if m.status == "failed"]
|
||
lost = sum(
|
||
1
|
||
for u in accepted
|
||
if u.attachment_id not in metas or metas[u.attachment_id].http != 200
|
||
)
|
||
stuck = sum(
|
||
1
|
||
for u in accepted
|
||
if u.attachment_id in metas
|
||
and metas[u.attachment_id].http == 200
|
||
and metas[u.attachment_id].status not in ("ready", "failed")
|
||
)
|
||
ready_at = [m.updated_at for m in ready if m.updated_at is not None]
|
||
|
||
def per_second(w0: float, w1: float) -> float | None:
|
||
if w1 <= w0:
|
||
return None
|
||
return round(sum(1 for t in ready_at if w0 <= t < w1) / (w1 - w0), 3)
|
||
|
||
waits: dict[str, list[float]] = {window: [] for window in WINDOWS}
|
||
for upload in accepted:
|
||
meta = metas.get(upload.attachment_id)
|
||
if meta and meta.status == "ready" and meta.updated_at is not None:
|
||
waits[upload.window].append(meta.updated_at - upload.answered)
|
||
|
||
jobs_over = [t for t in src.jobs if src.t0 <= t.t <= recovery_end]
|
||
peak = {
|
||
name: max((getattr(t, name) for t in jobs_over), default=None)
|
||
for name in (
|
||
"unfinished",
|
||
"outbox",
|
||
"waiting",
|
||
"published",
|
||
"processing",
|
||
"oldest_s",
|
||
)
|
||
}
|
||
mq_inside = [m for m in src.mq if src.t0 <= m[0] <= recovery_end]
|
||
# неподтверждённые против взятых: ближайший по времени тик заданий
|
||
prefetched = 0
|
||
for moment, _, unacked in mq_inside:
|
||
nearest = min(src.jobs, key=lambda t: abs(t.t - moment), default=None)
|
||
if nearest is not None:
|
||
prefetched = max(prefetched, unacked - nearest.processing)
|
||
|
||
worker_ready = events(src.worker_records, "Attachment became ready")
|
||
second_ready = events(src.second_records, "Attachment became ready")
|
||
both_ready = [*worker_ready, *second_ready]
|
||
started_jobs = events(
|
||
[*src.worker_records, *src.second_records],
|
||
"Started job attempt",
|
||
)
|
||
queue_waits = [
|
||
float(item["queue_wait_ms"])
|
||
for item in started_jobs
|
||
if item.get("queue_wait_ms") is not None
|
||
]
|
||
durations = [float(i["duration_ms"]) for i in both_ready]
|
||
renders = [float(i["render_ms"]) for i in both_ready if i.get("render_ms")]
|
||
render_peaks = [
|
||
float(i["render_peak_kib"]) / 1024
|
||
for i in both_ready
|
||
if i.get("render_peak_kib")
|
||
]
|
||
skipped: dict[str, int] = {}
|
||
for item in events(src.worker_records, "Skipped job message"):
|
||
reason = str(item.get("reason"))
|
||
skipped[reason] = skipped.get(reason, 0) + 1
|
||
relay_passes = events(src.relay_records, "Relay pass")
|
||
busy = rejected_by_reason(src.api_records, "busy")
|
||
backlog = rejected_by_reason(src.api_records, "backlog")
|
||
retry_values = sorted(
|
||
{u.retry_after for u in answers if u.status == 429},
|
||
)
|
||
|
||
worker_lives = src.cg.get("worker", [])
|
||
# только основной контейнер worker: лимит памяти у второго свой
|
||
peak_mem, anon_mem, kills = memory_by_lives(src.mp.get("worker", []))
|
||
# процесс 1, завершённый ядром, уносит cgroup раньше следующего тика: его
|
||
# счётчик не прочитать, поэтому — завершения, подтверждённые событиями Docker
|
||
causes = die_causes(src.docker_events, src.worker_name)
|
||
relay_causes = die_causes(src.docker_events, src.relay_name)
|
||
kills += causes["oom"]
|
||
per_job = None
|
||
delta = lives_delta(
|
||
[life for lives in worker_lives for life in lives],
|
||
src.t0,
|
||
recovery_end,
|
||
)
|
||
# CPU считается до конца восстановления, значит, и задания — до него: окно
|
||
# after в этот счёт не входит (verify W2)
|
||
ready_inside = sum(1 for moment in ready_at if moment <= recovery_end)
|
||
if delta is not None and ready_inside:
|
||
per_job = round(delta / 1e6 / ready_inside, 3)
|
||
|
||
after_start = src.extra.get("after_start")
|
||
after_end = src.extra.get("after_end")
|
||
bounds = {
|
||
"before": (src.t0, over_start),
|
||
"over": (over_start, over_end),
|
||
"recovery": (over_end, recovery_end),
|
||
"after": (after_start or recovery_end, after_end or recovery_end),
|
||
}
|
||
# queue lag по окнам: ожидание задания — по окну его прихода, возраст
|
||
# старейшего — по окну тика (решение 7, verify W1)
|
||
arrival = {u.attachment_id: u.window for u in accepted}
|
||
waits_by_window: dict[str, list[float]] = {w: [] for w in WINDOWS}
|
||
for item in started_jobs:
|
||
window = arrival.get(str(item.get("attachment_id")))
|
||
if window and item.get("queue_wait_ms") is not None:
|
||
waits_by_window[window].append(float(item["queue_wait_ms"]))
|
||
unexpected = None
|
||
if src.db_ids is not None:
|
||
unexpected = len(
|
||
src.db_ids - {u.attachment_id for u in accepted} - src.warm_ids,
|
||
)
|
||
|
||
row: dict[str, Any] = {
|
||
key: cell.get(key)
|
||
for key in (
|
||
"cell",
|
||
"config",
|
||
"block",
|
||
"class",
|
||
"concurrency",
|
||
"prefetch",
|
||
"repeat",
|
||
"order",
|
||
"rate_low",
|
||
"rate_high",
|
||
"over_s",
|
||
"limit",
|
||
"action",
|
||
)
|
||
}
|
||
row.update(
|
||
{
|
||
"accepted": len(accepted),
|
||
"s202": statuses.count(202),
|
||
"s429": statuses.count(429),
|
||
"s5xx": sum(1 for s in statuses if 500 <= s < 600),
|
||
"s_other": sum(
|
||
1
|
||
for s in statuses
|
||
if s not in (0, 202, 429) and not 500 <= s < 600
|
||
),
|
||
"s_conn": statuses.count(0),
|
||
"dropped": src.dropped,
|
||
"accepted_before": sum(
|
||
1 for u in accepted if u.window == "before"
|
||
),
|
||
"accepted_over": sum(1 for u in accepted if u.window == "over"),
|
||
"accepted_after": sum(1 for u in accepted if u.window == "after"),
|
||
"retry_after": " ".join(retry_values),
|
||
"busy_429": len(busy),
|
||
"backlog_429": len(backlog),
|
||
"first_busy_s": rounded(busy[0] - over_start) if busy else None,
|
||
"first_backlog_s": (
|
||
rounded(backlog[0] - over_start) if backlog else None
|
||
),
|
||
"upload_before_p50_ms": percentile(
|
||
[u.duration_ms for u in accepted if u.window == "before"],
|
||
50,
|
||
1,
|
||
),
|
||
"upload_over_p50_ms": percentile(
|
||
[u.duration_ms for u in accepted if u.window == "over"],
|
||
50,
|
||
1,
|
||
),
|
||
"upload_over_p95_ms": percentile(
|
||
[u.duration_ms for u in accepted if u.window == "over"],
|
||
95,
|
||
1,
|
||
),
|
||
"ready": len(ready),
|
||
"failed": len(failed),
|
||
"lost": lost,
|
||
"stuck": stuck,
|
||
"duplicate": src.sql.get("extra_jobs", 0)
|
||
+ src.sql.get("bad_variants", 0),
|
||
"sha_checked": len(src.sha_checks),
|
||
"sha_bad": src.sha_checks.count(False),
|
||
"ready_s_over": per_second(over_start, over_end),
|
||
"ready_s_recovery": per_second(over_end, recovery_end),
|
||
"over_end": round(over_end, 3),
|
||
"zero_at": rounded(zero_at, 3),
|
||
"action_at": rounded(src.action_at, 3),
|
||
"recovery_s": (
|
||
rounded(zero_at - load_end) if zero_at is not None else None
|
||
),
|
||
"peak_unfinished": peak["unfinished"],
|
||
"peak_outbox": peak["outbox"],
|
||
"peak_waiting": peak["waiting"],
|
||
"peak_published": peak["published"],
|
||
"peak_processing": peak["processing"],
|
||
"peak_broker_ready": max((m[1] for m in mq_inside), default=None),
|
||
"peak_broker_unacked": max(
|
||
(m[2] for m in mq_inside),
|
||
default=None,
|
||
),
|
||
"peak_prefetched": prefetched,
|
||
"peak_oldest_s": peak["oldest_s"],
|
||
"staging_ready_end": (
|
||
src.jobs[-1].staging_ready if src.jobs else None
|
||
),
|
||
"queue_left": sum(src.mq[-1][1:]) if src.mq else None,
|
||
"ready_before_list": " ".join(f"{v:.2f}" for v in waits["before"]),
|
||
"ready_after_list": " ".join(f"{v:.2f}" for v in waits["after"]),
|
||
"queue_wait_p50_ms": percentile(queue_waits, 50, 0),
|
||
"queue_wait_p95_ms": percentile(queue_waits, 95, 0),
|
||
"duration_p50_ms": percentile(durations, 50, 0),
|
||
"render_p50_ms": percentile(renders, 50, 0),
|
||
"worker_hwm_mib": (
|
||
round(src.hwm_kib / 1024, 1) if src.hwm_kib else None
|
||
),
|
||
"render_peak_p50_mib": percentile(render_peaks, 50, 1),
|
||
"render_peak_max_mib": rounded(
|
||
max(render_peaks, default=None),
|
||
1,
|
||
),
|
||
"memory_peak_mib": peak_mem,
|
||
"anon_max_mib": anon_mem,
|
||
"oom_kill": kills,
|
||
"worker_restarts": src.restarts,
|
||
"cpu_s3_accept": role_cores(
|
||
src.cg.get("s3", []),
|
||
over_start,
|
||
min(over_end, over_start + ACCEPT_S3_WINDOW_S),
|
||
),
|
||
"cpu_worker_per_job_s": per_job,
|
||
"broker_memory_mib": src.extra.get("broker_memory_mib"),
|
||
"broker_refused": src.extra.get("broker_refused", 0),
|
||
"attempts_over_one": src.sql.get("attempts_over_one"),
|
||
"skipped": json.dumps(skipped, sort_keys=True),
|
||
"relay_published": sum(
|
||
i.get("published", 0) for i in relay_passes
|
||
),
|
||
"relay_republished": sum(
|
||
i.get("republished", 0) for i in relay_passes
|
||
),
|
||
"relay_recovered": sum(
|
||
i.get("recovered", 0) for i in relay_passes
|
||
),
|
||
"relay_backlog_full": len(
|
||
events(src.relay_records, "Relay backlog is full"),
|
||
),
|
||
"failed_by": json.dumps(src.sql.get("failed", {}), sort_keys=True),
|
||
"second_jobs": len(second_ready) if src.second_records else None,
|
||
"swapouts": src.swapouts,
|
||
"lb_down": int(src.lb_down),
|
||
"relay_restarts": src.relay_restarts,
|
||
"relay_die": relay_causes["die"],
|
||
"unexpected_rows": unexpected,
|
||
"worker_die_oom": causes["oom"],
|
||
"worker_die_command": causes["command"],
|
||
"worker_die_other": causes["other"],
|
||
"worker_exit_codes": json.dumps(causes["codes"], sort_keys=True),
|
||
"worker_oom_events": causes["oom_events"],
|
||
"lb_up_min": src.lb_up_min,
|
||
},
|
||
)
|
||
for window in WINDOWS:
|
||
for q in (50, 95, 99):
|
||
row[f"ready_{window}_p{q}_s"] = percentile(waits[window], q)
|
||
for q in (50, 95):
|
||
row[f"queue_wait_{window}_p{q}_ms"] = percentile(
|
||
waits_by_window[window],
|
||
q,
|
||
0,
|
||
)
|
||
for window, (w0, w1) in bounds.items():
|
||
row[f"peak_oldest_{window}_s"] = max(
|
||
(t.oldest_s for t in src.jobs if w0 <= t.t < w1),
|
||
default=None,
|
||
)
|
||
for role in ROLES:
|
||
lives = src.cg.get(role, [])
|
||
flat = [life for each in lives for life in each]
|
||
row[f"cpu_{role}_over"] = role_cores(lives, over_start, over_end)
|
||
row[f"cpu_{role}_recovery"] = role_cores(lives, over_end, recovery_end)
|
||
for window in ("before", "after"):
|
||
row[f"cpu_{role}_{window}"] = role_cores(lives, *bounds[window])
|
||
for window, (w0, w1) in bounds.items():
|
||
row[f"throttle_{role}_{window}"] = (
|
||
throttle_share(flat, w0, w1) if w1 > w0 else None
|
||
)
|
||
action_at = src.extra.get("second_started_at")
|
||
if action_at is not None and src.second_records:
|
||
|
||
def done_at(item: Mapping[str, Any]) -> float:
|
||
return reconcile.timestamp(item["timestamp"])
|
||
|
||
def share(items: list[Mapping[str, Any]], total: int) -> float | None:
|
||
return round(len(items) / total, 3) if total else None
|
||
|
||
# столбец порога H3 — только разбор накопленного: окно after делится
|
||
# поровну при любом prefetch (design, «Уточнения apply», п. 2)
|
||
backlog_all = [
|
||
i for i in both_ready if action_at <= done_at(i) <= recovery_end
|
||
]
|
||
backlog_second = [
|
||
i for i in second_ready if action_at <= done_at(i) <= recovery_end
|
||
]
|
||
row["second_jobs"] = len(backlog_second)
|
||
row["second_share"] = share(backlog_second, len(backlog_all))
|
||
# буквальное прочтение порога и само окно after: числами (verify W3)
|
||
since_all = [i for i in both_ready if done_at(i) >= action_at]
|
||
since_second = [i for i in second_ready if done_at(i) >= action_at]
|
||
row["second_jobs_all"] = len(since_second)
|
||
row["second_share_all"] = share(since_second, len(since_all))
|
||
row["second_share_after"] = share(
|
||
[i for i in second_ready if done_at(i) > recovery_end],
|
||
sum(1 for i in both_ready if done_at(i) > recovery_end),
|
||
)
|
||
row.update(control_columns(src.control, bounds))
|
||
row["flags"] = " ".join(
|
||
cell_flags(row, zero_at is None, src.storage_down),
|
||
)
|
||
return row
|
||
|
||
|
||
def cell_flags(
|
||
row: Mapping[str, Any],
|
||
not_recovered: bool,
|
||
storage_down: bool,
|
||
) -> list[str]:
|
||
"""Пометки решения 11; `oom` — только подтверждённое завершение ядром, а любой
|
||
перезапуск — `worker_restart`, его причина — столбцы `worker_die_*`."""
|
||
flags = []
|
||
if num(row.get("oom_kill")) or num(row.get("worker_oom_events")):
|
||
flags.append("oom")
|
||
if num(row.get("worker_restarts")):
|
||
flags.append("worker_restart")
|
||
if num(row.get("worker_die_other")):
|
||
flags.append("worker_crash")
|
||
if (num(row.get("unexpected_rows")) or 0) > (
|
||
(num(row.get("s5xx")) or 0) + (num(row.get("s_conn")) or 0)
|
||
):
|
||
# строка после 5xx или обрыва законна: на неё клиент не получил 202
|
||
flags.append("unexpected")
|
||
if num(row.get("lost")):
|
||
flags.append("lost")
|
||
if num(row.get("stuck")):
|
||
flags.append("stuck")
|
||
if num(row.get("duplicate")):
|
||
flags.append("duplicate")
|
||
if num(row.get("sha_bad")):
|
||
flags.append("sha_mismatch")
|
||
if not_recovered:
|
||
flags.append("not_recovered")
|
||
if num(row.get("failed")):
|
||
flags.append("failed")
|
||
if num(row.get("dropped")):
|
||
flags.append("dropped")
|
||
if num(row.get("lb_down")):
|
||
flags.append("lb_down")
|
||
if num(row.get("swapouts")):
|
||
flags.append("host_swap")
|
||
if storage_down:
|
||
flags.append("storage_down")
|
||
return flags
|
||
|
||
|
||
def timeline_rows(
|
||
cell: Mapping[str, Any],
|
||
src: Sources,
|
||
) -> list[dict[str, Any]]:
|
||
"""Посекундная шкала от `q-t0`: окно, очередь в трёх местах, принято и
|
||
готово за секунду."""
|
||
over_start = src.t0 + cell["before_s"]
|
||
over_end = over_start + cell["over_s"]
|
||
after_start = src.extra.get("after_start")
|
||
accepted: dict[int, int] = {}
|
||
for upload in src.uploads:
|
||
if upload.status == 202:
|
||
second = int(upload.answered - src.t0)
|
||
accepted[second] = accepted.get(second, 0) + 1
|
||
ready: dict[int, int] = {}
|
||
for meta in src.metas.values():
|
||
if meta.status == "ready" and meta.updated_at is not None:
|
||
second = int(meta.updated_at - src.t0)
|
||
ready[second] = ready.get(second, 0) + 1
|
||
mq = {int(m[0] - src.t0): m for m in src.mq}
|
||
rows = []
|
||
for tick in src.jobs:
|
||
if tick.t < src.t0:
|
||
continue
|
||
second = int(tick.t - src.t0)
|
||
if tick.t < over_start:
|
||
window = "before"
|
||
elif tick.t < over_end:
|
||
window = "over"
|
||
elif after_start is None or tick.t < after_start:
|
||
window = "recovery"
|
||
else:
|
||
window = "after"
|
||
broker = mq.get(second)
|
||
rows.append(
|
||
{
|
||
"cell": cell["cell"],
|
||
"t": second,
|
||
"window": window,
|
||
"outbox": tick.outbox,
|
||
"waiting": tick.waiting,
|
||
"published": tick.published,
|
||
"processing": tick.processing,
|
||
"unfinished": tick.unfinished,
|
||
"broker_ready": broker[1] if broker else None,
|
||
"broker_unacked": broker[2] if broker else None,
|
||
"prefetched": (
|
||
max(0, broker[2] - tick.processing) if broker else None
|
||
),
|
||
"oldest_s": tick.oldest_s,
|
||
"staging_ready": tick.staging_ready,
|
||
"accepted": accepted.get(second, 0),
|
||
"ready": ready.get(second, 0),
|
||
},
|
||
)
|
||
return rows
|
||
|
||
|
||
# --------------------------------------------------------------- вердикты
|
||
|
||
|
||
def read_csv(path: Path) -> list[dict[str, Any]]:
|
||
if not path.exists():
|
||
return []
|
||
with path.open(encoding="utf-8", newline="") as file:
|
||
return list(csv.DictReader(file))
|
||
|
||
|
||
def num(value: Any) -> float | None:
|
||
return reconcile.num(value)
|
||
|
||
|
||
def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None:
|
||
return reconcile.med(row.get(key) for row in rows)
|
||
|
||
|
||
def select(
|
||
rows: Iterable[Mapping[str, Any]],
|
||
*,
|
||
block: str = "matrix",
|
||
klass: str | None = None,
|
||
setting: tuple[int, int] | None = None,
|
||
) -> list[Mapping[str, Any]]:
|
||
found = []
|
||
for row in rows:
|
||
if row.get("block") != block:
|
||
continue
|
||
if klass is not None and row.get("class") != klass:
|
||
continue
|
||
if setting is not None and (
|
||
num(row.get("concurrency")) != setting[0]
|
||
or num(row.get("prefetch")) != setting[1]
|
||
):
|
||
continue
|
||
found.append(row)
|
||
return found
|
||
|
||
|
||
def flagged(row: Mapping[str, Any], flag: str) -> bool:
|
||
return flag in (row.get("flags") or "").split()
|
||
|
||
|
||
def classes_of(rows: Iterable[Mapping[str, Any]]) -> list[str]:
|
||
return sorted({r["class"] for r in rows if r.get("block") == "matrix"})
|
||
|
||
|
||
def ratio(a: float | None, b: float | None) -> float | None:
|
||
return None if a is None or not b else a / b
|
||
|
||
|
||
class Check:
|
||
"""Условия гипотезы: `нет данных`, если нечем проверить условие; `нет` — если
|
||
условие нарушено, и причина названа."""
|
||
|
||
def __init__(self) -> None:
|
||
self.failed: list[str] = []
|
||
self.missing: list[str] = []
|
||
self.passed = 0
|
||
|
||
def within(
|
||
self,
|
||
name: str,
|
||
value: float | None,
|
||
low: float | None,
|
||
high: float | None,
|
||
) -> None:
|
||
if value is None:
|
||
self.missing.append(name)
|
||
elif (low is not None and value < low) or (
|
||
high is not None and value > high
|
||
):
|
||
self.failed.append(f"{name} = {value:.3g}")
|
||
else:
|
||
self.passed += 1
|
||
|
||
def true(self, name: str, value: bool | None) -> None:
|
||
if value is None:
|
||
self.missing.append(name)
|
||
elif not value:
|
||
self.failed.append(name)
|
||
else:
|
||
self.passed += 1
|
||
|
||
def verdict(self) -> tuple[str, str]:
|
||
if self.failed:
|
||
return REFUTED, "; ".join(self.failed[:6])
|
||
if self.missing:
|
||
return NO_DATA, "нет: " + ", ".join(self.missing[:6])
|
||
if not self.passed:
|
||
return NO_DATA, "нет ячеек"
|
||
return CONFIRMED, f"условий: {self.passed}"
|
||
|
||
|
||
def h1(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
for klass in classes_of(rows):
|
||
base = select(rows, klass=klass, setting=(2, 2))
|
||
wide = select(rows, klass=klass, setting=(4, 4))
|
||
check.within(
|
||
f"{klass}: (4,4)/(2,2) throughput",
|
||
ratio(
|
||
med(wide, "ready_s_recovery"),
|
||
med(base, "ready_s_recovery"),
|
||
),
|
||
None,
|
||
1.05,
|
||
)
|
||
for row in select(rows):
|
||
if (num(row.get("concurrency")) or 0) >= 2:
|
||
check.within(
|
||
f"{row['cell']}: CPU worker",
|
||
num(row.get("cpu_worker_over")),
|
||
0.85,
|
||
None,
|
||
)
|
||
for klass in ("m", "lp"):
|
||
base = select(rows, klass=klass, setting=(2, 2))
|
||
small = select(rows, klass=klass, setting=(1, 1))
|
||
check.within(
|
||
f"{klass}: (1,1)/(2,2) throughput",
|
||
ratio(
|
||
med(small, "ready_s_recovery"),
|
||
med(base, "ready_s_recovery"),
|
||
),
|
||
None,
|
||
1.0,
|
||
)
|
||
return check
|
||
|
||
|
||
def h2(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
limit = 1024.0
|
||
for klass in classes_of(rows):
|
||
base = select(rows, klass=klass, setting=(2, 2))
|
||
wide = select(rows, klass=klass, setting=(4, 4))
|
||
check.within(
|
||
f"{klass}: duration (4,4)/(2,2)",
|
||
ratio(med(wide, "duration_p50_ms"), med(base, "duration_p50_ms")),
|
||
1.5,
|
||
None,
|
||
)
|
||
for row in base:
|
||
check.within(
|
||
f"{row['cell']}: memory.peak",
|
||
num(row.get("memory_peak_mib")),
|
||
None,
|
||
limit - 0.001,
|
||
)
|
||
check.true(f"{row['cell']}: без oom", not flagged(row, "oom"))
|
||
wide_lp = select(rows, klass="lp", setting=(4, 4))
|
||
for row in wide_lp:
|
||
check.within(
|
||
f"{row['cell']}: memory.peak",
|
||
num(row.get("memory_peak_mib")),
|
||
0.95 * limit,
|
||
None,
|
||
)
|
||
check.true(
|
||
"lp (4,4): oom хотя бы в одном повторе",
|
||
any(flagged(r, "oom") for r in wide_lp) if wide_lp else None,
|
||
)
|
||
wide_lj = select(rows, klass="lj", setting=(4, 4))
|
||
for row in wide_lj:
|
||
check.true(f"{row['cell']}: без oom", not flagged(row, "oom"))
|
||
check.within(
|
||
f"{row['cell']}: memory.peak",
|
||
num(row.get("memory_peak_mib")),
|
||
None,
|
||
500,
|
||
)
|
||
if not wide_lj:
|
||
check.missing.append("lj (4,4)")
|
||
return check
|
||
|
||
|
||
def h3(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
for klass in ("s", "lp"):
|
||
base = select(rows, klass=klass, setting=(2, 2))
|
||
big = select(rows, klass=klass, setting=(2, 256))
|
||
check.within(
|
||
f"{klass}: throughput (2,256)/(2,2)",
|
||
ratio(med(big, "ready_s_recovery"), med(base, "ready_s_recovery")),
|
||
0.85,
|
||
1.15,
|
||
)
|
||
check.within(
|
||
f"{klass}: p95 (2,256)/(2,2)",
|
||
ratio(med(big, "ready_over_p95_s"), med(base, "ready_over_p95_s")),
|
||
0.8,
|
||
1.25,
|
||
)
|
||
for row in big:
|
||
check.within(
|
||
f"{row['cell']}: broker_ready / пик",
|
||
ratio(
|
||
num(row.get("peak_broker_ready")),
|
||
num(row.get("peak_unfinished")),
|
||
),
|
||
None,
|
||
0.10,
|
||
)
|
||
for row in base:
|
||
check.within(
|
||
f"{row['cell']}: broker_ready / пик",
|
||
ratio(
|
||
num(row.get("peak_broker_ready")),
|
||
num(row.get("peak_unfinished")),
|
||
),
|
||
0.70,
|
||
None,
|
||
)
|
||
p2 = select(rows, block="w2-p2")
|
||
p256 = select(rows, block="w2-p256")
|
||
check.within(
|
||
"доля второго при prefetch 2",
|
||
med(p2, "second_share"),
|
||
0.30,
|
||
None,
|
||
)
|
||
check.within(
|
||
"доля второго при prefetch 256",
|
||
med(p256, "second_share"),
|
||
None,
|
||
0.05,
|
||
)
|
||
check.within(
|
||
"восстановление p256 / p2",
|
||
ratio(med(p256, "recovery_s"), med(p2, "recovery_s")),
|
||
1.4,
|
||
None,
|
||
)
|
||
return check
|
||
|
||
|
||
def expected_wait(row: Mapping[str, Any]) -> float | None:
|
||
return ratio(num(row.get("peak_unfinished")), num(row.get("ready_s_over")))
|
||
|
||
|
||
def h4(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
for row in select(rows):
|
||
if flagged(row, "oom"):
|
||
continue
|
||
check.within(
|
||
f"{row['cell']}: p95 / (пик / throughput)",
|
||
ratio(num(row.get("ready_over_p95_s")), expected_wait(row)),
|
||
0.8,
|
||
1.25,
|
||
)
|
||
return check
|
||
|
||
|
||
def h5(rows: list[Mapping[str, Any]], limit: int) -> Check:
|
||
check = Check()
|
||
on = select(rows, block="gate-on")
|
||
off = select(rows, block="gate-off")
|
||
if not on or not off:
|
||
check.missing.append("блоки gate-on и gate-off")
|
||
return check
|
||
for row in on:
|
||
check.within(
|
||
f"{row['cell']}: пик",
|
||
num(row.get("peak_unfinished")),
|
||
None,
|
||
limit + IN_FLIGHT,
|
||
)
|
||
check.true(f"{row['cell']}: 429 есть", (num(row.get("s429")) or 0) > 0)
|
||
check.true(
|
||
f"{row['cell']}: Retry-After 5",
|
||
row.get("retry_after") == BACKLOG_RETRY_AFTER,
|
||
)
|
||
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
|
||
check.true(
|
||
f"{row['cell']}: 429 = записи backlog",
|
||
num(row.get("s429")) == num(row.get("backlog_429")),
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: p95 / (предел / throughput)",
|
||
ratio(
|
||
num(row.get("ready_over_p95_s")),
|
||
ratio(float(limit), num(row.get("ready_s_over"))),
|
||
),
|
||
0.8,
|
||
1.25,
|
||
)
|
||
for row in off:
|
||
check.within(
|
||
f"{row['cell']}: пик без предела",
|
||
num(row.get("peak_unfinished")),
|
||
1.5 * limit,
|
||
None,
|
||
)
|
||
check.within(
|
||
"p95 on / off",
|
||
ratio(med(on, "ready_over_p95_s"), med(off, "ready_over_p95_s")),
|
||
None,
|
||
0.75,
|
||
)
|
||
check.within(
|
||
"восстановление on / off",
|
||
ratio(med(on, "recovery_s"), med(off, "recovery_s")),
|
||
None,
|
||
0.75,
|
||
)
|
||
a = med(on, "upload_before_p50_ms")
|
||
b = med(off, "upload_before_p50_ms")
|
||
check.within(
|
||
"загрузка before p50 on − off, мс",
|
||
None if a is None or b is None else a - b,
|
||
None,
|
||
3,
|
||
)
|
||
return check
|
||
|
||
|
||
def pooled(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]:
|
||
values: list[float] = []
|
||
for row in rows:
|
||
values += [float(v) for v in (row.get(key) or "").split()]
|
||
return values
|
||
|
||
|
||
def h6(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
for row in rows:
|
||
for flag in H6_FLAGS:
|
||
check.true(f"{row['cell']}: нет {flag}", not flagged(row, flag))
|
||
if num(row.get("failed")):
|
||
only_exhausted = all(
|
||
key.startswith("retry_exhausted")
|
||
for key in json.loads(row.get("failed_by") or "{}")
|
||
)
|
||
check.true(
|
||
f"{row['cell']}: failed только при oom и retry_exhausted",
|
||
flagged(row, "oom") and only_exhausted,
|
||
)
|
||
if flagged(row, "oom") or row.get("block") not in (
|
||
"matrix",
|
||
"gate-on",
|
||
"gate-off",
|
||
):
|
||
continue
|
||
bound = expected_wait(row)
|
||
check.within(
|
||
f"{row['cell']}: восстановление / (пик / throughput)",
|
||
ratio(num(row.get("recovery_s")), bound),
|
||
None,
|
||
1.3,
|
||
)
|
||
by_config: dict[str, list[Mapping[str, Any]]] = {}
|
||
for row in rows:
|
||
by_config.setdefault(row["config"], []).append(row)
|
||
for config, items in by_config.items():
|
||
before = reconcile.percentile(pooled(items, "ready_before_list"), 95)
|
||
after = reconcile.percentile(pooled(items, "ready_after_list"), 95)
|
||
check.within(
|
||
f"{config}: p95 after / before",
|
||
ratio(after, before),
|
||
None,
|
||
1.5,
|
||
)
|
||
return check
|
||
|
||
|
||
def h7(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
for row in rows:
|
||
if row.get("block") == "accept":
|
||
continue
|
||
for window in CONTROL_WINDOWS:
|
||
for kind, limit in (
|
||
("write", 200),
|
||
("read", 200),
|
||
("download", 500),
|
||
):
|
||
value = num(row.get(f"ctrl_{kind}_{window}_p95"))
|
||
if value is not None:
|
||
check.within(
|
||
f"{row['cell']}: {kind} {window} p95",
|
||
value,
|
||
None,
|
||
limit,
|
||
)
|
||
check.true(
|
||
f"{row['cell']}: {kind} {window} 5xx нет",
|
||
not num(row.get(f"ctrl_{kind}_{window}_s5xx")),
|
||
)
|
||
check.true(
|
||
f"{row['cell']}: все реплики в LB",
|
||
not num(row.get("lb_down")),
|
||
)
|
||
return check
|
||
|
||
|
||
def h8(rows: list[Mapping[str, Any]]) -> Check:
|
||
check = Check()
|
||
kills = select(rows, block="kill-worker")
|
||
restarts = select(rows, block="restart-broker")
|
||
if not kills:
|
||
check.missing.append("блок kill-worker")
|
||
if not restarts:
|
||
check.missing.append("блок restart-broker")
|
||
for row in kills:
|
||
check.true(
|
||
f"{row['cell']}: всё ready",
|
||
num(row.get("ready")) == num(row.get("accepted")),
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: заданий с повтором",
|
||
num(row.get("attempts_over_one")),
|
||
None,
|
||
2,
|
||
)
|
||
end = num(row.get("over_end"))
|
||
action = num(row.get("action_at"))
|
||
wait = expected_wait(row)
|
||
zero = num(row.get("zero_at"))
|
||
if None in (end, action, wait, zero):
|
||
check.missing.append(f"{row['cell']}: моменты")
|
||
continue
|
||
check.within(
|
||
f"{row['cell']}: разбор до границы, с",
|
||
zero - max(end + 1.3 * wait, action + LEASE_S + 15),
|
||
None,
|
||
0,
|
||
)
|
||
for row in restarts:
|
||
check.true(
|
||
f"{row['cell']}: всё ready",
|
||
num(row.get("ready")) == num(row.get("accepted")),
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: заданий с повтором",
|
||
num(row.get("attempts_over_one")),
|
||
None,
|
||
0,
|
||
)
|
||
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
|
||
wait = expected_wait(row)
|
||
recovery = num(row.get("recovery_s"))
|
||
check.within(
|
||
f"{row['cell']}: восстановление, с",
|
||
None
|
||
if wait is None or recovery is None
|
||
else recovery - (1.3 * wait + 15),
|
||
None,
|
||
0,
|
||
)
|
||
return check
|
||
|
||
|
||
def h9(rows: list[Mapping[str, Any]], limit: int) -> Check:
|
||
check = Check()
|
||
cells = select(rows, block="accept")
|
||
if not cells:
|
||
check.missing.append("блок accept")
|
||
for row in cells:
|
||
busy = num(row.get("first_busy_s"))
|
||
backlog = num(row.get("first_backlog_s"))
|
||
check.true(
|
||
f"{row['cell']}: первый busy раньше backlog",
|
||
None if busy is None or backlog is None else busy < backlog,
|
||
)
|
||
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
|
||
check.within(
|
||
f"{row['cell']}: CPU S3 первые 30 с",
|
||
num(row.get("cpu_s3_accept")),
|
||
0.85,
|
||
None,
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: принято/с",
|
||
ratio(num(row.get("accepted_over")), num(row.get("over_s"))),
|
||
6,
|
||
None,
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: пик",
|
||
num(row.get("peak_unfinished")),
|
||
None,
|
||
limit + IN_FLIGHT,
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: готовых/с в окне",
|
||
num(row.get("ready_s_over")),
|
||
None,
|
||
1.5,
|
||
)
|
||
check.within(
|
||
f"{row['cell']}: готовых/с при восстановлении",
|
||
num(row.get("ready_s_recovery")),
|
||
2.4,
|
||
None,
|
||
)
|
||
for window in CONTROL_WINDOWS:
|
||
for kind, bound in (
|
||
("write", 200),
|
||
("read", 200),
|
||
("download", 1000),
|
||
):
|
||
value = num(row.get(f"ctrl_{kind}_{window}_p95"))
|
||
if value is not None:
|
||
check.within(
|
||
f"{row['cell']}: {kind} {window} p95",
|
||
value,
|
||
None,
|
||
bound,
|
||
)
|
||
check.true(
|
||
f"{row['cell']}: {kind} {window} 5xx нет",
|
||
not num(row.get(f"ctrl_{kind}_{window}_s5xx")),
|
||
)
|
||
return check
|
||
|
||
|
||
def verdicts(
|
||
rows: list[Mapping[str, Any]],
|
||
limit: int,
|
||
) -> dict[str, tuple[str, str]]:
|
||
return {
|
||
"H1": h1(rows).verdict(),
|
||
"H2": h2(rows).verdict(),
|
||
"H3": h3(rows).verdict(),
|
||
"H4": h4(rows).verdict(),
|
||
"H5": h5(rows, limit).verdict(),
|
||
"H6": h6(rows).verdict(),
|
||
"H7": h7(rows).verdict(),
|
||
"H8": h8(rows).verdict(),
|
||
"H9": h9(rows, limit).verdict(),
|
||
}
|
||
|
||
|
||
def settings_of(rows: Iterable[Mapping[str, Any]]) -> list[tuple[int, int]]:
|
||
return sorted(
|
||
{
|
||
(int(num(r["concurrency"])), int(num(r["prefetch"])))
|
||
for r in rows
|
||
if r.get("block") == "matrix"
|
||
},
|
||
)
|
||
|
||
|
||
def recommend(rows: list[Mapping[str, Any]]) -> dict[str, Any]:
|
||
"""Правило решения 12: кандидат — без `oom` и `failed` на всех классах, с
|
||
throughput не ниже 90 % лучшей на каждом классе; из кандидатов — наименьшая
|
||
конкуренция; `prefetch` равен конкуренции. Настройка, не снятая на части
|
||
классов, не кандидат: условие «на всех классах» для неё не проверено, она
|
||
названа в `partial` с этими классами (verify S6). В сравнении лучшей она
|
||
участвует: её снятые числа настоящие."""
|
||
settings = settings_of(rows)
|
||
candidates = []
|
||
reasons: dict[str, str] = {}
|
||
partial: dict[str, list[str]] = {}
|
||
for setting in settings:
|
||
problem = None
|
||
key = f"{setting[0]},{setting[1]}"
|
||
missing = [
|
||
klass
|
||
for klass in classes_of(rows)
|
||
if not select(rows, klass=klass, setting=setting)
|
||
]
|
||
if missing:
|
||
partial[key] = missing
|
||
for klass in classes_of(rows):
|
||
cells = select(rows, klass=klass, setting=setting)
|
||
if not cells:
|
||
continue
|
||
if any(flagged(r, "oom") or flagged(r, "failed") for r in cells):
|
||
problem = f"{klass}: oom или failed"
|
||
break
|
||
best = max(
|
||
(
|
||
value
|
||
for other in settings
|
||
if (
|
||
value := med(
|
||
select(rows, klass=klass, setting=other),
|
||
"ready_s_recovery",
|
||
)
|
||
)
|
||
is not None
|
||
),
|
||
default=None,
|
||
)
|
||
own = med(cells, "ready_s_recovery")
|
||
if best is None or own is None or own < 0.9 * best:
|
||
problem = f"{klass}: throughput ниже 90 % лучшей"
|
||
break
|
||
if problem:
|
||
reasons[key] = problem
|
||
elif not missing:
|
||
candidates.append(setting)
|
||
if not candidates:
|
||
return {
|
||
"setting": None,
|
||
"candidates": [],
|
||
"rejected": reasons,
|
||
"partial": partial,
|
||
}
|
||
concurrency = min(setting[0] for setting in candidates)
|
||
return {
|
||
"setting": (concurrency, concurrency),
|
||
"candidates": candidates,
|
||
"rejected": reasons,
|
||
"partial": partial,
|
||
}
|
||
|
||
|
||
# ----------------------------------------------------------------- сводка
|
||
|
||
|
||
def fmt(value: Any, digits: int = 1) -> str:
|
||
number = num(value)
|
||
return "—" if number is None else f"{number:.{digits}f}"
|
||
|
||
|
||
def spread(
|
||
rows: Iterable[Mapping[str, Any]],
|
||
key: str,
|
||
digits: int = 1,
|
||
) -> str:
|
||
return reconcile.spread((row.get(key) for row in rows), digits)
|
||
|
||
|
||
def config_table(rows: list[Mapping[str, Any]]) -> list[str]:
|
||
head = (
|
||
"| Конфигурация | Ячеек | Принято | 429 | Готовых/с: окно | восстановление | "
|
||
"Пик незаверш. | Пик брокера | До `ready` over p50 / p95 / p99, с | "
|
||
"Восст., с | `memory.peak`, МиБ | VmHWM, МиБ | Процесс p50 / макс., МиБ | "
|
||
"CPU worker, S3 | Троттлинг worker | OOM | Перезапуски | Пометки | "
|
||
"Queue lag: возраст старейшего, пик before / over / after, с | "
|
||
"Ожидание в очереди p50 / p95, мс: before; over; after | "
|
||
"Завершения worker: ядро / команда / прочие | Реплик в LB, минимум |"
|
||
)
|
||
lines = [head, "|" + "---|" * 22]
|
||
by_config: dict[str, list[Mapping[str, Any]]] = {}
|
||
for row in rows:
|
||
by_config.setdefault(row["config"], []).append(row)
|
||
for config in sorted(by_config):
|
||
items = by_config[config]
|
||
flags = sorted(
|
||
{f for r in items for f in (r.get("flags") or "").split()},
|
||
)
|
||
lines.append(
|
||
f"| `{config}` | {len(items)} | {spread(items, 'accepted', 0)} | "
|
||
f"{spread(items, 's429', 0)} | {spread(items, 'ready_s_over', 2)} | "
|
||
f"{spread(items, 'ready_s_recovery', 2)} | "
|
||
f"{spread(items, 'peak_unfinished', 0)} | "
|
||
f"{spread(items, 'peak_broker_ready', 0)} | "
|
||
f"{fmt(med(items, 'ready_over_p50_s'))} / "
|
||
f"{fmt(med(items, 'ready_over_p95_s'))} / "
|
||
f"{fmt(med(items, 'ready_over_p99_s'))} | "
|
||
f"{spread(items, 'recovery_s', 1)} | "
|
||
f"{spread(items, 'memory_peak_mib', 0)} | "
|
||
f"{fmt(med(items, 'worker_hwm_mib'), 0)} | "
|
||
f"{fmt(med(items, 'render_peak_p50_mib'), 0)} / "
|
||
f"{fmt(med(items, 'render_peak_max_mib'), 0)} | "
|
||
f"{fmt(med(items, 'cpu_worker_over'), 2)}, "
|
||
f"{fmt(med(items, 'cpu_s3_over'), 2)} | "
|
||
f"{fmt(med(items, 'throttle_worker_over'), 2)} | "
|
||
f"{sum(int(num(r.get('oom_kill')) or 0) for r in items)} | "
|
||
f"{sum(int(num(r.get('worker_restarts')) or 0) for r in items)} | "
|
||
f"{' '.join(flags) or '—'} | "
|
||
+ " / ".join(
|
||
fmt(med(items, f"peak_oldest_{w}_s")) for w in WINDOWS
|
||
)
|
||
+ " | "
|
||
+ "; ".join(
|
||
f"{fmt(med(items, f'queue_wait_{w}_p50_ms'), 0)} / "
|
||
f"{fmt(med(items, f'queue_wait_{w}_p95_ms'), 0)}"
|
||
for w in WINDOWS
|
||
)
|
||
+ " | "
|
||
+ " / ".join(
|
||
str(
|
||
sum(
|
||
int(num(r.get(f"worker_die_{k}")) or 0) for r in items
|
||
),
|
||
)
|
||
for k in ("oom", "command", "other")
|
||
)
|
||
+ " | "
|
||
+ fmt(
|
||
min(
|
||
(
|
||
value
|
||
for r in items
|
||
if (value := num(r.get("lb_up_min"))) is not None
|
||
),
|
||
default=None,
|
||
),
|
||
0,
|
||
)
|
||
+ " |",
|
||
)
|
||
return lines
|
||
|
||
|
||
def control_table(rows: list[Mapping[str, Any]]) -> list[str]:
|
||
lines = [
|
||
"| Конфигурация | Окно | Запись p50 / p95 / p99, мс | Чтение p50 / p95 / p99, "
|
||
"мс | Скачивание p50 / p95 / p99, мс | Ответов | 5xx |",
|
||
"|" + "---|" * 7,
|
||
]
|
||
by_config: dict[str, list[Mapping[str, Any]]] = {}
|
||
for row in rows:
|
||
by_config.setdefault(row["config"], []).append(row)
|
||
for config in sorted(by_config):
|
||
items = by_config[config]
|
||
for window in CONTROL_WINDOWS:
|
||
parts = []
|
||
total = errors = 0.0
|
||
for kind in CONTROL_KINDS.values():
|
||
prefix = f"ctrl_{kind}_{window}"
|
||
parts.append(
|
||
" / ".join(
|
||
fmt(med(items, f"{prefix}_{stat}"))
|
||
for stat in ("p50", "p95", "p99")
|
||
),
|
||
)
|
||
total += sum(num(r.get(f"{prefix}_n")) or 0 for r in items)
|
||
errors += sum(num(r.get(f"{prefix}_s5xx")) or 0 for r in items)
|
||
lines.append(
|
||
f"| `{config}` | {window} | {parts[0]} | {parts[1]} | {parts[2]} "
|
||
f"| {int(total)} | {int(errors)} |",
|
||
)
|
||
return lines
|
||
|
||
|
||
def block_table(rows: list[Mapping[str, Any]]) -> list[str]:
|
||
lines = [
|
||
"| Ячейка | Принято | 429 busy / backlog | Первый busy / backlog, с | Пик | "
|
||
"Готовых/с окно / восст. | Восст., с | Повторов | Второй worker: задания, доля "
|
||
"| Пометки | Второй: доля после запуска всего / в окне after | "
|
||
"Relay: перезапуски / завершения |",
|
||
"|" + "---|" * 12,
|
||
]
|
||
for row in sorted(rows, key=lambda r: r["cell"]):
|
||
if row.get("block") == "matrix":
|
||
continue
|
||
lines.append(
|
||
f"| `{row['cell']}` | {fmt(row.get('accepted'), 0)} | "
|
||
f"{fmt(row.get('busy_429'), 0)} / {fmt(row.get('backlog_429'), 0)} | "
|
||
f"{fmt(row.get('first_busy_s'))} / "
|
||
f"{fmt(row.get('first_backlog_s'))} | "
|
||
f"{fmt(row.get('peak_unfinished'), 0)} | "
|
||
f"{fmt(row.get('ready_s_over'), 2)} / "
|
||
f"{fmt(row.get('ready_s_recovery'), 2)} | "
|
||
f"{fmt(row.get('recovery_s'))} | "
|
||
f"{fmt(row.get('attempts_over_one'), 0)} | "
|
||
f"{fmt(row.get('second_jobs'), 0)}, "
|
||
f"{fmt(row.get('second_share'), 2)} | "
|
||
f"{row.get('flags') or '—'} | "
|
||
f"{fmt(row.get('second_share_all'), 3)} / "
|
||
f"{fmt(row.get('second_share_after'), 3)} | "
|
||
f"{fmt(row.get('relay_restarts'), 0)} / "
|
||
f"{fmt(row.get('relay_die'), 0)} |",
|
||
)
|
||
return lines
|
||
|
||
|
||
def render_summary(
|
||
meta: Mapping[str, Any],
|
||
rows: list[Mapping[str, Any]],
|
||
) -> str:
|
||
info = meta.get("queue", {})
|
||
limit = int(info.get("default_limit") or 250)
|
||
plan_total = len(info.get("plan", []))
|
||
lines = [
|
||
f"# Очередь под перегрузкой: метка {meta.get('label')}",
|
||
"",
|
||
f"commit `{meta.get('git', {}).get('commit')}`, "
|
||
f"dirty {meta.get('git', {}).get('dirty')}, статус "
|
||
f"{meta.get('status')}, ячеек {len(rows)} из {plan_total}.",
|
||
"",
|
||
]
|
||
fatal = [
|
||
f"`{row['cell']}`: {row['flags']}"
|
||
for row in rows
|
||
if any(flagged(row, flag) for flag in FATAL_FLAGS)
|
||
]
|
||
if fatal:
|
||
lines += [
|
||
"**Потеряна работа или лишние вложения:** " + "; ".join(fatal),
|
||
"",
|
||
]
|
||
lines += ["## Гипотезы", "", "| № | Вердикт | Детали |", "|---|---|---|"]
|
||
for name, (verdict, detail) in verdicts(rows, limit).items():
|
||
lines.append(f"| {name} | {verdict} | {detail} |")
|
||
advice = recommend(rows)
|
||
setting = advice["setting"]
|
||
lines += ["", "## Рекомендация", ""]
|
||
if setting is None:
|
||
lines.append("Кандидатов нет: правило не выбрало настройку.")
|
||
else:
|
||
keep = setting == DEFAULT_SETTING
|
||
lines.append(
|
||
f"Конкуренция {setting[0]}, prefetch {setting[1]}: "
|
||
+ (
|
||
"значения по умолчанию оставить."
|
||
if keep
|
||
else "отличается от (2, 2), решение — по правилу в отчёте."
|
||
),
|
||
)
|
||
for key, reason in advice["rejected"].items():
|
||
lines.append(f"- ({key.replace(',', ', ')}) не кандидат: {reason}")
|
||
for key, missing in advice["partial"].items():
|
||
lines.append(
|
||
f"- ({key.replace(',', ', ')}) снята не на всех классах (нет: "
|
||
f"{', '.join(missing)}): без данных не кандидат",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## Ячейки по конфигурациям",
|
||
"",
|
||
"Значения — медиана (min–max) повторов.",
|
||
"",
|
||
*config_table(rows),
|
||
"",
|
||
"## Блоки",
|
||
"",
|
||
*block_table(rows),
|
||
"",
|
||
"## Контрольный поток",
|
||
"",
|
||
*control_table(rows),
|
||
"",
|
||
]
|
||
return "\n".join(lines)
|
||
|
||
|
||
def setting_chart(
|
||
rows: list[Mapping[str, Any]],
|
||
key: str,
|
||
title: str,
|
||
unit: str,
|
||
) -> str:
|
||
"""Столбики: класс → настройки, медиана повторов."""
|
||
classes = classes_of(rows)
|
||
settings = settings_of(rows)
|
||
colors = ["#2a78d6", "#d98a00", "#3a9a5b", "#b0569c"]
|
||
width, height = 860, 420
|
||
svg = plots.Svg(width, height, title)
|
||
left, right, top, bottom = 70, 20, 50, 60
|
||
values = [
|
||
med(select(rows, klass=klass, setting=setting), key)
|
||
for klass in classes
|
||
for setting in settings
|
||
]
|
||
highest = max((v for v in values if v is not None), default=0.0) or 1.0
|
||
y_scale = (height - top - bottom) / (highest * 1.1)
|
||
group = (width - left - right) / max(1, len(classes))
|
||
bar = group / (len(settings) + 1)
|
||
base = height - bottom
|
||
svg.line(left, base, width - right, base, color=plots.AXIS)
|
||
svg.text(left, top - 14, unit, size=12)
|
||
for ci, klass in enumerate(classes):
|
||
x0 = left + ci * group
|
||
svg.text(x0 + group / 2 - 8, base + 22, klass, size=13)
|
||
for si, setting in enumerate(settings):
|
||
value = med(select(rows, klass=klass, setting=setting), key)
|
||
if value is None:
|
||
continue
|
||
bar_h = value * y_scale
|
||
svg.bar(
|
||
x0 + si * bar + bar / 2,
|
||
base - bar_h,
|
||
bar * 0.9,
|
||
bar_h,
|
||
colors[si % len(colors)],
|
||
f"{klass} ({setting[0]}, {setting[1]}): {value:.2f} {unit}",
|
||
top=True,
|
||
)
|
||
for si, setting in enumerate(settings):
|
||
x = left + si * 130
|
||
svg.bar(
|
||
x,
|
||
height - 28,
|
||
14,
|
||
14,
|
||
colors[si % len(colors)],
|
||
"",
|
||
top=False,
|
||
)
|
||
svg.text(x + 20, height - 16, f"({setting[0]}, {setting[1]})", size=12)
|
||
return svg.render()
|
||
|
||
|
||
def gate_chart(timeline: list[Mapping[str, Any]]) -> str:
|
||
"""Незавершённые посекундно: блоки предела, первый повтор."""
|
||
width, height = 860, 420
|
||
svg = plots.Svg(width, height, "Незавершённые задания: предел и без него")
|
||
left, right, top, bottom = 70, 20, 50, 50
|
||
series: dict[str, list[tuple[float, float]]] = {}
|
||
for row in timeline:
|
||
cell = row.get("cell", "")
|
||
if cell.startswith(("gate-on", "gate-off")) and cell.endswith("-r1"):
|
||
t = num(row.get("t"))
|
||
value = num(row.get("unfinished"))
|
||
if t is not None and value is not None:
|
||
series.setdefault(cell, []).append((t, value))
|
||
if not series:
|
||
svg.text(left, top + 30, "нет данных", size=13)
|
||
return svg.render()
|
||
max_t = max(p[0] for points in series.values() for p in points) or 1.0
|
||
max_v = max(p[1] for points in series.values() for p in points) or 1.0
|
||
colors = ["#2a78d6", "#d98a00"]
|
||
|
||
def sx(t: float) -> float:
|
||
return left + t / max_t * (width - left - right)
|
||
|
||
def sy(v: float) -> float:
|
||
return height - bottom - v / (max_v * 1.1) * (height - top - bottom)
|
||
|
||
svg.line(
|
||
left,
|
||
height - bottom,
|
||
width - right,
|
||
height - bottom,
|
||
color=plots.AXIS,
|
||
)
|
||
svg.text(
|
||
left,
|
||
top - 14,
|
||
f"до {max_v:.0f} заданий, до {max_t:.0f} с",
|
||
size=12,
|
||
)
|
||
for index, (cell, points) in enumerate(sorted(series.items())):
|
||
svg.polyline(
|
||
[(sx(t), sy(v)) for t, v in points],
|
||
colors[index % len(colors)],
|
||
)
|
||
svg.text(left + index * 260, height - 18, cell, size=12)
|
||
return svg.render()
|
||
|
||
|
||
def render_plots(
|
||
rows: list[Mapping[str, Any]],
|
||
timeline: list[Mapping[str, Any]],
|
||
) -> dict[str, str]:
|
||
return {
|
||
"throughput-by-setting.svg": setting_chart(
|
||
rows,
|
||
"ready_s_recovery",
|
||
"Throughput по настройкам и классам",
|
||
"готовых/с",
|
||
),
|
||
"p95-by-setting.svg": setting_chart(
|
||
rows,
|
||
"ready_over_p95_s",
|
||
"p95 времени до ready, окно over",
|
||
"с",
|
||
),
|
||
"memory-by-setting.svg": setting_chart(
|
||
rows,
|
||
"memory_peak_mib",
|
||
"memory.peak worker",
|
||
"МиБ",
|
||
),
|
||
"backlog-gate.svg": gate_chart(timeline),
|
||
}
|
||
|
||
|
||
# -------------------------------------------------------------- самопроверка
|
||
|
||
|
||
def base_row(**values: Any) -> dict[str, Any]:
|
||
row = {
|
||
"cell": "s-c2p2-r1",
|
||
"config": "s-c2p2",
|
||
"block": "matrix",
|
||
"class": "s",
|
||
"concurrency": "2",
|
||
"prefetch": "2",
|
||
"repeat": "1",
|
||
"accepted": "100",
|
||
"ready": "100",
|
||
"failed": "0",
|
||
"s429": "0",
|
||
"s5xx": "0",
|
||
"busy_429": "0",
|
||
"backlog_429": "0",
|
||
"ready_s_over": "3",
|
||
"ready_s_recovery": "3",
|
||
"peak_unfinished": "60",
|
||
"peak_broker_ready": "50",
|
||
"ready_over_p95_s": "20",
|
||
"recovery_s": "20",
|
||
"memory_peak_mib": "200",
|
||
"duration_p50_ms": "300",
|
||
"cpu_worker_over": "0.95",
|
||
"failed_by": "{}",
|
||
"ready_before_list": "1.0 1.1",
|
||
"ready_after_list": "1.0 1.2",
|
||
"attempts_over_one": "0",
|
||
"over_end": "100",
|
||
"zero_at": "120",
|
||
"action_at": "90",
|
||
"second_share": "",
|
||
"flags": "",
|
||
"lb_down": "0",
|
||
"retry_after": "",
|
||
}
|
||
for key, value in values.items():
|
||
row[key] = value
|
||
return row
|
||
|
||
|
||
def matrix_fixture() -> list[dict[str, Any]]:
|
||
rows = []
|
||
for klass in ("s", "m", "lj", "lp"):
|
||
rows.append(
|
||
base_row(
|
||
cell=f"{klass}-c2p2-r1",
|
||
config=f"{klass}-c2p2",
|
||
**{"class": klass},
|
||
),
|
||
)
|
||
wide = {
|
||
"duration_p50_ms": "600",
|
||
"memory_peak_mib": "400",
|
||
}
|
||
flags = ""
|
||
if klass == "lp":
|
||
wide["memory_peak_mib"] = "1024"
|
||
wide["oom_kill"] = "3"
|
||
flags = "oom"
|
||
rows.append(
|
||
base_row(
|
||
cell=f"{klass}-c4p4-r1",
|
||
config=f"{klass}-c4p4",
|
||
concurrency="4",
|
||
prefetch="4",
|
||
flags=flags,
|
||
**{"class": klass, **wide},
|
||
),
|
||
)
|
||
for klass in ("m", "lp"):
|
||
rows.append(
|
||
base_row(
|
||
cell=f"{klass}-c1p1-r1",
|
||
config=f"{klass}-c1p1",
|
||
concurrency="1",
|
||
prefetch="1",
|
||
ready_s_recovery="2.6",
|
||
cpu_worker_over="0.7",
|
||
**{"class": klass},
|
||
),
|
||
)
|
||
for klass in ("s", "lp"):
|
||
rows.append(
|
||
base_row(
|
||
cell=f"{klass}-c2p256-r1",
|
||
config=f"{klass}-c2p256",
|
||
prefetch="256",
|
||
peak_broker_ready="0",
|
||
**{"class": klass},
|
||
),
|
||
)
|
||
return rows
|
||
|
||
|
||
def gate_fixture() -> list[dict[str, Any]]:
|
||
on = base_row(
|
||
cell="gate-on-s-c2p2-r1",
|
||
config="gate-on-s-c2p2",
|
||
block="gate-on",
|
||
peak_unfinished="255",
|
||
s429="180",
|
||
backlog_429="180",
|
||
retry_after="5",
|
||
ready_over_p95_s="85",
|
||
recovery_s="85",
|
||
upload_before_p50_ms="15",
|
||
)
|
||
off = base_row(
|
||
cell="gate-off-s-c2p2-r1",
|
||
config="gate-off-s-c2p2",
|
||
block="gate-off",
|
||
peak_unfinished="430",
|
||
ready_over_p95_s="145",
|
||
recovery_s="145",
|
||
upload_before_p50_ms="14",
|
||
)
|
||
return [on, off]
|
||
|
||
|
||
def selftest() -> None:
|
||
# план: без повтора подряд на границе кругов, все ячейки
|
||
configs = [
|
||
{"block": "matrix", "class": k, "concurrency": 2, "prefetch": 2}
|
||
for k in ("s", "m", "lj")
|
||
]
|
||
plan = plan_cells(configs, 3, 7)
|
||
assert len(plan) == 9
|
||
for prev, cur in zip(plan, plan[1:], strict=False):
|
||
assert prev["config"] != cur["config"]
|
||
assert plan[0]["order"] == 1 and plan[-1]["order"] == 9
|
||
|
||
# разбор строк
|
||
t0, ups = parse_uploads(
|
||
"q-t0,1000000\n"
|
||
"q,before,1000000,10,202,-,a,id1\n"
|
||
"q,over,1010000,20,429,5,a,-\nбред\n",
|
||
)
|
||
assert t0 == 1000.0 and len(ups) == 2 and ups[1].retry_after == "5"
|
||
metas, checks = parse_sweep(
|
||
"qs,id1,200,ready,-,3,2026-11-01T10:00:00.123456Z\n"
|
||
"qs,id2,404,-,-,0,-\nqv,id1,original,1\nqv,id1,crop,0\n",
|
||
)
|
||
assert abs(metas["id1"].updated_at % 1 - 0.123456) < 1e-5
|
||
assert metas["id2"].updated_at is None and checks == [True, False]
|
||
jobs = parse_jobs("jobs|1000000|1|2|3|4|5.5|6\nсбой\n")
|
||
assert jobs[0].unfinished == 10 and jobs[0].oldest_s == 5.5
|
||
mq, missing = parse_mq("mq 1000000000 5 2\nmq 2000000000 missing\n")
|
||
assert mq == [(1.0, 5, 2)] and missing == 1
|
||
|
||
# жизни: перезапуск сбрасывает счётчики
|
||
points = [
|
||
(10**9, 100, 10, 1),
|
||
(2 * 10**9, 200, 20, 2),
|
||
(3 * 10**9, 50, 5, 0),
|
||
(4 * 10**9, 150, 15, 1),
|
||
]
|
||
lives = split_lives(points)
|
||
assert len(lives) == 2
|
||
assert lives_delta(lives, 0, 5) == 200
|
||
assert role_cores([lives], 1, 4) == round(200 / 1e6 / 3, 4)
|
||
mp = [
|
||
(10**9, 500 * MIB, 400 * MIB, 2),
|
||
(2 * 10**9, 700 * MIB, 600 * MIB, 3),
|
||
(3 * 10**9, 100 * MIB, 90 * MIB, 0),
|
||
(4 * 10**9, 300 * MIB, 200 * MIB, 1),
|
||
]
|
||
assert memory_by_lives(split_lives(mp, key=1)) == (700.0, 600.0, 4)
|
||
|
||
# LB: BACKEND и FRONTEND — не реплики, UP 1/2 — ещё в балансировке
|
||
both = [
|
||
{"svname": "api-1", "status": "UP"},
|
||
{"svname": "api-2", "status": "UP"},
|
||
{"svname": "api-3", "status": "MAINT"},
|
||
{"svname": "BACKEND", "status": "UP"},
|
||
]
|
||
one = [
|
||
{"svname": "api-1", "status": "UP"},
|
||
{"svname": "api-2", "status": "DOWN"},
|
||
{"svname": "BACKEND", "status": "UP"},
|
||
]
|
||
assert lb_up_count(both) == 2
|
||
assert lb_up_count(one) == 1, "реплика вышла, BACKEND посчитан как UP"
|
||
assert lb_up_min({1: both, 2: one}) == 1
|
||
assert lb_up_min({}) is None
|
||
assert not lb_down({1: both}, 2), "обе UP — реплика не выходила"
|
||
assert lb_down({1: both, 2: one}, 2), "одна UP и BACKEND UP — вышла"
|
||
assert not lb_down({}, 2)
|
||
assert not lb_down(
|
||
{1: [{"svname": "api-1", "status": "UP 1/2"}, *one[:1]]},
|
||
2,
|
||
), "UP 1/2 — ещё в балансировке"
|
||
|
||
# причины завершений: ядро, команда, прочее; процесс обработки — без die
|
||
happened = parse_docker_events(
|
||
"1000000000|oom|w|| \n"
|
||
"1000000000|oom|w||\n"
|
||
"9000000000|oom|w||\n"
|
||
"9050000000|die|w|137|\n"
|
||
"20000000000|kill|w||9\n"
|
||
"20050000000|die|w|137|\n"
|
||
"30000000000|die|w|3|\n"
|
||
"40000000000|die|w|137|\n"
|
||
"50000000000|die|other|137|\nбред\n",
|
||
)
|
||
found = die_causes(happened, "w")
|
||
assert found["oom"] == 1 and found["command"] == 1, found
|
||
assert found["other"] == 2 and found["die"] == 4, found
|
||
assert found["codes"] == {"3": 1, "137": 1}, found
|
||
assert found["oom_events"] == 3, found
|
||
|
||
# ячейка на границах окон: потерянное, зависшее, OOM, перезапуск
|
||
base = {
|
||
"cell": "s-c2p2-r1",
|
||
"config": "s-c2p2",
|
||
"block": "matrix",
|
||
"class": "s",
|
||
"concurrency": 2,
|
||
"prefetch": 2,
|
||
"repeat": 1,
|
||
"order": 1,
|
||
"before_s": 10,
|
||
"over_s": 20,
|
||
}
|
||
t = 1000.0
|
||
uploads_seen = [
|
||
Upload("before", int(t * 1000), 10, 202, "-", "a", "b1"),
|
||
Upload("over", int((t + 10) * 1000), 10, 202, "-", "a", "o1"),
|
||
Upload("over", int((t + 11) * 1000), 10, 202, "-", "a", "o2"),
|
||
Upload("over", int((t + 12) * 1000), 10, 202, "-", "a", "o3"),
|
||
Upload("over", int((t + 13) * 1000), 10, 429, "5", "a", "-"),
|
||
]
|
||
src = Sources(
|
||
uploads=uploads_seen,
|
||
t0=t,
|
||
metas={
|
||
"b1": Meta(200, "ready", "-", 3, t + 1),
|
||
# ровно на начале окна over: входит
|
||
"o1": Meta(200, "ready", "-", 3, t + 10),
|
||
"o2": Meta(200, "queued", "-", 0, None),
|
||
},
|
||
sha_checks=[True],
|
||
jobs=[
|
||
JobTick(t + 5, 0, 0, 0, 1, 0.1, 0),
|
||
JobTick(t + 20, 1, 1, 1, 2, 9.0, 1),
|
||
JobTick(t + 40, 0, 0, 0, 0, 0.0, 0),
|
||
],
|
||
mq=[(t + 20, 3, 2), (t + 40, 0, 0)],
|
||
mq_missing=0,
|
||
cg={
|
||
"worker": [
|
||
split_lives(
|
||
[(int((t + i) * 1e9), i * 10**6, i, 0) for i in range(41)],
|
||
),
|
||
],
|
||
},
|
||
mp={"worker": split_lives(mp)},
|
||
control=[],
|
||
worker_records=[],
|
||
second_records=[],
|
||
relay_records=[],
|
||
api_records=[],
|
||
sql={"extra_jobs": 0, "bad_variants": 0, "attempts_over_one": 1},
|
||
hwm_kib=2048,
|
||
restarts=1,
|
||
docker_events=[
|
||
DockerEvent(t + 15, "oom", "worker", "", ""),
|
||
DockerEvent(t + 15.05, "die", "worker", "137", ""),
|
||
DockerEvent(t + 16, "die", "relay", "1", ""),
|
||
],
|
||
relay_restarts=1,
|
||
lb_up_min=2,
|
||
db_ids={"b1", "o1", "o2", "o3", "warm"},
|
||
warm_ids={"warm"},
|
||
)
|
||
row = cell_row(base, src)
|
||
assert row["lost"] == 1 and row["stuck"] == 1
|
||
# счётчик cgroup 4 и завершение процесса 1, подтверждённое событиями
|
||
assert row["ready"] == 2 and row["oom_kill"] == 5
|
||
assert row["worker_die_oom"] == 1 and row["worker_die_other"] == 0
|
||
assert row["relay_restarts"] == 1 and row["relay_die"] == 1
|
||
assert row["unexpected_rows"] == 0 and "unexpected" not in row["flags"]
|
||
assert row["lb_up_min"] == 2
|
||
assert "lost" in row["flags"] and "stuck" in row["flags"]
|
||
|
||
# перезапуск не от ядра — не oom; лишняя строка без 5xx и обрыва — пометка
|
||
plain = replace(
|
||
src,
|
||
mp={},
|
||
docker_events=[DockerEvent(t + 15, "die", "worker", "3", "")],
|
||
db_ids={"b1", "o1", "o2", "o3", "warm", "чужое"},
|
||
)
|
||
crash = cell_row(base, plain)
|
||
assert crash["oom_kill"] == 0 and crash["worker_die_other"] == 1, crash
|
||
assert crash["worker_exit_codes"] == '{"3": 1}'
|
||
flags = crash["flags"].split()
|
||
assert "oom" not in flags and "worker_restart" in flags, flags
|
||
assert "worker_crash" in flags and "unexpected" in flags, flags
|
||
assert crash["unexpected_rows"] == 1
|
||
explained = replace(
|
||
plain,
|
||
uploads=[
|
||
*uploads_seen,
|
||
Upload("over", int((t + 14) * 1000), 10, 503, "-", "a", "-"),
|
||
],
|
||
)
|
||
assert "unexpected" not in cell_row(base, explained)["flags"].split()
|
||
|
||
# CPU до конца восстановления делится на задания до него: не на окно after
|
||
zero_end = replace(
|
||
src,
|
||
metas={
|
||
"b1": Meta(200, "ready", "-", 3, t + 1),
|
||
"o1": Meta(200, "ready", "-", 3, t + 20),
|
||
"o2": Meta(200, "ready", "-", 3, t + 30),
|
||
"o3": Meta(200, "ready", "-", 3, t + 39),
|
||
"a1": Meta(200, "ready", "-", 3, t + 45),
|
||
"a2": Meta(200, "ready", "-", 3, t + 46),
|
||
},
|
||
)
|
||
# от t + 0 до нуля незавершённых в t + 40: 40 с CPU по 1 ядру на 4 задания
|
||
assert cell_row(base, zero_end)["cpu_worker_per_job_s"] == 10.0
|
||
|
||
# queue lag по окнам: ожидание — по окну прихода, разогрева в счёте нет
|
||
laggy = replace(
|
||
src,
|
||
worker_records=[
|
||
{
|
||
"event": "Started job attempt",
|
||
"attachment_id": ident,
|
||
"queue_wait_ms": wait,
|
||
}
|
||
for ident, wait in (
|
||
("b1", 100),
|
||
("o1", 1000),
|
||
("o2", 3000),
|
||
("warm", 9000),
|
||
)
|
||
],
|
||
)
|
||
lag = cell_row(base, laggy)
|
||
assert lag["queue_wait_before_p50_ms"] == 100
|
||
assert lag["queue_wait_over_p50_ms"] == 1000
|
||
assert lag["queue_wait_over_p95_ms"] == 3000
|
||
assert lag["queue_wait_after_p50_ms"] is None
|
||
assert lag["queue_wait_p95_ms"] == 9000, "столбец на ячейку прежний"
|
||
assert lag["peak_oldest_before_s"] == 0.1
|
||
assert lag["peak_oldest_over_s"] == 9.0
|
||
assert set(CELL_FIELDS) >= set(lag), set(lag) - set(CELL_FIELDS)
|
||
assert len(CELL_FIELDS) == len(set(CELL_FIELDS))
|
||
|
||
# доля второго: порог H3 — только разбор, окно after — отдельным числом
|
||
done = [
|
||
{
|
||
"event": "Attachment became ready",
|
||
"timestamp": stamp,
|
||
"duration_ms": 300,
|
||
}
|
||
for stamp in (
|
||
"2026-11-01T10:00:10Z",
|
||
"2026-11-01T10:00:20Z",
|
||
"2026-11-01T10:00:50Z",
|
||
"2026-11-01T10:00:50Z",
|
||
)
|
||
]
|
||
other_done = done[3:]
|
||
done = done[:3]
|
||
second = reconcile.timestamp("2026-11-01T10:00:00Z")
|
||
shared = cell_row(
|
||
base,
|
||
replace(
|
||
src,
|
||
t0=second,
|
||
uploads=[
|
||
Upload(
|
||
"over",
|
||
int((second + 25) * 1000),
|
||
10,
|
||
202,
|
||
"-",
|
||
"a",
|
||
"o1",
|
||
),
|
||
],
|
||
jobs=[
|
||
JobTick(second + 28, 0, 0, 0, 1, 1.0, 0),
|
||
JobTick(second + 30, 0, 0, 0, 0, 0.0, 0),
|
||
],
|
||
worker_records=done,
|
||
second_records=other_done,
|
||
extra={"second_started_at": second + 5},
|
||
),
|
||
)
|
||
assert shared["second_jobs"] == 0 and shared["second_share"] == 0
|
||
assert shared["second_jobs_all"] == 1
|
||
assert shared["second_share_all"] == 0.25
|
||
assert shared["second_share_after"] == 0.5
|
||
# край окна — между тиками: CPU не занижен
|
||
assert row["cpu_worker_over"] == 1.0, row["cpu_worker_over"]
|
||
assert "oom" in row["flags"] and "worker_restart" in row["flags"]
|
||
assert row["ready_s_over"] == round(1 / 20, 3)
|
||
assert row["peak_unfinished"] == 5 and row["peak_prefetched"] == 0
|
||
assert row["recovery_s"] is not None and row["memory_peak_mib"] == 700.0
|
||
assert row["accepted_over"] == 3 and row["s429"] == 1
|
||
timeline = timeline_rows(base, src)
|
||
assert [r["window"] for r in timeline] == ["before", "over", "recovery"]
|
||
|
||
# вердикты: подтверждено, нет, нет данных
|
||
rows = matrix_fixture() + gate_fixture()
|
||
w2 = [
|
||
base_row(
|
||
cell="w2-p2-r1",
|
||
config="w2-p2",
|
||
block="w2-p2",
|
||
second_share="0.4",
|
||
recovery_s="20",
|
||
),
|
||
base_row(
|
||
cell="w2-p256-r1",
|
||
config="w2-p256",
|
||
block="w2-p256",
|
||
prefetch="256",
|
||
second_share="0",
|
||
recovery_s="30",
|
||
),
|
||
]
|
||
restarts = [
|
||
base_row(
|
||
cell="kill-r1",
|
||
config="kill",
|
||
block="kill-worker",
|
||
attempts_over_one="2",
|
||
),
|
||
base_row(cell="brk-r1", config="brk", block="restart-broker"),
|
||
]
|
||
accept = [
|
||
base_row(
|
||
cell="acc-r1",
|
||
config="acc",
|
||
block="accept",
|
||
first_busy_s="10",
|
||
first_backlog_s="35",
|
||
cpu_s3_accept="0.97",
|
||
accepted_over="300",
|
||
over_s="45",
|
||
peak_unfinished="280",
|
||
ready_s_over="0.6",
|
||
ready_s_recovery="2.8",
|
||
ctrl_download_over_p95="340",
|
||
ready_over_p95_s="93",
|
||
recovery_s="100",
|
||
),
|
||
]
|
||
everything = rows + w2 + restarts + accept
|
||
assert h1(everything).verdict()[0] == CONFIRMED, h1(everything).verdict()
|
||
assert h2(everything).verdict()[0] == CONFIRMED, h2(everything).verdict()
|
||
assert h3(everything).verdict()[0] == CONFIRMED, h3(everything).verdict()
|
||
assert h4(rows[:1]).verdict()[0] == CONFIRMED
|
||
assert h5(everything, 250).verdict()[0] == CONFIRMED, h5(
|
||
everything,
|
||
250,
|
||
).verdict()
|
||
assert h6(rows[:1]).verdict()[0] == CONFIRMED, h6(rows[:1]).verdict()
|
||
assert h7(everything).verdict()[0] == CONFIRMED
|
||
assert h8(everything).verdict()[0] == CONFIRMED, h8(everything).verdict()
|
||
assert h9(everything, 250).verdict()[0] == CONFIRMED, h9(
|
||
everything,
|
||
250,
|
||
).verdict()
|
||
|
||
empty: list[dict[str, Any]] = []
|
||
for name, (verdict, _) in verdicts(empty, 250).items():
|
||
assert verdict == NO_DATA, (name, verdict)
|
||
|
||
def broken(**values: Any) -> list[dict[str, Any]]:
|
||
return [base_row(**{**r, **values}) for r in everything]
|
||
|
||
assert h1(broken(cpu_worker_over="0.5")).verdict()[0] == REFUTED
|
||
assert (
|
||
h2(broken(duration_p50_ms="300", memory_peak_mib="1100")).verdict()[0]
|
||
== REFUTED
|
||
)
|
||
assert h3(broken(peak_broker_ready="60")).verdict()[0] == REFUTED
|
||
assert h4(broken(ready_over_p95_s="60")[:1]).verdict()[0] == REFUTED
|
||
assert h5(broken(retry_after="1"), 250).verdict()[0] == REFUTED
|
||
assert h6(broken(flags="lost")).verdict()[0] == REFUTED
|
||
assert h7(broken(ctrl_write_over_p95="500")).verdict()[0] == REFUTED
|
||
assert h7(broken(ctrl_read_over_s5xx="1")).verdict()[0] == REFUTED
|
||
assert h8(broken(attempts_over_one="5")).verdict()[0] == REFUTED
|
||
assert (
|
||
h9(broken(first_busy_s="50", first_backlog_s="10"), 250).verdict()[0]
|
||
== REFUTED
|
||
)
|
||
|
||
# рекомендация: (2, 2) и другая
|
||
assert recommend(matrix_fixture())["setting"] == (2, 2)
|
||
assert recommend(empty)["setting"] is None
|
||
# настройка, не снятая на части классов, не кандидат, даже если на снятых
|
||
# классах она лучшая: данных об остальных нет
|
||
partly = [
|
||
base_row(**{**r, "ready_s_recovery": "3"})
|
||
if r["concurrency"] == "1"
|
||
else r
|
||
for r in matrix_fixture()
|
||
]
|
||
advice = recommend(partly)
|
||
assert advice["setting"] == (2, 2), advice
|
||
assert advice["partial"]["1,1"] == ["lj", "s"], advice
|
||
assert (1, 1) not in advice["candidates"], advice
|
||
assert advice["partial"]["2,256"] == ["lj", "m"], advice
|
||
text = render_summary(
|
||
{"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}},
|
||
partly,
|
||
)
|
||
assert "(1, 1) снята не на всех классах (нет: lj, s)" in text
|
||
# снятая на всех классах и лучшая по правилу — другая рекомендация
|
||
other = [
|
||
*partly,
|
||
*[
|
||
base_row(
|
||
cell=f"{klass}-c1p1-r1",
|
||
config=f"{klass}-c1p1",
|
||
concurrency="1",
|
||
prefetch="1",
|
||
**{"class": klass},
|
||
)
|
||
for klass in ("s", "lj")
|
||
],
|
||
]
|
||
advice = recommend(other)
|
||
assert advice["setting"] == (1, 1), advice
|
||
assert "1,1" not in advice["partial"], advice
|
||
|
||
# сводка и графики собираются
|
||
text = render_summary(
|
||
{"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}},
|
||
everything,
|
||
)
|
||
assert "H9" in text and "Рекомендация" in text
|
||
pictures = render_plots(everything, [])
|
||
assert set(pictures) == {
|
||
"throughput-by-setting.svg",
|
||
"p95-by-setting.svg",
|
||
"memory-by-setting.svg",
|
||
"backlog-gate.svg",
|
||
}
|
||
print("overload selftest ok")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if "--selftest" in sys.argv:
|
||
selftest()
|
||
else:
|
||
print(__doc__)
|