"""Профиль очереди без docker (design change `track1-queue-overload`, решения 5–8, 11, 12): план кругами, разбор строк k6, сэмплера и журналов, строка ячейки и посекундная шкала, пометки, вердикты H1–H9, правило рекомендации, сводка и графики. Только stdlib; `--selftest`. Имя не `queue.py`: каталог скрипта первый в `sys.path`, и `queue` затенил бы модуль stdlib. """ from __future__ import annotations import csv import json import random import sys from collections.abc import Iterable, Mapping from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any import plots import reconcile WINDOWS = ("before", "over", "after") CONTROL_WINDOWS = ("before", "over", "recovery", "after") CONTROL_KINDS = {"w": "write", "r": "read", "d": "download"} CONTROL_STATS = ("n", "errors", "s5xx", "p50", "p95", "p99") ROLES = ( "worker", "s3", "postgres", "api", "relay", "rabbitmq", "cleaner", "haproxy", ) # сервисы телеметрии стенда при выключателе TELEMETRY=on (design change # track1-telemetry-cost, решение 13) TELEMETRY_ROLES = ("otel-collector", "prometheus", "loki", "tempo") # условие H6, записанное до метки: состав не меняется H6_FLAGS = ("lost", "stuck", "duplicate", "sha_mismatch") # пометки, при которых метка не зелёная: система потеряла работу (решение 11) или # в базе есть вложение, на которое никто не отвечал (verify S4) FATAL_FLAGS = (*H6_FLAGS, "unexpected") DEFAULT_SETTING = (2, 2) # `oom` и `die` одного завершения: на пробе ~50 мс, запас на нагруженную ВМ OOM_DIE_WINDOW_S = 2.0 MIB = 2**20 CONFIRMED = "подтверждено" REFUTED = "нет" NO_DATA = "нет данных" BACKLOG_RETRY_AFTER = "5" # запас превышения предела: загрузки в полёте двух реплик (решение 3) IN_FLIGHT = 40 LEASE_S = 70 # окно CPU S3 блока accept: до срабатывания предела (гипотеза H9) ACCEPT_S3_WINDOW_S = 30 CELL_FIELDS = [ "cell", "config", "block", "class", "concurrency", "prefetch", "repeat", "order", "rate_low", "rate_high", "over_s", "limit", "action", "accepted", "s202", "s429", "s5xx", "s_other", "s_conn", "dropped", "accepted_before", "accepted_over", "accepted_after", "retry_after", "busy_429", "backlog_429", "first_busy_s", "first_backlog_s", "upload_before_p50_ms", "upload_over_p50_ms", "upload_over_p95_ms", "ready", "failed", "lost", "stuck", "duplicate", "sha_checked", "sha_bad", "ready_s_over", "ready_s_recovery", "over_end", "zero_at", "action_at", "recovery_s", "peak_unfinished", "peak_outbox", "peak_waiting", "peak_published", "peak_processing", "peak_broker_ready", "peak_broker_unacked", "peak_prefetched", "peak_oldest_s", "staging_ready_end", "queue_left", "ready_before_p50_s", "ready_before_p95_s", "ready_before_p99_s", "ready_over_p50_s", "ready_over_p95_s", "ready_over_p99_s", "ready_after_p50_s", "ready_after_p95_s", "ready_after_p99_s", "ready_before_list", "ready_after_list", "queue_wait_p50_ms", "queue_wait_p95_ms", "duration_p50_ms", "render_p50_ms", "worker_hwm_mib", "render_peak_p50_mib", "render_peak_max_mib", "memory_peak_mib", "anon_max_mib", "oom_kill", "worker_restarts", *[f"cpu_{role}_over" for role in ROLES], *[f"cpu_{role}_recovery" for role in ROLES], *[f"throttle_{role}_over" for role in ("worker", "s3")], "cpu_s3_accept", "cpu_worker_per_job_s", "broker_memory_mib", "broker_refused", "attempts_over_one", "skipped", "relay_published", "relay_republished", "relay_recovered", "relay_backlog_full", "failed_by", "second_jobs", "second_share", *[ f"ctrl_{kind}_{window}_{stat}" for kind in CONTROL_KINDS.values() for window in CONTROL_WINDOWS for stat in CONTROL_STATS ], "swapouts", "lb_down", "flags", # новые столбцы — только в конец (verify change track1-queue-overload) *[f"queue_wait_{w}_{q}_ms" for w in WINDOWS for q in ("p50", "p95")], *[f"peak_oldest_{w}_s" for w in CONTROL_WINDOWS], "second_jobs_all", "second_share_all", "second_share_after", *[f"cpu_{role}_{w}" for role in ROLES for w in ("before", "after")], *[ f"throttle_{role}_{w}" for role in ROLES for w in CONTROL_WINDOWS if not (role in ("worker", "s3") and w == "over") ], "relay_restarts", "relay_die", "unexpected_rows", "worker_die_oom", "worker_die_command", "worker_die_other", "worker_exit_codes", "worker_oom_events", "lb_up_min", # выключатель телеметрии (design change track1-telemetry-cost, решение # 13): CPU и троттлинг окна over и пик памяти сервисов телеметрии; без # телеметрии — пусто *[ f"tel_{role.replace('-', '_')}_{key}" for role in TELEMETRY_ROLES for key in ("cpu_over", "throttle_over", "peak_mib") ], ] TIMELINE_FIELDS = [ "cell", "t", "window", "outbox", "waiting", "published", "processing", "unfinished", "broker_ready", "broker_unacked", "prefetched", "oldest_s", "staging_ready", "accepted", "ready", ] # ------------------------------------------------------------------ план def config_key( block: str, klass: str, concurrency: int, prefetch: int, ) -> str: setting = f"{klass}-c{concurrency}p{prefetch}" return setting if block == "matrix" else f"{block}-{setting}" def plan_cells( configs: Iterable[Mapping[str, Any]], repeats: int, seed: int, ) -> list[dict[str, Any]]: """Круги: на круге — перестановка конфигураций от зерна; круг не начинается с той, на которой кончился прошлый (как `scaling.plan_cells`).""" items = list(configs) plan: list[dict[str, Any]] = [] previous = None for round_no in range(1, repeats + 1): order = list(range(len(items))) random.Random(seed + round_no).shuffle(order) if len(order) > 1 and order[0] == previous: order[0], order[1] = order[1], order[0] for index in order: item = dict(items[index]) key = config_key( item["block"], item["class"], item["concurrency"], item["prefetch"], ) plan.append( { **item, "cell": f"{key}-r{round_no}", "config": key, "repeat": round_no, "order": len(plan) + 1, }, ) previous = order[-1] if order else None return plan # --------------------------------------------------------- разбор источников @dataclass(frozen=True) class Upload: window: str start_ms: int duration_ms: int status: int retry_after: str instance: str attachment_id: str @property def answered(self) -> float: return (self.start_ms + self.duration_ms) / 1000 @dataclass(frozen=True) class JobTick: t: float outbox: int waiting: int published: int processing: int oldest_s: float staging_ready: int @property def unfinished(self) -> int: return self.outbox + self.waiting + self.published + self.processing @dataclass(frozen=True) class Meta: http: int status: str error_code: str variants: int updated_at: float | None @dataclass class Sources: """Всё снятое за ячейку; сюда же — готовые разборы журналов.""" uploads: list[Upload] t0: float metas: dict[str, Meta] sha_checks: list[bool] jobs: list[JobTick] mq: list[tuple[float, int, int]] mq_missing: int cg: dict[str, list[list[tuple[int, ...]]]] mp: dict[str, list[list[tuple[int, ...]]]] control: list[Any] worker_records: list[dict[str, Any]] second_records: list[dict[str, Any]] relay_records: list[dict[str, Any]] api_records: list[dict[str, Any]] sql: dict[str, Any] hwm_kib: int | None = None restarts: int = 0 dropped: int = 0 action_at: float | None = None swapouts: int | None = None lb_down: bool = False storage_down: bool = False extra: dict[str, Any] = field(default_factory=dict) docker_events: list[DockerEvent] = field(default_factory=list) worker_name: str = "worker" relay_name: str = "relay" relay_restarts: int = 0 lb_up_min: int | None = None # все вложения стенда и разогрев: сверка в обратную сторону (verify S4) db_ids: set[str] | None = None warm_ids: set[str] = field(default_factory=set) def parse_uploads(text: str) -> tuple[float | None, list[Upload]]: """Строки `q-t0,<мс>` и `q,<окно>,<мс>,<мс>,<статус>,,<реплика>,`.""" t0 = None out = [] for raw in text.splitlines(): line = raw.strip() if line.startswith("q-t0,"): t0 = int(line.split(",")[1]) / 1000 elif line.startswith("q,"): parts = line.split(",") if len(parts) == 8: out.append( Upload( parts[1], int(parts[2]), int(parts[3]), int(parts[4]), parts[5], parts[6], parts[7], ), ) return t0, out def parse_sweep(text: str) -> tuple[dict[str, Meta], list[bool]]: metas: dict[str, Meta] = {} checks: list[bool] = [] for raw in text.splitlines(): parts = raw.strip().split(",") if parts[0] == "qs" and len(parts) == 7: updated = ( None if parts[6] == "-" else reconcile.timestamp(parts[6]) ) metas[parts[1]] = Meta( int(parts[2]), parts[3], parts[4], int(parts[5]), updated, ) elif parts[0] == "qv" and len(parts) == 4: checks.append(parts[3] == "1") return metas, checks def parse_jobs(text: str) -> list[JobTick]: out = [] for line in text.splitlines(): parts = line.strip().split("|") if len(parts) == 8 and parts[0] == "jobs": out.append( JobTick( int(parts[1]) / 1000, int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5]), float(parts[6]), int(parts[7]), ), ) return out def parse_mq(text: str) -> tuple[list[tuple[float, int, int]], int]: """Строки `mq`: значения и число `missing` (брокер не ответил).""" out = [] missing = 0 for line in text.splitlines(): parts = line.split(" ") if parts[0] != "mq": continue if len(parts) == 3 and parts[2] == "missing": missing += 1 elif len(parts) == 4: out.append( ( int(parts[1]) / 1e9, int(float(parts[2])), int(float(parts[3])), ), ) return out, missing def parse_mp(text: str) -> dict[str, list[tuple[int, ...]]]: """Строки `mp <нс> ` по контейнерам.""" out: dict[str, list[tuple[int, ...]]] = {} for line in text.splitlines(): parts = line.split(" ") if parts[0] == "mp" and len(parts) == 6: out.setdefault(parts[2], []).append( (int(parts[1]), *(int(v) for v in parts[3:6])), ) return out def split_lives( points: list[tuple[int, ...]], key: int = 1, ) -> list[list[tuple[int, ...]]]: """Жизни контейнера: перезапуск сбрасывает cgroup того же контейнера, и счётчик `points[i][key]` падает (решение 10).""" lives: list[list[tuple[int, ...]]] = [] for point in points: if lives and point[key] >= lives[-1][-1][key]: lives[-1].append(point) else: lives.append([point]) return lives def lives_delta( lives: Iterable[list[tuple[int, ...]]], w0: float, w1: float, key: int = 1, ) -> float | None: """Разность счётчика за окно [w0, w1] (секунды) — сумма разностей внутри жизней, на краях окна — линейно между соседними отсчётами: тик сэмплера не совпадает с краем, и разность только по внутренним отсчётам занижала бы короткое окно на ~10 %. Окно не задевает ни одну жизнь — `None`.""" def value_at(life: list[tuple[int, ...]], ns: float) -> float: for prev, cur in zip(life, life[1:], strict=False): if prev[0] <= ns <= cur[0]: share = (ns - prev[0]) / ((cur[0] - prev[0]) or 1) return prev[key] + (cur[key] - prev[key]) * share return float(life[-1][key]) total = 0.0 seen = False for life in lives: if len(life) < 2: continue a, b = max(w0 * 1e9, life[0][0]), min(w1 * 1e9, life[-1][0]) if b > a: total += value_at(life, b) - value_at(life, a) seen = True return total if seen else None def role_cores( lives_by_container: Iterable[list[list[tuple[int, ...]]]], w0: float, w1: float, ) -> float | None: """Ядер роли за окно: сумма по контейнерам, `usage_usec` — столбец 1.""" if w1 <= w0: return None total = None for lives in lives_by_container: delta = lives_delta(lives, w0, w1) if delta is not None: total = (total or 0.0) + delta return None if total is None else round(total / 1e6 / (w1 - w0), 4) def throttle_share( lives: Iterable[list[tuple[int, ...]]], w0: float, w1: float, ) -> float | None: """Доля периодов с троттлингом: столбцы 2 (`nr_periods`), 3 (`nr_throttled`).""" lives = list(lives) periods = lives_delta(lives, w0, w1, key=2) throttled = lives_delta(lives, w0, w1, key=3) if not periods or throttled is None: return None return round(throttled / periods, 4) def memory_by_lives( lives: Iterable[list[tuple[int, ...]]], ) -> tuple[float | None, float | None, int]: """`memory.peak` и `anon` — наибольшее по жизням, `oom_kill` — сумма: у каждой жизни считается наибольшее значение.""" peak = anon = None kills = 0 for life in lives: peak = max([p[1] for p in life] + ([peak] if peak is not None else [])) anon = max([p[2] for p in life] + ([anon] if anon is not None else [])) kills += max(p[3] for p in life) return ( None if peak is None else round(peak / MIB, 1), None if anon is None else round(anon / MIB, 1), kills, ) def lb_up_count(rows: Iterable[Mapping[str, str]]) -> int: """Реплик в балансировке в строках статистики одного тика LB: только слоты- серверы, `BACKEND` и `FRONTEND` — не реплики. `UP 1/2` — проверка уже не прошла, но реплика ещё получает запросы (verify C1).""" return sum( 1 for row in rows if row.get("svname") not in ("BACKEND", "FRONTEND") and (row.get("status") or "").split(" ")[0] == "UP" ) def lb_up_min(lb: Mapping[int, list[dict[str, str]]]) -> int | None: """Наименьшее число реплик в балансировке за все тики; тиков нет — `None`.""" return min((lb_up_count(rows) for rows in lb.values()), default=None) def lb_down(lb: Mapping[int, list[dict[str, str]]], replicas: int) -> bool: """Реплика выходила из балансировки хотя бы на одном тике.""" least = lb_up_min(lb) return least is not None and least < replicas @dataclass(frozen=True) class DockerEvent: """Событие `docker events` контейнера (часы ВМ).""" t: float action: str name: str exit_code: str signal: str def parse_docker_events(text: str) -> list[DockerEvent]: """Строки `<нс>|<действие>|<имя>|<код выхода>|<сигнал>`.""" out = [] for line in text.splitlines(): parts = line.strip().split("|") if len(parts) == 5 and parts[0].isdigit(): out.append( DockerEvent(int(parts[0]) / 1e9, *parts[1:]), ) return sorted(out, key=lambda event: event.t) def die_causes(events: Iterable[DockerEvent], name: str) -> dict[str, Any]: """Причины завершений контейнера (verify S5). Проба на Docker 29.9: ядро — `oom`, затем `die` с кодом 137 в те же десятки мс; `docker kill` — `kill` с сигналом, затем `die`; завершение процесса обработки — `oom` без `die`. Поэтому `command` — был `kill` с прошлого `die`, `oom` — код 137 рядом с `oom`, `other` — всё остальное, по кодам выхода.""" own = [event for event in events if event.name == name] ooms = [event.t for event in own if event.action == "oom"] causes: dict[str, Any] = { "oom": 0, "command": 0, "other": 0, "die": 0, "codes": {}, "oom_events": len(ooms), } killed = False for event in own: if event.action == "kill": killed = True elif event.action == "die": causes["die"] += 1 if killed: causes["command"] += 1 elif event.exit_code == "137" and any( abs(event.t - moment) <= OOM_DIE_WINDOW_S for moment in ooms ): causes["oom"] += 1 else: causes["other"] += 1 code = event.exit_code or "-" causes["codes"][code] = causes["codes"].get(code, 0) + 1 killed = False return causes def events( records: Iterable[Mapping[str, Any]], name: str, ) -> list[Mapping[str, Any]]: return [item for item in records if item.get("event") == name] def rejected_by_reason( records: Iterable[Mapping[str, Any]], reason: str, ) -> list[float]: """Моменты отказов «Rejected attachment» с этой причиной, по времени.""" return sorted( reconcile.timestamp(item["timestamp"]) for item in events(records, "Rejected attachment") if item.get("reason") == reason and item.get("timestamp") ) def rounded(value: float | None, digits: int = 2) -> float | None: return None if value is None else round(value, digits) def percentile(values: list[float], q: float, digits: int = 2) -> float | None: return rounded(reconcile.percentile(values, q), digits) # ---------------------------------------------------------------- ячейка def control_columns( requests: list[Any], bounds: Mapping[str, tuple[float, float]], ) -> dict[str, Any]: out: dict[str, Any] = {} for kind, name in CONTROL_KINDS.items(): for window, (w0, w1) in bounds.items(): stats = reconcile.window_stats(requests, kind, w0, w1) prefix = f"ctrl_{name}_{window}" out[f"{prefix}_n"] = stats["responses"] out[f"{prefix}_errors"] = stats["errors"] out[f"{prefix}_s5xx"] = stats["s5xx"] out[f"{prefix}_p50"] = rounded(stats["p50_ms"], 1) out[f"{prefix}_p95"] = rounded(stats["p95_ms"], 1) out[f"{prefix}_p99"] = rounded(stats["p99_ms"], 1) return out def cell_row(cell: Mapping[str, Any], src: Sources) -> dict[str, Any]: """Строка `cells.csv`. Окна — от начала `q-t0`: `before`, `over`, затем `recovery` до нуля незавершённых.""" before_s = cell["before_s"] over_s = cell["over_s"] over_start = src.t0 + before_s over_end = over_start + over_s answers = src.uploads in_over = [u for u in answers if u.window == "over"] # последняя загрузка окна: её ответ — конец приёма load_end = max((u.answered for u in in_over), default=over_end) zero_at = next( (t.t for t in src.jobs if t.t >= load_end and t.unfinished == 0), None, ) recovery_end = zero_at if zero_at is not None else load_end accepted = [u for u in answers if u.status == 202] statuses = [u.status for u in answers] metas = src.metas ready = [m for m in metas.values() if m.status == "ready"] failed = [m for m in metas.values() if m.status == "failed"] lost = sum( 1 for u in accepted if u.attachment_id not in metas or metas[u.attachment_id].http != 200 ) stuck = sum( 1 for u in accepted if u.attachment_id in metas and metas[u.attachment_id].http == 200 and metas[u.attachment_id].status not in ("ready", "failed") ) ready_at = [m.updated_at for m in ready if m.updated_at is not None] def per_second(w0: float, w1: float) -> float | None: if w1 <= w0: return None return round(sum(1 for t in ready_at if w0 <= t < w1) / (w1 - w0), 3) waits: dict[str, list[float]] = {window: [] for window in WINDOWS} for upload in accepted: meta = metas.get(upload.attachment_id) if meta and meta.status == "ready" and meta.updated_at is not None: waits[upload.window].append(meta.updated_at - upload.answered) jobs_over = [t for t in src.jobs if src.t0 <= t.t <= recovery_end] peak = { name: max((getattr(t, name) for t in jobs_over), default=None) for name in ( "unfinished", "outbox", "waiting", "published", "processing", "oldest_s", ) } mq_inside = [m for m in src.mq if src.t0 <= m[0] <= recovery_end] # неподтверждённые против взятых: ближайший по времени тик заданий prefetched = 0 for moment, _, unacked in mq_inside: nearest = min(src.jobs, key=lambda t: abs(t.t - moment), default=None) if nearest is not None: prefetched = max(prefetched, unacked - nearest.processing) worker_ready = events(src.worker_records, "Attachment became ready") second_ready = events(src.second_records, "Attachment became ready") both_ready = [*worker_ready, *second_ready] started_jobs = events( [*src.worker_records, *src.second_records], "Started job attempt", ) queue_waits = [ float(item["queue_wait_ms"]) for item in started_jobs if item.get("queue_wait_ms") is not None ] durations = [float(i["duration_ms"]) for i in both_ready] renders = [float(i["render_ms"]) for i in both_ready if i.get("render_ms")] render_peaks = [ float(i["render_peak_kib"]) / 1024 for i in both_ready if i.get("render_peak_kib") ] skipped: dict[str, int] = {} for item in events(src.worker_records, "Skipped job message"): reason = str(item.get("reason")) skipped[reason] = skipped.get(reason, 0) + 1 relay_passes = events(src.relay_records, "Relay pass") busy = rejected_by_reason(src.api_records, "busy") backlog = rejected_by_reason(src.api_records, "backlog") retry_values = sorted( {u.retry_after for u in answers if u.status == 429}, ) worker_lives = src.cg.get("worker", []) # только основной контейнер worker: лимит памяти у второго свой peak_mem, anon_mem, kills = memory_by_lives(src.mp.get("worker", [])) # процесс 1, завершённый ядром, уносит cgroup раньше следующего тика: его # счётчик не прочитать, поэтому — завершения, подтверждённые событиями Docker causes = die_causes(src.docker_events, src.worker_name) relay_causes = die_causes(src.docker_events, src.relay_name) kills += causes["oom"] per_job = None delta = lives_delta( [life for lives in worker_lives for life in lives], src.t0, recovery_end, ) # CPU считается до конца восстановления, значит, и задания — до него: окно # after в этот счёт не входит (verify W2) ready_inside = sum(1 for moment in ready_at if moment <= recovery_end) if delta is not None and ready_inside: per_job = round(delta / 1e6 / ready_inside, 3) after_start = src.extra.get("after_start") after_end = src.extra.get("after_end") bounds = { "before": (src.t0, over_start), "over": (over_start, over_end), "recovery": (over_end, recovery_end), "after": (after_start or recovery_end, after_end or recovery_end), } # queue lag по окнам: ожидание задания — по окну его прихода, возраст # старейшего — по окну тика (решение 7, verify W1) arrival = {u.attachment_id: u.window for u in accepted} waits_by_window: dict[str, list[float]] = {w: [] for w in WINDOWS} for item in started_jobs: window = arrival.get(str(item.get("attachment_id"))) if window and item.get("queue_wait_ms") is not None: waits_by_window[window].append(float(item["queue_wait_ms"])) unexpected = None if src.db_ids is not None: unexpected = len( src.db_ids - {u.attachment_id for u in accepted} - src.warm_ids, ) row: dict[str, Any] = { key: cell.get(key) for key in ( "cell", "config", "block", "class", "concurrency", "prefetch", "repeat", "order", "rate_low", "rate_high", "over_s", "limit", "action", ) } row.update( { "accepted": len(accepted), "s202": statuses.count(202), "s429": statuses.count(429), "s5xx": sum(1 for s in statuses if 500 <= s < 600), "s_other": sum( 1 for s in statuses if s not in (0, 202, 429) and not 500 <= s < 600 ), "s_conn": statuses.count(0), "dropped": src.dropped, "accepted_before": sum( 1 for u in accepted if u.window == "before" ), "accepted_over": sum(1 for u in accepted if u.window == "over"), "accepted_after": sum(1 for u in accepted if u.window == "after"), "retry_after": " ".join(retry_values), "busy_429": len(busy), "backlog_429": len(backlog), "first_busy_s": rounded(busy[0] - over_start) if busy else None, "first_backlog_s": ( rounded(backlog[0] - over_start) if backlog else None ), "upload_before_p50_ms": percentile( [u.duration_ms for u in accepted if u.window == "before"], 50, 1, ), "upload_over_p50_ms": percentile( [u.duration_ms for u in accepted if u.window == "over"], 50, 1, ), "upload_over_p95_ms": percentile( [u.duration_ms for u in accepted if u.window == "over"], 95, 1, ), "ready": len(ready), "failed": len(failed), "lost": lost, "stuck": stuck, "duplicate": src.sql.get("extra_jobs", 0) + src.sql.get("bad_variants", 0), "sha_checked": len(src.sha_checks), "sha_bad": src.sha_checks.count(False), "ready_s_over": per_second(over_start, over_end), "ready_s_recovery": per_second(over_end, recovery_end), "over_end": round(over_end, 3), "zero_at": rounded(zero_at, 3), "action_at": rounded(src.action_at, 3), "recovery_s": ( rounded(zero_at - load_end) if zero_at is not None else None ), "peak_unfinished": peak["unfinished"], "peak_outbox": peak["outbox"], "peak_waiting": peak["waiting"], "peak_published": peak["published"], "peak_processing": peak["processing"], "peak_broker_ready": max((m[1] for m in mq_inside), default=None), "peak_broker_unacked": max( (m[2] for m in mq_inside), default=None, ), "peak_prefetched": prefetched, "peak_oldest_s": peak["oldest_s"], "staging_ready_end": ( src.jobs[-1].staging_ready if src.jobs else None ), "queue_left": sum(src.mq[-1][1:]) if src.mq else None, "ready_before_list": " ".join(f"{v:.2f}" for v in waits["before"]), "ready_after_list": " ".join(f"{v:.2f}" for v in waits["after"]), "queue_wait_p50_ms": percentile(queue_waits, 50, 0), "queue_wait_p95_ms": percentile(queue_waits, 95, 0), "duration_p50_ms": percentile(durations, 50, 0), "render_p50_ms": percentile(renders, 50, 0), "worker_hwm_mib": ( round(src.hwm_kib / 1024, 1) if src.hwm_kib else None ), "render_peak_p50_mib": percentile(render_peaks, 50, 1), "render_peak_max_mib": rounded( max(render_peaks, default=None), 1, ), "memory_peak_mib": peak_mem, "anon_max_mib": anon_mem, "oom_kill": kills, "worker_restarts": src.restarts, "cpu_s3_accept": role_cores( src.cg.get("s3", []), over_start, min(over_end, over_start + ACCEPT_S3_WINDOW_S), ), "cpu_worker_per_job_s": per_job, "broker_memory_mib": src.extra.get("broker_memory_mib"), "broker_refused": src.extra.get("broker_refused", 0), "attempts_over_one": src.sql.get("attempts_over_one"), "skipped": json.dumps(skipped, sort_keys=True), "relay_published": sum( i.get("published", 0) for i in relay_passes ), "relay_republished": sum( i.get("republished", 0) for i in relay_passes ), "relay_recovered": sum( i.get("recovered", 0) for i in relay_passes ), "relay_backlog_full": len( events(src.relay_records, "Relay backlog is full"), ), "failed_by": json.dumps(src.sql.get("failed", {}), sort_keys=True), "second_jobs": len(second_ready) if src.second_records else None, "swapouts": src.swapouts, "lb_down": int(src.lb_down), "relay_restarts": src.relay_restarts, "relay_die": relay_causes["die"], "unexpected_rows": unexpected, "worker_die_oom": causes["oom"], "worker_die_command": causes["command"], "worker_die_other": causes["other"], "worker_exit_codes": json.dumps(causes["codes"], sort_keys=True), "worker_oom_events": causes["oom_events"], "lb_up_min": src.lb_up_min, }, ) for window in WINDOWS: for q in (50, 95, 99): row[f"ready_{window}_p{q}_s"] = percentile(waits[window], q) for q in (50, 95): row[f"queue_wait_{window}_p{q}_ms"] = percentile( waits_by_window[window], q, 0, ) for window, (w0, w1) in bounds.items(): row[f"peak_oldest_{window}_s"] = max( (t.oldest_s for t in src.jobs if w0 <= t.t < w1), default=None, ) for role in ROLES: lives = src.cg.get(role, []) flat = [life for each in lives for life in each] row[f"cpu_{role}_over"] = role_cores(lives, over_start, over_end) row[f"cpu_{role}_recovery"] = role_cores(lives, over_end, recovery_end) for window in ("before", "after"): row[f"cpu_{role}_{window}"] = role_cores(lives, *bounds[window]) for window, (w0, w1) in bounds.items(): row[f"throttle_{role}_{window}"] = ( throttle_share(flat, w0, w1) if w1 > w0 else None ) action_at = src.extra.get("second_started_at") if action_at is not None and src.second_records: def done_at(item: Mapping[str, Any]) -> float: return reconcile.timestamp(item["timestamp"]) def share(items: list[Mapping[str, Any]], total: int) -> float | None: return round(len(items) / total, 3) if total else None # столбец порога H3 — только разбор накопленного: окно after делится # поровну при любом prefetch (design, «Уточнения apply», п. 2) backlog_all = [ i for i in both_ready if action_at <= done_at(i) <= recovery_end ] backlog_second = [ i for i in second_ready if action_at <= done_at(i) <= recovery_end ] row["second_jobs"] = len(backlog_second) row["second_share"] = share(backlog_second, len(backlog_all)) # буквальное прочтение порога и само окно after: числами (verify W3) since_all = [i for i in both_ready if done_at(i) >= action_at] since_second = [i for i in second_ready if done_at(i) >= action_at] row["second_jobs_all"] = len(since_second) row["second_share_all"] = share(since_second, len(since_all)) row["second_share_after"] = share( [i for i in second_ready if done_at(i) > recovery_end], sum(1 for i in both_ready if done_at(i) > recovery_end), ) row.update(control_columns(src.control, bounds)) for role in TELEMETRY_ROLES: lives = src.cg.get(role, []) prefix = f"tel_{role.replace('-', '_')}" flat = [life for each in lives for life in each] peaks = [p[1] for each in src.mp.get(role, []) for p in each] row[f"{prefix}_cpu_over"] = ( role_cores(lives, over_start, over_end) if lives else None ) row[f"{prefix}_throttle_over"] = ( throttle_share(flat, over_start, over_end) if flat else None ) row[f"{prefix}_peak_mib"] = ( round(max(peaks) / 2**20, 1) if peaks else None ) row["flags"] = " ".join( cell_flags(row, zero_at is None, src.storage_down), ) return row def cell_flags( row: Mapping[str, Any], not_recovered: bool, storage_down: bool, ) -> list[str]: """Пометки решения 11; `oom` — только подтверждённое завершение ядром, а любой перезапуск — `worker_restart`, его причина — столбцы `worker_die_*`.""" flags = [] if num(row.get("oom_kill")) or num(row.get("worker_oom_events")): flags.append("oom") if num(row.get("worker_restarts")): flags.append("worker_restart") if num(row.get("worker_die_other")): flags.append("worker_crash") if (num(row.get("unexpected_rows")) or 0) > ( (num(row.get("s5xx")) or 0) + (num(row.get("s_conn")) or 0) ): # строка после 5xx или обрыва законна: на неё клиент не получил 202 flags.append("unexpected") if num(row.get("lost")): flags.append("lost") if num(row.get("stuck")): flags.append("stuck") if num(row.get("duplicate")): flags.append("duplicate") if num(row.get("sha_bad")): flags.append("sha_mismatch") if not_recovered: flags.append("not_recovered") if num(row.get("failed")): flags.append("failed") if num(row.get("dropped")): flags.append("dropped") if num(row.get("lb_down")): flags.append("lb_down") if num(row.get("swapouts")): flags.append("host_swap") if storage_down: flags.append("storage_down") return flags def timeline_rows( cell: Mapping[str, Any], src: Sources, ) -> list[dict[str, Any]]: """Посекундная шкала от `q-t0`: окно, очередь в трёх местах, принято и готово за секунду.""" over_start = src.t0 + cell["before_s"] over_end = over_start + cell["over_s"] after_start = src.extra.get("after_start") accepted: dict[int, int] = {} for upload in src.uploads: if upload.status == 202: second = int(upload.answered - src.t0) accepted[second] = accepted.get(second, 0) + 1 ready: dict[int, int] = {} for meta in src.metas.values(): if meta.status == "ready" and meta.updated_at is not None: second = int(meta.updated_at - src.t0) ready[second] = ready.get(second, 0) + 1 mq = {int(m[0] - src.t0): m for m in src.mq} rows = [] for tick in src.jobs: if tick.t < src.t0: continue second = int(tick.t - src.t0) if tick.t < over_start: window = "before" elif tick.t < over_end: window = "over" elif after_start is None or tick.t < after_start: window = "recovery" else: window = "after" broker = mq.get(second) rows.append( { "cell": cell["cell"], "t": second, "window": window, "outbox": tick.outbox, "waiting": tick.waiting, "published": tick.published, "processing": tick.processing, "unfinished": tick.unfinished, "broker_ready": broker[1] if broker else None, "broker_unacked": broker[2] if broker else None, "prefetched": ( max(0, broker[2] - tick.processing) if broker else None ), "oldest_s": tick.oldest_s, "staging_ready": tick.staging_ready, "accepted": accepted.get(second, 0), "ready": ready.get(second, 0), }, ) return rows # --------------------------------------------------------------- вердикты def read_csv(path: Path) -> list[dict[str, Any]]: if not path.exists(): return [] with path.open(encoding="utf-8", newline="") as file: return list(csv.DictReader(file)) def num(value: Any) -> float | None: return reconcile.num(value) def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None: return reconcile.med(row.get(key) for row in rows) def select( rows: Iterable[Mapping[str, Any]], *, block: str = "matrix", klass: str | None = None, setting: tuple[int, int] | None = None, ) -> list[Mapping[str, Any]]: found = [] for row in rows: if row.get("block") != block: continue if klass is not None and row.get("class") != klass: continue if setting is not None and ( num(row.get("concurrency")) != setting[0] or num(row.get("prefetch")) != setting[1] ): continue found.append(row) return found def flagged(row: Mapping[str, Any], flag: str) -> bool: return flag in (row.get("flags") or "").split() def classes_of(rows: Iterable[Mapping[str, Any]]) -> list[str]: return sorted({r["class"] for r in rows if r.get("block") == "matrix"}) def ratio(a: float | None, b: float | None) -> float | None: return None if a is None or not b else a / b class Check: """Условия гипотезы: `нет данных`, если нечем проверить условие; `нет` — если условие нарушено, и причина названа.""" def __init__(self) -> None: self.failed: list[str] = [] self.missing: list[str] = [] self.passed = 0 def within( self, name: str, value: float | None, low: float | None, high: float | None, ) -> None: if value is None: self.missing.append(name) elif (low is not None and value < low) or ( high is not None and value > high ): self.failed.append(f"{name} = {value:.3g}") else: self.passed += 1 def true(self, name: str, value: bool | None) -> None: if value is None: self.missing.append(name) elif not value: self.failed.append(name) else: self.passed += 1 def verdict(self) -> tuple[str, str]: if self.failed: return REFUTED, "; ".join(self.failed[:6]) if self.missing: return NO_DATA, "нет: " + ", ".join(self.missing[:6]) if not self.passed: return NO_DATA, "нет ячеек" return CONFIRMED, f"условий: {self.passed}" def h1(rows: list[Mapping[str, Any]]) -> Check: check = Check() for klass in classes_of(rows): base = select(rows, klass=klass, setting=(2, 2)) wide = select(rows, klass=klass, setting=(4, 4)) check.within( f"{klass}: (4,4)/(2,2) throughput", ratio( med(wide, "ready_s_recovery"), med(base, "ready_s_recovery"), ), None, 1.05, ) for row in select(rows): if (num(row.get("concurrency")) or 0) >= 2: check.within( f"{row['cell']}: CPU worker", num(row.get("cpu_worker_over")), 0.85, None, ) for klass in ("m", "lp"): base = select(rows, klass=klass, setting=(2, 2)) small = select(rows, klass=klass, setting=(1, 1)) check.within( f"{klass}: (1,1)/(2,2) throughput", ratio( med(small, "ready_s_recovery"), med(base, "ready_s_recovery"), ), None, 1.0, ) return check def h2(rows: list[Mapping[str, Any]]) -> Check: check = Check() limit = 1024.0 for klass in classes_of(rows): base = select(rows, klass=klass, setting=(2, 2)) wide = select(rows, klass=klass, setting=(4, 4)) check.within( f"{klass}: duration (4,4)/(2,2)", ratio(med(wide, "duration_p50_ms"), med(base, "duration_p50_ms")), 1.5, None, ) for row in base: check.within( f"{row['cell']}: memory.peak", num(row.get("memory_peak_mib")), None, limit - 0.001, ) check.true(f"{row['cell']}: без oom", not flagged(row, "oom")) wide_lp = select(rows, klass="lp", setting=(4, 4)) for row in wide_lp: check.within( f"{row['cell']}: memory.peak", num(row.get("memory_peak_mib")), 0.95 * limit, None, ) check.true( "lp (4,4): oom хотя бы в одном повторе", any(flagged(r, "oom") for r in wide_lp) if wide_lp else None, ) wide_lj = select(rows, klass="lj", setting=(4, 4)) for row in wide_lj: check.true(f"{row['cell']}: без oom", not flagged(row, "oom")) check.within( f"{row['cell']}: memory.peak", num(row.get("memory_peak_mib")), None, 500, ) if not wide_lj: check.missing.append("lj (4,4)") return check def h3(rows: list[Mapping[str, Any]]) -> Check: check = Check() for klass in ("s", "lp"): base = select(rows, klass=klass, setting=(2, 2)) big = select(rows, klass=klass, setting=(2, 256)) check.within( f"{klass}: throughput (2,256)/(2,2)", ratio(med(big, "ready_s_recovery"), med(base, "ready_s_recovery")), 0.85, 1.15, ) check.within( f"{klass}: p95 (2,256)/(2,2)", ratio(med(big, "ready_over_p95_s"), med(base, "ready_over_p95_s")), 0.8, 1.25, ) for row in big: check.within( f"{row['cell']}: broker_ready / пик", ratio( num(row.get("peak_broker_ready")), num(row.get("peak_unfinished")), ), None, 0.10, ) for row in base: check.within( f"{row['cell']}: broker_ready / пик", ratio( num(row.get("peak_broker_ready")), num(row.get("peak_unfinished")), ), 0.70, None, ) p2 = select(rows, block="w2-p2") p256 = select(rows, block="w2-p256") check.within( "доля второго при prefetch 2", med(p2, "second_share"), 0.30, None, ) check.within( "доля второго при prefetch 256", med(p256, "second_share"), None, 0.05, ) check.within( "восстановление p256 / p2", ratio(med(p256, "recovery_s"), med(p2, "recovery_s")), 1.4, None, ) return check def expected_wait(row: Mapping[str, Any]) -> float | None: return ratio(num(row.get("peak_unfinished")), num(row.get("ready_s_over"))) def h4(rows: list[Mapping[str, Any]]) -> Check: check = Check() for row in select(rows): if flagged(row, "oom"): continue check.within( f"{row['cell']}: p95 / (пик / throughput)", ratio(num(row.get("ready_over_p95_s")), expected_wait(row)), 0.8, 1.25, ) return check def h5(rows: list[Mapping[str, Any]], limit: int) -> Check: check = Check() on = select(rows, block="gate-on") off = select(rows, block="gate-off") if not on or not off: check.missing.append("блоки gate-on и gate-off") return check for row in on: check.within( f"{row['cell']}: пик", num(row.get("peak_unfinished")), None, limit + IN_FLIGHT, ) check.true(f"{row['cell']}: 429 есть", (num(row.get("s429")) or 0) > 0) check.true( f"{row['cell']}: Retry-After 5", row.get("retry_after") == BACKLOG_RETRY_AFTER, ) check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx"))) check.true( f"{row['cell']}: 429 = записи backlog", num(row.get("s429")) == num(row.get("backlog_429")), ) check.within( f"{row['cell']}: p95 / (предел / throughput)", ratio( num(row.get("ready_over_p95_s")), ratio(float(limit), num(row.get("ready_s_over"))), ), 0.8, 1.25, ) for row in off: check.within( f"{row['cell']}: пик без предела", num(row.get("peak_unfinished")), 1.5 * limit, None, ) check.within( "p95 on / off", ratio(med(on, "ready_over_p95_s"), med(off, "ready_over_p95_s")), None, 0.75, ) check.within( "восстановление on / off", ratio(med(on, "recovery_s"), med(off, "recovery_s")), None, 0.75, ) a = med(on, "upload_before_p50_ms") b = med(off, "upload_before_p50_ms") check.within( "загрузка before p50 on − off, мс", None if a is None or b is None else a - b, None, 3, ) return check def pooled(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]: values: list[float] = [] for row in rows: values += [float(v) for v in (row.get(key) or "").split()] return values def h6(rows: list[Mapping[str, Any]]) -> Check: check = Check() for row in rows: for flag in H6_FLAGS: check.true(f"{row['cell']}: нет {flag}", not flagged(row, flag)) if num(row.get("failed")): only_exhausted = all( key.startswith("retry_exhausted") for key in json.loads(row.get("failed_by") or "{}") ) check.true( f"{row['cell']}: failed только при oom и retry_exhausted", flagged(row, "oom") and only_exhausted, ) if flagged(row, "oom") or row.get("block") not in ( "matrix", "gate-on", "gate-off", ): continue bound = expected_wait(row) check.within( f"{row['cell']}: восстановление / (пик / throughput)", ratio(num(row.get("recovery_s")), bound), None, 1.3, ) by_config: dict[str, list[Mapping[str, Any]]] = {} for row in rows: by_config.setdefault(row["config"], []).append(row) for config, items in by_config.items(): before = reconcile.percentile(pooled(items, "ready_before_list"), 95) after = reconcile.percentile(pooled(items, "ready_after_list"), 95) check.within( f"{config}: p95 after / before", ratio(after, before), None, 1.5, ) return check def h7(rows: list[Mapping[str, Any]]) -> Check: check = Check() for row in rows: if row.get("block") == "accept": continue for window in CONTROL_WINDOWS: for kind, limit in ( ("write", 200), ("read", 200), ("download", 500), ): value = num(row.get(f"ctrl_{kind}_{window}_p95")) if value is not None: check.within( f"{row['cell']}: {kind} {window} p95", value, None, limit, ) check.true( f"{row['cell']}: {kind} {window} 5xx нет", not num(row.get(f"ctrl_{kind}_{window}_s5xx")), ) check.true( f"{row['cell']}: все реплики в LB", not num(row.get("lb_down")), ) return check def h8(rows: list[Mapping[str, Any]]) -> Check: check = Check() kills = select(rows, block="kill-worker") restarts = select(rows, block="restart-broker") if not kills: check.missing.append("блок kill-worker") if not restarts: check.missing.append("блок restart-broker") for row in kills: check.true( f"{row['cell']}: всё ready", num(row.get("ready")) == num(row.get("accepted")), ) check.within( f"{row['cell']}: заданий с повтором", num(row.get("attempts_over_one")), None, 2, ) end = num(row.get("over_end")) action = num(row.get("action_at")) wait = expected_wait(row) zero = num(row.get("zero_at")) if None in (end, action, wait, zero): check.missing.append(f"{row['cell']}: моменты") continue check.within( f"{row['cell']}: разбор до границы, с", zero - max(end + 1.3 * wait, action + LEASE_S + 15), None, 0, ) for row in restarts: check.true( f"{row['cell']}: всё ready", num(row.get("ready")) == num(row.get("accepted")), ) check.within( f"{row['cell']}: заданий с повтором", num(row.get("attempts_over_one")), None, 0, ) check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx"))) wait = expected_wait(row) recovery = num(row.get("recovery_s")) check.within( f"{row['cell']}: восстановление, с", None if wait is None or recovery is None else recovery - (1.3 * wait + 15), None, 0, ) return check def h9(rows: list[Mapping[str, Any]], limit: int) -> Check: check = Check() cells = select(rows, block="accept") if not cells: check.missing.append("блок accept") for row in cells: busy = num(row.get("first_busy_s")) backlog = num(row.get("first_backlog_s")) check.true( f"{row['cell']}: первый busy раньше backlog", None if busy is None or backlog is None else busy < backlog, ) check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx"))) check.within( f"{row['cell']}: CPU S3 первые 30 с", num(row.get("cpu_s3_accept")), 0.85, None, ) check.within( f"{row['cell']}: принято/с", ratio(num(row.get("accepted_over")), num(row.get("over_s"))), 6, None, ) check.within( f"{row['cell']}: пик", num(row.get("peak_unfinished")), None, limit + IN_FLIGHT, ) check.within( f"{row['cell']}: готовых/с в окне", num(row.get("ready_s_over")), None, 1.5, ) check.within( f"{row['cell']}: готовых/с при восстановлении", num(row.get("ready_s_recovery")), 2.4, None, ) for window in CONTROL_WINDOWS: for kind, bound in ( ("write", 200), ("read", 200), ("download", 1000), ): value = num(row.get(f"ctrl_{kind}_{window}_p95")) if value is not None: check.within( f"{row['cell']}: {kind} {window} p95", value, None, bound, ) check.true( f"{row['cell']}: {kind} {window} 5xx нет", not num(row.get(f"ctrl_{kind}_{window}_s5xx")), ) return check def verdicts( rows: list[Mapping[str, Any]], limit: int, ) -> dict[str, tuple[str, str]]: return { "H1": h1(rows).verdict(), "H2": h2(rows).verdict(), "H3": h3(rows).verdict(), "H4": h4(rows).verdict(), "H5": h5(rows, limit).verdict(), "H6": h6(rows).verdict(), "H7": h7(rows).verdict(), "H8": h8(rows).verdict(), "H9": h9(rows, limit).verdict(), } def settings_of(rows: Iterable[Mapping[str, Any]]) -> list[tuple[int, int]]: return sorted( { (int(num(r["concurrency"])), int(num(r["prefetch"]))) for r in rows if r.get("block") == "matrix" }, ) def recommend(rows: list[Mapping[str, Any]]) -> dict[str, Any]: """Правило решения 12: кандидат — без `oom` и `failed` на всех классах, с throughput не ниже 90 % лучшей на каждом классе; из кандидатов — наименьшая конкуренция; `prefetch` равен конкуренции. Настройка, не снятая на части классов, не кандидат: условие «на всех классах» для неё не проверено, она названа в `partial` с этими классами (verify S6). В сравнении лучшей она участвует: её снятые числа настоящие.""" settings = settings_of(rows) candidates = [] reasons: dict[str, str] = {} partial: dict[str, list[str]] = {} for setting in settings: problem = None key = f"{setting[0]},{setting[1]}" missing = [ klass for klass in classes_of(rows) if not select(rows, klass=klass, setting=setting) ] if missing: partial[key] = missing for klass in classes_of(rows): cells = select(rows, klass=klass, setting=setting) if not cells: continue if any(flagged(r, "oom") or flagged(r, "failed") for r in cells): problem = f"{klass}: oom или failed" break best = max( ( value for other in settings if ( value := med( select(rows, klass=klass, setting=other), "ready_s_recovery", ) ) is not None ), default=None, ) own = med(cells, "ready_s_recovery") if best is None or own is None or own < 0.9 * best: problem = f"{klass}: throughput ниже 90 % лучшей" break if problem: reasons[key] = problem elif not missing: candidates.append(setting) if not candidates: return { "setting": None, "candidates": [], "rejected": reasons, "partial": partial, } concurrency = min(setting[0] for setting in candidates) return { "setting": (concurrency, concurrency), "candidates": candidates, "rejected": reasons, "partial": partial, } # ----------------------------------------------------------------- сводка def fmt(value: Any, digits: int = 1) -> str: number = num(value) return "—" if number is None else f"{number:.{digits}f}" def spread( rows: Iterable[Mapping[str, Any]], key: str, digits: int = 1, ) -> str: return reconcile.spread((row.get(key) for row in rows), digits) def config_table(rows: list[Mapping[str, Any]]) -> list[str]: head = ( "| Конфигурация | Ячеек | Принято | 429 | Готовых/с: окно | восстановление | " "Пик незаверш. | Пик брокера | До `ready` over p50 / p95 / p99, с | " "Восст., с | `memory.peak`, МиБ | VmHWM, МиБ | Процесс p50 / макс., МиБ | " "CPU worker, S3 | Троттлинг worker | OOM | Перезапуски | Пометки | " "Queue lag: возраст старейшего, пик before / over / after, с | " "Ожидание в очереди p50 / p95, мс: before; over; after | " "Завершения worker: ядро / команда / прочие | Реплик в LB, минимум |" ) lines = [head, "|" + "---|" * 22] by_config: dict[str, list[Mapping[str, Any]]] = {} for row in rows: by_config.setdefault(row["config"], []).append(row) for config in sorted(by_config): items = by_config[config] flags = sorted( {f for r in items for f in (r.get("flags") or "").split()}, ) lines.append( f"| `{config}` | {len(items)} | {spread(items, 'accepted', 0)} | " f"{spread(items, 's429', 0)} | {spread(items, 'ready_s_over', 2)} | " f"{spread(items, 'ready_s_recovery', 2)} | " f"{spread(items, 'peak_unfinished', 0)} | " f"{spread(items, 'peak_broker_ready', 0)} | " f"{fmt(med(items, 'ready_over_p50_s'))} / " f"{fmt(med(items, 'ready_over_p95_s'))} / " f"{fmt(med(items, 'ready_over_p99_s'))} | " f"{spread(items, 'recovery_s', 1)} | " f"{spread(items, 'memory_peak_mib', 0)} | " f"{fmt(med(items, 'worker_hwm_mib'), 0)} | " f"{fmt(med(items, 'render_peak_p50_mib'), 0)} / " f"{fmt(med(items, 'render_peak_max_mib'), 0)} | " f"{fmt(med(items, 'cpu_worker_over'), 2)}, " f"{fmt(med(items, 'cpu_s3_over'), 2)} | " f"{fmt(med(items, 'throttle_worker_over'), 2)} | " f"{sum(int(num(r.get('oom_kill')) or 0) for r in items)} | " f"{sum(int(num(r.get('worker_restarts')) or 0) for r in items)} | " f"{' '.join(flags) or '—'} | " + " / ".join( fmt(med(items, f"peak_oldest_{w}_s")) for w in WINDOWS ) + " | " + "; ".join( f"{fmt(med(items, f'queue_wait_{w}_p50_ms'), 0)} / " f"{fmt(med(items, f'queue_wait_{w}_p95_ms'), 0)}" for w in WINDOWS ) + " | " + " / ".join( str( sum( int(num(r.get(f"worker_die_{k}")) or 0) for r in items ), ) for k in ("oom", "command", "other") ) + " | " + fmt( min( ( value for r in items if (value := num(r.get("lb_up_min"))) is not None ), default=None, ), 0, ) + " |", ) return lines def control_table(rows: list[Mapping[str, Any]]) -> list[str]: lines = [ "| Конфигурация | Окно | Запись p50 / p95 / p99, мс | Чтение p50 / p95 / p99, " "мс | Скачивание p50 / p95 / p99, мс | Ответов | 5xx |", "|" + "---|" * 7, ] by_config: dict[str, list[Mapping[str, Any]]] = {} for row in rows: by_config.setdefault(row["config"], []).append(row) for config in sorted(by_config): items = by_config[config] for window in CONTROL_WINDOWS: parts = [] total = errors = 0.0 for kind in CONTROL_KINDS.values(): prefix = f"ctrl_{kind}_{window}" parts.append( " / ".join( fmt(med(items, f"{prefix}_{stat}")) for stat in ("p50", "p95", "p99") ), ) total += sum(num(r.get(f"{prefix}_n")) or 0 for r in items) errors += sum(num(r.get(f"{prefix}_s5xx")) or 0 for r in items) lines.append( f"| `{config}` | {window} | {parts[0]} | {parts[1]} | {parts[2]} " f"| {int(total)} | {int(errors)} |", ) return lines def block_table(rows: list[Mapping[str, Any]]) -> list[str]: lines = [ "| Ячейка | Принято | 429 busy / backlog | Первый busy / backlog, с | Пик | " "Готовых/с окно / восст. | Восст., с | Повторов | Второй worker: задания, доля " "| Пометки | Второй: доля после запуска всего / в окне after | " "Relay: перезапуски / завершения |", "|" + "---|" * 12, ] for row in sorted(rows, key=lambda r: r["cell"]): if row.get("block") == "matrix": continue lines.append( f"| `{row['cell']}` | {fmt(row.get('accepted'), 0)} | " f"{fmt(row.get('busy_429'), 0)} / {fmt(row.get('backlog_429'), 0)} | " f"{fmt(row.get('first_busy_s'))} / " f"{fmt(row.get('first_backlog_s'))} | " f"{fmt(row.get('peak_unfinished'), 0)} | " f"{fmt(row.get('ready_s_over'), 2)} / " f"{fmt(row.get('ready_s_recovery'), 2)} | " f"{fmt(row.get('recovery_s'))} | " f"{fmt(row.get('attempts_over_one'), 0)} | " f"{fmt(row.get('second_jobs'), 0)}, " f"{fmt(row.get('second_share'), 2)} | " f"{row.get('flags') or '—'} | " f"{fmt(row.get('second_share_all'), 3)} / " f"{fmt(row.get('second_share_after'), 3)} | " f"{fmt(row.get('relay_restarts'), 0)} / " f"{fmt(row.get('relay_die'), 0)} |", ) return lines def telemetry_line(meta: Mapping[str, Any]) -> str: """Режим телеметрии метки (design change `track1-telemetry-cost`, решение 13) и предупреждение о сумме лимитов CPU больше ВМ.""" mode = meta.get("profile", {}).get("telemetry") or "off" if mode != "on": return ( "Телеметрия стенда выключена (TELEMETRY=off): SDK реплик, worker и " "relay выключен, сервисов телеметрии нет." ) budget = meta.get("resources", {}).get("budget", {}) text = ( "Телеметрия стенда включена (TELEMETRY=on): SDK реплик, worker и relay " "с настройками рабочего стенда, Collector, Prometheus, Loki и Tempo " "рабочего стенда; столбцы `tel_*` в конце `cells.csv`." ) if budget.get("cpus", 0) > budget.get("vm_cpus", 0) > 0: text += ( f" Сумма лимитов CPU {budget['cpus']:g} больше CPU ВМ " f"{budget['vm_cpus']}: лимиты делят время, а не резервируют ядра." ) return text def render_summary( meta: Mapping[str, Any], rows: list[Mapping[str, Any]], ) -> str: info = meta.get("queue", {}) limit = int(info.get("default_limit") or 250) plan_total = len(info.get("plan", [])) lines = [ f"# Очередь под перегрузкой: метка {meta.get('label')}", "", f"commit `{meta.get('git', {}).get('commit')}`, " f"dirty {meta.get('git', {}).get('dirty')}, статус " f"{meta.get('status')}, ячеек {len(rows)} из {plan_total}.", "", telemetry_line(meta), "", ] fatal = [ f"`{row['cell']}`: {row['flags']}" for row in rows if any(flagged(row, flag) for flag in FATAL_FLAGS) ] if fatal: lines += [ "**Потеряна работа или лишние вложения:** " + "; ".join(fatal), "", ] lines += ["## Гипотезы", "", "| № | Вердикт | Детали |", "|---|---|---|"] for name, (verdict, detail) in verdicts(rows, limit).items(): lines.append(f"| {name} | {verdict} | {detail} |") advice = recommend(rows) setting = advice["setting"] lines += ["", "## Рекомендация", ""] if setting is None: lines.append("Кандидатов нет: правило не выбрало настройку.") else: keep = setting == DEFAULT_SETTING lines.append( f"Конкуренция {setting[0]}, prefetch {setting[1]}: " + ( "значения по умолчанию оставить." if keep else "отличается от (2, 2), решение — по правилу в отчёте." ), ) for key, reason in advice["rejected"].items(): lines.append(f"- ({key.replace(',', ', ')}) не кандидат: {reason}") for key, missing in advice["partial"].items(): lines.append( f"- ({key.replace(',', ', ')}) снята не на всех классах (нет: " f"{', '.join(missing)}): без данных не кандидат", ) lines += [ "", "## Ячейки по конфигурациям", "", "Значения — медиана (min–max) повторов.", "", *config_table(rows), "", "## Блоки", "", *block_table(rows), "", "## Контрольный поток", "", *control_table(rows), "", ] return "\n".join(lines) def setting_chart( rows: list[Mapping[str, Any]], key: str, title: str, unit: str, ) -> str: """Столбики: класс → настройки, медиана повторов.""" classes = classes_of(rows) settings = settings_of(rows) colors = ["#2a78d6", "#d98a00", "#3a9a5b", "#b0569c"] width, height = 860, 420 svg = plots.Svg(width, height, title) left, right, top, bottom = 70, 20, 50, 60 values = [ med(select(rows, klass=klass, setting=setting), key) for klass in classes for setting in settings ] highest = max((v for v in values if v is not None), default=0.0) or 1.0 y_scale = (height - top - bottom) / (highest * 1.1) group = (width - left - right) / max(1, len(classes)) bar = group / (len(settings) + 1) base = height - bottom svg.line(left, base, width - right, base, color=plots.AXIS) svg.text(left, top - 14, unit, size=12) for ci, klass in enumerate(classes): x0 = left + ci * group svg.text(x0 + group / 2 - 8, base + 22, klass, size=13) for si, setting in enumerate(settings): value = med(select(rows, klass=klass, setting=setting), key) if value is None: continue bar_h = value * y_scale svg.bar( x0 + si * bar + bar / 2, base - bar_h, bar * 0.9, bar_h, colors[si % len(colors)], f"{klass} ({setting[0]}, {setting[1]}): {value:.2f} {unit}", top=True, ) for si, setting in enumerate(settings): x = left + si * 130 svg.bar( x, height - 28, 14, 14, colors[si % len(colors)], "", top=False, ) svg.text(x + 20, height - 16, f"({setting[0]}, {setting[1]})", size=12) return svg.render() def gate_chart(timeline: list[Mapping[str, Any]]) -> str: """Незавершённые посекундно: блоки предела, первый повтор.""" width, height = 860, 420 svg = plots.Svg(width, height, "Незавершённые задания: предел и без него") left, right, top, bottom = 70, 20, 50, 50 series: dict[str, list[tuple[float, float]]] = {} for row in timeline: cell = row.get("cell", "") if cell.startswith(("gate-on", "gate-off")) and cell.endswith("-r1"): t = num(row.get("t")) value = num(row.get("unfinished")) if t is not None and value is not None: series.setdefault(cell, []).append((t, value)) if not series: svg.text(left, top + 30, "нет данных", size=13) return svg.render() max_t = max(p[0] for points in series.values() for p in points) or 1.0 max_v = max(p[1] for points in series.values() for p in points) or 1.0 colors = ["#2a78d6", "#d98a00"] def sx(t: float) -> float: return left + t / max_t * (width - left - right) def sy(v: float) -> float: return height - bottom - v / (max_v * 1.1) * (height - top - bottom) svg.line( left, height - bottom, width - right, height - bottom, color=plots.AXIS, ) svg.text( left, top - 14, f"до {max_v:.0f} заданий, до {max_t:.0f} с", size=12, ) for index, (cell, points) in enumerate(sorted(series.items())): svg.polyline( [(sx(t), sy(v)) for t, v in points], colors[index % len(colors)], ) svg.text(left + index * 260, height - 18, cell, size=12) return svg.render() def render_plots( rows: list[Mapping[str, Any]], timeline: list[Mapping[str, Any]], ) -> dict[str, str]: return { "throughput-by-setting.svg": setting_chart( rows, "ready_s_recovery", "Throughput по настройкам и классам", "готовых/с", ), "p95-by-setting.svg": setting_chart( rows, "ready_over_p95_s", "p95 времени до ready, окно over", "с", ), "memory-by-setting.svg": setting_chart( rows, "memory_peak_mib", "memory.peak worker", "МиБ", ), "backlog-gate.svg": gate_chart(timeline), } # -------------------------------------------------------------- самопроверка def base_row(**values: Any) -> dict[str, Any]: row = { "cell": "s-c2p2-r1", "config": "s-c2p2", "block": "matrix", "class": "s", "concurrency": "2", "prefetch": "2", "repeat": "1", "accepted": "100", "ready": "100", "failed": "0", "s429": "0", "s5xx": "0", "busy_429": "0", "backlog_429": "0", "ready_s_over": "3", "ready_s_recovery": "3", "peak_unfinished": "60", "peak_broker_ready": "50", "ready_over_p95_s": "20", "recovery_s": "20", "memory_peak_mib": "200", "duration_p50_ms": "300", "cpu_worker_over": "0.95", "failed_by": "{}", "ready_before_list": "1.0 1.1", "ready_after_list": "1.0 1.2", "attempts_over_one": "0", "over_end": "100", "zero_at": "120", "action_at": "90", "second_share": "", "flags": "", "lb_down": "0", "retry_after": "", } for key, value in values.items(): row[key] = value return row def matrix_fixture() -> list[dict[str, Any]]: rows = [] for klass in ("s", "m", "lj", "lp"): rows.append( base_row( cell=f"{klass}-c2p2-r1", config=f"{klass}-c2p2", **{"class": klass}, ), ) wide = { "duration_p50_ms": "600", "memory_peak_mib": "400", } flags = "" if klass == "lp": wide["memory_peak_mib"] = "1024" wide["oom_kill"] = "3" flags = "oom" rows.append( base_row( cell=f"{klass}-c4p4-r1", config=f"{klass}-c4p4", concurrency="4", prefetch="4", flags=flags, **{"class": klass, **wide}, ), ) for klass in ("m", "lp"): rows.append( base_row( cell=f"{klass}-c1p1-r1", config=f"{klass}-c1p1", concurrency="1", prefetch="1", ready_s_recovery="2.6", cpu_worker_over="0.7", **{"class": klass}, ), ) for klass in ("s", "lp"): rows.append( base_row( cell=f"{klass}-c2p256-r1", config=f"{klass}-c2p256", prefetch="256", peak_broker_ready="0", **{"class": klass}, ), ) return rows def gate_fixture() -> list[dict[str, Any]]: on = base_row( cell="gate-on-s-c2p2-r1", config="gate-on-s-c2p2", block="gate-on", peak_unfinished="255", s429="180", backlog_429="180", retry_after="5", ready_over_p95_s="85", recovery_s="85", upload_before_p50_ms="15", ) off = base_row( cell="gate-off-s-c2p2-r1", config="gate-off-s-c2p2", block="gate-off", peak_unfinished="430", ready_over_p95_s="145", recovery_s="145", upload_before_p50_ms="14", ) return [on, off] def selftest() -> None: # план: без повтора подряд на границе кругов, все ячейки configs = [ {"block": "matrix", "class": k, "concurrency": 2, "prefetch": 2} for k in ("s", "m", "lj") ] plan = plan_cells(configs, 3, 7) assert len(plan) == 9 for prev, cur in zip(plan, plan[1:], strict=False): assert prev["config"] != cur["config"] assert plan[0]["order"] == 1 and plan[-1]["order"] == 9 # разбор строк t0, ups = parse_uploads( "q-t0,1000000\n" "q,before,1000000,10,202,-,a,id1\n" "q,over,1010000,20,429,5,a,-\nбред\n", ) assert t0 == 1000.0 and len(ups) == 2 and ups[1].retry_after == "5" metas, checks = parse_sweep( "qs,id1,200,ready,-,3,2026-11-01T10:00:00.123456Z\n" "qs,id2,404,-,-,0,-\nqv,id1,original,1\nqv,id1,crop,0\n", ) assert abs(metas["id1"].updated_at % 1 - 0.123456) < 1e-5 assert metas["id2"].updated_at is None and checks == [True, False] jobs = parse_jobs("jobs|1000000|1|2|3|4|5.5|6\nсбой\n") assert jobs[0].unfinished == 10 and jobs[0].oldest_s == 5.5 mq, missing = parse_mq("mq 1000000000 5 2\nmq 2000000000 missing\n") assert mq == [(1.0, 5, 2)] and missing == 1 # жизни: перезапуск сбрасывает счётчики points = [ (10**9, 100, 10, 1), (2 * 10**9, 200, 20, 2), (3 * 10**9, 50, 5, 0), (4 * 10**9, 150, 15, 1), ] lives = split_lives(points) assert len(lives) == 2 assert lives_delta(lives, 0, 5) == 200 assert role_cores([lives], 1, 4) == round(200 / 1e6 / 3, 4) mp = [ (10**9, 500 * MIB, 400 * MIB, 2), (2 * 10**9, 700 * MIB, 600 * MIB, 3), (3 * 10**9, 100 * MIB, 90 * MIB, 0), (4 * 10**9, 300 * MIB, 200 * MIB, 1), ] assert memory_by_lives(split_lives(mp, key=1)) == (700.0, 600.0, 4) # LB: BACKEND и FRONTEND — не реплики, UP 1/2 — ещё в балансировке both = [ {"svname": "api-1", "status": "UP"}, {"svname": "api-2", "status": "UP"}, {"svname": "api-3", "status": "MAINT"}, {"svname": "BACKEND", "status": "UP"}, ] one = [ {"svname": "api-1", "status": "UP"}, {"svname": "api-2", "status": "DOWN"}, {"svname": "BACKEND", "status": "UP"}, ] assert lb_up_count(both) == 2 assert lb_up_count(one) == 1, "реплика вышла, BACKEND посчитан как UP" assert lb_up_min({1: both, 2: one}) == 1 assert lb_up_min({}) is None assert not lb_down({1: both}, 2), "обе UP — реплика не выходила" assert lb_down({1: both, 2: one}, 2), "одна UP и BACKEND UP — вышла" assert not lb_down({}, 2) assert not lb_down( {1: [{"svname": "api-1", "status": "UP 1/2"}, *one[:1]]}, 2, ), "UP 1/2 — ещё в балансировке" # причины завершений: ядро, команда, прочее; процесс обработки — без die happened = parse_docker_events( "1000000000|oom|w|| \n" "1000000000|oom|w||\n" "9000000000|oom|w||\n" "9050000000|die|w|137|\n" "20000000000|kill|w||9\n" "20050000000|die|w|137|\n" "30000000000|die|w|3|\n" "40000000000|die|w|137|\n" "50000000000|die|other|137|\nбред\n", ) found = die_causes(happened, "w") assert found["oom"] == 1 and found["command"] == 1, found assert found["other"] == 2 and found["die"] == 4, found assert found["codes"] == {"3": 1, "137": 1}, found assert found["oom_events"] == 3, found # ячейка на границах окон: потерянное, зависшее, OOM, перезапуск base = { "cell": "s-c2p2-r1", "config": "s-c2p2", "block": "matrix", "class": "s", "concurrency": 2, "prefetch": 2, "repeat": 1, "order": 1, "before_s": 10, "over_s": 20, } t = 1000.0 uploads_seen = [ Upload("before", int(t * 1000), 10, 202, "-", "a", "b1"), Upload("over", int((t + 10) * 1000), 10, 202, "-", "a", "o1"), Upload("over", int((t + 11) * 1000), 10, 202, "-", "a", "o2"), Upload("over", int((t + 12) * 1000), 10, 202, "-", "a", "o3"), Upload("over", int((t + 13) * 1000), 10, 429, "5", "a", "-"), ] src = Sources( uploads=uploads_seen, t0=t, metas={ "b1": Meta(200, "ready", "-", 3, t + 1), # ровно на начале окна over: входит "o1": Meta(200, "ready", "-", 3, t + 10), "o2": Meta(200, "queued", "-", 0, None), }, sha_checks=[True], jobs=[ JobTick(t + 5, 0, 0, 0, 1, 0.1, 0), JobTick(t + 20, 1, 1, 1, 2, 9.0, 1), JobTick(t + 40, 0, 0, 0, 0, 0.0, 0), ], mq=[(t + 20, 3, 2), (t + 40, 0, 0)], mq_missing=0, cg={ "worker": [ split_lives( [(int((t + i) * 1e9), i * 10**6, i, 0) for i in range(41)], ), ], }, mp={"worker": split_lives(mp)}, control=[], worker_records=[], second_records=[], relay_records=[], api_records=[], sql={"extra_jobs": 0, "bad_variants": 0, "attempts_over_one": 1}, hwm_kib=2048, restarts=1, docker_events=[ DockerEvent(t + 15, "oom", "worker", "", ""), DockerEvent(t + 15.05, "die", "worker", "137", ""), DockerEvent(t + 16, "die", "relay", "1", ""), ], relay_restarts=1, lb_up_min=2, db_ids={"b1", "o1", "o2", "o3", "warm"}, warm_ids={"warm"}, ) row = cell_row(base, src) assert row["lost"] == 1 and row["stuck"] == 1 # счётчик cgroup 4 и завершение процесса 1, подтверждённое событиями assert row["ready"] == 2 and row["oom_kill"] == 5 assert row["worker_die_oom"] == 1 and row["worker_die_other"] == 0 assert row["relay_restarts"] == 1 and row["relay_die"] == 1 assert row["unexpected_rows"] == 0 and "unexpected" not in row["flags"] assert row["lb_up_min"] == 2 assert "lost" in row["flags"] and "stuck" in row["flags"] # перезапуск не от ядра — не oom; лишняя строка без 5xx и обрыва — пометка plain = replace( src, mp={}, docker_events=[DockerEvent(t + 15, "die", "worker", "3", "")], db_ids={"b1", "o1", "o2", "o3", "warm", "чужое"}, ) crash = cell_row(base, plain) assert crash["oom_kill"] == 0 and crash["worker_die_other"] == 1, crash assert crash["worker_exit_codes"] == '{"3": 1}' flags = crash["flags"].split() assert "oom" not in flags and "worker_restart" in flags, flags assert "worker_crash" in flags and "unexpected" in flags, flags assert crash["unexpected_rows"] == 1 explained = replace( plain, uploads=[ *uploads_seen, Upload("over", int((t + 14) * 1000), 10, 503, "-", "a", "-"), ], ) assert "unexpected" not in cell_row(base, explained)["flags"].split() # CPU до конца восстановления делится на задания до него: не на окно after zero_end = replace( src, metas={ "b1": Meta(200, "ready", "-", 3, t + 1), "o1": Meta(200, "ready", "-", 3, t + 20), "o2": Meta(200, "ready", "-", 3, t + 30), "o3": Meta(200, "ready", "-", 3, t + 39), "a1": Meta(200, "ready", "-", 3, t + 45), "a2": Meta(200, "ready", "-", 3, t + 46), }, ) # от t + 0 до нуля незавершённых в t + 40: 40 с CPU по 1 ядру на 4 задания assert cell_row(base, zero_end)["cpu_worker_per_job_s"] == 10.0 # queue lag по окнам: ожидание — по окну прихода, разогрева в счёте нет laggy = replace( src, worker_records=[ { "event": "Started job attempt", "attachment_id": ident, "queue_wait_ms": wait, } for ident, wait in ( ("b1", 100), ("o1", 1000), ("o2", 3000), ("warm", 9000), ) ], ) lag = cell_row(base, laggy) assert lag["queue_wait_before_p50_ms"] == 100 assert lag["queue_wait_over_p50_ms"] == 1000 assert lag["queue_wait_over_p95_ms"] == 3000 assert lag["queue_wait_after_p50_ms"] is None assert lag["queue_wait_p95_ms"] == 9000, "столбец на ячейку прежний" assert lag["peak_oldest_before_s"] == 0.1 assert lag["peak_oldest_over_s"] == 9.0 assert set(CELL_FIELDS) >= set(lag), set(lag) - set(CELL_FIELDS) assert len(CELL_FIELDS) == len(set(CELL_FIELDS)) # доля второго: порог H3 — только разбор, окно after — отдельным числом done = [ { "event": "Attachment became ready", "timestamp": stamp, "duration_ms": 300, } for stamp in ( "2026-11-01T10:00:10Z", "2026-11-01T10:00:20Z", "2026-11-01T10:00:50Z", "2026-11-01T10:00:50Z", ) ] other_done = done[3:] done = done[:3] second = reconcile.timestamp("2026-11-01T10:00:00Z") shared = cell_row( base, replace( src, t0=second, uploads=[ Upload( "over", int((second + 25) * 1000), 10, 202, "-", "a", "o1", ), ], jobs=[ JobTick(second + 28, 0, 0, 0, 1, 1.0, 0), JobTick(second + 30, 0, 0, 0, 0, 0.0, 0), ], worker_records=done, second_records=other_done, extra={"second_started_at": second + 5}, ), ) assert shared["second_jobs"] == 0 and shared["second_share"] == 0 assert shared["second_jobs_all"] == 1 assert shared["second_share_all"] == 0.25 assert shared["second_share_after"] == 0.5 # край окна — между тиками: CPU не занижен assert row["cpu_worker_over"] == 1.0, row["cpu_worker_over"] assert "oom" in row["flags"] and "worker_restart" in row["flags"] assert row["ready_s_over"] == round(1 / 20, 3) assert row["peak_unfinished"] == 5 and row["peak_prefetched"] == 0 assert row["recovery_s"] is not None and row["memory_peak_mib"] == 700.0 assert row["accepted_over"] == 3 and row["s429"] == 1 timeline = timeline_rows(base, src) assert [r["window"] for r in timeline] == ["before", "over", "recovery"] # вердикты: подтверждено, нет, нет данных rows = matrix_fixture() + gate_fixture() w2 = [ base_row( cell="w2-p2-r1", config="w2-p2", block="w2-p2", second_share="0.4", recovery_s="20", ), base_row( cell="w2-p256-r1", config="w2-p256", block="w2-p256", prefetch="256", second_share="0", recovery_s="30", ), ] restarts = [ base_row( cell="kill-r1", config="kill", block="kill-worker", attempts_over_one="2", ), base_row(cell="brk-r1", config="brk", block="restart-broker"), ] accept = [ base_row( cell="acc-r1", config="acc", block="accept", first_busy_s="10", first_backlog_s="35", cpu_s3_accept="0.97", accepted_over="300", over_s="45", peak_unfinished="280", ready_s_over="0.6", ready_s_recovery="2.8", ctrl_download_over_p95="340", ready_over_p95_s="93", recovery_s="100", ), ] everything = rows + w2 + restarts + accept assert h1(everything).verdict()[0] == CONFIRMED, h1(everything).verdict() assert h2(everything).verdict()[0] == CONFIRMED, h2(everything).verdict() assert h3(everything).verdict()[0] == CONFIRMED, h3(everything).verdict() assert h4(rows[:1]).verdict()[0] == CONFIRMED assert h5(everything, 250).verdict()[0] == CONFIRMED, h5( everything, 250, ).verdict() assert h6(rows[:1]).verdict()[0] == CONFIRMED, h6(rows[:1]).verdict() assert h7(everything).verdict()[0] == CONFIRMED assert h8(everything).verdict()[0] == CONFIRMED, h8(everything).verdict() assert h9(everything, 250).verdict()[0] == CONFIRMED, h9( everything, 250, ).verdict() empty: list[dict[str, Any]] = [] for name, (verdict, _) in verdicts(empty, 250).items(): assert verdict == NO_DATA, (name, verdict) def broken(**values: Any) -> list[dict[str, Any]]: return [base_row(**{**r, **values}) for r in everything] assert h1(broken(cpu_worker_over="0.5")).verdict()[0] == REFUTED assert ( h2(broken(duration_p50_ms="300", memory_peak_mib="1100")).verdict()[0] == REFUTED ) assert h3(broken(peak_broker_ready="60")).verdict()[0] == REFUTED assert h4(broken(ready_over_p95_s="60")[:1]).verdict()[0] == REFUTED assert h5(broken(retry_after="1"), 250).verdict()[0] == REFUTED assert h6(broken(flags="lost")).verdict()[0] == REFUTED assert h7(broken(ctrl_write_over_p95="500")).verdict()[0] == REFUTED assert h7(broken(ctrl_read_over_s5xx="1")).verdict()[0] == REFUTED assert h8(broken(attempts_over_one="5")).verdict()[0] == REFUTED assert ( h9(broken(first_busy_s="50", first_backlog_s="10"), 250).verdict()[0] == REFUTED ) # рекомендация: (2, 2) и другая assert recommend(matrix_fixture())["setting"] == (2, 2) assert recommend(empty)["setting"] is None # настройка, не снятая на части классов, не кандидат, даже если на снятых # классах она лучшая: данных об остальных нет partly = [ base_row(**{**r, "ready_s_recovery": "3"}) if r["concurrency"] == "1" else r for r in matrix_fixture() ] advice = recommend(partly) assert advice["setting"] == (2, 2), advice assert advice["partial"]["1,1"] == ["lj", "s"], advice assert (1, 1) not in advice["candidates"], advice assert advice["partial"]["2,256"] == ["lj", "m"], advice text = render_summary( {"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}}, partly, ) assert "(1, 1) снята не на всех классах (нет: lj, s)" in text # снятая на всех классах и лучшая по правилу — другая рекомендация other = [ *partly, *[ base_row( cell=f"{klass}-c1p1-r1", config=f"{klass}-c1p1", concurrency="1", prefetch="1", **{"class": klass}, ) for klass in ("s", "lj") ], ] advice = recommend(other) assert advice["setting"] == (1, 1), advice assert "1,1" not in advice["partial"], advice # сводка и графики собираются text = render_summary( {"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}}, everything, ) assert "H9" in text and "Рекомендация" in text pictures = render_plots(everything, []) assert set(pictures) == { "throughput-by-setting.svg", "p95-by-setting.svg", "memory-by-setting.svg", "backlog-gate.svg", } print("overload selftest ok") if __name__ == "__main__": if "--selftest" in sys.argv: selftest() else: print(__doc__)