Files
mtusi2026lab5/bench/overload.py
T
ricoromaandClaude Opus 5.5 d667fa3c57 feat: limit accepted attachment work and add queue overload bench
Reject uploads with 429 at [uploads] max_unfinished_jobs, counted by the existing partial indexes in the membership block. Add the bench_queue profile, hypotheses H1-H9 and the fifth failure-stand API.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 07:30:08 +03:00

2174 lines
72 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль очереди без docker (design change `track1-queue-overload`, решения
5–8, 11, 12): план кругами, разбор строк k6, сэмплера и журналов, строка ячейки и
посекундная шкала, пометки, вердикты H1–H9, правило рекомендации, сводка и
графики. Только stdlib; `--selftest`. Имя не `queue.py`: каталог скрипта первый в
`sys.path`, и `queue` затенил бы модуль stdlib.
"""
from __future__ import annotations
import csv
import json
import random
import sys
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import plots
import reconcile
WINDOWS = ("before", "over", "after")
CONTROL_WINDOWS = ("before", "over", "recovery", "after")
CONTROL_KINDS = {"w": "write", "r": "read", "d": "download"}
CONTROL_STATS = ("n", "errors", "s5xx", "p50", "p95", "p99")
ROLES = (
"worker",
"s3",
"postgres",
"api",
"relay",
"rabbitmq",
"cleaner",
"haproxy",
)
# пометки, при которых метка не зелёная: система потеряла работу (решение 11)
FATAL_FLAGS = ("lost", "stuck", "duplicate", "sha_mismatch")
DEFAULT_SETTING = (2, 2)
MIB = 2**20
CONFIRMED = "подтверждено"
REFUTED = "нет"
NO_DATA = "нет данных"
BACKLOG_RETRY_AFTER = "5"
# запас превышения предела: загрузки в полёте двух реплик (решение 3)
IN_FLIGHT = 40
LEASE_S = 70
# окно CPU S3 блока accept: до срабатывания предела (гипотеза H9)
ACCEPT_S3_WINDOW_S = 30
CELL_FIELDS = [
"cell",
"config",
"block",
"class",
"concurrency",
"prefetch",
"repeat",
"order",
"rate_low",
"rate_high",
"over_s",
"limit",
"action",
"accepted",
"s202",
"s429",
"s5xx",
"s_other",
"s_conn",
"dropped",
"accepted_before",
"accepted_over",
"accepted_after",
"retry_after",
"busy_429",
"backlog_429",
"first_busy_s",
"first_backlog_s",
"upload_before_p50_ms",
"upload_over_p50_ms",
"upload_over_p95_ms",
"ready",
"failed",
"lost",
"stuck",
"duplicate",
"sha_checked",
"sha_bad",
"ready_s_over",
"ready_s_recovery",
"over_end",
"zero_at",
"action_at",
"recovery_s",
"peak_unfinished",
"peak_outbox",
"peak_waiting",
"peak_published",
"peak_processing",
"peak_broker_ready",
"peak_broker_unacked",
"peak_prefetched",
"peak_oldest_s",
"staging_ready_end",
"queue_left",
"ready_before_p50_s",
"ready_before_p95_s",
"ready_before_p99_s",
"ready_over_p50_s",
"ready_over_p95_s",
"ready_over_p99_s",
"ready_after_p50_s",
"ready_after_p95_s",
"ready_after_p99_s",
"ready_before_list",
"ready_after_list",
"queue_wait_p50_ms",
"queue_wait_p95_ms",
"duration_p50_ms",
"render_p50_ms",
"worker_hwm_mib",
"render_peak_p50_mib",
"render_peak_max_mib",
"memory_peak_mib",
"anon_max_mib",
"oom_kill",
"worker_restarts",
*[f"cpu_{role}_over" for role in ROLES],
*[f"cpu_{role}_recovery" for role in ROLES],
*[f"throttle_{role}_over" for role in ("worker", "s3")],
"cpu_s3_accept",
"cpu_worker_per_job_s",
"broker_memory_mib",
"broker_refused",
"attempts_over_one",
"skipped",
"relay_published",
"relay_republished",
"relay_recovered",
"relay_backlog_full",
"failed_by",
"second_jobs",
"second_share",
*[
f"ctrl_{kind}_{window}_{stat}"
for kind in CONTROL_KINDS.values()
for window in CONTROL_WINDOWS
for stat in CONTROL_STATS
],
"swapouts",
"lb_down",
"flags",
]
TIMELINE_FIELDS = [
"cell",
"t",
"window",
"outbox",
"waiting",
"published",
"processing",
"unfinished",
"broker_ready",
"broker_unacked",
"prefetched",
"oldest_s",
"staging_ready",
"accepted",
"ready",
]
# ------------------------------------------------------------------ план
def config_key(
block: str,
klass: str,
concurrency: int,
prefetch: int,
) -> str:
setting = f"{klass}-c{concurrency}p{prefetch}"
return setting if block == "matrix" else f"{block}-{setting}"
def plan_cells(
configs: Iterable[Mapping[str, Any]],
repeats: int,
seed: int,
) -> list[dict[str, Any]]:
"""Круги: на круге — перестановка конфигураций от зерна; круг не начинается
с той, на которой кончился прошлый (как `scaling.plan_cells`)."""
items = list(configs)
plan: list[dict[str, Any]] = []
previous = None
for round_no in range(1, repeats + 1):
order = list(range(len(items)))
random.Random(seed + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for index in order:
item = dict(items[index])
key = config_key(
item["block"],
item["class"],
item["concurrency"],
item["prefetch"],
)
plan.append(
{
**item,
"cell": f"{key}-r{round_no}",
"config": key,
"repeat": round_no,
"order": len(plan) + 1,
},
)
previous = order[-1] if order else None
return plan
# --------------------------------------------------------- разбор источников
@dataclass(frozen=True)
class Upload:
window: str
start_ms: int
duration_ms: int
status: int
retry_after: str
instance: str
attachment_id: str
@property
def answered(self) -> float:
return (self.start_ms + self.duration_ms) / 1000
@dataclass(frozen=True)
class JobTick:
t: float
outbox: int
waiting: int
published: int
processing: int
oldest_s: float
staging_ready: int
@property
def unfinished(self) -> int:
return self.outbox + self.waiting + self.published + self.processing
@dataclass(frozen=True)
class Meta:
http: int
status: str
error_code: str
variants: int
updated_at: float | None
@dataclass
class Sources:
"""Всё снятое за ячейку; сюда же — готовые разборы журналов."""
uploads: list[Upload]
t0: float
metas: dict[str, Meta]
sha_checks: list[bool]
jobs: list[JobTick]
mq: list[tuple[float, int, int]]
mq_missing: int
cg: dict[str, list[list[tuple[int, ...]]]]
mp: dict[str, list[list[tuple[int, ...]]]]
control: list[Any]
worker_records: list[dict[str, Any]]
second_records: list[dict[str, Any]]
relay_records: list[dict[str, Any]]
api_records: list[dict[str, Any]]
sql: dict[str, Any]
hwm_kib: int | None = None
restarts: int = 0
dropped: int = 0
action_at: float | None = None
swapouts: int | None = None
lb_down: bool = False
storage_down: bool = False
extra: dict[str, Any] = field(default_factory=dict)
def parse_uploads(text: str) -> tuple[float | None, list[Upload]]:
"""Строки `q-t0,<мс>` и `q,<окно>,<мс>,<мс>,<статус>,<RA>,<реплика>,<id>`."""
t0 = None
out = []
for raw in text.splitlines():
line = raw.strip()
if line.startswith("q-t0,"):
t0 = int(line.split(",")[1]) / 1000
elif line.startswith("q,"):
parts = line.split(",")
if len(parts) == 8:
out.append(
Upload(
parts[1],
int(parts[2]),
int(parts[3]),
int(parts[4]),
parts[5],
parts[6],
parts[7],
),
)
return t0, out
def parse_sweep(text: str) -> tuple[dict[str, Meta], list[bool]]:
metas: dict[str, Meta] = {}
checks: list[bool] = []
for raw in text.splitlines():
parts = raw.strip().split(",")
if parts[0] == "qs" and len(parts) == 7:
updated = (
None if parts[6] == "-" else reconcile.timestamp(parts[6])
)
metas[parts[1]] = Meta(
int(parts[2]),
parts[3],
parts[4],
int(parts[5]),
updated,
)
elif parts[0] == "qv" and len(parts) == 4:
checks.append(parts[3] == "1")
return metas, checks
def parse_jobs(text: str) -> list[JobTick]:
out = []
for line in text.splitlines():
parts = line.strip().split("|")
if len(parts) == 8 and parts[0] == "jobs":
out.append(
JobTick(
int(parts[1]) / 1000,
int(parts[2]),
int(parts[3]),
int(parts[4]),
int(parts[5]),
float(parts[6]),
int(parts[7]),
),
)
return out
def parse_mq(text: str) -> tuple[list[tuple[float, int, int]], int]:
"""Строки `mq`: значения и число `missing` (брокер не ответил)."""
out = []
missing = 0
for line in text.splitlines():
parts = line.split(" ")
if parts[0] != "mq":
continue
if len(parts) == 3 and parts[2] == "missing":
missing += 1
elif len(parts) == 4:
out.append(
(
int(parts[1]) / 1e9,
int(float(parts[2])),
int(float(parts[3])),
),
)
return out, missing
def parse_mp(text: str) -> dict[str, list[tuple[int, ...]]]:
"""Строки `mp <нс> <id12> <memory.peak> <anon> <oom_kill>` по контейнерам."""
out: dict[str, list[tuple[int, ...]]] = {}
for line in text.splitlines():
parts = line.split(" ")
if parts[0] == "mp" and len(parts) == 6:
out.setdefault(parts[2], []).append(
(int(parts[1]), *(int(v) for v in parts[3:6])),
)
return out
def split_lives(
points: list[tuple[int, ...]],
key: int = 1,
) -> list[list[tuple[int, ...]]]:
"""Жизни контейнера: перезапуск сбрасывает cgroup того же контейнера, и
счётчик `points[i][key]` падает (решение 10)."""
lives: list[list[tuple[int, ...]]] = []
for point in points:
if lives and point[key] >= lives[-1][-1][key]:
lives[-1].append(point)
else:
lives.append([point])
return lives
def lives_delta(
lives: Iterable[list[tuple[int, ...]]],
w0: float,
w1: float,
key: int = 1,
) -> float | None:
"""Разность счётчика за окно [w0, w1] (секунды) — сумма разностей внутри
жизней, на краях окна — линейно между соседними отсчётами: тик сэмплера не
совпадает с краем, и разность только по внутренним отсчётам занижала бы
короткое окно на ~10 %. Окно не задевает ни одну жизнь — `None`."""
def value_at(life: list[tuple[int, ...]], ns: float) -> float:
for prev, cur in zip(life, life[1:], strict=False):
if prev[0] <= ns <= cur[0]:
share = (ns - prev[0]) / ((cur[0] - prev[0]) or 1)
return prev[key] + (cur[key] - prev[key]) * share
return float(life[-1][key])
total = 0.0
seen = False
for life in lives:
if len(life) < 2:
continue
a, b = max(w0 * 1e9, life[0][0]), min(w1 * 1e9, life[-1][0])
if b > a:
total += value_at(life, b) - value_at(life, a)
seen = True
return total if seen else None
def role_cores(
lives_by_container: Iterable[list[list[tuple[int, ...]]]],
w0: float,
w1: float,
) -> float | None:
"""Ядер роли за окно: сумма по контейнерам, `usage_usec` — столбец 1."""
if w1 <= w0:
return None
total = None
for lives in lives_by_container:
delta = lives_delta(lives, w0, w1)
if delta is not None:
total = (total or 0.0) + delta
return None if total is None else round(total / 1e6 / (w1 - w0), 4)
def throttle_share(
lives: Iterable[list[tuple[int, ...]]],
w0: float,
w1: float,
) -> float | None:
"""Доля периодов с троттлингом: столбцы 2 (`nr_periods`), 3 (`nr_throttled`)."""
lives = list(lives)
periods = lives_delta(lives, w0, w1, key=2)
throttled = lives_delta(lives, w0, w1, key=3)
if not periods or throttled is None:
return None
return round(throttled / periods, 4)
def memory_by_lives(
lives: Iterable[list[tuple[int, ...]]],
) -> tuple[float | None, float | None, int]:
"""`memory.peak` и `anon` — наибольшее по жизням, `oom_kill` — сумма: у
каждой жизни считается наибольшее значение."""
peak = anon = None
kills = 0
for life in lives:
peak = max([p[1] for p in life] + ([peak] if peak is not None else []))
anon = max([p[2] for p in life] + ([anon] if anon is not None else []))
kills += max(p[3] for p in life)
return (
None if peak is None else round(peak / MIB, 1),
None if anon is None else round(anon / MIB, 1),
kills,
)
def events(
records: Iterable[Mapping[str, Any]],
name: str,
) -> list[Mapping[str, Any]]:
return [item for item in records if item.get("event") == name]
def rejected_by_reason(
records: Iterable[Mapping[str, Any]],
reason: str,
) -> list[float]:
"""Моменты отказов «Rejected attachment» с этой причиной, по времени."""
return sorted(
reconcile.timestamp(item["timestamp"])
for item in events(records, "Rejected attachment")
if item.get("reason") == reason and item.get("timestamp")
)
def rounded(value: float | None, digits: int = 2) -> float | None:
return None if value is None else round(value, digits)
def percentile(values: list[float], q: float, digits: int = 2) -> float | None:
return rounded(reconcile.percentile(values, q), digits)
# ---------------------------------------------------------------- ячейка
def control_columns(
requests: list[Any],
bounds: Mapping[str, tuple[float, float]],
) -> dict[str, Any]:
out: dict[str, Any] = {}
for kind, name in CONTROL_KINDS.items():
for window, (w0, w1) in bounds.items():
stats = reconcile.window_stats(requests, kind, w0, w1)
prefix = f"ctrl_{name}_{window}"
out[f"{prefix}_n"] = stats["responses"]
out[f"{prefix}_errors"] = stats["errors"]
out[f"{prefix}_s5xx"] = stats["s5xx"]
out[f"{prefix}_p50"] = rounded(stats["p50_ms"], 1)
out[f"{prefix}_p95"] = rounded(stats["p95_ms"], 1)
out[f"{prefix}_p99"] = rounded(stats["p99_ms"], 1)
return out
def cell_row(cell: Mapping[str, Any], src: Sources) -> dict[str, Any]:
"""Строка `cells.csv`. Окна — от начала `q-t0`: `before`, `over`, затем
`recovery` до нуля незавершённых."""
before_s = cell["before_s"]
over_s = cell["over_s"]
over_start = src.t0 + before_s
over_end = over_start + over_s
answers = src.uploads
in_over = [u for u in answers if u.window == "over"]
# последняя загрузка окна: её ответ — конец приёма
load_end = max((u.answered for u in in_over), default=over_end)
zero_at = next(
(t.t for t in src.jobs if t.t >= load_end and t.unfinished == 0),
None,
)
recovery_end = zero_at if zero_at is not None else load_end
accepted = [u for u in answers if u.status == 202]
statuses = [u.status for u in answers]
metas = src.metas
ready = [m for m in metas.values() if m.status == "ready"]
failed = [m for m in metas.values() if m.status == "failed"]
lost = sum(
1
for u in accepted
if u.attachment_id not in metas or metas[u.attachment_id].http != 200
)
stuck = sum(
1
for u in accepted
if u.attachment_id in metas
and metas[u.attachment_id].http == 200
and metas[u.attachment_id].status not in ("ready", "failed")
)
ready_at = [m.updated_at for m in ready if m.updated_at is not None]
def per_second(w0: float, w1: float) -> float | None:
if w1 <= w0:
return None
return round(sum(1 for t in ready_at if w0 <= t < w1) / (w1 - w0), 3)
waits: dict[str, list[float]] = {window: [] for window in WINDOWS}
for upload in accepted:
meta = metas.get(upload.attachment_id)
if meta and meta.status == "ready" and meta.updated_at is not None:
waits[upload.window].append(meta.updated_at - upload.answered)
jobs_over = [t for t in src.jobs if src.t0 <= t.t <= recovery_end]
peak = {
name: max((getattr(t, name) for t in jobs_over), default=None)
for name in (
"unfinished",
"outbox",
"waiting",
"published",
"processing",
"oldest_s",
)
}
mq_inside = [m for m in src.mq if src.t0 <= m[0] <= recovery_end]
# неподтверждённые против взятых: ближайший по времени тик заданий
prefetched = 0
for moment, _, unacked in mq_inside:
nearest = min(src.jobs, key=lambda t: abs(t.t - moment), default=None)
if nearest is not None:
prefetched = max(prefetched, unacked - nearest.processing)
worker_ready = events(src.worker_records, "Attachment became ready")
second_ready = events(src.second_records, "Attachment became ready")
both_ready = [*worker_ready, *second_ready]
started_jobs = events(
[*src.worker_records, *src.second_records],
"Started job attempt",
)
queue_waits = [
float(item["queue_wait_ms"])
for item in started_jobs
if item.get("queue_wait_ms") is not None
]
durations = [float(i["duration_ms"]) for i in both_ready]
renders = [float(i["render_ms"]) for i in both_ready if i.get("render_ms")]
render_peaks = [
float(i["render_peak_kib"]) / 1024
for i in both_ready
if i.get("render_peak_kib")
]
skipped: dict[str, int] = {}
for item in events(src.worker_records, "Skipped job message"):
reason = str(item.get("reason"))
skipped[reason] = skipped.get(reason, 0) + 1
relay_passes = events(src.relay_records, "Relay pass")
busy = rejected_by_reason(src.api_records, "busy")
backlog = rejected_by_reason(src.api_records, "backlog")
retry_values = sorted(
{u.retry_after for u in answers if u.status == 429},
)
worker_lives = src.cg.get("worker", [])
# только основной контейнер worker: лимит памяти у второго свой
peak_mem, anon_mem, kills = memory_by_lives(src.mp.get("worker", []))
# процесс 1, завершённый ядром, уносит cgroup раньше следующего тика: его
# счётчик не прочитать, а политика перезапускает только аварийный выход
kills += src.restarts
per_job = None
delta = lives_delta(
[life for lives in worker_lives for life in lives],
src.t0,
recovery_end,
)
if delta is not None and ready_at:
per_job = round(delta / 1e6 / len(ready_at), 3)
row: dict[str, Any] = {
key: cell.get(key)
for key in (
"cell",
"config",
"block",
"class",
"concurrency",
"prefetch",
"repeat",
"order",
"rate_low",
"rate_high",
"over_s",
"limit",
"action",
)
}
row.update(
{
"accepted": len(accepted),
"s202": statuses.count(202),
"s429": statuses.count(429),
"s5xx": sum(1 for s in statuses if 500 <= s < 600),
"s_other": sum(
1
for s in statuses
if s not in (0, 202, 429) and not 500 <= s < 600
),
"s_conn": statuses.count(0),
"dropped": src.dropped,
"accepted_before": sum(
1 for u in accepted if u.window == "before"
),
"accepted_over": sum(1 for u in accepted if u.window == "over"),
"accepted_after": sum(1 for u in accepted if u.window == "after"),
"retry_after": " ".join(retry_values),
"busy_429": len(busy),
"backlog_429": len(backlog),
"first_busy_s": rounded(busy[0] - over_start) if busy else None,
"first_backlog_s": (
rounded(backlog[0] - over_start) if backlog else None
),
"upload_before_p50_ms": percentile(
[u.duration_ms for u in accepted if u.window == "before"],
50,
1,
),
"upload_over_p50_ms": percentile(
[u.duration_ms for u in accepted if u.window == "over"],
50,
1,
),
"upload_over_p95_ms": percentile(
[u.duration_ms for u in accepted if u.window == "over"],
95,
1,
),
"ready": len(ready),
"failed": len(failed),
"lost": lost,
"stuck": stuck,
"duplicate": src.sql.get("extra_jobs", 0)
+ src.sql.get("bad_variants", 0),
"sha_checked": len(src.sha_checks),
"sha_bad": src.sha_checks.count(False),
"ready_s_over": per_second(over_start, over_end),
"ready_s_recovery": per_second(over_end, recovery_end),
"over_end": round(over_end, 3),
"zero_at": rounded(zero_at, 3),
"action_at": rounded(src.action_at, 3),
"recovery_s": (
rounded(zero_at - load_end) if zero_at is not None else None
),
"peak_unfinished": peak["unfinished"],
"peak_outbox": peak["outbox"],
"peak_waiting": peak["waiting"],
"peak_published": peak["published"],
"peak_processing": peak["processing"],
"peak_broker_ready": max((m[1] for m in mq_inside), default=None),
"peak_broker_unacked": max(
(m[2] for m in mq_inside),
default=None,
),
"peak_prefetched": prefetched,
"peak_oldest_s": peak["oldest_s"],
"staging_ready_end": (
src.jobs[-1].staging_ready if src.jobs else None
),
"queue_left": sum(src.mq[-1][1:]) if src.mq else None,
"ready_before_list": " ".join(f"{v:.2f}" for v in waits["before"]),
"ready_after_list": " ".join(f"{v:.2f}" for v in waits["after"]),
"queue_wait_p50_ms": percentile(queue_waits, 50, 0),
"queue_wait_p95_ms": percentile(queue_waits, 95, 0),
"duration_p50_ms": percentile(durations, 50, 0),
"render_p50_ms": percentile(renders, 50, 0),
"worker_hwm_mib": (
round(src.hwm_kib / 1024, 1) if src.hwm_kib else None
),
"render_peak_p50_mib": percentile(render_peaks, 50, 1),
"render_peak_max_mib": rounded(
max(render_peaks, default=None),
1,
),
"memory_peak_mib": peak_mem,
"anon_max_mib": anon_mem,
"oom_kill": kills,
"worker_restarts": src.restarts,
"cpu_s3_accept": role_cores(
src.cg.get("s3", []),
over_start,
min(over_end, over_start + ACCEPT_S3_WINDOW_S),
),
"cpu_worker_per_job_s": per_job,
"broker_memory_mib": src.extra.get("broker_memory_mib"),
"broker_refused": src.extra.get("broker_refused", 0),
"attempts_over_one": src.sql.get("attempts_over_one"),
"skipped": json.dumps(skipped, sort_keys=True),
"relay_published": sum(
i.get("published", 0) for i in relay_passes
),
"relay_republished": sum(
i.get("republished", 0) for i in relay_passes
),
"relay_recovered": sum(
i.get("recovered", 0) for i in relay_passes
),
"relay_backlog_full": len(
events(src.relay_records, "Relay backlog is full"),
),
"failed_by": json.dumps(src.sql.get("failed", {}), sort_keys=True),
"second_jobs": len(second_ready) if src.second_records else None,
"swapouts": src.swapouts,
"lb_down": int(src.lb_down),
},
)
for window in WINDOWS:
for q in (50, 95, 99):
row[f"ready_{window}_p{q}_s"] = percentile(waits[window], q)
for role in ROLES:
lives = src.cg.get(role, [])
row[f"cpu_{role}_over"] = role_cores(lives, over_start, over_end)
row[f"cpu_{role}_recovery"] = role_cores(lives, over_end, recovery_end)
for role in ("worker", "s3"):
row[f"throttle_{role}_over"] = throttle_share(
[life for lives in src.cg.get(role, []) for life in lives],
over_start,
over_end,
)
action_at = src.extra.get("second_started_at")
if action_at is not None and src.second_records:
# только разбор накопленного: окно after делится поровну при любом
# prefetch и смазывало бы опыт (design, «Уточнения apply», п. 2)
def in_backlog(item: Mapping[str, Any]) -> bool:
moment = reconcile.timestamp(item["timestamp"])
return action_at <= moment <= recovery_end
after = sum(1 for item in both_ready if in_backlog(item))
taken = sum(1 for item in second_ready if in_backlog(item))
row["second_jobs"] = taken
row["second_share"] = round(taken / after, 3) if after else None
after_start = src.extra.get("after_start")
after_end = src.extra.get("after_end")
bounds = {
"before": (src.t0, over_start),
"over": (over_start, over_end),
"recovery": (over_end, recovery_end),
"after": (after_start or recovery_end, after_end or recovery_end),
}
row.update(control_columns(src.control, bounds))
row["flags"] = " ".join(
cell_flags(row, zero_at is None, src.storage_down),
)
return row
def cell_flags(
row: Mapping[str, Any],
not_recovered: bool,
storage_down: bool,
) -> list[str]:
"""Пометки решения 11."""
flags = []
if num(row.get("oom_kill")):
flags.append("oom")
if num(row.get("worker_restarts")):
flags.append("worker_restart")
if num(row.get("lost")):
flags.append("lost")
if num(row.get("stuck")):
flags.append("stuck")
if num(row.get("duplicate")):
flags.append("duplicate")
if num(row.get("sha_bad")):
flags.append("sha_mismatch")
if not_recovered:
flags.append("not_recovered")
if num(row.get("failed")):
flags.append("failed")
if num(row.get("dropped")):
flags.append("dropped")
if num(row.get("lb_down")):
flags.append("lb_down")
if num(row.get("swapouts")):
flags.append("host_swap")
if storage_down:
flags.append("storage_down")
return flags
def timeline_rows(
cell: Mapping[str, Any],
src: Sources,
) -> list[dict[str, Any]]:
"""Посекундная шкала от `q-t0`: окно, очередь в трёх местах, принято и
готово за секунду."""
over_start = src.t0 + cell["before_s"]
over_end = over_start + cell["over_s"]
after_start = src.extra.get("after_start")
accepted: dict[int, int] = {}
for upload in src.uploads:
if upload.status == 202:
second = int(upload.answered - src.t0)
accepted[second] = accepted.get(second, 0) + 1
ready: dict[int, int] = {}
for meta in src.metas.values():
if meta.status == "ready" and meta.updated_at is not None:
second = int(meta.updated_at - src.t0)
ready[second] = ready.get(second, 0) + 1
mq = {int(m[0] - src.t0): m for m in src.mq}
rows = []
for tick in src.jobs:
if tick.t < src.t0:
continue
second = int(tick.t - src.t0)
if tick.t < over_start:
window = "before"
elif tick.t < over_end:
window = "over"
elif after_start is None or tick.t < after_start:
window = "recovery"
else:
window = "after"
broker = mq.get(second)
rows.append(
{
"cell": cell["cell"],
"t": second,
"window": window,
"outbox": tick.outbox,
"waiting": tick.waiting,
"published": tick.published,
"processing": tick.processing,
"unfinished": tick.unfinished,
"broker_ready": broker[1] if broker else None,
"broker_unacked": broker[2] if broker else None,
"prefetched": (
max(0, broker[2] - tick.processing) if broker else None
),
"oldest_s": tick.oldest_s,
"staging_ready": tick.staging_ready,
"accepted": accepted.get(second, 0),
"ready": ready.get(second, 0),
},
)
return rows
# --------------------------------------------------------------- вердикты
def read_csv(path: Path) -> list[dict[str, Any]]:
if not path.exists():
return []
with path.open(encoding="utf-8", newline="") as file:
return list(csv.DictReader(file))
def num(value: Any) -> float | None:
return reconcile.num(value)
def med(rows: Iterable[Mapping[str, Any]], key: str) -> float | None:
return reconcile.med(row.get(key) for row in rows)
def select(
rows: Iterable[Mapping[str, Any]],
*,
block: str = "matrix",
klass: str | None = None,
setting: tuple[int, int] | None = None,
) -> list[Mapping[str, Any]]:
found = []
for row in rows:
if row.get("block") != block:
continue
if klass is not None and row.get("class") != klass:
continue
if setting is not None and (
num(row.get("concurrency")) != setting[0]
or num(row.get("prefetch")) != setting[1]
):
continue
found.append(row)
return found
def flagged(row: Mapping[str, Any], flag: str) -> bool:
return flag in (row.get("flags") or "").split()
def classes_of(rows: Iterable[Mapping[str, Any]]) -> list[str]:
return sorted({r["class"] for r in rows if r.get("block") == "matrix"})
def ratio(a: float | None, b: float | None) -> float | None:
return None if a is None or not b else a / b
class Check:
"""Условия гипотезы: `нет данных`, если нечем проверить условие; `нет` — если
условие нарушено, и причина названа."""
def __init__(self) -> None:
self.failed: list[str] = []
self.missing: list[str] = []
self.passed = 0
def within(
self,
name: str,
value: float | None,
low: float | None,
high: float | None,
) -> None:
if value is None:
self.missing.append(name)
elif (low is not None and value < low) or (
high is not None and value > high
):
self.failed.append(f"{name} = {value:.3g}")
else:
self.passed += 1
def true(self, name: str, value: bool | None) -> None:
if value is None:
self.missing.append(name)
elif not value:
self.failed.append(name)
else:
self.passed += 1
def verdict(self) -> tuple[str, str]:
if self.failed:
return REFUTED, "; ".join(self.failed[:6])
if self.missing:
return NO_DATA, "нет: " + ", ".join(self.missing[:6])
if not self.passed:
return NO_DATA, "нет ячеек"
return CONFIRMED, f"условий: {self.passed}"
def h1(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for klass in classes_of(rows):
base = select(rows, klass=klass, setting=(2, 2))
wide = select(rows, klass=klass, setting=(4, 4))
check.within(
f"{klass}: (4,4)/(2,2) throughput",
ratio(
med(wide, "ready_s_recovery"),
med(base, "ready_s_recovery"),
),
None,
1.05,
)
for row in select(rows):
if (num(row.get("concurrency")) or 0) >= 2:
check.within(
f"{row['cell']}: CPU worker",
num(row.get("cpu_worker_over")),
0.85,
None,
)
for klass in ("m", "lp"):
base = select(rows, klass=klass, setting=(2, 2))
small = select(rows, klass=klass, setting=(1, 1))
check.within(
f"{klass}: (1,1)/(2,2) throughput",
ratio(
med(small, "ready_s_recovery"),
med(base, "ready_s_recovery"),
),
None,
1.0,
)
return check
def h2(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
limit = 1024.0
for klass in classes_of(rows):
base = select(rows, klass=klass, setting=(2, 2))
wide = select(rows, klass=klass, setting=(4, 4))
check.within(
f"{klass}: duration (4,4)/(2,2)",
ratio(med(wide, "duration_p50_ms"), med(base, "duration_p50_ms")),
1.5,
None,
)
for row in base:
check.within(
f"{row['cell']}: memory.peak",
num(row.get("memory_peak_mib")),
None,
limit - 0.001,
)
check.true(f"{row['cell']}: без oom", not flagged(row, "oom"))
wide_lp = select(rows, klass="lp", setting=(4, 4))
for row in wide_lp:
check.within(
f"{row['cell']}: memory.peak",
num(row.get("memory_peak_mib")),
0.95 * limit,
None,
)
check.true(
"lp (4,4): oom хотя бы в одном повторе",
any(flagged(r, "oom") for r in wide_lp) if wide_lp else None,
)
wide_lj = select(rows, klass="lj", setting=(4, 4))
for row in wide_lj:
check.true(f"{row['cell']}: без oom", not flagged(row, "oom"))
check.within(
f"{row['cell']}: memory.peak",
num(row.get("memory_peak_mib")),
None,
500,
)
if not wide_lj:
check.missing.append("lj (4,4)")
return check
def h3(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for klass in ("s", "lp"):
base = select(rows, klass=klass, setting=(2, 2))
big = select(rows, klass=klass, setting=(2, 256))
check.within(
f"{klass}: throughput (2,256)/(2,2)",
ratio(med(big, "ready_s_recovery"), med(base, "ready_s_recovery")),
0.85,
1.15,
)
check.within(
f"{klass}: p95 (2,256)/(2,2)",
ratio(med(big, "ready_over_p95_s"), med(base, "ready_over_p95_s")),
0.8,
1.25,
)
for row in big:
check.within(
f"{row['cell']}: broker_ready / пик",
ratio(
num(row.get("peak_broker_ready")),
num(row.get("peak_unfinished")),
),
None,
0.10,
)
for row in base:
check.within(
f"{row['cell']}: broker_ready / пик",
ratio(
num(row.get("peak_broker_ready")),
num(row.get("peak_unfinished")),
),
0.70,
None,
)
p2 = select(rows, block="w2-p2")
p256 = select(rows, block="w2-p256")
check.within(
"доля второго при prefetch 2",
med(p2, "second_share"),
0.30,
None,
)
check.within(
"доля второго при prefetch 256",
med(p256, "second_share"),
None,
0.05,
)
check.within(
"восстановление p256 / p2",
ratio(med(p256, "recovery_s"), med(p2, "recovery_s")),
1.4,
None,
)
return check
def expected_wait(row: Mapping[str, Any]) -> float | None:
return ratio(num(row.get("peak_unfinished")), num(row.get("ready_s_over")))
def h4(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for row in select(rows):
if flagged(row, "oom"):
continue
check.within(
f"{row['cell']}: p95 / (пик / throughput)",
ratio(num(row.get("ready_over_p95_s")), expected_wait(row)),
0.8,
1.25,
)
return check
def h5(rows: list[Mapping[str, Any]], limit: int) -> Check:
check = Check()
on = select(rows, block="gate-on")
off = select(rows, block="gate-off")
if not on or not off:
check.missing.append("блоки gate-on и gate-off")
return check
for row in on:
check.within(
f"{row['cell']}: пик",
num(row.get("peak_unfinished")),
None,
limit + IN_FLIGHT,
)
check.true(f"{row['cell']}: 429 есть", (num(row.get("s429")) or 0) > 0)
check.true(
f"{row['cell']}: Retry-After 5",
row.get("retry_after") == BACKLOG_RETRY_AFTER,
)
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
check.true(
f"{row['cell']}: 429 = записи backlog",
num(row.get("s429")) == num(row.get("backlog_429")),
)
check.within(
f"{row['cell']}: p95 / (предел / throughput)",
ratio(
num(row.get("ready_over_p95_s")),
ratio(float(limit), num(row.get("ready_s_over"))),
),
0.8,
1.25,
)
for row in off:
check.within(
f"{row['cell']}: пик без предела",
num(row.get("peak_unfinished")),
1.5 * limit,
None,
)
check.within(
"p95 on / off",
ratio(med(on, "ready_over_p95_s"), med(off, "ready_over_p95_s")),
None,
0.75,
)
check.within(
"восстановление on / off",
ratio(med(on, "recovery_s"), med(off, "recovery_s")),
None,
0.75,
)
a = med(on, "upload_before_p50_ms")
b = med(off, "upload_before_p50_ms")
check.within(
"загрузка before p50 on − off, мс",
None if a is None or b is None else a - b,
None,
3,
)
return check
def pooled(rows: Iterable[Mapping[str, Any]], key: str) -> list[float]:
values: list[float] = []
for row in rows:
values += [float(v) for v in (row.get(key) or "").split()]
return values
def h6(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for row in rows:
for flag in FATAL_FLAGS:
check.true(f"{row['cell']}: нет {flag}", not flagged(row, flag))
if num(row.get("failed")):
only_exhausted = all(
key.startswith("retry_exhausted")
for key in json.loads(row.get("failed_by") or "{}")
)
check.true(
f"{row['cell']}: failed только при oom и retry_exhausted",
flagged(row, "oom") and only_exhausted,
)
if flagged(row, "oom") or row.get("block") not in (
"matrix",
"gate-on",
"gate-off",
):
continue
bound = expected_wait(row)
check.within(
f"{row['cell']}: восстановление / (пик / throughput)",
ratio(num(row.get("recovery_s")), bound),
None,
1.3,
)
by_config: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
by_config.setdefault(row["config"], []).append(row)
for config, items in by_config.items():
before = reconcile.percentile(pooled(items, "ready_before_list"), 95)
after = reconcile.percentile(pooled(items, "ready_after_list"), 95)
check.within(
f"{config}: p95 after / before",
ratio(after, before),
None,
1.5,
)
return check
def h7(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
for row in rows:
if row.get("block") == "accept":
continue
for window in CONTROL_WINDOWS:
for kind, limit in (
("write", 200),
("read", 200),
("download", 500),
):
value = num(row.get(f"ctrl_{kind}_{window}_p95"))
if value is not None:
check.within(
f"{row['cell']}: {kind} {window} p95",
value,
None,
limit,
)
check.true(
f"{row['cell']}: {kind} {window} 5xx нет",
not num(row.get(f"ctrl_{kind}_{window}_s5xx")),
)
check.true(
f"{row['cell']}: все реплики в LB",
not num(row.get("lb_down")),
)
return check
def h8(rows: list[Mapping[str, Any]]) -> Check:
check = Check()
kills = select(rows, block="kill-worker")
restarts = select(rows, block="restart-broker")
if not kills:
check.missing.append("блок kill-worker")
if not restarts:
check.missing.append("блок restart-broker")
for row in kills:
check.true(
f"{row['cell']}: всё ready",
num(row.get("ready")) == num(row.get("accepted")),
)
check.within(
f"{row['cell']}: заданий с повтором",
num(row.get("attempts_over_one")),
None,
2,
)
end = num(row.get("over_end"))
action = num(row.get("action_at"))
wait = expected_wait(row)
zero = num(row.get("zero_at"))
if None in (end, action, wait, zero):
check.missing.append(f"{row['cell']}: моменты")
continue
check.within(
f"{row['cell']}: разбор до границы, с",
zero - max(end + 1.3 * wait, action + LEASE_S + 15),
None,
0,
)
for row in restarts:
check.true(
f"{row['cell']}: всё ready",
num(row.get("ready")) == num(row.get("accepted")),
)
check.within(
f"{row['cell']}: заданий с повтором",
num(row.get("attempts_over_one")),
None,
0,
)
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
wait = expected_wait(row)
recovery = num(row.get("recovery_s"))
check.within(
f"{row['cell']}: восстановление, с",
None
if wait is None or recovery is None
else recovery - (1.3 * wait + 15),
None,
0,
)
return check
def h9(rows: list[Mapping[str, Any]], limit: int) -> Check:
check = Check()
cells = select(rows, block="accept")
if not cells:
check.missing.append("блок accept")
for row in cells:
busy = num(row.get("first_busy_s"))
backlog = num(row.get("first_backlog_s"))
check.true(
f"{row['cell']}: первый busy раньше backlog",
None if busy is None or backlog is None else busy < backlog,
)
check.true(f"{row['cell']}: 5xx нет", not num(row.get("s5xx")))
check.within(
f"{row['cell']}: CPU S3 первые 30 с",
num(row.get("cpu_s3_accept")),
0.85,
None,
)
check.within(
f"{row['cell']}: принято/с",
ratio(num(row.get("accepted_over")), num(row.get("over_s"))),
6,
None,
)
check.within(
f"{row['cell']}: пик",
num(row.get("peak_unfinished")),
None,
limit + IN_FLIGHT,
)
check.within(
f"{row['cell']}: готовых/с в окне",
num(row.get("ready_s_over")),
None,
1.5,
)
check.within(
f"{row['cell']}: готовых/с при восстановлении",
num(row.get("ready_s_recovery")),
2.4,
None,
)
for window in CONTROL_WINDOWS:
for kind, bound in (
("write", 200),
("read", 200),
("download", 1000),
):
value = num(row.get(f"ctrl_{kind}_{window}_p95"))
if value is not None:
check.within(
f"{row['cell']}: {kind} {window} p95",
value,
None,
bound,
)
check.true(
f"{row['cell']}: {kind} {window} 5xx нет",
not num(row.get(f"ctrl_{kind}_{window}_s5xx")),
)
return check
def verdicts(
rows: list[Mapping[str, Any]],
limit: int,
) -> dict[str, tuple[str, str]]:
return {
"H1": h1(rows).verdict(),
"H2": h2(rows).verdict(),
"H3": h3(rows).verdict(),
"H4": h4(rows).verdict(),
"H5": h5(rows, limit).verdict(),
"H6": h6(rows).verdict(),
"H7": h7(rows).verdict(),
"H8": h8(rows).verdict(),
"H9": h9(rows, limit).verdict(),
}
def settings_of(rows: Iterable[Mapping[str, Any]]) -> list[tuple[int, int]]:
return sorted(
{
(int(num(r["concurrency"])), int(num(r["prefetch"])))
for r in rows
if r.get("block") == "matrix"
},
)
def recommend(rows: list[Mapping[str, Any]]) -> dict[str, Any]:
"""Правило решения 12: кандидат — без `oom` и `failed` на всех классах, с
throughput не ниже 90 % лучшей на каждом классе; из кандидатов — наименьшая
конкуренция; `prefetch` равен конкуренции."""
settings = settings_of(rows)
candidates = []
reasons: dict[str, str] = {}
for setting in settings:
problem = None
for klass in classes_of(rows):
cells = select(rows, klass=klass, setting=setting)
if not cells:
continue
if any(flagged(r, "oom") or flagged(r, "failed") for r in cells):
problem = f"{klass}: oom или failed"
break
best = max(
(
value
for other in settings
if (
value := med(
select(rows, klass=klass, setting=other),
"ready_s_recovery",
)
)
is not None
),
default=None,
)
own = med(cells, "ready_s_recovery")
if best is None or own is None or own < 0.9 * best:
problem = f"{klass}: throughput ниже 90 % лучшей"
break
if problem:
reasons[f"{setting[0]},{setting[1]}"] = problem
else:
candidates.append(setting)
if not candidates:
return {"setting": None, "candidates": [], "rejected": reasons}
concurrency = min(setting[0] for setting in candidates)
return {
"setting": (concurrency, concurrency),
"candidates": candidates,
"rejected": reasons,
}
# ----------------------------------------------------------------- сводка
def fmt(value: Any, digits: int = 1) -> str:
number = num(value)
return "—" if number is None else f"{number:.{digits}f}"
def spread(
rows: Iterable[Mapping[str, Any]],
key: str,
digits: int = 1,
) -> str:
return reconcile.spread((row.get(key) for row in rows), digits)
def config_table(rows: list[Mapping[str, Any]]) -> list[str]:
head = (
"| Конфигурация | Ячеек | Принято | 429 | Готовых/с: окно | восстановление | "
"Пик незаверш. | Пик брокера | До `ready` over p50 / p95 / p99, с | "
"Восст., с | `memory.peak`, МиБ | VmHWM, МиБ | Процесс p50 / макс., МиБ | "
"CPU worker, S3 | Троттлинг worker | OOM | Перезапуски | Пометки |"
)
lines = [head, "|" + "---|" * 18]
by_config: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
by_config.setdefault(row["config"], []).append(row)
for config in sorted(by_config):
items = by_config[config]
flags = sorted(
{f for r in items for f in (r.get("flags") or "").split()},
)
lines.append(
f"| `{config}` | {len(items)} | {spread(items, 'accepted', 0)} | "
f"{spread(items, 's429', 0)} | {spread(items, 'ready_s_over', 2)} | "
f"{spread(items, 'ready_s_recovery', 2)} | "
f"{spread(items, 'peak_unfinished', 0)} | "
f"{spread(items, 'peak_broker_ready', 0)} | "
f"{fmt(med(items, 'ready_over_p50_s'))} / "
f"{fmt(med(items, 'ready_over_p95_s'))} / "
f"{fmt(med(items, 'ready_over_p99_s'))} | "
f"{spread(items, 'recovery_s', 1)} | "
f"{spread(items, 'memory_peak_mib', 0)} | "
f"{fmt(med(items, 'worker_hwm_mib'), 0)} | "
f"{fmt(med(items, 'render_peak_p50_mib'), 0)} / "
f"{fmt(med(items, 'render_peak_max_mib'), 0)} | "
f"{fmt(med(items, 'cpu_worker_over'), 2)}, "
f"{fmt(med(items, 'cpu_s3_over'), 2)} | "
f"{fmt(med(items, 'throttle_worker_over'), 2)} | "
f"{sum(int(num(r.get('oom_kill')) or 0) for r in items)} | "
f"{sum(int(num(r.get('worker_restarts')) or 0) for r in items)} | "
f"{' '.join(flags) or '—'} |",
)
return lines
def control_table(rows: list[Mapping[str, Any]]) -> list[str]:
lines = [
"| Конфигурация | Окно | Запись p50 / p95 / p99, мс | Чтение p50 / p95 / p99, "
"мс | Скачивание p50 / p95 / p99, мс | Ответов | 5xx |",
"|" + "---|" * 7,
]
by_config: dict[str, list[Mapping[str, Any]]] = {}
for row in rows:
by_config.setdefault(row["config"], []).append(row)
for config in sorted(by_config):
items = by_config[config]
for window in CONTROL_WINDOWS:
parts = []
total = errors = 0.0
for kind in CONTROL_KINDS.values():
prefix = f"ctrl_{kind}_{window}"
parts.append(
" / ".join(
fmt(med(items, f"{prefix}_{stat}"))
for stat in ("p50", "p95", "p99")
),
)
total += sum(num(r.get(f"{prefix}_n")) or 0 for r in items)
errors += sum(num(r.get(f"{prefix}_s5xx")) or 0 for r in items)
lines.append(
f"| `{config}` | {window} | {parts[0]} | {parts[1]} | {parts[2]} "
f"| {int(total)} | {int(errors)} |",
)
return lines
def block_table(rows: list[Mapping[str, Any]]) -> list[str]:
lines = [
"| Ячейка | Принято | 429 busy / backlog | Первый busy / backlog, с | Пик | "
"Готовых/с окно / восст. | Восст., с | Повторов | Второй worker: задания, доля "
"| Пометки |",
"|" + "---|" * 10,
]
for row in sorted(rows, key=lambda r: r["cell"]):
if row.get("block") == "matrix":
continue
lines.append(
f"| `{row['cell']}` | {fmt(row.get('accepted'), 0)} | "
f"{fmt(row.get('busy_429'), 0)} / {fmt(row.get('backlog_429'), 0)} | "
f"{fmt(row.get('first_busy_s'))} / "
f"{fmt(row.get('first_backlog_s'))} | "
f"{fmt(row.get('peak_unfinished'), 0)} | "
f"{fmt(row.get('ready_s_over'), 2)} / "
f"{fmt(row.get('ready_s_recovery'), 2)} | "
f"{fmt(row.get('recovery_s'))} | "
f"{fmt(row.get('attempts_over_one'), 0)} | "
f"{fmt(row.get('second_jobs'), 0)}, "
f"{fmt(row.get('second_share'), 2)} | "
f"{row.get('flags') or '—'} |",
)
return lines
def render_summary(
meta: Mapping[str, Any],
rows: list[Mapping[str, Any]],
) -> str:
info = meta.get("queue", {})
limit = int(info.get("default_limit") or 250)
plan_total = len(info.get("plan", []))
lines = [
f"# Очередь под перегрузкой: метка {meta.get('label')}",
"",
f"commit `{meta.get('git', {}).get('commit')}`, "
f"dirty {meta.get('git', {}).get('dirty')}, статус "
f"{meta.get('status')}, ячеек {len(rows)} из {plan_total}.",
"",
]
fatal = [
f"`{row['cell']}`: {row['flags']}"
for row in rows
if any(flagged(row, flag) for flag in FATAL_FLAGS)
]
if fatal:
lines += ["**Потеряна работа:** " + "; ".join(fatal), ""]
lines += ["## Гипотезы", "", "| № | Вердикт | Детали |", "|---|---|---|"]
for name, (verdict, detail) in verdicts(rows, limit).items():
lines.append(f"| {name} | {verdict} | {detail} |")
advice = recommend(rows)
setting = advice["setting"]
lines += ["", "## Рекомендация", ""]
if setting is None:
lines.append("Кандидатов нет: правило не выбрало настройку.")
else:
keep = setting == DEFAULT_SETTING
lines.append(
f"Конкуренция {setting[0]}, prefetch {setting[1]}: "
+ (
"значения по умолчанию оставить."
if keep
else "отличается от (2, 2), решение — по правилу в отчёте."
),
)
for key, reason in advice["rejected"].items():
lines.append(f"- ({key}) не кандидат: {reason}")
lines += [
"",
"## Ячейки по конфигурациям",
"",
"Значения — медиана (min–max) повторов.",
"",
*config_table(rows),
"",
"## Блоки",
"",
*block_table(rows),
"",
"## Контрольный поток",
"",
*control_table(rows),
"",
]
return "\n".join(lines)
def setting_chart(
rows: list[Mapping[str, Any]],
key: str,
title: str,
unit: str,
) -> str:
"""Столбики: класс → настройки, медиана повторов."""
classes = classes_of(rows)
settings = settings_of(rows)
colors = ["#2a78d6", "#d98a00", "#3a9a5b", "#b0569c"]
width, height = 860, 420
svg = plots.Svg(width, height, title)
left, right, top, bottom = 70, 20, 50, 60
values = [
med(select(rows, klass=klass, setting=setting), key)
for klass in classes
for setting in settings
]
highest = max((v for v in values if v is not None), default=0.0) or 1.0
y_scale = (height - top - bottom) / (highest * 1.1)
group = (width - left - right) / max(1, len(classes))
bar = group / (len(settings) + 1)
base = height - bottom
svg.line(left, base, width - right, base, color=plots.AXIS)
svg.text(left, top - 14, unit, size=12)
for ci, klass in enumerate(classes):
x0 = left + ci * group
svg.text(x0 + group / 2 - 8, base + 22, klass, size=13)
for si, setting in enumerate(settings):
value = med(select(rows, klass=klass, setting=setting), key)
if value is None:
continue
bar_h = value * y_scale
svg.bar(
x0 + si * bar + bar / 2,
base - bar_h,
bar * 0.9,
bar_h,
colors[si % len(colors)],
f"{klass} ({setting[0]}, {setting[1]}): {value:.2f} {unit}",
top=True,
)
for si, setting in enumerate(settings):
x = left + si * 130
svg.bar(
x,
height - 28,
14,
14,
colors[si % len(colors)],
"",
top=False,
)
svg.text(x + 20, height - 16, f"({setting[0]}, {setting[1]})", size=12)
return svg.render()
def gate_chart(timeline: list[Mapping[str, Any]]) -> str:
"""Незавершённые посекундно: блоки предела, первый повтор."""
width, height = 860, 420
svg = plots.Svg(width, height, "Незавершённые задания: предел и без него")
left, right, top, bottom = 70, 20, 50, 50
series: dict[str, list[tuple[float, float]]] = {}
for row in timeline:
cell = row.get("cell", "")
if cell.startswith(("gate-on", "gate-off")) and cell.endswith("-r1"):
t = num(row.get("t"))
value = num(row.get("unfinished"))
if t is not None and value is not None:
series.setdefault(cell, []).append((t, value))
if not series:
svg.text(left, top + 30, "нет данных", size=13)
return svg.render()
max_t = max(p[0] for points in series.values() for p in points) or 1.0
max_v = max(p[1] for points in series.values() for p in points) or 1.0
colors = ["#2a78d6", "#d98a00"]
def sx(t: float) -> float:
return left + t / max_t * (width - left - right)
def sy(v: float) -> float:
return height - bottom - v / (max_v * 1.1) * (height - top - bottom)
svg.line(
left,
height - bottom,
width - right,
height - bottom,
color=plots.AXIS,
)
svg.text(
left,
top - 14,
f"до {max_v:.0f} заданий, до {max_t:.0f} с",
size=12,
)
for index, (cell, points) in enumerate(sorted(series.items())):
svg.polyline(
[(sx(t), sy(v)) for t, v in points],
colors[index % len(colors)],
)
svg.text(left + index * 260, height - 18, cell, size=12)
return svg.render()
def render_plots(
rows: list[Mapping[str, Any]],
timeline: list[Mapping[str, Any]],
) -> dict[str, str]:
return {
"throughput-by-setting.svg": setting_chart(
rows,
"ready_s_recovery",
"Throughput по настройкам и классам",
"готовых/с",
),
"p95-by-setting.svg": setting_chart(
rows,
"ready_over_p95_s",
"p95 времени до ready, окно over",
"с",
),
"memory-by-setting.svg": setting_chart(
rows,
"memory_peak_mib",
"memory.peak worker",
"МиБ",
),
"backlog-gate.svg": gate_chart(timeline),
}
# -------------------------------------------------------------- самопроверка
def base_row(**values: Any) -> dict[str, Any]:
row = {
"cell": "s-c2p2-r1",
"config": "s-c2p2",
"block": "matrix",
"class": "s",
"concurrency": "2",
"prefetch": "2",
"repeat": "1",
"accepted": "100",
"ready": "100",
"failed": "0",
"s429": "0",
"s5xx": "0",
"busy_429": "0",
"backlog_429": "0",
"ready_s_over": "3",
"ready_s_recovery": "3",
"peak_unfinished": "60",
"peak_broker_ready": "50",
"ready_over_p95_s": "20",
"recovery_s": "20",
"memory_peak_mib": "200",
"duration_p50_ms": "300",
"cpu_worker_over": "0.95",
"failed_by": "{}",
"ready_before_list": "1.0 1.1",
"ready_after_list": "1.0 1.2",
"attempts_over_one": "0",
"over_end": "100",
"zero_at": "120",
"action_at": "90",
"second_share": "",
"flags": "",
"lb_down": "0",
"retry_after": "",
}
for key, value in values.items():
row[key] = value
return row
def matrix_fixture() -> list[dict[str, Any]]:
rows = []
for klass in ("s", "m", "lj", "lp"):
rows.append(
base_row(
cell=f"{klass}-c2p2-r1",
config=f"{klass}-c2p2",
**{"class": klass},
),
)
wide = {
"duration_p50_ms": "600",
"memory_peak_mib": "400",
}
flags = ""
if klass == "lp":
wide["memory_peak_mib"] = "1024"
wide["oom_kill"] = "3"
flags = "oom"
rows.append(
base_row(
cell=f"{klass}-c4p4-r1",
config=f"{klass}-c4p4",
concurrency="4",
prefetch="4",
flags=flags,
**{"class": klass, **wide},
),
)
for klass in ("m", "lp"):
rows.append(
base_row(
cell=f"{klass}-c1p1-r1",
config=f"{klass}-c1p1",
concurrency="1",
prefetch="1",
ready_s_recovery="2.6",
cpu_worker_over="0.7",
**{"class": klass},
),
)
for klass in ("s", "lp"):
rows.append(
base_row(
cell=f"{klass}-c2p256-r1",
config=f"{klass}-c2p256",
prefetch="256",
peak_broker_ready="0",
**{"class": klass},
),
)
return rows
def gate_fixture() -> list[dict[str, Any]]:
on = base_row(
cell="gate-on-s-c2p2-r1",
config="gate-on-s-c2p2",
block="gate-on",
peak_unfinished="255",
s429="180",
backlog_429="180",
retry_after="5",
ready_over_p95_s="85",
recovery_s="85",
upload_before_p50_ms="15",
)
off = base_row(
cell="gate-off-s-c2p2-r1",
config="gate-off-s-c2p2",
block="gate-off",
peak_unfinished="430",
ready_over_p95_s="145",
recovery_s="145",
upload_before_p50_ms="14",
)
return [on, off]
def selftest() -> None:
# план: без повтора подряд на границе кругов, все ячейки
configs = [
{"block": "matrix", "class": k, "concurrency": 2, "prefetch": 2}
for k in ("s", "m", "lj")
]
plan = plan_cells(configs, 3, 7)
assert len(plan) == 9
for prev, cur in zip(plan, plan[1:], strict=False):
assert prev["config"] != cur["config"]
assert plan[0]["order"] == 1 and plan[-1]["order"] == 9
# разбор строк
t0, ups = parse_uploads(
"q-t0,1000000\n"
"q,before,1000000,10,202,-,a,id1\n"
"q,over,1010000,20,429,5,a,-\nбред\n",
)
assert t0 == 1000.0 and len(ups) == 2 and ups[1].retry_after == "5"
metas, checks = parse_sweep(
"qs,id1,200,ready,-,3,2026-11-01T10:00:00.123456Z\n"
"qs,id2,404,-,-,0,-\nqv,id1,original,1\nqv,id1,crop,0\n",
)
assert abs(metas["id1"].updated_at % 1 - 0.123456) < 1e-5
assert metas["id2"].updated_at is None and checks == [True, False]
jobs = parse_jobs("jobs|1000000|1|2|3|4|5.5|6\nсбой\n")
assert jobs[0].unfinished == 10 and jobs[0].oldest_s == 5.5
mq, missing = parse_mq("mq 1000000000 5 2\nmq 2000000000 missing\n")
assert mq == [(1.0, 5, 2)] and missing == 1
# жизни: перезапуск сбрасывает счётчики
points = [
(10**9, 100, 10, 1),
(2 * 10**9, 200, 20, 2),
(3 * 10**9, 50, 5, 0),
(4 * 10**9, 150, 15, 1),
]
lives = split_lives(points)
assert len(lives) == 2
assert lives_delta(lives, 0, 5) == 200
assert role_cores([lives], 1, 4) == round(200 / 1e6 / 3, 4)
mp = [
(10**9, 500 * MIB, 400 * MIB, 2),
(2 * 10**9, 700 * MIB, 600 * MIB, 3),
(3 * 10**9, 100 * MIB, 90 * MIB, 0),
(4 * 10**9, 300 * MIB, 200 * MIB, 1),
]
assert memory_by_lives(split_lives(mp, key=1)) == (700.0, 600.0, 4)
# ячейка на границах окон: потерянное, зависшее, OOM, перезапуск
base = {
"cell": "s-c2p2-r1",
"config": "s-c2p2",
"block": "matrix",
"class": "s",
"concurrency": 2,
"prefetch": 2,
"repeat": 1,
"order": 1,
"before_s": 10,
"over_s": 20,
}
t = 1000.0
uploads_seen = [
Upload("before", int(t * 1000), 10, 202, "-", "a", "b1"),
Upload("over", int((t + 10) * 1000), 10, 202, "-", "a", "o1"),
Upload("over", int((t + 11) * 1000), 10, 202, "-", "a", "o2"),
Upload("over", int((t + 12) * 1000), 10, 202, "-", "a", "o3"),
Upload("over", int((t + 13) * 1000), 10, 429, "5", "a", "-"),
]
src = Sources(
uploads=uploads_seen,
t0=t,
metas={
"b1": Meta(200, "ready", "-", 3, t + 1),
# ровно на начале окна over: входит
"o1": Meta(200, "ready", "-", 3, t + 10),
"o2": Meta(200, "queued", "-", 0, None),
},
sha_checks=[True],
jobs=[
JobTick(t + 5, 0, 0, 0, 1, 0.1, 0),
JobTick(t + 20, 1, 1, 1, 2, 9.0, 1),
JobTick(t + 40, 0, 0, 0, 0, 0.0, 0),
],
mq=[(t + 20, 3, 2), (t + 40, 0, 0)],
mq_missing=0,
cg={
"worker": [
split_lives(
[(int((t + i) * 1e9), i * 10**6, i, 0) for i in range(41)],
),
],
},
mp={"worker": split_lives(mp)},
control=[],
worker_records=[],
second_records=[],
relay_records=[],
api_records=[],
sql={"extra_jobs": 0, "bad_variants": 0, "attempts_over_one": 1},
hwm_kib=2048,
restarts=1,
)
row = cell_row(base, src)
assert row["lost"] == 1 and row["stuck"] == 1
assert row["ready"] == 2 and row["oom_kill"] == 5
assert "lost" in row["flags"] and "stuck" in row["flags"]
# край окна — между тиками: CPU не занижен
assert row["cpu_worker_over"] == 1.0, row["cpu_worker_over"]
assert "oom" in row["flags"] and "worker_restart" in row["flags"]
assert row["ready_s_over"] == round(1 / 20, 3)
assert row["peak_unfinished"] == 5 and row["peak_prefetched"] == 0
assert row["recovery_s"] is not None and row["memory_peak_mib"] == 700.0
assert row["accepted_over"] == 3 and row["s429"] == 1
timeline = timeline_rows(base, src)
assert [r["window"] for r in timeline] == ["before", "over", "recovery"]
# вердикты: подтверждено, нет, нет данных
rows = matrix_fixture() + gate_fixture()
w2 = [
base_row(
cell="w2-p2-r1",
config="w2-p2",
block="w2-p2",
second_share="0.4",
recovery_s="20",
),
base_row(
cell="w2-p256-r1",
config="w2-p256",
block="w2-p256",
prefetch="256",
second_share="0",
recovery_s="30",
),
]
restarts = [
base_row(
cell="kill-r1",
config="kill",
block="kill-worker",
attempts_over_one="2",
),
base_row(cell="brk-r1", config="brk", block="restart-broker"),
]
accept = [
base_row(
cell="acc-r1",
config="acc",
block="accept",
first_busy_s="10",
first_backlog_s="35",
cpu_s3_accept="0.97",
accepted_over="300",
over_s="45",
peak_unfinished="280",
ready_s_over="0.6",
ready_s_recovery="2.8",
ctrl_download_over_p95="340",
ready_over_p95_s="93",
recovery_s="100",
),
]
everything = rows + w2 + restarts + accept
assert h1(everything).verdict()[0] == CONFIRMED, h1(everything).verdict()
assert h2(everything).verdict()[0] == CONFIRMED, h2(everything).verdict()
assert h3(everything).verdict()[0] == CONFIRMED, h3(everything).verdict()
assert h4(rows[:1]).verdict()[0] == CONFIRMED
assert h5(everything, 250).verdict()[0] == CONFIRMED, h5(
everything,
250,
).verdict()
assert h6(rows[:1]).verdict()[0] == CONFIRMED, h6(rows[:1]).verdict()
assert h7(everything).verdict()[0] == CONFIRMED
assert h8(everything).verdict()[0] == CONFIRMED, h8(everything).verdict()
assert h9(everything, 250).verdict()[0] == CONFIRMED, h9(
everything,
250,
).verdict()
empty: list[dict[str, Any]] = []
for name, (verdict, _) in verdicts(empty, 250).items():
assert verdict == NO_DATA, (name, verdict)
def broken(**values: Any) -> list[dict[str, Any]]:
return [base_row(**{**r, **values}) for r in everything]
assert h1(broken(cpu_worker_over="0.5")).verdict()[0] == REFUTED
assert (
h2(broken(duration_p50_ms="300", memory_peak_mib="1100")).verdict()[0]
== REFUTED
)
assert h3(broken(peak_broker_ready="60")).verdict()[0] == REFUTED
assert h4(broken(ready_over_p95_s="60")[:1]).verdict()[0] == REFUTED
assert h5(broken(retry_after="1"), 250).verdict()[0] == REFUTED
assert h6(broken(flags="lost")).verdict()[0] == REFUTED
assert h7(broken(ctrl_write_over_p95="500")).verdict()[0] == REFUTED
assert h7(broken(ctrl_read_over_s5xx="1")).verdict()[0] == REFUTED
assert h8(broken(attempts_over_one="5")).verdict()[0] == REFUTED
assert (
h9(broken(first_busy_s="50", first_backlog_s="10"), 250).verdict()[0]
== REFUTED
)
# рекомендация: (2, 2) и другая
assert recommend(matrix_fixture())["setting"] == (2, 2)
other = [
base_row(**{**r, "ready_s_recovery": "3"})
if r["concurrency"] == "1"
else r
for r in matrix_fixture()
]
assert recommend(other)["setting"] == (1, 1)
assert recommend(empty)["setting"] is None
# сводка и графики собираются
text = render_summary(
{"label": "x", "git": {"commit": "abc", "dirty": False}, "queue": {}},
everything,
)
assert "H9" in text and "Рекомендация" in text
pictures = render_plots(everything, [])
assert set(pictures) == {
"throughput-by-setting.svg",
"p95-by-setting.svg",
"memory-by-setting.svg",
"backlog-gate.svg",
}
print("overload selftest ok")
if __name__ == "__main__":
if "--selftest" in sys.argv:
selftest()
else:
print(__doc__)