Move publishing out of the upload path: a relay process publishes queued jobs with confirms. Add lease and attempt fencing, backoff with jitter, a per-job render process with CPU and memory limits, crop and optimized variants, the failed registry with audited replay, and staging cleanup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2296 lines
86 KiB
Python
2296 lines
86 KiB
Python
"""Профиль сверки: набор, план, разбор и сводка без docker.
|
||
|
||
Оркестрирует проходы `run.py` (профили `reconcile`, `reconcile_quick`),
|
||
здесь — чистые функции на stdlib: модель набора (id, содержимое и класс
|
||
объекта по номеру и зерну), строки CSV метаданных для `\\copy`, план
|
||
ячеек кругами, разбор записей уборщика, метрик шлюза S3, строк фоновой
|
||
нагрузки k6 и отсчётов сэмплера, окна и отношения p95 гипотезы 4, строки
|
||
`passes.csv` и `windows.csv`, вердикты H1–H4 и S4, `summary.md` и графики
|
||
SVG. Методика — design change `track2-reconcile-cost`, решения 1–3, 8–13;
|
||
правило — `.claude/rules/bench.md`, «Профиль сверки». Проверка —
|
||
`uv run python bench/reconcile.py --selftest`.
|
||
|
||
Генератор набора импортирует и загрузчик `bench/fill_objects.py` в
|
||
контейнере на образе стенда: модуль на верхнем уровне берёт только
|
||
stdlib, графики (`plots`) — внутри функций.
|
||
"""
|
||
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import random
|
||
import re
|
||
import statistics
|
||
import sys
|
||
import uuid
|
||
from collections.abc import Iterable
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
# класс объекта i — по i mod PERIOD: 0 — объект без метаданных, 1 —
|
||
# метаданные без объекта, 2 — другой размер, 3 — зависшая загрузка; по
|
||
# 0,25 % на класс (решение 2)
|
||
PERIOD = 400
|
||
CLASSES = ("orphan_object", "missing_object", "size_mismatch", "stale_pending")
|
||
# принудительный класс диапазона (набор S4): все согласованы или все —
|
||
# метаданные без объекта
|
||
FORCE_OK = "ok"
|
||
FORCE_MISSING = "missing_object"
|
||
CONTENT_TYPE = "image/png"
|
||
KIB = 1024
|
||
MIB = 2**20
|
||
# размер объекта набора отказа по пределу (S4)
|
||
S4_OBJECT_BYTES = 4 * KIB
|
||
# столбцы `\copy attachments`: created_at и updated_at — одно значение
|
||
COPY_COLUMNS = (
|
||
"id",
|
||
"chat_id",
|
||
"owner_id",
|
||
"filename",
|
||
"content_type",
|
||
"size_bytes",
|
||
"sha256",
|
||
"object_key",
|
||
"status",
|
||
"created_at",
|
||
"updated_at",
|
||
)
|
||
# ячейки: стратегия, глубина, CPU уборщика («stand» — лимит рабочего
|
||
# стенда, «fast» — управляемый опыт), предел скорости («default» — из
|
||
# настроек, иначе число), фоновая нагрузка (решения 1 и 10)
|
||
CELLS: dict[str, dict[str, Any]] = {
|
||
"batch": {"strategy": "batch", "deep": False, "cpu": "stand", "rate": 0},
|
||
"head": {"strategy": "head", "deep": False, "cpu": "stand", "rate": 0},
|
||
"deep": {"strategy": "batch", "deep": True, "cpu": "stand", "rate": 0},
|
||
"deep-fast": {
|
||
"strategy": "batch",
|
||
"deep": True,
|
||
"cpu": "fast",
|
||
"rate": 0,
|
||
},
|
||
"stand": {
|
||
"strategy": "batch",
|
||
"deep": False,
|
||
"cpu": "stand",
|
||
"rate": "default",
|
||
"load": True,
|
||
},
|
||
"stand-unlimited": {
|
||
"strategy": "batch",
|
||
"deep": False,
|
||
"cpu": "stand",
|
||
"rate": 0,
|
||
"load": True,
|
||
},
|
||
"fast-unlimited": {
|
||
"strategy": "batch",
|
||
"deep": False,
|
||
"cpu": "fast",
|
||
"rate": 0,
|
||
"load": True,
|
||
},
|
||
"fast-limited": {
|
||
"strategy": "batch",
|
||
"deep": False,
|
||
"cpu": "fast",
|
||
"rate": "default",
|
||
"load": True,
|
||
},
|
||
# отказ по пределу (решение 7): образ до change и checkout, --apply
|
||
# --max-changes 100; у checkout — без предела скорости
|
||
"before": {"apply": True, "before": True, "cpu": "stand", "rate": None},
|
||
"after": {
|
||
"strategy": "batch",
|
||
"deep": False,
|
||
"apply": True,
|
||
"cpu": "stand",
|
||
"rate": 0,
|
||
},
|
||
}
|
||
LOAD_CELLS = ("stand", "stand-unlimited", "fast-unlimited", "fast-limited")
|
||
LIMITED_LOAD = ("stand", "stand-unlimited", "fast-limited")
|
||
# пометки ошибок фоновой нагрузки у проходов блока H4: 5xx окон, неуспешные
|
||
# запросы и недосланные итерации по всему итогу k6 (verify S9)
|
||
BG_FLAGS = ("bg_5xx", "bg_failed", "bg_dropped")
|
||
# сценарии фоновой нагрузки: скачивания, запись, чтение истории
|
||
SCENARIOS = {"d": "download", "w": "write", "r": "read"}
|
||
WINDOWS = ("before", "pass", "after")
|
||
# пороги гипотез (решение 12)
|
||
H1_SPREAD = 0.20
|
||
H1_BYTES_PER_KEY = 300
|
||
H2_TIME_RATIO = 10
|
||
H2_HEAD_GROWTH_MIB = 5
|
||
# H2: запросов S3 и сеансов PostgreSQL на 1000 объектов — «~18 против
|
||
# ~2000» при 1 % расхождений, допуск ±25 % (verify S6)
|
||
H2_BATCH_PER_K = 18
|
||
H2_HEAD_PER_K = 2000
|
||
H2_PER_K_TOLERANCE = 0.25
|
||
H3_DEEP_RATIO = 5
|
||
H3_MS_PER_MIB = 5
|
||
H3_CLEANER_SHARE = 0.9
|
||
H3_S3_CORES = 0.5
|
||
H3_FAST_RATIO = 1.5
|
||
H4_THRESHOLD = 1.5
|
||
S4_AFTER_EXTRA_MS = 1000
|
||
S4_MS_PER_CANDIDATE = 5
|
||
# размер страницы списка и строк (`[reconciler] page_size`) и выборки
|
||
# зависших (`[cleaner] batch_size`) — значения по умолчанию, их берут
|
||
# проходы стенда; по ним H1 считает ожидаемые LIST и сеансы
|
||
PAGE_SIZE = 1000
|
||
STALE_BATCH = 100
|
||
# страниц списка staging за проход: staging набора пуст, строки и объекты
|
||
# пишет оркестратор мимо API
|
||
STAGING_PAGES = 1
|
||
# записи журнала уборщика, которые идут в raw/<проход>-cleaner.json
|
||
STARTED = "Started cleaner"
|
||
FINISHED = "Finished reconcile pass"
|
||
EXCEEDED = "Reconcile changes exceed limit"
|
||
FOUND = "Found storage discrepancy"
|
||
FIXED = "Fixed storage discrepancy"
|
||
# операции S3 в итоге прохода и типы запросов шлюза: HEAD шлюз считает
|
||
# типом GET (проба, design, Context)
|
||
S3_OPS = ("list", "head", "get", "delete")
|
||
GATEWAY_TYPES = ("LIST", "GET", "DELETE")
|
||
|
||
PASS_FIELDS = [
|
||
"pass",
|
||
"stage",
|
||
"cell",
|
||
"repeat",
|
||
"order",
|
||
"strategy",
|
||
"deep",
|
||
"apply",
|
||
"image",
|
||
"cleaner_cpus",
|
||
"max_objects_per_second",
|
||
"load",
|
||
"objects",
|
||
"object_bytes",
|
||
"ready_rows",
|
||
"candidates",
|
||
*(f"expected_{kind}" for kind in CLASSES),
|
||
*(f"found_{kind}" for kind in CLASSES),
|
||
"found_checksum_mismatch",
|
||
"found_other",
|
||
"listed_objects",
|
||
"checked_rows",
|
||
"exit_code",
|
||
"duration_ms",
|
||
"phase_list_ms",
|
||
"phase_rows_ms",
|
||
"phase_stale_ms",
|
||
"phase_recheck_ms",
|
||
"phase_act_ms",
|
||
"us_per_object",
|
||
"storage_objects",
|
||
*(f"s3_{op}" for op in S3_OPS),
|
||
"read_bytes",
|
||
"db_sessions",
|
||
"unchecked",
|
||
"refused",
|
||
"limit_changes",
|
||
"gw_list",
|
||
"gw_get",
|
||
"gw_delete",
|
||
"gw_other",
|
||
"pg_xact",
|
||
"pg_tup_returned",
|
||
"pg_tup_fetched",
|
||
"pg_blks_hit",
|
||
"pg_blks_read",
|
||
"pg_sessions",
|
||
"cpu_cleaner",
|
||
"cpu_s3",
|
||
"cpu_postgres",
|
||
"cpu_api",
|
||
"cpu_haproxy",
|
||
"mem_start_mib",
|
||
"mem_peak_mib",
|
||
"mem_growth_bytes_per_object",
|
||
"flags",
|
||
# с verify S6: исправлено проходом по итогу и записей «Fixed» —
|
||
# неизменность данных у отказа по пределу (S4)
|
||
"fixed_total",
|
||
"fixed_records",
|
||
]
|
||
WINDOW_FIELDS = [
|
||
"pass",
|
||
"stage",
|
||
"cell",
|
||
"repeat",
|
||
"window",
|
||
"scenario",
|
||
"start_s",
|
||
"end_s",
|
||
"responses",
|
||
"errors",
|
||
"s5xx",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
]
|
||
|
||
|
||
# ------------------------------------------------------------------ набор
|
||
|
||
|
||
def object_class(index: int, force: str | None = None) -> str | None:
|
||
"""Класс расхождения объекта `index`; `None` — согласован."""
|
||
if force == FORCE_OK:
|
||
return None
|
||
if force is not None:
|
||
return force
|
||
slot = index % PERIOD
|
||
return CLASSES[slot] if slot < len(CLASSES) else None
|
||
|
||
|
||
def object_id(seed: int, index: int) -> str:
|
||
"""UUID v4 из SHA-256 строки `<зерно>:<номер>` — запись сервера."""
|
||
digest = hashlib.sha256(f"{seed}:{index}".encode()).digest()
|
||
return str(uuid.UUID(bytes=digest[:16], version=4))
|
||
|
||
|
||
def content(seed: int, index: int, size: int) -> bytes:
|
||
"""Повторённый SHA-256 строки `<зерно>:<номер>:c`, обрезанный до
|
||
размера."""
|
||
block = hashlib.sha256(f"{seed}:{index}:c".encode()).digest()
|
||
return (block * (size // len(block) + 1))[:size]
|
||
|
||
|
||
def object_key(namespace: str, attachment_id: str) -> str:
|
||
return f"{namespace}/attachments/{attachment_id}/original"
|
||
|
||
|
||
def has_object(kind: str | None) -> bool:
|
||
return kind != "missing_object"
|
||
|
||
|
||
def has_row(kind: str | None) -> bool:
|
||
return kind != "orphan_object"
|
||
|
||
|
||
def object_indices(
|
||
start: int,
|
||
end: int,
|
||
force: str | None = None,
|
||
) -> list[int]:
|
||
"""Номера объектов диапазона, у которых есть объект в bucket."""
|
||
return [i for i in range(start, end) if has_object(object_class(i, force))]
|
||
|
||
|
||
def rows_csv(
|
||
*,
|
||
seed: int,
|
||
start: int,
|
||
end: int,
|
||
size: int,
|
||
namespace: str,
|
||
chat_id: str,
|
||
owner_id: str,
|
||
created_at: str,
|
||
force: str | None = None,
|
||
) -> str:
|
||
"""Строки CSV `\\copy attachments (COPY_COLUMNS)` диапазона:
|
||
`ready` — у согласованных, метаданных без объекта и другого размера
|
||
(`size_bytes` на байт больше), `pending` — у зависшей загрузки; объект
|
||
без метаданных строки не имеет. SHA-256 — от того же содержимого."""
|
||
lines = []
|
||
for index in range(start, end):
|
||
kind = object_class(index, force)
|
||
if not has_row(kind):
|
||
continue
|
||
attachment_id = object_id(seed, index)
|
||
digest = hashlib.sha256(content(seed, index, size)).hexdigest()
|
||
size_bytes = size + 1 if kind == "size_mismatch" else size
|
||
status = "pending" if kind == "stale_pending" else "ready"
|
||
lines.append(
|
||
",".join(
|
||
(
|
||
attachment_id,
|
||
chat_id,
|
||
owner_id,
|
||
f"bench-{index}.png",
|
||
CONTENT_TYPE,
|
||
str(size_bytes),
|
||
f"\\x{digest}",
|
||
object_key(namespace, attachment_id),
|
||
status,
|
||
created_at,
|
||
created_at,
|
||
),
|
||
),
|
||
)
|
||
return "\n".join(lines) + ("\n" if lines else "")
|
||
|
||
|
||
def class_count(n: int, slot: int) -> int:
|
||
"""Сколько номеров `i < n` дают `i mod PERIOD == slot`."""
|
||
return n // PERIOD + (1 if n % PERIOD > slot else 0)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Model:
|
||
"""Ожидаемое у набора: объекты в bucket, строки `ready`, расхождения
|
||
по классам (внесено)."""
|
||
|
||
objects: int
|
||
ready_rows: int
|
||
pending_rows: int
|
||
found: dict[str, int]
|
||
|
||
@property
|
||
def candidates(self) -> int:
|
||
return sum(self.found.values())
|
||
|
||
|
||
def model(n: int, *, clean: int = 0, missing: int = 0) -> Model:
|
||
"""Набор из `n` объектов с классами по номеру; у набора S4 — `clean`
|
||
согласованных и `missing` строк без объектов (тогда `n` не важен)."""
|
||
if clean or missing:
|
||
return Model(
|
||
objects=clean,
|
||
ready_rows=clean + missing,
|
||
pending_rows=0,
|
||
found={"missing_object": missing} if missing else {},
|
||
)
|
||
found = {kind: class_count(n, slot) for slot, kind in enumerate(CLASSES)}
|
||
return Model(
|
||
objects=n - found["missing_object"],
|
||
ready_rows=n - found["orphan_object"] - found["stale_pending"],
|
||
pending_rows=found["stale_pending"],
|
||
found={kind: count for kind, count in found.items() if count},
|
||
)
|
||
|
||
|
||
def downloads(seed: int, n: int, count: int) -> list[str]:
|
||
"""`id` согласованных вложений набора для фоновых скачиваний: выборка
|
||
от зерна (решение 10)."""
|
||
pool = [i for i in range(n) if object_class(i) is None]
|
||
rng = random.Random(seed)
|
||
chosen = rng.sample(pool, min(count, len(pool)))
|
||
return [object_id(seed, i) for i in chosen]
|
||
|
||
|
||
# ------------------------------------------------------------------- план
|
||
|
||
|
||
def plan_passes(
|
||
stages: Iterable[tuple[str, tuple[str, ...]]],
|
||
repeats: int,
|
||
seed: int,
|
||
) -> list[dict[str, Any]]:
|
||
"""Проходы этапов кругами: в круге r — перестановка ячеек этапа
|
||
генератором `Random(seed + номер этапа × 100 + r)`, круг не начинается
|
||
с ячейки, которой кончился прежний (как `uploads.plan_runs`). У группы
|
||
из одной ячейки (D без нагрузки — только `batch`) повторы идут подряд:
|
||
остальные проходы набора — блок под нагрузкой (спек `load-benchmark`,
|
||
«Профиль сверки», verify S8)."""
|
||
plan: list[dict[str, Any]] = []
|
||
for stage_no, (stage, cells) in enumerate(stages):
|
||
previous = None
|
||
for round_no in range(1, repeats + 1):
|
||
order = list(range(len(cells)))
|
||
random.Random(seed + stage_no * 100 + round_no).shuffle(order)
|
||
if len(order) > 1 and order[0] == previous:
|
||
order[0], order[1] = order[1], order[0]
|
||
for index in order:
|
||
cell = cells[index]
|
||
plan.append(
|
||
{
|
||
"pass": f"{stage}-{cell}-r{round_no}",
|
||
"stage": stage,
|
||
"cell": cell,
|
||
"repeat": round_no,
|
||
"order": len(plan) + 1,
|
||
},
|
||
)
|
||
previous = order[-1]
|
||
return plan
|
||
|
||
|
||
# ----------------------------------------------------------------- разбор
|
||
|
||
|
||
def parse_records(text: str) -> list[dict[str, Any]]:
|
||
"""JSON-записи журнала уборщика; остальное (вывод compose) — мимо."""
|
||
records = []
|
||
for line in text.splitlines():
|
||
line = line.strip()
|
||
if not line.startswith("{"):
|
||
continue
|
||
try:
|
||
records.append(json.loads(line))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
return records
|
||
|
||
|
||
def pick(records: list[dict[str, Any]], event: str) -> dict[str, Any] | None:
|
||
found = [item for item in records if item.get("event") == event]
|
||
return found[-1] if found else None
|
||
|
||
|
||
def kept_records(records: list[dict[str, Any]]) -> dict[str, Any]:
|
||
"""`raw/<проход>-cleaner.json`: записи старта, итога и предела; записи
|
||
о расхождениях — только числом (их до тысячи на проход)."""
|
||
return {
|
||
"started": pick(records, STARTED),
|
||
"finished": pick(records, FINISHED),
|
||
"exceeded": pick(records, EXCEEDED),
|
||
"found_records": sum(1 for r in records if r.get("event") == FOUND),
|
||
"fixed_records": sum(1 for r in records if r.get("event") == FIXED),
|
||
}
|
||
|
||
|
||
def timestamp(value: str) -> float:
|
||
"""ISO 8601 с `Z` (записи structlog, Docker) → секунды эпохи; дробь
|
||
Docker — до наносекунд, Python берёт микросекунды."""
|
||
text = value.strip().replace("Z", "+00:00")
|
||
match = re.match(r"(.*\.\d{6})\d*(.*)", text)
|
||
if match:
|
||
text = match.group(1) + match.group(2)
|
||
return datetime.fromisoformat(text).timestamp()
|
||
|
||
|
||
def parse_metrics(text: str) -> dict[str, float]:
|
||
"""`SeaweedFS_s3_request_total` по типам запроса (сумма по прочим
|
||
меткам)."""
|
||
totals: dict[str, float] = {}
|
||
for line in text.splitlines():
|
||
if not line.startswith("SeaweedFS_s3_request_total{"):
|
||
continue
|
||
kind = re.search(r'type="([^"]+)"', line)
|
||
try:
|
||
value = float(line.rsplit(" ", 1)[1])
|
||
except (IndexError, ValueError):
|
||
continue
|
||
name = kind.group(1) if kind else "?"
|
||
totals[name] = totals.get(name, 0.0) + value
|
||
return totals
|
||
|
||
|
||
def metrics_delta(
|
||
before: dict[str, float],
|
||
after: dict[str, float],
|
||
) -> dict[str, int]:
|
||
names = set(before) | set(after)
|
||
return {
|
||
name: round(after.get(name, 0.0) - before.get(name, 0.0))
|
||
for name in sorted(names)
|
||
}
|
||
|
||
|
||
def gateway_mismatch(
|
||
s3_requests: dict[str, int],
|
||
gateway: dict[str, int],
|
||
) -> bool:
|
||
"""Счёт сверки и метрики шлюза: LIST = list, GET = head + get, DELETE
|
||
= delete; запросов других типов быть не должно (решение 8)."""
|
||
expected = {
|
||
"LIST": s3_requests.get("list", 0),
|
||
"GET": s3_requests.get("head", 0) + s3_requests.get("get", 0),
|
||
"DELETE": s3_requests.get("delete", 0),
|
||
}
|
||
other = sum(v for k, v in gateway.items() if k not in GATEWAY_TYPES)
|
||
return other != 0 or any(
|
||
gateway.get(name, 0) != value for name, value in expected.items()
|
||
)
|
||
|
||
|
||
PG_STAT_SQL = (
|
||
"SELECT xact_commit + xact_rollback, tup_returned, tup_fetched, "
|
||
"blks_hit, blks_read, sessions FROM pg_stat_database "
|
||
"WHERE datname = current_database();\n"
|
||
)
|
||
PG_STAT_NAMES = (
|
||
"pg_xact",
|
||
"pg_tup_returned",
|
||
"pg_tup_fetched",
|
||
"pg_blks_hit",
|
||
"pg_blks_read",
|
||
"pg_sessions",
|
||
)
|
||
|
||
|
||
def parse_pg_stat(text: str) -> dict[str, int]:
|
||
values = [int(v) for v in text.strip().split("|")]
|
||
return dict(zip(PG_STAT_NAMES, values, strict=True))
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Request:
|
||
"""Запрос фоновой нагрузки: сценарий, отправка (мс часов ВМ),
|
||
длительность и статус."""
|
||
|
||
kind: str
|
||
start_ms: int
|
||
duration_ms: float
|
||
status: int
|
||
|
||
|
||
def parse_background(text: str) -> list[Request]:
|
||
"""Строки `bg,<d|w|r>,<отправка мс>,<длительность мс>,<статус>`."""
|
||
out = []
|
||
for line in text.splitlines():
|
||
line = line.strip()
|
||
if not line.startswith("bg,"):
|
||
continue
|
||
parts = line.split(",")
|
||
if len(parts) != 5 or parts[1] not in SCENARIOS:
|
||
continue
|
||
try:
|
||
out.append(
|
||
Request(
|
||
kind=parts[1],
|
||
start_ms=int(parts[2]),
|
||
duration_ms=float(parts[3]),
|
||
status=int(parts[4]),
|
||
),
|
||
)
|
||
except ValueError:
|
||
continue
|
||
return out
|
||
|
||
|
||
def summary_count(k6_data: dict[str, Any], name: str) -> int | None:
|
||
value = k6_data.get("metrics", {}).get(name, {}).get("values", {})
|
||
count = value.get("count")
|
||
return None if count is None else int(count)
|
||
|
||
|
||
def background_errors(k6_data: dict[str, Any]) -> dict[str, int]:
|
||
"""Ошибки фоновой нагрузки по всему итогу k6, а не по окнам (verify
|
||
S9): неуспешные запросы (`http_req_failed`: у Rate-метрики `passes` —
|
||
число неуспешных) и недосланные итерации (`dropped_iterations`)."""
|
||
metrics = k6_data.get("metrics", {})
|
||
failed = metrics.get("http_req_failed", {}).get("values", {})
|
||
dropped = metrics.get("dropped_iterations", {}).get("values", {})
|
||
return {
|
||
"failed": int(failed.get("passes", 0) or 0),
|
||
"dropped": int(dropped.get("count", 0) or 0),
|
||
}
|
||
|
||
|
||
def background_flags(errors: dict[str, int]) -> list[str]:
|
||
"""Пометки проходов блока по `background_errors`."""
|
||
flags = []
|
||
if errors.get("failed"):
|
||
flags.append("bg_failed")
|
||
if errors.get("dropped"):
|
||
flags.append("bg_dropped")
|
||
return flags
|
||
|
||
|
||
def line_problems(
|
||
requests: list[Request],
|
||
k6_data: dict[str, Any],
|
||
*,
|
||
max_vus: int,
|
||
) -> tuple[list[str], dict[str, int]]:
|
||
"""Полнота строк `bg,…` по итогу k6 (design, «Уточнения apply», п.
|
||
15). Итерацию, которую остановка SIGINT прервала уже после строки, k6
|
||
не считает ни в `iterations`, ни (если прервал до сброса отсчёта) в
|
||
счётчике `bg_lines_<сценарий>`: у прерванной строка может быть, а
|
||
отсчётов нет. Поэтому снизу — строк не меньше итераций и не меньше
|
||
счётчика (у каждой завершённой итерации строка есть: пропажу видно),
|
||
сверху — лишних строк против каждого из них не больше VU сценария и
|
||
не больше VU, созданных k6 за прогон (`vus_max` итога): столько
|
||
итераций может быть в полёте. Возвращает расхождения и число
|
||
прерванных (строк сверх итераций) по сценариям."""
|
||
problems = []
|
||
interrupted: dict[str, int] = {}
|
||
# в полёте не больше VU, созданных k6 за прогон (`vus_max` итога —
|
||
# на все сценарии), и не больше VU сценария (verify S4)
|
||
created = k6_data.get("metrics", {}).get("vus_max", {}).get("values", {})
|
||
if created.get("max") is not None:
|
||
max_vus = min(max_vus, int(created["max"]))
|
||
for kind, scenario in SCENARIOS.items():
|
||
lines = sum(1 for r in requests if r.kind == kind)
|
||
counted = summary_count(k6_data, f"bg_lines_{scenario}")
|
||
iterations = summary_count(
|
||
k6_data,
|
||
f"iterations{{scenario:{scenario}}}",
|
||
)
|
||
if counted is None or iterations is None:
|
||
problems.append(
|
||
f"{scenario}: no bg_lines or iterations in the k6 summary",
|
||
)
|
||
continue
|
||
interrupted[scenario] = lines - iterations
|
||
for name, value in (("iterations", iterations), ("bg_lines", counted)):
|
||
if lines < value:
|
||
problems.append(
|
||
f"{scenario}: {lines} k6 lines, {value} {name} "
|
||
f"(lines lost: {value - lines})",
|
||
)
|
||
elif lines - value > max_vus:
|
||
problems.append(
|
||
f"{scenario}: {lines} k6 lines, {value} {name} (extra "
|
||
f"{lines - value} > {max_vus} VU: not interruptions)",
|
||
)
|
||
return problems, interrupted
|
||
|
||
|
||
def percentile(values: list[float], q: float) -> float | None:
|
||
"""Перцентиль ближайшего ранга, как у k6 (p(95))."""
|
||
if not values:
|
||
return None
|
||
ordered = sorted(values)
|
||
rank = max(1, math.ceil(q / 100 * len(ordered)))
|
||
return float(ordered[rank - 1])
|
||
|
||
|
||
OK_STATUS = {"d": 200, "w": 201, "r": 200}
|
||
|
||
|
||
def window_stats(
|
||
requests: list[Request],
|
||
kind: str,
|
||
start: float,
|
||
end: float,
|
||
) -> dict[str, Any]:
|
||
"""Ответы сценария, отправленные в [start, end) (секунды часов ВМ):
|
||
число, ошибки, 5xx и перцентили успешных."""
|
||
inside = [
|
||
r
|
||
for r in requests
|
||
if r.kind == kind and start * 1000 <= r.start_ms < end * 1000
|
||
]
|
||
ok = [r.duration_ms for r in inside if r.status == OK_STATUS[kind]]
|
||
return {
|
||
"responses": len(inside),
|
||
"errors": len(inside) - len(ok),
|
||
"s5xx": sum(1 for r in inside if 500 <= r.status < 600),
|
||
"p50_ms": percentile(ok, 50),
|
||
"p95_ms": percentile(ok, 95),
|
||
"p99_ms": percentile(ok, 99),
|
||
}
|
||
|
||
|
||
def ratio(a: float | None, b: float | None) -> float | None:
|
||
if a is None or b is None or b <= 0:
|
||
return None
|
||
return round(a / b, 3)
|
||
|
||
|
||
def windows_of(
|
||
requests: list[Request],
|
||
bounds: dict[str, tuple[float, float]],
|
||
) -> tuple[list[dict[str, Any]], dict[str, dict[str, float | None]]]:
|
||
"""Строки окон прохода по сценариям и отношения: `r` — p95 окна
|
||
прохода к p95 окон до и после вместе (строка `quiet`), `a` — p95 до к
|
||
p95 после."""
|
||
rows = []
|
||
ratios: dict[str, dict[str, float | None]] = {}
|
||
for kind, scenario in SCENARIOS.items():
|
||
stats = {}
|
||
for name in WINDOWS:
|
||
w0, w1 = bounds[name]
|
||
stats[name] = window_stats(requests, kind, w0, w1)
|
||
rows.append(
|
||
{
|
||
"window": name,
|
||
"scenario": scenario,
|
||
"start_s": round(w0, 3),
|
||
"end_s": round(w1, 3),
|
||
**stats[name],
|
||
},
|
||
)
|
||
quiet = [
|
||
r
|
||
for r in requests
|
||
if any(
|
||
bounds[name][0] * 1000 <= r.start_ms < bounds[name][1] * 1000
|
||
for name in ("before", "after")
|
||
)
|
||
]
|
||
stats["quiet"] = window_stats(quiet, kind, -math.inf, math.inf)
|
||
rows.append(
|
||
{
|
||
"window": "quiet",
|
||
"scenario": scenario,
|
||
"start_s": round(bounds["before"][0], 3),
|
||
"end_s": round(bounds["after"][1], 3),
|
||
**stats["quiet"],
|
||
},
|
||
)
|
||
if "list" in bounds:
|
||
# шаг списка (`phase_ms.list` от начала прохода): справочно, без
|
||
# вердикта — в нём у `batch` нагружен S3 (решение 10)
|
||
w0, w1 = bounds["list"]
|
||
rows.append(
|
||
{
|
||
"window": "list",
|
||
"scenario": scenario,
|
||
"start_s": round(w0, 3),
|
||
"end_s": round(w1, 3),
|
||
**window_stats(requests, kind, w0, w1),
|
||
},
|
||
)
|
||
ratios[scenario] = {
|
||
"r": ratio(stats["pass"]["p95_ms"], stats["quiet"]["p95_ms"]),
|
||
"a": ratio(stats["before"]["p95_ms"], stats["after"]["p95_ms"]),
|
||
"s5xx": sum(stats[name]["s5xx"] for name in WINDOWS),
|
||
}
|
||
return rows, ratios
|
||
|
||
|
||
# ------------------------------------------------------------- сэмплер
|
||
|
||
|
||
def usage_at(points: list[tuple[int, ...]], t: float) -> float | None:
|
||
"""`usage_usec` контейнера на момент `t` (секунды): линейно между
|
||
соседними отсчётами; вне отсчётов — None."""
|
||
if not points:
|
||
return None
|
||
ns = t * 1e9
|
||
if ns < points[0][0] or ns > points[-1][0]:
|
||
return None
|
||
for prev, cur in zip(points, points[1:], strict=False):
|
||
if prev[0] <= ns <= cur[0]:
|
||
if cur[0] == prev[0]:
|
||
return float(cur[1])
|
||
share = (ns - prev[0]) / (cur[0] - prev[0])
|
||
return prev[1] + (cur[1] - prev[1]) * share
|
||
return float(points[-1][1])
|
||
|
||
|
||
def window_cpu(
|
||
points: list[tuple[int, ...]],
|
||
w0: float,
|
||
w1: float,
|
||
) -> float | None:
|
||
"""Ядер в среднем за окно [w0, w1]: разность `usage_usec` на концах.
|
||
Окно у края отсчётов — по ближайшим отсчётам внутри. Короткий проход
|
||
в конце жизни контейнера (последний отсчёт раньше конца окна, внутри
|
||
меньше половины окна) — по отсчётам, охватывающим окно: среднее за
|
||
чуть большее время, а не пропуск."""
|
||
if w1 <= w0 or len(points) < 2:
|
||
return None
|
||
first, last = points[0][0] / 1e9, points[-1][0] / 1e9
|
||
a, b = max(w0, first), min(w1, last)
|
||
if b - a >= (w1 - w0) / 2:
|
||
ua, ub = usage_at(points, a), usage_at(points, b)
|
||
if ua is not None and ub is not None:
|
||
return round((ub - ua) / 1e6 / (b - a), 4)
|
||
before = [p for p in points if p[0] <= w0 * 1e9]
|
||
after = [p for p in points if p[0] >= w1 * 1e9]
|
||
p0 = before[-1] if before else points[0]
|
||
p1 = after[0] if after else points[-1]
|
||
if p1[0] <= p0[0]:
|
||
return None
|
||
return round((p1[1] - p0[1]) * 1e3 / (p1[0] - p0[0]), 4)
|
||
|
||
|
||
def memory_start_peak(
|
||
points: list[tuple[int, ...]],
|
||
w0: float,
|
||
) -> tuple[float | None, float | None]:
|
||
"""Память контейнера (МиБ): последняя выборка не позже начала окна и
|
||
наибольшая за жизнь (решение 9)."""
|
||
if not points:
|
||
return None, None
|
||
before = [p for p in points if p[0] <= w0 * 1e9]
|
||
start = (before[-1] if before else points[0])[5] / MIB
|
||
peak = max(p[5] for p in points) / MIB
|
||
return round(start, 1), round(peak, 1)
|
||
|
||
|
||
# ---------------------------------------------------------- строка прохода
|
||
|
||
|
||
def pass_row(
|
||
entry: dict[str, Any],
|
||
*,
|
||
cell: dict[str, Any],
|
||
expected: Model,
|
||
object_bytes: int,
|
||
kept: dict[str, Any],
|
||
exit_code: int,
|
||
gateway: dict[str, int] | None,
|
||
pg: dict[str, int] | None,
|
||
cpu: dict[str, float | None],
|
||
memory: tuple[float | None, float | None],
|
||
cleaner_cpus: str,
|
||
rate: float | None,
|
||
image: str,
|
||
) -> dict[str, Any]:
|
||
"""Строка `passes.csv`: модель и найденное, итог прохода, метрики
|
||
шлюза и PostgreSQL, CPU и память за окно; пометки — `count_mismatch`
|
||
(счёт сверки ≠ метрикам шлюза в ячейке без нагрузки)."""
|
||
finished = kept.get("finished") or {}
|
||
exceeded = kept.get("exceeded") or {}
|
||
found = finished.get("found") or {}
|
||
phases = finished.get("phase_ms") or {}
|
||
s3 = finished.get("s3_requests") or {}
|
||
duration = finished.get("duration_ms")
|
||
row: dict[str, Any] = {
|
||
**{key: entry[key] for key in ("pass", "stage", "cell", "repeat")},
|
||
"order": entry["order"],
|
||
"strategy": finished.get("strategy", cell.get("strategy", "batch")),
|
||
"deep": finished.get("deep", cell.get("deep", False)),
|
||
"apply": bool(cell.get("apply")),
|
||
"image": image,
|
||
"cleaner_cpus": cleaner_cpus,
|
||
"max_objects_per_second": finished.get(
|
||
"max_objects_per_second",
|
||
rate,
|
||
),
|
||
"load": bool(cell.get("load")),
|
||
"objects": expected.objects,
|
||
"object_bytes": object_bytes,
|
||
"ready_rows": expected.ready_rows,
|
||
"candidates": expected.candidates,
|
||
"listed_objects": finished.get("listed_objects"),
|
||
"checked_rows": finished.get("checked_rows"),
|
||
"exit_code": exit_code,
|
||
"duration_ms": duration,
|
||
"us_per_object": (
|
||
round(duration * 1000 / expected.objects, 2)
|
||
if duration is not None and expected.objects
|
||
else None
|
||
),
|
||
"storage_objects": finished.get("storage_objects"),
|
||
"read_bytes": finished.get("read_bytes"),
|
||
"db_sessions": finished.get("db_sessions"),
|
||
"unchecked": finished.get("unchecked"),
|
||
"refused": finished.get("refused"),
|
||
"limit_changes": exceeded.get("changes"),
|
||
"cpu_cleaner": cpu.get("cleaner"),
|
||
"cpu_s3": cpu.get("s3"),
|
||
"cpu_postgres": cpu.get("postgres"),
|
||
"cpu_api": cpu.get("api"),
|
||
"cpu_haproxy": cpu.get("haproxy"),
|
||
"mem_start_mib": memory[0],
|
||
"mem_peak_mib": memory[1],
|
||
}
|
||
for kind in CLASSES:
|
||
row[f"expected_{kind}"] = expected.found.get(kind, 0)
|
||
row[f"found_{kind}"] = found.get(kind, 0)
|
||
row["found_checksum_mismatch"] = found.get("checksum_mismatch", 0)
|
||
row["found_other"] = sum(
|
||
count
|
||
for kind, count in found.items()
|
||
if kind not in CLASSES and kind != "checksum_mismatch"
|
||
)
|
||
for name in ("list", "rows", "stale", "recheck", "act"):
|
||
row[f"phase_{name}_ms"] = phases.get(name)
|
||
for op in S3_OPS:
|
||
row[f"s3_{op}"] = s3.get(op)
|
||
if memory[0] is not None and memory[1] is not None and expected.objects:
|
||
row["mem_growth_bytes_per_object"] = round(
|
||
(memory[1] - memory[0]) * MIB / expected.objects,
|
||
1,
|
||
)
|
||
flags = []
|
||
if gateway is not None:
|
||
row["gw_list"] = gateway.get("LIST", 0)
|
||
row["gw_get"] = gateway.get("GET", 0)
|
||
row["gw_delete"] = gateway.get("DELETE", 0)
|
||
row["gw_other"] = sum(
|
||
v for k, v in gateway.items() if k not in GATEWAY_TYPES
|
||
)
|
||
if s3 and not cell.get("load") and gateway_mismatch(s3, gateway):
|
||
flags.append("count_mismatch")
|
||
if pg is not None:
|
||
row.update(pg)
|
||
row["flags"] = " ".join(flags)
|
||
row["fixed_total"] = sum((finished.get("fixed") or {}).values())
|
||
row["fixed_records"] = kept.get("fixed_records")
|
||
return row
|
||
|
||
|
||
def found_problems(
|
||
entry: dict[str, Any],
|
||
expected: dict[str, int],
|
||
kept: dict[str, Any],
|
||
) -> list[str]:
|
||
"""Найдено не то, что внесено: класс, ячейка и оба числа (спек
|
||
`load-benchmark`, «Найдено не то, что внесено»)."""
|
||
finished = kept.get("finished")
|
||
if finished is None:
|
||
return [f"{entry['pass']}: no «{FINISHED}» record"]
|
||
found = finished.get("found") or {}
|
||
problems = []
|
||
for kind in sorted(set(expected) | set(found)):
|
||
if found.get(kind, 0) != expected.get(kind, 0):
|
||
problems.append(
|
||
f"{entry['pass']}: {kind} found {found.get(kind, 0)}, "
|
||
f"injected {expected.get(kind, 0)}",
|
||
)
|
||
return problems
|
||
|
||
|
||
# ------------------------------------------------------------- агрегаты
|
||
|
||
|
||
def num(value: Any) -> float | None:
|
||
if value is None or value == "":
|
||
return None
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def med(values: Iterable[Any]) -> float | None:
|
||
present = [v for v in (num(x) for x in values) if v is not None]
|
||
return statistics.median(present) if present else None
|
||
|
||
|
||
def spread(values: Iterable[Any], digits: int = 1) -> str:
|
||
present = [v for v in (num(x) for x in values) if v is not None]
|
||
if not present:
|
||
return "—"
|
||
mid = statistics.median(present)
|
||
if len(present) == 1:
|
||
return f"{mid:.{digits}f}"
|
||
return (
|
||
f"{mid:.{digits}f} ({min(present):.{digits}f}–"
|
||
f"{max(present):.{digits}f})"
|
||
)
|
||
|
||
|
||
def cell_rows(
|
||
passes: list[dict[str, Any]],
|
||
stage: str,
|
||
cell: str,
|
||
) -> list[dict[str, Any]]:
|
||
return [
|
||
row
|
||
for row in passes
|
||
if row.get("stage") == stage and row.get("cell") == cell
|
||
]
|
||
|
||
|
||
def cell_median(
|
||
passes: list[dict[str, Any]],
|
||
stage: str,
|
||
cell: str,
|
||
column: str,
|
||
) -> float | None:
|
||
return med(row.get(column) for row in cell_rows(passes, stage, cell))
|
||
|
||
|
||
def load_ratios(
|
||
window_rows: list[dict[str, Any]],
|
||
) -> dict[str, dict[str, dict[str, float | None]]]:
|
||
"""Медиана `r` и `a` по повторам: ячейка → сценарий → значение;
|
||
`s5xx` — сумма."""
|
||
by_pass: dict[str, dict[str, Any]] = {}
|
||
for row in window_rows:
|
||
item = by_pass.setdefault(
|
||
row["pass"],
|
||
{"cell": row["cell"], "stats": {}},
|
||
)
|
||
item["stats"][(row["scenario"], row["window"])] = row
|
||
values: dict[str, dict[str, dict[str, list[float]]]] = {}
|
||
fives: dict[str, dict[str, int]] = {}
|
||
for item in by_pass.values():
|
||
cell = item["cell"]
|
||
for scenario in SCENARIOS.values():
|
||
stats = item["stats"]
|
||
p = {
|
||
name: num(stats.get((scenario, name), {}).get("p95_ms"))
|
||
for name in WINDOWS
|
||
}
|
||
quiet = num(stats.get((scenario, "quiet"), {}).get("p95_ms"))
|
||
r = ratio(p["pass"], quiet)
|
||
a = ratio(p["before"], p["after"])
|
||
bucket = values.setdefault(cell, {}).setdefault(
|
||
scenario,
|
||
{"r": [], "a": []},
|
||
)
|
||
if r is not None:
|
||
bucket["r"].append(r)
|
||
if a is not None:
|
||
bucket["a"].append(a)
|
||
fives.setdefault(cell, {}).setdefault(scenario, 0)
|
||
fives[cell][scenario] += sum(
|
||
int(num(stats.get((scenario, name), {}).get("s5xx")) or 0)
|
||
for name in WINDOWS
|
||
)
|
||
return {
|
||
cell: {
|
||
scenario: {
|
||
"r": med(v["r"]),
|
||
"a": med(v["a"]),
|
||
"s5xx": fives[cell][scenario],
|
||
}
|
||
for scenario, v in scenarios.items()
|
||
}
|
||
for cell, scenarios in values.items()
|
||
}
|
||
|
||
|
||
# -------------------------------------------------------------- вердикты
|
||
|
||
|
||
CONFIRMED = "подтверждена"
|
||
REFUTED = "опровергнута"
|
||
UNTESTED = "не проверена"
|
||
NOISY = "не различимо на этом хосте"
|
||
|
||
|
||
@dataclass
|
||
class Verdict:
|
||
"""Строка гипотезы: имя, критерий, измерено, вывод."""
|
||
|
||
name: str
|
||
criterion: str
|
||
measured: str
|
||
verdict: str
|
||
details: list[str] = field(default_factory=list)
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 2) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def stage_n(meta: dict[str, Any], stage: str) -> int | None:
|
||
for item in meta.get("reconcile", {}).get("stages", []):
|
||
if item["stage"] == stage:
|
||
return item["objects_total"]
|
||
return None
|
||
|
||
|
||
def expected_batch_requests(row: dict[str, Any]) -> tuple[int, int, int]:
|
||
"""LIST, HEAD и сеансы PostgreSQL поверхностного dry-run `batch` по
|
||
модели прохода (H1): страницы списка (страница без ключей — только у
|
||
пустого bucket), одна страница списка staging (пустая: загрузок в наборе
|
||
нет, сеанса с БД не даёт; design change `pipeline-reliability`, решение
|
||
16) и проба bucket, если есть кандидаты без объекта или с другим размером;
|
||
HEAD — по одному на такого кандидата и объект без строки; сеанс — на
|
||
страницу списка с ключами, на каждую страницу строк `ready` и выборку
|
||
зависших (последняя — неполная) и на перепроверку кандидата, кроме
|
||
зависших."""
|
||
|
||
def count(column: str) -> int:
|
||
return int(num(row.get(column)) or 0)
|
||
|
||
objects, ready = count("objects"), count("ready_rows")
|
||
orphan, missing = (
|
||
count("expected_orphan_object"),
|
||
count(
|
||
"expected_missing_object",
|
||
),
|
||
)
|
||
size, stale = (
|
||
count("expected_size_mismatch"),
|
||
count(
|
||
"expected_stale_pending",
|
||
),
|
||
)
|
||
probe = 1 if missing or size else 0
|
||
lists = max(1, math.ceil(objects / PAGE_SIZE)) + STAGING_PAGES + probe
|
||
heads = orphan + missing + size
|
||
sessions = (
|
||
math.ceil(objects / PAGE_SIZE)
|
||
+ ready // PAGE_SIZE
|
||
+ 1
|
||
+ stale // STALE_BATCH
|
||
+ 1
|
||
+ heads
|
||
)
|
||
return lists, heads, sessions
|
||
|
||
|
||
def verdict_h1(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
|
||
"""Линейность `batch`: цена на объект, запросы S3, сеансы, память."""
|
||
stages = [
|
||
s
|
||
for s in meta.get("reconcile", {}).get("h1_stages", [])
|
||
if cell_rows(passes, s, "batch")
|
||
]
|
||
if len(stages) < 2:
|
||
return Verdict(
|
||
"H1",
|
||
"цена на объект `batch` при трёх N в пределах 20 %",
|
||
"нет ячеек",
|
||
UNTESTED,
|
||
)
|
||
costs = {
|
||
s: cell_median(passes, s, "batch", "us_per_object") for s in stages
|
||
}
|
||
present = [v for v in costs.values() if v]
|
||
spread_ratio = max(present) / min(present) - 1 if present else None
|
||
details = []
|
||
ok_requests = True
|
||
for s in stages:
|
||
row = cell_rows(passes, s, "batch")[0]
|
||
n = int(num(row["objects"]) or 0) + int(
|
||
num(row["expected_missing_object"]) or 0,
|
||
)
|
||
want_list, want_head, want_sessions = expected_batch_requests(row)
|
||
got_list = cell_median(passes, s, "batch", "s3_list")
|
||
got_head = cell_median(passes, s, "batch", "s3_head")
|
||
got_sessions = cell_median(passes, s, "batch", "db_sessions")
|
||
# счёт точный: набор неизменен, допуска нет (verify S6)
|
||
ok_requests &= got_list == want_list
|
||
ok_requests &= got_head == want_head
|
||
ok_requests &= got_sessions == want_sessions
|
||
details.append(
|
||
f"{s} (N = {n}): {fmt(costs[s])} мкс на объект, LIST "
|
||
f"{fmt(got_list, 0)} (ожидалось {want_list}), HEAD "
|
||
f"{fmt(got_head, 0)} (ожидалось {want_head}), сеансов "
|
||
f"{fmt(got_sessions, 0)} (ожидалось {want_sessions})",
|
||
)
|
||
slope = memory_slope(passes, stages, "batch")
|
||
memory_ok = slope is not None and slope <= H1_BYTES_PER_KEY
|
||
holds = (
|
||
spread_ratio is not None
|
||
and spread_ratio <= H1_SPREAD
|
||
and ok_requests
|
||
and memory_ok
|
||
)
|
||
spread_pct = None if spread_ratio is None else spread_ratio * 100
|
||
return Verdict(
|
||
"H1",
|
||
f"цена на объект `batch` различается не больше чем на "
|
||
f"{H1_SPREAD:.0%}; LIST = ⌈N/1000⌉ + 2 (страницы вложений, страница "
|
||
"staging, проба bucket), HEAD — только у кандидатов; "
|
||
"сеансов — по одному на страницу списка, страницу строк `ready` и "
|
||
"выборку зависших плюс по одному на кандидата, кроме зависших; "
|
||
f"прирост пика памяти уборщика от N ≤ {H1_BYTES_PER_KEY} Б на ключ",
|
||
f"разброс цены {fmt(spread_pct, 1)} %, наклон пика памяти "
|
||
f"{fmt(slope, 0)} Б на ключ",
|
||
CONFIRMED if holds else REFUTED,
|
||
details,
|
||
)
|
||
|
||
|
||
def memory_slope(
|
||
passes: list[dict[str, Any]],
|
||
stages: list[str],
|
||
cell: str,
|
||
) -> float | None:
|
||
"""Прирост пика памяти уборщика на объект между наименьшим и
|
||
наибольшим набором (Б на ключ): постоянная часть прохода (модели
|
||
botocore, пул соединений) в разность не входит."""
|
||
points = []
|
||
for stage in stages:
|
||
rows = cell_rows(passes, stage, cell)
|
||
peak = cell_median(passes, stage, cell, "mem_peak_mib")
|
||
if rows and peak is not None:
|
||
points.append((num(rows[0]["objects"]) or 0, peak))
|
||
if len(points) < 2:
|
||
return None
|
||
(n0, m0), (n1, m1) = min(points), max(points)
|
||
if n1 <= n0:
|
||
return None
|
||
return (m1 - m0) * MIB / (n1 - n0)
|
||
|
||
|
||
def verdict_h2(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
|
||
stages = [
|
||
s
|
||
for s in meta.get("reconcile", {}).get("h2_stages", [])
|
||
if cell_rows(passes, s, "batch") and cell_rows(passes, s, "head")
|
||
]
|
||
if not stages:
|
||
return Verdict(
|
||
"H2",
|
||
f"HEAD дороже `batch` по времени на объект ≥ {H2_TIME_RATIO}×",
|
||
"нет ячеек",
|
||
UNTESTED,
|
||
)
|
||
details = []
|
||
ok = True
|
||
peaks = {s: cell_median(passes, s, "head", "mem_peak_mib") for s in stages}
|
||
known = [v for v in peaks.values() if v is not None]
|
||
# память HEAD от N не растёт: разность пиков при наибольшем и наименьшем
|
||
# N; при одном N критерий не проверяется
|
||
head_growth = max(known) - min(known) if len(known) > 1 else None
|
||
if head_growth is not None:
|
||
ok &= head_growth <= H2_HEAD_GROWTH_MIB
|
||
for s in stages:
|
||
batch = cell_median(passes, s, "batch", "us_per_object")
|
||
head = cell_median(passes, s, "head", "us_per_object")
|
||
time_ratio = ratio(head, batch)
|
||
n = num(cell_rows(passes, s, "batch")[0]["objects"]) or 1
|
||
|
||
def per_k(cell: str, n: float = n, s: str = s) -> float | None:
|
||
total = 0.0
|
||
for column in ("s3_list", "s3_head", "s3_get", "db_sessions"):
|
||
value = cell_median(passes, s, cell, column)
|
||
if value is None:
|
||
return None
|
||
total += value
|
||
return total / n * 1000
|
||
|
||
ok &= time_ratio is not None and time_ratio >= H2_TIME_RATIO
|
||
# «~18 против ~2000» запросов S3 и сеансов на 1000 объектов
|
||
for cell, target in (
|
||
("batch", H2_BATCH_PER_K),
|
||
("head", H2_HEAD_PER_K),
|
||
):
|
||
value = per_k(cell)
|
||
ok &= (
|
||
value is not None
|
||
and abs(value - target) <= target * H2_PER_K_TOLERANCE
|
||
)
|
||
details.append(
|
||
f"{s}: HEAD / batch по времени {fmt(time_ratio, 1)}×; запросов "
|
||
f"S3 и сеансов на 1000 объектов — batch {fmt(per_k('batch'), 1)}, "
|
||
f"HEAD {fmt(per_k('head'), 0)}; пик памяти HEAD "
|
||
f"{fmt(peaks[s], 1)} МиБ",
|
||
)
|
||
details.append(
|
||
f"рост пика памяти HEAD от N: {fmt(head_growth, 1)} МиБ"
|
||
+ (" (одно N — не проверялся)" if head_growth is None else ""),
|
||
)
|
||
return Verdict(
|
||
"H2",
|
||
f"HEAD дороже `batch` по времени на объект ≥ {H2_TIME_RATIO}×; "
|
||
f"запросов S3 и сеансов на 1000 объектов — ~{H2_BATCH_PER_K} у "
|
||
f"`batch` и ~{H2_HEAD_PER_K} у HEAD (±{H2_PER_K_TOLERANCE:.0%}); "
|
||
f"пик памяти HEAD от N растёт не больше чем на {H2_HEAD_GROWTH_MIB} "
|
||
"МиБ",
|
||
"; ".join(details),
|
||
CONFIRMED if ok else REFUTED,
|
||
details,
|
||
)
|
||
|
||
|
||
def verdict_h3(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
|
||
info = meta.get("reconcile", {})
|
||
small, large = info.get("h3_small_stage"), info.get("h3_large_stage")
|
||
batch_small = cell_median(passes, small, "batch", "us_per_object")
|
||
deep_small = cell_median(passes, small, "deep", "us_per_object")
|
||
deep_large = cell_median(passes, large, "deep", "us_per_object")
|
||
fast_large = cell_median(passes, large, "deep-fast", "us_per_object")
|
||
if deep_small is None and deep_large is None:
|
||
return Verdict("H3", "глубокая дороже", "нет ячеек", UNTESTED)
|
||
checks = []
|
||
details = []
|
||
# GET на каждую строку того же размера с объектом (verify S6): строки
|
||
# `ready` без объекта и с другим размером не читаются
|
||
for stage, cell in (
|
||
(small, "deep"),
|
||
(large, "deep"),
|
||
(large, "deep-fast"),
|
||
):
|
||
rows = cell_rows(passes, stage, cell)
|
||
if not rows:
|
||
continue
|
||
row = rows[0]
|
||
want_get = int(num(row.get("ready_rows")) or 0) - sum(
|
||
int(num(row.get(f"expected_{kind}")) or 0)
|
||
for kind in ("missing_object", "size_mismatch")
|
||
)
|
||
got_get = cell_median(passes, stage, cell, "s3_get")
|
||
checks.append(got_get == want_get)
|
||
details.append(
|
||
f"{stage} {cell}: GET {fmt(got_get, 0)} (ожидалось {want_get})",
|
||
)
|
||
deep_ratio = ratio(deep_small, batch_small)
|
||
if deep_ratio is not None:
|
||
checks.append(deep_ratio >= H3_DEEP_RATIO)
|
||
details.append(
|
||
f"4 KiB: глубокая / batch по времени {fmt(deep_ratio, 1)}×",
|
||
)
|
||
sizes = info.get("object_bytes", {})
|
||
if deep_large is not None and deep_small is not None:
|
||
mib = (sizes.get(large, MIB) - sizes.get(small, 4 * KIB)) / MIB
|
||
per_mib = (deep_large - deep_small) / 1000 / mib if mib else None
|
||
checks.append(per_mib is not None and per_mib >= H3_MS_PER_MIB)
|
||
details.append(f"цена байтов {fmt(per_mib, 2)} мс на MiB")
|
||
if deep_large is not None:
|
||
cleaner = cell_median(passes, large, "deep", "cpu_cleaner")
|
||
s3 = cell_median(passes, large, "deep", "cpu_s3")
|
||
limit = num(info.get("cleaner_cpus", {}).get("stand"))
|
||
share = ratio(cleaner, limit)
|
||
checks.append(share is not None and share >= H3_CLEANER_SHARE)
|
||
checks.append(s3 is not None and s3 <= H3_S3_CORES)
|
||
details.append(
|
||
f"1 MiB на лимите стенда: уборщик {fmt(cleaner, 3)} ядра "
|
||
f"({fmt(share * 100 if share else None, 0)} % лимита), S3 "
|
||
f"{fmt(s3, 3)} ядра",
|
||
)
|
||
if fast_large is not None and deep_large is not None:
|
||
fast = ratio(deep_large, fast_large)
|
||
checks.append(fast is not None and fast >= H3_FAST_RATIO)
|
||
details.append(f"1 CPU быстрее в {fmt(fast, 2)} раза")
|
||
holds = bool(checks) and all(checks)
|
||
return Verdict(
|
||
"H3",
|
||
f"глубокая на 4 KiB дороже `batch` ≥ {H3_DEEP_RATIO}×, GET на "
|
||
f"каждую строку того же размера; ≥ {H3_MS_PER_MIB} мс на MiB; на "
|
||
"лимите стенда — CPU уборщика ≥ "
|
||
f"{H3_CLEANER_SHARE:.0%} лимита, S3 ≤ {H3_S3_CORES} ядра; с 1 CPU "
|
||
f"быстрее ≥ {H3_FAST_RATIO}×",
|
||
"; ".join(details),
|
||
CONFIRMED if holds else REFUTED,
|
||
details,
|
||
)
|
||
|
||
|
||
def verdict_h4(
|
||
window_rows: list[dict[str, Any]],
|
||
passes: Iterable[dict[str, Any]] = (),
|
||
) -> Verdict:
|
||
"""Порог решения 10. Ошибки фона — 5xx окон и пометки проходов
|
||
`bg_5xx`, `bg_failed` (`http_req_failed` итога k6), `bg_dropped`
|
||
(`dropped_iterations`): с ними H4 не подтверждается."""
|
||
ratios = load_ratios(window_rows)
|
||
if not ratios:
|
||
return Verdict(
|
||
"H4",
|
||
"медиана r ≤ 1,5",
|
||
"нет проходов под нагрузкой",
|
||
UNTESTED,
|
||
)
|
||
details = []
|
||
noisy = False
|
||
fives = False
|
||
holds = True
|
||
for cell in LOAD_CELLS:
|
||
if cell not in ratios:
|
||
continue
|
||
parts = []
|
||
for scenario, value in ratios[cell].items():
|
||
r, a = value["r"], value["a"]
|
||
parts.append(f"{scenario} r {fmt(r)} a {fmt(a)}")
|
||
if a is not None and not 1 / H4_THRESHOLD <= a <= H4_THRESHOLD:
|
||
noisy = True
|
||
if value["s5xx"]:
|
||
fives = True
|
||
if cell in LIMITED_LOAD and (r is None or r > H4_THRESHOLD):
|
||
holds = False
|
||
details.append(f"{cell}: " + ", ".join(parts))
|
||
flagged = sorted(
|
||
{
|
||
flag
|
||
for row in passes
|
||
if row.get("cell") in LOAD_CELLS
|
||
for flag in str(row.get("flags") or "").split()
|
||
if flag in BG_FLAGS
|
||
},
|
||
)
|
||
if flagged:
|
||
fives = True
|
||
details.append("пометки фона: " + ", ".join(flagged))
|
||
contrast = ratios.get("fast-unlimited", {}).get("download", {}).get("r")
|
||
if fives:
|
||
verdict = REFUTED if not holds else UNTESTED
|
||
elif noisy:
|
||
verdict = NOISY
|
||
elif contrast is None or contrast <= H4_THRESHOLD:
|
||
verdict = UNTESTED if holds else REFUTED
|
||
else:
|
||
verdict = CONFIRMED if holds else REFUTED
|
||
return Verdict(
|
||
"H4",
|
||
f"`stand`, `stand-unlimited`, `fast-limited` — медиана r ≤ "
|
||
f"{H4_THRESHOLD} по трём сценариям; контраст `fast-unlimited` — r "
|
||
f"скачиваний > {H4_THRESHOLD}; A/A a в [1/{H4_THRESHOLD}; "
|
||
f"{H4_THRESHOLD}]; без 5xx",
|
||
f"контраст r скачиваний {fmt(contrast)}",
|
||
verdict,
|
||
details,
|
||
)
|
||
|
||
|
||
def verdict_s4(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
|
||
info = meta.get("reconcile", {})
|
||
base_stage = info.get("s4_base_stage")
|
||
base = cell_median(passes, base_stage, "batch", "duration_ms")
|
||
details = []
|
||
checks = []
|
||
# «до» уже отказывает рано (commit переноса, `bench.md`): сравнивать не с
|
||
# чем, подтвердить выигрыш нельзя
|
||
same_before = False
|
||
for count in info.get("s4_candidates", []):
|
||
stage = f"S4-{count}"
|
||
before = cell_median(passes, stage, "before", "duration_ms")
|
||
after = cell_median(passes, stage, "after", "duration_ms")
|
||
heads = cell_median(passes, stage, "after", "s3_head")
|
||
heads_before = cell_median(passes, stage, "before", "s3_head")
|
||
both = [
|
||
row
|
||
for cell in ("before", "after")
|
||
for row in cell_rows(passes, stage, cell)
|
||
]
|
||
codes = {int(num(row["exit_code"]) or -1) for row in both}
|
||
# данные не менялись (verify S6): ни одного исправления в итоге и
|
||
# ни одной записи «Fixed»; нет столбца (метка до правки) — не
|
||
# доказано
|
||
unchanged = all(
|
||
num(row.get("fixed_total")) == 0
|
||
and num(row.get("fixed_records")) == 0
|
||
for row in both
|
||
)
|
||
if before is None or after is None:
|
||
continue
|
||
extra_before = None if base is None else before - base
|
||
extra_after = None if base is None else after - base
|
||
limit = info.get("s4_max_changes", 100)
|
||
checks.append(
|
||
extra_after is not None and extra_after <= S4_AFTER_EXTRA_MS,
|
||
)
|
||
checks.append(heads is not None and heads <= limit + 1)
|
||
checks.append(codes == {3})
|
||
checks.append(unchanged)
|
||
early_before = heads_before is not None and heads_before <= limit + 1
|
||
same_before = same_before or early_before
|
||
before_note = (
|
||
f"HEAD до {fmt(heads_before, 0)}: «до» уже с ранним отказом"
|
||
if early_before
|
||
else f"ожидалось ~{count * S4_MS_PER_CANDIDATE / 1000:.0f} с"
|
||
)
|
||
details.append(
|
||
f"{count} кандидатов: до {fmt(before / 1000, 1)} с (+"
|
||
f"{fmt(None if extra_before is None else extra_before / 1000, 1)} "
|
||
f"с, {before_note}), "
|
||
f"после {fmt(after / 1000, 1)} с (+"
|
||
f"{fmt(None if extra_after is None else extra_after / 1000, 2)} "
|
||
f"с), HEAD после {fmt(heads, 0)}, коды {sorted(codes)}, данные "
|
||
f"{'не менялись' if unchanged else 'менялись или не известны'}",
|
||
)
|
||
if not checks:
|
||
return Verdict("S4", "цена отказа", "нет ячеек", UNTESTED)
|
||
if not all(checks):
|
||
verdict = REFUTED
|
||
else:
|
||
verdict = UNTESTED if same_before else CONFIRMED
|
||
return Verdict(
|
||
"S4",
|
||
f"после — не дольше базы на {S4_AFTER_EXTRA_MS / 1000:.0f} с (база "
|
||
f"— поверхностный dry-run `batch` этапа {base_stage} того же N: "
|
||
"перепроверка его кандидатов и выборка зависших, без отказа по "
|
||
"пределу), HEAD ≤ предела + 1, код 3 у обоих, ни одного исправления; "
|
||
"у «до» HEAD больше предела + 1, иначе сравнивать не с чем",
|
||
"; ".join(details),
|
||
verdict,
|
||
details,
|
||
)
|
||
|
||
|
||
def verdicts(
|
||
meta: dict[str, Any],
|
||
passes: list[dict[str, Any]],
|
||
window_rows: list[dict[str, Any]],
|
||
) -> list[Verdict]:
|
||
return [
|
||
verdict_h1(meta, passes),
|
||
verdict_h2(meta, passes),
|
||
verdict_h3(meta, passes),
|
||
verdict_h4(window_rows, passes),
|
||
verdict_s4(meta, passes),
|
||
]
|
||
|
||
|
||
# ---------------------------------------------------------------- сводка
|
||
|
||
|
||
def render_summary(
|
||
meta: dict[str, Any],
|
||
passes: list[dict[str, Any]],
|
||
window_rows: list[dict[str, Any]],
|
||
) -> str:
|
||
info = meta.get("reconcile", {})
|
||
profile = meta.get("profile", {})
|
||
lines = [
|
||
f"# Профиль сверки: метка `{meta.get('label')}`",
|
||
"",
|
||
f"- commit `{meta.get('git', {}).get('commit')}`, dirty "
|
||
f"{meta.get('git', {}).get('dirty')}; статус {meta.get('status')}",
|
||
f"- профиль `{profile.get('name')}`, повторов "
|
||
f"{profile.get('repeats')}, зерно {profile.get('seed')}, grace "
|
||
f"проходов {info.get('grace_seconds')} с, предел скорости по "
|
||
f"умолчанию {info.get('default_rate')}/с",
|
||
f"- образ «до» — commit `{info.get('before_commit')}`",
|
||
f"- CPU уборщика: {info.get('cleaner_cpus')}",
|
||
f"- фоновая нагрузка: {info.get('background')}",
|
||
f"- аргументы S3: `{' '.join(info.get('s3_command') or [])}`",
|
||
f"- хост: {meta.get('host_state', {}).get('start')}; "
|
||
f"предупреждения {meta.get('host_warnings')}; посторонние "
|
||
f"контейнеры {meta.get('other_containers')}",
|
||
"",
|
||
"## Проходы без фоновой нагрузки",
|
||
"",
|
||
"Медиана и min–max повторов. Цена — микросекунд на объект bucket.",
|
||
"",
|
||
"| Этап | N | Ячейка | Проход, с | мкс/объект | LIST | HEAD | GET "
|
||
"| DELETE | Сеансов | Прочитано, МиБ | CPU уборщика | CPU S3 | CPU "
|
||
"PG | Память, МиБ (старт → пик) | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
stage_cells: dict[str, list[str]] = {}
|
||
for row in passes:
|
||
if row.get("load") in (True, "True"):
|
||
continue
|
||
cells = stage_cells.setdefault(row["stage"], [])
|
||
if row["cell"] not in cells:
|
||
cells.append(row["cell"])
|
||
for stage, cells in stage_cells.items():
|
||
for cell in cells:
|
||
rows = cell_rows(passes, stage, cell)
|
||
flags = sorted({f for r in rows for f in str(r["flags"]).split()})
|
||
lines.append(
|
||
f"| {stage} | {rows[0]['objects']} | {cell} | "
|
||
f"{spread([num(r['duration_ms']) / 1000 if num(r['duration_ms']) is not None else None for r in rows], 2)} | "
|
||
f"{spread([r['us_per_object'] for r in rows], 1)} | "
|
||
f"{spread([r['s3_list'] for r in rows], 0)} | "
|
||
f"{spread([r['s3_head'] for r in rows], 0)} | "
|
||
f"{spread([r['s3_get'] for r in rows], 0)} | "
|
||
f"{spread([r['s3_delete'] for r in rows], 0)} | "
|
||
f"{spread([r['db_sessions'] for r in rows], 0)} | "
|
||
f"{spread([num(r['read_bytes']) / MIB if num(r['read_bytes']) is not None else None for r in rows], 1)} | "
|
||
f"{spread([r['cpu_cleaner'] for r in rows], 3)} | "
|
||
f"{spread([r['cpu_s3'] for r in rows], 3)} | "
|
||
f"{spread([r['cpu_postgres'] for r in rows], 3)} | "
|
||
f"{spread([r['mem_start_mib'] for r in rows], 1)} → "
|
||
f"{spread([r['mem_peak_mib'] for r in rows], 1)} | "
|
||
f"{' '.join(flags) or '—'} |",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## Фоновая нагрузка (H4)",
|
||
"",
|
||
"r — p95 окна прохода к p95 окон до и после; a — p95 до к p95 после "
|
||
"(A/A). Медиана по повторам.",
|
||
"",
|
||
"| Ячейка | Проход, с | Сценарий | r | a | 5xx |",
|
||
"|---|---|---|---|---|---|",
|
||
]
|
||
ratios = load_ratios(window_rows)
|
||
for cell in LOAD_CELLS:
|
||
if cell not in ratios:
|
||
continue
|
||
stage = next(
|
||
(r["stage"] for r in passes if r["cell"] == cell),
|
||
"",
|
||
)
|
||
duration = cell_median(passes, stage, cell, "duration_ms")
|
||
for scenario, value in ratios[cell].items():
|
||
lines.append(
|
||
f"| {cell} | {fmt(duration / 1000 if duration else None, 1)}"
|
||
f" | {scenario} | {fmt(value['r'])} | {fmt(value['a'])} | "
|
||
f"{value['s5xx']} |",
|
||
)
|
||
lines += [
|
||
"",
|
||
"## Отказ по пределу (S4)",
|
||
"",
|
||
"| Кандидатов | Ячейка | Проход, с | HEAD | Не перепроверено | "
|
||
"Изменений в ошибке | Код |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
for count in info.get("s4_candidates", []):
|
||
stage = f"S4-{count}"
|
||
for cell in ("before", "after"):
|
||
rows = cell_rows(passes, stage, cell)
|
||
if not rows:
|
||
continue
|
||
lines.append(
|
||
f"| {count} | {cell} | "
|
||
f"{spread([num(r['duration_ms']) / 1000 if num(r['duration_ms']) is not None else None for r in rows], 2)} | "
|
||
f"{spread([r['s3_head'] for r in rows], 0)} | "
|
||
f"{spread([r['unchecked'] for r in rows], 0)} | "
|
||
f"{spread([r['limit_changes'] for r in rows], 0)} | "
|
||
f"{sorted({r['exit_code'] for r in rows})} |",
|
||
)
|
||
lines += ["", "## Гипотезы", ""]
|
||
for item in verdicts(meta, passes, window_rows):
|
||
lines.append(f"### {item.name}: {item.verdict}")
|
||
lines += ["", f"- критерий: {item.criterion}"]
|
||
lines.append(f"- измерено: {item.measured}")
|
||
lines += [f" - {line}" for line in item.details]
|
||
lines.append("")
|
||
lines += [
|
||
"Графики: `cost-by-n.svg`, `requests-by-n.svg`, `background-p95.svg`.",
|
||
"",
|
||
]
|
||
return "\n".join(lines)
|
||
|
||
|
||
def render_plots(
|
||
meta: dict[str, Any],
|
||
passes: list[dict[str, Any]],
|
||
window_rows: list[dict[str, Any]],
|
||
) -> dict[str, str]:
|
||
"""SVG метки: время прохода от N, запросы от N (лог. шкала), r фоновой
|
||
нагрузки по ячейкам и сценариям."""
|
||
import plots
|
||
|
||
info = meta.get("reconcile", {})
|
||
stages = info.get("h1_stages", [])
|
||
points: dict[str, list[tuple[float, float]]] = {}
|
||
for cell in ("batch", "head", "deep"):
|
||
for stage in stages:
|
||
value = cell_median(passes, stage, cell, "duration_ms")
|
||
n = stage_n(meta, stage)
|
||
if value is not None and n:
|
||
points.setdefault(cell, []).append((n, value / 1000))
|
||
out = {}
|
||
svg = plots.Svg(720, 420, "Время прохода сверки от числа объектов")
|
||
out["cost-by-n.svg"] = xy_chart(
|
||
svg,
|
||
points,
|
||
"объектов в bucket",
|
||
"проход, с",
|
||
)
|
||
req: dict[str, list[tuple[float, float]]] = {}
|
||
for cell in ("batch", "head"):
|
||
for stage in stages:
|
||
n = stage_n(meta, stage)
|
||
s3 = sum(
|
||
cell_median(passes, stage, cell, f"s3_{op}") or 0
|
||
for op in S3_OPS
|
||
)
|
||
db = cell_median(passes, stage, cell, "db_sessions")
|
||
if n and cell_rows(passes, stage, cell):
|
||
req.setdefault(f"{cell}: S3", []).append((n, max(s3, 1)))
|
||
if db:
|
||
req.setdefault(f"{cell}: PostgreSQL", []).append((n, db))
|
||
svg = plots.Svg(720, 420, "Запросы прохода от числа объектов")
|
||
out["requests-by-n.svg"] = xy_chart(
|
||
svg,
|
||
req,
|
||
"объектов в bucket",
|
||
"запросов (лог. шкала)",
|
||
log=True,
|
||
)
|
||
out["background-p95.svg"] = ratio_chart(load_ratios(window_rows))
|
||
return out
|
||
|
||
|
||
def xy_chart(
|
||
svg: Any,
|
||
series: dict[str, list[tuple[float, float]]],
|
||
x_title: str,
|
||
y_title: str,
|
||
*,
|
||
log: bool = False,
|
||
) -> str:
|
||
import plots
|
||
|
||
left, top, width, height = 80, 60, 600, 290
|
||
xs = [x for pts in series.values() for x, _ in pts] or [1]
|
||
ys = [y for pts in series.values() for _, y in pts] or [1]
|
||
x_hi = max(xs) * 1.05
|
||
if log:
|
||
y_lo = 10 ** math.floor(math.log10(max(min(ys), 1)))
|
||
y_hi = 10 ** math.ceil(math.log10(max(ys) * 1.01))
|
||
ticks = plots.log_ticks(y_lo, y_hi)
|
||
else:
|
||
ticks = plots.nice_ticks(max(ys) * 1.1)
|
||
y_lo, y_hi = 0.0, ticks[-1]
|
||
|
||
def sx(x: float) -> float:
|
||
return left + width * x / x_hi
|
||
|
||
def sy(y: float) -> float:
|
||
if log:
|
||
span = math.log10(y_hi) - math.log10(y_lo)
|
||
share = (math.log10(max(y, y_lo)) - math.log10(y_lo)) / span
|
||
else:
|
||
share = (y - y_lo) / (y_hi - y_lo) if y_hi > y_lo else 0
|
||
return top + height * (1 - share)
|
||
|
||
for tick in ticks:
|
||
svg.line(left, sy(tick), left + width, sy(tick))
|
||
svg.text(left - 8, sy(tick) + 4, plots.compact(tick), anchor="end")
|
||
for x in sorted(set(xs)):
|
||
svg.text(sx(x), top + height + 18, plots.compact(x), anchor="middle")
|
||
svg.text(left + width / 2, top + height + 40, x_title, anchor="middle")
|
||
svg.text(24, top + height / 2, y_title, anchor="middle", rotate=True)
|
||
legend = []
|
||
for index, (name, pts) in enumerate(series.items()):
|
||
color = plots.SERIES[index % len(plots.SERIES)]
|
||
ordered = sorted(pts)
|
||
svg.polyline([(sx(x), sy(y)) for x, y in ordered], color)
|
||
for x, y in ordered:
|
||
svg.dot(sx(x), sy(y), color, f"{name}: N {x:g}, {y:g}")
|
||
legend.append((color, name, "line"))
|
||
svg.legend(legend, left, 48, width)
|
||
return svg.render()
|
||
|
||
|
||
def ratio_chart(ratios: dict[str, dict[str, dict[str, Any]]]) -> str:
|
||
import plots
|
||
|
||
svg = plots.Svg(720, 420, "Фоновая нагрузка: r = p95 прохода / p95 тихих")
|
||
left, top, width, height = 80, 70, 600, 280
|
||
cells = [cell for cell in LOAD_CELLS if cell in ratios]
|
||
values = [
|
||
v["r"]
|
||
for cell in cells
|
||
for v in ratios[cell].values()
|
||
if v["r"] is not None
|
||
]
|
||
high = max([H4_THRESHOLD * 1.2, *values])
|
||
ticks = plots.nice_ticks(high)
|
||
|
||
def sy(y: float) -> float:
|
||
return top + height * (1 - y / ticks[-1])
|
||
|
||
for tick in ticks:
|
||
svg.line(left, sy(tick), left + width, sy(tick))
|
||
svg.text(left - 8, sy(tick) + 4, f"{tick:g}", anchor="end")
|
||
# полоса A/A: [1/1,5; 1,5] и линия порога
|
||
svg.parts.append(
|
||
f'<rect x="{left}" y="{sy(H4_THRESHOLD):.1f}" width="{width}" '
|
||
f'height="{sy(1 / H4_THRESHOLD) - sy(H4_THRESHOLD):.1f}" '
|
||
f'fill="{plots.GRID}" opacity="0.5"/>',
|
||
)
|
||
svg.line(
|
||
left,
|
||
sy(H4_THRESHOLD),
|
||
left + width,
|
||
sy(H4_THRESHOLD),
|
||
color=plots.INK_2,
|
||
dash="5 4",
|
||
)
|
||
group = width / max(1, len(cells))
|
||
bar = group / (len(SCENARIOS) + 1)
|
||
for gi, cell in enumerate(cells):
|
||
x0 = left + gi * group + bar / 2
|
||
for si, scenario in enumerate(SCENARIOS.values()):
|
||
r = ratios[cell].get(scenario, {}).get("r")
|
||
if r is None:
|
||
continue
|
||
color = plots.SERIES[si]
|
||
svg.bar(
|
||
x0 + si * bar,
|
||
sy(r),
|
||
bar * 0.9,
|
||
sy(0) - sy(r),
|
||
color,
|
||
f"{cell} {scenario}: r {r:.2f}",
|
||
top=True,
|
||
)
|
||
svg.text(
|
||
left + gi * group + group / 2,
|
||
top + height + 18,
|
||
cell,
|
||
anchor="middle",
|
||
)
|
||
svg.legend(
|
||
[
|
||
(plots.SERIES[i], name, "box")
|
||
for i, name in enumerate(SCENARIOS.values())
|
||
]
|
||
+ [(plots.INK_2, f"порог {H4_THRESHOLD}", "dash")],
|
||
left,
|
||
56,
|
||
width,
|
||
)
|
||
return svg.render()
|
||
|
||
|
||
# ------------------------------------------------------------ самопроверка
|
||
|
||
|
||
def selftest() -> int:
|
||
failures: list[str] = []
|
||
|
||
def expect(name: str, got: Any, want: Any) -> None:
|
||
if got != want:
|
||
failures.append(f"{name}: got {got!r}, want {want!r}")
|
||
|
||
# набор: классы, id, содержимое, модель
|
||
expect("class 0", object_class(0), "orphan_object")
|
||
expect("class 3", object_class(403), "stale_pending")
|
||
expect("class ok", object_class(4), None)
|
||
expect("force ok", object_class(0, FORCE_OK), None)
|
||
expect("force missing", object_class(5, FORCE_MISSING), "missing_object")
|
||
first = object_id(7, 1)
|
||
expect("id deterministic", first, object_id(7, 1))
|
||
expect("id v4", uuid.UUID(first).version, 4)
|
||
expect("id lower", first, str(uuid.UUID(first)))
|
||
expect("content size", len(content(7, 1, 4096)), 4096)
|
||
expect("content differs", content(7, 1, 64) == content(7, 2, 64), False)
|
||
m = model(10_000)
|
||
expect("model found", m.found, dict.fromkeys(CLASSES, 25))
|
||
expect("model objects", m.objects, 9975)
|
||
expect("model ready", m.ready_rows, 9950)
|
||
expect("model 1000", model(1000).found["orphan_object"], 3)
|
||
expect("model 1000 stale", model(1000).found["stale_pending"], 3)
|
||
expect("model 1000 ready", model(1000).ready_rows, 994)
|
||
s4 = model(0, clean=10_000, missing=1000)
|
||
expect(
|
||
"s4 model",
|
||
(s4.objects, s4.ready_rows, s4.found),
|
||
(
|
||
10_000,
|
||
11_000,
|
||
{"missing_object": 1000},
|
||
),
|
||
)
|
||
text = rows_csv(
|
||
seed=7,
|
||
start=0,
|
||
end=800,
|
||
size=4096,
|
||
namespace="bench",
|
||
chat_id="c",
|
||
owner_id="o",
|
||
created_at="2026-10-22T00:00:00+00:00",
|
||
)
|
||
lines = text.splitlines()
|
||
expect("csv rows", len(lines), 800 - 2)
|
||
expect("csv columns", len(lines[0].split(",")), len(COPY_COLUMNS))
|
||
by_index = {line.split(",")[3]: line.split(",") for line in lines}
|
||
expect("csv size mismatch", by_index["bench-2.png"][5], "4097")
|
||
expect("csv pending", by_index["bench-3.png"][8], "pending")
|
||
expect("csv no orphan row", "bench-0.png" in by_index, False)
|
||
expect(
|
||
"csv sha",
|
||
by_index["bench-4.png"][6],
|
||
"\\x" + hashlib.sha256(content(7, 4, 4096)).hexdigest(),
|
||
)
|
||
expect("objects without missing", 1 in object_indices(0, 10), False)
|
||
expect("objects count", len(object_indices(0, 800)), 798)
|
||
picked = downloads(7, 2000, 5000)
|
||
expect("downloads only consistent", len(picked), 2000 - 20)
|
||
|
||
# план кругами
|
||
plan = plan_passes([("A", ("x", "y", "z")), ("B", ("x", "y"))], 3, 5)
|
||
expect("plan length", len(plan), 15)
|
||
expect("plan unique", len({p["pass"] for p in plan}), 15)
|
||
expect("plan stage order", [p["stage"] for p in plan][:9], ["A"] * 9)
|
||
for i in range(1, len(plan)):
|
||
same = plan[i]["cell"] == plan[i - 1]["cell"]
|
||
if same and plan[i]["stage"] == plan[i - 1]["stage"]:
|
||
failures.append(f"plan repeats a cell in a row at {i}")
|
||
|
||
# записи уборщика
|
||
log = "\n".join(
|
||
[
|
||
"Container mtusi-bench-reconcile-1 Creating",
|
||
json.dumps({"event": STARTED, "reconcile_strategy": "batch"}),
|
||
json.dumps({"event": FOUND, "kind": "orphan_object"}),
|
||
json.dumps({"event": FOUND, "kind": "orphan_object"}),
|
||
json.dumps(
|
||
{
|
||
"event": FINISHED,
|
||
"found": {"orphan_object": 2},
|
||
"duration_ms": 1500,
|
||
"timestamp": "2026-10-23T01:02:03.456789Z",
|
||
"s3_requests": {
|
||
"list": 3,
|
||
"head": 2,
|
||
"get": 0,
|
||
"delete": 0,
|
||
},
|
||
"phase_ms": {"list": 1000, "rows": 200},
|
||
"listed_objects": 1000,
|
||
"storage_objects": 1003,
|
||
},
|
||
),
|
||
],
|
||
)
|
||
kept = kept_records(parse_records(log))
|
||
expect("kept found count", kept["found_records"], 2)
|
||
expect("kept finished", kept["finished"]["duration_ms"], 1500)
|
||
expect(
|
||
"timestamp",
|
||
timestamp("2026-10-23T01:02:03.456789Z"),
|
||
1792717323.456789,
|
||
)
|
||
expect(
|
||
"timestamp nano",
|
||
timestamp("2026-10-23T01:02:03.456789123Z"),
|
||
1792717323.456789,
|
||
)
|
||
problems = found_problems(
|
||
{"pass": "B-batch-r1"},
|
||
{"orphan_object": 2, "missing_object": 1},
|
||
kept,
|
||
)
|
||
expect(
|
||
"found problem",
|
||
problems,
|
||
["B-batch-r1: missing_object found 0, injected 1"],
|
||
)
|
||
expect(
|
||
"found ok",
|
||
found_problems({"pass": "p"}, {"orphan_object": 2}, kept),
|
||
[],
|
||
)
|
||
|
||
# метрики шлюза
|
||
metrics_a = (
|
||
'# HELP x\nSeaweedFS_s3_request_total{bucket="bench",code="200",'
|
||
'type="LIST"} 4\nSeaweedFS_s3_request_total{bucket="bench",'
|
||
'code="200",type="GET"} 10\nSeaweedFS_s3_request_total{bucket='
|
||
'"bench",code="404",type="GET"} 2\nother_metric 5\n'
|
||
)
|
||
metrics_b = metrics_a.replace("} 4", "} 7").replace("} 10", "} 11")
|
||
delta = metrics_delta(parse_metrics(metrics_a), parse_metrics(metrics_b))
|
||
expect("metrics delta", delta, {"GET": 1, "LIST": 3})
|
||
expect(
|
||
"gateway equal",
|
||
gateway_mismatch({"list": 3, "head": 1, "get": 0, "delete": 0}, delta),
|
||
False,
|
||
)
|
||
expect(
|
||
"gateway differ",
|
||
gateway_mismatch({"list": 3, "head": 2, "get": 0, "delete": 0}, delta),
|
||
True,
|
||
)
|
||
expect("pg stat", parse_pg_stat("1|2|3|4|5|6")["pg_sessions"], 6)
|
||
|
||
# фоновая нагрузка, окна и отношения
|
||
reqs = []
|
||
for second in range(30):
|
||
for kind, base in (("d", 10), ("w", 20), ("r", 5)):
|
||
duration = base * (3 if 10 <= second < 20 else 1)
|
||
reqs.append(
|
||
Request(kind, second * 1000 + 1, duration, OK_STATUS[kind]),
|
||
)
|
||
reqs.append(Request("d", 15_000, 1, 503))
|
||
console = "\n".join(
|
||
f"bg,{r.kind},{r.start_ms},{r.duration_ms},{r.status}" for r in reqs
|
||
)
|
||
parsed = parse_background(console + "\nnoise\nbg,x,1,2,3\n")
|
||
expect("bg parsed", len(parsed), len(reqs))
|
||
# полнота строк: снизу — итерации и счётчик, сверху — прерванные ≤ VU
|
||
counts = {
|
||
kind: sum(1 for r in parsed if r.kind == kind) for kind in SCENARIOS
|
||
}
|
||
|
||
def summary(iterations_less: int, counter_less: int) -> dict[str, Any]:
|
||
"""Итог k6, где у скачиваний итераций и отсчётов счётчика на
|
||
столько меньше строк (отрицательное — больше)."""
|
||
metrics = {}
|
||
for kind, scenario in SCENARIOS.items():
|
||
shift = (iterations_less, counter_less) if kind == "d" else (0, 0)
|
||
metrics[f"bg_lines_{scenario}"] = {
|
||
"values": {"count": counts[kind] - shift[1]},
|
||
}
|
||
metrics[f"iterations{{scenario:{scenario}}}"] = {
|
||
"values": {"count": counts[kind] - shift[0]},
|
||
}
|
||
return {"metrics": metrics}
|
||
|
||
def problems_of(iterations_less: int, counter_less: int) -> list[str]:
|
||
return line_problems(
|
||
parsed,
|
||
summary(iterations_less, counter_less),
|
||
max_vus=5,
|
||
)[0]
|
||
|
||
expect("lines ok", problems_of(0, 0), [])
|
||
# метка aaf3fa3: строка прерванной итерации, отсчёт счётчика есть
|
||
problems, interrupted = line_problems(parsed, summary(1, 0), max_vus=5)
|
||
expect(
|
||
"lines interrupted, counted",
|
||
(problems, interrupted["download"]),
|
||
([], 1),
|
||
)
|
||
# метка a42dd27: строка на 1 больше и счётчика, и итераций
|
||
expect("lines interrupted, not counted", problems_of(1, 1), [])
|
||
expect("lines lost vs counter", len(problems_of(0, -1)), 1)
|
||
expect("lines lost vs iterations", len(problems_of(-1, 0)), 1)
|
||
expect(
|
||
"lost message names direction",
|
||
"lines lost" in problems_of(-1, 0)[0],
|
||
True,
|
||
)
|
||
expect("lines too many interrupted", len(problems_of(6, 0)), 1)
|
||
expect("lines too many over both", len(problems_of(6, 6)), 2)
|
||
expect(
|
||
"lines no counter",
|
||
len(line_problems(parsed, {"metrics": {}}, max_vus=5)[0]),
|
||
3,
|
||
)
|
||
# верхний допуск — не больше VU, созданных k6 (`vus_max` итога)
|
||
capped = summary(4, 0)
|
||
capped["metrics"]["vus_max"] = {"values": {"max": 3}}
|
||
expect(
|
||
"lines over vus_max",
|
||
len(line_problems(parsed, capped, max_vus=5)[0]),
|
||
1,
|
||
)
|
||
capped["metrics"]["vus_max"] = {"values": {"max": 300}}
|
||
expect(
|
||
"lines vus_max above scenario VU",
|
||
line_problems(parsed, capped, max_vus=5)[0],
|
||
[],
|
||
)
|
||
# ошибки фона по всему итогу k6
|
||
k6_errors = {
|
||
"metrics": {
|
||
"http_req_failed": {"values": {"passes": 2, "fails": 10}},
|
||
"dropped_iterations": {"values": {"count": 1}},
|
||
},
|
||
}
|
||
expect(
|
||
"bg errors",
|
||
background_errors(k6_errors),
|
||
{"failed": 2, "dropped": 1},
|
||
)
|
||
expect(
|
||
"bg flags",
|
||
background_flags(background_errors(k6_errors)),
|
||
["bg_failed", "bg_dropped"],
|
||
)
|
||
expect("bg no errors", background_flags(background_errors({})), [])
|
||
bounds = {"before": (0, 10), "pass": (10, 20), "after": (20, 30)}
|
||
rows, ratios_one = windows_of(parsed, bounds)
|
||
expect("window rows", len(rows), 12)
|
||
expect("r download", ratios_one["download"]["r"], 3.0)
|
||
expect("a download", ratios_one["download"]["a"], 1.0)
|
||
expect("5xx download", ratios_one["download"]["s5xx"], 1)
|
||
expect("percentile", percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 95), 10.0)
|
||
|
||
# окно CPU и память по отсчётам сэмплера (нс, usage_usec, …, memory)
|
||
points = [
|
||
(int(t * 1e9), int(t * 250_000), 0, 0, 0, (100 + t) * MIB, 0)
|
||
for t in (0, 1, 2, 3, 4)
|
||
]
|
||
expect("window cpu", window_cpu(points, 0.5, 3.5), 0.25)
|
||
# короткий проход после последнего отсчёта — по охватывающим отсчётам
|
||
expect("window cpu tail", window_cpu(points, 3.8, 4.3), 0.25)
|
||
expect("memory", memory_start_peak(points, 2.5), (102.0, 104.0))
|
||
|
||
# строка прохода и пометка count_mismatch
|
||
entry = {
|
||
"pass": "B-batch-r1",
|
||
"stage": "B",
|
||
"cell": "batch",
|
||
"repeat": 1,
|
||
"order": 1,
|
||
}
|
||
row = pass_row(
|
||
entry,
|
||
cell=CELLS["batch"],
|
||
expected=model(1000),
|
||
object_bytes=4096,
|
||
kept=kept,
|
||
exit_code=3,
|
||
gateway={"LIST": 3, "GET": 3},
|
||
pg=None,
|
||
cpu={"cleaner": 0.25},
|
||
memory=(90.0, 91.0),
|
||
cleaner_cpus="0.25",
|
||
rate=0,
|
||
image="checkout",
|
||
)
|
||
expect("row flags", row["flags"], "count_mismatch")
|
||
expect(
|
||
"row us per object",
|
||
row["us_per_object"],
|
||
round(1_500_000 / 997, 2),
|
||
)
|
||
expect("row fields", set(row) <= set(PASS_FIELDS), True)
|
||
expect("row fixed", (row["fixed_total"], row["fixed_records"]), (0, 0))
|
||
|
||
# вердикты и сводка на синтетических проходах
|
||
meta = {
|
||
"label": "selftest",
|
||
"profile": {"name": "reconcile_quick", "repeats": 1},
|
||
"reconcile": {
|
||
"h1_stages": ["B", "C"],
|
||
"h2_stages": ["B"],
|
||
"h3_small_stage": "B",
|
||
"h3_large_stage": "A",
|
||
"s4_base_stage": "B",
|
||
"s4_candidates": [300],
|
||
"s4_max_changes": 100,
|
||
"cleaner_cpus": {"stand": "0.25", "fast": "1"},
|
||
"object_bytes": {"A": MIB, "B": 4 * KIB, "C": 4 * KIB},
|
||
"stages": [
|
||
{"stage": "B", "objects_total": 10_000},
|
||
{"stage": "C", "objects_total": 30_000},
|
||
],
|
||
},
|
||
}
|
||
|
||
def synthetic(
|
||
stage: str,
|
||
cell: str,
|
||
n: int,
|
||
us: float,
|
||
**extra: Any,
|
||
) -> dict[str, Any]:
|
||
mdl = model(n)
|
||
base = dict.fromkeys(PASS_FIELDS)
|
||
base.update(
|
||
{
|
||
"pass": f"{stage}-{cell}-r1",
|
||
"stage": stage,
|
||
"cell": cell,
|
||
"load": cell in LOAD_CELLS,
|
||
"objects": mdl.objects,
|
||
"ready_rows": mdl.ready_rows,
|
||
"us_per_object": us,
|
||
"duration_ms": us * mdl.objects / 1000,
|
||
"s3_list": math.ceil(mdl.objects / 1000) + 2,
|
||
"s3_head": sum(
|
||
mdl.found[k]
|
||
for k in (
|
||
"orphan_object",
|
||
"missing_object",
|
||
"size_mismatch",
|
||
)
|
||
),
|
||
"s3_get": 0,
|
||
"s3_delete": 0,
|
||
"mem_start_mib": 97.0,
|
||
"mem_peak_mib": 97.0 + mdl.objects * 150 / MIB,
|
||
"mem_growth_bytes_per_object": 150.0,
|
||
"exit_code": 3,
|
||
"flags": "",
|
||
"fixed_total": 0,
|
||
"fixed_records": 0,
|
||
**{f"expected_{k}": mdl.found.get(k, 0) for k in CLASSES},
|
||
},
|
||
)
|
||
base["db_sessions"] = expected_batch_requests(base)[2]
|
||
base.update(extra)
|
||
return base
|
||
|
||
b_batch = synthetic("B", "batch", 10_000, 300)
|
||
expect(
|
||
"batch requests B",
|
||
expected_batch_requests(b_batch),
|
||
(12, 75, 96),
|
||
)
|
||
expect(
|
||
"batch requests D",
|
||
expected_batch_requests(synthetic("D", "batch", 100_000, 300)),
|
||
(102, 750, 953),
|
||
)
|
||
passes = [
|
||
b_batch,
|
||
synthetic("C", "batch", 30_000, 310),
|
||
synthetic(
|
||
"B",
|
||
"head",
|
||
10_000,
|
||
5000,
|
||
mem_peak_mib=98.0,
|
||
s3_head=9950,
|
||
db_sessions=10061,
|
||
),
|
||
synthetic("B", "deep", 10_000, 2700, s3_get=9900),
|
||
synthetic(
|
||
"A",
|
||
"deep",
|
||
1000,
|
||
10_500,
|
||
cpu_cleaner=0.245,
|
||
cpu_s3=0.19,
|
||
s3_get=988,
|
||
),
|
||
synthetic("A", "deep-fast", 1000, 4800, s3_get=988),
|
||
synthetic(
|
||
"S4-300",
|
||
"before",
|
||
10_000,
|
||
0,
|
||
duration_ms=4500,
|
||
exit_code=3,
|
||
s3_head=300,
|
||
),
|
||
synthetic("S4-300", "after", 10_000, 0, duration_ms=3200, s3_head=101),
|
||
]
|
||
found = {
|
||
item.name: item.verdict
|
||
for item in verdicts(meta, passes, rows_with_cells(rows))
|
||
}
|
||
expect("verdict H1", found["H1"], CONFIRMED)
|
||
expect("verdict H2", found["H2"], CONFIRMED)
|
||
expect("verdict H3", found["H3"], CONFIRMED)
|
||
expect("verdict S4", found["S4"], CONFIRMED)
|
||
expect("verdict H4 contrast", found["H4"], UNTESTED)
|
||
slow = [*passes[:1], synthetic("C", "batch", 30_000, 500), *passes[2:]]
|
||
found = {item.name: item.verdict for item in verdicts(meta, slow, [])}
|
||
expect("verdict H1 refuted", found["H1"], REFUTED)
|
||
|
||
def refuted(name: str, index: int, **change: Any) -> None:
|
||
"""Вердикт `name` опровергнут, если у прохода `index` изменить
|
||
столбцы `change` (критерии verify S6)."""
|
||
changed = [dict(row) for row in passes]
|
||
changed[index].update(change)
|
||
verdict = {
|
||
item.name: item.verdict for item in verdicts(meta, changed, [])
|
||
}[name]
|
||
expect(f"verdict {name} refuted by {sorted(change)}", verdict, REFUTED)
|
||
|
||
refuted("H1", 0, db_sessions=97)
|
||
refuted("H1", 0, s3_list=13)
|
||
refuted("H2", 2, db_sessions=500)
|
||
refuted("H3", 3, s3_get=9950)
|
||
refuted("S4", 7, fixed_total=1)
|
||
refuted("S4", 6, fixed_records=None)
|
||
# «до» с тем же ранним отказом: S4 не подтверждена и не ждёт полной цены
|
||
early = [dict(row) for row in passes]
|
||
early[6]["s3_head"] = 101
|
||
early_s4 = verdict_s4(meta, early)
|
||
expect("verdict S4 same before", early_s4.verdict, UNTESTED)
|
||
expect(
|
||
"S4 same before: no expected",
|
||
"ожидалось" in early_s4.measured,
|
||
False,
|
||
)
|
||
expect(
|
||
"S4 old before: expected",
|
||
"ожидалось" in verdict_s4(meta, passes).measured,
|
||
True,
|
||
)
|
||
# H4: без 5xx окон контраст подтверждает, пометка фона — нет
|
||
clean, _ = windows_of([r for r in parsed if r.status < 500], bounds)
|
||
expect(
|
||
"verdict H4 clean",
|
||
verdict_h4(rows_with_cells(clean)).verdict,
|
||
CONFIRMED,
|
||
)
|
||
expect(
|
||
"verdict H4 bg_failed",
|
||
verdict_h4(
|
||
rows_with_cells(clean),
|
||
[{"cell": "fast-unlimited", "flags": "bg_failed"}],
|
||
).verdict,
|
||
UNTESTED,
|
||
)
|
||
summary = render_summary(meta, passes, rows_with_cells(rows))
|
||
for needle in ("## Гипотезы", "### H1", "### S4", "| B | 9975 | batch"):
|
||
if needle not in summary:
|
||
failures.append(f"summary lacks {needle!r}")
|
||
svgs = render_plots(meta, passes, rows_with_cells(rows))
|
||
expect(
|
||
"plots",
|
||
sorted(svgs),
|
||
[
|
||
"background-p95.svg",
|
||
"cost-by-n.svg",
|
||
"requests-by-n.svg",
|
||
],
|
||
)
|
||
for name, text in svgs.items():
|
||
if not text.startswith("<svg"):
|
||
failures.append(f"{name} is not svg")
|
||
|
||
for failure in failures:
|
||
print(f"FAIL {failure}", flush=True)
|
||
print(
|
||
f"selftest: {'ok' if not failures else f'{len(failures)} failed'}",
|
||
flush=True,
|
||
)
|
||
return 1 if failures else 0
|
||
|
||
|
||
def rows_with_cells(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
"""Самопроверка: строки окон одного прохода ячейки `fast-unlimited`
|
||
с окном «тихих» для `load_ratios`."""
|
||
out = []
|
||
for row in rows:
|
||
out.append(
|
||
{"pass": "D-fast-unlimited-r1", "cell": "fast-unlimited", **row},
|
||
)
|
||
return out
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if sys.argv[1:] == ["--selftest"]:
|
||
sys.exit(selftest())
|
||
print("usage: uv run python bench/reconcile.py --selftest", flush=True)
|
||
sys.exit(2)
|