Files
mtusi2026lab6/bench/reconcile.py
T
ricoromaandClaude Opus 5.5 9cc6c0c7a6 chore: port lab 5 solution into the lab 6 skeleton
Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e.
Contracts, public tests and evidence of lab 5 are not carried over.
The compose project, image and CI prefix are renamed to lab6; the first
broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the
OpenSpec changes of lab 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 04:46:56 +03:00

2296 lines
86 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль сверки: набор, план, разбор и сводка без docker.
Оркестрирует проходы `run.py` (профили `reconcile`, `reconcile_quick`),
здесь — чистые функции на stdlib: модель набора (id, содержимое и класс
объекта по номеру и зерну), строки CSV метаданных для `\\copy`, план
ячеек кругами, разбор записей уборщика, метрик шлюза S3, строк фоновой
нагрузки k6 и отсчётов сэмплера, окна и отношения p95 гипотезы 4, строки
`passes.csv` и `windows.csv`, вердикты H1–H4 и S4, `summary.md` и графики
SVG. Методика — design change `track2-reconcile-cost`, решения 1–3, 8–13;
правило — `.claude/rules/bench.md`, «Профиль сверки». Проверка —
`uv run python bench/reconcile.py --selftest`.
Генератор набора импортирует и загрузчик `bench/fill_objects.py` в
контейнере на образе стенда: модуль на верхнем уровне берёт только
stdlib, графики (`plots`) — внутри функций.
"""
import hashlib
import json
import math
import random
import re
import statistics
import sys
import uuid
from collections.abc import Iterable
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
# класс объекта i — по i mod PERIOD: 0 — объект без метаданных, 1 —
# метаданные без объекта, 2 — другой размер, 3 — зависшая загрузка; по
# 0,25 % на класс (решение 2)
PERIOD = 400
CLASSES = ("orphan_object", "missing_object", "size_mismatch", "stale_pending")
# принудительный класс диапазона (набор S4): все согласованы или все —
# метаданные без объекта
FORCE_OK = "ok"
FORCE_MISSING = "missing_object"
CONTENT_TYPE = "image/png"
KIB = 1024
MIB = 2**20
# размер объекта набора отказа по пределу (S4)
S4_OBJECT_BYTES = 4 * KIB
# столбцы `\copy attachments`: created_at и updated_at — одно значение
COPY_COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"object_key",
"status",
"created_at",
"updated_at",
)
# ячейки: стратегия, глубина, CPU уборщика («stand» — лимит рабочего
# стенда, «fast» — управляемый опыт), предел скорости («default» — из
# настроек, иначе число), фоновая нагрузка (решения 1 и 10)
CELLS: dict[str, dict[str, Any]] = {
"batch": {"strategy": "batch", "deep": False, "cpu": "stand", "rate": 0},
"head": {"strategy": "head", "deep": False, "cpu": "stand", "rate": 0},
"deep": {"strategy": "batch", "deep": True, "cpu": "stand", "rate": 0},
"deep-fast": {
"strategy": "batch",
"deep": True,
"cpu": "fast",
"rate": 0,
},
"stand": {
"strategy": "batch",
"deep": False,
"cpu": "stand",
"rate": "default",
"load": True,
},
"stand-unlimited": {
"strategy": "batch",
"deep": False,
"cpu": "stand",
"rate": 0,
"load": True,
},
"fast-unlimited": {
"strategy": "batch",
"deep": False,
"cpu": "fast",
"rate": 0,
"load": True,
},
"fast-limited": {
"strategy": "batch",
"deep": False,
"cpu": "fast",
"rate": "default",
"load": True,
},
# отказ по пределу (решение 7): образ до change и checkout, --apply
# --max-changes 100; у checkout — без предела скорости
"before": {"apply": True, "before": True, "cpu": "stand", "rate": None},
"after": {
"strategy": "batch",
"deep": False,
"apply": True,
"cpu": "stand",
"rate": 0,
},
}
LOAD_CELLS = ("stand", "stand-unlimited", "fast-unlimited", "fast-limited")
LIMITED_LOAD = ("stand", "stand-unlimited", "fast-limited")
# пометки ошибок фоновой нагрузки у проходов блока H4: 5xx окон, неуспешные
# запросы и недосланные итерации по всему итогу k6 (verify S9)
BG_FLAGS = ("bg_5xx", "bg_failed", "bg_dropped")
# сценарии фоновой нагрузки: скачивания, запись, чтение истории
SCENARIOS = {"d": "download", "w": "write", "r": "read"}
WINDOWS = ("before", "pass", "after")
# пороги гипотез (решение 12)
H1_SPREAD = 0.20
H1_BYTES_PER_KEY = 300
H2_TIME_RATIO = 10
H2_HEAD_GROWTH_MIB = 5
# H2: запросов S3 и сеансов PostgreSQL на 1000 объектов — «~18 против
# ~2000» при 1 % расхождений, допуск ±25 % (verify S6)
H2_BATCH_PER_K = 18
H2_HEAD_PER_K = 2000
H2_PER_K_TOLERANCE = 0.25
H3_DEEP_RATIO = 5
H3_MS_PER_MIB = 5
H3_CLEANER_SHARE = 0.9
H3_S3_CORES = 0.5
H3_FAST_RATIO = 1.5
H4_THRESHOLD = 1.5
S4_AFTER_EXTRA_MS = 1000
S4_MS_PER_CANDIDATE = 5
# размер страницы списка и строк (`[reconciler] page_size`) и выборки
# зависших (`[cleaner] batch_size`) — значения по умолчанию, их берут
# проходы стенда; по ним H1 считает ожидаемые LIST и сеансы
PAGE_SIZE = 1000
STALE_BATCH = 100
# страниц списка staging за проход: staging набора пуст, строки и объекты
# пишет оркестратор мимо API
STAGING_PAGES = 1
# записи журнала уборщика, которые идут в raw/<проход>-cleaner.json
STARTED = "Started cleaner"
FINISHED = "Finished reconcile pass"
EXCEEDED = "Reconcile changes exceed limit"
FOUND = "Found storage discrepancy"
FIXED = "Fixed storage discrepancy"
# операции S3 в итоге прохода и типы запросов шлюза: HEAD шлюз считает
# типом GET (проба, design, Context)
S3_OPS = ("list", "head", "get", "delete")
GATEWAY_TYPES = ("LIST", "GET", "DELETE")
PASS_FIELDS = [
"pass",
"stage",
"cell",
"repeat",
"order",
"strategy",
"deep",
"apply",
"image",
"cleaner_cpus",
"max_objects_per_second",
"load",
"objects",
"object_bytes",
"ready_rows",
"candidates",
*(f"expected_{kind}" for kind in CLASSES),
*(f"found_{kind}" for kind in CLASSES),
"found_checksum_mismatch",
"found_other",
"listed_objects",
"checked_rows",
"exit_code",
"duration_ms",
"phase_list_ms",
"phase_rows_ms",
"phase_stale_ms",
"phase_recheck_ms",
"phase_act_ms",
"us_per_object",
"storage_objects",
*(f"s3_{op}" for op in S3_OPS),
"read_bytes",
"db_sessions",
"unchecked",
"refused",
"limit_changes",
"gw_list",
"gw_get",
"gw_delete",
"gw_other",
"pg_xact",
"pg_tup_returned",
"pg_tup_fetched",
"pg_blks_hit",
"pg_blks_read",
"pg_sessions",
"cpu_cleaner",
"cpu_s3",
"cpu_postgres",
"cpu_api",
"cpu_haproxy",
"mem_start_mib",
"mem_peak_mib",
"mem_growth_bytes_per_object",
"flags",
# с verify S6: исправлено проходом по итогу и записей «Fixed» —
# неизменность данных у отказа по пределу (S4)
"fixed_total",
"fixed_records",
]
WINDOW_FIELDS = [
"pass",
"stage",
"cell",
"repeat",
"window",
"scenario",
"start_s",
"end_s",
"responses",
"errors",
"s5xx",
"p50_ms",
"p95_ms",
"p99_ms",
]
# ------------------------------------------------------------------ набор
def object_class(index: int, force: str | None = None) -> str | None:
"""Класс расхождения объекта `index`; `None` — согласован."""
if force == FORCE_OK:
return None
if force is not None:
return force
slot = index % PERIOD
return CLASSES[slot] if slot < len(CLASSES) else None
def object_id(seed: int, index: int) -> str:
"""UUID v4 из SHA-256 строки `<зерно>:<номер>` — запись сервера."""
digest = hashlib.sha256(f"{seed}:{index}".encode()).digest()
return str(uuid.UUID(bytes=digest[:16], version=4))
def content(seed: int, index: int, size: int) -> bytes:
"""Повторённый SHA-256 строки `<зерно>:<номер>:c`, обрезанный до
размера."""
block = hashlib.sha256(f"{seed}:{index}:c".encode()).digest()
return (block * (size // len(block) + 1))[:size]
def object_key(namespace: str, attachment_id: str) -> str:
return f"{namespace}/attachments/{attachment_id}/original"
def has_object(kind: str | None) -> bool:
return kind != "missing_object"
def has_row(kind: str | None) -> bool:
return kind != "orphan_object"
def object_indices(
start: int,
end: int,
force: str | None = None,
) -> list[int]:
"""Номера объектов диапазона, у которых есть объект в bucket."""
return [i for i in range(start, end) if has_object(object_class(i, force))]
def rows_csv(
*,
seed: int,
start: int,
end: int,
size: int,
namespace: str,
chat_id: str,
owner_id: str,
created_at: str,
force: str | None = None,
) -> str:
"""Строки CSV `\\copy attachments (COPY_COLUMNS)` диапазона:
`ready` — у согласованных, метаданных без объекта и другого размера
(`size_bytes` на байт больше), `pending` — у зависшей загрузки; объект
без метаданных строки не имеет. SHA-256 — от того же содержимого."""
lines = []
for index in range(start, end):
kind = object_class(index, force)
if not has_row(kind):
continue
attachment_id = object_id(seed, index)
digest = hashlib.sha256(content(seed, index, size)).hexdigest()
size_bytes = size + 1 if kind == "size_mismatch" else size
status = "pending" if kind == "stale_pending" else "ready"
lines.append(
",".join(
(
attachment_id,
chat_id,
owner_id,
f"bench-{index}.png",
CONTENT_TYPE,
str(size_bytes),
f"\\x{digest}",
object_key(namespace, attachment_id),
status,
created_at,
created_at,
),
),
)
return "\n".join(lines) + ("\n" if lines else "")
def class_count(n: int, slot: int) -> int:
"""Сколько номеров `i < n` дают `i mod PERIOD == slot`."""
return n // PERIOD + (1 if n % PERIOD > slot else 0)
@dataclass(frozen=True)
class Model:
"""Ожидаемое у набора: объекты в bucket, строки `ready`, расхождения
по классам (внесено)."""
objects: int
ready_rows: int
pending_rows: int
found: dict[str, int]
@property
def candidates(self) -> int:
return sum(self.found.values())
def model(n: int, *, clean: int = 0, missing: int = 0) -> Model:
"""Набор из `n` объектов с классами по номеру; у набора S4 — `clean`
согласованных и `missing` строк без объектов (тогда `n` не важен)."""
if clean or missing:
return Model(
objects=clean,
ready_rows=clean + missing,
pending_rows=0,
found={"missing_object": missing} if missing else {},
)
found = {kind: class_count(n, slot) for slot, kind in enumerate(CLASSES)}
return Model(
objects=n - found["missing_object"],
ready_rows=n - found["orphan_object"] - found["stale_pending"],
pending_rows=found["stale_pending"],
found={kind: count for kind, count in found.items() if count},
)
def downloads(seed: int, n: int, count: int) -> list[str]:
"""`id` согласованных вложений набора для фоновых скачиваний: выборка
от зерна (решение 10)."""
pool = [i for i in range(n) if object_class(i) is None]
rng = random.Random(seed)
chosen = rng.sample(pool, min(count, len(pool)))
return [object_id(seed, i) for i in chosen]
# ------------------------------------------------------------------- план
def plan_passes(
stages: Iterable[tuple[str, tuple[str, ...]]],
repeats: int,
seed: int,
) -> list[dict[str, Any]]:
"""Проходы этапов кругами: в круге r — перестановка ячеек этапа
генератором `Random(seed + номер этапа × 100 + r)`, круг не начинается
с ячейки, которой кончился прежний (как `uploads.plan_runs`). У группы
из одной ячейки (D без нагрузки — только `batch`) повторы идут подряд:
остальные проходы набора — блок под нагрузкой (спек `load-benchmark`,
«Профиль сверки», verify S8)."""
plan: list[dict[str, Any]] = []
for stage_no, (stage, cells) in enumerate(stages):
previous = None
for round_no in range(1, repeats + 1):
order = list(range(len(cells)))
random.Random(seed + stage_no * 100 + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for index in order:
cell = cells[index]
plan.append(
{
"pass": f"{stage}-{cell}-r{round_no}",
"stage": stage,
"cell": cell,
"repeat": round_no,
"order": len(plan) + 1,
},
)
previous = order[-1]
return plan
# ----------------------------------------------------------------- разбор
def parse_records(text: str) -> list[dict[str, Any]]:
"""JSON-записи журнала уборщика; остальное (вывод compose) — мимо."""
records = []
for line in text.splitlines():
line = line.strip()
if not line.startswith("{"):
continue
try:
records.append(json.loads(line))
except json.JSONDecodeError:
continue
return records
def pick(records: list[dict[str, Any]], event: str) -> dict[str, Any] | None:
found = [item for item in records if item.get("event") == event]
return found[-1] if found else None
def kept_records(records: list[dict[str, Any]]) -> dict[str, Any]:
"""`raw/<проход>-cleaner.json`: записи старта, итога и предела; записи
о расхождениях — только числом (их до тысячи на проход)."""
return {
"started": pick(records, STARTED),
"finished": pick(records, FINISHED),
"exceeded": pick(records, EXCEEDED),
"found_records": sum(1 for r in records if r.get("event") == FOUND),
"fixed_records": sum(1 for r in records if r.get("event") == FIXED),
}
def timestamp(value: str) -> float:
"""ISO 8601 с `Z` (записи structlog, Docker) → секунды эпохи; дробь
Docker — до наносекунд, Python берёт микросекунды."""
text = value.strip().replace("Z", "+00:00")
match = re.match(r"(.*\.\d{6})\d*(.*)", text)
if match:
text = match.group(1) + match.group(2)
return datetime.fromisoformat(text).timestamp()
def parse_metrics(text: str) -> dict[str, float]:
"""`SeaweedFS_s3_request_total` по типам запроса (сумма по прочим
меткам)."""
totals: dict[str, float] = {}
for line in text.splitlines():
if not line.startswith("SeaweedFS_s3_request_total{"):
continue
kind = re.search(r'type="([^"]+)"', line)
try:
value = float(line.rsplit(" ", 1)[1])
except (IndexError, ValueError):
continue
name = kind.group(1) if kind else "?"
totals[name] = totals.get(name, 0.0) + value
return totals
def metrics_delta(
before: dict[str, float],
after: dict[str, float],
) -> dict[str, int]:
names = set(before) | set(after)
return {
name: round(after.get(name, 0.0) - before.get(name, 0.0))
for name in sorted(names)
}
def gateway_mismatch(
s3_requests: dict[str, int],
gateway: dict[str, int],
) -> bool:
"""Счёт сверки и метрики шлюза: LIST = list, GET = head + get, DELETE
= delete; запросов других типов быть не должно (решение 8)."""
expected = {
"LIST": s3_requests.get("list", 0),
"GET": s3_requests.get("head", 0) + s3_requests.get("get", 0),
"DELETE": s3_requests.get("delete", 0),
}
other = sum(v for k, v in gateway.items() if k not in GATEWAY_TYPES)
return other != 0 or any(
gateway.get(name, 0) != value for name, value in expected.items()
)
PG_STAT_SQL = (
"SELECT xact_commit + xact_rollback, tup_returned, tup_fetched, "
"blks_hit, blks_read, sessions FROM pg_stat_database "
"WHERE datname = current_database();\n"
)
PG_STAT_NAMES = (
"pg_xact",
"pg_tup_returned",
"pg_tup_fetched",
"pg_blks_hit",
"pg_blks_read",
"pg_sessions",
)
def parse_pg_stat(text: str) -> dict[str, int]:
values = [int(v) for v in text.strip().split("|")]
return dict(zip(PG_STAT_NAMES, values, strict=True))
@dataclass(frozen=True)
class Request:
"""Запрос фоновой нагрузки: сценарий, отправка (мс часов ВМ),
длительность и статус."""
kind: str
start_ms: int
duration_ms: float
status: int
def parse_background(text: str) -> list[Request]:
"""Строки `bg,<d|w|r>,<отправка мс>,<длительность мс>,<статус>`."""
out = []
for line in text.splitlines():
line = line.strip()
if not line.startswith("bg,"):
continue
parts = line.split(",")
if len(parts) != 5 or parts[1] not in SCENARIOS:
continue
try:
out.append(
Request(
kind=parts[1],
start_ms=int(parts[2]),
duration_ms=float(parts[3]),
status=int(parts[4]),
),
)
except ValueError:
continue
return out
def summary_count(k6_data: dict[str, Any], name: str) -> int | None:
value = k6_data.get("metrics", {}).get(name, {}).get("values", {})
count = value.get("count")
return None if count is None else int(count)
def background_errors(k6_data: dict[str, Any]) -> dict[str, int]:
"""Ошибки фоновой нагрузки по всему итогу k6, а не по окнам (verify
S9): неуспешные запросы (`http_req_failed`: у Rate-метрики `passes` —
число неуспешных) и недосланные итерации (`dropped_iterations`)."""
metrics = k6_data.get("metrics", {})
failed = metrics.get("http_req_failed", {}).get("values", {})
dropped = metrics.get("dropped_iterations", {}).get("values", {})
return {
"failed": int(failed.get("passes", 0) or 0),
"dropped": int(dropped.get("count", 0) or 0),
}
def background_flags(errors: dict[str, int]) -> list[str]:
"""Пометки проходов блока по `background_errors`."""
flags = []
if errors.get("failed"):
flags.append("bg_failed")
if errors.get("dropped"):
flags.append("bg_dropped")
return flags
def line_problems(
requests: list[Request],
k6_data: dict[str, Any],
*,
max_vus: int,
) -> tuple[list[str], dict[str, int]]:
"""Полнота строк `bg,…` по итогу k6 (design, «Уточнения apply», п.
15). Итерацию, которую остановка SIGINT прервала уже после строки, k6
не считает ни в `iterations`, ни (если прервал до сброса отсчёта) в
счётчике `bg_lines_<сценарий>`: у прерванной строка может быть, а
отсчётов нет. Поэтому снизу — строк не меньше итераций и не меньше
счётчика (у каждой завершённой итерации строка есть: пропажу видно),
сверху — лишних строк против каждого из них не больше VU сценария и
не больше VU, созданных k6 за прогон (`vus_max` итога): столько
итераций может быть в полёте. Возвращает расхождения и число
прерванных (строк сверх итераций) по сценариям."""
problems = []
interrupted: dict[str, int] = {}
# в полёте не больше VU, созданных k6 за прогон (`vus_max` итога —
# на все сценарии), и не больше VU сценария (verify S4)
created = k6_data.get("metrics", {}).get("vus_max", {}).get("values", {})
if created.get("max") is not None:
max_vus = min(max_vus, int(created["max"]))
for kind, scenario in SCENARIOS.items():
lines = sum(1 for r in requests if r.kind == kind)
counted = summary_count(k6_data, f"bg_lines_{scenario}")
iterations = summary_count(
k6_data,
f"iterations{{scenario:{scenario}}}",
)
if counted is None or iterations is None:
problems.append(
f"{scenario}: no bg_lines or iterations in the k6 summary",
)
continue
interrupted[scenario] = lines - iterations
for name, value in (("iterations", iterations), ("bg_lines", counted)):
if lines < value:
problems.append(
f"{scenario}: {lines} k6 lines, {value} {name} "
f"(lines lost: {value - lines})",
)
elif lines - value > max_vus:
problems.append(
f"{scenario}: {lines} k6 lines, {value} {name} (extra "
f"{lines - value} > {max_vus} VU: not interruptions)",
)
return problems, interrupted
def percentile(values: list[float], q: float) -> float | None:
"""Перцентиль ближайшего ранга, как у k6 (p(95))."""
if not values:
return None
ordered = sorted(values)
rank = max(1, math.ceil(q / 100 * len(ordered)))
return float(ordered[rank - 1])
OK_STATUS = {"d": 200, "w": 201, "r": 200}
def window_stats(
requests: list[Request],
kind: str,
start: float,
end: float,
) -> dict[str, Any]:
"""Ответы сценария, отправленные в [start, end) (секунды часов ВМ):
число, ошибки, 5xx и перцентили успешных."""
inside = [
r
for r in requests
if r.kind == kind and start * 1000 <= r.start_ms < end * 1000
]
ok = [r.duration_ms for r in inside if r.status == OK_STATUS[kind]]
return {
"responses": len(inside),
"errors": len(inside) - len(ok),
"s5xx": sum(1 for r in inside if 500 <= r.status < 600),
"p50_ms": percentile(ok, 50),
"p95_ms": percentile(ok, 95),
"p99_ms": percentile(ok, 99),
}
def ratio(a: float | None, b: float | None) -> float | None:
if a is None or b is None or b <= 0:
return None
return round(a / b, 3)
def windows_of(
requests: list[Request],
bounds: dict[str, tuple[float, float]],
) -> tuple[list[dict[str, Any]], dict[str, dict[str, float | None]]]:
"""Строки окон прохода по сценариям и отношения: `r` — p95 окна
прохода к p95 окон до и после вместе (строка `quiet`), `a` — p95 до к
p95 после."""
rows = []
ratios: dict[str, dict[str, float | None]] = {}
for kind, scenario in SCENARIOS.items():
stats = {}
for name in WINDOWS:
w0, w1 = bounds[name]
stats[name] = window_stats(requests, kind, w0, w1)
rows.append(
{
"window": name,
"scenario": scenario,
"start_s": round(w0, 3),
"end_s": round(w1, 3),
**stats[name],
},
)
quiet = [
r
for r in requests
if any(
bounds[name][0] * 1000 <= r.start_ms < bounds[name][1] * 1000
for name in ("before", "after")
)
]
stats["quiet"] = window_stats(quiet, kind, -math.inf, math.inf)
rows.append(
{
"window": "quiet",
"scenario": scenario,
"start_s": round(bounds["before"][0], 3),
"end_s": round(bounds["after"][1], 3),
**stats["quiet"],
},
)
if "list" in bounds:
# шаг списка (`phase_ms.list` от начала прохода): справочно, без
# вердикта — в нём у `batch` нагружен S3 (решение 10)
w0, w1 = bounds["list"]
rows.append(
{
"window": "list",
"scenario": scenario,
"start_s": round(w0, 3),
"end_s": round(w1, 3),
**window_stats(requests, kind, w0, w1),
},
)
ratios[scenario] = {
"r": ratio(stats["pass"]["p95_ms"], stats["quiet"]["p95_ms"]),
"a": ratio(stats["before"]["p95_ms"], stats["after"]["p95_ms"]),
"s5xx": sum(stats[name]["s5xx"] for name in WINDOWS),
}
return rows, ratios
# ------------------------------------------------------------- сэмплер
def usage_at(points: list[tuple[int, ...]], t: float) -> float | None:
"""`usage_usec` контейнера на момент `t` (секунды): линейно между
соседними отсчётами; вне отсчётов — None."""
if not points:
return None
ns = t * 1e9
if ns < points[0][0] or ns > points[-1][0]:
return None
for prev, cur in zip(points, points[1:], strict=False):
if prev[0] <= ns <= cur[0]:
if cur[0] == prev[0]:
return float(cur[1])
share = (ns - prev[0]) / (cur[0] - prev[0])
return prev[1] + (cur[1] - prev[1]) * share
return float(points[-1][1])
def window_cpu(
points: list[tuple[int, ...]],
w0: float,
w1: float,
) -> float | None:
"""Ядер в среднем за окно [w0, w1]: разность `usage_usec` на концах.
Окно у края отсчётов — по ближайшим отсчётам внутри. Короткий проход
в конце жизни контейнера (последний отсчёт раньше конца окна, внутри
меньше половины окна) — по отсчётам, охватывающим окно: среднее за
чуть большее время, а не пропуск."""
if w1 <= w0 or len(points) < 2:
return None
first, last = points[0][0] / 1e9, points[-1][0] / 1e9
a, b = max(w0, first), min(w1, last)
if b - a >= (w1 - w0) / 2:
ua, ub = usage_at(points, a), usage_at(points, b)
if ua is not None and ub is not None:
return round((ub - ua) / 1e6 / (b - a), 4)
before = [p for p in points if p[0] <= w0 * 1e9]
after = [p for p in points if p[0] >= w1 * 1e9]
p0 = before[-1] if before else points[0]
p1 = after[0] if after else points[-1]
if p1[0] <= p0[0]:
return None
return round((p1[1] - p0[1]) * 1e3 / (p1[0] - p0[0]), 4)
def memory_start_peak(
points: list[tuple[int, ...]],
w0: float,
) -> tuple[float | None, float | None]:
"""Память контейнера (МиБ): последняя выборка не позже начала окна и
наибольшая за жизнь (решение 9)."""
if not points:
return None, None
before = [p for p in points if p[0] <= w0 * 1e9]
start = (before[-1] if before else points[0])[5] / MIB
peak = max(p[5] for p in points) / MIB
return round(start, 1), round(peak, 1)
# ---------------------------------------------------------- строка прохода
def pass_row(
entry: dict[str, Any],
*,
cell: dict[str, Any],
expected: Model,
object_bytes: int,
kept: dict[str, Any],
exit_code: int,
gateway: dict[str, int] | None,
pg: dict[str, int] | None,
cpu: dict[str, float | None],
memory: tuple[float | None, float | None],
cleaner_cpus: str,
rate: float | None,
image: str,
) -> dict[str, Any]:
"""Строка `passes.csv`: модель и найденное, итог прохода, метрики
шлюза и PostgreSQL, CPU и память за окно; пометки — `count_mismatch`
(счёт сверки ≠ метрикам шлюза в ячейке без нагрузки)."""
finished = kept.get("finished") or {}
exceeded = kept.get("exceeded") or {}
found = finished.get("found") or {}
phases = finished.get("phase_ms") or {}
s3 = finished.get("s3_requests") or {}
duration = finished.get("duration_ms")
row: dict[str, Any] = {
**{key: entry[key] for key in ("pass", "stage", "cell", "repeat")},
"order": entry["order"],
"strategy": finished.get("strategy", cell.get("strategy", "batch")),
"deep": finished.get("deep", cell.get("deep", False)),
"apply": bool(cell.get("apply")),
"image": image,
"cleaner_cpus": cleaner_cpus,
"max_objects_per_second": finished.get(
"max_objects_per_second",
rate,
),
"load": bool(cell.get("load")),
"objects": expected.objects,
"object_bytes": object_bytes,
"ready_rows": expected.ready_rows,
"candidates": expected.candidates,
"listed_objects": finished.get("listed_objects"),
"checked_rows": finished.get("checked_rows"),
"exit_code": exit_code,
"duration_ms": duration,
"us_per_object": (
round(duration * 1000 / expected.objects, 2)
if duration is not None and expected.objects
else None
),
"storage_objects": finished.get("storage_objects"),
"read_bytes": finished.get("read_bytes"),
"db_sessions": finished.get("db_sessions"),
"unchecked": finished.get("unchecked"),
"refused": finished.get("refused"),
"limit_changes": exceeded.get("changes"),
"cpu_cleaner": cpu.get("cleaner"),
"cpu_s3": cpu.get("s3"),
"cpu_postgres": cpu.get("postgres"),
"cpu_api": cpu.get("api"),
"cpu_haproxy": cpu.get("haproxy"),
"mem_start_mib": memory[0],
"mem_peak_mib": memory[1],
}
for kind in CLASSES:
row[f"expected_{kind}"] = expected.found.get(kind, 0)
row[f"found_{kind}"] = found.get(kind, 0)
row["found_checksum_mismatch"] = found.get("checksum_mismatch", 0)
row["found_other"] = sum(
count
for kind, count in found.items()
if kind not in CLASSES and kind != "checksum_mismatch"
)
for name in ("list", "rows", "stale", "recheck", "act"):
row[f"phase_{name}_ms"] = phases.get(name)
for op in S3_OPS:
row[f"s3_{op}"] = s3.get(op)
if memory[0] is not None and memory[1] is not None and expected.objects:
row["mem_growth_bytes_per_object"] = round(
(memory[1] - memory[0]) * MIB / expected.objects,
1,
)
flags = []
if gateway is not None:
row["gw_list"] = gateway.get("LIST", 0)
row["gw_get"] = gateway.get("GET", 0)
row["gw_delete"] = gateway.get("DELETE", 0)
row["gw_other"] = sum(
v for k, v in gateway.items() if k not in GATEWAY_TYPES
)
if s3 and not cell.get("load") and gateway_mismatch(s3, gateway):
flags.append("count_mismatch")
if pg is not None:
row.update(pg)
row["flags"] = " ".join(flags)
row["fixed_total"] = sum((finished.get("fixed") or {}).values())
row["fixed_records"] = kept.get("fixed_records")
return row
def found_problems(
entry: dict[str, Any],
expected: dict[str, int],
kept: dict[str, Any],
) -> list[str]:
"""Найдено не то, что внесено: класс, ячейка и оба числа (спек
`load-benchmark`, «Найдено не то, что внесено»)."""
finished = kept.get("finished")
if finished is None:
return [f"{entry['pass']}: no «{FINISHED}» record"]
found = finished.get("found") or {}
problems = []
for kind in sorted(set(expected) | set(found)):
if found.get(kind, 0) != expected.get(kind, 0):
problems.append(
f"{entry['pass']}: {kind} found {found.get(kind, 0)}, "
f"injected {expected.get(kind, 0)}",
)
return problems
# ------------------------------------------------------------- агрегаты
def num(value: Any) -> float | None:
if value is None or value == "":
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def med(values: Iterable[Any]) -> float | None:
present = [v for v in (num(x) for x in values) if v is not None]
return statistics.median(present) if present else None
def spread(values: Iterable[Any], digits: int = 1) -> str:
present = [v for v in (num(x) for x in values) if v is not None]
if not present:
return "—"
mid = statistics.median(present)
if len(present) == 1:
return f"{mid:.{digits}f}"
return (
f"{mid:.{digits}f} ({min(present):.{digits}f}–"
f"{max(present):.{digits}f})"
)
def cell_rows(
passes: list[dict[str, Any]],
stage: str,
cell: str,
) -> list[dict[str, Any]]:
return [
row
for row in passes
if row.get("stage") == stage and row.get("cell") == cell
]
def cell_median(
passes: list[dict[str, Any]],
stage: str,
cell: str,
column: str,
) -> float | None:
return med(row.get(column) for row in cell_rows(passes, stage, cell))
def load_ratios(
window_rows: list[dict[str, Any]],
) -> dict[str, dict[str, dict[str, float | None]]]:
"""Медиана `r` и `a` по повторам: ячейка → сценарий → значение;
`s5xx` — сумма."""
by_pass: dict[str, dict[str, Any]] = {}
for row in window_rows:
item = by_pass.setdefault(
row["pass"],
{"cell": row["cell"], "stats": {}},
)
item["stats"][(row["scenario"], row["window"])] = row
values: dict[str, dict[str, dict[str, list[float]]]] = {}
fives: dict[str, dict[str, int]] = {}
for item in by_pass.values():
cell = item["cell"]
for scenario in SCENARIOS.values():
stats = item["stats"]
p = {
name: num(stats.get((scenario, name), {}).get("p95_ms"))
for name in WINDOWS
}
quiet = num(stats.get((scenario, "quiet"), {}).get("p95_ms"))
r = ratio(p["pass"], quiet)
a = ratio(p["before"], p["after"])
bucket = values.setdefault(cell, {}).setdefault(
scenario,
{"r": [], "a": []},
)
if r is not None:
bucket["r"].append(r)
if a is not None:
bucket["a"].append(a)
fives.setdefault(cell, {}).setdefault(scenario, 0)
fives[cell][scenario] += sum(
int(num(stats.get((scenario, name), {}).get("s5xx")) or 0)
for name in WINDOWS
)
return {
cell: {
scenario: {
"r": med(v["r"]),
"a": med(v["a"]),
"s5xx": fives[cell][scenario],
}
for scenario, v in scenarios.items()
}
for cell, scenarios in values.items()
}
# -------------------------------------------------------------- вердикты
CONFIRMED = "подтверждена"
REFUTED = "опровергнута"
UNTESTED = "не проверена"
NOISY = "не различимо на этом хосте"
@dataclass
class Verdict:
"""Строка гипотезы: имя, критерий, измерено, вывод."""
name: str
criterion: str
measured: str
verdict: str
details: list[str] = field(default_factory=list)
def fmt(value: float | None, digits: int = 2) -> str:
return "—" if value is None else f"{value:.{digits}f}"
def stage_n(meta: dict[str, Any], stage: str) -> int | None:
for item in meta.get("reconcile", {}).get("stages", []):
if item["stage"] == stage:
return item["objects_total"]
return None
def expected_batch_requests(row: dict[str, Any]) -> tuple[int, int, int]:
"""LIST, HEAD и сеансы PostgreSQL поверхностного dry-run `batch` по
модели прохода (H1): страницы списка (страница без ключей — только у
пустого bucket), одна страница списка staging (пустая: загрузок в наборе
нет, сеанса с БД не даёт; design change `pipeline-reliability`, решение
16) и проба bucket, если есть кандидаты без объекта или с другим размером;
HEAD — по одному на такого кандидата и объект без строки; сеанс — на
страницу списка с ключами, на каждую страницу строк `ready` и выборку
зависших (последняя — неполная) и на перепроверку кандидата, кроме
зависших."""
def count(column: str) -> int:
return int(num(row.get(column)) or 0)
objects, ready = count("objects"), count("ready_rows")
orphan, missing = (
count("expected_orphan_object"),
count(
"expected_missing_object",
),
)
size, stale = (
count("expected_size_mismatch"),
count(
"expected_stale_pending",
),
)
probe = 1 if missing or size else 0
lists = max(1, math.ceil(objects / PAGE_SIZE)) + STAGING_PAGES + probe
heads = orphan + missing + size
sessions = (
math.ceil(objects / PAGE_SIZE)
+ ready // PAGE_SIZE
+ 1
+ stale // STALE_BATCH
+ 1
+ heads
)
return lists, heads, sessions
def verdict_h1(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
"""Линейность `batch`: цена на объект, запросы S3, сеансы, память."""
stages = [
s
for s in meta.get("reconcile", {}).get("h1_stages", [])
if cell_rows(passes, s, "batch")
]
if len(stages) < 2:
return Verdict(
"H1",
"цена на объект `batch` при трёх N в пределах 20 %",
"нет ячеек",
UNTESTED,
)
costs = {
s: cell_median(passes, s, "batch", "us_per_object") for s in stages
}
present = [v for v in costs.values() if v]
spread_ratio = max(present) / min(present) - 1 if present else None
details = []
ok_requests = True
for s in stages:
row = cell_rows(passes, s, "batch")[0]
n = int(num(row["objects"]) or 0) + int(
num(row["expected_missing_object"]) or 0,
)
want_list, want_head, want_sessions = expected_batch_requests(row)
got_list = cell_median(passes, s, "batch", "s3_list")
got_head = cell_median(passes, s, "batch", "s3_head")
got_sessions = cell_median(passes, s, "batch", "db_sessions")
# счёт точный: набор неизменен, допуска нет (verify S6)
ok_requests &= got_list == want_list
ok_requests &= got_head == want_head
ok_requests &= got_sessions == want_sessions
details.append(
f"{s} (N = {n}): {fmt(costs[s])} мкс на объект, LIST "
f"{fmt(got_list, 0)} (ожидалось {want_list}), HEAD "
f"{fmt(got_head, 0)} (ожидалось {want_head}), сеансов "
f"{fmt(got_sessions, 0)} (ожидалось {want_sessions})",
)
slope = memory_slope(passes, stages, "batch")
memory_ok = slope is not None and slope <= H1_BYTES_PER_KEY
holds = (
spread_ratio is not None
and spread_ratio <= H1_SPREAD
and ok_requests
and memory_ok
)
spread_pct = None if spread_ratio is None else spread_ratio * 100
return Verdict(
"H1",
f"цена на объект `batch` различается не больше чем на "
f"{H1_SPREAD:.0%}; LIST = ⌈N/1000⌉ + 2 (страницы вложений, страница "
"staging, проба bucket), HEAD — только у кандидатов; "
"сеансов — по одному на страницу списка, страницу строк `ready` и "
"выборку зависших плюс по одному на кандидата, кроме зависших; "
f"прирост пика памяти уборщика от N ≤ {H1_BYTES_PER_KEY} Б на ключ",
f"разброс цены {fmt(spread_pct, 1)} %, наклон пика памяти "
f"{fmt(slope, 0)} Б на ключ",
CONFIRMED if holds else REFUTED,
details,
)
def memory_slope(
passes: list[dict[str, Any]],
stages: list[str],
cell: str,
) -> float | None:
"""Прирост пика памяти уборщика на объект между наименьшим и
наибольшим набором (Б на ключ): постоянная часть прохода (модели
botocore, пул соединений) в разность не входит."""
points = []
for stage in stages:
rows = cell_rows(passes, stage, cell)
peak = cell_median(passes, stage, cell, "mem_peak_mib")
if rows and peak is not None:
points.append((num(rows[0]["objects"]) or 0, peak))
if len(points) < 2:
return None
(n0, m0), (n1, m1) = min(points), max(points)
if n1 <= n0:
return None
return (m1 - m0) * MIB / (n1 - n0)
def verdict_h2(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
stages = [
s
for s in meta.get("reconcile", {}).get("h2_stages", [])
if cell_rows(passes, s, "batch") and cell_rows(passes, s, "head")
]
if not stages:
return Verdict(
"H2",
f"HEAD дороже `batch` по времени на объект ≥ {H2_TIME_RATIO}×",
"нет ячеек",
UNTESTED,
)
details = []
ok = True
peaks = {s: cell_median(passes, s, "head", "mem_peak_mib") for s in stages}
known = [v for v in peaks.values() if v is not None]
# память HEAD от N не растёт: разность пиков при наибольшем и наименьшем
# N; при одном N критерий не проверяется
head_growth = max(known) - min(known) if len(known) > 1 else None
if head_growth is not None:
ok &= head_growth <= H2_HEAD_GROWTH_MIB
for s in stages:
batch = cell_median(passes, s, "batch", "us_per_object")
head = cell_median(passes, s, "head", "us_per_object")
time_ratio = ratio(head, batch)
n = num(cell_rows(passes, s, "batch")[0]["objects"]) or 1
def per_k(cell: str, n: float = n, s: str = s) -> float | None:
total = 0.0
for column in ("s3_list", "s3_head", "s3_get", "db_sessions"):
value = cell_median(passes, s, cell, column)
if value is None:
return None
total += value
return total / n * 1000
ok &= time_ratio is not None and time_ratio >= H2_TIME_RATIO
# «~18 против ~2000» запросов S3 и сеансов на 1000 объектов
for cell, target in (
("batch", H2_BATCH_PER_K),
("head", H2_HEAD_PER_K),
):
value = per_k(cell)
ok &= (
value is not None
and abs(value - target) <= target * H2_PER_K_TOLERANCE
)
details.append(
f"{s}: HEAD / batch по времени {fmt(time_ratio, 1)}×; запросов "
f"S3 и сеансов на 1000 объектов — batch {fmt(per_k('batch'), 1)}, "
f"HEAD {fmt(per_k('head'), 0)}; пик памяти HEAD "
f"{fmt(peaks[s], 1)} МиБ",
)
details.append(
f"рост пика памяти HEAD от N: {fmt(head_growth, 1)} МиБ"
+ (" (одно N — не проверялся)" if head_growth is None else ""),
)
return Verdict(
"H2",
f"HEAD дороже `batch` по времени на объект ≥ {H2_TIME_RATIO}×; "
f"запросов S3 и сеансов на 1000 объектов — ~{H2_BATCH_PER_K} у "
f"`batch` и ~{H2_HEAD_PER_K} у HEAD (±{H2_PER_K_TOLERANCE:.0%}); "
f"пик памяти HEAD от N растёт не больше чем на {H2_HEAD_GROWTH_MIB} "
"МиБ",
"; ".join(details),
CONFIRMED if ok else REFUTED,
details,
)
def verdict_h3(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
info = meta.get("reconcile", {})
small, large = info.get("h3_small_stage"), info.get("h3_large_stage")
batch_small = cell_median(passes, small, "batch", "us_per_object")
deep_small = cell_median(passes, small, "deep", "us_per_object")
deep_large = cell_median(passes, large, "deep", "us_per_object")
fast_large = cell_median(passes, large, "deep-fast", "us_per_object")
if deep_small is None and deep_large is None:
return Verdict("H3", "глубокая дороже", "нет ячеек", UNTESTED)
checks = []
details = []
# GET на каждую строку того же размера с объектом (verify S6): строки
# `ready` без объекта и с другим размером не читаются
for stage, cell in (
(small, "deep"),
(large, "deep"),
(large, "deep-fast"),
):
rows = cell_rows(passes, stage, cell)
if not rows:
continue
row = rows[0]
want_get = int(num(row.get("ready_rows")) or 0) - sum(
int(num(row.get(f"expected_{kind}")) or 0)
for kind in ("missing_object", "size_mismatch")
)
got_get = cell_median(passes, stage, cell, "s3_get")
checks.append(got_get == want_get)
details.append(
f"{stage} {cell}: GET {fmt(got_get, 0)} (ожидалось {want_get})",
)
deep_ratio = ratio(deep_small, batch_small)
if deep_ratio is not None:
checks.append(deep_ratio >= H3_DEEP_RATIO)
details.append(
f"4 KiB: глубокая / batch по времени {fmt(deep_ratio, 1)}×",
)
sizes = info.get("object_bytes", {})
if deep_large is not None and deep_small is not None:
mib = (sizes.get(large, MIB) - sizes.get(small, 4 * KIB)) / MIB
per_mib = (deep_large - deep_small) / 1000 / mib if mib else None
checks.append(per_mib is not None and per_mib >= H3_MS_PER_MIB)
details.append(f"цена байтов {fmt(per_mib, 2)} мс на MiB")
if deep_large is not None:
cleaner = cell_median(passes, large, "deep", "cpu_cleaner")
s3 = cell_median(passes, large, "deep", "cpu_s3")
limit = num(info.get("cleaner_cpus", {}).get("stand"))
share = ratio(cleaner, limit)
checks.append(share is not None and share >= H3_CLEANER_SHARE)
checks.append(s3 is not None and s3 <= H3_S3_CORES)
details.append(
f"1 MiB на лимите стенда: уборщик {fmt(cleaner, 3)} ядра "
f"({fmt(share * 100 if share else None, 0)} % лимита), S3 "
f"{fmt(s3, 3)} ядра",
)
if fast_large is not None and deep_large is not None:
fast = ratio(deep_large, fast_large)
checks.append(fast is not None and fast >= H3_FAST_RATIO)
details.append(f"1 CPU быстрее в {fmt(fast, 2)} раза")
holds = bool(checks) and all(checks)
return Verdict(
"H3",
f"глубокая на 4 KiB дороже `batch` ≥ {H3_DEEP_RATIO}×, GET на "
f"каждую строку того же размера; ≥ {H3_MS_PER_MIB} мс на MiB; на "
"лимите стенда — CPU уборщика ≥ "
f"{H3_CLEANER_SHARE:.0%} лимита, S3 ≤ {H3_S3_CORES} ядра; с 1 CPU "
f"быстрее ≥ {H3_FAST_RATIO}×",
"; ".join(details),
CONFIRMED if holds else REFUTED,
details,
)
def verdict_h4(
window_rows: list[dict[str, Any]],
passes: Iterable[dict[str, Any]] = (),
) -> Verdict:
"""Порог решения 10. Ошибки фона — 5xx окон и пометки проходов
`bg_5xx`, `bg_failed` (`http_req_failed` итога k6), `bg_dropped`
(`dropped_iterations`): с ними H4 не подтверждается."""
ratios = load_ratios(window_rows)
if not ratios:
return Verdict(
"H4",
"медиана r ≤ 1,5",
"нет проходов под нагрузкой",
UNTESTED,
)
details = []
noisy = False
fives = False
holds = True
for cell in LOAD_CELLS:
if cell not in ratios:
continue
parts = []
for scenario, value in ratios[cell].items():
r, a = value["r"], value["a"]
parts.append(f"{scenario} r {fmt(r)} a {fmt(a)}")
if a is not None and not 1 / H4_THRESHOLD <= a <= H4_THRESHOLD:
noisy = True
if value["s5xx"]:
fives = True
if cell in LIMITED_LOAD and (r is None or r > H4_THRESHOLD):
holds = False
details.append(f"{cell}: " + ", ".join(parts))
flagged = sorted(
{
flag
for row in passes
if row.get("cell") in LOAD_CELLS
for flag in str(row.get("flags") or "").split()
if flag in BG_FLAGS
},
)
if flagged:
fives = True
details.append("пометки фона: " + ", ".join(flagged))
contrast = ratios.get("fast-unlimited", {}).get("download", {}).get("r")
if fives:
verdict = REFUTED if not holds else UNTESTED
elif noisy:
verdict = NOISY
elif contrast is None or contrast <= H4_THRESHOLD:
verdict = UNTESTED if holds else REFUTED
else:
verdict = CONFIRMED if holds else REFUTED
return Verdict(
"H4",
f"`stand`, `stand-unlimited`, `fast-limited` — медиана r ≤ "
f"{H4_THRESHOLD} по трём сценариям; контраст `fast-unlimited` — r "
f"скачиваний > {H4_THRESHOLD}; A/A a в [1/{H4_THRESHOLD}; "
f"{H4_THRESHOLD}]; без 5xx",
f"контраст r скачиваний {fmt(contrast)}",
verdict,
details,
)
def verdict_s4(meta: dict[str, Any], passes: list[dict[str, Any]]) -> Verdict:
info = meta.get("reconcile", {})
base_stage = info.get("s4_base_stage")
base = cell_median(passes, base_stage, "batch", "duration_ms")
details = []
checks = []
# «до» уже отказывает рано (commit переноса, `bench.md`): сравнивать не с
# чем, подтвердить выигрыш нельзя
same_before = False
for count in info.get("s4_candidates", []):
stage = f"S4-{count}"
before = cell_median(passes, stage, "before", "duration_ms")
after = cell_median(passes, stage, "after", "duration_ms")
heads = cell_median(passes, stage, "after", "s3_head")
heads_before = cell_median(passes, stage, "before", "s3_head")
both = [
row
for cell in ("before", "after")
for row in cell_rows(passes, stage, cell)
]
codes = {int(num(row["exit_code"]) or -1) for row in both}
# данные не менялись (verify S6): ни одного исправления в итоге и
# ни одной записи «Fixed»; нет столбца (метка до правки) — не
# доказано
unchanged = all(
num(row.get("fixed_total")) == 0
and num(row.get("fixed_records")) == 0
for row in both
)
if before is None or after is None:
continue
extra_before = None if base is None else before - base
extra_after = None if base is None else after - base
limit = info.get("s4_max_changes", 100)
checks.append(
extra_after is not None and extra_after <= S4_AFTER_EXTRA_MS,
)
checks.append(heads is not None and heads <= limit + 1)
checks.append(codes == {3})
checks.append(unchanged)
early_before = heads_before is not None and heads_before <= limit + 1
same_before = same_before or early_before
before_note = (
f"HEAD до {fmt(heads_before, 0)}: «до» уже с ранним отказом"
if early_before
else f"ожидалось ~{count * S4_MS_PER_CANDIDATE / 1000:.0f} с"
)
details.append(
f"{count} кандидатов: до {fmt(before / 1000, 1)} с (+"
f"{fmt(None if extra_before is None else extra_before / 1000, 1)} "
f"с, {before_note}), "
f"после {fmt(after / 1000, 1)} с (+"
f"{fmt(None if extra_after is None else extra_after / 1000, 2)} "
f"с), HEAD после {fmt(heads, 0)}, коды {sorted(codes)}, данные "
f"{'не менялись' if unchanged else 'менялись или не известны'}",
)
if not checks:
return Verdict("S4", "цена отказа", "нет ячеек", UNTESTED)
if not all(checks):
verdict = REFUTED
else:
verdict = UNTESTED if same_before else CONFIRMED
return Verdict(
"S4",
f"после — не дольше базы на {S4_AFTER_EXTRA_MS / 1000:.0f} с (база "
f"— поверхностный dry-run `batch` этапа {base_stage} того же N: "
"перепроверка его кандидатов и выборка зависших, без отказа по "
"пределу), HEAD ≤ предела + 1, код 3 у обоих, ни одного исправления; "
"у «до» HEAD больше предела + 1, иначе сравнивать не с чем",
"; ".join(details),
verdict,
details,
)
def verdicts(
meta: dict[str, Any],
passes: list[dict[str, Any]],
window_rows: list[dict[str, Any]],
) -> list[Verdict]:
return [
verdict_h1(meta, passes),
verdict_h2(meta, passes),
verdict_h3(meta, passes),
verdict_h4(window_rows, passes),
verdict_s4(meta, passes),
]
# ---------------------------------------------------------------- сводка
def render_summary(
meta: dict[str, Any],
passes: list[dict[str, Any]],
window_rows: list[dict[str, Any]],
) -> str:
info = meta.get("reconcile", {})
profile = meta.get("profile", {})
lines = [
f"# Профиль сверки: метка `{meta.get('label')}`",
"",
f"- commit `{meta.get('git', {}).get('commit')}`, dirty "
f"{meta.get('git', {}).get('dirty')}; статус {meta.get('status')}",
f"- профиль `{profile.get('name')}`, повторов "
f"{profile.get('repeats')}, зерно {profile.get('seed')}, grace "
f"проходов {info.get('grace_seconds')} с, предел скорости по "
f"умолчанию {info.get('default_rate')}/с",
f"- образ «до» — commit `{info.get('before_commit')}`",
f"- CPU уборщика: {info.get('cleaner_cpus')}",
f"- фоновая нагрузка: {info.get('background')}",
f"- аргументы S3: `{' '.join(info.get('s3_command') or [])}`",
f"- хост: {meta.get('host_state', {}).get('start')}; "
f"предупреждения {meta.get('host_warnings')}; посторонние "
f"контейнеры {meta.get('other_containers')}",
"",
"## Проходы без фоновой нагрузки",
"",
"Медиана и min–max повторов. Цена — микросекунд на объект bucket.",
"",
"| Этап | N | Ячейка | Проход, с | мкс/объект | LIST | HEAD | GET "
"| DELETE | Сеансов | Прочитано, МиБ | CPU уборщика | CPU S3 | CPU "
"PG | Память, МиБ (старт → пик) | Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|",
]
stage_cells: dict[str, list[str]] = {}
for row in passes:
if row.get("load") in (True, "True"):
continue
cells = stage_cells.setdefault(row["stage"], [])
if row["cell"] not in cells:
cells.append(row["cell"])
for stage, cells in stage_cells.items():
for cell in cells:
rows = cell_rows(passes, stage, cell)
flags = sorted({f for r in rows for f in str(r["flags"]).split()})
lines.append(
f"| {stage} | {rows[0]['objects']} | {cell} | "
f"{spread([num(r['duration_ms']) / 1000 if num(r['duration_ms']) is not None else None for r in rows], 2)} | "
f"{spread([r['us_per_object'] for r in rows], 1)} | "
f"{spread([r['s3_list'] for r in rows], 0)} | "
f"{spread([r['s3_head'] for r in rows], 0)} | "
f"{spread([r['s3_get'] for r in rows], 0)} | "
f"{spread([r['s3_delete'] for r in rows], 0)} | "
f"{spread([r['db_sessions'] for r in rows], 0)} | "
f"{spread([num(r['read_bytes']) / MIB if num(r['read_bytes']) is not None else None for r in rows], 1)} | "
f"{spread([r['cpu_cleaner'] for r in rows], 3)} | "
f"{spread([r['cpu_s3'] for r in rows], 3)} | "
f"{spread([r['cpu_postgres'] for r in rows], 3)} | "
f"{spread([r['mem_start_mib'] for r in rows], 1)} → "
f"{spread([r['mem_peak_mib'] for r in rows], 1)} | "
f"{' '.join(flags) or '—'} |",
)
lines += [
"",
"## Фоновая нагрузка (H4)",
"",
"r — p95 окна прохода к p95 окон до и после; a — p95 до к p95 после "
"(A/A). Медиана по повторам.",
"",
"| Ячейка | Проход, с | Сценарий | r | a | 5xx |",
"|---|---|---|---|---|---|",
]
ratios = load_ratios(window_rows)
for cell in LOAD_CELLS:
if cell not in ratios:
continue
stage = next(
(r["stage"] for r in passes if r["cell"] == cell),
"",
)
duration = cell_median(passes, stage, cell, "duration_ms")
for scenario, value in ratios[cell].items():
lines.append(
f"| {cell} | {fmt(duration / 1000 if duration else None, 1)}"
f" | {scenario} | {fmt(value['r'])} | {fmt(value['a'])} | "
f"{value['s5xx']} |",
)
lines += [
"",
"## Отказ по пределу (S4)",
"",
"| Кандидатов | Ячейка | Проход, с | HEAD | Не перепроверено | "
"Изменений в ошибке | Код |",
"|---|---|---|---|---|---|---|",
]
for count in info.get("s4_candidates", []):
stage = f"S4-{count}"
for cell in ("before", "after"):
rows = cell_rows(passes, stage, cell)
if not rows:
continue
lines.append(
f"| {count} | {cell} | "
f"{spread([num(r['duration_ms']) / 1000 if num(r['duration_ms']) is not None else None for r in rows], 2)} | "
f"{spread([r['s3_head'] for r in rows], 0)} | "
f"{spread([r['unchecked'] for r in rows], 0)} | "
f"{spread([r['limit_changes'] for r in rows], 0)} | "
f"{sorted({r['exit_code'] for r in rows})} |",
)
lines += ["", "## Гипотезы", ""]
for item in verdicts(meta, passes, window_rows):
lines.append(f"### {item.name}: {item.verdict}")
lines += ["", f"- критерий: {item.criterion}"]
lines.append(f"- измерено: {item.measured}")
lines += [f" - {line}" for line in item.details]
lines.append("")
lines += [
"Графики: `cost-by-n.svg`, `requests-by-n.svg`, `background-p95.svg`.",
"",
]
return "\n".join(lines)
def render_plots(
meta: dict[str, Any],
passes: list[dict[str, Any]],
window_rows: list[dict[str, Any]],
) -> dict[str, str]:
"""SVG метки: время прохода от N, запросы от N (лог. шкала), r фоновой
нагрузки по ячейкам и сценариям."""
import plots
info = meta.get("reconcile", {})
stages = info.get("h1_stages", [])
points: dict[str, list[tuple[float, float]]] = {}
for cell in ("batch", "head", "deep"):
for stage in stages:
value = cell_median(passes, stage, cell, "duration_ms")
n = stage_n(meta, stage)
if value is not None and n:
points.setdefault(cell, []).append((n, value / 1000))
out = {}
svg = plots.Svg(720, 420, "Время прохода сверки от числа объектов")
out["cost-by-n.svg"] = xy_chart(
svg,
points,
"объектов в bucket",
"проход, с",
)
req: dict[str, list[tuple[float, float]]] = {}
for cell in ("batch", "head"):
for stage in stages:
n = stage_n(meta, stage)
s3 = sum(
cell_median(passes, stage, cell, f"s3_{op}") or 0
for op in S3_OPS
)
db = cell_median(passes, stage, cell, "db_sessions")
if n and cell_rows(passes, stage, cell):
req.setdefault(f"{cell}: S3", []).append((n, max(s3, 1)))
if db:
req.setdefault(f"{cell}: PostgreSQL", []).append((n, db))
svg = plots.Svg(720, 420, "Запросы прохода от числа объектов")
out["requests-by-n.svg"] = xy_chart(
svg,
req,
"объектов в bucket",
"запросов (лог. шкала)",
log=True,
)
out["background-p95.svg"] = ratio_chart(load_ratios(window_rows))
return out
def xy_chart(
svg: Any,
series: dict[str, list[tuple[float, float]]],
x_title: str,
y_title: str,
*,
log: bool = False,
) -> str:
import plots
left, top, width, height = 80, 60, 600, 290
xs = [x for pts in series.values() for x, _ in pts] or [1]
ys = [y for pts in series.values() for _, y in pts] or [1]
x_hi = max(xs) * 1.05
if log:
y_lo = 10 ** math.floor(math.log10(max(min(ys), 1)))
y_hi = 10 ** math.ceil(math.log10(max(ys) * 1.01))
ticks = plots.log_ticks(y_lo, y_hi)
else:
ticks = plots.nice_ticks(max(ys) * 1.1)
y_lo, y_hi = 0.0, ticks[-1]
def sx(x: float) -> float:
return left + width * x / x_hi
def sy(y: float) -> float:
if log:
span = math.log10(y_hi) - math.log10(y_lo)
share = (math.log10(max(y, y_lo)) - math.log10(y_lo)) / span
else:
share = (y - y_lo) / (y_hi - y_lo) if y_hi > y_lo else 0
return top + height * (1 - share)
for tick in ticks:
svg.line(left, sy(tick), left + width, sy(tick))
svg.text(left - 8, sy(tick) + 4, plots.compact(tick), anchor="end")
for x in sorted(set(xs)):
svg.text(sx(x), top + height + 18, plots.compact(x), anchor="middle")
svg.text(left + width / 2, top + height + 40, x_title, anchor="middle")
svg.text(24, top + height / 2, y_title, anchor="middle", rotate=True)
legend = []
for index, (name, pts) in enumerate(series.items()):
color = plots.SERIES[index % len(plots.SERIES)]
ordered = sorted(pts)
svg.polyline([(sx(x), sy(y)) for x, y in ordered], color)
for x, y in ordered:
svg.dot(sx(x), sy(y), color, f"{name}: N {x:g}, {y:g}")
legend.append((color, name, "line"))
svg.legend(legend, left, 48, width)
return svg.render()
def ratio_chart(ratios: dict[str, dict[str, dict[str, Any]]]) -> str:
import plots
svg = plots.Svg(720, 420, "Фоновая нагрузка: r = p95 прохода / p95 тихих")
left, top, width, height = 80, 70, 600, 280
cells = [cell for cell in LOAD_CELLS if cell in ratios]
values = [
v["r"]
for cell in cells
for v in ratios[cell].values()
if v["r"] is not None
]
high = max([H4_THRESHOLD * 1.2, *values])
ticks = plots.nice_ticks(high)
def sy(y: float) -> float:
return top + height * (1 - y / ticks[-1])
for tick in ticks:
svg.line(left, sy(tick), left + width, sy(tick))
svg.text(left - 8, sy(tick) + 4, f"{tick:g}", anchor="end")
# полоса A/A: [1/1,5; 1,5] и линия порога
svg.parts.append(
f'<rect x="{left}" y="{sy(H4_THRESHOLD):.1f}" width="{width}" '
f'height="{sy(1 / H4_THRESHOLD) - sy(H4_THRESHOLD):.1f}" '
f'fill="{plots.GRID}" opacity="0.5"/>',
)
svg.line(
left,
sy(H4_THRESHOLD),
left + width,
sy(H4_THRESHOLD),
color=plots.INK_2,
dash="5 4",
)
group = width / max(1, len(cells))
bar = group / (len(SCENARIOS) + 1)
for gi, cell in enumerate(cells):
x0 = left + gi * group + bar / 2
for si, scenario in enumerate(SCENARIOS.values()):
r = ratios[cell].get(scenario, {}).get("r")
if r is None:
continue
color = plots.SERIES[si]
svg.bar(
x0 + si * bar,
sy(r),
bar * 0.9,
sy(0) - sy(r),
color,
f"{cell} {scenario}: r {r:.2f}",
top=True,
)
svg.text(
left + gi * group + group / 2,
top + height + 18,
cell,
anchor="middle",
)
svg.legend(
[
(plots.SERIES[i], name, "box")
for i, name in enumerate(SCENARIOS.values())
]
+ [(plots.INK_2, f"порог {H4_THRESHOLD}", "dash")],
left,
56,
width,
)
return svg.render()
# ------------------------------------------------------------ самопроверка
def selftest() -> int:
failures: list[str] = []
def expect(name: str, got: Any, want: Any) -> None:
if got != want:
failures.append(f"{name}: got {got!r}, want {want!r}")
# набор: классы, id, содержимое, модель
expect("class 0", object_class(0), "orphan_object")
expect("class 3", object_class(403), "stale_pending")
expect("class ok", object_class(4), None)
expect("force ok", object_class(0, FORCE_OK), None)
expect("force missing", object_class(5, FORCE_MISSING), "missing_object")
first = object_id(7, 1)
expect("id deterministic", first, object_id(7, 1))
expect("id v4", uuid.UUID(first).version, 4)
expect("id lower", first, str(uuid.UUID(first)))
expect("content size", len(content(7, 1, 4096)), 4096)
expect("content differs", content(7, 1, 64) == content(7, 2, 64), False)
m = model(10_000)
expect("model found", m.found, dict.fromkeys(CLASSES, 25))
expect("model objects", m.objects, 9975)
expect("model ready", m.ready_rows, 9950)
expect("model 1000", model(1000).found["orphan_object"], 3)
expect("model 1000 stale", model(1000).found["stale_pending"], 3)
expect("model 1000 ready", model(1000).ready_rows, 994)
s4 = model(0, clean=10_000, missing=1000)
expect(
"s4 model",
(s4.objects, s4.ready_rows, s4.found),
(
10_000,
11_000,
{"missing_object": 1000},
),
)
text = rows_csv(
seed=7,
start=0,
end=800,
size=4096,
namespace="bench",
chat_id="c",
owner_id="o",
created_at="2026-10-22T00:00:00+00:00",
)
lines = text.splitlines()
expect("csv rows", len(lines), 800 - 2)
expect("csv columns", len(lines[0].split(",")), len(COPY_COLUMNS))
by_index = {line.split(",")[3]: line.split(",") for line in lines}
expect("csv size mismatch", by_index["bench-2.png"][5], "4097")
expect("csv pending", by_index["bench-3.png"][8], "pending")
expect("csv no orphan row", "bench-0.png" in by_index, False)
expect(
"csv sha",
by_index["bench-4.png"][6],
"\\x" + hashlib.sha256(content(7, 4, 4096)).hexdigest(),
)
expect("objects without missing", 1 in object_indices(0, 10), False)
expect("objects count", len(object_indices(0, 800)), 798)
picked = downloads(7, 2000, 5000)
expect("downloads only consistent", len(picked), 2000 - 20)
# план кругами
plan = plan_passes([("A", ("x", "y", "z")), ("B", ("x", "y"))], 3, 5)
expect("plan length", len(plan), 15)
expect("plan unique", len({p["pass"] for p in plan}), 15)
expect("plan stage order", [p["stage"] for p in plan][:9], ["A"] * 9)
for i in range(1, len(plan)):
same = plan[i]["cell"] == plan[i - 1]["cell"]
if same and plan[i]["stage"] == plan[i - 1]["stage"]:
failures.append(f"plan repeats a cell in a row at {i}")
# записи уборщика
log = "\n".join(
[
"Container mtusi-bench-reconcile-1 Creating",
json.dumps({"event": STARTED, "reconcile_strategy": "batch"}),
json.dumps({"event": FOUND, "kind": "orphan_object"}),
json.dumps({"event": FOUND, "kind": "orphan_object"}),
json.dumps(
{
"event": FINISHED,
"found": {"orphan_object": 2},
"duration_ms": 1500,
"timestamp": "2026-10-23T01:02:03.456789Z",
"s3_requests": {
"list": 3,
"head": 2,
"get": 0,
"delete": 0,
},
"phase_ms": {"list": 1000, "rows": 200},
"listed_objects": 1000,
"storage_objects": 1003,
},
),
],
)
kept = kept_records(parse_records(log))
expect("kept found count", kept["found_records"], 2)
expect("kept finished", kept["finished"]["duration_ms"], 1500)
expect(
"timestamp",
timestamp("2026-10-23T01:02:03.456789Z"),
1792717323.456789,
)
expect(
"timestamp nano",
timestamp("2026-10-23T01:02:03.456789123Z"),
1792717323.456789,
)
problems = found_problems(
{"pass": "B-batch-r1"},
{"orphan_object": 2, "missing_object": 1},
kept,
)
expect(
"found problem",
problems,
["B-batch-r1: missing_object found 0, injected 1"],
)
expect(
"found ok",
found_problems({"pass": "p"}, {"orphan_object": 2}, kept),
[],
)
# метрики шлюза
metrics_a = (
'# HELP x\nSeaweedFS_s3_request_total{bucket="bench",code="200",'
'type="LIST"} 4\nSeaweedFS_s3_request_total{bucket="bench",'
'code="200",type="GET"} 10\nSeaweedFS_s3_request_total{bucket='
'"bench",code="404",type="GET"} 2\nother_metric 5\n'
)
metrics_b = metrics_a.replace("} 4", "} 7").replace("} 10", "} 11")
delta = metrics_delta(parse_metrics(metrics_a), parse_metrics(metrics_b))
expect("metrics delta", delta, {"GET": 1, "LIST": 3})
expect(
"gateway equal",
gateway_mismatch({"list": 3, "head": 1, "get": 0, "delete": 0}, delta),
False,
)
expect(
"gateway differ",
gateway_mismatch({"list": 3, "head": 2, "get": 0, "delete": 0}, delta),
True,
)
expect("pg stat", parse_pg_stat("1|2|3|4|5|6")["pg_sessions"], 6)
# фоновая нагрузка, окна и отношения
reqs = []
for second in range(30):
for kind, base in (("d", 10), ("w", 20), ("r", 5)):
duration = base * (3 if 10 <= second < 20 else 1)
reqs.append(
Request(kind, second * 1000 + 1, duration, OK_STATUS[kind]),
)
reqs.append(Request("d", 15_000, 1, 503))
console = "\n".join(
f"bg,{r.kind},{r.start_ms},{r.duration_ms},{r.status}" for r in reqs
)
parsed = parse_background(console + "\nnoise\nbg,x,1,2,3\n")
expect("bg parsed", len(parsed), len(reqs))
# полнота строк: снизу — итерации и счётчик, сверху — прерванные ≤ VU
counts = {
kind: sum(1 for r in parsed if r.kind == kind) for kind in SCENARIOS
}
def summary(iterations_less: int, counter_less: int) -> dict[str, Any]:
"""Итог k6, где у скачиваний итераций и отсчётов счётчика на
столько меньше строк (отрицательное — больше)."""
metrics = {}
for kind, scenario in SCENARIOS.items():
shift = (iterations_less, counter_less) if kind == "d" else (0, 0)
metrics[f"bg_lines_{scenario}"] = {
"values": {"count": counts[kind] - shift[1]},
}
metrics[f"iterations{{scenario:{scenario}}}"] = {
"values": {"count": counts[kind] - shift[0]},
}
return {"metrics": metrics}
def problems_of(iterations_less: int, counter_less: int) -> list[str]:
return line_problems(
parsed,
summary(iterations_less, counter_less),
max_vus=5,
)[0]
expect("lines ok", problems_of(0, 0), [])
# метка aaf3fa3: строка прерванной итерации, отсчёт счётчика есть
problems, interrupted = line_problems(parsed, summary(1, 0), max_vus=5)
expect(
"lines interrupted, counted",
(problems, interrupted["download"]),
([], 1),
)
# метка a42dd27: строка на 1 больше и счётчика, и итераций
expect("lines interrupted, not counted", problems_of(1, 1), [])
expect("lines lost vs counter", len(problems_of(0, -1)), 1)
expect("lines lost vs iterations", len(problems_of(-1, 0)), 1)
expect(
"lost message names direction",
"lines lost" in problems_of(-1, 0)[0],
True,
)
expect("lines too many interrupted", len(problems_of(6, 0)), 1)
expect("lines too many over both", len(problems_of(6, 6)), 2)
expect(
"lines no counter",
len(line_problems(parsed, {"metrics": {}}, max_vus=5)[0]),
3,
)
# верхний допуск — не больше VU, созданных k6 (`vus_max` итога)
capped = summary(4, 0)
capped["metrics"]["vus_max"] = {"values": {"max": 3}}
expect(
"lines over vus_max",
len(line_problems(parsed, capped, max_vus=5)[0]),
1,
)
capped["metrics"]["vus_max"] = {"values": {"max": 300}}
expect(
"lines vus_max above scenario VU",
line_problems(parsed, capped, max_vus=5)[0],
[],
)
# ошибки фона по всему итогу k6
k6_errors = {
"metrics": {
"http_req_failed": {"values": {"passes": 2, "fails": 10}},
"dropped_iterations": {"values": {"count": 1}},
},
}
expect(
"bg errors",
background_errors(k6_errors),
{"failed": 2, "dropped": 1},
)
expect(
"bg flags",
background_flags(background_errors(k6_errors)),
["bg_failed", "bg_dropped"],
)
expect("bg no errors", background_flags(background_errors({})), [])
bounds = {"before": (0, 10), "pass": (10, 20), "after": (20, 30)}
rows, ratios_one = windows_of(parsed, bounds)
expect("window rows", len(rows), 12)
expect("r download", ratios_one["download"]["r"], 3.0)
expect("a download", ratios_one["download"]["a"], 1.0)
expect("5xx download", ratios_one["download"]["s5xx"], 1)
expect("percentile", percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 95), 10.0)
# окно CPU и память по отсчётам сэмплера (нс, usage_usec, …, memory)
points = [
(int(t * 1e9), int(t * 250_000), 0, 0, 0, (100 + t) * MIB, 0)
for t in (0, 1, 2, 3, 4)
]
expect("window cpu", window_cpu(points, 0.5, 3.5), 0.25)
# короткий проход после последнего отсчёта — по охватывающим отсчётам
expect("window cpu tail", window_cpu(points, 3.8, 4.3), 0.25)
expect("memory", memory_start_peak(points, 2.5), (102.0, 104.0))
# строка прохода и пометка count_mismatch
entry = {
"pass": "B-batch-r1",
"stage": "B",
"cell": "batch",
"repeat": 1,
"order": 1,
}
row = pass_row(
entry,
cell=CELLS["batch"],
expected=model(1000),
object_bytes=4096,
kept=kept,
exit_code=3,
gateway={"LIST": 3, "GET": 3},
pg=None,
cpu={"cleaner": 0.25},
memory=(90.0, 91.0),
cleaner_cpus="0.25",
rate=0,
image="checkout",
)
expect("row flags", row["flags"], "count_mismatch")
expect(
"row us per object",
row["us_per_object"],
round(1_500_000 / 997, 2),
)
expect("row fields", set(row) <= set(PASS_FIELDS), True)
expect("row fixed", (row["fixed_total"], row["fixed_records"]), (0, 0))
# вердикты и сводка на синтетических проходах
meta = {
"label": "selftest",
"profile": {"name": "reconcile_quick", "repeats": 1},
"reconcile": {
"h1_stages": ["B", "C"],
"h2_stages": ["B"],
"h3_small_stage": "B",
"h3_large_stage": "A",
"s4_base_stage": "B",
"s4_candidates": [300],
"s4_max_changes": 100,
"cleaner_cpus": {"stand": "0.25", "fast": "1"},
"object_bytes": {"A": MIB, "B": 4 * KIB, "C": 4 * KIB},
"stages": [
{"stage": "B", "objects_total": 10_000},
{"stage": "C", "objects_total": 30_000},
],
},
}
def synthetic(
stage: str,
cell: str,
n: int,
us: float,
**extra: Any,
) -> dict[str, Any]:
mdl = model(n)
base = dict.fromkeys(PASS_FIELDS)
base.update(
{
"pass": f"{stage}-{cell}-r1",
"stage": stage,
"cell": cell,
"load": cell in LOAD_CELLS,
"objects": mdl.objects,
"ready_rows": mdl.ready_rows,
"us_per_object": us,
"duration_ms": us * mdl.objects / 1000,
"s3_list": math.ceil(mdl.objects / 1000) + 2,
"s3_head": sum(
mdl.found[k]
for k in (
"orphan_object",
"missing_object",
"size_mismatch",
)
),
"s3_get": 0,
"s3_delete": 0,
"mem_start_mib": 97.0,
"mem_peak_mib": 97.0 + mdl.objects * 150 / MIB,
"mem_growth_bytes_per_object": 150.0,
"exit_code": 3,
"flags": "",
"fixed_total": 0,
"fixed_records": 0,
**{f"expected_{k}": mdl.found.get(k, 0) for k in CLASSES},
},
)
base["db_sessions"] = expected_batch_requests(base)[2]
base.update(extra)
return base
b_batch = synthetic("B", "batch", 10_000, 300)
expect(
"batch requests B",
expected_batch_requests(b_batch),
(12, 75, 96),
)
expect(
"batch requests D",
expected_batch_requests(synthetic("D", "batch", 100_000, 300)),
(102, 750, 953),
)
passes = [
b_batch,
synthetic("C", "batch", 30_000, 310),
synthetic(
"B",
"head",
10_000,
5000,
mem_peak_mib=98.0,
s3_head=9950,
db_sessions=10061,
),
synthetic("B", "deep", 10_000, 2700, s3_get=9900),
synthetic(
"A",
"deep",
1000,
10_500,
cpu_cleaner=0.245,
cpu_s3=0.19,
s3_get=988,
),
synthetic("A", "deep-fast", 1000, 4800, s3_get=988),
synthetic(
"S4-300",
"before",
10_000,
0,
duration_ms=4500,
exit_code=3,
s3_head=300,
),
synthetic("S4-300", "after", 10_000, 0, duration_ms=3200, s3_head=101),
]
found = {
item.name: item.verdict
for item in verdicts(meta, passes, rows_with_cells(rows))
}
expect("verdict H1", found["H1"], CONFIRMED)
expect("verdict H2", found["H2"], CONFIRMED)
expect("verdict H3", found["H3"], CONFIRMED)
expect("verdict S4", found["S4"], CONFIRMED)
expect("verdict H4 contrast", found["H4"], UNTESTED)
slow = [*passes[:1], synthetic("C", "batch", 30_000, 500), *passes[2:]]
found = {item.name: item.verdict for item in verdicts(meta, slow, [])}
expect("verdict H1 refuted", found["H1"], REFUTED)
def refuted(name: str, index: int, **change: Any) -> None:
"""Вердикт `name` опровергнут, если у прохода `index` изменить
столбцы `change` (критерии verify S6)."""
changed = [dict(row) for row in passes]
changed[index].update(change)
verdict = {
item.name: item.verdict for item in verdicts(meta, changed, [])
}[name]
expect(f"verdict {name} refuted by {sorted(change)}", verdict, REFUTED)
refuted("H1", 0, db_sessions=97)
refuted("H1", 0, s3_list=13)
refuted("H2", 2, db_sessions=500)
refuted("H3", 3, s3_get=9950)
refuted("S4", 7, fixed_total=1)
refuted("S4", 6, fixed_records=None)
# «до» с тем же ранним отказом: S4 не подтверждена и не ждёт полной цены
early = [dict(row) for row in passes]
early[6]["s3_head"] = 101
early_s4 = verdict_s4(meta, early)
expect("verdict S4 same before", early_s4.verdict, UNTESTED)
expect(
"S4 same before: no expected",
"ожидалось" in early_s4.measured,
False,
)
expect(
"S4 old before: expected",
"ожидалось" in verdict_s4(meta, passes).measured,
True,
)
# H4: без 5xx окон контраст подтверждает, пометка фона — нет
clean, _ = windows_of([r for r in parsed if r.status < 500], bounds)
expect(
"verdict H4 clean",
verdict_h4(rows_with_cells(clean)).verdict,
CONFIRMED,
)
expect(
"verdict H4 bg_failed",
verdict_h4(
rows_with_cells(clean),
[{"cell": "fast-unlimited", "flags": "bg_failed"}],
).verdict,
UNTESTED,
)
summary = render_summary(meta, passes, rows_with_cells(rows))
for needle in ("## Гипотезы", "### H1", "### S4", "| B | 9975 | batch"):
if needle not in summary:
failures.append(f"summary lacks {needle!r}")
svgs = render_plots(meta, passes, rows_with_cells(rows))
expect(
"plots",
sorted(svgs),
[
"background-p95.svg",
"cost-by-n.svg",
"requests-by-n.svg",
],
)
for name, text in svgs.items():
if not text.startswith("<svg"):
failures.append(f"{name} is not svg")
for failure in failures:
print(f"FAIL {failure}", flush=True)
print(
f"selftest: {'ok' if not failures else f'{len(failures)} failed'}",
flush=True,
)
return 1 if failures else 0
def rows_with_cells(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Самопроверка: строки окон одного прохода ячейки `fast-unlimited`
с окном «тихих» для `load_ratios`."""
out = []
for row in rows:
out.append(
{"pass": "D-fast-unlimited-r1", "cell": "fast-unlimited", **row},
)
return out
if __name__ == "__main__":
if sys.argv[1:] == ["--selftest"]:
sys.exit(selftest())
print("usage: uv run python bench/reconcile.py --selftest", flush=True)
sys.exit(2)