Files
mtusi2026lab5/bench/queue.py
T

1483 lines
60 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль queue: план, разбор журналов, строки и вердикты без docker. Только stdlib.
Трек 1 лабы 5 «Очередь под перегрузкой». `run.py` (профили `queue`,
`queue-quick`) гоняет ячейки «настройка worker × класс изображения»; на
ячейку — пять фаз: `drain` (ёмкость: N тел при остановленном worker),
`steady` (0,6 ёмкости ячейки), `fixed` (одна интенсивность на класс для
всех настроек: задержки сравнимы при равной нагрузке), `overload` (2
ёмкости, окно дотягивается до предела незавершённых), `recovery`
(опустошение после перегрузки и снова 0,6 ёмкости). Здесь — чистые
функции: план ячеек кругами, разбор журнала worker, журнала генератора и
строк SQL,
строки `cells.csv`, `timeline.csv`, `jobs.csv.gz`, вердикты Q1–Q5 с
порогами-константами (гипотезы записываются в `REPORT.md` до замеров),
рекомендация и `summary.md`. Самопроверка — `uv run python
bench/queue.py --selftest`.
Имя совпадает с модулем stdlib `queue`: в процессах bench/ каталог
скрипта стоит первым в `sys.path`, и `import queue` находит этот файл.
Модули стенда stdlib `queue` не используют; импорт, которому он нужен
(`concurrent.futures`, `multiprocessing`, `logging.handlers`), в `run.py`
не добавлять.
"""
import bisect
import math
import random
import re
import statistics
import sys
from collections.abc import Iterable
from dataclasses import dataclass, field
from datetime import UTC, datetime
MIB = 2**20
PHASES = ("drain", "steady", "fixed", "overload", "recovery")
PHASE_NAMES = {
"drain": "ёмкость",
"steady": "ровная",
"fixed": "общая интенсивность",
"overload": "перегрузка",
"recovery": "восстановление",
}
# Retry-After у 429 «незавершённых заданий не меньше предела» и «места
# загрузки заняты» (src/constants.py, копия: стенд код приложения не
# импортирует)
BACKLOG_RETRY_AFTER = "5"
SLOTS_RETRY_AFTER = "1"
# пороги гипотез — до замеров (REPORT.md, «Очередь: гипотезы»)
Q1_CLASSES = ("l", "x")
Q1_SCALE = 1.6
Q1_FLAT = 1.15
Q2_THROUGHPUT = 0.10
Q2_P95 = 0.20
Q3_BIG_CLASS = "x"
Q3_BIG_RATIO = 2.5
Q3_SMALL_CLASS = "s"
Q3_SMALL_RATIO = 1.5
Q4_SHARE = 0.30
Q5_DRAIN_FACTOR = 1.3
Q5_DRAIN_EXTRA_S = 5.0
Q5_P95_FACTOR = 1.3
Q5_P95_EXTRA_MS = 500.0
# рекомендация — только среди настроек с пиком worker не выше этой доли
# лимита памяти контейнера; p95 фазы fixed в пределах этой доли от
# лучшего — равны, и выбирается меньший пик
RECOMMEND_MEMORY_SHARE = 0.8
RECOMMEND_P95_TIE = 0.10
def overload_window(
window_s: float, backlog_limit: int, throughput: float, *, fill: float, cap: float
) -> float:
"""Окно перегрузки: не короче окна профиля и достаточно долгое, чтобы
при приходе 2 × T очередь (растёт на T в секунду) дошла до предела с
запасом `fill`; не длиннее `cap`."""
return min(cap, max(window_s, fill * backlog_limit / throughput))
CELL_COLUMNS = (
"cell",
"config",
"concurrency",
"prefetch",
"size_class",
"body_bytes",
"repeat",
"order",
"phase",
"rate",
"window_s",
"scheduled",
"sent",
"accepted",
"rejected_429",
"retry_after_5",
"retry_after_1",
"errors_5xx",
"other_errors",
"dropped",
"share_429",
"share_429_saturated",
"backlog_reached",
"throughput",
"drain_s",
"startup_s",
"ttr_p50_ms",
"ttr_p95_ms",
"ttr_p99_ms",
"ttr_max_ms",
"lag_p50_ms",
"lag_p95_ms",
"lag_max_ms",
"took_p50_ms",
"took_p95_ms",
"max_unfinished",
"max_unfinished_sampled",
"max_outbox",
"max_in_broker",
"max_oldest_age_s",
"recovery_s",
"jobs",
"ready",
"failed",
"missing",
"unknown_jobs",
"retries",
"skipped",
"worker_cpu_cores",
"worker_mem_max_mib",
"worker_peak_mib",
"render_peak_mib",
"upload_p95_ms",
"flags",
)
TIMELINE_COLUMNS = (
"cell",
"config",
"size_class",
"repeat",
"phase",
"ts",
"t_s",
"unfinished",
"outbox",
"in_broker",
"processing",
"oldest_age_s",
"worker_cpu_cores",
"worker_mem_mib",
"accepted",
"rejected_429",
"errors",
)
JOB_COLUMNS = (
"cell",
"phase",
"attachment_id",
"created_at",
"ready_at",
"attempt",
"waited_ms",
"took_ms",
"state",
"job_created_at",
"finished_at",
"render_peak_kib",
)
# состояние заданий одним запросом: часы сервера (ВМ Docker), счётчики по
# частичному индексу незавершённых
TIMELINE_SQL = """
SELECT extract(epoch FROM clock_timestamp()),
count(*),
count(*) FILTER (WHERE state = 'queued' AND published_at IS NULL),
count(*) FILTER (WHERE state = 'queued' AND published_at IS NOT NULL),
count(*) FILTER (WHERE state = 'processing'),
coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)
FROM attachment_jobs
WHERE state IN ('queued', 'processing');
"""
NOW_SQL = "SELECT extract(epoch FROM clock_timestamp());\n"
def jobs_sql(since: float) -> str:
"""Вложения ячейки с их заданием: всё принятое с начала ячейки. LEFT
JOIN — принятая строка без задания тоже видна (тогда она «missing»)."""
return (
"SELECT a.id, extract(epoch FROM a.created_at), coalesce(j.id::text, ''), "
"coalesce(extract(epoch FROM j.created_at)::text, ''), "
"coalesce(j.state, ''), coalesce(extract(epoch FROM j.updated_at)::text, ''), "
"coalesce(j.attempt::text, ''), a.status "
"FROM attachments a LEFT JOIN attachment_jobs j ON j.attachment_id = a.id "
f"WHERE a.created_at >= to_timestamp({since:.6f}) "
"ORDER BY a.created_at, a.id;\n"
)
# --- план --------------------------------------------------------------------------
def plan_cells(
configs: Iterable[tuple[str, int, int]],
classes: Iterable[str],
repeats: int,
seed: int,
) -> list[dict]:
"""Ячейки кругами: в круге r — перестановка (настройка × класс)
`Random(seed + r)`; круг не начинается с ячейки, которой кончился
прежний (одна и та же настройка подряд не попадает на границу)."""
items = [(c, k) for c in configs for k in classes]
plan: list[dict] = []
previous = None
for round_no in range(1, repeats + 1):
order = list(range(len(items)))
random.Random(seed + round_no).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for index in order:
(name, concurrency, prefetch), size_class = items[index]
plan.append(
{
"cell": f"{name}-{size_class}-r{round_no}",
"config": name,
"concurrency": concurrency,
"prefetch": prefetch,
"size_class": size_class,
"repeat": round_no,
"order": len(plan) + 1,
},
)
previous = order[-1]
return plan
# --- разбор ------------------------------------------------------------------------
LOG_LINE = re.compile(
r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3})Z \S+\s+\S+ \[rid=\S*\] (.*)$"
)
FIELD = re.compile(r"(\w+)=(\S+)")
def log_time(stamp: str) -> float:
"""`2026-11-05T16:23:22.870` (UTC, src/log.py) → секунды эпохи."""
moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%S.%f")
return moment.replace(tzinfo=UTC).timestamp()
def _ms(value: str) -> float | None:
try:
return float(value.removesuffix("ms"))
except ValueError:
return None
@dataclass
class WorkerLog:
"""Журнал worker одной ячейки: старты потребителя и итоги доставок."""
consuming: list[float] = field(default_factory=list)
# attachment_id → ts, job_id, attempt, render_peak_kib, took_ms, waited_ms
processed: dict[str, dict] = field(default_factory=dict)
# job_id → число пропущенных доставок (дубль, чужая попытка)
skipped: dict[str, int] = field(default_factory=dict)
# attachment_id → error_code
failed: dict[str, str] = field(default_factory=dict)
# attachment_id → число назначенных повторов
retries: dict[str, int] = field(default_factory=dict)
def parse_worker_log(text: str) -> WorkerLog:
out = WorkerLog()
for line in text.splitlines():
match = LOG_LINE.match(line)
if not match:
continue
ts, message = log_time(match.group(1)), match.group(2)
fields = dict(FIELD.findall(message))
if message.startswith("broker consuming"):
out.consuming.append(ts)
elif message.startswith("attachment processed"):
out.processed[fields.get("attachment_id", "")] = {
"ts": ts,
"job_id": fields.get("job_id", ""),
"attempt": int(fields.get("attempt", "0") or 0),
"render_peak_kib": int(fields.get("render_peak_kib", "0") or 0),
"took_ms": _ms(fields.get("took", "")),
"waited_ms": _ms(fields.get("waited", "")),
}
elif message.startswith("job delivery skipped"):
job = fields.get("job_id", "")
out.skipped[job] = out.skipped.get(job, 0) + 1
elif message.startswith("attachment processing failed"):
out.failed[fields.get("attachment_id", "")] = fields.get("error_code", "")
elif message.startswith("job retry scheduled"):
key = fields.get("attachment_id", "")
out.retries[key] = out.retries.get(key, 0) + 1
return out
@dataclass(frozen=True)
class Upload:
"""Строка журнала генератора `queue` (loadgen.py); времена — секунды
часов ВМ, `None` — загрузка не ушла."""
scheduled: float
sent: float | None
done: float | None
status: int
retry_after: str
attachment_id: str
error: str
@property
def accepted(self) -> bool:
return self.status in (201, 202)
def _sec(value: str) -> float | None:
return float(value) / 1000 if value else None
def parse_gen_log(text: str) -> list[Upload]:
uploads = []
for line in text.splitlines():
parts = line.split(",")
if len(parts) != 9 or parts[0] != "q":
continue
uploads.append(
Upload(
scheduled=float(parts[1]) / 1000,
sent=_sec(parts[2]),
done=_sec(parts[3]),
status=int(parts[4] or 0),
retry_after=parts[5],
attachment_id=parts[6],
error=parts[8],
)
)
return uploads
@dataclass
class Job:
"""Вложение ячейки и его задание (`jobs_sql`); времена — секунды эпохи
часов сервера. `finished_at` — `updated_at` задания в `ready`/`failed`."""
attachment_id: str
created_at: float
job_id: str
job_created_at: float | None
state: str
finished_at: float | None
attempt: int | None
status: str
phase: str = ""
@property
def ready_at(self) -> float | None:
return self.finished_at if self.state == "ready" else None
@property
def ttr_ms(self) -> float | None:
ready = self.ready_at
return None if ready is None else (ready - self.created_at) * 1000
def _opt(value: str) -> float | None:
return float(value) if value else None
def parse_jobs(text: str) -> list[Job]:
jobs = []
for line in text.splitlines():
parts = line.split("|")
if len(parts) != 8:
continue
state = parts[4]
jobs.append(
Job(
attachment_id=parts[0],
created_at=float(parts[1]),
job_id=parts[2],
job_created_at=_opt(parts[3]),
state=state,
finished_at=_opt(parts[5]) if state in ("ready", "failed") else None,
attempt=int(parts[6]) if parts[6] else None,
status=parts[7],
)
)
return jobs
def parse_sample(text: str) -> dict | None:
"""Строка `TIMELINE_SQL` → выборка шкалы."""
parts = text.strip().split("|")
if len(parts) != 6:
return None
return {
"ts": float(parts[0]),
"unfinished": int(parts[1]),
"outbox": int(parts[2]),
"in_broker": int(parts[3]),
"processing": int(parts[4]),
"oldest_age_s": round(float(parts[5]), 2),
}
@dataclass(frozen=True)
class Tick:
"""Одна строка `scan` по worker: CPU с прошлой строки и текущая память cgroup."""
ts: float
cpu_cores: float | None
memory: int
def worker_ticks(lines: list[dict], container: str) -> list[Tick]:
"""Строки сэмплера (bench/sampler.py) → выборки контейнера `container`
(id12). После перезапуска контейнера счётчик CPU начинается заново:
интервал через перезапуск CPU не получает."""
ticks: list[Tick] = []
previous = None
for line in lines:
values = line.get("cg", {}).get(container)
if values is None:
previous = None
continue
cpu = None
if previous is not None and values[0] >= previous[1][0]:
seconds = line["ts"] - previous[0]
if seconds > 0:
cpu = round((values[0] - previous[1][0]) / seconds / 1e6, 3)
ticks.append(Tick(ts=line["ts"], cpu_cores=cpu, memory=values[4]))
previous = (line["ts"], values)
return ticks
# --- метрики -----------------------------------------------------------------------
def percentile(values: Iterable[float], fraction: float) -> float | None:
"""Ближайший ранг, как у генератора (loadgen.percentile)."""
ordered = sorted(values)
if not ordered:
return None
rank = max(1, math.ceil(fraction * len(ordered)))
return ordered[min(rank, len(ordered)) - 1]
def _r(value: float | None, digits: int = 1) -> float | None:
return None if value is None else round(value, digits)
def max_unfinished(jobs: Iterable[Job], start: float, end: float) -> int:
"""Наибольшее число незавершённых заданий в [start, end] по событиям:
+1 при создании задания, −1 при его ready/failed. Точнее выборок раз в
секунду: пик между ними тоже виден."""
events: list[tuple[float, int]] = []
level = 0
for job in jobs:
if job.job_created_at is None:
continue
finish = job.finished_at
if job.job_created_at <= start:
if finish is None or finish > start:
level += 1
elif job.job_created_at <= end:
events.append((job.job_created_at, 1))
if finish is not None and start < finish <= end and job.job_created_at <= end:
events.append((finish, -1))
peak = level
# при равном времени сначала уход, потом приход: пик не завышается
for _, step in sorted(events, key=lambda e: (e[0], e[1])):
level += step
peak = max(peak, level)
return peak
def drain_throughput(jobs: Iterable[Job], consuming_ts: float | None) -> float | None:
"""Ready/с фазы ёмкости: готовых заданий на секунды от начала работы
потребителя (`broker consuming`) до последнего ready. Старт процесса
worker сюда не входит — он в `startup_s`."""
ready = [j.ready_at for j in jobs if j.ready_at is not None]
if consuming_ts is None or not ready:
return None
span = max(ready) - consuming_ts
return len(ready) / span if span > 0 else None
def assign_phases(jobs: list[Job], marks: dict[str, float]) -> None:
"""Фаза вложения — по `created_at` между отметками начала фаз."""
bounds = sorted((marks[p], p) for p in PHASES if p in marks)
starts = [b[0] for b in bounds]
for job in jobs:
index = bisect.bisect_right(starts, job.created_at) - 1
job.phase = bounds[index][1] if index >= 0 else ""
def window_usage(ticks: list[Tick], start: float, end: float) -> dict:
inside = [t for t in ticks if start < t.ts <= end]
cpu = [t.cpu_cores for t in inside if t.cpu_cores is not None]
return {
"worker_cpu_cores": _r(statistics.fmean(cpu), 3) if cpu else None,
"worker_mem_max_mib": _r(max(t.memory for t in inside) / MIB)
if inside
else None,
}
@dataclass
class CellData:
"""Всё, что `run.py` собрал за ячейку (часы ВМ Docker)."""
item: dict
body_bytes: int
# начало фаз и прочие отметки: drain, worker_start, steady, overload,
# recovery, end
marks: dict[str, float]
consuming_ts: float | None
# фаза → итог генератора и его строки
gens: dict[str, dict]
uploads: dict[str, list[Upload]]
jobs: list[Job]
log: WorkerLog
ticks: list[Tick]
samples: list[dict]
peak_bytes: int | None
backlog_limit: int
def _phase_end(marks: dict[str, float], phase: str) -> float:
later = [marks[p] for p in PHASES[PHASES.index(phase) + 1 :] if p in marks]
return later[0] if later else marks["end"]
def _gen_fields(summary: dict, uploads: list[Upload]) -> dict:
sent = summary.get("sent") or 0
rejected = summary.get("rejected_429") or 0
pauses = summary.get("retry_after") or {}
# доля 429 после предела: от первого отказа «работы слишком много» до
# конца окна — тогда backpressure уже включён
first = min(
(
u.scheduled
for u in uploads
if u.status == 429 and u.retry_after == BACKLOG_RETRY_AFTER
),
default=None,
)
saturated = None
if first is not None:
after = [u for u in uploads if u.scheduled >= first and u.sent is not None]
if after:
saturated = round(sum(u.status == 429 for u in after) / len(after), 4)
return {
"rate": _r(summary.get("rate"), 3),
"window_s": summary.get("duration_s"),
"scheduled": summary.get("scheduled"),
"sent": sent,
"accepted": summary.get("accepted"),
"rejected_429": rejected,
"retry_after_5": pauses.get(BACKLOG_RETRY_AFTER, 0),
"retry_after_1": pauses.get(SLOTS_RETRY_AFTER, 0),
"errors_5xx": summary.get("errors_5xx"),
"other_errors": summary.get("other_errors"),
"dropped": summary.get("dropped"),
"share_429": round(rejected / sent, 4) if sent else None,
"share_429_saturated": saturated,
"backlog_reached": first is not None,
"upload_p95_ms": summary.get("upload_p95_ms"),
}
def cell_rows(data: CellData) -> list[dict]:
"""Строки `cells.csv` ячейки — по одной на фазу."""
item, marks = data.item, data.marks
assign_phases(data.jobs, marks)
by_id = {job.attachment_id: job for job in data.jobs}
log = data.log
peak_mib = _r(data.peak_bytes / MIB) if data.peak_bytes else None
rows = []
for phase in PHASES:
if phase not in marks:
continue
start, end = marks[phase], _phase_end(marks, phase)
jobs = [j for j in data.jobs if j.phase == phase]
uploads = data.uploads.get(phase, [])
accepted_ids = {u.attachment_id for u in uploads if u.accepted}
processed = [
log.processed[j.attachment_id]
for j in jobs
if j.attachment_id in log.processed
]
waited = [p["waited_ms"] for p in processed if p["waited_ms"] is not None]
took = [p["took_ms"] for p in processed if p["took_ms"] is not None]
ttr = [j.ttr_ms for j in jobs if j.ttr_ms is not None]
samples = [s for s in data.samples if s.get("phase") == phase]
flags = []
row = {
**{k: item[k] for k in ("cell", "config", "concurrency", "prefetch")},
"size_class": item["size_class"],
"body_bytes": data.body_bytes,
"repeat": item["repeat"],
"order": item["order"],
"phase": phase,
**_gen_fields(data.gens.get(phase, {}), uploads),
"ttr_p50_ms": _r(percentile(ttr, 0.50)),
"ttr_p95_ms": _r(percentile(ttr, 0.95)),
"ttr_p99_ms": _r(percentile(ttr, 0.99)),
"ttr_max_ms": _r(max(ttr)) if ttr else None,
"lag_p50_ms": _r(percentile(waited, 0.50)),
"lag_p95_ms": _r(percentile(waited, 0.95)),
"lag_max_ms": _r(max(waited)) if waited else None,
"took_p50_ms": _r(percentile(took, 0.50)),
"took_p95_ms": _r(percentile(took, 0.95)),
"max_unfinished_sampled": max(
(s["unfinished"] for s in samples), default=None
),
"max_outbox": max((s["outbox"] for s in samples), default=None),
"max_in_broker": max((s["in_broker"] for s in samples), default=None),
"max_oldest_age_s": max((s["oldest_age_s"] for s in samples), default=None),
"jobs": len(jobs),
"ready": sum(j.state == "ready" for j in jobs),
"failed": sum(
by_id[i].state == "failed" for i in accepted_ids if i in by_id
),
# принято (201/202), но не ready и не failed к концу ячейки
"missing": sum(
1
for i in accepted_ids
if i not in by_id or by_id[i].state not in ("ready", "failed")
),
# задание есть, а 202 генератор не увидел (срок ответа, обрыв)
"unknown_jobs": sum(j.attachment_id not in accepted_ids for j in jobs),
"retries": sum(log.retries.get(j.attachment_id, 0) for j in jobs),
"skipped": sum(log.skipped.get(j.job_id, 0) for j in jobs),
"worker_peak_mib": peak_mib,
"render_peak_mib": _r(max(p["render_peak_kib"] for p in processed) / 1024)
if processed
else None,
}
usage_start = marks.get("worker_start", start) if phase == "drain" else start
row.update(window_usage(data.ticks, usage_start, end))
if phase == "drain":
throughput = drain_throughput(jobs, data.consuming_ts)
if throughput:
row["throughput"] = _r(throughput, 3)
row["drain_s"] = _r(row["ready"] / throughput, 3)
if data.consuming_ts is not None and "worker_start" in marks:
row["startup_s"] = _r(data.consuming_ts - marks["worker_start"], 3)
if (row["accepted"] or 0) < (row["scheduled"] or 0):
flags.append("drain_rejected")
if phase == "overload":
stop = marks.get("recovery", end)
row["max_unfinished"] = max_unfinished(data.jobs, start, stop)
if phase == "recovery":
overload_end = data.gens.get("overload", {}).get("end")
earlier = [
j.finished_at
for j in data.jobs
if j.phase != "recovery" and j.finished_at is not None
]
if overload_end is not None and earlier:
row["recovery_s"] = _r(max(0.0, max(earlier) - overload_end), 3)
if row["failed"]:
flags.append("failed")
if row["missing"]:
flags.append("missing")
if row["errors_5xx"]:
flags.append("5xx")
if row["dropped"]:
flags.append("gen_dropped")
row["flags"] = " ".join(flags)
rows.append(row)
return rows
def timeline_rows(data: CellData) -> list[dict]:
"""Шкала ячейки раз в секунду: выборки SQL, ближайшая выборка cgroup
worker и ответы генератора с прошлой выборки."""
item = data.item
done = sorted(
(u.done, u.status)
for phase in data.uploads.values()
for u in phase
if u.done is not None
)
done_ts = [d[0] for d in done]
tick_ts = [t.ts for t in data.ticks]
rows = []
previous = None
origin = data.marks.get("drain", data.samples[0]["ts"] if data.samples else 0.0)
for sample in data.samples:
ts = sample["ts"]
lo = bisect.bisect_right(done_ts, previous) if previous is not None else 0
hi = bisect.bisect_right(done_ts, ts)
window = [status for _, status in done[lo:hi]]
index = bisect.bisect_right(tick_ts, ts) - 1
tick = data.ticks[index] if index >= 0 and ts - tick_ts[index] <= 2.5 else None
rows.append(
{
"cell": item["cell"],
"config": item["config"],
"size_class": item["size_class"],
"repeat": item["repeat"],
"phase": sample.get("phase", ""),
"ts": round(ts, 3),
"t_s": round(ts - origin, 2),
**{
k: sample[k]
for k in (
"unfinished",
"outbox",
"in_broker",
"processing",
"oldest_age_s",
)
},
"worker_cpu_cores": tick.cpu_cores if tick else None,
"worker_mem_mib": _r(tick.memory / MIB) if tick else None,
"accepted": sum(s in (201, 202) for s in window),
"rejected_429": sum(s == 429 for s in window),
"errors": sum(s not in (201, 202, 429) for s in window),
}
)
previous = ts
return rows
def job_rows(data: CellData) -> list[dict]:
rows = []
for job in data.jobs:
processed = data.log.processed.get(job.attachment_id, {})
rows.append(
{
"cell": data.item["cell"],
"phase": job.phase,
"attachment_id": job.attachment_id,
"created_at": round(job.created_at, 6),
"ready_at": _r(job.ready_at, 6),
"attempt": job.attempt,
"waited_ms": processed.get("waited_ms"),
"took_ms": processed.get("took_ms"),
"state": job.state or job.status,
"job_created_at": _r(job.job_created_at, 6),
"finished_at": _r(job.finished_at, 6),
"render_peak_kib": processed.get("render_peak_kib"),
}
)
return rows
# --- вердикты ----------------------------------------------------------------------
def num(value) -> float | None:
if value is None or value == "":
return None
if isinstance(value, bool):
return float(value)
try:
return float(value)
except (TypeError, ValueError):
return None
def median(values: Iterable) -> float | None:
present = [v for v in (num(x) for x in values) if v is not None]
return statistics.median(present) if present else None
def spread(values: Iterable, digits: int = 1, scale: float = 1.0) -> str:
present = [v / scale for v in (num(x) for x in values) if v is not None]
if not present:
return "—"
mid = statistics.median(present)
if len(present) == 1:
return f"{mid:.{digits}f}"
return f"{mid:.{digits}f} ({min(present):.{digits}f}–{max(present):.{digits}f})"
def fmt(value: float | None, digits: int = 2) -> str:
return "—" if value is None else f"{value:.{digits}f}"
def group(rows: list[dict]) -> dict[tuple[str, str, str], list[dict]]:
out: dict[tuple[str, str, str], list[dict]] = {}
for row in rows:
key = (row["config"], row["size_class"], row["phase"])
out.setdefault(key, []).append(row)
return out
def metric(
grouped: dict, config: str, size_class: str, phase: str, name: str
) -> float | None:
return median(r.get(name) for r in grouped.get((config, size_class, phase), []))
@dataclass(frozen=True)
class Verdict:
hypothesis: str
result: str
numbers: str
CONFIRMED = "подтверждена"
REFUTED = "опровергнута"
UNTESTED = "нет данных"
def _result(checked: int, failed: bool, missing: bool) -> str:
"""Опровергнута, если не прошла хоть одна проверенная часть; нет
данных, если части не хватает и опровергать нечем."""
if failed:
return REFUTED
if missing or not checked:
return UNTESTED
return CONFIRMED
def verdict_q1(grouped: dict) -> Verdict:
"""Q1: на `l` и `x` c2p2 ≥ 1,6 × c1p1, а c4p4 ≤ 1,15 × c2p2 (у worker
2 CPU: больше двух рендеров сразу не ускоряют)."""
parts, checked, failed, missing = [], 0, False, False
for size_class in Q1_CLASSES:
t1, t2, t4 = (
metric(grouped, c, size_class, "drain", "throughput")
for c in ("c1p1", "c2p2", "c4p4")
)
if not t1 or not t2 or t4 is None:
missing = True
parts.append(f"{size_class}: нет ячеек c1p1, c2p2 и c4p4")
continue
checked += 1
up, flat = t2 / t1, t4 / t2
failed |= not (up >= Q1_SCALE and flat <= Q1_FLAT)
parts.append(
f"{size_class}: c2p2/c1p1 = {t2:.3f}/{t1:.3f} = {up:.2f} (≥ {Q1_SCALE}), "
f"c4p4/c2p2 = {t4:.3f}/{t2:.3f} = {flat:.2f} (≤ {Q1_FLAT}) гот./с"
)
return Verdict("Q1", _result(checked, failed, missing), "; ".join(parts))
def verdict_q2(grouped: dict, classes: list[str]) -> Verdict:
"""Q2: prefetch 32 против 2 при concurrency 2 — пропускная в ±10 %,
p95 фазы fixed (одна интенсивность для обеих) в ±20 % на каждом
классе."""
parts, checked, failed, missing = [], 0, False, False
for size_class in classes:
t2 = metric(grouped, "c2p2", size_class, "drain", "throughput")
t32 = metric(grouped, "c2p32", size_class, "drain", "throughput")
p2 = metric(grouped, "c2p2", size_class, "fixed", "ttr_p95_ms")
p32 = metric(grouped, "c2p32", size_class, "fixed", "ttr_p95_ms")
if not t2 or t32 is None or not p2 or p32 is None:
missing = True
parts.append(f"{size_class}: нет ячеек c2p2 и c2p32")
continue
checked += 1
dt, dp = t32 / t2 - 1, p32 / p2 - 1
failed |= not (abs(dt) <= Q2_THROUGHPUT and abs(dp) <= Q2_P95)
parts.append(
f"{size_class}: пропускная {t32:.3f} к {t2:.3f} ({dt:+.0%}, допуск ±{Q2_THROUGHPUT:.0%}), "
f"p95 {p32 / 1000:.2f} к {p2 / 1000:.2f} с ({dp:+.0%}, допуск ±{Q2_P95:.0%})"
)
return Verdict("Q2", _result(checked, failed, missing), "; ".join(parts))
def verdict_q3(grouped: dict) -> Verdict:
"""Q3: память растёт с concurrency, когда дорого декодирование: на `x`
пик worker c4p4 ≥ 2,5 × c1p1, на `s` — ≤ 1,5 × c1p1."""
parts, checked, failed, missing = [], 0, False, False
for size_class, ratio, bigger in (
(Q3_BIG_CLASS, Q3_BIG_RATIO, True),
(Q3_SMALL_CLASS, Q3_SMALL_RATIO, False),
):
m1 = metric(grouped, "c1p1", size_class, "drain", "worker_peak_mib")
m4 = metric(grouped, "c4p4", size_class, "drain", "worker_peak_mib")
if not m1 or m4 is None:
missing = True
parts.append(f"{size_class}: нет пиков c1p1 и c4p4")
continue
checked += 1
value = m4 / m1
ok = value >= ratio if bigger else value <= ratio
failed |= not ok
sign = "≥" if bigger else "≤"
parts.append(
f"{size_class}: пик c4p4/c1p1 = {m4:.0f}/{m1:.0f} MiB = {value:.2f} ({sign} {ratio})"
)
return Verdict("Q3", _result(checked, failed, missing), "; ".join(parts))
def verdict_q4(rows: list[dict], *, backlog_limit: int, slack: int) -> Verdict:
"""Q4: в перегрузке незавершённых не больше предела + мест загрузок
стенда, после достижения предела 429 ≥ 30 % загрузок, 5xx нет. Доля
429 — с первого отказа «работы слишком много» до конца окна: пока
очередь не дошла до предела, отказывать не в чем. Ячейки, где предел
не достигнут, долю не проверяют (названы в числах)."""
over = [r for r in rows if r["phase"] == "overload"]
if not over:
return Verdict("Q4", UNTESTED, "нет фаз перегрузки")
bound = backlog_limit + slack
peaks = [num(r.get("max_unfinished")) for r in over]
peak = max((p for p in peaks if p is not None), default=None)
fives = sum(int(num(r.get("errors_5xx")) or 0) for r in over)
reached = [
r for r in over if str(r.get("backlog_reached")) in ("True", "true", "1")
]
shares = [num(r.get("share_429_saturated")) for r in reached]
shares = [s for s in shares if s is not None]
whole = [num(r.get("share_429")) for r in over]
whole = [w for w in whole if w is not None]
not_reached = sorted({r["cell"] for r in over if r not in reached})
failed = (peak is not None and peak > bound) or fives > 0
failed |= any(s < Q4_SHARE for s in shares)
missing = peak is None or not shares
numbers = (
f"макс. незавершённых {fmt(peak, 0)} (граница {backlog_limit} + {slack} = {bound}); "
f"429 после предела: min {fmt(min(shares) * 100 if shares else None, 0)} % "
f"(порог {Q4_SHARE:.0%}) в {len(shares)} ячейках; за всё окно "
+ (f"{min(whole):.0%}–{max(whole):.0%}" if whole else "—")
+ f"; 5xx {fives}"
+ (f"; предел не достигнут: {', '.join(not_reached)}" if not_reached else "")
)
return Verdict("Q4", _result(len(over), failed, missing), numbers)
def verdict_q5(rows: list[dict], *, backlog_limit: int) -> Verdict:
"""Q5: после перегрузки очередь пустеет за ≤ 1,3 × предел / пропускная
+ 5 с, p95 ровной фазы после — ≤ 1,3 × p95 до + 0,5 с, и каждая
принятая загрузка B–D стала ready. Проверяется каждая ячейка × повтор."""
cells: dict[str, dict[str, dict]] = {}
for row in rows:
cells.setdefault(row["cell"], {})[row["phase"]] = row
parts, checked, failed, missing = [], 0, False, False
lost = 0
worst_drain = worst_p95 = None
for name, phases in sorted(cells.items()):
drain, steady, recovery = (
phases.get(p) for p in ("drain", "steady", "recovery")
)
if not drain or not steady or not recovery:
missing = True
continue
throughput = num(drain.get("throughput"))
recovery_s = num(recovery.get("recovery_s"))
before, after = num(steady.get("ttr_p95_ms")), num(recovery.get("ttr_p95_ms"))
if not throughput or recovery_s is None or before is None or after is None:
missing = True
parts.append(f"{name}: нет чисел")
continue
checked += 1
drain_bound = Q5_DRAIN_FACTOR * backlog_limit / throughput + Q5_DRAIN_EXTRA_S
p95_bound = Q5_P95_FACTOR * before + Q5_P95_EXTRA_MS
bad = sum(
int(num(phases[p].get(k)) or 0)
for p in ("steady", "fixed", "overload", "recovery")
if p in phases
for k in ("failed", "missing")
)
lost += bad
drain_share, p95_share = recovery_s / drain_bound, after / p95_bound
worst_drain = max(worst_drain or 0.0, drain_share)
worst_p95 = max(worst_p95 or 0.0, p95_share)
if recovery_s > drain_bound or after > p95_bound or bad:
failed = True
parts.append(
f"{name}: опустошение {recovery_s:.1f} с (граница {drain_bound:.1f}), "
f"p95 {after / 1000:.2f} с (граница {p95_bound / 1000:.2f}), потеряно {bad}"
)
head = (
f"{checked} ячеек; худшее опустошение — {fmt(worst_drain)} границы, "
f"худший p95 после — {fmt(worst_p95)} границы; failed + missing {lost}"
)
return Verdict("Q5", _result(checked, failed, missing), "; ".join([head, *parts]))
def recommendation(
grouped: dict, configs: list[str], classes: list[str], worker_limit_mib: float
) -> str:
"""Класс → настройка с наименьшим p95 фазы fixed (одна интенсивность
для всех настроек класса) среди тех, чей пик worker не выше доли
лимита памяти; p95 в пределах `RECOMMEND_P95_TIE` от лучшего считаются
равными, и из них берётся меньший пик."""
cap = worker_limit_mib * RECOMMEND_MEMORY_SHARE
parts = []
for size_class in classes:
found = []
for config in configs:
p95 = metric(grouped, config, size_class, "fixed", "ttr_p95_ms")
peak = metric(grouped, config, size_class, "drain", "worker_peak_mib")
if p95 is None or peak is None or peak > cap:
continue
found.append((config, p95, peak))
if not found:
parts.append(f"{size_class} — нет настройки с пиком ≤ {cap:.0f} MiB")
continue
fastest = min(p for _, p, _ in found)
near = [f for f in found if f[1] <= fastest * (1 + RECOMMEND_P95_TIE)]
config, p95, peak = min(near, key=lambda f: (f[2], f[1]))
throughput = metric(grouped, config, size_class, "drain", "throughput")
others = ", ".join(
f"{c} {p / 1000:.2f} с" for c, p, _ in sorted(found, key=lambda f: f[1])
)
parts.append(
f"{size_class} — {config} (p95 fixed {p95 / 1000:.2f} с, пропускная "
f"{fmt(throughput, 3)} гот./с, пик {peak:.0f} MiB; p95 fixed: {others})"
)
return (
f"рекомендация (наименьший p95 фазы fixed при пике worker ≤ "
f"{RECOMMEND_MEMORY_SHARE:.0%} лимита, {cap:.0f} MiB; p95 в пределах "
f"{RECOMMEND_P95_TIE:.0%} — меньший пик): " + "; ".join(parts)
)
def verdicts(
rows: list[dict], *, backlog_limit: int, slack: int, classes: list[str]
) -> list[Verdict]:
grouped = group(rows)
return [
verdict_q1(grouped),
verdict_q2(grouped, classes),
verdict_q3(grouped),
verdict_q4(rows, backlog_limit=backlog_limit, slack=slack),
verdict_q5(rows, backlog_limit=backlog_limit),
]
# --- сводка ------------------------------------------------------------------------
def _flags(items: list[dict]) -> str:
found = sorted({f for r in items for f in str(r.get("flags") or "").split()})
return " ".join(found) or "—"
def render_summary(meta: dict, rows: list[dict]) -> str:
profile = meta["profile"]
limit = profile["backlog_limit"]
slack = meta.get("upload_slack", 0)
configs = [c[0] for c in profile["configs"]]
classes = [c[0] for c in profile["classes"]]
worker_mib = float(profile["worker_memory"].rstrip("m"))
status = "прерван: " + meta["aborted"] if meta.get("aborted") else "готово"
bodies = meta.get("bodies", {})
lines = [
f"# Очередь под перегрузкой: метка `{meta['label']}`",
"",
f"- Статус: **{status}**",
f"- Код (`src_sha256` образа): `{meta.get('code', {}).get('src_sha256', '—')}`",
f"- Профиль `{profile['name']}`: настройки "
+ ", ".join(
f"{n} (concurrency {c}, prefetch {p})" for n, c, p in profile["configs"]
)
+ f"; повторов {profile['repeats']}; окно {profile['window_s']:g} с, "
f"ровная {profile['steady_factor']:g} ёмкости ячейки, общая интенсивность "
"класса (fixed) "
+ ", ".join(f"{k} {v:g}/с" for k, v in profile["fixed_rates"])
+ f", перегрузка {profile['overload_factor']:g} ёмкости в окне "
f"min({profile['overload_max_s']:g} с, max(окно, "
f"{profile['overload_fill']:g} × предел / ёмкость)), после перегрузки — "
f"ровная {profile['window_s'] / 2:g} с; предел незавершённых "
f"`UPLOAD_MAX_UNFINISHED={limit}`",
"- Тела: "
+ ", ".join(
f"{name} — {item['kind']} {item['width']} × {item['height']}, "
f"{item['bytes']} байт, SHA-256 `{item['sha256'][:16]}…`"
for name, item in bodies.items()
),
f"- Стенд: LB, {profile['api_replicas']} реплики по {profile['api_cpus']:g} CPU / "
f"{profile['api_memory']}, worker {profile['worker_cpus']:g} CPU / "
f"{profile['worker_memory']} (`RENDER_MEMORY_MB={profile['render_memory_mb']}`), "
f"брокер {profile['mq_cpus']:g} CPU / {profile['mq_memory']}, S3 "
f"{profile['s3_cpus']:g} CPU / {profile['s3_memory']}",
"- Пропускная — ready/с фазы ёмкости: N тел ждут в брокере, отсчёт — "
"от строки `broker consuming` worker до последнего ready. Время до "
"ready — `attachment_jobs.updated_at − attachments.created_at` (часы "
"PostgreSQL); queue lag — `waited` журнала worker (приём → захват). "
"Пик worker — `memory.peak` cgroup контейнера за ячейку (контейнер "
"новый на ячейку), пик рендера — наибольший `render_peak_kib`. "
"Незавершённых — точный счёт по созданию и завершению заданий. "
"Восстановление — от конца окна перегрузки до последнего ready. "
"Значения — медиана (min–max) повторов",
"",
]
grouped = group(rows)
for size_class in classes:
lines += [
f"## Класс `{size_class}`",
"",
"| Настройка | Пропускная, гот./с | CPU worker, ядер | p95 до ready "
"(0,6 ёмкости), с | queue lag p95, с | p95 fixed, с | Окно перегрузки, с "
"| 429 за окно, % | 429 после предела, % | Retry-After 5 / 1 "
"| Макс. незавершённых | Восстановление, с | p95 после, с | Пик worker, MiB | "
"Пик рендера, MiB | failed / missing | Пометки |",
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|",
]
for config in configs:
drain, steady, fixed, over, rec = (
grouped.get((config, size_class, phase), []) for phase in PHASES
)
if not drain:
continue
accepted = steady + fixed + over + rec
lost = sum(
int(num(r.get(k)) or 0) for r in accepted for k in ("failed", "missing")
)
failed = sum(int(num(r.get("failed")) or 0) for r in accepted)
lines.append(
f"| {config} | {spread((r['throughput'] for r in drain), 3)} | "
f"{spread((r['worker_cpu_cores'] for r in drain), 2)} | "
f"{spread((r['ttr_p95_ms'] for r in steady), 2, 1000)} | "
f"{spread((r['lag_p95_ms'] for r in steady), 2, 1000)} | "
f"{spread((r['ttr_p95_ms'] for r in fixed), 2, 1000)} | "
f"{spread((r['window_s'] for r in over), 0)} | "
f"{spread((r['share_429'] for r in over), 0, 0.01)} | "
f"{spread((r['share_429_saturated'] for r in over), 0, 0.01)} | "
f"{spread((r['retry_after_5'] for r in over), 0)} / "
f"{spread((r['retry_after_1'] for r in over), 0)} | "
f"{spread((r['max_unfinished'] for r in over), 0)} | "
f"{spread((r['recovery_s'] for r in rec), 1)} | "
f"{spread((r['ttr_p95_ms'] for r in rec), 2, 1000)} | "
f"{spread((r['worker_peak_mib'] for r in drain), 0)} | "
f"{spread((r['render_peak_mib'] for r in drain + accepted), 0)} | "
f"{failed} / {lost - failed} | {_flags(drain + accepted)} |"
)
lines.append("")
lines += [
"## Гипотезы",
"",
"Тексты и пороги — `REPORT.md`, «Очередь: гипотезы» (записаны до "
f"замеров); граница незавершённых — предел {limit} + места загрузок "
f"стенда {slack}.",
"",
"| Гипотеза | Итог | Числа |",
"|---|---|---|",
]
lines += [
f"| {v.hypothesis} | {v.result} | {v.numbers} |"
for v in verdicts(rows, backlog_limit=limit, slack=slack, classes=classes)
]
lines += ["", "- " + recommendation(grouped, configs, classes, worker_mib)]
warnings = meta.get("host_warnings") or []
if warnings:
lines += ["", "## Предупреждения о хосте", "", *(f"- {w}" for w in warnings)]
return "\n".join(lines) + "\n"
# --- --selftest: проверка расчётов без стенда --------------------------------------
def _synthetic_cell(
item: dict, *, throughput: float, peak_mib: float, steady_s: float, fixed_s: float
) -> CellData:
"""Ячейка без docker: отметки, журнал генератора и задания под заданную
пропускную и время до ready ровной фазы и фазы fixed; в перегрузке 15
принятых и 5 отказов после предела."""
base = 1_800_000_000.0
marks = {
"drain": base,
"worker_start": base + 5,
"steady": base + 30,
"fixed": base + 45,
"overload": base + 60,
"recovery": base + 100,
"end": base + 130,
}
gens: dict[str, dict] = {}
uploads: dict[str, list[Upload]] = {}
jobs: list[Job] = []
processed: dict[str, dict] = {}
number = 0
def add(phase: str, at: float, status: int, ready_after: float | None) -> None:
nonlocal number
number += 1
attachment = f"a{number}" if status == 202 else ""
uploads.setdefault(phase, []).append(
Upload(
at, at, at + 0.05, status, "5" if status == 429 else "", attachment, ""
)
)
if status != 202:
return
finish = at + ready_after if ready_after is not None else None
jobs.append(
Job(
attachment,
at,
f"j{number}",
at + 0.01,
"ready" if finish else "queued",
finish,
1,
"ready",
)
)
if finish is not None:
processed[attachment] = {
"ts": finish,
"job_id": f"j{number}",
"attempt": 1,
"render_peak_kib": 50_000,
"took_ms": 200.0,
"waited_ms": 100.0,
}
consuming = marks["worker_start"] + 1
for n in range(10):
add(
"drain",
base + 1 + n * 0.1,
202,
consuming + (n + 1) / throughput - (base + 1 + n * 0.1),
)
for n in range(6):
add("steady", marks["steady"] + n, 202, steady_s + n * 0.01)
add("fixed", marks["fixed"] + n, 202, fixed_s + n * 0.01)
for n in range(20):
status = 202 if n < 10 or n % 2 else 429
add(
"overload",
marks["overload"] + n * 0.5,
status,
30.0 if status == 202 else None,
)
for n in range(4):
add("recovery", marks["recovery"] + 5 + n, 202, 0.6)
for phase, items in uploads.items():
accepted = sum(u.accepted for u in items)
rejected = sum(u.status == 429 for u in items)
gens[phase] = {
"rate": 1.0,
"duration_s": 10.0,
"scheduled": len(items),
"sent": len(items),
"accepted": accepted,
"rejected_429": rejected,
"retry_after": {"5": rejected} if rejected else {},
"errors_5xx": 0,
"other_errors": 0,
"dropped": 0,
"end": marks["overload"] + 10 if phase == "overload" else None,
}
samples = [
{
"ts": base + t,
"phase": "overload",
"unfinished": 9,
"outbox": 0,
"in_broker": 8,
"processing": 1,
"oldest_age_s": 3.0,
}
for t in range(61, 70)
]
ticks = [Tick(base + t, 1.5, int(peak_mib * MIB * 0.9)) for t in range(0, 130)]
return CellData(
item=item,
body_bytes=1000,
marks=marks,
consuming_ts=consuming,
gens=gens,
uploads=uploads,
jobs=jobs,
log=WorkerLog(consuming=[consuming], processed=processed),
ticks=ticks,
samples=samples,
peak_bytes=int(peak_mib * MIB),
backlog_limit=10,
)
def selftest() -> None:
configs = (("c1p1", 1, 1), ("c2p2", 2, 2), ("c4p4", 4, 4), ("c2p32", 2, 32))
plan = plan_cells(configs, ("s", "l", "x"), 2, 7)
assert [p["order"] for p in plan] == list(range(1, 25))
assert len({p["cell"] for p in plan}) == 24
assert plan[11]["cell"].rsplit("-", 1)[0] != plan[12]["cell"].rsplit("-", 1)[0]
assert plan == plan_cells(configs, ("s", "l", "x"), 2, 7)
text = "\n".join(
[
"2026-11-05T16:23:20.100Z INFO src.broker [rid=-] broker consuming queue=bench.attachment-jobs prefetch=2",
"2026-11-05T16:23:22.870Z INFO src.services.processing [rid=-] attachment processed attachment_id=a1 job_id=j1 attempt=1 variants=3 render_peak_kib=98236 took=201ms waited=241ms",
"2026-11-05T16:23:23.000Z INFO src.services.processing [rid=-] job delivery skipped job_id=j1 attempt=1 outcome=done",
"2026-11-05T16:23:24.000Z WARNING src.services.processing [rid=-] attachment processing failed attachment_id=a2 job_id=j2 attempt=1 error_code=image_too_large detail=x fenced=False",
"2026-11-05T16:23:25.000Z INFO src.services.processing [rid=-] job retry scheduled job_id=j3 attachment_id=a3 attempt=1 reason=s3_read delay=1.2s fenced=False",
"garbage line",
]
)
log = parse_worker_log(text)
assert log.consuming == [log_time("2026-11-05T16:23:20.100")]
assert abs(log.consuming[0] - 1793895800.1) < 1e-3, log.consuming
assert log.processed["a1"]["waited_ms"] == 241.0
assert log.processed["a1"]["render_peak_kib"] == 98236
assert (log.skipped, log.failed, log.retries) == (
{"j1": 1},
{"a2": "image_too_large"},
{"a3": 1},
)
gen = parse_gen_log(
"q,1000.0,1001.0,1500.5,202,,a1,abc,\n"
"q,2000.0,2001.0,2002.0,429,5,,abc,\n"
"q,3000.0,,,0,,,,dropped\n"
)
assert [u.status for u in gen] == [202, 429, 0]
assert (gen[0].accepted, gen[0].done, gen[2].sent) == (True, 1.5005, None)
jobs = parse_jobs(
"a1|100.0|j1|100.1|ready|101.5|1|ready\n"
"a2|100.5|j2|100.6|failed|102.0|2|failed\n"
"a3|101.0|j3|101.1|queued|101.2|1|queued\n"
"a4|101.0||||||pending\n"
)
assert (jobs[0].ttr_ms, jobs[1].ready_at, jobs[2].finished_at) == (
1500.0,
None,
None,
)
assert jobs[3].job_created_at is None
# 100.1 → +1, 100.6 → +2, 101.1 → +3, 101.5 → 2, 102.0 → 1
assert max_unfinished(jobs, 100.0, 103.0) == 3
assert max_unfinished(jobs, 101.6, 103.0) == 2
assert max_unfinished(jobs, 100.0, 100.55) == 1
sample = parse_sample("1800000000.5|7|1|4|2|3.456\n")
assert sample == {
"ts": 1800000000.5,
"unfinished": 7,
"outbox": 1,
"in_broker": 4,
"processing": 2,
"oldest_age_s": 3.46,
}
lines = [
{"ts": 1.0, "cg": {"w": [1_000_000, 0, 0, 0, 100 * MIB]}},
{"ts": 2.0, "cg": {"w": [2_500_000, 0, 0, 0, 120 * MIB]}},
{"ts": 3.0, "cg": {}},
{"ts": 4.0, "cg": {"w": [10_000, 0, 0, 0, 50 * MIB]}},
{"ts": 5.0, "cg": {"w": [5_000, 0, 0, 0, 60 * MIB]}},
]
ticks = worker_ticks(lines, "w")
assert [t.cpu_cores for t in ticks] == [None, 1.5, None, None], ticks
assert percentile([5, 1, 3, 2, 4], 0.95) == 5
assert percentile([5, 1, 3, 2, 4], 0.5) == 3
assert percentile([], 0.5) is None
rows: list[dict] = []
classes = ["s", "l", "x"]
# пик c4p4 на x выше 80 % лимита 1536 MiB: в рекомендацию он не идёт
peaks = {"c1p1": 400.0, "c2p2": 700.0, "c4p4": 1300.0, "c2p32": 720.0}
rates = {"c1p1": 1.0, "c2p2": 1.8, "c4p4": 1.9, "c2p32": 1.85}
steady_s = {"c1p1": 0.9, "c2p2": 0.5, "c4p4": 0.45, "c2p32": 0.55}
# c2p32 быстрее c2p2 на 4 % — в пределах 10 %: рекомендация по пику
fixed_s = {"c1p1": 0.9, "c2p2": 0.5, "c4p4": 0.45, "c2p32": 0.48}
for item in plan:
config = item["config"]
peak = peaks[config] if item["size_class"] == "x" else 300.0
data = _synthetic_cell(
item,
throughput=rates[config],
peak_mib=peak,
steady_s=steady_s[config],
fixed_s=fixed_s[config],
)
cell = cell_rows(data)
assert [r["phase"] for r in cell] == list(PHASES)
assert all(set(r) <= set(CELL_COLUMNS) for r in cell), set(cell[0]) - set(
CELL_COLUMNS
)
drain, steady, fixed, over, rec = cell
assert abs(fixed["ttr_p95_ms"] - (fixed_s[config] + 0.05) * 1000) < 0.2
assert (fixed["jobs"], fixed["ready"], fixed["missing"]) == (6, 6, 0)
assert abs(drain["throughput"] - rates[config]) < 0.01, drain
assert (drain["startup_s"], drain["worker_peak_mib"]) == (1.0, peak)
assert over["max_unfinished"] == 10 + 5, over["max_unfinished"]
assert (over["backlog_reached"], over["share_429_saturated"]) == (True, 0.5)
assert (over["share_429"], over["retry_after_5"]) == (0.25, 5)
assert (rec["recovery_s"], rec["missing"]) == (29.5, 0), rec
assert abs(steady["ttr_p95_ms"] - (steady_s[config] + 0.05) * 1000) < 0.2
assert steady["lag_p95_ms"] == 100.0
assert drain["worker_cpu_cores"] == 1.5
timeline = timeline_rows(data)
assert len(timeline) == 9
assert set(timeline[0]) <= set(TIMELINE_COLUMNS)
assert sum(r["accepted"] + r["rejected_429"] for r in timeline) > 0
assert all(set(r) <= set(JOB_COLUMNS) for r in job_rows(data))
rows += cell
found = {
v.hypothesis: v.result
for v in verdicts(rows, backlog_limit=10, slack=8, classes=classes)
}
# c4p4/c2p2 = 1,9/1,8 ≤ 1,15, c2p2/c1p1 = 1,8 ≥ 1,6; пики x 1300/400;
# c2p32 против c2p2: +3 % пропускной, −4 % p95 фазы fixed
assert found == {
"Q1": CONFIRMED,
"Q2": CONFIRMED,
"Q3": CONFIRMED,
"Q4": CONFIRMED,
"Q5": REFUTED,
}, found
# восстановление синтетической ячейки — 30 с на пределе 10 при 1 гот./с:
# граница 1,3 × 10 / 1 + 5 = 18 с
assert verdict_q5(rows, backlog_limit=100).result == CONFIRMED
assert verdict_q4(rows, backlog_limit=5, slack=8).result == REFUTED
quick = [
r
for r in rows
if r["config"] in ("c1p1", "c4p4") and r["size_class"] in ("s", "x")
]
quick_found = {
v.hypothesis: v.result
for v in verdicts(quick, backlog_limit=100, slack=8, classes=["s", "x"])
}
assert quick_found == {
"Q1": UNTESTED,
"Q2": UNTESTED,
"Q3": CONFIRMED,
"Q4": CONFIRMED,
"Q5": CONFIRMED,
}, quick_found
text = recommendation(group(rows), list(peaks), classes, 1536)
# x: c4p4 выше 80 % лимита, c2p32 быстрее c2p2 меньше чем на 10 % —
# берётся меньший пик; s: пики равны — меньший p95
assert "x — c2p2" in text, text
assert "s — c4p4" in text, text
assert overload_window(40, 60, 1.0, fill=1.5, cap=150) == 90
assert overload_window(40, 60, 10.0, fill=1.5, cap=150) == 40
assert overload_window(40, 60, 0.3, fill=1.5, cap=150) == 150
meta = {
"label": "selftest",
"profile": {
"name": "queue",
"configs": configs,
"classes": (
("s", "noise", 320, 200, 100),
("l", "noise", 1800, 1800, 40),
("x", "flat", 6000, 6000, 20),
),
"repeats": 2,
"window_s": 40.0,
"steady_factor": 0.6,
"fixed_rates": (("s", 2.5), ("l", 1.0), ("x", 0.5)),
"overload_factor": 2.0,
"overload_fill": 1.5,
"overload_max_s": 150.0,
"backlog_limit": 10,
"api_replicas": 2,
"api_cpus": 1.0,
"api_memory": "512m",
"worker_cpus": 2.0,
"worker_memory": "1536m",
"render_memory_mb": 768,
"mq_cpus": 1.0,
"mq_memory": "512m",
"s3_cpus": 1.0,
"s3_memory": "512m",
},
"upload_slack": 8,
"bodies": {
"s": {
"kind": "noise",
"width": 320,
"height": 200,
"bytes": 1,
"sha256": "0" * 64,
}
},
}
summary = render_summary(meta, rows)
for part in ("## Класс `x`", "| Q5 |", "рекомендация"):
assert part in summary, part
print("queue selftest: ok")
if __name__ == "__main__":
if sys.argv[1:] == ["--selftest"]:
selftest()
else:
raise SystemExit("usage: uv run python bench/queue.py --selftest")