1483 lines
60 KiB
Python
1483 lines
60 KiB
Python
"""Профиль queue: план, разбор журналов, строки и вердикты без docker. Только stdlib.
|
||
|
||
Трек 1 лабы 5 «Очередь под перегрузкой». `run.py` (профили `queue`,
|
||
`queue-quick`) гоняет ячейки «настройка worker × класс изображения»; на
|
||
ячейку — пять фаз: `drain` (ёмкость: N тел при остановленном worker),
|
||
`steady` (0,6 ёмкости ячейки), `fixed` (одна интенсивность на класс для
|
||
всех настроек: задержки сравнимы при равной нагрузке), `overload` (2
|
||
ёмкости, окно дотягивается до предела незавершённых), `recovery`
|
||
(опустошение после перегрузки и снова 0,6 ёмкости). Здесь — чистые
|
||
функции: план ячеек кругами, разбор журнала worker, журнала генератора и
|
||
строк SQL,
|
||
строки `cells.csv`, `timeline.csv`, `jobs.csv.gz`, вердикты Q1–Q5 с
|
||
порогами-константами (гипотезы записываются в `REPORT.md` до замеров),
|
||
рекомендация и `summary.md`. Самопроверка — `uv run python
|
||
bench/queue.py --selftest`.
|
||
|
||
Имя совпадает с модулем stdlib `queue`: в процессах bench/ каталог
|
||
скрипта стоит первым в `sys.path`, и `import queue` находит этот файл.
|
||
Модули стенда stdlib `queue` не используют; импорт, которому он нужен
|
||
(`concurrent.futures`, `multiprocessing`, `logging.handlers`), в `run.py`
|
||
не добавлять.
|
||
"""
|
||
|
||
import bisect
|
||
import math
|
||
import random
|
||
import re
|
||
import statistics
|
||
import sys
|
||
from collections.abc import Iterable
|
||
from dataclasses import dataclass, field
|
||
from datetime import UTC, datetime
|
||
|
||
MIB = 2**20
|
||
PHASES = ("drain", "steady", "fixed", "overload", "recovery")
|
||
PHASE_NAMES = {
|
||
"drain": "ёмкость",
|
||
"steady": "ровная",
|
||
"fixed": "общая интенсивность",
|
||
"overload": "перегрузка",
|
||
"recovery": "восстановление",
|
||
}
|
||
# Retry-After у 429 «незавершённых заданий не меньше предела» и «места
|
||
# загрузки заняты» (src/constants.py, копия: стенд код приложения не
|
||
# импортирует)
|
||
BACKLOG_RETRY_AFTER = "5"
|
||
SLOTS_RETRY_AFTER = "1"
|
||
# пороги гипотез — до замеров (REPORT.md, «Очередь: гипотезы»)
|
||
Q1_CLASSES = ("l", "x")
|
||
Q1_SCALE = 1.6
|
||
Q1_FLAT = 1.15
|
||
Q2_THROUGHPUT = 0.10
|
||
Q2_P95 = 0.20
|
||
Q3_BIG_CLASS = "x"
|
||
Q3_BIG_RATIO = 2.5
|
||
Q3_SMALL_CLASS = "s"
|
||
Q3_SMALL_RATIO = 1.5
|
||
Q4_SHARE = 0.30
|
||
Q5_DRAIN_FACTOR = 1.3
|
||
Q5_DRAIN_EXTRA_S = 5.0
|
||
Q5_P95_FACTOR = 1.3
|
||
Q5_P95_EXTRA_MS = 500.0
|
||
# рекомендация — только среди настроек с пиком worker не выше этой доли
|
||
# лимита памяти контейнера; p95 фазы fixed в пределах этой доли от
|
||
# лучшего — равны, и выбирается меньший пик
|
||
RECOMMEND_MEMORY_SHARE = 0.8
|
||
RECOMMEND_P95_TIE = 0.10
|
||
|
||
|
||
def overload_window(
|
||
window_s: float, backlog_limit: int, throughput: float, *, fill: float, cap: float
|
||
) -> float:
|
||
"""Окно перегрузки: не короче окна профиля и достаточно долгое, чтобы
|
||
при приходе 2 × T очередь (растёт на T в секунду) дошла до предела с
|
||
запасом `fill`; не длиннее `cap`."""
|
||
return min(cap, max(window_s, fill * backlog_limit / throughput))
|
||
|
||
|
||
CELL_COLUMNS = (
|
||
"cell",
|
||
"config",
|
||
"concurrency",
|
||
"prefetch",
|
||
"size_class",
|
||
"body_bytes",
|
||
"repeat",
|
||
"order",
|
||
"phase",
|
||
"rate",
|
||
"window_s",
|
||
"scheduled",
|
||
"sent",
|
||
"accepted",
|
||
"rejected_429",
|
||
"retry_after_5",
|
||
"retry_after_1",
|
||
"errors_5xx",
|
||
"other_errors",
|
||
"dropped",
|
||
"share_429",
|
||
"share_429_saturated",
|
||
"backlog_reached",
|
||
"throughput",
|
||
"drain_s",
|
||
"startup_s",
|
||
"ttr_p50_ms",
|
||
"ttr_p95_ms",
|
||
"ttr_p99_ms",
|
||
"ttr_max_ms",
|
||
"lag_p50_ms",
|
||
"lag_p95_ms",
|
||
"lag_max_ms",
|
||
"took_p50_ms",
|
||
"took_p95_ms",
|
||
"max_unfinished",
|
||
"max_unfinished_sampled",
|
||
"max_outbox",
|
||
"max_in_broker",
|
||
"max_oldest_age_s",
|
||
"recovery_s",
|
||
"jobs",
|
||
"ready",
|
||
"failed",
|
||
"missing",
|
||
"unknown_jobs",
|
||
"retries",
|
||
"skipped",
|
||
"worker_cpu_cores",
|
||
"worker_mem_max_mib",
|
||
"worker_peak_mib",
|
||
"render_peak_mib",
|
||
"upload_p95_ms",
|
||
"flags",
|
||
)
|
||
TIMELINE_COLUMNS = (
|
||
"cell",
|
||
"config",
|
||
"size_class",
|
||
"repeat",
|
||
"phase",
|
||
"ts",
|
||
"t_s",
|
||
"unfinished",
|
||
"outbox",
|
||
"in_broker",
|
||
"processing",
|
||
"oldest_age_s",
|
||
"worker_cpu_cores",
|
||
"worker_mem_mib",
|
||
"accepted",
|
||
"rejected_429",
|
||
"errors",
|
||
)
|
||
JOB_COLUMNS = (
|
||
"cell",
|
||
"phase",
|
||
"attachment_id",
|
||
"created_at",
|
||
"ready_at",
|
||
"attempt",
|
||
"waited_ms",
|
||
"took_ms",
|
||
"state",
|
||
"job_created_at",
|
||
"finished_at",
|
||
"render_peak_kib",
|
||
)
|
||
|
||
# состояние заданий одним запросом: часы сервера (ВМ Docker), счётчики по
|
||
# частичному индексу незавершённых
|
||
TIMELINE_SQL = """
|
||
SELECT extract(epoch FROM clock_timestamp()),
|
||
count(*),
|
||
count(*) FILTER (WHERE state = 'queued' AND published_at IS NULL),
|
||
count(*) FILTER (WHERE state = 'queued' AND published_at IS NOT NULL),
|
||
count(*) FILTER (WHERE state = 'processing'),
|
||
coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0)
|
||
FROM attachment_jobs
|
||
WHERE state IN ('queued', 'processing');
|
||
"""
|
||
NOW_SQL = "SELECT extract(epoch FROM clock_timestamp());\n"
|
||
|
||
|
||
def jobs_sql(since: float) -> str:
|
||
"""Вложения ячейки с их заданием: всё принятое с начала ячейки. LEFT
|
||
JOIN — принятая строка без задания тоже видна (тогда она «missing»)."""
|
||
return (
|
||
"SELECT a.id, extract(epoch FROM a.created_at), coalesce(j.id::text, ''), "
|
||
"coalesce(extract(epoch FROM j.created_at)::text, ''), "
|
||
"coalesce(j.state, ''), coalesce(extract(epoch FROM j.updated_at)::text, ''), "
|
||
"coalesce(j.attempt::text, ''), a.status "
|
||
"FROM attachments a LEFT JOIN attachment_jobs j ON j.attachment_id = a.id "
|
||
f"WHERE a.created_at >= to_timestamp({since:.6f}) "
|
||
"ORDER BY a.created_at, a.id;\n"
|
||
)
|
||
|
||
|
||
# --- план --------------------------------------------------------------------------
|
||
|
||
|
||
def plan_cells(
|
||
configs: Iterable[tuple[str, int, int]],
|
||
classes: Iterable[str],
|
||
repeats: int,
|
||
seed: int,
|
||
) -> list[dict]:
|
||
"""Ячейки кругами: в круге r — перестановка (настройка × класс)
|
||
`Random(seed + r)`; круг не начинается с ячейки, которой кончился
|
||
прежний (одна и та же настройка подряд не попадает на границу)."""
|
||
items = [(c, k) for c in configs for k in classes]
|
||
plan: list[dict] = []
|
||
previous = None
|
||
for round_no in range(1, repeats + 1):
|
||
order = list(range(len(items)))
|
||
random.Random(seed + round_no).shuffle(order)
|
||
if len(order) > 1 and order[0] == previous:
|
||
order[0], order[1] = order[1], order[0]
|
||
for index in order:
|
||
(name, concurrency, prefetch), size_class = items[index]
|
||
plan.append(
|
||
{
|
||
"cell": f"{name}-{size_class}-r{round_no}",
|
||
"config": name,
|
||
"concurrency": concurrency,
|
||
"prefetch": prefetch,
|
||
"size_class": size_class,
|
||
"repeat": round_no,
|
||
"order": len(plan) + 1,
|
||
},
|
||
)
|
||
previous = order[-1]
|
||
return plan
|
||
|
||
|
||
# --- разбор ------------------------------------------------------------------------
|
||
|
||
|
||
LOG_LINE = re.compile(
|
||
r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3})Z \S+\s+\S+ \[rid=\S*\] (.*)$"
|
||
)
|
||
FIELD = re.compile(r"(\w+)=(\S+)")
|
||
|
||
|
||
def log_time(stamp: str) -> float:
|
||
"""`2026-11-05T16:23:22.870` (UTC, src/log.py) → секунды эпохи."""
|
||
moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%S.%f")
|
||
return moment.replace(tzinfo=UTC).timestamp()
|
||
|
||
|
||
def _ms(value: str) -> float | None:
|
||
try:
|
||
return float(value.removesuffix("ms"))
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
@dataclass
|
||
class WorkerLog:
|
||
"""Журнал worker одной ячейки: старты потребителя и итоги доставок."""
|
||
|
||
consuming: list[float] = field(default_factory=list)
|
||
# attachment_id → ts, job_id, attempt, render_peak_kib, took_ms, waited_ms
|
||
processed: dict[str, dict] = field(default_factory=dict)
|
||
# job_id → число пропущенных доставок (дубль, чужая попытка)
|
||
skipped: dict[str, int] = field(default_factory=dict)
|
||
# attachment_id → error_code
|
||
failed: dict[str, str] = field(default_factory=dict)
|
||
# attachment_id → число назначенных повторов
|
||
retries: dict[str, int] = field(default_factory=dict)
|
||
|
||
|
||
def parse_worker_log(text: str) -> WorkerLog:
|
||
out = WorkerLog()
|
||
for line in text.splitlines():
|
||
match = LOG_LINE.match(line)
|
||
if not match:
|
||
continue
|
||
ts, message = log_time(match.group(1)), match.group(2)
|
||
fields = dict(FIELD.findall(message))
|
||
if message.startswith("broker consuming"):
|
||
out.consuming.append(ts)
|
||
elif message.startswith("attachment processed"):
|
||
out.processed[fields.get("attachment_id", "")] = {
|
||
"ts": ts,
|
||
"job_id": fields.get("job_id", ""),
|
||
"attempt": int(fields.get("attempt", "0") or 0),
|
||
"render_peak_kib": int(fields.get("render_peak_kib", "0") or 0),
|
||
"took_ms": _ms(fields.get("took", "")),
|
||
"waited_ms": _ms(fields.get("waited", "")),
|
||
}
|
||
elif message.startswith("job delivery skipped"):
|
||
job = fields.get("job_id", "")
|
||
out.skipped[job] = out.skipped.get(job, 0) + 1
|
||
elif message.startswith("attachment processing failed"):
|
||
out.failed[fields.get("attachment_id", "")] = fields.get("error_code", "")
|
||
elif message.startswith("job retry scheduled"):
|
||
key = fields.get("attachment_id", "")
|
||
out.retries[key] = out.retries.get(key, 0) + 1
|
||
return out
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Upload:
|
||
"""Строка журнала генератора `queue` (loadgen.py); времена — секунды
|
||
часов ВМ, `None` — загрузка не ушла."""
|
||
|
||
scheduled: float
|
||
sent: float | None
|
||
done: float | None
|
||
status: int
|
||
retry_after: str
|
||
attachment_id: str
|
||
error: str
|
||
|
||
@property
|
||
def accepted(self) -> bool:
|
||
return self.status in (201, 202)
|
||
|
||
|
||
def _sec(value: str) -> float | None:
|
||
return float(value) / 1000 if value else None
|
||
|
||
|
||
def parse_gen_log(text: str) -> list[Upload]:
|
||
uploads = []
|
||
for line in text.splitlines():
|
||
parts = line.split(",")
|
||
if len(parts) != 9 or parts[0] != "q":
|
||
continue
|
||
uploads.append(
|
||
Upload(
|
||
scheduled=float(parts[1]) / 1000,
|
||
sent=_sec(parts[2]),
|
||
done=_sec(parts[3]),
|
||
status=int(parts[4] or 0),
|
||
retry_after=parts[5],
|
||
attachment_id=parts[6],
|
||
error=parts[8],
|
||
)
|
||
)
|
||
return uploads
|
||
|
||
|
||
@dataclass
|
||
class Job:
|
||
"""Вложение ячейки и его задание (`jobs_sql`); времена — секунды эпохи
|
||
часов сервера. `finished_at` — `updated_at` задания в `ready`/`failed`."""
|
||
|
||
attachment_id: str
|
||
created_at: float
|
||
job_id: str
|
||
job_created_at: float | None
|
||
state: str
|
||
finished_at: float | None
|
||
attempt: int | None
|
||
status: str
|
||
phase: str = ""
|
||
|
||
@property
|
||
def ready_at(self) -> float | None:
|
||
return self.finished_at if self.state == "ready" else None
|
||
|
||
@property
|
||
def ttr_ms(self) -> float | None:
|
||
ready = self.ready_at
|
||
return None if ready is None else (ready - self.created_at) * 1000
|
||
|
||
|
||
def _opt(value: str) -> float | None:
|
||
return float(value) if value else None
|
||
|
||
|
||
def parse_jobs(text: str) -> list[Job]:
|
||
jobs = []
|
||
for line in text.splitlines():
|
||
parts = line.split("|")
|
||
if len(parts) != 8:
|
||
continue
|
||
state = parts[4]
|
||
jobs.append(
|
||
Job(
|
||
attachment_id=parts[0],
|
||
created_at=float(parts[1]),
|
||
job_id=parts[2],
|
||
job_created_at=_opt(parts[3]),
|
||
state=state,
|
||
finished_at=_opt(parts[5]) if state in ("ready", "failed") else None,
|
||
attempt=int(parts[6]) if parts[6] else None,
|
||
status=parts[7],
|
||
)
|
||
)
|
||
return jobs
|
||
|
||
|
||
def parse_sample(text: str) -> dict | None:
|
||
"""Строка `TIMELINE_SQL` → выборка шкалы."""
|
||
parts = text.strip().split("|")
|
||
if len(parts) != 6:
|
||
return None
|
||
return {
|
||
"ts": float(parts[0]),
|
||
"unfinished": int(parts[1]),
|
||
"outbox": int(parts[2]),
|
||
"in_broker": int(parts[3]),
|
||
"processing": int(parts[4]),
|
||
"oldest_age_s": round(float(parts[5]), 2),
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Tick:
|
||
"""Одна строка `scan` по worker: CPU с прошлой строки и текущая память cgroup."""
|
||
|
||
ts: float
|
||
cpu_cores: float | None
|
||
memory: int
|
||
|
||
|
||
def worker_ticks(lines: list[dict], container: str) -> list[Tick]:
|
||
"""Строки сэмплера (bench/sampler.py) → выборки контейнера `container`
|
||
(id12). После перезапуска контейнера счётчик CPU начинается заново:
|
||
интервал через перезапуск CPU не получает."""
|
||
ticks: list[Tick] = []
|
||
previous = None
|
||
for line in lines:
|
||
values = line.get("cg", {}).get(container)
|
||
if values is None:
|
||
previous = None
|
||
continue
|
||
cpu = None
|
||
if previous is not None and values[0] >= previous[1][0]:
|
||
seconds = line["ts"] - previous[0]
|
||
if seconds > 0:
|
||
cpu = round((values[0] - previous[1][0]) / seconds / 1e6, 3)
|
||
ticks.append(Tick(ts=line["ts"], cpu_cores=cpu, memory=values[4]))
|
||
previous = (line["ts"], values)
|
||
return ticks
|
||
|
||
|
||
# --- метрики -----------------------------------------------------------------------
|
||
|
||
|
||
def percentile(values: Iterable[float], fraction: float) -> float | None:
|
||
"""Ближайший ранг, как у генератора (loadgen.percentile)."""
|
||
ordered = sorted(values)
|
||
if not ordered:
|
||
return None
|
||
rank = max(1, math.ceil(fraction * len(ordered)))
|
||
return ordered[min(rank, len(ordered)) - 1]
|
||
|
||
|
||
def _r(value: float | None, digits: int = 1) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def max_unfinished(jobs: Iterable[Job], start: float, end: float) -> int:
|
||
"""Наибольшее число незавершённых заданий в [start, end] по событиям:
|
||
+1 при создании задания, −1 при его ready/failed. Точнее выборок раз в
|
||
секунду: пик между ними тоже виден."""
|
||
events: list[tuple[float, int]] = []
|
||
level = 0
|
||
for job in jobs:
|
||
if job.job_created_at is None:
|
||
continue
|
||
finish = job.finished_at
|
||
if job.job_created_at <= start:
|
||
if finish is None or finish > start:
|
||
level += 1
|
||
elif job.job_created_at <= end:
|
||
events.append((job.job_created_at, 1))
|
||
if finish is not None and start < finish <= end and job.job_created_at <= end:
|
||
events.append((finish, -1))
|
||
peak = level
|
||
# при равном времени сначала уход, потом приход: пик не завышается
|
||
for _, step in sorted(events, key=lambda e: (e[0], e[1])):
|
||
level += step
|
||
peak = max(peak, level)
|
||
return peak
|
||
|
||
|
||
def drain_throughput(jobs: Iterable[Job], consuming_ts: float | None) -> float | None:
|
||
"""Ready/с фазы ёмкости: готовых заданий на секунды от начала работы
|
||
потребителя (`broker consuming`) до последнего ready. Старт процесса
|
||
worker сюда не входит — он в `startup_s`."""
|
||
ready = [j.ready_at for j in jobs if j.ready_at is not None]
|
||
if consuming_ts is None or not ready:
|
||
return None
|
||
span = max(ready) - consuming_ts
|
||
return len(ready) / span if span > 0 else None
|
||
|
||
|
||
def assign_phases(jobs: list[Job], marks: dict[str, float]) -> None:
|
||
"""Фаза вложения — по `created_at` между отметками начала фаз."""
|
||
bounds = sorted((marks[p], p) for p in PHASES if p in marks)
|
||
starts = [b[0] for b in bounds]
|
||
for job in jobs:
|
||
index = bisect.bisect_right(starts, job.created_at) - 1
|
||
job.phase = bounds[index][1] if index >= 0 else ""
|
||
|
||
|
||
def window_usage(ticks: list[Tick], start: float, end: float) -> dict:
|
||
inside = [t for t in ticks if start < t.ts <= end]
|
||
cpu = [t.cpu_cores for t in inside if t.cpu_cores is not None]
|
||
return {
|
||
"worker_cpu_cores": _r(statistics.fmean(cpu), 3) if cpu else None,
|
||
"worker_mem_max_mib": _r(max(t.memory for t in inside) / MIB)
|
||
if inside
|
||
else None,
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class CellData:
|
||
"""Всё, что `run.py` собрал за ячейку (часы ВМ Docker)."""
|
||
|
||
item: dict
|
||
body_bytes: int
|
||
# начало фаз и прочие отметки: drain, worker_start, steady, overload,
|
||
# recovery, end
|
||
marks: dict[str, float]
|
||
consuming_ts: float | None
|
||
# фаза → итог генератора и его строки
|
||
gens: dict[str, dict]
|
||
uploads: dict[str, list[Upload]]
|
||
jobs: list[Job]
|
||
log: WorkerLog
|
||
ticks: list[Tick]
|
||
samples: list[dict]
|
||
peak_bytes: int | None
|
||
backlog_limit: int
|
||
|
||
|
||
def _phase_end(marks: dict[str, float], phase: str) -> float:
|
||
later = [marks[p] for p in PHASES[PHASES.index(phase) + 1 :] if p in marks]
|
||
return later[0] if later else marks["end"]
|
||
|
||
|
||
def _gen_fields(summary: dict, uploads: list[Upload]) -> dict:
|
||
sent = summary.get("sent") or 0
|
||
rejected = summary.get("rejected_429") or 0
|
||
pauses = summary.get("retry_after") or {}
|
||
# доля 429 после предела: от первого отказа «работы слишком много» до
|
||
# конца окна — тогда backpressure уже включён
|
||
first = min(
|
||
(
|
||
u.scheduled
|
||
for u in uploads
|
||
if u.status == 429 and u.retry_after == BACKLOG_RETRY_AFTER
|
||
),
|
||
default=None,
|
||
)
|
||
saturated = None
|
||
if first is not None:
|
||
after = [u for u in uploads if u.scheduled >= first and u.sent is not None]
|
||
if after:
|
||
saturated = round(sum(u.status == 429 for u in after) / len(after), 4)
|
||
return {
|
||
"rate": _r(summary.get("rate"), 3),
|
||
"window_s": summary.get("duration_s"),
|
||
"scheduled": summary.get("scheduled"),
|
||
"sent": sent,
|
||
"accepted": summary.get("accepted"),
|
||
"rejected_429": rejected,
|
||
"retry_after_5": pauses.get(BACKLOG_RETRY_AFTER, 0),
|
||
"retry_after_1": pauses.get(SLOTS_RETRY_AFTER, 0),
|
||
"errors_5xx": summary.get("errors_5xx"),
|
||
"other_errors": summary.get("other_errors"),
|
||
"dropped": summary.get("dropped"),
|
||
"share_429": round(rejected / sent, 4) if sent else None,
|
||
"share_429_saturated": saturated,
|
||
"backlog_reached": first is not None,
|
||
"upload_p95_ms": summary.get("upload_p95_ms"),
|
||
}
|
||
|
||
|
||
def cell_rows(data: CellData) -> list[dict]:
|
||
"""Строки `cells.csv` ячейки — по одной на фазу."""
|
||
item, marks = data.item, data.marks
|
||
assign_phases(data.jobs, marks)
|
||
by_id = {job.attachment_id: job for job in data.jobs}
|
||
log = data.log
|
||
peak_mib = _r(data.peak_bytes / MIB) if data.peak_bytes else None
|
||
rows = []
|
||
for phase in PHASES:
|
||
if phase not in marks:
|
||
continue
|
||
start, end = marks[phase], _phase_end(marks, phase)
|
||
jobs = [j for j in data.jobs if j.phase == phase]
|
||
uploads = data.uploads.get(phase, [])
|
||
accepted_ids = {u.attachment_id for u in uploads if u.accepted}
|
||
processed = [
|
||
log.processed[j.attachment_id]
|
||
for j in jobs
|
||
if j.attachment_id in log.processed
|
||
]
|
||
waited = [p["waited_ms"] for p in processed if p["waited_ms"] is not None]
|
||
took = [p["took_ms"] for p in processed if p["took_ms"] is not None]
|
||
ttr = [j.ttr_ms for j in jobs if j.ttr_ms is not None]
|
||
samples = [s for s in data.samples if s.get("phase") == phase]
|
||
flags = []
|
||
row = {
|
||
**{k: item[k] for k in ("cell", "config", "concurrency", "prefetch")},
|
||
"size_class": item["size_class"],
|
||
"body_bytes": data.body_bytes,
|
||
"repeat": item["repeat"],
|
||
"order": item["order"],
|
||
"phase": phase,
|
||
**_gen_fields(data.gens.get(phase, {}), uploads),
|
||
"ttr_p50_ms": _r(percentile(ttr, 0.50)),
|
||
"ttr_p95_ms": _r(percentile(ttr, 0.95)),
|
||
"ttr_p99_ms": _r(percentile(ttr, 0.99)),
|
||
"ttr_max_ms": _r(max(ttr)) if ttr else None,
|
||
"lag_p50_ms": _r(percentile(waited, 0.50)),
|
||
"lag_p95_ms": _r(percentile(waited, 0.95)),
|
||
"lag_max_ms": _r(max(waited)) if waited else None,
|
||
"took_p50_ms": _r(percentile(took, 0.50)),
|
||
"took_p95_ms": _r(percentile(took, 0.95)),
|
||
"max_unfinished_sampled": max(
|
||
(s["unfinished"] for s in samples), default=None
|
||
),
|
||
"max_outbox": max((s["outbox"] for s in samples), default=None),
|
||
"max_in_broker": max((s["in_broker"] for s in samples), default=None),
|
||
"max_oldest_age_s": max((s["oldest_age_s"] for s in samples), default=None),
|
||
"jobs": len(jobs),
|
||
"ready": sum(j.state == "ready" for j in jobs),
|
||
"failed": sum(
|
||
by_id[i].state == "failed" for i in accepted_ids if i in by_id
|
||
),
|
||
# принято (201/202), но не ready и не failed к концу ячейки
|
||
"missing": sum(
|
||
1
|
||
for i in accepted_ids
|
||
if i not in by_id or by_id[i].state not in ("ready", "failed")
|
||
),
|
||
# задание есть, а 202 генератор не увидел (срок ответа, обрыв)
|
||
"unknown_jobs": sum(j.attachment_id not in accepted_ids for j in jobs),
|
||
"retries": sum(log.retries.get(j.attachment_id, 0) for j in jobs),
|
||
"skipped": sum(log.skipped.get(j.job_id, 0) for j in jobs),
|
||
"worker_peak_mib": peak_mib,
|
||
"render_peak_mib": _r(max(p["render_peak_kib"] for p in processed) / 1024)
|
||
if processed
|
||
else None,
|
||
}
|
||
usage_start = marks.get("worker_start", start) if phase == "drain" else start
|
||
row.update(window_usage(data.ticks, usage_start, end))
|
||
if phase == "drain":
|
||
throughput = drain_throughput(jobs, data.consuming_ts)
|
||
if throughput:
|
||
row["throughput"] = _r(throughput, 3)
|
||
row["drain_s"] = _r(row["ready"] / throughput, 3)
|
||
if data.consuming_ts is not None and "worker_start" in marks:
|
||
row["startup_s"] = _r(data.consuming_ts - marks["worker_start"], 3)
|
||
if (row["accepted"] or 0) < (row["scheduled"] or 0):
|
||
flags.append("drain_rejected")
|
||
if phase == "overload":
|
||
stop = marks.get("recovery", end)
|
||
row["max_unfinished"] = max_unfinished(data.jobs, start, stop)
|
||
if phase == "recovery":
|
||
overload_end = data.gens.get("overload", {}).get("end")
|
||
earlier = [
|
||
j.finished_at
|
||
for j in data.jobs
|
||
if j.phase != "recovery" and j.finished_at is not None
|
||
]
|
||
if overload_end is not None and earlier:
|
||
row["recovery_s"] = _r(max(0.0, max(earlier) - overload_end), 3)
|
||
if row["failed"]:
|
||
flags.append("failed")
|
||
if row["missing"]:
|
||
flags.append("missing")
|
||
if row["errors_5xx"]:
|
||
flags.append("5xx")
|
||
if row["dropped"]:
|
||
flags.append("gen_dropped")
|
||
row["flags"] = " ".join(flags)
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def timeline_rows(data: CellData) -> list[dict]:
|
||
"""Шкала ячейки раз в секунду: выборки SQL, ближайшая выборка cgroup
|
||
worker и ответы генератора с прошлой выборки."""
|
||
item = data.item
|
||
done = sorted(
|
||
(u.done, u.status)
|
||
for phase in data.uploads.values()
|
||
for u in phase
|
||
if u.done is not None
|
||
)
|
||
done_ts = [d[0] for d in done]
|
||
tick_ts = [t.ts for t in data.ticks]
|
||
rows = []
|
||
previous = None
|
||
origin = data.marks.get("drain", data.samples[0]["ts"] if data.samples else 0.0)
|
||
for sample in data.samples:
|
||
ts = sample["ts"]
|
||
lo = bisect.bisect_right(done_ts, previous) if previous is not None else 0
|
||
hi = bisect.bisect_right(done_ts, ts)
|
||
window = [status for _, status in done[lo:hi]]
|
||
index = bisect.bisect_right(tick_ts, ts) - 1
|
||
tick = data.ticks[index] if index >= 0 and ts - tick_ts[index] <= 2.5 else None
|
||
rows.append(
|
||
{
|
||
"cell": item["cell"],
|
||
"config": item["config"],
|
||
"size_class": item["size_class"],
|
||
"repeat": item["repeat"],
|
||
"phase": sample.get("phase", ""),
|
||
"ts": round(ts, 3),
|
||
"t_s": round(ts - origin, 2),
|
||
**{
|
||
k: sample[k]
|
||
for k in (
|
||
"unfinished",
|
||
"outbox",
|
||
"in_broker",
|
||
"processing",
|
||
"oldest_age_s",
|
||
)
|
||
},
|
||
"worker_cpu_cores": tick.cpu_cores if tick else None,
|
||
"worker_mem_mib": _r(tick.memory / MIB) if tick else None,
|
||
"accepted": sum(s in (201, 202) for s in window),
|
||
"rejected_429": sum(s == 429 for s in window),
|
||
"errors": sum(s not in (201, 202, 429) for s in window),
|
||
}
|
||
)
|
||
previous = ts
|
||
return rows
|
||
|
||
|
||
def job_rows(data: CellData) -> list[dict]:
|
||
rows = []
|
||
for job in data.jobs:
|
||
processed = data.log.processed.get(job.attachment_id, {})
|
||
rows.append(
|
||
{
|
||
"cell": data.item["cell"],
|
||
"phase": job.phase,
|
||
"attachment_id": job.attachment_id,
|
||
"created_at": round(job.created_at, 6),
|
||
"ready_at": _r(job.ready_at, 6),
|
||
"attempt": job.attempt,
|
||
"waited_ms": processed.get("waited_ms"),
|
||
"took_ms": processed.get("took_ms"),
|
||
"state": job.state or job.status,
|
||
"job_created_at": _r(job.job_created_at, 6),
|
||
"finished_at": _r(job.finished_at, 6),
|
||
"render_peak_kib": processed.get("render_peak_kib"),
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
# --- вердикты ----------------------------------------------------------------------
|
||
|
||
|
||
def num(value) -> float | None:
|
||
if value is None or value == "":
|
||
return None
|
||
if isinstance(value, bool):
|
||
return float(value)
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def median(values: Iterable) -> float | None:
|
||
present = [v for v in (num(x) for x in values) if v is not None]
|
||
return statistics.median(present) if present else None
|
||
|
||
|
||
def spread(values: Iterable, digits: int = 1, scale: float = 1.0) -> str:
|
||
present = [v / scale for v in (num(x) for x in values) if v is not None]
|
||
if not present:
|
||
return "—"
|
||
mid = statistics.median(present)
|
||
if len(present) == 1:
|
||
return f"{mid:.{digits}f}"
|
||
return f"{mid:.{digits}f} ({min(present):.{digits}f}–{max(present):.{digits}f})"
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 2) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def group(rows: list[dict]) -> dict[tuple[str, str, str], list[dict]]:
|
||
out: dict[tuple[str, str, str], list[dict]] = {}
|
||
for row in rows:
|
||
key = (row["config"], row["size_class"], row["phase"])
|
||
out.setdefault(key, []).append(row)
|
||
return out
|
||
|
||
|
||
def metric(
|
||
grouped: dict, config: str, size_class: str, phase: str, name: str
|
||
) -> float | None:
|
||
return median(r.get(name) for r in grouped.get((config, size_class, phase), []))
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Verdict:
|
||
hypothesis: str
|
||
result: str
|
||
numbers: str
|
||
|
||
|
||
CONFIRMED = "подтверждена"
|
||
REFUTED = "опровергнута"
|
||
UNTESTED = "нет данных"
|
||
|
||
|
||
def _result(checked: int, failed: bool, missing: bool) -> str:
|
||
"""Опровергнута, если не прошла хоть одна проверенная часть; нет
|
||
данных, если части не хватает и опровергать нечем."""
|
||
if failed:
|
||
return REFUTED
|
||
if missing or not checked:
|
||
return UNTESTED
|
||
return CONFIRMED
|
||
|
||
|
||
def verdict_q1(grouped: dict) -> Verdict:
|
||
"""Q1: на `l` и `x` c2p2 ≥ 1,6 × c1p1, а c4p4 ≤ 1,15 × c2p2 (у worker
|
||
2 CPU: больше двух рендеров сразу не ускоряют)."""
|
||
parts, checked, failed, missing = [], 0, False, False
|
||
for size_class in Q1_CLASSES:
|
||
t1, t2, t4 = (
|
||
metric(grouped, c, size_class, "drain", "throughput")
|
||
for c in ("c1p1", "c2p2", "c4p4")
|
||
)
|
||
if not t1 or not t2 or t4 is None:
|
||
missing = True
|
||
parts.append(f"{size_class}: нет ячеек c1p1, c2p2 и c4p4")
|
||
continue
|
||
checked += 1
|
||
up, flat = t2 / t1, t4 / t2
|
||
failed |= not (up >= Q1_SCALE and flat <= Q1_FLAT)
|
||
parts.append(
|
||
f"{size_class}: c2p2/c1p1 = {t2:.3f}/{t1:.3f} = {up:.2f} (≥ {Q1_SCALE}), "
|
||
f"c4p4/c2p2 = {t4:.3f}/{t2:.3f} = {flat:.2f} (≤ {Q1_FLAT}) гот./с"
|
||
)
|
||
return Verdict("Q1", _result(checked, failed, missing), "; ".join(parts))
|
||
|
||
|
||
def verdict_q2(grouped: dict, classes: list[str]) -> Verdict:
|
||
"""Q2: prefetch 32 против 2 при concurrency 2 — пропускная в ±10 %,
|
||
p95 фазы fixed (одна интенсивность для обеих) в ±20 % на каждом
|
||
классе."""
|
||
parts, checked, failed, missing = [], 0, False, False
|
||
for size_class in classes:
|
||
t2 = metric(grouped, "c2p2", size_class, "drain", "throughput")
|
||
t32 = metric(grouped, "c2p32", size_class, "drain", "throughput")
|
||
p2 = metric(grouped, "c2p2", size_class, "fixed", "ttr_p95_ms")
|
||
p32 = metric(grouped, "c2p32", size_class, "fixed", "ttr_p95_ms")
|
||
if not t2 or t32 is None or not p2 or p32 is None:
|
||
missing = True
|
||
parts.append(f"{size_class}: нет ячеек c2p2 и c2p32")
|
||
continue
|
||
checked += 1
|
||
dt, dp = t32 / t2 - 1, p32 / p2 - 1
|
||
failed |= not (abs(dt) <= Q2_THROUGHPUT and abs(dp) <= Q2_P95)
|
||
parts.append(
|
||
f"{size_class}: пропускная {t32:.3f} к {t2:.3f} ({dt:+.0%}, допуск ±{Q2_THROUGHPUT:.0%}), "
|
||
f"p95 {p32 / 1000:.2f} к {p2 / 1000:.2f} с ({dp:+.0%}, допуск ±{Q2_P95:.0%})"
|
||
)
|
||
return Verdict("Q2", _result(checked, failed, missing), "; ".join(parts))
|
||
|
||
|
||
def verdict_q3(grouped: dict) -> Verdict:
|
||
"""Q3: память растёт с concurrency, когда дорого декодирование: на `x`
|
||
пик worker c4p4 ≥ 2,5 × c1p1, на `s` — ≤ 1,5 × c1p1."""
|
||
parts, checked, failed, missing = [], 0, False, False
|
||
for size_class, ratio, bigger in (
|
||
(Q3_BIG_CLASS, Q3_BIG_RATIO, True),
|
||
(Q3_SMALL_CLASS, Q3_SMALL_RATIO, False),
|
||
):
|
||
m1 = metric(grouped, "c1p1", size_class, "drain", "worker_peak_mib")
|
||
m4 = metric(grouped, "c4p4", size_class, "drain", "worker_peak_mib")
|
||
if not m1 or m4 is None:
|
||
missing = True
|
||
parts.append(f"{size_class}: нет пиков c1p1 и c4p4")
|
||
continue
|
||
checked += 1
|
||
value = m4 / m1
|
||
ok = value >= ratio if bigger else value <= ratio
|
||
failed |= not ok
|
||
sign = "≥" if bigger else "≤"
|
||
parts.append(
|
||
f"{size_class}: пик c4p4/c1p1 = {m4:.0f}/{m1:.0f} MiB = {value:.2f} ({sign} {ratio})"
|
||
)
|
||
return Verdict("Q3", _result(checked, failed, missing), "; ".join(parts))
|
||
|
||
|
||
def verdict_q4(rows: list[dict], *, backlog_limit: int, slack: int) -> Verdict:
|
||
"""Q4: в перегрузке незавершённых не больше предела + мест загрузок
|
||
стенда, после достижения предела 429 ≥ 30 % загрузок, 5xx нет. Доля
|
||
429 — с первого отказа «работы слишком много» до конца окна: пока
|
||
очередь не дошла до предела, отказывать не в чем. Ячейки, где предел
|
||
не достигнут, долю не проверяют (названы в числах)."""
|
||
over = [r for r in rows if r["phase"] == "overload"]
|
||
if not over:
|
||
return Verdict("Q4", UNTESTED, "нет фаз перегрузки")
|
||
bound = backlog_limit + slack
|
||
peaks = [num(r.get("max_unfinished")) for r in over]
|
||
peak = max((p for p in peaks if p is not None), default=None)
|
||
fives = sum(int(num(r.get("errors_5xx")) or 0) for r in over)
|
||
reached = [
|
||
r for r in over if str(r.get("backlog_reached")) in ("True", "true", "1")
|
||
]
|
||
shares = [num(r.get("share_429_saturated")) for r in reached]
|
||
shares = [s for s in shares if s is not None]
|
||
whole = [num(r.get("share_429")) for r in over]
|
||
whole = [w for w in whole if w is not None]
|
||
not_reached = sorted({r["cell"] for r in over if r not in reached})
|
||
failed = (peak is not None and peak > bound) or fives > 0
|
||
failed |= any(s < Q4_SHARE for s in shares)
|
||
missing = peak is None or not shares
|
||
numbers = (
|
||
f"макс. незавершённых {fmt(peak, 0)} (граница {backlog_limit} + {slack} = {bound}); "
|
||
f"429 после предела: min {fmt(min(shares) * 100 if shares else None, 0)} % "
|
||
f"(порог {Q4_SHARE:.0%}) в {len(shares)} ячейках; за всё окно "
|
||
+ (f"{min(whole):.0%}–{max(whole):.0%}" if whole else "—")
|
||
+ f"; 5xx {fives}"
|
||
+ (f"; предел не достигнут: {', '.join(not_reached)}" if not_reached else "")
|
||
)
|
||
return Verdict("Q4", _result(len(over), failed, missing), numbers)
|
||
|
||
|
||
def verdict_q5(rows: list[dict], *, backlog_limit: int) -> Verdict:
|
||
"""Q5: после перегрузки очередь пустеет за ≤ 1,3 × предел / пропускная
|
||
+ 5 с, p95 ровной фазы после — ≤ 1,3 × p95 до + 0,5 с, и каждая
|
||
принятая загрузка B–D стала ready. Проверяется каждая ячейка × повтор."""
|
||
cells: dict[str, dict[str, dict]] = {}
|
||
for row in rows:
|
||
cells.setdefault(row["cell"], {})[row["phase"]] = row
|
||
parts, checked, failed, missing = [], 0, False, False
|
||
lost = 0
|
||
worst_drain = worst_p95 = None
|
||
for name, phases in sorted(cells.items()):
|
||
drain, steady, recovery = (
|
||
phases.get(p) for p in ("drain", "steady", "recovery")
|
||
)
|
||
if not drain or not steady or not recovery:
|
||
missing = True
|
||
continue
|
||
throughput = num(drain.get("throughput"))
|
||
recovery_s = num(recovery.get("recovery_s"))
|
||
before, after = num(steady.get("ttr_p95_ms")), num(recovery.get("ttr_p95_ms"))
|
||
if not throughput or recovery_s is None or before is None or after is None:
|
||
missing = True
|
||
parts.append(f"{name}: нет чисел")
|
||
continue
|
||
checked += 1
|
||
drain_bound = Q5_DRAIN_FACTOR * backlog_limit / throughput + Q5_DRAIN_EXTRA_S
|
||
p95_bound = Q5_P95_FACTOR * before + Q5_P95_EXTRA_MS
|
||
bad = sum(
|
||
int(num(phases[p].get(k)) or 0)
|
||
for p in ("steady", "fixed", "overload", "recovery")
|
||
if p in phases
|
||
for k in ("failed", "missing")
|
||
)
|
||
lost += bad
|
||
drain_share, p95_share = recovery_s / drain_bound, after / p95_bound
|
||
worst_drain = max(worst_drain or 0.0, drain_share)
|
||
worst_p95 = max(worst_p95 or 0.0, p95_share)
|
||
if recovery_s > drain_bound or after > p95_bound or bad:
|
||
failed = True
|
||
parts.append(
|
||
f"{name}: опустошение {recovery_s:.1f} с (граница {drain_bound:.1f}), "
|
||
f"p95 {after / 1000:.2f} с (граница {p95_bound / 1000:.2f}), потеряно {bad}"
|
||
)
|
||
head = (
|
||
f"{checked} ячеек; худшее опустошение — {fmt(worst_drain)} границы, "
|
||
f"худший p95 после — {fmt(worst_p95)} границы; failed + missing {lost}"
|
||
)
|
||
return Verdict("Q5", _result(checked, failed, missing), "; ".join([head, *parts]))
|
||
|
||
|
||
def recommendation(
|
||
grouped: dict, configs: list[str], classes: list[str], worker_limit_mib: float
|
||
) -> str:
|
||
"""Класс → настройка с наименьшим p95 фазы fixed (одна интенсивность
|
||
для всех настроек класса) среди тех, чей пик worker не выше доли
|
||
лимита памяти; p95 в пределах `RECOMMEND_P95_TIE` от лучшего считаются
|
||
равными, и из них берётся меньший пик."""
|
||
cap = worker_limit_mib * RECOMMEND_MEMORY_SHARE
|
||
parts = []
|
||
for size_class in classes:
|
||
found = []
|
||
for config in configs:
|
||
p95 = metric(grouped, config, size_class, "fixed", "ttr_p95_ms")
|
||
peak = metric(grouped, config, size_class, "drain", "worker_peak_mib")
|
||
if p95 is None or peak is None or peak > cap:
|
||
continue
|
||
found.append((config, p95, peak))
|
||
if not found:
|
||
parts.append(f"{size_class} — нет настройки с пиком ≤ {cap:.0f} MiB")
|
||
continue
|
||
fastest = min(p for _, p, _ in found)
|
||
near = [f for f in found if f[1] <= fastest * (1 + RECOMMEND_P95_TIE)]
|
||
config, p95, peak = min(near, key=lambda f: (f[2], f[1]))
|
||
throughput = metric(grouped, config, size_class, "drain", "throughput")
|
||
others = ", ".join(
|
||
f"{c} {p / 1000:.2f} с" for c, p, _ in sorted(found, key=lambda f: f[1])
|
||
)
|
||
parts.append(
|
||
f"{size_class} — {config} (p95 fixed {p95 / 1000:.2f} с, пропускная "
|
||
f"{fmt(throughput, 3)} гот./с, пик {peak:.0f} MiB; p95 fixed: {others})"
|
||
)
|
||
return (
|
||
f"рекомендация (наименьший p95 фазы fixed при пике worker ≤ "
|
||
f"{RECOMMEND_MEMORY_SHARE:.0%} лимита, {cap:.0f} MiB; p95 в пределах "
|
||
f"{RECOMMEND_P95_TIE:.0%} — меньший пик): " + "; ".join(parts)
|
||
)
|
||
|
||
|
||
def verdicts(
|
||
rows: list[dict], *, backlog_limit: int, slack: int, classes: list[str]
|
||
) -> list[Verdict]:
|
||
grouped = group(rows)
|
||
return [
|
||
verdict_q1(grouped),
|
||
verdict_q2(grouped, classes),
|
||
verdict_q3(grouped),
|
||
verdict_q4(rows, backlog_limit=backlog_limit, slack=slack),
|
||
verdict_q5(rows, backlog_limit=backlog_limit),
|
||
]
|
||
|
||
|
||
# --- сводка ------------------------------------------------------------------------
|
||
|
||
|
||
def _flags(items: list[dict]) -> str:
|
||
found = sorted({f for r in items for f in str(r.get("flags") or "").split()})
|
||
return " ".join(found) or "—"
|
||
|
||
|
||
def render_summary(meta: dict, rows: list[dict]) -> str:
|
||
profile = meta["profile"]
|
||
limit = profile["backlog_limit"]
|
||
slack = meta.get("upload_slack", 0)
|
||
configs = [c[0] for c in profile["configs"]]
|
||
classes = [c[0] for c in profile["classes"]]
|
||
worker_mib = float(profile["worker_memory"].rstrip("m"))
|
||
status = "прерван: " + meta["aborted"] if meta.get("aborted") else "готово"
|
||
bodies = meta.get("bodies", {})
|
||
lines = [
|
||
f"# Очередь под перегрузкой: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{status}**",
|
||
f"- Код (`src_sha256` образа): `{meta.get('code', {}).get('src_sha256', '—')}`",
|
||
f"- Профиль `{profile['name']}`: настройки "
|
||
+ ", ".join(
|
||
f"{n} (concurrency {c}, prefetch {p})" for n, c, p in profile["configs"]
|
||
)
|
||
+ f"; повторов {profile['repeats']}; окно {profile['window_s']:g} с, "
|
||
f"ровная {profile['steady_factor']:g} ёмкости ячейки, общая интенсивность "
|
||
"класса (fixed) "
|
||
+ ", ".join(f"{k} {v:g}/с" for k, v in profile["fixed_rates"])
|
||
+ f", перегрузка {profile['overload_factor']:g} ёмкости в окне "
|
||
f"min({profile['overload_max_s']:g} с, max(окно, "
|
||
f"{profile['overload_fill']:g} × предел / ёмкость)), после перегрузки — "
|
||
f"ровная {profile['window_s'] / 2:g} с; предел незавершённых "
|
||
f"`UPLOAD_MAX_UNFINISHED={limit}`",
|
||
"- Тела: "
|
||
+ ", ".join(
|
||
f"{name} — {item['kind']} {item['width']} × {item['height']}, "
|
||
f"{item['bytes']} байт, SHA-256 `{item['sha256'][:16]}…`"
|
||
for name, item in bodies.items()
|
||
),
|
||
f"- Стенд: LB, {profile['api_replicas']} реплики по {profile['api_cpus']:g} CPU / "
|
||
f"{profile['api_memory']}, worker {profile['worker_cpus']:g} CPU / "
|
||
f"{profile['worker_memory']} (`RENDER_MEMORY_MB={profile['render_memory_mb']}`), "
|
||
f"брокер {profile['mq_cpus']:g} CPU / {profile['mq_memory']}, S3 "
|
||
f"{profile['s3_cpus']:g} CPU / {profile['s3_memory']}",
|
||
"- Пропускная — ready/с фазы ёмкости: N тел ждут в брокере, отсчёт — "
|
||
"от строки `broker consuming` worker до последнего ready. Время до "
|
||
"ready — `attachment_jobs.updated_at − attachments.created_at` (часы "
|
||
"PostgreSQL); queue lag — `waited` журнала worker (приём → захват). "
|
||
"Пик worker — `memory.peak` cgroup контейнера за ячейку (контейнер "
|
||
"новый на ячейку), пик рендера — наибольший `render_peak_kib`. "
|
||
"Незавершённых — точный счёт по созданию и завершению заданий. "
|
||
"Восстановление — от конца окна перегрузки до последнего ready. "
|
||
"Значения — медиана (min–max) повторов",
|
||
"",
|
||
]
|
||
grouped = group(rows)
|
||
for size_class in classes:
|
||
lines += [
|
||
f"## Класс `{size_class}`",
|
||
"",
|
||
"| Настройка | Пропускная, гот./с | CPU worker, ядер | p95 до ready "
|
||
"(0,6 ёмкости), с | queue lag p95, с | p95 fixed, с | Окно перегрузки, с "
|
||
"| 429 за окно, % | 429 после предела, % | Retry-After 5 / 1 "
|
||
"| Макс. незавершённых | Восстановление, с | p95 после, с | Пик worker, MiB | "
|
||
"Пик рендера, MiB | failed / missing | Пометки |",
|
||
"|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|",
|
||
]
|
||
for config in configs:
|
||
drain, steady, fixed, over, rec = (
|
||
grouped.get((config, size_class, phase), []) for phase in PHASES
|
||
)
|
||
if not drain:
|
||
continue
|
||
accepted = steady + fixed + over + rec
|
||
lost = sum(
|
||
int(num(r.get(k)) or 0) for r in accepted for k in ("failed", "missing")
|
||
)
|
||
failed = sum(int(num(r.get("failed")) or 0) for r in accepted)
|
||
lines.append(
|
||
f"| {config} | {spread((r['throughput'] for r in drain), 3)} | "
|
||
f"{spread((r['worker_cpu_cores'] for r in drain), 2)} | "
|
||
f"{spread((r['ttr_p95_ms'] for r in steady), 2, 1000)} | "
|
||
f"{spread((r['lag_p95_ms'] for r in steady), 2, 1000)} | "
|
||
f"{spread((r['ttr_p95_ms'] for r in fixed), 2, 1000)} | "
|
||
f"{spread((r['window_s'] for r in over), 0)} | "
|
||
f"{spread((r['share_429'] for r in over), 0, 0.01)} | "
|
||
f"{spread((r['share_429_saturated'] for r in over), 0, 0.01)} | "
|
||
f"{spread((r['retry_after_5'] for r in over), 0)} / "
|
||
f"{spread((r['retry_after_1'] for r in over), 0)} | "
|
||
f"{spread((r['max_unfinished'] for r in over), 0)} | "
|
||
f"{spread((r['recovery_s'] for r in rec), 1)} | "
|
||
f"{spread((r['ttr_p95_ms'] for r in rec), 2, 1000)} | "
|
||
f"{spread((r['worker_peak_mib'] for r in drain), 0)} | "
|
||
f"{spread((r['render_peak_mib'] for r in drain + accepted), 0)} | "
|
||
f"{failed} / {lost - failed} | {_flags(drain + accepted)} |"
|
||
)
|
||
lines.append("")
|
||
lines += [
|
||
"## Гипотезы",
|
||
"",
|
||
"Тексты и пороги — `REPORT.md`, «Очередь: гипотезы» (записаны до "
|
||
f"замеров); граница незавершённых — предел {limit} + места загрузок "
|
||
f"стенда {slack}.",
|
||
"",
|
||
"| Гипотеза | Итог | Числа |",
|
||
"|---|---|---|",
|
||
]
|
||
lines += [
|
||
f"| {v.hypothesis} | {v.result} | {v.numbers} |"
|
||
for v in verdicts(rows, backlog_limit=limit, slack=slack, classes=classes)
|
||
]
|
||
lines += ["", "- " + recommendation(grouped, configs, classes, worker_mib)]
|
||
warnings = meta.get("host_warnings") or []
|
||
if warnings:
|
||
lines += ["", "## Предупреждения о хосте", "", *(f"- {w}" for w in warnings)]
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
# --- --selftest: проверка расчётов без стенда --------------------------------------
|
||
|
||
|
||
def _synthetic_cell(
|
||
item: dict, *, throughput: float, peak_mib: float, steady_s: float, fixed_s: float
|
||
) -> CellData:
|
||
"""Ячейка без docker: отметки, журнал генератора и задания под заданную
|
||
пропускную и время до ready ровной фазы и фазы fixed; в перегрузке 15
|
||
принятых и 5 отказов после предела."""
|
||
base = 1_800_000_000.0
|
||
marks = {
|
||
"drain": base,
|
||
"worker_start": base + 5,
|
||
"steady": base + 30,
|
||
"fixed": base + 45,
|
||
"overload": base + 60,
|
||
"recovery": base + 100,
|
||
"end": base + 130,
|
||
}
|
||
gens: dict[str, dict] = {}
|
||
uploads: dict[str, list[Upload]] = {}
|
||
jobs: list[Job] = []
|
||
processed: dict[str, dict] = {}
|
||
number = 0
|
||
|
||
def add(phase: str, at: float, status: int, ready_after: float | None) -> None:
|
||
nonlocal number
|
||
number += 1
|
||
attachment = f"a{number}" if status == 202 else ""
|
||
uploads.setdefault(phase, []).append(
|
||
Upload(
|
||
at, at, at + 0.05, status, "5" if status == 429 else "", attachment, ""
|
||
)
|
||
)
|
||
if status != 202:
|
||
return
|
||
finish = at + ready_after if ready_after is not None else None
|
||
jobs.append(
|
||
Job(
|
||
attachment,
|
||
at,
|
||
f"j{number}",
|
||
at + 0.01,
|
||
"ready" if finish else "queued",
|
||
finish,
|
||
1,
|
||
"ready",
|
||
)
|
||
)
|
||
if finish is not None:
|
||
processed[attachment] = {
|
||
"ts": finish,
|
||
"job_id": f"j{number}",
|
||
"attempt": 1,
|
||
"render_peak_kib": 50_000,
|
||
"took_ms": 200.0,
|
||
"waited_ms": 100.0,
|
||
}
|
||
|
||
consuming = marks["worker_start"] + 1
|
||
for n in range(10):
|
||
add(
|
||
"drain",
|
||
base + 1 + n * 0.1,
|
||
202,
|
||
consuming + (n + 1) / throughput - (base + 1 + n * 0.1),
|
||
)
|
||
for n in range(6):
|
||
add("steady", marks["steady"] + n, 202, steady_s + n * 0.01)
|
||
add("fixed", marks["fixed"] + n, 202, fixed_s + n * 0.01)
|
||
for n in range(20):
|
||
status = 202 if n < 10 or n % 2 else 429
|
||
add(
|
||
"overload",
|
||
marks["overload"] + n * 0.5,
|
||
status,
|
||
30.0 if status == 202 else None,
|
||
)
|
||
for n in range(4):
|
||
add("recovery", marks["recovery"] + 5 + n, 202, 0.6)
|
||
for phase, items in uploads.items():
|
||
accepted = sum(u.accepted for u in items)
|
||
rejected = sum(u.status == 429 for u in items)
|
||
gens[phase] = {
|
||
"rate": 1.0,
|
||
"duration_s": 10.0,
|
||
"scheduled": len(items),
|
||
"sent": len(items),
|
||
"accepted": accepted,
|
||
"rejected_429": rejected,
|
||
"retry_after": {"5": rejected} if rejected else {},
|
||
"errors_5xx": 0,
|
||
"other_errors": 0,
|
||
"dropped": 0,
|
||
"end": marks["overload"] + 10 if phase == "overload" else None,
|
||
}
|
||
samples = [
|
||
{
|
||
"ts": base + t,
|
||
"phase": "overload",
|
||
"unfinished": 9,
|
||
"outbox": 0,
|
||
"in_broker": 8,
|
||
"processing": 1,
|
||
"oldest_age_s": 3.0,
|
||
}
|
||
for t in range(61, 70)
|
||
]
|
||
ticks = [Tick(base + t, 1.5, int(peak_mib * MIB * 0.9)) for t in range(0, 130)]
|
||
return CellData(
|
||
item=item,
|
||
body_bytes=1000,
|
||
marks=marks,
|
||
consuming_ts=consuming,
|
||
gens=gens,
|
||
uploads=uploads,
|
||
jobs=jobs,
|
||
log=WorkerLog(consuming=[consuming], processed=processed),
|
||
ticks=ticks,
|
||
samples=samples,
|
||
peak_bytes=int(peak_mib * MIB),
|
||
backlog_limit=10,
|
||
)
|
||
|
||
|
||
def selftest() -> None:
|
||
configs = (("c1p1", 1, 1), ("c2p2", 2, 2), ("c4p4", 4, 4), ("c2p32", 2, 32))
|
||
plan = plan_cells(configs, ("s", "l", "x"), 2, 7)
|
||
assert [p["order"] for p in plan] == list(range(1, 25))
|
||
assert len({p["cell"] for p in plan}) == 24
|
||
assert plan[11]["cell"].rsplit("-", 1)[0] != plan[12]["cell"].rsplit("-", 1)[0]
|
||
assert plan == plan_cells(configs, ("s", "l", "x"), 2, 7)
|
||
|
||
text = "\n".join(
|
||
[
|
||
"2026-11-05T16:23:20.100Z INFO src.broker [rid=-] broker consuming queue=bench.attachment-jobs prefetch=2",
|
||
"2026-11-05T16:23:22.870Z INFO src.services.processing [rid=-] attachment processed attachment_id=a1 job_id=j1 attempt=1 variants=3 render_peak_kib=98236 took=201ms waited=241ms",
|
||
"2026-11-05T16:23:23.000Z INFO src.services.processing [rid=-] job delivery skipped job_id=j1 attempt=1 outcome=done",
|
||
"2026-11-05T16:23:24.000Z WARNING src.services.processing [rid=-] attachment processing failed attachment_id=a2 job_id=j2 attempt=1 error_code=image_too_large detail=x fenced=False",
|
||
"2026-11-05T16:23:25.000Z INFO src.services.processing [rid=-] job retry scheduled job_id=j3 attachment_id=a3 attempt=1 reason=s3_read delay=1.2s fenced=False",
|
||
"garbage line",
|
||
]
|
||
)
|
||
log = parse_worker_log(text)
|
||
assert log.consuming == [log_time("2026-11-05T16:23:20.100")]
|
||
assert abs(log.consuming[0] - 1793895800.1) < 1e-3, log.consuming
|
||
assert log.processed["a1"]["waited_ms"] == 241.0
|
||
assert log.processed["a1"]["render_peak_kib"] == 98236
|
||
assert (log.skipped, log.failed, log.retries) == (
|
||
{"j1": 1},
|
||
{"a2": "image_too_large"},
|
||
{"a3": 1},
|
||
)
|
||
gen = parse_gen_log(
|
||
"q,1000.0,1001.0,1500.5,202,,a1,abc,\n"
|
||
"q,2000.0,2001.0,2002.0,429,5,,abc,\n"
|
||
"q,3000.0,,,0,,,,dropped\n"
|
||
)
|
||
assert [u.status for u in gen] == [202, 429, 0]
|
||
assert (gen[0].accepted, gen[0].done, gen[2].sent) == (True, 1.5005, None)
|
||
jobs = parse_jobs(
|
||
"a1|100.0|j1|100.1|ready|101.5|1|ready\n"
|
||
"a2|100.5|j2|100.6|failed|102.0|2|failed\n"
|
||
"a3|101.0|j3|101.1|queued|101.2|1|queued\n"
|
||
"a4|101.0||||||pending\n"
|
||
)
|
||
assert (jobs[0].ttr_ms, jobs[1].ready_at, jobs[2].finished_at) == (
|
||
1500.0,
|
||
None,
|
||
None,
|
||
)
|
||
assert jobs[3].job_created_at is None
|
||
# 100.1 → +1, 100.6 → +2, 101.1 → +3, 101.5 → 2, 102.0 → 1
|
||
assert max_unfinished(jobs, 100.0, 103.0) == 3
|
||
assert max_unfinished(jobs, 101.6, 103.0) == 2
|
||
assert max_unfinished(jobs, 100.0, 100.55) == 1
|
||
sample = parse_sample("1800000000.5|7|1|4|2|3.456\n")
|
||
assert sample == {
|
||
"ts": 1800000000.5,
|
||
"unfinished": 7,
|
||
"outbox": 1,
|
||
"in_broker": 4,
|
||
"processing": 2,
|
||
"oldest_age_s": 3.46,
|
||
}
|
||
lines = [
|
||
{"ts": 1.0, "cg": {"w": [1_000_000, 0, 0, 0, 100 * MIB]}},
|
||
{"ts": 2.0, "cg": {"w": [2_500_000, 0, 0, 0, 120 * MIB]}},
|
||
{"ts": 3.0, "cg": {}},
|
||
{"ts": 4.0, "cg": {"w": [10_000, 0, 0, 0, 50 * MIB]}},
|
||
{"ts": 5.0, "cg": {"w": [5_000, 0, 0, 0, 60 * MIB]}},
|
||
]
|
||
ticks = worker_ticks(lines, "w")
|
||
assert [t.cpu_cores for t in ticks] == [None, 1.5, None, None], ticks
|
||
assert percentile([5, 1, 3, 2, 4], 0.95) == 5
|
||
assert percentile([5, 1, 3, 2, 4], 0.5) == 3
|
||
assert percentile([], 0.5) is None
|
||
|
||
rows: list[dict] = []
|
||
classes = ["s", "l", "x"]
|
||
# пик c4p4 на x выше 80 % лимита 1536 MiB: в рекомендацию он не идёт
|
||
peaks = {"c1p1": 400.0, "c2p2": 700.0, "c4p4": 1300.0, "c2p32": 720.0}
|
||
rates = {"c1p1": 1.0, "c2p2": 1.8, "c4p4": 1.9, "c2p32": 1.85}
|
||
steady_s = {"c1p1": 0.9, "c2p2": 0.5, "c4p4": 0.45, "c2p32": 0.55}
|
||
# c2p32 быстрее c2p2 на 4 % — в пределах 10 %: рекомендация по пику
|
||
fixed_s = {"c1p1": 0.9, "c2p2": 0.5, "c4p4": 0.45, "c2p32": 0.48}
|
||
for item in plan:
|
||
config = item["config"]
|
||
peak = peaks[config] if item["size_class"] == "x" else 300.0
|
||
data = _synthetic_cell(
|
||
item,
|
||
throughput=rates[config],
|
||
peak_mib=peak,
|
||
steady_s=steady_s[config],
|
||
fixed_s=fixed_s[config],
|
||
)
|
||
cell = cell_rows(data)
|
||
assert [r["phase"] for r in cell] == list(PHASES)
|
||
assert all(set(r) <= set(CELL_COLUMNS) for r in cell), set(cell[0]) - set(
|
||
CELL_COLUMNS
|
||
)
|
||
drain, steady, fixed, over, rec = cell
|
||
assert abs(fixed["ttr_p95_ms"] - (fixed_s[config] + 0.05) * 1000) < 0.2
|
||
assert (fixed["jobs"], fixed["ready"], fixed["missing"]) == (6, 6, 0)
|
||
assert abs(drain["throughput"] - rates[config]) < 0.01, drain
|
||
assert (drain["startup_s"], drain["worker_peak_mib"]) == (1.0, peak)
|
||
assert over["max_unfinished"] == 10 + 5, over["max_unfinished"]
|
||
assert (over["backlog_reached"], over["share_429_saturated"]) == (True, 0.5)
|
||
assert (over["share_429"], over["retry_after_5"]) == (0.25, 5)
|
||
assert (rec["recovery_s"], rec["missing"]) == (29.5, 0), rec
|
||
assert abs(steady["ttr_p95_ms"] - (steady_s[config] + 0.05) * 1000) < 0.2
|
||
assert steady["lag_p95_ms"] == 100.0
|
||
assert drain["worker_cpu_cores"] == 1.5
|
||
timeline = timeline_rows(data)
|
||
assert len(timeline) == 9
|
||
assert set(timeline[0]) <= set(TIMELINE_COLUMNS)
|
||
assert sum(r["accepted"] + r["rejected_429"] for r in timeline) > 0
|
||
assert all(set(r) <= set(JOB_COLUMNS) for r in job_rows(data))
|
||
rows += cell
|
||
found = {
|
||
v.hypothesis: v.result
|
||
for v in verdicts(rows, backlog_limit=10, slack=8, classes=classes)
|
||
}
|
||
# c4p4/c2p2 = 1,9/1,8 ≤ 1,15, c2p2/c1p1 = 1,8 ≥ 1,6; пики x 1300/400;
|
||
# c2p32 против c2p2: +3 % пропускной, −4 % p95 фазы fixed
|
||
assert found == {
|
||
"Q1": CONFIRMED,
|
||
"Q2": CONFIRMED,
|
||
"Q3": CONFIRMED,
|
||
"Q4": CONFIRMED,
|
||
"Q5": REFUTED,
|
||
}, found
|
||
# восстановление синтетической ячейки — 30 с на пределе 10 при 1 гот./с:
|
||
# граница 1,3 × 10 / 1 + 5 = 18 с
|
||
assert verdict_q5(rows, backlog_limit=100).result == CONFIRMED
|
||
assert verdict_q4(rows, backlog_limit=5, slack=8).result == REFUTED
|
||
quick = [
|
||
r
|
||
for r in rows
|
||
if r["config"] in ("c1p1", "c4p4") and r["size_class"] in ("s", "x")
|
||
]
|
||
quick_found = {
|
||
v.hypothesis: v.result
|
||
for v in verdicts(quick, backlog_limit=100, slack=8, classes=["s", "x"])
|
||
}
|
||
assert quick_found == {
|
||
"Q1": UNTESTED,
|
||
"Q2": UNTESTED,
|
||
"Q3": CONFIRMED,
|
||
"Q4": CONFIRMED,
|
||
"Q5": CONFIRMED,
|
||
}, quick_found
|
||
text = recommendation(group(rows), list(peaks), classes, 1536)
|
||
# x: c4p4 выше 80 % лимита, c2p32 быстрее c2p2 меньше чем на 10 % —
|
||
# берётся меньший пик; s: пики равны — меньший p95
|
||
assert "x — c2p2" in text, text
|
||
assert "s — c4p4" in text, text
|
||
assert overload_window(40, 60, 1.0, fill=1.5, cap=150) == 90
|
||
assert overload_window(40, 60, 10.0, fill=1.5, cap=150) == 40
|
||
assert overload_window(40, 60, 0.3, fill=1.5, cap=150) == 150
|
||
meta = {
|
||
"label": "selftest",
|
||
"profile": {
|
||
"name": "queue",
|
||
"configs": configs,
|
||
"classes": (
|
||
("s", "noise", 320, 200, 100),
|
||
("l", "noise", 1800, 1800, 40),
|
||
("x", "flat", 6000, 6000, 20),
|
||
),
|
||
"repeats": 2,
|
||
"window_s": 40.0,
|
||
"steady_factor": 0.6,
|
||
"fixed_rates": (("s", 2.5), ("l", 1.0), ("x", 0.5)),
|
||
"overload_factor": 2.0,
|
||
"overload_fill": 1.5,
|
||
"overload_max_s": 150.0,
|
||
"backlog_limit": 10,
|
||
"api_replicas": 2,
|
||
"api_cpus": 1.0,
|
||
"api_memory": "512m",
|
||
"worker_cpus": 2.0,
|
||
"worker_memory": "1536m",
|
||
"render_memory_mb": 768,
|
||
"mq_cpus": 1.0,
|
||
"mq_memory": "512m",
|
||
"s3_cpus": 1.0,
|
||
"s3_memory": "512m",
|
||
},
|
||
"upload_slack": 8,
|
||
"bodies": {
|
||
"s": {
|
||
"kind": "noise",
|
||
"width": 320,
|
||
"height": 200,
|
||
"bytes": 1,
|
||
"sha256": "0" * 64,
|
||
}
|
||
},
|
||
}
|
||
summary = render_summary(meta, rows)
|
||
for part in ("## Класс `x`", "| Q5 |", "рекомендация"):
|
||
assert part in summary, part
|
||
print("queue selftest: ok")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if sys.argv[1:] == ["--selftest"]:
|
||
selftest()
|
||
else:
|
||
raise SystemExit("usage: uv run python bench/queue.py --selftest")
|