"""Профиль queue: план, разбор журналов, строки и вердикты без docker. Только stdlib. Трек 1 лабы 5 «Очередь под перегрузкой». `run.py` (профили `queue`, `queue-quick`) гоняет ячейки «настройка worker × класс изображения»; на ячейку — пять фаз: `drain` (ёмкость: N тел при остановленном worker), `steady` (0,6 ёмкости ячейки), `fixed` (одна интенсивность на класс для всех настроек: задержки сравнимы при равной нагрузке), `overload` (2 ёмкости, окно дотягивается до предела незавершённых), `recovery` (опустошение после перегрузки и снова 0,6 ёмкости). Здесь — чистые функции: план ячеек кругами, разбор журнала worker, журнала генератора и строк SQL, строки `cells.csv`, `timeline.csv`, `jobs.csv.gz`, вердикты Q1–Q5 с порогами-константами (гипотезы записываются в `REPORT.md` до замеров), рекомендация и `summary.md`. Самопроверка — `uv run python bench/queue.py --selftest`. Имя совпадает с модулем stdlib `queue`: в процессах bench/ каталог скрипта стоит первым в `sys.path`, и `import queue` находит этот файл. Модули стенда stdlib `queue` не используют; импорт, которому он нужен (`concurrent.futures`, `multiprocessing`, `logging.handlers`), в `run.py` не добавлять. """ import bisect import math import random import re import statistics import sys from collections.abc import Iterable from dataclasses import dataclass, field from datetime import UTC, datetime MIB = 2**20 PHASES = ("drain", "steady", "fixed", "overload", "recovery") PHASE_NAMES = { "drain": "ёмкость", "steady": "ровная", "fixed": "общая интенсивность", "overload": "перегрузка", "recovery": "восстановление", } # Retry-After у 429 «незавершённых заданий не меньше предела» и «места # загрузки заняты» (src/constants.py, копия: стенд код приложения не # импортирует) BACKLOG_RETRY_AFTER = "5" SLOTS_RETRY_AFTER = "1" # пороги гипотез — до замеров (REPORT.md, «Очередь: гипотезы») Q1_CLASSES = ("l", "x") Q1_SCALE = 1.6 Q1_FLAT = 1.15 Q2_THROUGHPUT = 0.10 Q2_P95 = 0.20 Q3_BIG_CLASS = "x" Q3_BIG_RATIO = 2.5 Q3_SMALL_CLASS = "s" Q3_SMALL_RATIO = 1.5 Q4_SHARE = 0.30 Q5_DRAIN_FACTOR = 1.3 Q5_DRAIN_EXTRA_S = 5.0 Q5_P95_FACTOR = 1.3 Q5_P95_EXTRA_MS = 500.0 # рекомендация — только среди настроек с пиком worker не выше этой доли # лимита памяти контейнера; p95 фазы fixed в пределах этой доли от # лучшего — равны, и выбирается меньший пик RECOMMEND_MEMORY_SHARE = 0.8 RECOMMEND_P95_TIE = 0.10 def overload_window( window_s: float, backlog_limit: int, throughput: float, *, fill: float, cap: float ) -> float: """Окно перегрузки: не короче окна профиля и достаточно долгое, чтобы при приходе 2 × T очередь (растёт на T в секунду) дошла до предела с запасом `fill`; не длиннее `cap`.""" return min(cap, max(window_s, fill * backlog_limit / throughput)) CELL_COLUMNS = ( "cell", "config", "concurrency", "prefetch", "size_class", "body_bytes", "repeat", "order", "phase", "rate", "window_s", "scheduled", "sent", "accepted", "rejected_429", "retry_after_5", "retry_after_1", "errors_5xx", "other_errors", "dropped", "share_429", "share_429_saturated", "backlog_reached", "throughput", "drain_s", "startup_s", "ttr_p50_ms", "ttr_p95_ms", "ttr_p99_ms", "ttr_max_ms", "lag_p50_ms", "lag_p95_ms", "lag_max_ms", "took_p50_ms", "took_p95_ms", "max_unfinished", "max_unfinished_sampled", "max_outbox", "max_in_broker", "max_oldest_age_s", "recovery_s", "jobs", "ready", "failed", "missing", "unknown_jobs", "retries", "skipped", "worker_cpu_cores", "worker_mem_max_mib", "worker_peak_mib", "render_peak_mib", "upload_p95_ms", "flags", ) TIMELINE_COLUMNS = ( "cell", "config", "size_class", "repeat", "phase", "ts", "t_s", "unfinished", "outbox", "in_broker", "processing", "oldest_age_s", "worker_cpu_cores", "worker_mem_mib", "accepted", "rejected_429", "errors", ) JOB_COLUMNS = ( "cell", "phase", "attachment_id", "created_at", "ready_at", "attempt", "waited_ms", "took_ms", "state", "job_created_at", "finished_at", "render_peak_kib", ) # состояние заданий одним запросом: часы сервера (ВМ Docker), счётчики по # частичному индексу незавершённых TIMELINE_SQL = """ SELECT extract(epoch FROM clock_timestamp()), count(*), count(*) FILTER (WHERE state = 'queued' AND published_at IS NULL), count(*) FILTER (WHERE state = 'queued' AND published_at IS NOT NULL), count(*) FILTER (WHERE state = 'processing'), coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0) FROM attachment_jobs WHERE state IN ('queued', 'processing'); """ NOW_SQL = "SELECT extract(epoch FROM clock_timestamp());\n" def jobs_sql(since: float) -> str: """Вложения ячейки с их заданием: всё принятое с начала ячейки. LEFT JOIN — принятая строка без задания тоже видна (тогда она «missing»).""" return ( "SELECT a.id, extract(epoch FROM a.created_at), coalesce(j.id::text, ''), " "coalesce(extract(epoch FROM j.created_at)::text, ''), " "coalesce(j.state, ''), coalesce(extract(epoch FROM j.updated_at)::text, ''), " "coalesce(j.attempt::text, ''), a.status " "FROM attachments a LEFT JOIN attachment_jobs j ON j.attachment_id = a.id " f"WHERE a.created_at >= to_timestamp({since:.6f}) " "ORDER BY a.created_at, a.id;\n" ) # --- план -------------------------------------------------------------------------- def plan_cells( configs: Iterable[tuple[str, int, int]], classes: Iterable[str], repeats: int, seed: int, ) -> list[dict]: """Ячейки кругами: в круге r — перестановка (настройка × класс) `Random(seed + r)`; круг не начинается с ячейки, которой кончился прежний (одна и та же настройка подряд не попадает на границу).""" items = [(c, k) for c in configs for k in classes] plan: list[dict] = [] previous = None for round_no in range(1, repeats + 1): order = list(range(len(items))) random.Random(seed + round_no).shuffle(order) if len(order) > 1 and order[0] == previous: order[0], order[1] = order[1], order[0] for index in order: (name, concurrency, prefetch), size_class = items[index] plan.append( { "cell": f"{name}-{size_class}-r{round_no}", "config": name, "concurrency": concurrency, "prefetch": prefetch, "size_class": size_class, "repeat": round_no, "order": len(plan) + 1, }, ) previous = order[-1] return plan # --- разбор ------------------------------------------------------------------------ LOG_LINE = re.compile( r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3})Z \S+\s+\S+ \[rid=\S*\] (.*)$" ) FIELD = re.compile(r"(\w+)=(\S+)") def log_time(stamp: str) -> float: """`2026-11-05T16:23:22.870` (UTC, src/log.py) → секунды эпохи.""" moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%S.%f") return moment.replace(tzinfo=UTC).timestamp() def _ms(value: str) -> float | None: try: return float(value.removesuffix("ms")) except ValueError: return None @dataclass class WorkerLog: """Журнал worker одной ячейки: старты потребителя и итоги доставок.""" consuming: list[float] = field(default_factory=list) # attachment_id → ts, job_id, attempt, render_peak_kib, took_ms, waited_ms processed: dict[str, dict] = field(default_factory=dict) # job_id → число пропущенных доставок (дубль, чужая попытка) skipped: dict[str, int] = field(default_factory=dict) # attachment_id → error_code failed: dict[str, str] = field(default_factory=dict) # attachment_id → число назначенных повторов retries: dict[str, int] = field(default_factory=dict) def parse_worker_log(text: str) -> WorkerLog: out = WorkerLog() for line in text.splitlines(): match = LOG_LINE.match(line) if not match: continue ts, message = log_time(match.group(1)), match.group(2) fields = dict(FIELD.findall(message)) if message.startswith("broker consuming"): out.consuming.append(ts) elif message.startswith("attachment processed"): out.processed[fields.get("attachment_id", "")] = { "ts": ts, "job_id": fields.get("job_id", ""), "attempt": int(fields.get("attempt", "0") or 0), "render_peak_kib": int(fields.get("render_peak_kib", "0") or 0), "took_ms": _ms(fields.get("took", "")), "waited_ms": _ms(fields.get("waited", "")), } elif message.startswith("job delivery skipped"): job = fields.get("job_id", "") out.skipped[job] = out.skipped.get(job, 0) + 1 elif message.startswith("attachment processing failed"): out.failed[fields.get("attachment_id", "")] = fields.get("error_code", "") elif message.startswith("job retry scheduled"): key = fields.get("attachment_id", "") out.retries[key] = out.retries.get(key, 0) + 1 return out @dataclass(frozen=True) class Upload: """Строка журнала генератора `queue` (loadgen.py); времена — секунды часов ВМ, `None` — загрузка не ушла.""" scheduled: float sent: float | None done: float | None status: int retry_after: str attachment_id: str error: str @property def accepted(self) -> bool: return self.status in (201, 202) def _sec(value: str) -> float | None: return float(value) / 1000 if value else None def parse_gen_log(text: str) -> list[Upload]: uploads = [] for line in text.splitlines(): parts = line.split(",") if len(parts) != 9 or parts[0] != "q": continue uploads.append( Upload( scheduled=float(parts[1]) / 1000, sent=_sec(parts[2]), done=_sec(parts[3]), status=int(parts[4] or 0), retry_after=parts[5], attachment_id=parts[6], error=parts[8], ) ) return uploads @dataclass class Job: """Вложение ячейки и его задание (`jobs_sql`); времена — секунды эпохи часов сервера. `finished_at` — `updated_at` задания в `ready`/`failed`.""" attachment_id: str created_at: float job_id: str job_created_at: float | None state: str finished_at: float | None attempt: int | None status: str phase: str = "" @property def ready_at(self) -> float | None: return self.finished_at if self.state == "ready" else None @property def ttr_ms(self) -> float | None: ready = self.ready_at return None if ready is None else (ready - self.created_at) * 1000 def _opt(value: str) -> float | None: return float(value) if value else None def parse_jobs(text: str) -> list[Job]: jobs = [] for line in text.splitlines(): parts = line.split("|") if len(parts) != 8: continue state = parts[4] jobs.append( Job( attachment_id=parts[0], created_at=float(parts[1]), job_id=parts[2], job_created_at=_opt(parts[3]), state=state, finished_at=_opt(parts[5]) if state in ("ready", "failed") else None, attempt=int(parts[6]) if parts[6] else None, status=parts[7], ) ) return jobs def parse_sample(text: str) -> dict | None: """Строка `TIMELINE_SQL` → выборка шкалы.""" parts = text.strip().split("|") if len(parts) != 6: return None return { "ts": float(parts[0]), "unfinished": int(parts[1]), "outbox": int(parts[2]), "in_broker": int(parts[3]), "processing": int(parts[4]), "oldest_age_s": round(float(parts[5]), 2), } @dataclass(frozen=True) class Tick: """Одна строка `scan` по worker: CPU с прошлой строки и текущая память cgroup.""" ts: float cpu_cores: float | None memory: int def worker_ticks(lines: list[dict], container: str) -> list[Tick]: """Строки сэмплера (bench/sampler.py) → выборки контейнера `container` (id12). После перезапуска контейнера счётчик CPU начинается заново: интервал через перезапуск CPU не получает.""" ticks: list[Tick] = [] previous = None for line in lines: values = line.get("cg", {}).get(container) if values is None: previous = None continue cpu = None if previous is not None and values[0] >= previous[1][0]: seconds = line["ts"] - previous[0] if seconds > 0: cpu = round((values[0] - previous[1][0]) / seconds / 1e6, 3) ticks.append(Tick(ts=line["ts"], cpu_cores=cpu, memory=values[4])) previous = (line["ts"], values) return ticks # --- метрики ----------------------------------------------------------------------- def percentile(values: Iterable[float], fraction: float) -> float | None: """Ближайший ранг, как у генератора (loadgen.percentile).""" ordered = sorted(values) if not ordered: return None rank = max(1, math.ceil(fraction * len(ordered))) return ordered[min(rank, len(ordered)) - 1] def _r(value: float | None, digits: int = 1) -> float | None: return None if value is None else round(value, digits) def max_unfinished(jobs: Iterable[Job], start: float, end: float) -> int: """Наибольшее число незавершённых заданий в [start, end] по событиям: +1 при создании задания, −1 при его ready/failed. Точнее выборок раз в секунду: пик между ними тоже виден.""" events: list[tuple[float, int]] = [] level = 0 for job in jobs: if job.job_created_at is None: continue finish = job.finished_at if job.job_created_at <= start: if finish is None or finish > start: level += 1 elif job.job_created_at <= end: events.append((job.job_created_at, 1)) if finish is not None and start < finish <= end and job.job_created_at <= end: events.append((finish, -1)) peak = level # при равном времени сначала уход, потом приход: пик не завышается for _, step in sorted(events, key=lambda e: (e[0], e[1])): level += step peak = max(peak, level) return peak def drain_throughput(jobs: Iterable[Job], consuming_ts: float | None) -> float | None: """Ready/с фазы ёмкости: готовых заданий на секунды от начала работы потребителя (`broker consuming`) до последнего ready. Старт процесса worker сюда не входит — он в `startup_s`.""" ready = [j.ready_at for j in jobs if j.ready_at is not None] if consuming_ts is None or not ready: return None span = max(ready) - consuming_ts return len(ready) / span if span > 0 else None def assign_phases(jobs: list[Job], marks: dict[str, float]) -> None: """Фаза вложения — по `created_at` между отметками начала фаз.""" bounds = sorted((marks[p], p) for p in PHASES if p in marks) starts = [b[0] for b in bounds] for job in jobs: index = bisect.bisect_right(starts, job.created_at) - 1 job.phase = bounds[index][1] if index >= 0 else "" def window_usage(ticks: list[Tick], start: float, end: float) -> dict: inside = [t for t in ticks if start < t.ts <= end] cpu = [t.cpu_cores for t in inside if t.cpu_cores is not None] return { "worker_cpu_cores": _r(statistics.fmean(cpu), 3) if cpu else None, "worker_mem_max_mib": _r(max(t.memory for t in inside) / MIB) if inside else None, } @dataclass class CellData: """Всё, что `run.py` собрал за ячейку (часы ВМ Docker).""" item: dict body_bytes: int # начало фаз и прочие отметки: drain, worker_start, steady, overload, # recovery, end marks: dict[str, float] consuming_ts: float | None # фаза → итог генератора и его строки gens: dict[str, dict] uploads: dict[str, list[Upload]] jobs: list[Job] log: WorkerLog ticks: list[Tick] samples: list[dict] peak_bytes: int | None backlog_limit: int def _phase_end(marks: dict[str, float], phase: str) -> float: later = [marks[p] for p in PHASES[PHASES.index(phase) + 1 :] if p in marks] return later[0] if later else marks["end"] def _gen_fields(summary: dict, uploads: list[Upload]) -> dict: sent = summary.get("sent") or 0 rejected = summary.get("rejected_429") or 0 pauses = summary.get("retry_after") or {} # доля 429 после предела: от первого отказа «работы слишком много» до # конца окна — тогда backpressure уже включён first = min( ( u.scheduled for u in uploads if u.status == 429 and u.retry_after == BACKLOG_RETRY_AFTER ), default=None, ) saturated = None if first is not None: after = [u for u in uploads if u.scheduled >= first and u.sent is not None] if after: saturated = round(sum(u.status == 429 for u in after) / len(after), 4) return { "rate": _r(summary.get("rate"), 3), "window_s": summary.get("duration_s"), "scheduled": summary.get("scheduled"), "sent": sent, "accepted": summary.get("accepted"), "rejected_429": rejected, "retry_after_5": pauses.get(BACKLOG_RETRY_AFTER, 0), "retry_after_1": pauses.get(SLOTS_RETRY_AFTER, 0), "errors_5xx": summary.get("errors_5xx"), "other_errors": summary.get("other_errors"), "dropped": summary.get("dropped"), "share_429": round(rejected / sent, 4) if sent else None, "share_429_saturated": saturated, "backlog_reached": first is not None, "upload_p95_ms": summary.get("upload_p95_ms"), } def cell_rows(data: CellData) -> list[dict]: """Строки `cells.csv` ячейки — по одной на фазу.""" item, marks = data.item, data.marks assign_phases(data.jobs, marks) by_id = {job.attachment_id: job for job in data.jobs} log = data.log peak_mib = _r(data.peak_bytes / MIB) if data.peak_bytes else None rows = [] for phase in PHASES: if phase not in marks: continue start, end = marks[phase], _phase_end(marks, phase) jobs = [j for j in data.jobs if j.phase == phase] uploads = data.uploads.get(phase, []) accepted_ids = {u.attachment_id for u in uploads if u.accepted} processed = [ log.processed[j.attachment_id] for j in jobs if j.attachment_id in log.processed ] waited = [p["waited_ms"] for p in processed if p["waited_ms"] is not None] took = [p["took_ms"] for p in processed if p["took_ms"] is not None] ttr = [j.ttr_ms for j in jobs if j.ttr_ms is not None] samples = [s for s in data.samples if s.get("phase") == phase] flags = [] row = { **{k: item[k] for k in ("cell", "config", "concurrency", "prefetch")}, "size_class": item["size_class"], "body_bytes": data.body_bytes, "repeat": item["repeat"], "order": item["order"], "phase": phase, **_gen_fields(data.gens.get(phase, {}), uploads), "ttr_p50_ms": _r(percentile(ttr, 0.50)), "ttr_p95_ms": _r(percentile(ttr, 0.95)), "ttr_p99_ms": _r(percentile(ttr, 0.99)), "ttr_max_ms": _r(max(ttr)) if ttr else None, "lag_p50_ms": _r(percentile(waited, 0.50)), "lag_p95_ms": _r(percentile(waited, 0.95)), "lag_max_ms": _r(max(waited)) if waited else None, "took_p50_ms": _r(percentile(took, 0.50)), "took_p95_ms": _r(percentile(took, 0.95)), "max_unfinished_sampled": max( (s["unfinished"] for s in samples), default=None ), "max_outbox": max((s["outbox"] for s in samples), default=None), "max_in_broker": max((s["in_broker"] for s in samples), default=None), "max_oldest_age_s": max((s["oldest_age_s"] for s in samples), default=None), "jobs": len(jobs), "ready": sum(j.state == "ready" for j in jobs), "failed": sum( by_id[i].state == "failed" for i in accepted_ids if i in by_id ), # принято (201/202), но не ready и не failed к концу ячейки "missing": sum( 1 for i in accepted_ids if i not in by_id or by_id[i].state not in ("ready", "failed") ), # задание есть, а 202 генератор не увидел (срок ответа, обрыв) "unknown_jobs": sum(j.attachment_id not in accepted_ids for j in jobs), "retries": sum(log.retries.get(j.attachment_id, 0) for j in jobs), "skipped": sum(log.skipped.get(j.job_id, 0) for j in jobs), "worker_peak_mib": peak_mib, "render_peak_mib": _r(max(p["render_peak_kib"] for p in processed) / 1024) if processed else None, } usage_start = marks.get("worker_start", start) if phase == "drain" else start row.update(window_usage(data.ticks, usage_start, end)) if phase == "drain": throughput = drain_throughput(jobs, data.consuming_ts) if throughput: row["throughput"] = _r(throughput, 3) row["drain_s"] = _r(row["ready"] / throughput, 3) if data.consuming_ts is not None and "worker_start" in marks: row["startup_s"] = _r(data.consuming_ts - marks["worker_start"], 3) if (row["accepted"] or 0) < (row["scheduled"] or 0): flags.append("drain_rejected") if phase == "overload": stop = marks.get("recovery", end) row["max_unfinished"] = max_unfinished(data.jobs, start, stop) if phase == "recovery": overload_end = data.gens.get("overload", {}).get("end") earlier = [ j.finished_at for j in data.jobs if j.phase != "recovery" and j.finished_at is not None ] if overload_end is not None and earlier: row["recovery_s"] = _r(max(0.0, max(earlier) - overload_end), 3) if row["failed"]: flags.append("failed") if row["missing"]: flags.append("missing") if row["errors_5xx"]: flags.append("5xx") if row["dropped"]: flags.append("gen_dropped") row["flags"] = " ".join(flags) rows.append(row) return rows def timeline_rows(data: CellData) -> list[dict]: """Шкала ячейки раз в секунду: выборки SQL, ближайшая выборка cgroup worker и ответы генератора с прошлой выборки.""" item = data.item done = sorted( (u.done, u.status) for phase in data.uploads.values() for u in phase if u.done is not None ) done_ts = [d[0] for d in done] tick_ts = [t.ts for t in data.ticks] rows = [] previous = None origin = data.marks.get("drain", data.samples[0]["ts"] if data.samples else 0.0) for sample in data.samples: ts = sample["ts"] lo = bisect.bisect_right(done_ts, previous) if previous is not None else 0 hi = bisect.bisect_right(done_ts, ts) window = [status for _, status in done[lo:hi]] index = bisect.bisect_right(tick_ts, ts) - 1 tick = data.ticks[index] if index >= 0 and ts - tick_ts[index] <= 2.5 else None rows.append( { "cell": item["cell"], "config": item["config"], "size_class": item["size_class"], "repeat": item["repeat"], "phase": sample.get("phase", ""), "ts": round(ts, 3), "t_s": round(ts - origin, 2), **{ k: sample[k] for k in ( "unfinished", "outbox", "in_broker", "processing", "oldest_age_s", ) }, "worker_cpu_cores": tick.cpu_cores if tick else None, "worker_mem_mib": _r(tick.memory / MIB) if tick else None, "accepted": sum(s in (201, 202) for s in window), "rejected_429": sum(s == 429 for s in window), "errors": sum(s not in (201, 202, 429) for s in window), } ) previous = ts return rows def job_rows(data: CellData) -> list[dict]: rows = [] for job in data.jobs: processed = data.log.processed.get(job.attachment_id, {}) rows.append( { "cell": data.item["cell"], "phase": job.phase, "attachment_id": job.attachment_id, "created_at": round(job.created_at, 6), "ready_at": _r(job.ready_at, 6), "attempt": job.attempt, "waited_ms": processed.get("waited_ms"), "took_ms": processed.get("took_ms"), "state": job.state or job.status, "job_created_at": _r(job.job_created_at, 6), "finished_at": _r(job.finished_at, 6), "render_peak_kib": processed.get("render_peak_kib"), } ) return rows # --- вердикты ---------------------------------------------------------------------- def num(value) -> float | None: if value is None or value == "": return None if isinstance(value, bool): return float(value) try: return float(value) except (TypeError, ValueError): return None def median(values: Iterable) -> float | None: present = [v for v in (num(x) for x in values) if v is not None] return statistics.median(present) if present else None def spread(values: Iterable, digits: int = 1, scale: float = 1.0) -> str: present = [v / scale for v in (num(x) for x in values) if v is not None] if not present: return "—" mid = statistics.median(present) if len(present) == 1: return f"{mid:.{digits}f}" return f"{mid:.{digits}f} ({min(present):.{digits}f}–{max(present):.{digits}f})" def fmt(value: float | None, digits: int = 2) -> str: return "—" if value is None else f"{value:.{digits}f}" def group(rows: list[dict]) -> dict[tuple[str, str, str], list[dict]]: out: dict[tuple[str, str, str], list[dict]] = {} for row in rows: key = (row["config"], row["size_class"], row["phase"]) out.setdefault(key, []).append(row) return out def metric( grouped: dict, config: str, size_class: str, phase: str, name: str ) -> float | None: return median(r.get(name) for r in grouped.get((config, size_class, phase), [])) @dataclass(frozen=True) class Verdict: hypothesis: str result: str numbers: str CONFIRMED = "подтверждена" REFUTED = "опровергнута" UNTESTED = "нет данных" def _result(checked: int, failed: bool, missing: bool) -> str: """Опровергнута, если не прошла хоть одна проверенная часть; нет данных, если части не хватает и опровергать нечем.""" if failed: return REFUTED if missing or not checked: return UNTESTED return CONFIRMED def verdict_q1(grouped: dict) -> Verdict: """Q1: на `l` и `x` c2p2 ≥ 1,6 × c1p1, а c4p4 ≤ 1,15 × c2p2 (у worker 2 CPU: больше двух рендеров сразу не ускоряют).""" parts, checked, failed, missing = [], 0, False, False for size_class in Q1_CLASSES: t1, t2, t4 = ( metric(grouped, c, size_class, "drain", "throughput") for c in ("c1p1", "c2p2", "c4p4") ) if not t1 or not t2 or t4 is None: missing = True parts.append(f"{size_class}: нет ячеек c1p1, c2p2 и c4p4") continue checked += 1 up, flat = t2 / t1, t4 / t2 failed |= not (up >= Q1_SCALE and flat <= Q1_FLAT) parts.append( f"{size_class}: c2p2/c1p1 = {t2:.3f}/{t1:.3f} = {up:.2f} (≥ {Q1_SCALE}), " f"c4p4/c2p2 = {t4:.3f}/{t2:.3f} = {flat:.2f} (≤ {Q1_FLAT}) гот./с" ) return Verdict("Q1", _result(checked, failed, missing), "; ".join(parts)) def verdict_q2(grouped: dict, classes: list[str]) -> Verdict: """Q2: prefetch 32 против 2 при concurrency 2 — пропускная в ±10 %, p95 фазы fixed (одна интенсивность для обеих) в ±20 % на каждом классе.""" parts, checked, failed, missing = [], 0, False, False for size_class in classes: t2 = metric(grouped, "c2p2", size_class, "drain", "throughput") t32 = metric(grouped, "c2p32", size_class, "drain", "throughput") p2 = metric(grouped, "c2p2", size_class, "fixed", "ttr_p95_ms") p32 = metric(grouped, "c2p32", size_class, "fixed", "ttr_p95_ms") if not t2 or t32 is None or not p2 or p32 is None: missing = True parts.append(f"{size_class}: нет ячеек c2p2 и c2p32") continue checked += 1 dt, dp = t32 / t2 - 1, p32 / p2 - 1 failed |= not (abs(dt) <= Q2_THROUGHPUT and abs(dp) <= Q2_P95) parts.append( f"{size_class}: пропускная {t32:.3f} к {t2:.3f} ({dt:+.0%}, допуск ±{Q2_THROUGHPUT:.0%}), " f"p95 {p32 / 1000:.2f} к {p2 / 1000:.2f} с ({dp:+.0%}, допуск ±{Q2_P95:.0%})" ) return Verdict("Q2", _result(checked, failed, missing), "; ".join(parts)) def verdict_q3(grouped: dict) -> Verdict: """Q3: память растёт с concurrency, когда дорого декодирование: на `x` пик worker c4p4 ≥ 2,5 × c1p1, на `s` — ≤ 1,5 × c1p1.""" parts, checked, failed, missing = [], 0, False, False for size_class, ratio, bigger in ( (Q3_BIG_CLASS, Q3_BIG_RATIO, True), (Q3_SMALL_CLASS, Q3_SMALL_RATIO, False), ): m1 = metric(grouped, "c1p1", size_class, "drain", "worker_peak_mib") m4 = metric(grouped, "c4p4", size_class, "drain", "worker_peak_mib") if not m1 or m4 is None: missing = True parts.append(f"{size_class}: нет пиков c1p1 и c4p4") continue checked += 1 value = m4 / m1 ok = value >= ratio if bigger else value <= ratio failed |= not ok sign = "≥" if bigger else "≤" parts.append( f"{size_class}: пик c4p4/c1p1 = {m4:.0f}/{m1:.0f} MiB = {value:.2f} ({sign} {ratio})" ) return Verdict("Q3", _result(checked, failed, missing), "; ".join(parts)) def verdict_q4(rows: list[dict], *, backlog_limit: int, slack: int) -> Verdict: """Q4: в перегрузке незавершённых не больше предела + мест загрузок стенда, после достижения предела 429 ≥ 30 % загрузок, 5xx нет. Доля 429 — с первого отказа «работы слишком много» до конца окна: пока очередь не дошла до предела, отказывать не в чем. Ячейки, где предел не достигнут, долю не проверяют (названы в числах).""" over = [r for r in rows if r["phase"] == "overload"] if not over: return Verdict("Q4", UNTESTED, "нет фаз перегрузки") bound = backlog_limit + slack peaks = [num(r.get("max_unfinished")) for r in over] peak = max((p for p in peaks if p is not None), default=None) fives = sum(int(num(r.get("errors_5xx")) or 0) for r in over) reached = [ r for r in over if str(r.get("backlog_reached")) in ("True", "true", "1") ] shares = [num(r.get("share_429_saturated")) for r in reached] shares = [s for s in shares if s is not None] whole = [num(r.get("share_429")) for r in over] whole = [w for w in whole if w is not None] not_reached = sorted({r["cell"] for r in over if r not in reached}) failed = (peak is not None and peak > bound) or fives > 0 failed |= any(s < Q4_SHARE for s in shares) missing = peak is None or not shares numbers = ( f"макс. незавершённых {fmt(peak, 0)} (граница {backlog_limit} + {slack} = {bound}); " f"429 после предела: min {fmt(min(shares) * 100 if shares else None, 0)} % " f"(порог {Q4_SHARE:.0%}) в {len(shares)} ячейках; за всё окно " + (f"{min(whole):.0%}–{max(whole):.0%}" if whole else "—") + f"; 5xx {fives}" + (f"; предел не достигнут: {', '.join(not_reached)}" if not_reached else "") ) return Verdict("Q4", _result(len(over), failed, missing), numbers) def verdict_q5(rows: list[dict], *, backlog_limit: int) -> Verdict: """Q5: после перегрузки очередь пустеет за ≤ 1,3 × предел / пропускная + 5 с, p95 ровной фазы после — ≤ 1,3 × p95 до + 0,5 с, и каждая принятая загрузка B–D стала ready. Проверяется каждая ячейка × повтор.""" cells: dict[str, dict[str, dict]] = {} for row in rows: cells.setdefault(row["cell"], {})[row["phase"]] = row parts, checked, failed, missing = [], 0, False, False lost = 0 worst_drain = worst_p95 = None for name, phases in sorted(cells.items()): drain, steady, recovery = ( phases.get(p) for p in ("drain", "steady", "recovery") ) if not drain or not steady or not recovery: missing = True continue throughput = num(drain.get("throughput")) recovery_s = num(recovery.get("recovery_s")) before, after = num(steady.get("ttr_p95_ms")), num(recovery.get("ttr_p95_ms")) if not throughput or recovery_s is None or before is None or after is None: missing = True parts.append(f"{name}: нет чисел") continue checked += 1 drain_bound = Q5_DRAIN_FACTOR * backlog_limit / throughput + Q5_DRAIN_EXTRA_S p95_bound = Q5_P95_FACTOR * before + Q5_P95_EXTRA_MS bad = sum( int(num(phases[p].get(k)) or 0) for p in ("steady", "fixed", "overload", "recovery") if p in phases for k in ("failed", "missing") ) lost += bad drain_share, p95_share = recovery_s / drain_bound, after / p95_bound worst_drain = max(worst_drain or 0.0, drain_share) worst_p95 = max(worst_p95 or 0.0, p95_share) if recovery_s > drain_bound or after > p95_bound or bad: failed = True parts.append( f"{name}: опустошение {recovery_s:.1f} с (граница {drain_bound:.1f}), " f"p95 {after / 1000:.2f} с (граница {p95_bound / 1000:.2f}), потеряно {bad}" ) head = ( f"{checked} ячеек; худшее опустошение — {fmt(worst_drain)} границы, " f"худший p95 после — {fmt(worst_p95)} границы; failed + missing {lost}" ) return Verdict("Q5", _result(checked, failed, missing), "; ".join([head, *parts])) def recommendation( grouped: dict, configs: list[str], classes: list[str], worker_limit_mib: float ) -> str: """Класс → настройка с наименьшим p95 фазы fixed (одна интенсивность для всех настроек класса) среди тех, чей пик worker не выше доли лимита памяти; p95 в пределах `RECOMMEND_P95_TIE` от лучшего считаются равными, и из них берётся меньший пик.""" cap = worker_limit_mib * RECOMMEND_MEMORY_SHARE parts = [] for size_class in classes: found = [] for config in configs: p95 = metric(grouped, config, size_class, "fixed", "ttr_p95_ms") peak = metric(grouped, config, size_class, "drain", "worker_peak_mib") if p95 is None or peak is None or peak > cap: continue found.append((config, p95, peak)) if not found: parts.append(f"{size_class} — нет настройки с пиком ≤ {cap:.0f} MiB") continue fastest = min(p for _, p, _ in found) near = [f for f in found if f[1] <= fastest * (1 + RECOMMEND_P95_TIE)] config, p95, peak = min(near, key=lambda f: (f[2], f[1])) throughput = metric(grouped, config, size_class, "drain", "throughput") others = ", ".join( f"{c} {p / 1000:.2f} с" for c, p, _ in sorted(found, key=lambda f: f[1]) ) parts.append( f"{size_class} — {config} (p95 fixed {p95 / 1000:.2f} с, пропускная " f"{fmt(throughput, 3)} гот./с, пик {peak:.0f} MiB; p95 fixed: {others})" ) return ( f"рекомендация (наименьший p95 фазы fixed при пике worker ≤ " f"{RECOMMEND_MEMORY_SHARE:.0%} лимита, {cap:.0f} MiB; p95 в пределах " f"{RECOMMEND_P95_TIE:.0%} — меньший пик): " + "; ".join(parts) ) def verdicts( rows: list[dict], *, backlog_limit: int, slack: int, classes: list[str] ) -> list[Verdict]: grouped = group(rows) return [ verdict_q1(grouped), verdict_q2(grouped, classes), verdict_q3(grouped), verdict_q4(rows, backlog_limit=backlog_limit, slack=slack), verdict_q5(rows, backlog_limit=backlog_limit), ] # --- сводка ------------------------------------------------------------------------ def _flags(items: list[dict]) -> str: found = sorted({f for r in items for f in str(r.get("flags") or "").split()}) return " ".join(found) or "—" def render_summary(meta: dict, rows: list[dict]) -> str: profile = meta["profile"] limit = profile["backlog_limit"] slack = meta.get("upload_slack", 0) configs = [c[0] for c in profile["configs"]] classes = [c[0] for c in profile["classes"]] worker_mib = float(profile["worker_memory"].rstrip("m")) status = "прерван: " + meta["aborted"] if meta.get("aborted") else "готово" bodies = meta.get("bodies", {}) lines = [ f"# Очередь под перегрузкой: метка `{meta['label']}`", "", f"- Статус: **{status}**", f"- Код (`src_sha256` образа): `{meta.get('code', {}).get('src_sha256', '—')}`", f"- Профиль `{profile['name']}`: настройки " + ", ".join( f"{n} (concurrency {c}, prefetch {p})" for n, c, p in profile["configs"] ) + f"; повторов {profile['repeats']}; окно {profile['window_s']:g} с, " f"ровная {profile['steady_factor']:g} ёмкости ячейки, общая интенсивность " "класса (fixed) " + ", ".join(f"{k} {v:g}/с" for k, v in profile["fixed_rates"]) + f", перегрузка {profile['overload_factor']:g} ёмкости в окне " f"min({profile['overload_max_s']:g} с, max(окно, " f"{profile['overload_fill']:g} × предел / ёмкость)), после перегрузки — " f"ровная {profile['window_s'] / 2:g} с; предел незавершённых " f"`UPLOAD_MAX_UNFINISHED={limit}`", "- Тела: " + ", ".join( f"{name} — {item['kind']} {item['width']} × {item['height']}, " f"{item['bytes']} байт, SHA-256 `{item['sha256'][:16]}…`" for name, item in bodies.items() ), f"- Стенд: LB, {profile['api_replicas']} реплики по {profile['api_cpus']:g} CPU / " f"{profile['api_memory']}, worker {profile['worker_cpus']:g} CPU / " f"{profile['worker_memory']} (`RENDER_MEMORY_MB={profile['render_memory_mb']}`), " f"брокер {profile['mq_cpus']:g} CPU / {profile['mq_memory']}, S3 " f"{profile['s3_cpus']:g} CPU / {profile['s3_memory']}", "- Пропускная — ready/с фазы ёмкости: N тел ждут в брокере, отсчёт — " "от строки `broker consuming` worker до последнего ready. Время до " "ready — `attachment_jobs.updated_at − attachments.created_at` (часы " "PostgreSQL); queue lag — `waited` журнала worker (приём → захват). " "Пик worker — `memory.peak` cgroup контейнера за ячейку (контейнер " "новый на ячейку), пик рендера — наибольший `render_peak_kib`. " "Незавершённых — точный счёт по созданию и завершению заданий. " "Восстановление — от конца окна перегрузки до последнего ready. " "Значения — медиана (min–max) повторов", "", ] grouped = group(rows) for size_class in classes: lines += [ f"## Класс `{size_class}`", "", "| Настройка | Пропускная, гот./с | CPU worker, ядер | p95 до ready " "(0,6 ёмкости), с | queue lag p95, с | p95 fixed, с | Окно перегрузки, с " "| 429 за окно, % | 429 после предела, % | Retry-After 5 / 1 " "| Макс. незавершённых | Восстановление, с | p95 после, с | Пик worker, MiB | " "Пик рендера, MiB | failed / missing | Пометки |", "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|", ] for config in configs: drain, steady, fixed, over, rec = ( grouped.get((config, size_class, phase), []) for phase in PHASES ) if not drain: continue accepted = steady + fixed + over + rec lost = sum( int(num(r.get(k)) or 0) for r in accepted for k in ("failed", "missing") ) failed = sum(int(num(r.get("failed")) or 0) for r in accepted) lines.append( f"| {config} | {spread((r['throughput'] for r in drain), 3)} | " f"{spread((r['worker_cpu_cores'] for r in drain), 2)} | " f"{spread((r['ttr_p95_ms'] for r in steady), 2, 1000)} | " f"{spread((r['lag_p95_ms'] for r in steady), 2, 1000)} | " f"{spread((r['ttr_p95_ms'] for r in fixed), 2, 1000)} | " f"{spread((r['window_s'] for r in over), 0)} | " f"{spread((r['share_429'] for r in over), 0, 0.01)} | " f"{spread((r['share_429_saturated'] for r in over), 0, 0.01)} | " f"{spread((r['retry_after_5'] for r in over), 0)} / " f"{spread((r['retry_after_1'] for r in over), 0)} | " f"{spread((r['max_unfinished'] for r in over), 0)} | " f"{spread((r['recovery_s'] for r in rec), 1)} | " f"{spread((r['ttr_p95_ms'] for r in rec), 2, 1000)} | " f"{spread((r['worker_peak_mib'] for r in drain), 0)} | " f"{spread((r['render_peak_mib'] for r in drain + accepted), 0)} | " f"{failed} / {lost - failed} | {_flags(drain + accepted)} |" ) lines.append("") lines += [ "## Гипотезы", "", "Тексты и пороги — `REPORT.md`, «Очередь: гипотезы» (записаны до " f"замеров); граница незавершённых — предел {limit} + места загрузок " f"стенда {slack}.", "", "| Гипотеза | Итог | Числа |", "|---|---|---|", ] lines += [ f"| {v.hypothesis} | {v.result} | {v.numbers} |" for v in verdicts(rows, backlog_limit=limit, slack=slack, classes=classes) ] lines += ["", "- " + recommendation(grouped, configs, classes, worker_mib)] warnings = meta.get("host_warnings") or [] if warnings: lines += ["", "## Предупреждения о хосте", "", *(f"- {w}" for w in warnings)] return "\n".join(lines) + "\n" # --- --selftest: проверка расчётов без стенда -------------------------------------- def _synthetic_cell( item: dict, *, throughput: float, peak_mib: float, steady_s: float, fixed_s: float ) -> CellData: """Ячейка без docker: отметки, журнал генератора и задания под заданную пропускную и время до ready ровной фазы и фазы fixed; в перегрузке 15 принятых и 5 отказов после предела.""" base = 1_800_000_000.0 marks = { "drain": base, "worker_start": base + 5, "steady": base + 30, "fixed": base + 45, "overload": base + 60, "recovery": base + 100, "end": base + 130, } gens: dict[str, dict] = {} uploads: dict[str, list[Upload]] = {} jobs: list[Job] = [] processed: dict[str, dict] = {} number = 0 def add(phase: str, at: float, status: int, ready_after: float | None) -> None: nonlocal number number += 1 attachment = f"a{number}" if status == 202 else "" uploads.setdefault(phase, []).append( Upload( at, at, at + 0.05, status, "5" if status == 429 else "", attachment, "" ) ) if status != 202: return finish = at + ready_after if ready_after is not None else None jobs.append( Job( attachment, at, f"j{number}", at + 0.01, "ready" if finish else "queued", finish, 1, "ready", ) ) if finish is not None: processed[attachment] = { "ts": finish, "job_id": f"j{number}", "attempt": 1, "render_peak_kib": 50_000, "took_ms": 200.0, "waited_ms": 100.0, } consuming = marks["worker_start"] + 1 for n in range(10): add( "drain", base + 1 + n * 0.1, 202, consuming + (n + 1) / throughput - (base + 1 + n * 0.1), ) for n in range(6): add("steady", marks["steady"] + n, 202, steady_s + n * 0.01) add("fixed", marks["fixed"] + n, 202, fixed_s + n * 0.01) for n in range(20): status = 202 if n < 10 or n % 2 else 429 add( "overload", marks["overload"] + n * 0.5, status, 30.0 if status == 202 else None, ) for n in range(4): add("recovery", marks["recovery"] + 5 + n, 202, 0.6) for phase, items in uploads.items(): accepted = sum(u.accepted for u in items) rejected = sum(u.status == 429 for u in items) gens[phase] = { "rate": 1.0, "duration_s": 10.0, "scheduled": len(items), "sent": len(items), "accepted": accepted, "rejected_429": rejected, "retry_after": {"5": rejected} if rejected else {}, "errors_5xx": 0, "other_errors": 0, "dropped": 0, "end": marks["overload"] + 10 if phase == "overload" else None, } samples = [ { "ts": base + t, "phase": "overload", "unfinished": 9, "outbox": 0, "in_broker": 8, "processing": 1, "oldest_age_s": 3.0, } for t in range(61, 70) ] ticks = [Tick(base + t, 1.5, int(peak_mib * MIB * 0.9)) for t in range(0, 130)] return CellData( item=item, body_bytes=1000, marks=marks, consuming_ts=consuming, gens=gens, uploads=uploads, jobs=jobs, log=WorkerLog(consuming=[consuming], processed=processed), ticks=ticks, samples=samples, peak_bytes=int(peak_mib * MIB), backlog_limit=10, ) def selftest() -> None: configs = (("c1p1", 1, 1), ("c2p2", 2, 2), ("c4p4", 4, 4), ("c2p32", 2, 32)) plan = plan_cells(configs, ("s", "l", "x"), 2, 7) assert [p["order"] for p in plan] == list(range(1, 25)) assert len({p["cell"] for p in plan}) == 24 assert plan[11]["cell"].rsplit("-", 1)[0] != plan[12]["cell"].rsplit("-", 1)[0] assert plan == plan_cells(configs, ("s", "l", "x"), 2, 7) text = "\n".join( [ "2026-11-05T16:23:20.100Z INFO src.broker [rid=-] broker consuming queue=bench.attachment-jobs prefetch=2", "2026-11-05T16:23:22.870Z INFO src.services.processing [rid=-] attachment processed attachment_id=a1 job_id=j1 attempt=1 variants=3 render_peak_kib=98236 took=201ms waited=241ms", "2026-11-05T16:23:23.000Z INFO src.services.processing [rid=-] job delivery skipped job_id=j1 attempt=1 outcome=done", "2026-11-05T16:23:24.000Z WARNING src.services.processing [rid=-] attachment processing failed attachment_id=a2 job_id=j2 attempt=1 error_code=image_too_large detail=x fenced=False", "2026-11-05T16:23:25.000Z INFO src.services.processing [rid=-] job retry scheduled job_id=j3 attachment_id=a3 attempt=1 reason=s3_read delay=1.2s fenced=False", "garbage line", ] ) log = parse_worker_log(text) assert log.consuming == [log_time("2026-11-05T16:23:20.100")] assert abs(log.consuming[0] - 1793895800.1) < 1e-3, log.consuming assert log.processed["a1"]["waited_ms"] == 241.0 assert log.processed["a1"]["render_peak_kib"] == 98236 assert (log.skipped, log.failed, log.retries) == ( {"j1": 1}, {"a2": "image_too_large"}, {"a3": 1}, ) gen = parse_gen_log( "q,1000.0,1001.0,1500.5,202,,a1,abc,\n" "q,2000.0,2001.0,2002.0,429,5,,abc,\n" "q,3000.0,,,0,,,,dropped\n" ) assert [u.status for u in gen] == [202, 429, 0] assert (gen[0].accepted, gen[0].done, gen[2].sent) == (True, 1.5005, None) jobs = parse_jobs( "a1|100.0|j1|100.1|ready|101.5|1|ready\n" "a2|100.5|j2|100.6|failed|102.0|2|failed\n" "a3|101.0|j3|101.1|queued|101.2|1|queued\n" "a4|101.0||||||pending\n" ) assert (jobs[0].ttr_ms, jobs[1].ready_at, jobs[2].finished_at) == ( 1500.0, None, None, ) assert jobs[3].job_created_at is None # 100.1 → +1, 100.6 → +2, 101.1 → +3, 101.5 → 2, 102.0 → 1 assert max_unfinished(jobs, 100.0, 103.0) == 3 assert max_unfinished(jobs, 101.6, 103.0) == 2 assert max_unfinished(jobs, 100.0, 100.55) == 1 sample = parse_sample("1800000000.5|7|1|4|2|3.456\n") assert sample == { "ts": 1800000000.5, "unfinished": 7, "outbox": 1, "in_broker": 4, "processing": 2, "oldest_age_s": 3.46, } lines = [ {"ts": 1.0, "cg": {"w": [1_000_000, 0, 0, 0, 100 * MIB]}}, {"ts": 2.0, "cg": {"w": [2_500_000, 0, 0, 0, 120 * MIB]}}, {"ts": 3.0, "cg": {}}, {"ts": 4.0, "cg": {"w": [10_000, 0, 0, 0, 50 * MIB]}}, {"ts": 5.0, "cg": {"w": [5_000, 0, 0, 0, 60 * MIB]}}, ] ticks = worker_ticks(lines, "w") assert [t.cpu_cores for t in ticks] == [None, 1.5, None, None], ticks assert percentile([5, 1, 3, 2, 4], 0.95) == 5 assert percentile([5, 1, 3, 2, 4], 0.5) == 3 assert percentile([], 0.5) is None rows: list[dict] = [] classes = ["s", "l", "x"] # пик c4p4 на x выше 80 % лимита 1536 MiB: в рекомендацию он не идёт peaks = {"c1p1": 400.0, "c2p2": 700.0, "c4p4": 1300.0, "c2p32": 720.0} rates = {"c1p1": 1.0, "c2p2": 1.8, "c4p4": 1.9, "c2p32": 1.85} steady_s = {"c1p1": 0.9, "c2p2": 0.5, "c4p4": 0.45, "c2p32": 0.55} # c2p32 быстрее c2p2 на 4 % — в пределах 10 %: рекомендация по пику fixed_s = {"c1p1": 0.9, "c2p2": 0.5, "c4p4": 0.45, "c2p32": 0.48} for item in plan: config = item["config"] peak = peaks[config] if item["size_class"] == "x" else 300.0 data = _synthetic_cell( item, throughput=rates[config], peak_mib=peak, steady_s=steady_s[config], fixed_s=fixed_s[config], ) cell = cell_rows(data) assert [r["phase"] for r in cell] == list(PHASES) assert all(set(r) <= set(CELL_COLUMNS) for r in cell), set(cell[0]) - set( CELL_COLUMNS ) drain, steady, fixed, over, rec = cell assert abs(fixed["ttr_p95_ms"] - (fixed_s[config] + 0.05) * 1000) < 0.2 assert (fixed["jobs"], fixed["ready"], fixed["missing"]) == (6, 6, 0) assert abs(drain["throughput"] - rates[config]) < 0.01, drain assert (drain["startup_s"], drain["worker_peak_mib"]) == (1.0, peak) assert over["max_unfinished"] == 10 + 5, over["max_unfinished"] assert (over["backlog_reached"], over["share_429_saturated"]) == (True, 0.5) assert (over["share_429"], over["retry_after_5"]) == (0.25, 5) assert (rec["recovery_s"], rec["missing"]) == (29.5, 0), rec assert abs(steady["ttr_p95_ms"] - (steady_s[config] + 0.05) * 1000) < 0.2 assert steady["lag_p95_ms"] == 100.0 assert drain["worker_cpu_cores"] == 1.5 timeline = timeline_rows(data) assert len(timeline) == 9 assert set(timeline[0]) <= set(TIMELINE_COLUMNS) assert sum(r["accepted"] + r["rejected_429"] for r in timeline) > 0 assert all(set(r) <= set(JOB_COLUMNS) for r in job_rows(data)) rows += cell found = { v.hypothesis: v.result for v in verdicts(rows, backlog_limit=10, slack=8, classes=classes) } # c4p4/c2p2 = 1,9/1,8 ≤ 1,15, c2p2/c1p1 = 1,8 ≥ 1,6; пики x 1300/400; # c2p32 против c2p2: +3 % пропускной, −4 % p95 фазы fixed assert found == { "Q1": CONFIRMED, "Q2": CONFIRMED, "Q3": CONFIRMED, "Q4": CONFIRMED, "Q5": REFUTED, }, found # восстановление синтетической ячейки — 30 с на пределе 10 при 1 гот./с: # граница 1,3 × 10 / 1 + 5 = 18 с assert verdict_q5(rows, backlog_limit=100).result == CONFIRMED assert verdict_q4(rows, backlog_limit=5, slack=8).result == REFUTED quick = [ r for r in rows if r["config"] in ("c1p1", "c4p4") and r["size_class"] in ("s", "x") ] quick_found = { v.hypothesis: v.result for v in verdicts(quick, backlog_limit=100, slack=8, classes=["s", "x"]) } assert quick_found == { "Q1": UNTESTED, "Q2": UNTESTED, "Q3": CONFIRMED, "Q4": CONFIRMED, "Q5": CONFIRMED, }, quick_found text = recommendation(group(rows), list(peaks), classes, 1536) # x: c4p4 выше 80 % лимита, c2p32 быстрее c2p2 меньше чем на 10 % — # берётся меньший пик; s: пики равны — меньший p95 assert "x — c2p2" in text, text assert "s — c4p4" in text, text assert overload_window(40, 60, 1.0, fill=1.5, cap=150) == 90 assert overload_window(40, 60, 10.0, fill=1.5, cap=150) == 40 assert overload_window(40, 60, 0.3, fill=1.5, cap=150) == 150 meta = { "label": "selftest", "profile": { "name": "queue", "configs": configs, "classes": ( ("s", "noise", 320, 200, 100), ("l", "noise", 1800, 1800, 40), ("x", "flat", 6000, 6000, 20), ), "repeats": 2, "window_s": 40.0, "steady_factor": 0.6, "fixed_rates": (("s", 2.5), ("l", 1.0), ("x", 0.5)), "overload_factor": 2.0, "overload_fill": 1.5, "overload_max_s": 150.0, "backlog_limit": 10, "api_replicas": 2, "api_cpus": 1.0, "api_memory": "512m", "worker_cpus": 2.0, "worker_memory": "1536m", "render_memory_mb": 768, "mq_cpus": 1.0, "mq_memory": "512m", "s3_cpus": 1.0, "s3_memory": "512m", }, "upload_slack": 8, "bodies": { "s": { "kind": "noise", "width": 320, "height": 200, "bytes": 1, "sha256": "0" * 64, } }, } summary = render_summary(meta, rows) for part in ("## Класс `x`", "| Q5 |", "рекомендация"): assert part in summary, part print("queue selftest: ok") if __name__ == "__main__": if sys.argv[1:] == ["--selftest"]: selftest() else: raise SystemExit("usage: uv run python bench/queue.py --selftest")