Files
mtusi2026lab6/bench/run.py
T

3918 lines
149 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Оркестратор нагрузочного стенда лаб 3–6. Только stdlib.
uv run python bench/run.py telemetry [--label baseline] # трек 1 лабы 6
uv run python bench/run.py telemetry-quick
uv run python bench/run.py queue [--label baseline] # трек 1 лабы 5
uv run python bench/run.py queue-quick
uv run python bench/run.py failover [--label baseline] # уровень 4 лабы 3
uv run python bench/run.py failover-quick # проверка стенда
uv run python bench/run.py rollout [--label baseline] # уровень 4 лабы 3
uv run python bench/run.py rollout-quick
uv run python bench/run.py uploads [--label baseline] # уровень 4 лабы 4
uv run python bench/run.py uploads-quick
uv run python bench/run.py reconcile [--label baseline] # трек 2 лабы 4
uv run python bench/run.py reconcile-quick
uv run python bench/run.py scaling --series a|b|db|ab|quick [--label L]
[--configs "4x1-pg0.5 4x1"] [--resume] # трек 1 лабы 3
Профили (поля — в `PROFILES`, целиком пишутся в `meta.json`):
- `failover` — стенд с лимитами рабочего (две реплики по 1 CPU / 512 МБ,
PostgreSQL 2 CPU / 1 ГБ, Valkey 0,5 / 128 МБ, HAProxy 1 / 256 МБ),
постоянная интенсивность записи с ключом и чтения; каждая реплика по
очереди в режимах `stop`, `kill`, `pause` с возвратом. Посекундная шкала,
время исключения из LB и возврата, ошибки, время восстановления, повтор
неуспешных записей с тем же ключом и сверка «без потерь и дублей»,
CPU и память компонентов по окнам фазы. Результат —
`evidence/failover/<метка>/`;
- `rollout` — тот же стенд и нагрузка, посреди неё
`scripts/rolling-update.sh` меняет обе реплики на новый образ с
expand-ревизией схемы (`9001_rollout_probe`, только в образе стенда).
Результат — `evidence/rollout/<метка>/`;
- `scaling` — серии трека 1 «Цена масштабирования»: конфигурации реплик
и лимитов, на каждую ячейку (конфигурация × повтор) свежий стенд,
открытая модель, фиксированные уровни и поиск R* по SLO, сэмплер
контейнеров, LB и PostgreSQL. Результат — `evidence/scaling/<метка>/`;
- `uploads` (лаба 4) — одна реплика 1 CPU / 512 МБ без swap, S3 рабочего
стенда; варианты «с лимитом» (места загрузок по умолчанию) и «без
лимита» (1000 мест, без очереди) × тела ~1 и ~10 MiB × 1/10/20/40
клиентов; на прогон — новые S3 и реплика, пик `memory.peak` окна,
OOM, ответы по кодам, загрузок/с. Результат — `evidence/uploads/<метка>/`;
- `reconcile` (лаба 4, трек 2) — стоимость сверки bucket и БД: набор мимо
API (`COPY` метаданных, PUT объектов `fill_objects.py`) с расхождениями
всех классов, разовые проходы уборщика `batch`, `head`, глубокий, под
фоновой нагрузкой с пределом скорости и без, отказ по пределу.
Результат — `evidence/reconcile/<метка>/`;
- `queue` (лаба 5, трек 1) — очередь под перегрузкой: две реплики, брокер,
relay и один worker 2 CPU / 1536 МБ; настройки concurrency/prefetch ×
классы изображений, на ячейку — ёмкость (N тел при остановленном
worker), ровная нагрузка 0,6 ёмкости, общая на класс интенсивность
(`fixed`), перегрузка 2 ёмкости при пределе `UPLOAD_MAX_UNFINISHED` (окно
дотягивается до предела), восстановление. Результат —
`evidence/queue/<метка>/`;
- `telemetry` (лаба 6, трек 1) — цена наблюдаемости: стенд очереди и
сервисы телеметрии рабочего стенда; варианты `off`, `full`, `head`,
`tail`, `balanced` (`telemetry.VARIANTS`) × повторы кругами при одной
открытой нагрузке с редкими событиями известного счёта; стенд новый на
повтор, телеметрия с пустыми хранилищами и процессы приложения — на
прогон. Результат — `evidence/telemetry-cost/<метка>/`.
Стенд — чёрный ящик: запросы делает только генератор (`loadgen.py`) по
HTTP через HAProxy стенда; к PostgreSQL оркестратор обращается напрямую
только для обслуживания и сверки (VACUUM, CHECKPOINT, число строк,
ревизия схемы), у профиля `reconcile` — ещё для наполнения набора, у
`queue` — для чтения состояния заданий (только SELECT).
Метки `quick-*` проверяют стенд и в отчёт не идут. Правила —
`bench/AGENTS.md`, порядок работы — `bench/RUNBOOK.md`.
"""
import argparse
import csv
import dataclasses
import gzip
import hashlib
import json
# bench/queue.py, а не модуль stdlib: каталог скрипта первым в sys.path
import queue
import random
import re
import statistics
import subprocess
import threading
import time
from collections.abc import Callable
from pathlib import Path
import analysis
import plots
import reconcile
import telemetry
import uploads
from common import (
BENCH,
CACHE,
COLLECTOR,
DB,
EVIDENCE,
FILL,
FIXTURE,
GEN,
IMAGE_REPO,
LB,
LOKI,
NET,
PROJECT,
PROM,
RECONCILE_PREFIX,
RELAY,
ROOT,
S3,
SAMPLER,
SERVICE_TAG,
TELEMETRY_NET,
TEMPO,
WORKER,
Limits,
Stand,
StandError,
bench_env,
bench_lock,
build_image,
code_state,
docker,
finish_host_state,
gib,
image_info,
new_label_dir,
preflight,
probe_migration_context,
quick_label,
say,
sh,
swapouts,
temp_workdir,
utc_now,
write_json,
)
# лимиты рабочего стенда compose.yaml: профили уровня 4 отвечают, что видит
# клиент `make up` при отказе или замене реплики
WORKING = Limits(api_replicas=2, api_cpus=1.0, api_memory="512m")
@dataclasses.dataclass(frozen=True)
class LoadProfile:
"""Профили уровня 4 (`failover`, `rollout`). Часть методики."""
name: str
limits: Limits
users: int
read_messages: int
write_rate: float
read_rate: float
gen_procs: int
session_ttl: int
seed: int
# failover: режимы отказа и сколько реплик по очереди
modes: tuple[str, ...] = ()
targets: int = 0
before_s: float = 0.0
fail_s: float = 0.0
after_s: float = 0.0
# запас на возврат реплики в LB после восстановления
restore_s: float = 10.0
# rollout: окна до скрипта, бюджет замены и после
rollout_before_s: float = 0.0
rollout_budget_s: float = 0.0
rollout_after_s: float = 0.0
# худшее обнаружение отказа по формуле haproxy.cfg, с
bounds: tuple[tuple[str, float], ...] = (
("stop", 2.5),
("kill", 2.5),
("pause", 6.5),
)
@dataclasses.dataclass(frozen=True)
class ScalingProfile:
"""Серии трека 1. Часть методики: метки сравнимы при равном профиле."""
name: str
series: str
configs: tuple[str, ...]
repeats: int
users: int = 20
read_messages: int = 10_000
write_chats: int = 10
fill_clients: int = 16
# смесь read,write,me: история (70), отправка с ключом (20), профиль (10)
mix: str = "70,20,10"
fixed_rates: tuple[float, ...] = (250.0, 500.0)
ladder_base: float = 100.0
ladder_factor: float = 1.25
# поиск начинается не ниже этого × число реплик
start_per_replica: float = 300.0
bisections: int = 2
rate_cap: float = 8000.0
warmup_first_s: float = 10.0
warmup_ramp_s: float = 5.0
window_s: float = 15.0
pause_after_fail_s: float = 10.0
slo_p95_ms: float = 200.0
slo_fail_share: float = 0.01
abort_error_share: float = 0.5
abort_p95_ms: float = 2000.0
request_timeout_s: float = 10.0
gen_procs: int = 4
session_ttl: int = 21_600
order_seed: int = 20261014
# лимиты без реплик: PostgreSQL, Valkey, LB, генератор, сэмплер
base: Limits = Limits(api_replicas=1, api_cpus=1.0, api_memory="512m")
FAILOVER = LoadProfile(
name="failover",
limits=WORKING,
users=4,
read_messages=1_000,
write_rate=100.0,
read_rate=100.0,
gen_procs=2,
session_ttl=21_600,
seed=20261014,
modes=("stop", "kill", "pause"),
targets=2,
before_s=10.0,
fail_s=15.0,
after_s=10.0,
)
ROLLOUT = dataclasses.replace(
FAILOVER,
name="rollout",
modes=(),
targets=0,
rollout_before_s=10.0,
rollout_budget_s=90.0,
rollout_after_s=15.0,
)
PROFILES: dict[str, LoadProfile] = {
"failover": FAILOVER,
"failover-quick": dataclasses.replace(
FAILOVER,
name="failover-quick",
write_rate=50.0,
read_rate=50.0,
modes=("stop", "kill"),
targets=1,
before_s=5.0,
fail_s=8.0,
after_s=5.0,
),
"rollout": ROLLOUT,
"rollout-quick": dataclasses.replace(
ROLLOUT,
name="rollout-quick",
write_rate=50.0,
read_rate=50.0,
rollout_before_s=5.0,
rollout_budget_s=60.0,
rollout_after_s=5.0,
),
}
SERIES: dict[str, ScalingProfile] = {
# бюджет приложения 4 CPU / 2 ГиБ: один процесс, два, четыре
"a": ScalingProfile(
name="scaling", series="a", configs=("1x4", "2x2", "4x1"), repeats=3
),
# рост ресурсов: реплики по 1 CPU, PostgreSQL 2 CPU фиксирован
"b": ScalingProfile(
name="scaling", series="b", configs=("1x1", "2x1", "3x1", "4x1"), repeats=3
),
# управляемое насыщение PostgreSQL: 0,5 CPU при 1, 2, 4 репликах
"db": ScalingProfile(
name="scaling",
series="db",
configs=("1x1-pg0.5", "2x1-pg0.5", "4x1-pg0.5"),
repeats=3,
),
# «до/после» улучшения и A/A: ячейки задаёт --configs
"ab": ScalingProfile(
name="scaling", series="ab", configs=("4x1-pg0.5", "4x1"), repeats=3
),
# самопроверка серий: малые данные, короткие окна, один повтор
"quick": ScalingProfile(
name="scaling-quick",
series="quick",
configs=("1x1", "2x1"),
repeats=1,
users=5,
read_messages=1_000,
write_chats=3,
fixed_rates=(100.0, 200.0),
start_per_replica=150.0,
bisections=1,
warmup_first_s=4.0,
warmup_ramp_s=3.0,
window_s=6.0,
pause_after_fail_s=5.0,
),
}
MIB = 2**20
KIB = 1024
@dataclasses.dataclass(frozen=True)
class UploadsProfile:
"""Параметры профиля uploads (лаба 4, уровень 4: сколько памяти ест
реплика под параллельными загрузками). Это методика, поэтому все поля
сохраняются в `meta.json`."""
name: str
variants: tuple[str, ...]
# имя тела → наибольший размер (байт): PNG из шума не больше него
sizes: tuple[tuple[str, int], ...]
clients: tuple[int, ...]
repeats: int
warmup_s: float = 3.0
window_s: float = 20.0
upload_timeout_s: float = 90.0
sample_interval_s: float = 0.2
seed: int = 20261023
api_cpus: float = 1.0
api_memory: str = "512m"
s3_cpus: float = 1.0
s3_memory: str = "512m"
session_ttl: int = 21_600
# «без лимита»: место каждому клиенту, очереди нет (override реплики)
unlimited_env: tuple[tuple[str, str], ...] = (
("UPLOAD_MAX_CONCURRENT", "1000"),
("UPLOAD_MAX_WAITING", "0"),
)
def limits(self) -> Limits:
return Limits(
api_replicas=1,
api_cpus=self.api_cpus,
api_memory=self.api_memory,
with_s3=True,
s3_cpus=self.s3_cpus,
s3_memory=self.s3_memory,
)
UPLOADS = UploadsProfile(
name="uploads",
variants=uploads.VARIANTS,
sizes=(("1mib", MIB), ("10mib", 10 * MIB)),
clients=(1, 10, 20, 40),
repeats=3,
)
UPLOADS_PROFILES = {
"uploads": UPLOADS,
"uploads-quick": dataclasses.replace(
UPLOADS,
name="uploads-quick",
sizes=(("10mib", 10 * MIB),),
clients=(1, 10, 40),
repeats=1,
window_s=10.0,
),
}
@dataclasses.dataclass(frozen=True)
class ReconcileProfile:
"""Параметры профиля reconcile (лаба 4, трек 2: цена сверки S3 с базой); методика."""
name: str
# этап: имя, объектов набора после этапа, размер объекта, свежий стенд,
# ячейки, строк без объекта сверх набора (>0 — набор отказа по пределу:
# объекты все согласованы)
stages: tuple[tuple[str, int, int, bool, tuple[str, ...], int], ...]
repeats: int
refuse_max_changes: int = 100
seed: int = 20261023
# grace прохода, с: больше границы живой загрузки (21 с), меньше паузы
# после наполнения набора
grace_s: int = 30
cleaner_cpus: float = 0.25
cleaner_memory: str = "256m"
# управляемый опыт: уборщик на 1 CPU
fast_cpus: float = 1.0
# загрузчик и уборщик управляемого опыта не работают одновременно: на
# пике стенд с ними — 9,75 CPU, как профили лабы 3
fill_cpus: float = 1.0
fill_memory: str = "512m"
fill_processes: int = 3
fill_threads: int = 16
sample_interval_s: float = 0.5
users: int = 4
# фоновая нагрузка ячеек load-*: скачивания согласованных вложений,
# запись с ключом и чтение истории через LB
load_download_rate: float = 200.0
load_write_rate: float = 50.0
load_read_rate: float = 50.0
load_downloads: int = 2000
load_procs: int = 2
quiet_s: float = 20.0
pass_budget_s: float = 600.0
session_ttl: int = 21_600
def limits(self) -> Limits:
return dataclasses.replace(
WORKING,
with_s3=True,
cleaner_cpus=self.cleaner_cpus,
cleaner_memory=self.cleaner_memory,
)
RECONCILE = ReconcileProfile(
name="reconcile",
stages=(
("a", 1_000, MIB, True, ("batch", "deep", "deep-fast"), 0),
("b", 10_000, 4 * KIB, True, ("batch", "head", "deep"), 0),
("c", 30_000, 4 * KIB, False, ("batch", "head"), 0),
("d", 100_000, 4 * KIB, False, ("batch", *reconcile.LOAD_CELLS), 0),
("e", 10_000, 4 * KIB, True, ("batch", "refuse"), 1_000),
),
repeats=3,
)
RECONCILE_PROFILES = {
"reconcile": RECONCILE,
"reconcile-quick": dataclasses.replace(
RECONCILE,
name="reconcile-quick",
stages=(
(
"q",
2_000,
4 * KIB,
True,
("batch", "head", "deep", "load-stand", "load-fast"),
0,
),
("qe", 500, 4 * KIB, True, ("batch", "refuse"), 100),
),
repeats=1,
refuse_max_changes=10,
quiet_s=5.0,
pass_budget_s=300.0,
),
}
if not hasattr(queue, "CELL_COLUMNS"):
raise SystemExit(
"stdlib queue shadows bench/queue.py: run bench/run.py as a script"
)
@dataclasses.dataclass(frozen=True)
class QueueProfile:
"""Очередь под перегрузкой (трек 1 лабы 5). Часть методики: поля
целиком пишутся в `meta.json`; ресурсы worker одни на все настройки —
меняются только concurrency и prefetch."""
name: str
# настройка worker: имя, WORKER_CONCURRENCY, WORKER_PREFETCH
configs: tuple[tuple[str, int, int], ...]
# класс изображения: имя, PNG (noise — шум, flat — один цвет), ширина,
# высота, тел в фазе ёмкости (меньше backlog_limit: все должны пройти)
classes: tuple[tuple[str, str, int, int, int], ...]
repeats: int
window_s: float = 40.0
# доли ёмкости ячейки: ровная нагрузка (и после перегрузки) и перегрузка
steady_factor: float = 0.6
overload_factor: float = 2.0
# окно перегрузки: min(overload_max_s, max(window_s, overload_fill ×
# backlog_limit / ёмкость)) — предел достигается на любом классе
overload_fill: float = 1.5
overload_max_s: float = 150.0
# фаза fixed: одна интенсивность на класс для всех настроек, загрузок/с
# (≈ 0,5 ёмкости c1p1 по пробе 5 ноября: s 5,87, l 1,79, x 1,09 гот./с
# — вниз до простого числа)
fixed_rates: tuple[tuple[str, float], ...] = (("s", 2.5), ("l", 0.8), ("x", 0.5))
# UPLOAD_MAX_UNFINISHED реплик
backlog_limit: int = 60
# загрузок/с при наполнении фазы ёмкости: быстрее, чем worker разберёт,
# но без 429 по местам загрузок
fill_rates: tuple[tuple[str, float], ...] = (("s", 25.0), ("l", 4.0), ("x", 10.0))
upload_timeout_s: float = 60.0
# потолок ожидания пустой очереди: много больше любой фазы
settle_timeout_s: float = 900.0
sample_interval_s: float = 1.0
seed: int = 20261105
api_replicas: int = 2
api_cpus: float = 1.0
api_memory: str = "512m"
s3_cpus: float = 1.0
s3_memory: str = "512m"
mq_cpus: float = 1.0
mq_memory: str = "512m"
worker_cpus: float = 2.0
worker_memory: str = "1536m"
render_memory_mb: int = 768
# генератор открытой модели загрузок — один процесс: ему хватает 1 CPU,
# а 512 МБ оставляют бюджет ВМ 12 CPU / 7,7 ГиБ с запасом
gen_cpus: float = 1.0
gen_memory: str = "512m"
session_ttl: int = 21_600
def limits(self) -> Limits:
return Limits(
api_replicas=self.api_replicas,
api_cpus=self.api_cpus,
api_memory=self.api_memory,
gen_cpus=self.gen_cpus,
gen_memory=self.gen_memory,
with_s3=True,
s3_cpus=self.s3_cpus,
s3_memory=self.s3_memory,
with_cleaner=False,
with_pipeline=True,
mq_cpus=self.mq_cpus,
mq_memory=self.mq_memory,
worker_cpus=self.worker_cpus,
worker_memory=self.worker_memory,
)
QUEUE_CONFIGS = (("c1p1", 1, 1), ("c2p2", 2, 2), ("c4p4", 4, 4), ("c2p32", 2, 32))
# тел фазы ёмкости меньше предела 60: при остановленном worker все они
# незавершённые, и каждое должно получить 202
QUEUE_CLASSES = (
("s", "noise", 320, 200, 50),
("l", "noise", 1800, 1800, 40),
("x", "flat", 6000, 6000, 20),
)
QUEUE = QueueProfile(
name="queue", configs=QUEUE_CONFIGS, classes=QUEUE_CLASSES, repeats=2
)
QUEUE_PROFILES = {
"queue": QUEUE,
"queue-quick": dataclasses.replace(
QUEUE,
name="queue-quick",
configs=(QUEUE_CONFIGS[0], QUEUE_CONFIGS[2]),
classes=(QUEUE_CLASSES[0], QUEUE_CLASSES[2]),
repeats=1,
window_s=15.0,
),
}
# лимит тела загрузки (src/constants.py, копия): класс l обязан в него влезть
ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024
@dataclasses.dataclass(frozen=True)
class TelemetryProfile:
"""Цена наблюдаемости (трек 1 лабы 6). Часть методики: поля целиком
пишутся в `meta.json`; варианты отличаются только телеметрией
(`telemetry.VARIANTS`), нагрузка и лимиты у всех одни."""
name: str
variants: tuple[str, ...]
repeats: int
# окна одного запуска генератора: разогрев, чистое (по нему цена),
# отказы (блокировки таблицы сообщений в моменты `locks` от начала окна)
warmup_s: float = 20.0
clean_s: float = 90.0
fault_s: float = 30.0
locks: tuple[float, ...] = (5.0, 15.0, 25.0)
lock_s: float = 0.6
# открытая нагрузка, запросов/с: ниже насыщения реплик и worker (проба
# 16 ноября при 140/с — реплика занята ~18 %, CPU генератора 3 %); редкие
# события — с известным счётом за прогон
rates: tuple[tuple[str, float], ...] = (
("read", 100.0),
("write", 30.0),
("me", 40.0),
("chat", 30.0),
("download", 8.0),
("upload", 0.5),
("bad", 0.1),
("broken", 0.2),
)
# после нагрузки и пустой очереди: решение tail sampling (до 10 с),
# пакет Collector (2 с) и приём Tempo
settle_s: float = 15.0
drain_timeout_s: float = 180.0
users: int = 4
read_messages: int = 200
warm_uploads: int = 3
upload_width: int = 480
upload_height: int = 320
seed: int = 20261116
sample_interval_s: float = 1.0
api_replicas: int = 2
api_cpus: float = 1.0
api_memory: str = "512m"
# бюджет — ВМ 10 CPU / 7,7 ГиБ (машина замеров): PostgreSQL, LB, worker,
# генератор и сэмплер ниже рабочего стенда, сервисы телеметрии — как у
# него (compose.yaml); при этой нагрузке ни один из них не упирается
db_cpus: float = 1.0
db_memory: str = "768m"
lb_cpus: float = 0.5
lb_memory: str = "256m"
worker_cpus: float = 1.0
worker_memory: str = "768m"
gen_cpus: float = 1.0
gen_memory: str = "256m"
sampler_memory: str = "128m"
session_ttl: int = 21_600
def limits(self) -> Limits:
return Limits(
api_replicas=self.api_replicas,
api_cpus=self.api_cpus,
api_memory=self.api_memory,
db_cpus=self.db_cpus,
db_memory=self.db_memory,
lb_cpus=self.lb_cpus,
lb_memory=self.lb_memory,
gen_cpus=self.gen_cpus,
gen_memory=self.gen_memory,
sampler_memory=self.sampler_memory,
with_s3=True,
with_cleaner=False,
with_pipeline=True,
worker_cpus=self.worker_cpus,
worker_memory=self.worker_memory,
with_telemetry=any(v != "off" for v in self.variants),
)
TELEMETRY = TelemetryProfile(
name="telemetry", variants=tuple(telemetry.VARIANTS), repeats=3
)
TELEMETRY_PROFILES = {
"telemetry": TELEMETRY,
"telemetry-quick": dataclasses.replace(
TELEMETRY,
name="telemetry-quick",
variants=("off", "full", "head", "tail", "balanced"),
repeats=1,
warmup_s=5.0,
clean_s=20.0,
fault_s=12.0,
locks=(3.0, 8.0),
settle_s=12.0,
),
}
TELEMETRY_SERVICES = ("otel-collector", "prometheus", "loki", "tempo")
TELEMETRY_VOLUMES = ("promdata", "lokidata", "tempodata")
RUN_COLUMNS = (
"series",
"config",
"repeat",
"cell",
"step",
"phase",
"rate",
"rate_from",
"warmup_s",
"window_s",
"scheduled",
"ok",
"errors",
"short_pages",
"dropped",
"unsent",
"fail_share",
"achieved_rps",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"service_p95_ms",
"lag_p99_ms",
"slo_pass",
"abort",
"api_cpu_cores",
"api_cpu_max",
"api_throttled_max",
"api_memory_max_mib",
"db_cpu_cores",
"db_throttled_share",
"lb_cpu_cores",
"cache_cpu_cores",
"gen_cpu_cores",
"vm_busy_share",
"pg_conn_max",
"pg_conn_replica_max",
"pg_active_mean",
"pg_lock_wait_mean",
"pg_new_sessions",
"pg_commits_per_s",
"lb_retries",
"lb_redispatches",
"lb_conn_errors",
"api_cpu_us_per_req",
"db_cpu_us_per_req",
"instances",
"instance_share_min",
"signals",
"error_kinds",
)
SEARCH_COLUMNS = (
"series",
"config",
"replicas",
"api_cpus",
"db_cpus",
"repeat",
"cell",
"r_star",
"first_fail",
"p95_at_r_star",
"signals_at_fail",
"generator_bound",
"steps",
"fill_rps",
"duration_s",
"flags",
)
SAMPLE_COLUMNS = (
"config",
"repeat",
"step",
"rate",
"t",
"role",
"container",
"cpu_cores",
"throttled_share",
"memory_mib",
)
# --- общие шаги ---------------------------------------------------------------------
def write_csv(path: Path, columns: tuple[str, ...], rows: list[dict]) -> None:
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore")
writer.writeheader()
writer.writerows(rows)
def append_csv(path: Path, columns: tuple[str, ...], row: dict) -> None:
new = not path.exists()
with path.open("a", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore")
if new:
writer.writeheader()
writer.writerow(row)
def read_csv(path: Path) -> list[dict]:
if not path.exists():
return []
with path.open(encoding="utf-8") as handle:
return list(csv.DictReader(handle))
def prepare_image(label: str) -> dict:
"""Образ из рабочего дерева под тегом метки; на него смотрит сервис api.
Образ полной метки остаётся в docker: по нему метку можно сверить
(`docker image inspect`); пробные `quick-*` убирает `drop_quick_image`.
"""
tag = f"{IMAGE_REPO}:bench-{label}"
say(f"build image {tag}")
build_image(tag)
docker("tag", tag, SERVICE_TAG)
return image_info(tag)
def drop_quick_image(label: str) -> None:
if label.startswith("quick-"):
docker("rmi", f"{IMAGE_REPO}:bench-{label}", check=False)
def clock_offset() -> float:
"""Часы ВМ Docker минус часы хоста, с: по контейнеру генератора.
Три замера, берётся с наименьшей задержкой вызова. Нужен только для
отметок, которые оркестратор ставит сам (начало и конец скрипта
замены); остальные отметки — часы ВМ.
"""
best = None
for _ in range(3):
before = time.time()
out = docker("exec", GEN, "python", "-c", "import time; print(time.time())")
after = time.time()
rtt = after - before
offset = float(out.stdout.strip()) - (before + after) / 2
if best is None or rtt < best[0]:
best = (rtt, offset)
return best[1]
def roles_of(stand: Stand) -> dict[str, str]:
"""id12 контейнера → роль: api, db, cache, lb, gen."""
roles = {container[:12]: "api" for container in stand.api_ids()}
for role, name in (
("db", DB),
("cache", CACHE),
("lb", LB),
("gen", GEN),
("sampler", SAMPLER),
):
done = docker("inspect", "-f", "{{.Id}}", name, check=False)
if done.returncode == 0:
roles[done.stdout.strip()[:12]] = role
return roles
def full_ids(stand: Stand) -> list[str]:
ids = list(stand.api_ids())
for name in (DB, CACHE, LB, GEN):
ids.append(docker("inspect", "-f", "{{.Id}}", name).stdout.strip())
return ids
def resources_by_role(ticks: list[dict], roles: dict[str, str]) -> dict:
"""CPU и память по ролям за окно; у api — сумма и максимум реплики."""
usage = {cid: analysis.cgroup_usage(ticks, cid) for cid in roles}
out: dict = {}
api = [u for cid, u in usage.items() if u and roles[cid] == "api"]
if api:
out["api_cpu_cores"] = round(sum(u["cpu_cores"] for u in api), 3)
out["api_cpu_max"] = max(u["cpu_cores"] for u in api)
out["api_throttled_max"] = max(u["throttled_share"] for u in api)
out["api_memory_max_mib"] = max(u["memory_mib"] for u in api)
for cid, role in roles.items():
if role == "api" or not usage.get(cid):
continue
out[f"{role}_cpu_cores"] = usage[cid]["cpu_cores"]
out[f"{role}_throttled_share"] = usage[cid]["throttled_share"]
out[f"{role}_memory_mib"] = usage[cid]["memory_mib"]
out["vm_busy_share"] = analysis.vm_busy(ticks)
return out
# --- failover -------------------------------------------------------------------------
def run_failover(profile: LoadProfile, label: str) -> Path:
workdir = temp_workdir()
folder = new_label_dir("failover", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
pre = preflight(profile.limits.total_cpus(), profile.limits.total_memory_gib())
image = prepare_image(label)
meta = {
"kind": "failover",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
}
write_json(folder / "meta.json", meta)
stand = Stand(profile.limits, session_ttl=profile.session_ttl, workdir=workdir)
phases, timeline_rows, resource_rows = [], [], []
try:
say("stand: postgres, valkey, migrate, 2 replicas, haproxy")
stand.start()
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
"1",
)
stand.maintain()
meta["setup"] = {k: v for k, v in setup.items() if k != "write_chats"}
meta["stand"] = stand.passport()
write_chat = setup["write_chats"][0]
targets = stand.api_ids()[: profile.targets]
roles = roles_of(stand)
stand.start_sampler("scan")
number = 0
for index, target in enumerate(targets, start=1):
for mode in profile.modes:
number += 1
name = f"p{number}-{mode}-r{index}"
say(f"phase {name}: target {target[:12]}")
phase, rows = failover_phase(
stand, profile, name, mode, target, write_chat, folder
)
phases.append(phase)
timeline_rows += rows
ticks = stand.sampler_lines()
roles.update(roles_of(stand))
for phase in phases:
for window, (start, end) in phase.pop("_windows").items():
usage = resources_by_role(
analysis.window_ticks(ticks, start / 1000, end / 1000), roles
)
ok_rps = phase[f"ok_rps_{window}"]
resource_rows.append(
{
"phase": phase["phase"],
"window": window,
**usage,
"api_cpu_us_per_req": _per_req(
usage.get("api_cpu_cores"), ok_rps
),
"db_cpu_us_per_req": _per_req(
usage.get("db_cpu_cores"), ok_rps
),
},
)
meta["sampler_ticks"] = len(ticks)
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
finally:
stand.stop_sampler()
stand.stop()
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(folder / "phases.csv", PHASE_COLUMNS, phases)
write_csv(folder / "timeline.csv", TIMELINE_COLUMNS, timeline_rows)
write_csv(folder / "resources.csv", RESOURCE_COLUMNS, resource_rows)
(folder / "summary.md").write_text(
failover_summary(meta, phases, resource_rows), encoding="utf-8"
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
PHASE_COLUMNS = (
"phase",
"mode",
"replica",
"target",
"slot",
"action_at_ms",
"restore_at_ms",
"detection_s",
"bound_s",
"over_bound",
"lb_return_s",
"errors_before",
"errors_fail",
"errors_after",
"error_kinds",
"first_error_s",
"last_error_s",
"recovery_s",
"ok_rps_before",
"ok_rps_fail",
"ok_rps_after",
"p95_before_ms",
"p95_fail_ms",
"p95_after_ms",
"max_fail_ms",
"target_after_down",
"writes",
"retried",
"recovered",
"rows",
"distinct_texts",
"verified",
"lb_errors",
)
TIMELINE_COLUMNS = (
"phase",
"second",
"sent",
"ok_write",
"ok_read",
"ok_target",
"ok_other",
"ok_old",
"ok_new",
"errors",
"error_kinds",
"p95_ms",
"max_ms",
)
RESOURCE_COLUMNS = (
"phase",
"window",
"api_cpu_cores",
"api_cpu_max",
"api_throttled_max",
"api_memory_max_mib",
"db_cpu_cores",
"db_memory_mib",
"lb_cpu_cores",
"cache_cpu_cores",
"gen_cpu_cores",
"vm_busy_share",
"api_cpu_us_per_req",
"db_cpu_us_per_req",
)
def _per_req(cores: float | None, rps: float | None) -> float | None:
if not cores or not rps:
return None
return round(cores * 1e6 / rps, 1)
def start_events(path: Path, *filters: str) -> subprocess.Popen:
"""Поток событий Docker в файл: буфер демона хранит лишь последние
события всех контейнеров, поэтому — поток с начала фазы."""
handle = path.open("w", encoding="utf-8")
process = subprocess.Popen(
["docker", "events", *filters, "--format", "{{json .}}"],
stdout=handle,
stderr=subprocess.DEVNULL,
cwd=ROOT,
)
process.handle = handle # type: ignore[attr-defined]
return process
def stop_events(process: subprocess.Popen) -> None:
time.sleep(0.5)
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.handle.close() # type: ignore[attr-defined]
def lb_log_since(since_wall: float) -> str:
"""Журнал HAProxy стенда с отметками ВМ (`docker logs -t`)."""
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(since_wall - 5))
done = docker("logs", "-t", "--since", stamp, LB, check=False)
return done.stdout + done.stderr
def collect_requests(stand: Stand, logs: list[str]) -> str:
return "".join(stand.gen_file(path) for path in logs)
def verify_writes(stand: Stand, chat: str, prefix: str) -> tuple[int, int]:
"""Строк и разных текстов в чате записи с ключами фазы."""
row = stand.psql(
"SELECT count(*), count(DISTINCT text) FROM messages "
f"WHERE chat_id = '{chat}' AND text LIKE '%{prefix}.%';\n",
)
rows, distinct = (int(value) for value in row.split("|"))
return rows, distinct
def failover_phase(
stand: Stand,
profile: LoadProfile,
name: str,
mode: str,
target: str,
write_chat: str,
folder: Path,
) -> tuple[dict, list[dict]]:
stand.wait_lb_up(profile.limits.api_replicas)
address = stand.address(target)
slot = next(s["slot"] for s in stand.lb_slots() if s["addr"] == address)
events_path = stand.workdir / f"{name}-events.log"
events = start_events(events_path, "--filter", f"container={target}")
since = time.time()
duration = profile.before_s + profile.fail_s + profile.restore_s + profile.after_s
log = f"/tmp/{name}.log"
load = stand.gen_popen(
"constant",
"--fixture",
FIXTURE,
"--write-rate",
str(profile.write_rate),
"--read-rate",
str(profile.read_rate),
"--duration",
str(duration),
"--procs",
str(profile.gen_procs),
"--seed",
str(profile.seed),
"--key-prefix",
name,
"--log",
log,
)
try:
time.sleep(profile.before_s + 0.3)
if mode == "stop":
# как scripts/stop-one.sh: graceful, срок 9 с
docker("stop", "--timeout", "9", target)
elif mode == "kill":
docker("kill", target)
else:
docker("pause", target)
time.sleep(profile.fail_s)
docker("unpause" if mode == "pause" else "start", target)
stand.wait_slot(address, "UP", timeout=profile.restore_s + 20)
out, err = load.communicate(timeout=duration + 60)
finally:
if load.poll() is None:
load.kill()
stop_events(events)
if load.returncode != 0:
raise StandError(f"generator failed in {name}: {err.strip()[-500:]}")
summary = json.loads(out.strip().splitlines()[-1])
raw = collect_requests(stand, summary["logs"])
requests = analysis.parse_requests(raw)
retry = stand.gen("retry", "--fixture", FIXTURE, *summary["logs"])
rows, distinct = verify_writes(stand, write_chat, name)
lb_text = lb_log_since(since)
events_list = analysis.parse_events(events_path.read_text(encoding="utf-8"))
(folder / "raw" / f"{name}-events.log").write_text(
events_path.read_text(encoding="utf-8"), encoding="utf-8"
)
(folder / "raw" / f"{name}-lb.log").write_text(lb_text, encoding="utf-8")
with gzip.open(
folder / "raw" / f"{name}-requests.csv.gz", "wt", encoding="ascii"
) as fh:
fh.write(raw)
action_names = {
"stop": ("kill", "stop", "die"),
"kill": ("kill", "die"),
"pause": ("pause",),
}
action_at = next(t for t, a, _ in events_list if a in action_names[mode])
restore_at = next(
t
for t, a, _ in events_list
if a == ("unpause" if mode == "pause" else "start") and t > action_at
)
lb_events = [e for e in analysis.parse_lb_log(lb_text) if e[1] == slot]
down = next(
(
t
for t, _, s in lb_events
if t >= action_at and s in ("is DOWN", "is going DOWN")
),
None,
)
up = next((t for t, _, s in lb_events if t >= restore_at and s == "is UP"), None)
load_start = summary["start"] * 1000
load_end = load_start + duration * 1000
windows = {
"before": (load_start + 2000, action_at),
"fail": (action_at, restore_at),
"after": (up or restore_at, load_end),
}
stats = {w: analysis.window_stats(requests, s, e) for w, (s, e) in windows.items()}
errors_all = [r for r in requests if not r.ok and r.scheduled >= load_start]
bound = dict(profile.bounds)[mode]
detection = round((down - action_at) / 1000, 2) if down else None
target_id = target[:12]
target_after = sum(
1
for r in requests
if r.ok
and r.instance == target_id
and down
and down + 1000 <= r.scheduled < restore_at
)
phase = {
"phase": name,
"mode": mode,
"replica": name.rsplit("-", 1)[1],
"target": target_id,
"slot": slot,
"action_at_ms": round(action_at, 1),
"restore_at_ms": round(restore_at, 1),
"detection_s": detection,
"bound_s": bound,
"over_bound": bool(detection is None or detection > bound),
"lb_return_s": round((up - restore_at) / 1000, 2) if up else None,
"errors_before": stats["before"]["errors"],
"errors_fail": stats["fail"]["errors"],
"errors_after": stats["after"]["errors"],
"error_kinds": ";".join(
f"{k}:{v}"
for k, v in sorted(
{
kind: sum(analysis.error_label(r) == kind for r in errors_all)
for kind in {analysis.error_label(r) for r in errors_all}
}.items()
)
),
"first_error_s": round((errors_all[0].scheduled - action_at) / 1000, 2)
if errors_all
else None,
"last_error_s": round((errors_all[-1].scheduled - action_at) / 1000, 2)
if errors_all
else None,
"recovery_s": analysis.recovery_seconds(
requests, action_at, restore_at, profile.write_rate + profile.read_rate
),
"ok_rps_before": stats["before"]["ok_rps"],
"ok_rps_fail": stats["fail"]["ok_rps"],
"ok_rps_after": stats["after"]["ok_rps"],
"p95_before_ms": stats["before"]["p95_ms"],
"p95_fail_ms": stats["fail"]["p95_ms"],
"p95_after_ms": stats["after"]["p95_ms"],
"max_fail_ms": stats["fail"]["max_ms"],
"target_after_down": target_after,
"writes": retry["keys"],
"retried": retry["failed"],
"recovered": retry["recovered"],
"rows": rows,
"distinct_texts": distinct,
"verified": rows == distinct == retry["keys"],
"lb_errors": ";".join(
f"{k}:{v}" for k, v in sorted(analysis.lb_http_errors(lb_text).items())
),
"_windows": windows,
}
if not phase["verified"]:
raise StandError(
f"{name}: {retry['keys']} keys sent, {rows} rows, {distinct} distinct texts"
)
groups = {
"target": {target_id},
"other": {s[:12] for s in stand.api_ids()} - {target_id},
}
rows_timeline = [
{"phase": name, **row}
for row in analysis.timeline(requests, action_at, load_start, load_end, groups)
]
return phase, rows_timeline
def failover_summary(meta: dict, phases: list[dict], resources: list[dict]) -> str:
profile = meta["profile"]
lines = [
f"# Отказ реплики под нагрузкой: {meta['label']}",
"",
_passport_lines(meta),
"",
f"Нагрузка: открытая модель, {profile['write_rate']:.0f} записей с "
f"`Idempotency-Key` и {profile['read_rate']:.0f} чтений страницы истории "
"в секунду через HAProxy стенда; две реплики с лимитами рабочего стенда. "
f"Фаза: {profile['before_s']:.0f} с до действия, действие, "
f"{profile['fail_s']:.0f} с отказа, возврат той же реплики, ещё "
f"{profile['after_s']:.0f} с.",
"",
"Отметки — часы ВМ Docker: действие и возврат — события Docker, "
"исключение и возврат в LB — журнал HAProxy, запросы — генератор "
"(момент по расписанию). Обнаружение — от действия до `DOWN` слота; "
"восстановление — от действия до начала секунды, после которой до "
"возврата нет ошибок и успешных ≥ 90 % поданной нагрузки.",
"",
"| Фаза | Режим | Реплика | Обнаружение, с (граница) | Возврат в LB, с | Ошибки до / отказ / после | Виды ошибок | Восстановление, с | Успешных/с до / отказ / после | p95, мс до / отказ / после | max в отказе, мс | Повтор: не прошло → прошло | Сверка |",
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
]
for p in phases:
lines.append(
f"| {p['phase']} | {p['mode']} | {p['replica']} ({p['target']}) | "
f"{_v(p['detection_s'])} ({p['bound_s']}){' ⚠' if p['over_bound'] else ''} | "
f"{_v(p['lb_return_s'])} | {p['errors_before']} / {p['errors_fail']} / "
f"{p['errors_after']} | {p['error_kinds'] or '—'} | {_v(p['recovery_s'])} | "
f"{p['ok_rps_before']} / {p['ok_rps_fail']} / {p['ok_rps_after']} | "
f"{_v(p['p95_before_ms'])} / {_v(p['p95_fail_ms'])} / {_v(p['p95_after_ms'])} | "
f"{_v(p['max_fail_ms'])} | {p['retried']} → {p['recovered']} | "
f"{'ok' if p['verified'] else 'нет'}: {p['writes']} ключей = {p['rows']} строк |"
)
if resources:
lines += [
"",
"## CPU и память по окнам фазы",
"",
"Сэмплер (cgroup каждого контейнера раз в секунду). Окна: до "
"действия (от первого запроса + 2 с), отказ (действие → возврат), "
"после (возврат в LB → конец нагрузки). CPU реплик — сумма; на "
"успешный запрос — только по окнам с нагрузкой.",
"",
"| Фаза | Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | Valkey | Генератор | ВМ занята | API, мкс/запрос | PostgreSQL, мкс/запрос |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for r in resources:
lines.append(
f"| {r['phase']} | {r['window']} | {_v(r.get('api_cpu_cores'))} "
f"({_v(r.get('api_cpu_max'))}) | {_v(r.get('db_cpu_cores'))} | "
f"{_v(r.get('lb_cpu_cores'))} | {_v(r.get('cache_cpu_cores'))} | "
f"{_v(r.get('gen_cpu_cores'))} | {_v(r.get('vm_busy_share'))} | "
f"{_v(r.get('api_cpu_us_per_req'))} | {_v(r.get('db_cpu_us_per_req'))} |"
)
lines += _warning_lines(meta)
return "\n".join(lines) + "\n"
def _v(value) -> str:
if value is None or value == "":
return "—"
if isinstance(value, float):
return f"{value:g}"
return str(value)
def _passport_lines(meta: dict) -> str:
code = meta.get("code", {})
commit = (code.get("git_commit") or "")[:12] or "git нет"
dock = meta.get("docker", {})
host = meta.get("host", {})
return (
f"Код: src_sha256 `{code.get('src_sha256', '')[:16]}`, commit {commit}. "
f"Хост: {host.get('cpu_model')}, {host.get('ram_gb')} ГиБ; ВМ Docker "
f"{dock.get('NCPU')} CPU, {round((dock.get('MemTotal') or 0) / 2**30, 1)} ГиБ, "
f"Docker {dock.get('ServerVersion')}, compose {dock.get('compose_version')}. "
f"Начало {meta.get('started')}, конец {meta.get('finished')}."
)
def _warning_lines(meta: dict) -> list[str]:
lines = []
if meta.get("host_warnings"):
lines += ["", "## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
if meta.get("aborted"):
lines += ["", f"**Прогон прерван:** {meta['aborted']}"]
return lines
# --- rollout --------------------------------------------------------------------------
def run_rollout(profile: LoadProfile, label: str) -> Path:
workdir = temp_workdir()
folder = new_label_dir("rollout", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
# на пике — реплик на одну больше
pre = preflight(
profile.limits.total_cpus(profile.limits.api_replicas + 1),
profile.limits.total_memory_gib(profile.limits.api_replicas + 1),
)
old = prepare_image(label)
say("build the new image: working tree + expand revision 9001_rollout_probe")
context, head = probe_migration_context(workdir)
new_tag = f"{IMAGE_REPO}:bench-{label}-new"
build_image(new_tag, context)
new = image_info(new_tag)
docker("tag", old["tag"], SERVICE_TAG)
meta = {
"kind": "rollout",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": old,
"image_new": new,
"code": code_state(old),
"rollout": {"schema_head_before": head, "probe_revision": "9001"},
}
write_json(folder / "meta.json", meta)
stand = Stand(profile.limits, session_ttl=profile.session_ttl, workdir=workdir)
windows_rows, timeline_rows, resource_rows = [], [], []
try:
say("stand: postgres, valkey, migrate, 2 replicas on the old image, haproxy")
stand.start()
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
"1",
)
stand.maintain()
meta["setup"] = {k: v for k, v in setup.items() if k != "write_chats"}
meta["stand"] = stand.passport()
write_chat = setup["write_chats"][0]
before_ids = stand.api_ids()
roles = roles_of(stand)
rollout = meta["rollout"]
rollout["replicas_before"] = [c[:12] for c in before_ids]
rollout["images_before"] = sorted(
{
docker("inspect", "-f", "{{.Image}}", c).stdout.strip()
for c in before_ids
}
)
rollout["alembic_before"] = stand.psql(
"SELECT version_num FROM alembic_version;\n"
)
offset = clock_offset()
rollout["clock_offset_s"] = round(offset, 4)
stand.start_sampler("scan")
events_path = workdir / "rollout-events.log"
events = start_events(
events_path, "--filter", f"label=com.docker.compose.project={PROJECT}"
)
since = time.time()
duration = (
profile.rollout_before_s
+ profile.rollout_budget_s
+ profile.rollout_after_s
)
load = stand.gen_popen(
"constant",
"--fixture",
FIXTURE,
"--write-rate",
str(profile.write_rate),
"--read-rate",
str(profile.read_rate),
"--duration",
str(duration),
"--procs",
str(profile.gen_procs),
"--seed",
str(profile.seed),
"--key-prefix",
"rollout",
"--log",
"/tmp/rollout.log",
)
try:
time.sleep(profile.rollout_before_s + 0.3)
docker("tag", new_tag, SERVICE_TAG)
script_start = time.time() + offset
script = subprocess.run(
[str(ROOT / "scripts" / "rolling-update.sh")],
cwd=ROOT,
env={
**stand.script_env(),
"API_REPLICAS": str(profile.limits.api_replicas),
"ROLLOUT_BUILD": "0",
},
capture_output=True,
text=True,
timeout=profile.rollout_budget_s + 60,
check=False,
)
script_end = time.time() + offset
out, err = load.communicate(timeout=duration + 60)
finally:
if load.poll() is None:
load.kill()
stop_events(events)
script_lines = [
line
for line in (script.stdout + script.stderr).splitlines()
if "rolling-update:" in line
]
(folder / "raw" / "rollout-script.log").write_text(
"\n".join(script_lines) + "\n", encoding="utf-8"
)
rollout["script_exit"] = script.returncode
rollout["script_seconds"] = round(script_end - script_start, 1)
if load.returncode != 0:
raise StandError(f"generator failed: {err.strip()[-500:]}")
if script.returncode != 0:
raise StandError(
f"rolling-update.sh exit {script.returncode}:\n"
+ "\n".join(script_lines[-10:])
)
summary = json.loads(out.strip().splitlines()[-1])
raw = collect_requests(stand, summary["logs"])
requests = analysis.parse_requests(raw)
retry = stand.gen("retry", "--fixture", FIXTURE, *summary["logs"])
rows, distinct = verify_writes(stand, write_chat, "rollout")
lb_text = lb_log_since(since)
(folder / "raw" / "rollout-lb.log").write_text(lb_text, encoding="utf-8")
(folder / "raw" / "rollout-events.log").write_text(
events_path.read_text(encoding="utf-8"), encoding="utf-8"
)
with gzip.open(
folder / "raw" / "rollout-requests.csv.gz", "wt", encoding="ascii"
) as fh:
fh.write(raw)
after_ids = stand.api_ids()
rollout["replicas_after"] = [c[:12] for c in after_ids]
rollout["images_after"] = sorted(
{docker("inspect", "-f", "{{.Image}}", c).stdout.strip() for c in after_ids}
)
rollout["alembic_after"] = stand.psql(
"SELECT version_num FROM alembic_version;\n"
)
rollout["probe_column"] = (
stand.psql(
"SELECT count(*) FROM information_schema.columns "
"WHERE table_name = 'messages' AND column_name = 'rollout_probe';\n"
)
== "1"
)
stand.wait_lb_up(profile.limits.api_replicas, timeout=30)
events_list = analysis.parse_events(events_path.read_text(encoding="utf-8"))
old_ids = {c[:12] for c in before_ids}
new_ids = {c[:12] for c in after_ids}
rollout["checks"] = {
"replicas_after": len(after_ids) == profile.limits.api_replicas,
"no_old_replica": not (old_ids & new_ids),
# образ контейнера — digest конфигурации (хранилище containerd), не
# Id тега: сравниваются контейнеры до и после между собой
"single_new_image": len(rollout["images_after"]) == 1
and rollout["images_after"] != rollout["images_before"],
"migrated": rollout["alembic_after"] == "9001",
"probe_column": rollout["probe_column"],
"writes_verified": rows == distinct == retry["keys"],
}
load_start = summary["start"] * 1000
load_end = load_start + duration * 1000
windows = {
"before": (load_start + 2000, script_start * 1000),
"during": (script_start * 1000, script_end * 1000),
"after": (script_end * 1000, load_end),
}
for window, (start, end) in windows.items():
stats = analysis.window_stats(requests, start, end)
windows_rows.append(
{
"window": window,
**{k: v for k, v in stats.items() if k != "error_kinds"},
"error_kinds": ";".join(
f"{k}:{v}" for k, v in sorted(stats["error_kinds"].items())
),
"ok_old": sum(
r.ok and r.instance in old_ids
for r in requests
if start <= r.scheduled < end
),
"ok_new": sum(
r.ok and r.instance in new_ids
for r in requests
if start <= r.scheduled < end
),
},
)
container_events = [
{
"t_s": round((t - script_start * 1000) / 1000, 2),
"action": a,
"container": c[:12],
# остальные контейнеры проекта за замену — миграция
"role": "old"
if c[:12] in old_ids
else "new"
if c[:12] in new_ids
else "migrate",
}
for t, a, c in events_list
if a in ("create", "start", "kill", "die", "stop", "destroy")
]
rollout["container_events"] = container_events
rollout["retry"] = retry
rollout["rows"] = rows
rollout["distinct_texts"] = distinct
rollout["lb_errors"] = analysis.lb_http_errors(lb_text)
timeline_rows = [
{"phase": "rollout", **row}
for row in analysis.timeline(
requests,
script_start * 1000,
load_start,
load_end,
{"old": old_ids, "new": new_ids},
)
]
ticks = stand.sampler_lines()
meta["sampler_ticks"] = len(ticks)
for cid in new_ids:
roles[cid] = "api"
# созданные за замену контейнеры, кроме реплик, — контейнер миграции
for _, action, container in events_list:
if action == "create" and container[:12] not in roles:
roles[container[:12]] = "migrate"
for window, (start, end) in windows.items():
usage = resources_by_role(
analysis.window_ticks(ticks, start / 1000, end / 1000), roles
)
ok_rps = next(w["ok_rps"] for w in windows_rows if w["window"] == window)
resource_rows.append(
{
"phase": "rollout",
"window": window,
**usage,
"api_cpu_us_per_req": _per_req(usage.get("api_cpu_cores"), ok_rps),
"db_cpu_us_per_req": _per_req(usage.get("db_cpu_cores"), ok_rps),
},
)
if not all(rollout["checks"].values()):
raise StandError(f"rollout checks failed: {rollout['checks']}")
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
finally:
stand.stop_sampler()
stand.stop()
docker("rmi", f"{IMAGE_REPO}:bench-{label}-new", check=False)
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(
folder / "windows.csv",
(
"window",
"seconds",
"requests",
"ok",
"ok_rps",
"ok_write_rps",
"ok_read_rps",
"errors",
"error_kinds",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"ok_old",
"ok_new",
),
windows_rows,
)
write_csv(folder / "timeline.csv", TIMELINE_COLUMNS, timeline_rows)
write_csv(folder / "resources.csv", RESOURCE_COLUMNS, resource_rows)
(folder / "summary.md").write_text(
rollout_summary(meta, windows_rows, resource_rows), encoding="utf-8"
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def rollout_summary(meta: dict, windows: list[dict], resources: list[dict]) -> str:
profile, rollout = meta["profile"], meta.get("rollout", {})
lines = [
f"# Поочерёдная замена под нагрузкой: {meta['label']}",
"",
_passport_lines(meta),
"",
f"Нагрузка: {profile['write_rate']:.0f} записей с ключом и "
f"{profile['read_rate']:.0f} чтений в секунду через HAProxy стенда, "
"открытая модель. Две реплики на образе рабочего дерева; посреди "
"нагрузки `scripts/rolling-update.sh` (тот же, что `make rolling-update`) "
"меняет их на образ с expand-ревизией `9001_rollout_probe` (колонка "
"`messages.rollout_probe` NULL): миграция один раз, затем по одной "
"реплике — новая `UP` в LB, старая `drain` → `maint` → stop → rm.",
"",
f"- Скрипт: код выхода {rollout.get('script_exit')}, "
f"{rollout.get('script_seconds')} с; ревизия схемы "
f"{rollout.get('alembic_before')} → {rollout.get('alembic_after')}; колонка "
f"есть: {rollout.get('probe_column')}.",
f"- Реплики до: {', '.join(rollout.get('replicas_before', []))}; после: "
f"{', '.join(rollout.get('replicas_after', []))}.",
f"- Проверки: {rollout.get('checks')}.",
f"- Повтор неуспешных записей с тем же ключом: {rollout.get('retry')}; "
f"строк в чате записи {rollout.get('rows')}, разных текстов "
f"{rollout.get('distinct_texts')}.",
f"- Ответы самого HAProxy с ошибкой: {rollout.get('lb_errors') or 'нет'}.",
"",
"| Окно | с | Запросов | Успешных/с | Ошибок | Виды | p95, мс | max, мс | Ответов старых | Ответов новых |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for w in windows:
lines.append(
f"| {w['window']} | {w['seconds']} | {w['requests']} | {w['ok_rps']} | "
f"{w['errors']} | {w['error_kinds'] or '—'} | {_v(w['p95_ms'])} | "
f"{_v(w['max_ms'])} | {w['ok_old']} | {w['ok_new']} |"
)
lines += ["", "## Контейнеры (секунды от начала скрипта)", ""]
for event in rollout.get("container_events", []):
lines.append(
f"- {event['t_s']:+.2f} с {event['action']} {event['role']} {event['container']}"
)
if resources:
lines += [
"",
"## CPU и память по окнам",
"",
"| Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | ВМ занята | API, мкс/запрос |",
"|---|---|---|---|---|---|",
]
for r in resources:
lines.append(
f"| {r['window']} | {_v(r.get('api_cpu_cores'))} ({_v(r.get('api_cpu_max'))}) | "
f"{_v(r.get('db_cpu_cores'))} | {_v(r.get('lb_cpu_cores'))} | "
f"{_v(r.get('vm_busy_share'))} | {_v(r.get('api_cpu_us_per_req'))} |"
)
lines += _warning_lines(meta)
return "\n".join(lines) + "\n"
# --- scaling (трек 1) -----------------------------------------------------------------
def parse_config(name: str, base: Limits) -> Limits:
"""`4x1` — 4 реплики по 1 CPU; `-pg0.5` — PostgreSQL 0,5 CPU;
`-mc50` — max_connections 50. Память реплики — 512 МиБ на CPU."""
head, *options = name.split("-")
replicas, cpus = head.split("x")
limits = dataclasses.replace(
base,
api_replicas=int(replicas),
api_cpus=float(cpus),
api_memory=f"{int(512 * float(cpus))}m",
)
for option in options:
if option.startswith("pg"):
limits = dataclasses.replace(limits, db_cpus=float(option[2:]))
elif option.startswith("mc"):
limits = dataclasses.replace(limits, db_max_connections=int(option[2:]))
else:
raise SystemExit(f"unknown config option {option!r} in {name!r}")
return limits
def cell_plan(profile: ScalingProfile) -> list[dict]:
"""Ячейки по кругам: в каждом круге — все конфигурации в случайном
порядке (зерно профиля); одна конфигурация не идёт дважды подряд."""
plan, previous = [], None
for repeat in range(1, profile.repeats + 1):
order = list(profile.configs)
random.Random(profile.order_seed + repeat).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for config in order:
plan.append({"cell": len(plan) + 1, "config": config, "repeat": repeat})
previous = order[-1]
return plan
def run_scaling(profile: ScalingProfile, label: str, resume: bool) -> Path:
workdir = temp_workdir()
limits_all = {c: parse_config(c, profile.base) for c in profile.configs}
worst_cpu = max(limits.total_cpus() for limits in limits_all.values())
worst_mem = max(limits.total_memory_gib() for limits in limits_all.values())
folder = EVIDENCE / "scaling" / label
if resume:
if not folder.exists():
raise SystemExit(
f"nothing to resume: {folder.relative_to(ROOT)} is missing"
)
meta = json.loads((folder / "meta.json").read_text(encoding="utf-8"))
if meta["profile"] != json.loads(json.dumps(dataclasses.asdict(profile))):
raise SystemExit("resume: profile differs from the label passport")
pre = preflight(worst_cpu, worst_mem)
image = prepare_image(label)
if image["src_sha256"] != meta["image"]["src_sha256"]:
raise SystemExit(
"resume: code differs from the label passport (src_sha256)"
)
meta.setdefault("resumed", []).append(utc_now())
else:
folder = new_label_dir("scaling", label)
(folder / "raw").mkdir()
pre = preflight(worst_cpu, worst_mem)
image = prepare_image(label)
meta = {
"kind": "scaling",
"label": label,
"series": profile.series,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"configs": {
c: dataclasses.asdict(limits) for c, limits in limits_all.items()
},
"budget": {
c: {
"cpus": limits.total_cpus(),
"memory_gib": round(limits.total_memory_gib(), 2),
}
for c, limits in limits_all.items()
},
"vm": pre["budget"],
"host": pre["host"],
"docker": pre["docker"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
"thresholds": analysis.THRESHOLDS,
"plan": cell_plan(profile),
}
started_mono, started_wall = time.monotonic(), time.time()
write_json(folder / "meta.json", meta)
done_cells = {int(row["cell"]) for row in read_csv(folder / "searches.csv")}
pool = image["app_defaults"]
pool_cap = pool["pool_size"] + pool["pool_overflow"]
try:
for cell in meta["plan"]:
if cell["cell"] in done_cells:
continue
say(
f"cell {cell['cell']}/{len(meta['plan'])}: {cell['config']} "
f"repeat {cell['repeat']}"
)
scaling_cell(
profile, limits_all[cell["config"]], cell, folder, workdir, pool_cap
)
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']} (continue: --resume)")
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
searches = read_csv(folder / "searches.csv")
runs = read_csv(folder / "runs.csv")
(folder / "summary.md").write_text(
scaling_summary(meta, searches, runs), encoding="utf-8"
)
if searches:
plots.render_label(folder)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def scaling_cell(
profile: ScalingProfile,
limits: Limits,
cell: dict,
folder: Path,
workdir: Path,
pool_cap: int,
) -> None:
started = time.monotonic()
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
limit_dict = dataclasses.asdict(limits)
try:
stand.start()
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
str(profile.write_chats),
"--fill-clients",
str(profile.fill_clients),
)
stand.maintain()
if cell["cell"] == 1 or not (folder / "stand.json").exists():
write_json(folder / "stand.json", stand.passport())
roles = roles_of(stand)
replica_addrs = {stand.address(c) for c in stand.api_ids()}
stand.start_sampler("list", full_ids(stand))
search = analysis.Search(
fixed=profile.fixed_rates,
base=profile.ladder_base,
factor=profile.ladder_factor,
start=profile.start_per_replica * limits.api_replicas,
bisections=profile.bisections,
cap=profile.rate_cap,
)
previous_rate, fresh = 0.0, True
step_no = 0
rows: list[dict] = []
while (step := search.next_rate()) is not None:
rate, phase = step
step_no += 1
warmup = profile.warmup_first_s if fresh else profile.warmup_ramp_s
rate_from = 0.0 if fresh else previous_rate
result = stand.gen(
"step",
"--fixture",
FIXTURE,
"--rate",
str(rate),
"--rate-from",
str(rate_from),
"--warmup",
str(warmup),
"--window",
str(profile.window_s),
"--mix",
profile.mix,
"--procs",
str(profile.gen_procs),
"--seed",
str(profile.order_seed + cell["cell"] * 100 + step_no),
"--timeout",
str(profile.request_timeout_s),
"--key-prefix",
f"c{cell['cell']}s{step_no}",
"--abort-error-share",
str(profile.abort_error_share),
"--abort-p95-ms",
str(profile.abort_p95_ms),
timeout=profile.warmup_first_s + profile.window_s + 120,
)
ticks = analysis.window_ticks(
stand.sampler_lines(), result["window_start"], result["window_end"]
)
row = step_row(
profile, cell, step_no, phase, result, ticks, roles, replica_addrs
)
row["slo_pass"] = bool(
not result["abort"]
and result["p95_ms"] is not None
and result["p95_ms"] <= profile.slo_p95_ms
and result["fail_share"] <= profile.slo_fail_share
)
row["signals"] = ";".join(analysis.signals(row, limit_dict, pool_cap))
append_csv(folder / "runs.csv", RUN_COLUMNS, row)
for sample in step_samples(
cell, step_no, rate, ticks, roles, result["window_start"]
):
append_csv(folder / "samples.csv", SAMPLE_COLUMNS, sample)
(folder / "raw" / f"c{cell['cell']:02d}-s{step_no:02d}.json").write_text(
json.dumps(result, ensure_ascii=False), encoding="utf-8"
)
rows.append(row)
say(
f" step {step_no} {phase} {rate:.0f}/s: p95 {result['p95_ms']} ms, "
f"fail {result['fail_share']:.4f}, {'pass' if row['slo_pass'] else 'FAIL'} "
f"[{row['signals']}]"
)
search.record(rate, row["slo_pass"], phase)
previous_rate = rate
fresh = not row["slo_pass"]
if fresh:
time.sleep(profile.pause_after_fail_s)
stand.gen("ready", "--timeout", "60")
r_star = search.r_star()
first_fail = search.first_fail()
fail_row = next(
(
r
for r in rows
if r["phase"] != "fixed" and float(r["rate"]) == first_fail
),
None,
)
star_row = next(
(r for r in rows if r["phase"] != "fixed" and float(r["rate"]) == r_star),
None,
)
flags = []
if fail_row and "generator" in fail_row["signals"].split(";"):
flags.append("generator_bound")
if r_star is not None and r_star >= profile.rate_cap:
flags.append("cap")
down = [name for name in (DB, CACHE, LB) if not _running(name)]
if down:
flags.append("stand_down:" + ",".join(down))
append_csv(
folder / "searches.csv",
SEARCH_COLUMNS,
{
"series": profile.series,
"config": cell["config"],
"replicas": limits.api_replicas,
"api_cpus": limits.api_cpus,
"db_cpus": limits.db_cpus,
"repeat": cell["repeat"],
"cell": cell["cell"],
"r_star": r_star,
"first_fail": first_fail,
"p95_at_r_star": star_row["p95_ms"] if star_row else None,
"signals_at_fail": fail_row["signals"] if fail_row else "",
"generator_bound": "generator_bound" in flags,
"steps": step_no,
"fill_rps": setup["fill_rps"],
"duration_s": round(time.monotonic() - started, 1),
"flags": ";".join(flags),
},
)
say(f" R* = {r_star} (first fail {first_fail})")
finally:
stand.stop_sampler()
stand.stop()
def _running(name: str) -> bool:
done = docker(
"inspect",
"-f",
"{{.State.Running}} {{.RestartCount}} {{.State.OOMKilled}}",
name,
check=False,
)
return done.stdout.strip() == "true 0 false"
def step_row(
profile: ScalingProfile,
cell: dict,
step_no: int,
phase: str,
result: dict,
ticks: list[dict],
roles: dict[str, str],
replica_addrs: set[str],
) -> dict:
usage = resources_by_role(ticks, roles)
pg = analysis.pg_stats(ticks, replica_addrs)
instances = result.get("instances") or {}
total = sum(instances.values())
row = {
"series": profile.series,
"config": cell["config"],
"repeat": cell["repeat"],
"cell": cell["cell"],
"step": step_no,
"phase": phase,
**{
key: result.get(key)
for key in (
"rate",
"rate_from",
"warmup_s",
"window_s",
"scheduled",
"ok",
"errors",
"short_pages",
"dropped",
"unsent",
"fail_share",
"achieved_rps",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"service_p95_ms",
"lag_p99_ms",
"abort",
)
},
**usage,
**pg,
"lb_retries": analysis.lb_delta(ticks, "wretr"),
"lb_redispatches": analysis.lb_delta(ticks, "wredis"),
"lb_conn_errors": analysis.lb_delta(ticks, "econ"),
"instances": len(instances),
"instance_share_min": round(min(instances.values()) / total, 3)
if total
else None,
"error_kinds": ";".join(
f"{k}:{v}" for k, v in sorted((result.get("error_kinds") or {}).items())
),
}
if row.get("gen_cpu_cores") is None:
# нет тиков cgroup генератора: его собственное время процессов
row["gen_cpu_cores"] = round(
result["gen_cpu_s"] / (result["warmup_s"] + result["window_s"]), 3
)
row["api_cpu_us_per_req"] = _per_req(
row.get("api_cpu_cores"), result.get("achieved_rps")
)
row["db_cpu_us_per_req"] = _per_req(
row.get("db_cpu_cores"), result.get("achieved_rps")
)
return row
def step_samples(
cell: dict,
step_no: int,
rate: float,
ticks: list[dict],
roles: dict[str, str],
window_start: float,
) -> list[dict]:
"""Посекундные интервалы окна: CPU, троттлинг, память каждого контейнера."""
out = []
for a, b in zip(ticks, ticks[1:], strict=False):
seconds = b["ts"] - a["ts"]
if seconds <= 0:
continue
for cid, role in roles.items():
x, y = a["cg"].get(cid), b["cg"].get(cid)
if not x or not y or y[0] < x[0]:
continue
periods = y[1] - x[1]
out.append(
{
"config": cell["config"],
"repeat": cell["repeat"],
"step": step_no,
"rate": rate,
"t": round(b["ts"] - window_start, 2),
"role": role,
"container": cid,
"cpu_cores": round((y[0] - x[0]) / seconds / 1e6, 3),
"throttled_share": round((y[2] - x[2]) / periods, 3)
if periods
else 0.0,
"memory_mib": round(y[4] / 2**20, 1),
}
)
return out
def _f(value) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None
def scaling_summary(meta: dict, searches: list[dict], runs: list[dict]) -> str:
profile = meta["profile"]
lines = [
f"# Серия трека 1 «{meta['series']}»: {meta['label']}",
"",
_passport_lines(meta),
"",
f"Открытая модель через HAProxy стенда, смесь read/write/me "
f"{profile['mix']} (страница истории читаемого чата из "
f"{profile['read_messages']} сообщений, отправка с новым ключом в один из "
f"{profile['write_chats']} чатов, профиль). Ступень: прогрев "
f"{profile['warmup_first_s']:g} с с нуля или {profile['warmup_ramp_s']:g} с "
f"подъёма, окно {profile['window_s']:g} с. Фиксированные уровни "
f"{', '.join(f'{r:g}' for r in profile['fixed_rates'])}/с, затем поиск R*: "
f"лестница {profile['ladder_base']:g} × {profile['ladder_factor']:g}^i от "
f"{profile['start_per_replica']:g} × реплик, {profile['bisections']} деления. "
f"SLO: p95 успешных ≤ {profile['slo_p95_ms']:g} мс и (ошибки + неполные "
f"страницы + недосланные) ≤ {profile['slo_fail_share'] * 100:g} %. "
f"Ячейка — конфигурация × повтор ({profile['repeats']}), свежий стенд.",
"",
"| Конфигурация | Реплик × CPU | PostgreSQL CPU | R*, медиана (min–max) | Эффективность к 1 × 1 | p95 на "
+ " / ".join(f"{r:g}" for r in profile["fixed_rates"])
+ " /с, мс | API мкс/запрос на "
+ f"{profile['fixed_rates'][-1]:g}/с | PostgreSQL мкс/запрос | Признаки на первой нарушившей |",
"|---|---|---|---|---|---|---|---|---|",
]
by_config: dict[str, list[dict]] = {}
for row in searches:
by_config.setdefault(row["config"], []).append(row)
base = by_config.get("1x1")
base_r = statistics.median(_f(r["r_star"]) or 0 for r in base) if base else None
for config in profile["configs"]:
rows = by_config.get(config, [])
if not rows:
lines.append(f"| {config} | — | — | не снята | — | — | — | — | — |")
continue
values = [_f(r["r_star"]) or 0 for r in rows]
median = statistics.median(values)
replicas = int(rows[0]["replicas"])
efficiency = (
f"{median / (replicas * base_r):.2f}" if base_r and config != "1x1" else "—"
)
fixed = []
for rate in profile["fixed_rates"]:
p95 = [
_f(r["p95_ms"])
for r in runs
if r["config"] == config
and r["phase"] == "fixed"
and _f(r["rate"]) == rate
]
p95 = [v for v in p95 if v is not None]
fixed.append(f"{statistics.median(p95):g}" if p95 else "—")
last_fixed = [
r
for r in runs
if r["config"] == config
and r["phase"] == "fixed"
and _f(r["rate"]) == profile["fixed_rates"][-1]
]
api_us = [
_f(r["api_cpu_us_per_req"])
for r in last_fixed
if _f(r["api_cpu_us_per_req"])
]
db_us = [
_f(r["db_cpu_us_per_req"]) for r in last_fixed if _f(r["db_cpu_us_per_req"])
]
signals: dict[str, int] = {}
for r in rows:
for s in (r["signals_at_fail"] or "").split(";"):
if s:
signals[s] = signals.get(s, 0) + 1
flags = sorted({f for r in rows for f in (r["flags"] or "").split(";") if f})
api_text = f"{statistics.median(api_us):g}" if api_us else "—"
db_text = f"{statistics.median(db_us):g}" if db_us else "—"
signal_text = ", ".join(f"{k} ×{v}" for k, v in sorted(signals.items())) or "—"
if flags:
signal_text += f" ⚠ {', '.join(flags)}"
lines.append(
f"| {config} | {replicas} × {_f(rows[0]['api_cpus']):g} | "
f"{_f(rows[0]['db_cpus']):g} | {median:g} ({min(values):g}–{max(values):g}) | "
f"{efficiency} | {' / '.join(fixed)} | {api_text} | {db_text} | {signal_text} |"
)
lines += [
"",
"Признаки (пороги — `THRESHOLDS` в `bench/analysis.py`, заданы до замеров): "
"`app` — CPU реплики у лимита или троттлинг; `db_cpu` — CPU PostgreSQL у "
"лимита; `db_lock` — ожидание блокировок; `pool` — соединений реплики у "
"предела пула; `churn` — переоткрытие соединений (новые сессии PostgreSQL "
"за окно); `lb` — CPU HAProxy; `host` — ВМ Docker занята; `generator` — "
"генератор у своего лимита или отстаёт от расписания (R* — нижняя граница).",
"",
"Файлы: `runs.csv` — строка на ступень, `searches.csv` — строка на ячейку, "
"`samples.csv` — посекундно по контейнерам в окнах ступеней, "
"`p95-by-rate.svg`, `cpu-by-component.svg`, `raw/` — итоги генератора.",
]
lines += _warning_lines(meta)
return "\n".join(lines) + "\n"
# --- uploads (лаба 4) -----------------------------------------------------------------
def variant_override(path: Path, env: tuple[tuple[str, str], ...]) -> Path:
"""Override окружения реплик варианта: третий `-f` у compose стенда.
Окружение варианта — не `BENCH_*` основного override: тогда у прочих
профилей окружение реплик (`stand.app_env`) не меняется.
"""
lines = [
"# окружение реплик варианта профиля uploads (bench/run.py)",
"services:",
" api:",
" environment:",
*(f' {name}: "{value}"' for name, value in env),
]
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
return path
def wait_process_ready(container: str, timeout: float = 60) -> None:
"""Процесс реплики сам отвечает на /health/ready (команда HEALTHCHECK
образа через `docker exec`): новый контейнер может получить прежний IP,
и слот LB ещё `UP` от прежнего."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
done = docker("exec", container, "python", "-m", "src.healthcheck", check=False)
if done.returncode == 0:
return
time.sleep(0.3)
raise StandError(f"replica {container[:12]} is not ready in {timeout:.0f}s")
def container_state(container: str) -> dict:
raw = docker("inspect", "-f", "{{json .State}}", container, check=False)
if raw.returncode != 0:
return {}
state = json.loads(raw.stdout)
return {
"oom_killed": state.get("OOMKilled"),
"exit_code": state.get("ExitCode"),
"running": state.get("Running"),
}
def full_id(name: str) -> str:
return docker("inspect", "-f", "{{.Id}}", name).stdout.strip()
def run_uploads(profile: UploadsProfile, label: str) -> Path:
workdir = temp_workdir()
folder = new_label_dir("uploads", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
limits = profile.limits()
pre = preflight(limits.total_cpus(), limits.total_memory_gib())
image = prepare_image(label)
plan = uploads.plan_runs(
profile.variants,
[name for name, _ in profile.sizes],
profile.clients,
profile.repeats,
profile.seed,
)
meta = {
"kind": "uploads",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
"plan": plan,
}
write_json(folder / "meta.json", meta)
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
override = variant_override(
workdir / "variant-unlimited.compose.yml", profile.unlimited_env
)
rows: list[dict] = []
try:
say("stand: postgres, valkey, s3, migrate, 1 replica, haproxy")
stand.start(with_s3=True)
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
"1",
"--read-messages",
"0",
"--write-chats",
"1",
)
meta["setup"] = {
k: v for k, v in setup.items() if k not in ("write_chats", "owner")
}
meta["bodies"] = {}
for name, size in profile.sizes:
body = stand.gen(
"body",
"--size",
str(size),
"--seed",
str(profile.seed),
"--out",
f"/tmp/body-{name}.png",
)
meta["bodies"][name] = {
key: body[key] for key in ("bytes", "width", "height", "sha256")
}
meta["stand"] = stand.passport()
meta["app_env"] = {}
write_json(folder / "meta.json", meta)
for item in plan:
say(f"run {item['order']}/{len(plan)} {item['run']}")
row = uploads_run(stand, profile, item, meta, folder, override)
rows.append(row)
append_csv(folder / "runs.csv", uploads.RUN_COLUMNS, row)
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
finally:
stand.stop_sampler()
stand.extra_files = []
stand.stop()
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(folder / "runs.csv", uploads.RUN_COLUMNS, rows)
(folder / "summary.md").write_text(
uploads.render_summary(meta, rows), encoding="utf-8"
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def uploads_run(
stand: Stand,
profile: UploadsProfile,
item: dict,
meta: dict,
folder: Path,
override: Path,
) -> dict:
"""Прогон: новый S3 на пустом volume, новая реплика с окружением
варианта, разогревающая загрузка, сброс пика, C клиентов окна, пик."""
stand.reset_s3()
stand.extra_files = [override] if item["variant"] == "unlimited" else []
stand.compose("up", "-d", "--no-deps", "--no-build", "--force-recreate", "api")
api = stand.api_ids()[0]
wait_process_ready(api)
# слот LB по адресу новой реплики: прежний мог ещё числиться UP
if stand.wait_slot(stand.address(api), "UP", timeout=30) is None:
raise StandError(f"{item['run']}: LB does not see the new replica UP")
if not stand.gen("ready", "--timeout", "60")["ready"]:
raise StandError("API is not ready through the LB:\n" + stand.app_tail())
body = f"/tmp/body-{item['size']}.png"
# разогрев: первое соединение S3 и первые страницы кода — до сброса пика
warm = stand.gen(
"uploads",
"--fixture",
FIXTURE,
"--body",
body,
"--clients",
"1",
"--warmup",
"0",
"--window",
"0.5",
)
if not warm["ok"]:
raise StandError(f"{item['run']}: warm-up upload failed: {warm['statuses']}")
if item["variant"] not in meta["app_env"]:
env = json.loads(docker("inspect", "-f", "{{json .Config.Env}}", api).stdout)
meta["app_env"][item["variant"]] = dict(
sorted(e.split("=", 1) for e in env if e.startswith("UPLOAD_"))
)
stand.start_peak_sampler(
{"api": api, "s3": full_id(S3), "lb": full_id(LB)}, profile.sample_interval_s
)
swap_before = swapouts()
try:
result = stand.gen(
"uploads",
"--fixture",
FIXTURE,
"--body",
body,
"--clients",
str(item["clients"]),
"--warmup",
str(profile.warmup_s),
"--window",
str(profile.window_s),
"--timeout",
str(profile.upload_timeout_s),
timeout=profile.warmup_s + profile.window_s + profile.upload_timeout_s + 60,
)
finally:
lines = stand.stop_peak_sampler()
swap_after = swapouts()
state = container_state(api)
(folder / "raw" / f"{item['run']}-gen.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
(folder / "raw" / f"{item['run']}-peak.jsonl").write_text(
"".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8"
)
delta = (
swap_after - swap_before
if swap_before is not None and swap_after is not None
else None
)
row = uploads.run_row(
item,
body_bytes=meta["bodies"][item["size"]]["bytes"],
peaks=uploads.parse_peak(lines),
result=result,
state=state,
swapouts=delta,
)
say(
f" peak {row['peak_mib']} MiB, accepted {row['ok']}, 429 {row['rejected_429']}, "
f"5xx {row['errors_5xx']}, {row['uploads_per_s']} uploads/s {row['flags']}"
)
return row
# --- reconcile (лаба 4, трек 2) -------------------------------------------------------
def fill_stage(
stand: Stand,
profile: ReconcileProfile,
*,
start: int,
end: int,
size: int,
missing: int,
chat: str,
owner: str,
) -> dict:
"""Набор мимо API: строки `COPY` в PostgreSQL стенда и объекты PUT
загрузчиком на образе стенда. Номера [start, end) — классы по модели;
`missing` > 0 — все объекты согласованы, плюс столько строк без
объекта (номера [end, end + missing))."""
created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 86400))
# пространство ключей S3 приложения — имя базы (src/s3.py)
namespace = bench_env("POSTGRES_DB")
force = reconcile.FORCE_OK if missing else None
common = {
"seed": profile.seed,
"size": size,
"namespace": namespace,
"chat_id": chat,
"owner_id": owner,
"created_at": created,
}
data = reconcile.rows_csv(start=start, end=end, force=force, **common)
if missing:
data += reconcile.rows_csv(
start=end, end=end + missing, force=reconcile.FORCE_MISSING, **common
)
started = time.monotonic()
columns = ", ".join(reconcile.COPY_COLUMNS)
stand.psql(
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv);\n{data}\\.\n"
)
copy_s = time.monotonic() - started
# ключи пользователя API — окружение docker run без значения в argv
keys = {
"S3_ACCESS_KEY_ID": stand.env["S3_API_ACCESS_KEY"],
"S3_SECRET_ACCESS_KEY": stand.env["S3_API_SECRET_KEY"],
}
sh(
[
"docker",
"run",
"-d",
"--name",
FILL,
"--network",
NET,
"--no-healthcheck",
"--cpus",
str(profile.fill_cpus),
"--memory",
profile.fill_memory,
"--memory-swap",
profile.fill_memory,
"-e",
"S3_ACCESS_KEY_ID",
"-e",
"S3_SECRET_ACCESS_KEY",
"--entrypoint",
"python",
SERVICE_TAG,
"-c",
"import time; time.sleep(86400)",
],
env={**stand.env, **keys},
)
try:
for name in ("reconcile.py", "fill_objects.py"):
docker("cp", str(BENCH / name), f"{FILL}:/tmp/{name}")
args = [
"--endpoint",
"http://s3:8333",
"--bucket",
bench_env("S3_BUCKET"),
"--namespace",
namespace,
"--seed",
str(profile.seed),
"--start",
str(start),
"--end",
str(end),
"--size",
str(size),
"--processes",
str(profile.fill_processes),
"--threads",
str(profile.fill_threads),
]
if force:
args += ["--force", force]
done = docker(
"exec", FILL, "python", "/tmp/fill_objects.py", *args, timeout=3600
)
finally:
docker("rm", "-f", FILL, check=False)
put = json.loads(done.stdout.strip().splitlines()[-1])
return {
"start": start,
"end": end,
"missing": missing,
"rows_copy_s": round(copy_s, 2),
**put,
"last_put_at": time.time(),
}
def run_reconcile(profile: ReconcileProfile, label: str) -> Path:
workdir = temp_workdir()
folder = new_label_dir("reconcile", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
limits = profile.limits()
# на пике: уборщик управляемого опыта или загрузчик, не оба сразу
peak_extra = max(profile.fast_cpus, profile.fill_cpus) - limits.cleaner_cpus
pre = preflight(
limits.total_cpus() + peak_extra,
limits.total_memory_gib() + gib(profile.fill_memory),
)
image = prepare_image(label)
plan = reconcile.plan_passes(
[(stage[0], stage[4]) for stage in profile.stages],
profile.repeats,
profile.seed,
)
stages = {
stage[0]: {"n": stage[1], "size": stage[2], "missing": stage[5]}
for stage in profile.stages
}
models = {
stage[0]: (
reconcile.model(0, clean=stage[1], missing=stage[5])
if stage[5]
else reconcile.model(stage[1])
)
for stage in profile.stages
}
default_rate = image["app_defaults"]["cleaner"]["reconcile_rate"]
meta = {
"kind": "reconcile",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
"reconcile": {
"plan": plan,
"models": {k: dataclasses.asdict(m) for k, m in models.items()},
"default_rate": default_rate,
"fills": [],
},
}
write_json(folder / "meta.json", meta)
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
passes: list[dict] = []
windows: list[dict] = []
number = 0
try:
setup: dict = {}
filled = 0
for name, n, size, fresh, cells, missing in profile.stages:
if fresh:
if setup:
stand.stop_sampler()
stand.stop()
say(f"stage {name}: fresh stand with s3, 2 replicas")
stand.start(with_s3=True)
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
"200",
"--write-chats",
"1",
)
if "stand" not in meta:
meta["stand"] = stand.passport()
stand.start_sampler("scan", interval=profile.sample_interval_s)
filled = 0
say(f"stage {name}: fill [{filled}, {n}) x {size} B, missing {missing}")
fill = fill_stage(
stand,
profile,
start=filled,
end=n,
size=size,
missing=missing,
chat=setup["read_chat"],
owner=setup["owner"],
)
meta["reconcile"]["fills"].append({"stage": name, **fill})
say(f" {fill['objects']} objects, {fill['put_per_s']} put/s")
filled = n
stand.maintain()
if any(reconcile.CELLS[c].get("load") for c in cells):
ids = reconcile.downloads(profile.seed, n, profile.load_downloads)
path = stand.workdir / "downloads.json"
path.write_text(json.dumps(ids), encoding="ascii")
docker("cp", str(path), f"{GEN}:/tmp/downloads.json")
# объекты этапа старше grace: иначе они «свежие» и не кандидаты
wait = fill["last_put_at"] + profile.grace_s + 2 - time.time()
if wait > 0:
time.sleep(wait)
write_json(folder / "meta.json", meta)
for item in [p for p in plan if p["stage"] == name]:
number += 1
say(f"pass {item['order']}/{len(plan)} {item['pass']}")
row, rows = reconcile_pass(
stand,
profile,
item,
n=n,
size=size,
expected=models[name],
default_rate=default_rate,
number=number,
folder=folder,
)
passes.append(row)
windows += rows
append_csv(folder / "passes.csv", reconcile.PASS_COLUMNS, row)
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
finally:
stand.stop_sampler()
stand.stop()
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(folder / "passes.csv", reconcile.PASS_COLUMNS, passes)
write_csv(folder / "windows.csv", reconcile.WINDOW_COLUMNS, windows)
verdicts = [
reconcile.verdict_h1(stages, passes),
reconcile.verdict_h2(stages, passes),
reconcile.verdict_h3(stages, passes, models),
reconcile.verdict_h4(passes, windows),
reconcile.verdict_h5(passes),
]
meta_profile = {
**meta["profile"],
"default_rate": default_rate,
"page_size": image["app_defaults"]["cleaner"]["reconcile_page"],
}
(folder / "summary.md").write_text(
reconcile.render_summary(
{**meta, "profile": meta_profile}, passes, windows, verdicts
),
encoding="utf-8",
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def reconcile_pass(
stand: Stand,
profile: ReconcileProfile,
item: dict,
*,
n: int,
size: int,
expected: reconcile.Model,
default_rate: float,
number: int,
folder: Path,
) -> tuple[dict, list[dict]]:
"""Разовый проход уборщика `docker compose run` с лимитом CPU ячейки;
у ячеек load-* — под фоновой нагрузкой с тихими окнами до и после."""
cell = reconcile.CELLS[item["cell"]]
cpus = profile.fast_cpus if cell["cpu"] == "fast" else profile.cleaner_cpus
rate = default_rate if cell["rate"] == "default" else float(cell["rate"])
args = ["reconcile", "--strategy", cell["strategy"]]
if cell.get("deep"):
args.append("--deep")
if cell["rate"] != "default":
args += ["--rate", f"{rate:g}"]
if cell.get("apply"):
args += ["--apply", "--max-changes", str(profile.refuse_max_changes)]
name = f"{RECONCILE_PREFIX}{number}"
load = None
if cell.get("load"):
docker("exec", GEN, "rm", "-f", "/tmp/load.stop")
load = stand.gen_popen(
"constant",
"--fixture",
FIXTURE,
"--write-rate",
str(profile.load_write_rate),
"--read-rate",
str(profile.load_read_rate),
"--download-rate",
str(profile.load_download_rate),
"--downloads",
"/tmp/downloads.json",
"--duration",
str(2 * profile.quiet_s + profile.pass_budget_s),
"--procs",
str(profile.load_procs),
"--seed",
str(profile.seed),
"--key-prefix",
item["pass"],
"--log",
f"/tmp/{item['pass']}.log",
"--stop-file",
"/tmp/load.stop",
)
time.sleep(profile.quiet_s + 0.3)
try:
stand.compose(
"run",
"-d",
"--name",
name,
"--no-deps",
"-e",
f"CLEANER_GRACE={profile.grace_s}",
"cleaner",
*args,
env={"BENCH_CLEANER_CPUS": str(cpus)},
)
waited = docker("wait", name, timeout=profile.pass_budget_s + 60)
exit_code = int(waited.stdout.strip())
logs = docker("logs", name, check=False)
log_text = logs.stdout + logs.stderr
nano = docker("inspect", "-f", "{{.HostConfig.NanoCpus}}", name).stdout
finally:
docker("rm", "-f", name, check=False)
(folder / "raw" / f"{item['pass']}-cleaner.log").write_text(
log_text, encoding="utf-8"
)
fields = reconcile.parse_finished(log_text)
window_rows: list[dict] = []
load_flags: list[str] = []
if load is not None:
try:
time.sleep(profile.quiet_s)
docker("exec", GEN, "touch", "/tmp/load.stop")
out, err = load.communicate(timeout=profile.pass_budget_s + 120)
finally:
if load.poll() is None:
load.kill()
if load.returncode != 0:
raise StandError(f"generator failed in {item['pass']}: {err[-500:]}")
summary = json.loads(out.strip().splitlines()[-1])
if summary["dropped"]:
load_flags.append("bg_dropped")
raw = collect_requests(stand, summary["logs"])
with gzip.open(
folder / "raw" / f"{item['pass']}-requests.csv.gz", "wt", encoding="ascii"
) as fh:
fh.write(raw)
requests = analysis.parse_requests(raw)
if fields and requests:
end = fields["ts"] * 1000
begin = end - (fields["took_ms"] or 0)
bounds = {
"before": (summary["start"] * 1000 + 2000, begin),
"pass": (begin, end),
"after": (end, requests[-1].scheduled + 1),
}
window_rows = reconcile.window_rows(
item, requests, bounds, analysis.window_stats
)
if not fields:
raise StandError(
f"{item['pass']}: no 'reconcile pass finished' line, exit {exit_code}:\n"
+ "\n".join(log_text.splitlines()[-15:])
)
end = fields["ts"]
begin = end - (fields["took_ms"] or 0) / 1000
ticks = analysis.window_ticks(stand.sampler_lines(), begin, end)
usage = {}
for role, ids in (
("s3", [full_id(S3)]),
("db", [full_id(DB)]),
("api", stand.api_ids()),
):
values = [analysis.cgroup_usage(ticks, cid) for cid in ids]
values = [v for v in values if v]
if values:
usage[f"{role}_cpu_cores"] = round(sum(v["cpu_cores"] for v in values), 3)
problems = reconcile.found_problems(fields, expected, deep=bool(cell.get("deep")))
row = reconcile.pass_row(
item,
n=n,
size=size,
cell=cell,
cleaner_cpus=cpus,
rate=rate,
fields=fields,
exit_code=exit_code,
nano_cpus=int(nano.strip() or 0) or None,
usage=usage,
problems=problems,
extra_flags=load_flags,
)
say(
f" {row['took_ms']} ms, cpu {row['cpu_ms']} ms, s3 list/head/get "
f"{row['s3_list']}/{row['s3_head']}/{row['s3_get']}, db {row['db_queries']}"
+ (f", flags {row['flags']}" if row["flags"] else "")
)
if problems:
raise StandError(f"{item['pass']}: found differs from the model: {problems}")
return row, window_rows
# --- queue (лаба 5, трек 1) ------------------------------------------------------------
class TimelinePoller(threading.Thread):
"""Шкала ячейки: состояние заданий одним SELECT раз в интервал (часы
PostgreSQL) с меткой текущей фазы. Отдельный поток: основной в это
время ждёт генератор и worker."""
def __init__(self, stand: Stand, interval: float) -> None:
super().__init__(daemon=True)
self.stand = stand
self.interval = interval
self.phase = ""
self.samples: list[dict] = []
self.failures = 0
# не `_stop`: так называется внутренний метод threading.Thread
self._halt = threading.Event()
def run(self) -> None:
next_tick = time.monotonic()
while not self._halt.is_set():
phase = self.phase
try:
sample = queue.parse_sample(self.stand.psql(queue.TIMELINE_SQL))
except StandError:
sample = None
if sample is None:
self.failures += 1
else:
self.samples.append({**sample, "phase": phase})
next_tick += self.interval
self._halt.wait(max(0.0, next_tick - time.monotonic()))
def finish(self) -> list[dict]:
self._halt.set()
self.join(timeout=10)
return self.samples
def server_now(stand: Stand) -> float:
"""Часы PostgreSQL стенда (ВМ Docker): отметки фаз в тех же часах, что
`created_at` вложений, журнал worker и генератор."""
return float(stand.psql(queue.NOW_SQL))
def pipeline_state(stand: Stand) -> dict:
sample = queue.parse_sample(stand.psql(queue.TIMELINE_SQL))
if sample is None:
raise StandError("cannot read the attachment_jobs state")
return sample
def worker_logs() -> str:
done = docker("logs", WORKER, check=False)
return done.stdout + done.stderr
def pipeline_tail(lines: int = 40) -> str:
done = docker("logs", "--tail", str(lines), WORKER, check=False)
return done.stdout + done.stderr
def _empty(state: dict) -> bool:
return state["unfinished"] == 0
def wait_state(
stand: Stand, what: str, done: Callable[[dict], bool], timeout: float
) -> dict:
deadline = time.monotonic() + timeout
while True:
state = pipeline_state(stand)
if done(state):
return state
if time.monotonic() > deadline:
raise StandError(
f"{what}: not reached in {timeout:.0f}s, state {state}\n"
+ pipeline_tail()
)
time.sleep(0.5)
def wait_consuming(before: int, timeout: float = 90) -> float:
"""Ждать в журнале worker строку `broker consuming` номер `before + 1`;
её время (часы ВМ) — начало работы потребителя."""
deadline = time.monotonic() + timeout
while True:
consuming = queue.parse_worker_log(worker_logs()).consuming
if len(consuming) > before:
return consuming[before]
if time.monotonic() > deadline:
raise StandError(
f"worker does not consume in {timeout:.0f}s:\n" + pipeline_tail()
)
time.sleep(0.3)
def queue_gen(
stand: Stand,
profile: QueueProfile,
folder: Path,
item: dict,
phase: str,
*,
rate: float,
duration: float,
count: int = 0,
) -> tuple[dict, list[queue.Upload]]:
"""Генератор `queue` фазы; итог и журнал загрузок — в `raw/`."""
if rate <= 0:
raise StandError(f"{item['cell']}: {phase} rate {rate} is not positive")
log = f"/tmp/{item['cell']}-{phase}.log"
args = [
"queue",
"--fixture",
FIXTURE,
"--body",
f"/tmp/body-{item['size_class']}.png",
"--rate",
f"{rate:.6g}",
"--duration",
f"{duration:.6g}",
"--log",
log,
"--timeout",
f"{profile.upload_timeout_s:g}",
]
if count:
args += ["--count", str(count)]
result = stand.gen(*args, timeout=duration + 2 * profile.upload_timeout_s + 60)
text = stand.gen_file(log)
docker("exec", GEN, "rm", "-f", log, check=False)
base = folder / "raw" / f"{item['cell']}-{phase}"
base.with_name(base.name + "-gen.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
base.with_name(base.name + "-gen.csv").write_text(text, encoding="ascii")
return result, queue.parse_gen_log(text)
def run_queue(profile: QueueProfile, label: str) -> Path:
fixed = dict(profile.fixed_rates)
for name, _, _, _, count in profile.classes:
# тела фазы ёмкости все незавершённые: при N ≥ предела часть получит 429
if count >= profile.backlog_limit:
raise SystemExit(f"class {name}: drain count {count} >= backlog limit")
if fixed.get(name, 0) <= 0:
raise SystemExit(f"class {name}: no fixed rate in the profile")
workdir = temp_workdir()
folder = new_label_dir("queue", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
limits = profile.limits()
pre = preflight(limits.total_cpus(), limits.total_memory_gib())
image = prepare_image(label)
plan = queue.plan_cells(
profile.configs, [c[0] for c in profile.classes], profile.repeats, profile.seed
)
meta = {
"kind": "queue",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
"plan": plan,
# граница Q4: предел незавершённых пропускает загрузки, которые уже
# прошли проверку, — не больше мест загрузок всех реплик
"upload_slack": profile.api_replicas
* image["app_defaults"]["uploads"]["max_concurrent"],
"bodies": {},
"app_env": {},
"cells": {},
}
write_json(folder / "meta.json", meta)
stand = Stand(
limits,
session_ttl=profile.session_ttl,
workdir=workdir,
upload_max_unfinished=profile.backlog_limit,
)
rows: list[dict] = []
timeline: list[dict] = []
jobs: list[dict] = []
try:
for repeat in range(1, profile.repeats + 1):
if repeat > 1:
stand.stop_sampler()
stand.stop()
say(
f"repeat {repeat}: stand with s3, rabbitmq, relay, worker, "
f"{profile.api_replicas} replicas"
)
stand.start(with_s3=True, with_pipeline=True)
stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
"1",
"--read-messages",
"0",
"--write-chats",
"1",
)
for index, (name, kind, width, height, _) in enumerate(profile.classes):
args = ["body", "--width", str(width), "--height", str(height)]
args += [
"--seed",
str(profile.seed + index),
"--out",
f"/tmp/body-{name}.png",
]
if kind == "flat":
args.append("--flat")
body = stand.gen(*args, timeout=600)
if body["bytes"] > ATTACHMENT_MAX_BYTES:
raise StandError(f"body {name}: {body['bytes']} B > upload limit")
known = meta["bodies"].setdefault(
name,
{
k: body[k]
for k in ("kind", "width", "height", "bytes", "sha256")
},
)
if known["sha256"] != body["sha256"]:
raise StandError(f"body {name} differs between repeats")
if repeat == 1:
meta["stand"] = stand.passport()
write_json(folder / "meta.json", meta)
stand.start_sampler("scan", interval=profile.sample_interval_s)
for item in [p for p in plan if p["repeat"] == repeat]:
say(f"cell {item['order']}/{len(plan)} {item['cell']}")
cell, cell_timeline, cell_jobs = queue_cell(
stand, profile, item, meta, folder
)
rows += cell
timeline += cell_timeline
jobs += cell_jobs
for row in cell:
append_csv(folder / "cells.csv", queue.CELL_COLUMNS, row)
write_json(folder / "meta.json", meta)
with gzip.open(
folder / "raw" / f"sampler-r{repeat}.jsonl.gz", "wt", encoding="ascii"
) as fh:
for line in stand.sampler_lines():
fh.write(json.dumps(line, separators=(",", ":")) + "\n")
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
tail = stand.compose(
"logs", "--tail", "80", "relay", "worker", "rabbitmq", check=False
)
(folder / "app-tail.log").write_text(
stand.app_tail(80) + tail.stdout + tail.stderr, encoding="utf-8"
)
finally:
stand.stop_sampler()
stand.stop()
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(folder / "cells.csv", queue.CELL_COLUMNS, rows)
write_csv(folder / "timeline.csv", queue.TIMELINE_COLUMNS, timeline)
with gzip.open(folder / "jobs.csv.gz", "wt", newline="", encoding="utf-8") as fh:
writer = csv.DictWriter(fh, fieldnames=queue.JOB_COLUMNS, extrasaction="ignore")
writer.writeheader()
writer.writerows(jobs)
(folder / "summary.md").write_text(
queue.render_summary(meta, rows), encoding="utf-8"
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def queue_cell(
stand: Stand, profile: QueueProfile, item: dict, meta: dict, folder: Path
) -> tuple[list[dict], list[dict], list[dict]]:
"""Ячейка: новый worker с настройкой ячейки, затем фазы ёмкости, ровной
нагрузки, общей интенсивности класса, перегрузки и восстановления;
между фазами очередь пустеет."""
name = item["cell"]
drain_n = {c[0]: c[4] for c in profile.classes}[item["size_class"]]
fill_rate = dict(profile.fill_rates)[item["size_class"]]
env = {
"BENCH_WORKER_CONCURRENCY": str(item["concurrency"]),
"BENCH_WORKER_PREFETCH": str(item["prefetch"]),
"BENCH_WORKER_RENDER_MEMORY_MB": str(profile.render_memory_mb),
}
stand.compose(
"up", "-d", "--no-deps", "--no-build", "--force-recreate", "worker", env=env
)
wait_consuming(0)
wait_state(
stand, f"{name}: empty queue before the cell", _empty, profile.settle_timeout_s
)
if item["config"] not in meta["app_env"]:
names = ("WORKER_", "RENDER_", "JOB_", "DB_POOL_", "LOG_")
values = json.loads(
docker("inspect", "-f", "{{json .Config.Env}}", WORKER).stdout
)
meta["app_env"][item["config"]] = dict(
sorted(v.split("=", 1) for v in values if v.startswith(names))
)
worker_id = full_id(WORKER)
marks: dict[str, float] = {}
gens: dict[str, dict] = {}
uploads_by: dict[str, list[queue.Upload]] = {}
poller = TimelinePoller(stand, profile.sample_interval_s)
# A — ёмкость: N тел ждут в брокере, worker стартует на готовую очередь
poller.phase = "drain"
poller.start()
try:
marks["drain"] = server_now(stand)
docker("stop", WORKER)
gens["drain"], uploads_by["drain"] = queue_gen(
stand,
profile,
folder,
item,
"drain",
rate=fill_rate,
duration=drain_n / fill_rate + 1,
count=drain_n,
)
wait_state(
stand, f"{name}: relay published the fill", lambda s: s["outbox"] == 0, 60
)
before = len(queue.parse_worker_log(worker_logs()).consuming)
marks["worker_start"] = server_now(stand)
docker("start", WORKER)
consuming_ts = wait_consuming(before)
wait_state(stand, f"{name}: drain", _empty, profile.settle_timeout_s)
fill_jobs = queue.parse_jobs(stand.psql(queue.jobs_sql(marks["drain"])))
throughput = queue.drain_throughput(fill_jobs, consuming_ts)
if not throughput:
raise StandError(
f"{name}: no ready job after the drain:\n" + pipeline_tail()
)
say(f" drain {drain_n}: {throughput:.3f} ready/s")
# B — ровная нагрузка, затем ждём, пока принятое станет ready
poller.phase = "steady"
marks["steady"] = server_now(stand)
gens["steady"], uploads_by["steady"] = queue_gen(
stand,
profile,
folder,
item,
"steady",
rate=profile.steady_factor * throughput,
duration=profile.window_s,
)
wait_state(stand, f"{name}: steady settle", _empty, profile.settle_timeout_s)
# B' — общая интенсивность класса: задержки настроек при равной
# нагрузке, а не при равной доле своей ёмкости
poller.phase = "fixed"
marks["fixed"] = server_now(stand)
gens["fixed"], uploads_by["fixed"] = queue_gen(
stand,
profile,
folder,
item,
"fixed",
rate=dict(profile.fixed_rates)[item["size_class"]],
duration=profile.window_s,
)
wait_state(stand, f"{name}: fixed settle", _empty, profile.settle_timeout_s)
# C — перегрузка: очередь упирается в предел, лишнее получает 429;
# окно не короче, чем нужно очереди, чтобы дойти до предела
poller.phase = "overload"
marks["overload"] = server_now(stand)
gens["overload"], uploads_by["overload"] = queue_gen(
stand,
profile,
folder,
item,
"overload",
rate=profile.overload_factor * throughput,
duration=queue.overload_window(
profile.window_s,
profile.backlog_limit,
throughput,
fill=profile.overload_fill,
cap=profile.overload_max_s,
),
)
# D — опустошение после перегрузки и снова ровная нагрузка
poller.phase = "recovery"
marks["recovery"] = server_now(stand)
wait_state(stand, f"{name}: recovery drain", _empty, profile.settle_timeout_s)
gens["recovery"], uploads_by["recovery"] = queue_gen(
stand,
profile,
folder,
item,
"recovery",
rate=profile.steady_factor * throughput,
duration=profile.window_s / 2,
)
wait_state(stand, f"{name}: recovery settle", _empty, profile.settle_timeout_s)
marks["end"] = server_now(stand)
finally:
samples = poller.finish()
# worker новый на ячейку, перезапуск в фазе A заводит cgroup заново:
# memory.peak — пик ячейки с начала фазы ёмкости
peak = stand.cgroup_value(worker_id, "memory.peak")
log_text = worker_logs()
(folder / "raw" / f"{name}-worker.log").write_text(log_text, encoding="utf-8")
data = queue.CellData(
item=item,
body_bytes=meta["bodies"][item["size_class"]]["bytes"],
marks=marks,
consuming_ts=consuming_ts,
gens=gens,
uploads=uploads_by,
jobs=queue.parse_jobs(stand.psql(queue.jobs_sql(marks["drain"]))),
log=queue.parse_worker_log(log_text),
ticks=queue.worker_ticks(stand.sampler_lines(), worker_id[:12]),
samples=samples,
peak_bytes=peak,
backlog_limit=profile.backlog_limit,
)
rows = queue.cell_rows(data)
meta["cells"][name] = {
"worker_id": worker_id,
"marks": marks,
"consuming_ts": consuming_ts,
"memory_peak_bytes": peak,
"timeline_samples": len(samples),
"timeline_failures": poller.failures,
}
by_phase = {row["phase"]: row for row in rows}
over, rec = by_phase["overload"], by_phase["recovery"]
say(
f" steady p95 {by_phase['steady']['ttr_p95_ms']} ms, fixed p95 "
f"{by_phase['fixed']['ttr_p95_ms']} ms, overload {over['window_s']} s: 429 "
f"{over['rejected_429']}/{over['sent']} (after limit "
f"{over['share_429_saturated']}), max unfinished {over['max_unfinished']}, "
f"recovery {rec['recovery_s']} s, worker peak {rows[0]['worker_peak_mib']} MiB"
+ (
f", flags {' '.join(r['flags'] for r in rows if r['flags'])}"
if any(r["flags"] for r in rows)
else ""
)
)
return rows, queue.timeline_rows(data), queue.job_rows(data)
# --- профиль telemetry (трек 1 лабы 6) -------------------------------------------------
# блокировка таблицы сообщений в своей сессии: запись ждёт, чтение идёт;
# момент — часы PostgreSQL (ВМ Docker)
LOCK_SQL = (
"BEGIN;\nLOCK TABLE messages IN SHARE MODE;\n"
"SELECT extract(epoch FROM clock_timestamp());\n"
"SELECT pg_sleep({seconds});\nCOMMIT;\n"
)
# строка журнала логгера SDK уровня WARNING и выше — признак потерь экспорта
SDK_WARNING = re.compile(r"\s(WARNING|ERROR|CRITICAL)\s+opentelemetry")
def compose_image(prefix: str) -> str:
"""Закреплённый образ сервиса из compose.yaml (тот же, что у стенда)."""
text = (ROOT / "compose.yaml").read_text(encoding="utf-8")
match = re.search(rf"image: ({re.escape(prefix)}[^\s]+)", text)
if not match:
raise SystemExit(f"no {prefix} image in compose.yaml")
return match.group(1)
def sha256_file(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def telemetry_preflight(profile: TelemetryProfile) -> dict:
"""До стенда: копия Prometheus совпадает с рабочей вне своих блоков,
слои Collector проходят `validate` вместе с рабочей конфигурацией,
копия — `promtool check config`. Иначе код 2."""
collector = compose_image("otel/opentelemetry-collector-contrib:")
prometheus = compose_image("prom/prometheus:")
base = ROOT / "infra" / "otel" / "collector.yaml"
rules = telemetry.PROM_BASE.with_name("rules.yml")
files: dict[str, str] = {}
for name in profile.variants:
variant = telemetry.VARIANTS[name]
if variant.collector_layer:
layer = telemetry.OTEL_DIR / variant.collector_layer
files[variant.collector_layer] = sha256_file(layer)
done = docker(
"run",
"--rm",
"--network",
"none",
"-v",
f"{base}:/etc/otelcol-contrib/collector.yaml:ro",
"-v",
f"{layer}:/etc/otelcol-contrib/variant.yaml:ro",
collector,
"validate",
"--config=/etc/otelcol-contrib/collector.yaml",
"--config=/etc/otelcol-contrib/variant.yaml",
check=False,
timeout=120,
)
if done.returncode != 0:
say(f"collector layer {layer.name} is invalid:\n{done.stderr[-2000:]}")
raise SystemExit(2)
if variant.prometheus:
lean = telemetry.OTEL_DIR / variant.prometheus
files[variant.prometheus] = sha256_file(lean)
problems = telemetry.lean_problems(
telemetry.PROM_BASE.read_text(encoding="utf-8"),
lean.read_text(encoding="utf-8"),
)
if problems:
say(f"{lean.name} differs from prometheus.yml: {problems}")
raise SystemExit(2)
done = docker(
"run",
"--rm",
"--network",
"none",
"--entrypoint",
"promtool",
"-v",
f"{lean}:/etc/prometheus/prometheus.yml:ro",
"-v",
f"{rules}:/etc/prometheus/rules.yml:ro",
prometheus,
"check",
"config",
"/etc/prometheus/prometheus.yml",
check=False,
timeout=120,
)
if done.returncode != 0:
say(
f"promtool rejects {lean.name}:\n{done.stdout[-1500:]}{done.stderr[-1500:]}"
)
raise SystemExit(2)
return {
"collector_image": collector,
"prometheus_image": prometheus,
"collector_yaml_sha256": sha256_file(base),
"prometheus_yml_sha256": sha256_file(telemetry.PROM_BASE),
"variant_files_sha256": files,
}
def lookup(*args: str, timeout: float = 120) -> dict:
"""`bench/lookup.py` в контейнере генератора (сеть телеметрии)."""
done = docker("exec", GEN, "python", "/tmp/lookup.py", *args, timeout=timeout)
return json.loads(done.stdout.strip().splitlines()[-1])
def wait_new_replicas(stand: Stand, replicas: int, timeout: float = 90) -> None:
"""После пересоздания реплик LB должен видеть `UP` именно их адреса:
слот прежнего контейнера какое-то время ещё числится живым."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
ids = stand.api_ids()
addresses = {stand.address(cid) for cid in ids}
up = {s["addr"] for s in stand.lb_slots() if s["status"] == "UP"}
if len(ids) == replicas and addresses <= up:
return
time.sleep(0.5)
raise StandError("LB does not see the recreated replicas UP:\n" + stand.app_tail())
def startup_lines(names: list[str], timeout: float = 30) -> dict[str, str]:
"""Строка старта каждого процесса со сведениями о телеметрии."""
deadline = time.monotonic() + timeout
found: dict[str, str] = {}
while True:
for name in names:
if name in found:
continue
done = docker("logs", name, check=False)
for line in (done.stdout + done.stderr).splitlines():
if "telemetry=" in line:
found[name] = "telemetry=" + line.split("telemetry=", 1)[1]
break
if len(found) == len(names) or time.monotonic() > deadline:
return found
time.sleep(0.5)
def sdk_warnings(names: list[str]) -> int:
count = 0
for name in names:
done = docker("logs", name, check=False)
count += sum(
1
for line in (done.stdout + done.stderr).splitlines()
if SDK_WARNING.search(line)
)
return count
def telemetry_prepare(
stand: Stand, profile: TelemetryProfile, workdir: Path, meta: dict
) -> None:
"""Данные повтора: пользователи и чаты, тела загрузок, готовые вложения
для скачиваний и строка `ready` без объекта (500 на скачивание)."""
stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
"1",
)
body = stand.gen(
"body",
"--width",
str(profile.upload_width),
"--height",
str(profile.upload_height),
"--seed",
str(profile.seed),
"--out",
"/tmp/body-upload.png",
)
bad = telemetry.jpeg_without_quant(
(ROOT / "student-tests" / "fixtures" / "sample.jpg").read_bytes()
)
(workdir / "body-bad.jpg").write_bytes(bad)
docker("cp", str(workdir / "body-bad.jpg"), f"{GEN}:/tmp/body-bad.jpg")
bodies = {
"upload": {k: body[k] for k in ("width", "height", "bytes", "sha256")},
"bad": {"bytes": len(bad), "sha256": hashlib.sha256(bad).hexdigest()},
}
if meta.setdefault("bodies", bodies) != bodies:
raise StandError("upload bodies differ between repeats")
warm = stand.gen(
"queue",
"--fixture",
FIXTURE,
"--body",
"/tmp/body-upload.png",
"--rate",
"5",
"--duration",
"2",
"--count",
str(profile.warm_uploads),
"--log",
"/tmp/warm.log",
)
ids = [
u.attachment_id for u in queue.parse_gen_log(stand.gen_file("/tmp/warm.log"))
]
ids = [i for i in ids if i]
if warm["accepted"] != profile.warm_uploads or not ids:
raise StandError(f"warm uploads: {warm['statuses']}")
wait_state(stand, "warm uploads", _empty, profile.drain_timeout_s)
ready = stand.psql(
"SELECT count(*) FROM attachments WHERE status = 'ready' AND id IN ("
+ ",".join(f"'{i}'" for i in ids)
+ ");\n"
)
if int(ready) != len(ids):
raise StandError(f"warm uploads: {ready} of {len(ids)} ready")
# строка ready с ключом, которого в S3 нет: скачивание — 500
# (ContentMissing), как у потерянного объекта до сверки
broken = stand.psql(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, "
"size_bytes, sha256, object_key, status, created_at, updated_at) "
"SELECT gen_random_uuid(), chat_id, owner_id, 'broken.png', content_type, "
"size_bytes, sha256, object_key || '.missing', 'ready', created_at, "
f"updated_at FROM attachments WHERE id = '{ids[0]}' RETURNING id;\n"
).splitlines()[0]
targets = {"download": ids[-1], "broken": broken}
write_json(workdir / "targets.json", targets)
docker("cp", str(workdir / "targets.json"), f"{GEN}:/tmp/targets.json")
docker("cp", str(BENCH / "lookup.py"), f"{GEN}:/tmp/lookup.py")
def run_telemetry(profile: TelemetryProfile, label: str) -> Path:
unknown = [v for v in profile.variants if v not in telemetry.VARIANTS]
if unknown:
raise SystemExit(f"unknown variants: {unknown}")
checked = telemetry_preflight(profile)
workdir = temp_workdir()
folder = new_label_dir("telemetry-cost", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
limits = profile.limits()
pre = preflight(limits.total_cpus(), limits.total_memory_gib())
image = prepare_image(label)
plan = telemetry.plan_runs(profile.variants, profile.repeats, profile.seed)
meta = {
"kind": "telemetry-cost",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
"plan": plan,
"telemetry_config": checked,
"variants": {
name: {
"env": telemetry.VARIANTS[name].env(),
"startup": telemetry.VARIANTS[name].startup(),
"collector_layer": telemetry.VARIANTS[name].collector_layer,
"prometheus": telemetry.VARIANTS[name].prometheus,
}
for name in profile.variants
},
"runs": {},
}
write_json(folder / "meta.json", meta)
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
rows: list[dict] = []
class_rows: list[dict] = []
try:
for repeat in range(1, profile.repeats + 1):
if repeat > 1:
stand.stop_sampler()
stand.stop()
# стенд повтора стартует без телеметрии; вариант — на прогон
stand.extra_files = []
stand.env.update(telemetry.VARIANTS["off"].env())
say(f"repeat {repeat}: stand with s3, rabbitmq, relay, worker")
stand.start(with_s3=True, with_pipeline=True)
telemetry_prepare(stand, profile, workdir, meta)
if repeat == 1:
meta["stand"] = stand.passport()
write_json(folder / "meta.json", meta)
stand.start_sampler("scan", interval=profile.sample_interval_s)
for item in [p for p in plan if p["repeat"] == repeat]:
say(f"run {item['order']}/{len(plan)} {item['run']}")
row, classes = telemetry_run(
stand, profile, item, meta, folder, workdir
)
rows.append(row)
class_rows += classes
append_csv(folder / "runs.csv", telemetry.RUN_COLUMNS, row)
write_json(folder / "meta.json", meta)
with gzip.open(
folder / "raw" / f"sampler-r{repeat}.jsonl.gz", "wt", encoding="ascii"
) as fh:
for line in stand.sampler_lines():
fh.write(json.dumps(line, separators=(",", ":")) + "\n")
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
tail = stand.compose(
"logs", "--tail", "60", "relay", "worker", *TELEMETRY_SERVICES, check=False
)
(folder / "app-tail.log").write_text(
stand.app_tail(60) + tail.stdout + tail.stderr, encoding="utf-8"
)
finally:
stand.stop_sampler()
stand.stop()
drop_quick_image(label)
meta["finished"] = utc_now()
meta["duration_s"] = round(time.monotonic() - started_mono, 1)
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(folder / "runs.csv", telemetry.RUN_COLUMNS, rows)
write_csv(folder / "classes.csv", telemetry.CLASS_COLUMNS, class_rows)
(folder / "summary.md").write_text(
telemetry.render_summary(meta, rows, class_rows), encoding="utf-8"
)
say(f"done in {meta['duration_s'] / 60:.1f} min: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def telemetry_run(
stand: Stand,
profile: TelemetryProfile,
item: dict,
meta: dict,
folder: Path,
workdir: Path,
) -> tuple[dict, list[dict]]:
"""Прогон: телеметрия варианта с пустыми хранилищами, новые api, relay и
worker с его окружением, проверка, что вариант применён, нагрузка с
окнами и блокировками, ожидание очереди и решений выборки, поиск."""
variant = telemetry.VARIANTS[item["variant"]]
name = item["run"]
marks: dict = {"begin": time.monotonic()}
stand.compose("rm", "-f", "-s", "-v", *TELEMETRY_SERVICES, check=False)
for volume in TELEMETRY_VOLUMES:
docker("volume", "rm", "-f", f"{PROJECT}_{volume}", check=False)
stand.env.update(variant.env())
override = workdir / "variant.compose.yml"
override.write_text(telemetry.override_yaml(variant), encoding="utf-8")
stand.extra_files = [override]
if variant.telemetry:
stand.compose("up", "-d", "--no-deps", "--wait", *TELEMETRY_SERVICES)
docker("network", "connect", TELEMETRY_NET, GEN, check=False)
ready = lookup("ready")
if not ready["ready"]:
raise StandError(f"{name}: telemetry is not ready: {ready}")
stand.compose(
"up",
"-d",
"--no-deps",
"--no-build",
"--force-recreate",
"api",
"relay",
"worker",
)
wait_new_replicas(stand, profile.api_replicas)
if not stand.gen("ready", "--timeout", "60")["ready"]:
raise StandError(f"{name}: API is not ready:\n" + stand.app_tail())
api_ids = stand.api_ids()
processes = [*(i[:12] for i in api_ids), WORKER, RELAY]
lines = startup_lines(processes)
expected = variant.startup()
problems = [
f"{p}: {lines.get(p, 'no startup line')}"
for p in processes
if not lines.get(p, "").startswith(expected)
]
if problems:
raise StandError(
f"{name}: variant is not applied, want {expected!r}: {problems}"
)
env = json.loads(docker("inspect", "-f", "{{json .Config.Env}}", api_ids[0]).stdout)
marks["prepared"] = time.monotonic()
duration = profile.warmup_s + profile.clean_s + profile.fault_s
log = f"/tmp/{name}-mix.log"
proc = stand.gen_popen(
"mix",
"--fixture",
FIXTURE,
"--targets",
"/tmp/targets.json",
"--rates",
",".join(f"{k}={v:g}" for k, v in profile.rates),
"--duration",
f"{duration:g}",
"--upload-body",
"/tmp/body-upload.png",
"--bad-body",
"/tmp/body-bad.jpg",
"--tag",
f"r{item['order']}",
"--log",
log,
)
launched = time.monotonic() + 0.3
def until(offset: float) -> None:
time.sleep(max(0.0, launched + offset - time.monotonic()))
snaps: dict[str, dict] = {}
if variant.telemetry:
until(profile.warmup_s)
snaps["clean_start"] = lookup("snap")
until(profile.warmup_s + profile.clean_s)
snaps["clean_end"] = lookup("snap")
locks = []
for offset in profile.locks:
until(profile.warmup_s + profile.clean_s + offset)
out = stand.psql(LOCK_SQL.format(seconds=profile.lock_s))
locks.append(float(out.splitlines()[0]))
out, err = proc.communicate(timeout=duration + 120)
if proc.returncode != 0:
raise StandError(f"{name}: generator exit {proc.returncode}: {err[-1500:]}")
result = json.loads(out.strip().splitlines()[-1])
text = stand.gen_file(log)
docker("exec", GEN, "rm", "-f", log, check=False)
with gzip.open(folder / "raw" / f"{name}-mix.csv.gz", "wt", encoding="ascii") as fh:
fh.write(text)
wait_state(stand, f"{name}: queue drain", _empty, profile.drain_timeout_s)
reqs = telemetry.parse_mix(text)
windows = telemetry.Windows(
result["start"], profile.warmup_s, profile.clean_s, profile.fault_s
)
found: dict = {}
layer_problems: list[str] = []
roles = {"api": api_ids, "worker": [full_id(WORKER)], "relay": [full_id(RELAY)]}
down: list[str] = []
if variant.telemetry:
time.sleep(profile.settle_s)
snaps["end"] = lookup("snap")
request = telemetry.lookup_request(reqs, windows)
write_json(workdir / "find.json", request)
docker("cp", str(workdir / "find.json"), f"{GEN}:/tmp/find.json")
found = lookup("find", "--input", "/tmp/find.json", timeout=600)
write_json(folder / "raw" / f"{name}-find.json", found)
for service in (COLLECTOR, PROM, LOKI, TEMPO):
state = container_state(service)
if not state.get("running") or state.get("oom_killed"):
down.append(f"{service} {state}")
tail_seen = telemetry.has_prefix(
snaps["end"].get("collector"), "otelcol_processor_tail_sampling"
)
if tail_seen != bool(variant.collector_layer):
layer_problems.append(f"tail_sampling metrics present: {tail_seen}")
if snaps["end"].get("prom_relabel") != bool(variant.prometheus):
layer_problems.append(
f"prometheus relabel: {snaps['end'].get('prom_relabel')}"
)
# у погибшего сервиса нет и рядов: это пометка storage_down, а не
# «слой не применён»
if layer_problems and not down:
raise StandError(
f"{name}: variant layers are not applied: {layer_problems}"
)
roles.update(
collector=[full_id(COLLECTOR)],
prometheus=[full_id(PROM)],
loki=[full_id(LOKI)],
tempo=[full_id(TEMPO)],
)
write_json(folder / "raw" / f"{name}-snaps.json", snaps)
peaks: dict[str, float | None] = {}
for role, ids in roles.items():
found_peaks = [
p for cid in ids if (p := stand.cgroup_value(cid, "memory.peak"))
]
peaks[role] = max(found_peaks) if found_peaks else None
data = telemetry.RunData(
item=item,
variant=variant.name,
windows=windows,
reqs=reqs,
sampler=stand.sampler_lines(),
ids=roles,
peaks=peaks,
snaps=snaps,
found=found,
uploads=telemetry.parse_uploads(
stand.psql(telemetry.uploads_sql(windows.start - 1))
),
layer_problems=layer_problems,
down=down,
sdk_warnings=sdk_warnings(processes),
locks=locks,
)
row, classes = telemetry.run_rows(data)
meta["runs"][name] = {
"variant": variant.name,
"gen": {k: v for k, v in result.items() if k != "log"},
"windows": {"clean": windows.clean, "fault": windows.fault},
"locks_at": locks,
"telemetry_down": down,
"startup": lines,
"app_env": dict(
sorted(
v.split("=", 1) for v in env if v.startswith(("OTEL_", "TELEMETRY_"))
)
),
"snap_ts": {k: v.get("ts") for k, v in snaps.items()},
"prepare_s": round(marks["prepared"] - marks["begin"], 1),
"run_s": round(time.monotonic() - marks["begin"], 1),
}
say(
f" {variant.name}: light p50/p95/p99 {row['light_p50_ms']}/{row['light_p95_ms']}/"
f"{row['light_p99_ms']} ms, api {row['api_cores']} cores "
f"({row['api_cpu_ms_per_req']} ms/req), tempo {row.get('tempo_spans_per_s')} "
f"spans/s, 500 kept {row.get('broken_kept')}/{row.get('broken_n')}, slow "
f"{row.get('slow_kept')}/{row.get('slow_n')}, bad {row.get('bad_kept')}/"
f"{row.get('bad_n')}, run {meta['runs'][name]['run_s']} s"
+ (f", flags {row['flags']}" if row["flags"] else "")
)
return row, classes
# --- CLI ------------------------------------------------------------------------------
def main() -> None:
root = argparse.ArgumentParser(description=__doc__.splitlines()[0])
root.add_argument(
"profile",
choices=(
*PROFILES,
*UPLOADS_PROFILES,
*RECONCILE_PROFILES,
*QUEUE_PROFILES,
*TELEMETRY_PROFILES,
"scaling",
),
)
root.add_argument("--label")
root.add_argument("--series", choices=sorted(SERIES))
# ячейки серии вместо профильных: у ab — «до/после»; у остальных — только
# если конфигурация не помещается в ВМ Docker (пишется в паспорт метки)
root.add_argument("--configs", help="ячейки серии: '4x1-pg0.5 4x1'")
root.add_argument("--resume", action="store_true")
args = root.parse_args()
with bench_lock():
if args.profile == "scaling":
if not args.series:
raise SystemExit("scaling needs --series")
profile = SERIES[args.series]
if args.configs:
profile = dataclasses.replace(
profile, configs=tuple(args.configs.split())
)
quick = args.series == "quick"
label = args.label or (
quick_label("scaling") if quick else f"scaling-{args.series}"
)
run_scaling(profile, label, args.resume)
return
kind = args.profile.removesuffix("-quick")
if args.profile.endswith("quick"):
label = args.label or quick_label(kind)
else:
label = args.label or "baseline"
if kind == "telemetry":
run_telemetry(TELEMETRY_PROFILES[args.profile], label)
elif kind == "queue":
run_queue(QUEUE_PROFILES[args.profile], label)
elif kind == "uploads":
run_uploads(UPLOADS_PROFILES[args.profile], label)
elif kind == "reconcile":
run_reconcile(RECONCILE_PROFILES[args.profile], label)
elif kind == "failover":
run_failover(PROFILES[args.profile], label)
else:
run_rollout(PROFILES[args.profile], label)
if __name__ == "__main__":
main()