3276 lines
123 KiB
Python
3276 lines
123 KiB
Python
"""Оркестратор нагрузочного стенда лаб 3–5. Только stdlib.
|
||
|
||
uv run python bench/run.py queue [--label baseline] # трек 1 лабы 5
|
||
uv run python bench/run.py queue-quick
|
||
uv run python bench/run.py failover [--label baseline] # уровень 4 лабы 3
|
||
uv run python bench/run.py failover-quick # проверка стенда
|
||
uv run python bench/run.py rollout [--label baseline] # уровень 4 лабы 3
|
||
uv run python bench/run.py rollout-quick
|
||
uv run python bench/run.py uploads [--label baseline] # уровень 4 лабы 4
|
||
uv run python bench/run.py uploads-quick
|
||
uv run python bench/run.py reconcile [--label baseline] # трек 2 лабы 4
|
||
uv run python bench/run.py reconcile-quick
|
||
uv run python bench/run.py scaling --series a|b|db|ab|quick [--label L]
|
||
[--configs "4x1-pg0.5 4x1"] [--resume] # трек 1 лабы 3
|
||
|
||
Профили (поля — в `PROFILES`, целиком пишутся в `meta.json`):
|
||
|
||
- `failover` — стенд с лимитами рабочего (две реплики по 1 CPU / 512 МБ,
|
||
PostgreSQL 2 CPU / 1 ГБ, Valkey 0,5 / 128 МБ, HAProxy 1 / 256 МБ),
|
||
постоянная интенсивность записи с ключом и чтения; каждая реплика по
|
||
очереди в режимах `stop`, `kill`, `pause` с возвратом. Посекундная шкала,
|
||
время исключения из LB и возврата, ошибки, время восстановления, повтор
|
||
неуспешных записей с тем же ключом и сверка «без потерь и дублей»,
|
||
CPU и память компонентов по окнам фазы. Результат —
|
||
`evidence/failover/<метка>/`;
|
||
- `rollout` — тот же стенд и нагрузка, посреди неё
|
||
`scripts/rolling-update.sh` меняет обе реплики на новый образ с
|
||
expand-ревизией схемы (`9001_rollout_probe`, только в образе стенда).
|
||
Результат — `evidence/rollout/<метка>/`;
|
||
- `scaling` — серии трека 1 «Цена масштабирования»: конфигурации реплик
|
||
и лимитов, на каждую ячейку (конфигурация × повтор) свежий стенд,
|
||
открытая модель, фиксированные уровни и поиск R* по SLO, сэмплер
|
||
контейнеров, LB и PostgreSQL. Результат — `evidence/scaling/<метка>/`;
|
||
- `uploads` (лаба 4) — одна реплика 1 CPU / 512 МБ без swap, S3 рабочего
|
||
стенда; варианты «с лимитом» (места загрузок по умолчанию) и «без
|
||
лимита» (1000 мест, без очереди) × тела ~1 и ~10 MiB × 1/10/20/40
|
||
клиентов; на прогон — новые S3 и реплика, пик `memory.peak` окна,
|
||
OOM, ответы по кодам, загрузок/с. Результат — `evidence/uploads/<метка>/`;
|
||
- `reconcile` (лаба 4, трек 2) — стоимость сверки bucket и БД: набор мимо
|
||
API (`COPY` метаданных, PUT объектов `fill_objects.py`) с расхождениями
|
||
всех классов, разовые проходы уборщика `batch`, `head`, глубокий, под
|
||
фоновой нагрузкой с пределом скорости и без, отказ по пределу.
|
||
Результат — `evidence/reconcile/<метка>/`;
|
||
- `queue` (лаба 5, трек 1) — очередь под перегрузкой: две реплики, брокер,
|
||
relay и один worker 2 CPU / 1536 МБ; настройки concurrency/prefetch ×
|
||
классы изображений, на ячейку — ёмкость (N тел при остановленном
|
||
worker), ровная нагрузка 0,6 ёмкости, общая на класс интенсивность
|
||
(`fixed`), перегрузка 2 ёмкости при пределе `UPLOAD_MAX_UNFINISHED` (окно
|
||
дотягивается до предела), восстановление. Результат —
|
||
`evidence/queue/<метка>/`.
|
||
|
||
Стенд — чёрный ящик: запросы делает только генератор (`loadgen.py`) по
|
||
HTTP через HAProxy стенда; к PostgreSQL оркестратор обращается напрямую
|
||
только для обслуживания и сверки (VACUUM, CHECKPOINT, число строк,
|
||
ревизия схемы), у профиля `reconcile` — ещё для наполнения набора, у
|
||
`queue` — для чтения состояния заданий (только SELECT).
|
||
Метки `quick-*` проверяют стенд и в отчёт не идут. Правила —
|
||
`bench/AGENTS.md`, порядок работы — `bench/RUNBOOK.md`.
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import dataclasses
|
||
import gzip
|
||
import json
|
||
|
||
# bench/queue.py, а не модуль stdlib: каталог скрипта первым в sys.path
|
||
import queue
|
||
import random
|
||
import statistics
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
from collections.abc import Callable
|
||
from pathlib import Path
|
||
|
||
import analysis
|
||
import plots
|
||
import reconcile
|
||
import uploads
|
||
from common import (
|
||
BENCH,
|
||
CACHE,
|
||
DB,
|
||
EVIDENCE,
|
||
FILL,
|
||
FIXTURE,
|
||
GEN,
|
||
IMAGE_REPO,
|
||
LB,
|
||
NET,
|
||
PROJECT,
|
||
RECONCILE_PREFIX,
|
||
ROOT,
|
||
S3,
|
||
SAMPLER,
|
||
SERVICE_TAG,
|
||
WORKER,
|
||
Limits,
|
||
Stand,
|
||
StandError,
|
||
bench_env,
|
||
bench_lock,
|
||
build_image,
|
||
code_state,
|
||
docker,
|
||
finish_host_state,
|
||
gib,
|
||
image_info,
|
||
new_label_dir,
|
||
preflight,
|
||
probe_migration_context,
|
||
quick_label,
|
||
say,
|
||
sh,
|
||
swapouts,
|
||
temp_workdir,
|
||
utc_now,
|
||
write_json,
|
||
)
|
||
|
||
# лимиты рабочего стенда compose.yaml: профили уровня 4 отвечают, что видит
|
||
# клиент `make up` при отказе или замене реплики
|
||
WORKING = Limits(api_replicas=2, api_cpus=1.0, api_memory="512m")
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class LoadProfile:
|
||
"""Профили уровня 4 (`failover`, `rollout`). Часть методики."""
|
||
|
||
name: str
|
||
limits: Limits
|
||
users: int
|
||
read_messages: int
|
||
write_rate: float
|
||
read_rate: float
|
||
gen_procs: int
|
||
session_ttl: int
|
||
seed: int
|
||
# failover: режимы отказа и сколько реплик по очереди
|
||
modes: tuple[str, ...] = ()
|
||
targets: int = 0
|
||
before_s: float = 0.0
|
||
fail_s: float = 0.0
|
||
after_s: float = 0.0
|
||
# запас на возврат реплики в LB после восстановления
|
||
restore_s: float = 10.0
|
||
# rollout: окна до скрипта, бюджет замены и после
|
||
rollout_before_s: float = 0.0
|
||
rollout_budget_s: float = 0.0
|
||
rollout_after_s: float = 0.0
|
||
# худшее обнаружение отказа по формуле haproxy.cfg, с
|
||
bounds: tuple[tuple[str, float], ...] = (
|
||
("stop", 2.5),
|
||
("kill", 2.5),
|
||
("pause", 6.5),
|
||
)
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class ScalingProfile:
|
||
"""Серии трека 1. Часть методики: метки сравнимы при равном профиле."""
|
||
|
||
name: str
|
||
series: str
|
||
configs: tuple[str, ...]
|
||
repeats: int
|
||
users: int = 20
|
||
read_messages: int = 10_000
|
||
write_chats: int = 10
|
||
fill_clients: int = 16
|
||
# смесь read,write,me: история (70), отправка с ключом (20), профиль (10)
|
||
mix: str = "70,20,10"
|
||
fixed_rates: tuple[float, ...] = (250.0, 500.0)
|
||
ladder_base: float = 100.0
|
||
ladder_factor: float = 1.25
|
||
# поиск начинается не ниже этого × число реплик
|
||
start_per_replica: float = 300.0
|
||
bisections: int = 2
|
||
rate_cap: float = 8000.0
|
||
warmup_first_s: float = 10.0
|
||
warmup_ramp_s: float = 5.0
|
||
window_s: float = 15.0
|
||
pause_after_fail_s: float = 10.0
|
||
slo_p95_ms: float = 200.0
|
||
slo_fail_share: float = 0.01
|
||
abort_error_share: float = 0.5
|
||
abort_p95_ms: float = 2000.0
|
||
request_timeout_s: float = 10.0
|
||
gen_procs: int = 4
|
||
session_ttl: int = 21_600
|
||
order_seed: int = 20261014
|
||
# лимиты без реплик: PostgreSQL, Valkey, LB, генератор, сэмплер
|
||
base: Limits = Limits(api_replicas=1, api_cpus=1.0, api_memory="512m")
|
||
|
||
|
||
FAILOVER = LoadProfile(
|
||
name="failover",
|
||
limits=WORKING,
|
||
users=4,
|
||
read_messages=1_000,
|
||
write_rate=100.0,
|
||
read_rate=100.0,
|
||
gen_procs=2,
|
||
session_ttl=21_600,
|
||
seed=20261014,
|
||
modes=("stop", "kill", "pause"),
|
||
targets=2,
|
||
before_s=10.0,
|
||
fail_s=15.0,
|
||
after_s=10.0,
|
||
)
|
||
ROLLOUT = dataclasses.replace(
|
||
FAILOVER,
|
||
name="rollout",
|
||
modes=(),
|
||
targets=0,
|
||
rollout_before_s=10.0,
|
||
rollout_budget_s=90.0,
|
||
rollout_after_s=15.0,
|
||
)
|
||
PROFILES: dict[str, LoadProfile] = {
|
||
"failover": FAILOVER,
|
||
"failover-quick": dataclasses.replace(
|
||
FAILOVER,
|
||
name="failover-quick",
|
||
write_rate=50.0,
|
||
read_rate=50.0,
|
||
modes=("stop", "kill"),
|
||
targets=1,
|
||
before_s=5.0,
|
||
fail_s=8.0,
|
||
after_s=5.0,
|
||
),
|
||
"rollout": ROLLOUT,
|
||
"rollout-quick": dataclasses.replace(
|
||
ROLLOUT,
|
||
name="rollout-quick",
|
||
write_rate=50.0,
|
||
read_rate=50.0,
|
||
rollout_before_s=5.0,
|
||
rollout_budget_s=60.0,
|
||
rollout_after_s=5.0,
|
||
),
|
||
}
|
||
|
||
SERIES: dict[str, ScalingProfile] = {
|
||
# бюджет приложения 4 CPU / 2 ГиБ: один процесс, два, четыре
|
||
"a": ScalingProfile(
|
||
name="scaling", series="a", configs=("1x4", "2x2", "4x1"), repeats=3
|
||
),
|
||
# рост ресурсов: реплики по 1 CPU, PostgreSQL 2 CPU фиксирован
|
||
"b": ScalingProfile(
|
||
name="scaling", series="b", configs=("1x1", "2x1", "3x1", "4x1"), repeats=3
|
||
),
|
||
# управляемое насыщение PostgreSQL: 0,5 CPU при 1, 2, 4 репликах
|
||
"db": ScalingProfile(
|
||
name="scaling",
|
||
series="db",
|
||
configs=("1x1-pg0.5", "2x1-pg0.5", "4x1-pg0.5"),
|
||
repeats=3,
|
||
),
|
||
# «до/после» улучшения и A/A: ячейки задаёт --configs
|
||
"ab": ScalingProfile(
|
||
name="scaling", series="ab", configs=("4x1-pg0.5", "4x1"), repeats=3
|
||
),
|
||
# самопроверка серий: малые данные, короткие окна, один повтор
|
||
"quick": ScalingProfile(
|
||
name="scaling-quick",
|
||
series="quick",
|
||
configs=("1x1", "2x1"),
|
||
repeats=1,
|
||
users=5,
|
||
read_messages=1_000,
|
||
write_chats=3,
|
||
fixed_rates=(100.0, 200.0),
|
||
start_per_replica=150.0,
|
||
bisections=1,
|
||
warmup_first_s=4.0,
|
||
warmup_ramp_s=3.0,
|
||
window_s=6.0,
|
||
pause_after_fail_s=5.0,
|
||
),
|
||
}
|
||
|
||
MIB = 2**20
|
||
KIB = 1024
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class UploadsProfile:
|
||
"""Параметры профиля uploads (лаба 4, уровень 4: сколько памяти ест
|
||
реплика под параллельными загрузками). Это методика, поэтому все поля
|
||
сохраняются в `meta.json`."""
|
||
|
||
name: str
|
||
variants: tuple[str, ...]
|
||
# имя тела → наибольший размер (байт): PNG из шума не больше него
|
||
sizes: tuple[tuple[str, int], ...]
|
||
clients: tuple[int, ...]
|
||
repeats: int
|
||
warmup_s: float = 3.0
|
||
window_s: float = 20.0
|
||
upload_timeout_s: float = 90.0
|
||
sample_interval_s: float = 0.2
|
||
seed: int = 20261023
|
||
api_cpus: float = 1.0
|
||
api_memory: str = "512m"
|
||
s3_cpus: float = 1.0
|
||
s3_memory: str = "512m"
|
||
session_ttl: int = 21_600
|
||
# «без лимита»: место каждому клиенту, очереди нет (override реплики)
|
||
unlimited_env: tuple[tuple[str, str], ...] = (
|
||
("UPLOAD_MAX_CONCURRENT", "1000"),
|
||
("UPLOAD_MAX_WAITING", "0"),
|
||
)
|
||
|
||
def limits(self) -> Limits:
|
||
return Limits(
|
||
api_replicas=1,
|
||
api_cpus=self.api_cpus,
|
||
api_memory=self.api_memory,
|
||
with_s3=True,
|
||
s3_cpus=self.s3_cpus,
|
||
s3_memory=self.s3_memory,
|
||
)
|
||
|
||
|
||
UPLOADS = UploadsProfile(
|
||
name="uploads",
|
||
variants=uploads.VARIANTS,
|
||
sizes=(("1mib", MIB), ("10mib", 10 * MIB)),
|
||
clients=(1, 10, 20, 40),
|
||
repeats=3,
|
||
)
|
||
UPLOADS_PROFILES = {
|
||
"uploads": UPLOADS,
|
||
"uploads-quick": dataclasses.replace(
|
||
UPLOADS,
|
||
name="uploads-quick",
|
||
sizes=(("10mib", 10 * MIB),),
|
||
clients=(1, 10, 40),
|
||
repeats=1,
|
||
window_s=10.0,
|
||
),
|
||
}
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class ReconcileProfile:
|
||
"""Параметры профиля reconcile (лаба 4, трек 2: цена сверки S3 с базой); методика."""
|
||
|
||
name: str
|
||
# этап: имя, объектов набора после этапа, размер объекта, свежий стенд,
|
||
# ячейки, строк без объекта сверх набора (>0 — набор отказа по пределу:
|
||
# объекты все согласованы)
|
||
stages: tuple[tuple[str, int, int, bool, tuple[str, ...], int], ...]
|
||
repeats: int
|
||
refuse_max_changes: int = 100
|
||
seed: int = 20261023
|
||
# grace прохода, с: больше границы живой загрузки (21 с), меньше паузы
|
||
# после наполнения набора
|
||
grace_s: int = 30
|
||
cleaner_cpus: float = 0.25
|
||
cleaner_memory: str = "256m"
|
||
# управляемый опыт: уборщик на 1 CPU
|
||
fast_cpus: float = 1.0
|
||
# загрузчик и уборщик управляемого опыта не работают одновременно: на
|
||
# пике стенд с ними — 9,75 CPU, как профили лабы 3
|
||
fill_cpus: float = 1.0
|
||
fill_memory: str = "512m"
|
||
fill_processes: int = 3
|
||
fill_threads: int = 16
|
||
sample_interval_s: float = 0.5
|
||
users: int = 4
|
||
# фоновая нагрузка ячеек load-*: скачивания согласованных вложений,
|
||
# запись с ключом и чтение истории через LB
|
||
load_download_rate: float = 200.0
|
||
load_write_rate: float = 50.0
|
||
load_read_rate: float = 50.0
|
||
load_downloads: int = 2000
|
||
load_procs: int = 2
|
||
quiet_s: float = 20.0
|
||
pass_budget_s: float = 600.0
|
||
session_ttl: int = 21_600
|
||
|
||
def limits(self) -> Limits:
|
||
return dataclasses.replace(
|
||
WORKING,
|
||
with_s3=True,
|
||
cleaner_cpus=self.cleaner_cpus,
|
||
cleaner_memory=self.cleaner_memory,
|
||
)
|
||
|
||
|
||
RECONCILE = ReconcileProfile(
|
||
name="reconcile",
|
||
stages=(
|
||
("a", 1_000, MIB, True, ("batch", "deep", "deep-fast"), 0),
|
||
("b", 10_000, 4 * KIB, True, ("batch", "head", "deep"), 0),
|
||
("c", 30_000, 4 * KIB, False, ("batch", "head"), 0),
|
||
("d", 100_000, 4 * KIB, False, ("batch", *reconcile.LOAD_CELLS), 0),
|
||
("e", 10_000, 4 * KIB, True, ("batch", "refuse"), 1_000),
|
||
),
|
||
repeats=3,
|
||
)
|
||
RECONCILE_PROFILES = {
|
||
"reconcile": RECONCILE,
|
||
"reconcile-quick": dataclasses.replace(
|
||
RECONCILE,
|
||
name="reconcile-quick",
|
||
stages=(
|
||
(
|
||
"q",
|
||
2_000,
|
||
4 * KIB,
|
||
True,
|
||
("batch", "head", "deep", "load-stand", "load-fast"),
|
||
0,
|
||
),
|
||
("qe", 500, 4 * KIB, True, ("batch", "refuse"), 100),
|
||
),
|
||
repeats=1,
|
||
refuse_max_changes=10,
|
||
quiet_s=5.0,
|
||
pass_budget_s=300.0,
|
||
),
|
||
}
|
||
|
||
if not hasattr(queue, "CELL_COLUMNS"):
|
||
raise SystemExit(
|
||
"stdlib queue shadows bench/queue.py: run bench/run.py as a script"
|
||
)
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class QueueProfile:
|
||
"""Очередь под перегрузкой (трек 1 лабы 5). Часть методики: поля
|
||
целиком пишутся в `meta.json`; ресурсы worker одни на все настройки —
|
||
меняются только concurrency и prefetch."""
|
||
|
||
name: str
|
||
# настройка worker: имя, WORKER_CONCURRENCY, WORKER_PREFETCH
|
||
configs: tuple[tuple[str, int, int], ...]
|
||
# класс изображения: имя, PNG (noise — шум, flat — один цвет), ширина,
|
||
# высота, тел в фазе ёмкости (меньше backlog_limit: все должны пройти)
|
||
classes: tuple[tuple[str, str, int, int, int], ...]
|
||
repeats: int
|
||
window_s: float = 40.0
|
||
# доли ёмкости ячейки: ровная нагрузка (и после перегрузки) и перегрузка
|
||
steady_factor: float = 0.6
|
||
overload_factor: float = 2.0
|
||
# окно перегрузки: min(overload_max_s, max(window_s, overload_fill ×
|
||
# backlog_limit / ёмкость)) — предел достигается на любом классе
|
||
overload_fill: float = 1.5
|
||
overload_max_s: float = 150.0
|
||
# фаза fixed: одна интенсивность на класс для всех настроек, загрузок/с
|
||
# (≈ 0,5 ёмкости c1p1 по пробе 5 ноября: s 5,87, l 1,79, x 1,09 гот./с
|
||
# — вниз до простого числа)
|
||
fixed_rates: tuple[tuple[str, float], ...] = (("s", 2.5), ("l", 0.8), ("x", 0.5))
|
||
# UPLOAD_MAX_UNFINISHED реплик
|
||
backlog_limit: int = 60
|
||
# загрузок/с при наполнении фазы ёмкости: быстрее, чем worker разберёт,
|
||
# но без 429 по местам загрузок
|
||
fill_rates: tuple[tuple[str, float], ...] = (("s", 25.0), ("l", 4.0), ("x", 10.0))
|
||
upload_timeout_s: float = 60.0
|
||
# потолок ожидания пустой очереди: много больше любой фазы
|
||
settle_timeout_s: float = 900.0
|
||
sample_interval_s: float = 1.0
|
||
seed: int = 20261105
|
||
api_replicas: int = 2
|
||
api_cpus: float = 1.0
|
||
api_memory: str = "512m"
|
||
# бюджет стенда 9,5 CPU оставляет ВМ с 10 CPU запас 0,5 CPU
|
||
db_cpus: float = 1.0
|
||
lb_cpus: float = 0.5
|
||
s3_cpus: float = 1.0
|
||
s3_memory: str = "768m"
|
||
mq_cpus: float = 1.0
|
||
mq_memory: str = "512m"
|
||
worker_cpus: float = 2.0
|
||
worker_memory: str = "1536m"
|
||
render_memory_mb: int = 768
|
||
# генератор открытой модели загрузок — один процесс: ему хватает 1 CPU,
|
||
# а 512 МБ укладываются в бюджет ВМ 10 CPU / 7,65 ГиБ
|
||
gen_cpus: float = 1.0
|
||
gen_memory: str = "512m"
|
||
session_ttl: int = 21_600
|
||
|
||
def limits(self) -> Limits:
|
||
return Limits(
|
||
api_replicas=self.api_replicas,
|
||
api_cpus=self.api_cpus,
|
||
api_memory=self.api_memory,
|
||
db_cpus=self.db_cpus,
|
||
lb_cpus=self.lb_cpus,
|
||
gen_cpus=self.gen_cpus,
|
||
gen_memory=self.gen_memory,
|
||
with_s3=True,
|
||
s3_cpus=self.s3_cpus,
|
||
s3_memory=self.s3_memory,
|
||
with_cleaner=False,
|
||
with_pipeline=True,
|
||
mq_cpus=self.mq_cpus,
|
||
mq_memory=self.mq_memory,
|
||
worker_cpus=self.worker_cpus,
|
||
worker_memory=self.worker_memory,
|
||
)
|
||
|
||
|
||
QUEUE_CONFIGS = (("c1p1", 1, 1), ("c2p2", 2, 2), ("c4p4", 4, 4), ("c2p32", 2, 32))
|
||
# тел фазы ёмкости меньше предела 60: при остановленном worker все они
|
||
# незавершённые, и каждое должно получить 202
|
||
QUEUE_CLASSES = (
|
||
("s", "noise", 320, 200, 50),
|
||
("l", "noise", 1800, 1800, 40),
|
||
("x", "flat", 6000, 6000, 20),
|
||
)
|
||
QUEUE = QueueProfile(
|
||
name="queue", configs=QUEUE_CONFIGS, classes=QUEUE_CLASSES, repeats=2
|
||
)
|
||
QUEUE_PROFILES = {
|
||
"queue": QUEUE,
|
||
"queue-quick": dataclasses.replace(
|
||
QUEUE,
|
||
name="queue-quick",
|
||
configs=(QUEUE_CONFIGS[0], QUEUE_CONFIGS[2]),
|
||
classes=(QUEUE_CLASSES[0], QUEUE_CLASSES[2]),
|
||
repeats=1,
|
||
window_s=15.0,
|
||
),
|
||
}
|
||
# лимит тела загрузки (src/constants.py, копия): класс l обязан в него влезть
|
||
ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024
|
||
|
||
RUN_COLUMNS = (
|
||
"series",
|
||
"config",
|
||
"repeat",
|
||
"cell",
|
||
"step",
|
||
"phase",
|
||
"rate",
|
||
"rate_from",
|
||
"warmup_s",
|
||
"window_s",
|
||
"scheduled",
|
||
"ok",
|
||
"errors",
|
||
"short_pages",
|
||
"dropped",
|
||
"unsent",
|
||
"fail_share",
|
||
"achieved_rps",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"service_p95_ms",
|
||
"lag_p99_ms",
|
||
"slo_pass",
|
||
"abort",
|
||
"api_cpu_cores",
|
||
"api_cpu_max",
|
||
"api_throttled_max",
|
||
"api_memory_max_mib",
|
||
"db_cpu_cores",
|
||
"db_throttled_share",
|
||
"lb_cpu_cores",
|
||
"cache_cpu_cores",
|
||
"gen_cpu_cores",
|
||
"vm_busy_share",
|
||
"pg_conn_max",
|
||
"pg_conn_replica_max",
|
||
"pg_active_mean",
|
||
"pg_lock_wait_mean",
|
||
"pg_new_sessions",
|
||
"pg_commits_per_s",
|
||
"lb_retries",
|
||
"lb_redispatches",
|
||
"lb_conn_errors",
|
||
"api_cpu_us_per_req",
|
||
"db_cpu_us_per_req",
|
||
"instances",
|
||
"instance_share_min",
|
||
"signals",
|
||
"error_kinds",
|
||
)
|
||
SEARCH_COLUMNS = (
|
||
"series",
|
||
"config",
|
||
"replicas",
|
||
"api_cpus",
|
||
"db_cpus",
|
||
"repeat",
|
||
"cell",
|
||
"r_star",
|
||
"first_fail",
|
||
"p95_at_r_star",
|
||
"signals_at_fail",
|
||
"generator_bound",
|
||
"steps",
|
||
"fill_rps",
|
||
"duration_s",
|
||
"flags",
|
||
)
|
||
SAMPLE_COLUMNS = (
|
||
"config",
|
||
"repeat",
|
||
"step",
|
||
"rate",
|
||
"t",
|
||
"role",
|
||
"container",
|
||
"cpu_cores",
|
||
"throttled_share",
|
||
"memory_mib",
|
||
)
|
||
|
||
|
||
# --- общие шаги ---------------------------------------------------------------------
|
||
|
||
|
||
def write_csv(path: Path, columns: tuple[str, ...], rows: list[dict]) -> None:
|
||
with path.open("w", newline="", encoding="utf-8") as handle:
|
||
writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore")
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
|
||
|
||
def append_csv(path: Path, columns: tuple[str, ...], row: dict) -> None:
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as handle:
|
||
writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore")
|
||
if new:
|
||
writer.writeheader()
|
||
writer.writerow(row)
|
||
|
||
|
||
def read_csv(path: Path) -> list[dict]:
|
||
if not path.exists():
|
||
return []
|
||
with path.open(encoding="utf-8") as handle:
|
||
return list(csv.DictReader(handle))
|
||
|
||
|
||
def prepare_image(label: str) -> dict:
|
||
"""Образ из рабочего дерева под тегом метки; на него смотрит сервис api.
|
||
|
||
Образ полной метки остаётся в docker: по нему метку можно сверить
|
||
(`docker image inspect`); пробные `quick-*` убирает `drop_quick_image`.
|
||
"""
|
||
tag = f"{IMAGE_REPO}:bench-{label}"
|
||
say(f"build image {tag}")
|
||
build_image(tag)
|
||
docker("tag", tag, SERVICE_TAG)
|
||
return image_info(tag)
|
||
|
||
|
||
def drop_quick_image(label: str) -> None:
|
||
if label.startswith("quick-"):
|
||
docker("rmi", f"{IMAGE_REPO}:bench-{label}", check=False)
|
||
|
||
|
||
def clock_offset() -> float:
|
||
"""Часы ВМ Docker минус часы хоста, с: по контейнеру генератора.
|
||
|
||
Три замера, берётся с наименьшей задержкой вызова. Нужен только для
|
||
отметок, которые оркестратор ставит сам (начало и конец скрипта
|
||
замены); остальные отметки — часы ВМ.
|
||
"""
|
||
best = None
|
||
for _ in range(3):
|
||
before = time.time()
|
||
out = docker("exec", GEN, "python", "-c", "import time; print(time.time())")
|
||
after = time.time()
|
||
rtt = after - before
|
||
offset = float(out.stdout.strip()) - (before + after) / 2
|
||
if best is None or rtt < best[0]:
|
||
best = (rtt, offset)
|
||
return best[1]
|
||
|
||
|
||
def roles_of(stand: Stand) -> dict[str, str]:
|
||
"""id12 контейнера → роль: api, db, cache, lb, gen."""
|
||
roles = {container[:12]: "api" for container in stand.api_ids()}
|
||
for role, name in (
|
||
("db", DB),
|
||
("cache", CACHE),
|
||
("lb", LB),
|
||
("gen", GEN),
|
||
("sampler", SAMPLER),
|
||
):
|
||
done = docker("inspect", "-f", "{{.Id}}", name, check=False)
|
||
if done.returncode == 0:
|
||
roles[done.stdout.strip()[:12]] = role
|
||
return roles
|
||
|
||
|
||
def full_ids(stand: Stand) -> list[str]:
|
||
ids = list(stand.api_ids())
|
||
for name in (DB, CACHE, LB, GEN):
|
||
ids.append(docker("inspect", "-f", "{{.Id}}", name).stdout.strip())
|
||
return ids
|
||
|
||
|
||
def resources_by_role(ticks: list[dict], roles: dict[str, str]) -> dict:
|
||
"""CPU и память по ролям за окно; у api — сумма и максимум реплики."""
|
||
usage = {cid: analysis.cgroup_usage(ticks, cid) for cid in roles}
|
||
out: dict = {}
|
||
api = [u for cid, u in usage.items() if u and roles[cid] == "api"]
|
||
if api:
|
||
out["api_cpu_cores"] = round(sum(u["cpu_cores"] for u in api), 3)
|
||
out["api_cpu_max"] = max(u["cpu_cores"] for u in api)
|
||
out["api_throttled_max"] = max(u["throttled_share"] for u in api)
|
||
out["api_memory_max_mib"] = max(u["memory_mib"] for u in api)
|
||
for cid, role in roles.items():
|
||
if role == "api" or not usage.get(cid):
|
||
continue
|
||
out[f"{role}_cpu_cores"] = usage[cid]["cpu_cores"]
|
||
out[f"{role}_throttled_share"] = usage[cid]["throttled_share"]
|
||
out[f"{role}_memory_mib"] = usage[cid]["memory_mib"]
|
||
out["vm_busy_share"] = analysis.vm_busy(ticks)
|
||
return out
|
||
|
||
|
||
# --- failover -------------------------------------------------------------------------
|
||
|
||
|
||
def run_failover(profile: LoadProfile, label: str) -> Path:
|
||
workdir = temp_workdir()
|
||
folder = new_label_dir("failover", label)
|
||
(folder / "raw").mkdir()
|
||
started_mono, started_wall = time.monotonic(), time.time()
|
||
pre = preflight(profile.limits.total_cpus(), profile.limits.total_memory_gib())
|
||
image = prepare_image(label)
|
||
meta = {
|
||
"kind": "failover",
|
||
"label": label,
|
||
"started": utc_now(),
|
||
"profile": dataclasses.asdict(profile),
|
||
"host": pre["host"],
|
||
"docker": pre["docker"],
|
||
"budget": pre["budget"],
|
||
"host_warnings": pre["warnings"],
|
||
"image": image,
|
||
"code": code_state(image),
|
||
}
|
||
write_json(folder / "meta.json", meta)
|
||
stand = Stand(profile.limits, session_ttl=profile.session_ttl, workdir=workdir)
|
||
phases, timeline_rows, resource_rows = [], [], []
|
||
try:
|
||
say("stand: postgres, valkey, migrate, 2 replicas, haproxy")
|
||
stand.start()
|
||
setup = stand.gen(
|
||
"setup",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--users",
|
||
str(profile.users),
|
||
"--read-messages",
|
||
str(profile.read_messages),
|
||
"--write-chats",
|
||
"1",
|
||
)
|
||
stand.maintain()
|
||
meta["setup"] = {k: v for k, v in setup.items() if k != "write_chats"}
|
||
meta["stand"] = stand.passport()
|
||
write_chat = setup["write_chats"][0]
|
||
targets = stand.api_ids()[: profile.targets]
|
||
roles = roles_of(stand)
|
||
stand.start_sampler("scan")
|
||
number = 0
|
||
for index, target in enumerate(targets, start=1):
|
||
for mode in profile.modes:
|
||
number += 1
|
||
name = f"p{number}-{mode}-r{index}"
|
||
say(f"phase {name}: target {target[:12]}")
|
||
phase, rows = failover_phase(
|
||
stand, profile, name, mode, target, write_chat, folder
|
||
)
|
||
phases.append(phase)
|
||
timeline_rows += rows
|
||
ticks = stand.sampler_lines()
|
||
roles.update(roles_of(stand))
|
||
for phase in phases:
|
||
for window, (start, end) in phase.pop("_windows").items():
|
||
usage = resources_by_role(
|
||
analysis.window_ticks(ticks, start / 1000, end / 1000), roles
|
||
)
|
||
ok_rps = phase[f"ok_rps_{window}"]
|
||
resource_rows.append(
|
||
{
|
||
"phase": phase["phase"],
|
||
"window": window,
|
||
**usage,
|
||
"api_cpu_us_per_req": _per_req(
|
||
usage.get("api_cpu_cores"), ok_rps
|
||
),
|
||
"db_cpu_us_per_req": _per_req(
|
||
usage.get("db_cpu_cores"), ok_rps
|
||
),
|
||
},
|
||
)
|
||
meta["sampler_ticks"] = len(ticks)
|
||
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
|
||
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
|
||
say(f"aborted: {meta['aborted']}")
|
||
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
|
||
finally:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
drop_quick_image(label)
|
||
meta["finished"] = utc_now()
|
||
finish_host_state(meta, started_mono, started_wall)
|
||
write_json(folder / "meta.json", meta)
|
||
write_csv(folder / "phases.csv", PHASE_COLUMNS, phases)
|
||
write_csv(folder / "timeline.csv", TIMELINE_COLUMNS, timeline_rows)
|
||
write_csv(folder / "resources.csv", RESOURCE_COLUMNS, resource_rows)
|
||
(folder / "summary.md").write_text(
|
||
failover_summary(meta, phases, resource_rows), encoding="utf-8"
|
||
)
|
||
say(f"done: {folder.relative_to(ROOT)}")
|
||
if meta.get("aborted"):
|
||
raise SystemExit(1)
|
||
return folder
|
||
|
||
|
||
PHASE_COLUMNS = (
|
||
"phase",
|
||
"mode",
|
||
"replica",
|
||
"target",
|
||
"slot",
|
||
"action_at_ms",
|
||
"restore_at_ms",
|
||
"detection_s",
|
||
"bound_s",
|
||
"over_bound",
|
||
"lb_return_s",
|
||
"errors_before",
|
||
"errors_fail",
|
||
"errors_after",
|
||
"error_kinds",
|
||
"first_error_s",
|
||
"last_error_s",
|
||
"recovery_s",
|
||
"ok_rps_before",
|
||
"ok_rps_fail",
|
||
"ok_rps_after",
|
||
"p95_before_ms",
|
||
"p95_fail_ms",
|
||
"p95_after_ms",
|
||
"max_fail_ms",
|
||
"target_after_down",
|
||
"writes",
|
||
"retried",
|
||
"recovered",
|
||
"rows",
|
||
"distinct_texts",
|
||
"verified",
|
||
"lb_errors",
|
||
)
|
||
TIMELINE_COLUMNS = (
|
||
"phase",
|
||
"second",
|
||
"sent",
|
||
"ok_write",
|
||
"ok_read",
|
||
"ok_target",
|
||
"ok_other",
|
||
"ok_old",
|
||
"ok_new",
|
||
"errors",
|
||
"error_kinds",
|
||
"p95_ms",
|
||
"max_ms",
|
||
)
|
||
RESOURCE_COLUMNS = (
|
||
"phase",
|
||
"window",
|
||
"api_cpu_cores",
|
||
"api_cpu_max",
|
||
"api_throttled_max",
|
||
"api_memory_max_mib",
|
||
"db_cpu_cores",
|
||
"db_memory_mib",
|
||
"lb_cpu_cores",
|
||
"cache_cpu_cores",
|
||
"gen_cpu_cores",
|
||
"vm_busy_share",
|
||
"api_cpu_us_per_req",
|
||
"db_cpu_us_per_req",
|
||
)
|
||
|
||
|
||
def _per_req(cores: float | None, rps: float | None) -> float | None:
|
||
if not cores or not rps:
|
||
return None
|
||
return round(cores * 1e6 / rps, 1)
|
||
|
||
|
||
def start_events(path: Path, *filters: str) -> subprocess.Popen:
|
||
"""Поток событий Docker в файл: буфер демона хранит лишь последние
|
||
события всех контейнеров, поэтому — поток с начала фазы."""
|
||
handle = path.open("w", encoding="utf-8")
|
||
process = subprocess.Popen(
|
||
["docker", "events", *filters, "--format", "{{json .}}"],
|
||
stdout=handle,
|
||
stderr=subprocess.DEVNULL,
|
||
cwd=ROOT,
|
||
)
|
||
process.handle = handle # type: ignore[attr-defined]
|
||
return process
|
||
|
||
|
||
def stop_events(process: subprocess.Popen) -> None:
|
||
time.sleep(0.5)
|
||
process.terminate()
|
||
try:
|
||
process.wait(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
process.kill()
|
||
process.handle.close() # type: ignore[attr-defined]
|
||
|
||
|
||
def lb_log_since(since_wall: float) -> str:
|
||
"""Журнал HAProxy стенда с отметками ВМ (`docker logs -t`)."""
|
||
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(since_wall - 5))
|
||
done = docker("logs", "-t", "--since", stamp, LB, check=False)
|
||
return done.stdout + done.stderr
|
||
|
||
|
||
def collect_requests(stand: Stand, logs: list[str]) -> str:
|
||
return "".join(stand.gen_file(path) for path in logs)
|
||
|
||
|
||
def verify_writes(stand: Stand, chat: str, prefix: str) -> tuple[int, int]:
|
||
"""Строк и разных текстов в чате записи с ключами фазы."""
|
||
row = stand.psql(
|
||
"SELECT count(*), count(DISTINCT text) FROM messages "
|
||
f"WHERE chat_id = '{chat}' AND text LIKE '%{prefix}.%';\n",
|
||
)
|
||
rows, distinct = (int(value) for value in row.split("|"))
|
||
return rows, distinct
|
||
|
||
|
||
def failover_phase(
|
||
stand: Stand,
|
||
profile: LoadProfile,
|
||
name: str,
|
||
mode: str,
|
||
target: str,
|
||
write_chat: str,
|
||
folder: Path,
|
||
) -> tuple[dict, list[dict]]:
|
||
stand.wait_lb_up(profile.limits.api_replicas)
|
||
address = stand.address(target)
|
||
slot = next(s["slot"] for s in stand.lb_slots() if s["addr"] == address)
|
||
events_path = stand.workdir / f"{name}-events.log"
|
||
events = start_events(events_path, "--filter", f"container={target}")
|
||
since = time.time()
|
||
duration = profile.before_s + profile.fail_s + profile.restore_s + profile.after_s
|
||
log = f"/tmp/{name}.log"
|
||
load = stand.gen_popen(
|
||
"constant",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--write-rate",
|
||
str(profile.write_rate),
|
||
"--read-rate",
|
||
str(profile.read_rate),
|
||
"--duration",
|
||
str(duration),
|
||
"--procs",
|
||
str(profile.gen_procs),
|
||
"--seed",
|
||
str(profile.seed),
|
||
"--key-prefix",
|
||
name,
|
||
"--log",
|
||
log,
|
||
)
|
||
try:
|
||
time.sleep(profile.before_s + 0.3)
|
||
if mode == "stop":
|
||
# как scripts/stop-one.sh: graceful, срок 9 с
|
||
docker("stop", "--timeout", "9", target)
|
||
elif mode == "kill":
|
||
docker("kill", target)
|
||
else:
|
||
docker("pause", target)
|
||
time.sleep(profile.fail_s)
|
||
docker("unpause" if mode == "pause" else "start", target)
|
||
stand.wait_slot(address, "UP", timeout=profile.restore_s + 20)
|
||
out, err = load.communicate(timeout=duration + 60)
|
||
finally:
|
||
if load.poll() is None:
|
||
load.kill()
|
||
stop_events(events)
|
||
if load.returncode != 0:
|
||
raise StandError(f"generator failed in {name}: {err.strip()[-500:]}")
|
||
summary = json.loads(out.strip().splitlines()[-1])
|
||
raw = collect_requests(stand, summary["logs"])
|
||
requests = analysis.parse_requests(raw)
|
||
retry = stand.gen("retry", "--fixture", FIXTURE, *summary["logs"])
|
||
rows, distinct = verify_writes(stand, write_chat, name)
|
||
lb_text = lb_log_since(since)
|
||
events_list = analysis.parse_events(events_path.read_text(encoding="utf-8"))
|
||
(folder / "raw" / f"{name}-events.log").write_text(
|
||
events_path.read_text(encoding="utf-8"), encoding="utf-8"
|
||
)
|
||
(folder / "raw" / f"{name}-lb.log").write_text(lb_text, encoding="utf-8")
|
||
with gzip.open(
|
||
folder / "raw" / f"{name}-requests.csv.gz", "wt", encoding="ascii"
|
||
) as fh:
|
||
fh.write(raw)
|
||
|
||
action_names = {
|
||
"stop": ("kill", "stop", "die"),
|
||
"kill": ("kill", "die"),
|
||
"pause": ("pause",),
|
||
}
|
||
action_at = next(t for t, a, _ in events_list if a in action_names[mode])
|
||
restore_at = next(
|
||
t
|
||
for t, a, _ in events_list
|
||
if a == ("unpause" if mode == "pause" else "start") and t > action_at
|
||
)
|
||
lb_events = [e for e in analysis.parse_lb_log(lb_text) if e[1] == slot]
|
||
down = next(
|
||
(
|
||
t
|
||
for t, _, s in lb_events
|
||
if t >= action_at and s in ("is DOWN", "is going DOWN")
|
||
),
|
||
None,
|
||
)
|
||
up = next((t for t, _, s in lb_events if t >= restore_at and s == "is UP"), None)
|
||
load_start = summary["start"] * 1000
|
||
load_end = load_start + duration * 1000
|
||
windows = {
|
||
"before": (load_start + 2000, action_at),
|
||
"fail": (action_at, restore_at),
|
||
"after": (up or restore_at, load_end),
|
||
}
|
||
stats = {w: analysis.window_stats(requests, s, e) for w, (s, e) in windows.items()}
|
||
errors_all = [r for r in requests if not r.ok and r.scheduled >= load_start]
|
||
bound = dict(profile.bounds)[mode]
|
||
detection = round((down - action_at) / 1000, 2) if down else None
|
||
target_id = target[:12]
|
||
target_after = sum(
|
||
1
|
||
for r in requests
|
||
if r.ok
|
||
and r.instance == target_id
|
||
and down
|
||
and down + 1000 <= r.scheduled < restore_at
|
||
)
|
||
phase = {
|
||
"phase": name,
|
||
"mode": mode,
|
||
"replica": name.rsplit("-", 1)[1],
|
||
"target": target_id,
|
||
"slot": slot,
|
||
"action_at_ms": round(action_at, 1),
|
||
"restore_at_ms": round(restore_at, 1),
|
||
"detection_s": detection,
|
||
"bound_s": bound,
|
||
"over_bound": bool(detection is None or detection > bound),
|
||
"lb_return_s": round((up - restore_at) / 1000, 2) if up else None,
|
||
"errors_before": stats["before"]["errors"],
|
||
"errors_fail": stats["fail"]["errors"],
|
||
"errors_after": stats["after"]["errors"],
|
||
"error_kinds": ";".join(
|
||
f"{k}:{v}"
|
||
for k, v in sorted(
|
||
{
|
||
kind: sum(analysis.error_label(r) == kind for r in errors_all)
|
||
for kind in {analysis.error_label(r) for r in errors_all}
|
||
}.items()
|
||
)
|
||
),
|
||
"first_error_s": round((errors_all[0].scheduled - action_at) / 1000, 2)
|
||
if errors_all
|
||
else None,
|
||
"last_error_s": round((errors_all[-1].scheduled - action_at) / 1000, 2)
|
||
if errors_all
|
||
else None,
|
||
"recovery_s": analysis.recovery_seconds(
|
||
requests, action_at, restore_at, profile.write_rate + profile.read_rate
|
||
),
|
||
"ok_rps_before": stats["before"]["ok_rps"],
|
||
"ok_rps_fail": stats["fail"]["ok_rps"],
|
||
"ok_rps_after": stats["after"]["ok_rps"],
|
||
"p95_before_ms": stats["before"]["p95_ms"],
|
||
"p95_fail_ms": stats["fail"]["p95_ms"],
|
||
"p95_after_ms": stats["after"]["p95_ms"],
|
||
"max_fail_ms": stats["fail"]["max_ms"],
|
||
"target_after_down": target_after,
|
||
"writes": retry["keys"],
|
||
"retried": retry["failed"],
|
||
"recovered": retry["recovered"],
|
||
"rows": rows,
|
||
"distinct_texts": distinct,
|
||
"verified": rows == distinct == retry["keys"],
|
||
"lb_errors": ";".join(
|
||
f"{k}:{v}" for k, v in sorted(analysis.lb_http_errors(lb_text).items())
|
||
),
|
||
"_windows": windows,
|
||
}
|
||
if not phase["verified"]:
|
||
raise StandError(
|
||
f"{name}: {retry['keys']} keys sent, {rows} rows, {distinct} distinct texts"
|
||
)
|
||
groups = {
|
||
"target": {target_id},
|
||
"other": {s[:12] for s in stand.api_ids()} - {target_id},
|
||
}
|
||
rows_timeline = [
|
||
{"phase": name, **row}
|
||
for row in analysis.timeline(requests, action_at, load_start, load_end, groups)
|
||
]
|
||
return phase, rows_timeline
|
||
|
||
|
||
def failover_summary(meta: dict, phases: list[dict], resources: list[dict]) -> str:
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Отказ реплики под нагрузкой: {meta['label']}",
|
||
"",
|
||
_passport_lines(meta),
|
||
"",
|
||
f"Нагрузка: открытая модель, {profile['write_rate']:.0f} записей с "
|
||
f"`Idempotency-Key` и {profile['read_rate']:.0f} чтений страницы истории "
|
||
"в секунду через HAProxy стенда; две реплики с лимитами рабочего стенда. "
|
||
f"Фаза: {profile['before_s']:.0f} с до действия, действие, "
|
||
f"{profile['fail_s']:.0f} с отказа, возврат той же реплики, ещё "
|
||
f"{profile['after_s']:.0f} с.",
|
||
"",
|
||
"Отметки — часы ВМ Docker: действие и возврат — события Docker, "
|
||
"исключение и возврат в LB — журнал HAProxy, запросы — генератор "
|
||
"(момент по расписанию). Обнаружение — от действия до `DOWN` слота; "
|
||
"восстановление — от действия до начала секунды, после которой до "
|
||
"возврата нет ошибок и успешных ≥ 90 % поданной нагрузки.",
|
||
"",
|
||
"| Фаза | Режим | Реплика | Обнаружение, с (граница) | Возврат в LB, с | Ошибки до / отказ / после | Виды ошибок | Восстановление, с | Успешных/с до / отказ / после | p95, мс до / отказ / после | max в отказе, мс | Повтор: не прошло → прошло | Сверка |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for p in phases:
|
||
lines.append(
|
||
f"| {p['phase']} | {p['mode']} | {p['replica']} ({p['target']}) | "
|
||
f"{_v(p['detection_s'])} ({p['bound_s']}){' ⚠' if p['over_bound'] else ''} | "
|
||
f"{_v(p['lb_return_s'])} | {p['errors_before']} / {p['errors_fail']} / "
|
||
f"{p['errors_after']} | {p['error_kinds'] or '—'} | {_v(p['recovery_s'])} | "
|
||
f"{p['ok_rps_before']} / {p['ok_rps_fail']} / {p['ok_rps_after']} | "
|
||
f"{_v(p['p95_before_ms'])} / {_v(p['p95_fail_ms'])} / {_v(p['p95_after_ms'])} | "
|
||
f"{_v(p['max_fail_ms'])} | {p['retried']} → {p['recovered']} | "
|
||
f"{'ok' if p['verified'] else 'нет'}: {p['writes']} ключей = {p['rows']} строк |"
|
||
)
|
||
if resources:
|
||
lines += [
|
||
"",
|
||
"## CPU и память по окнам фазы",
|
||
"",
|
||
"Сэмплер (cgroup каждого контейнера раз в секунду). Окна: до "
|
||
"действия (от первого запроса + 2 с), отказ (действие → возврат), "
|
||
"после (возврат в LB → конец нагрузки). CPU реплик — сумма; на "
|
||
"успешный запрос — только по окнам с нагрузкой.",
|
||
"",
|
||
"| Фаза | Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | Valkey | Генератор | ВМ занята | API, мкс/запрос | PostgreSQL, мкс/запрос |",
|
||
"|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in resources:
|
||
lines.append(
|
||
f"| {r['phase']} | {r['window']} | {_v(r.get('api_cpu_cores'))} "
|
||
f"({_v(r.get('api_cpu_max'))}) | {_v(r.get('db_cpu_cores'))} | "
|
||
f"{_v(r.get('lb_cpu_cores'))} | {_v(r.get('cache_cpu_cores'))} | "
|
||
f"{_v(r.get('gen_cpu_cores'))} | {_v(r.get('vm_busy_share'))} | "
|
||
f"{_v(r.get('api_cpu_us_per_req'))} | {_v(r.get('db_cpu_us_per_req'))} |"
|
||
)
|
||
lines += _warning_lines(meta)
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def _v(value) -> str:
|
||
if value is None or value == "":
|
||
return "—"
|
||
if isinstance(value, float):
|
||
return f"{value:g}"
|
||
return str(value)
|
||
|
||
|
||
def _passport_lines(meta: dict) -> str:
|
||
code = meta.get("code", {})
|
||
commit = (code.get("git_commit") or "")[:12] or "git нет"
|
||
dock = meta.get("docker", {})
|
||
host = meta.get("host", {})
|
||
return (
|
||
f"Код: src_sha256 `{code.get('src_sha256', '')[:16]}`, commit {commit}. "
|
||
f"Хост: {host.get('cpu_model')}, {host.get('ram_gb')} ГиБ; ВМ Docker "
|
||
f"{dock.get('NCPU')} CPU, {round((dock.get('MemTotal') or 0) / 2**30, 1)} ГиБ, "
|
||
f"Docker {dock.get('ServerVersion')}, compose {dock.get('compose_version')}. "
|
||
f"Начало {meta.get('started')}, конец {meta.get('finished')}."
|
||
)
|
||
|
||
|
||
def _warning_lines(meta: dict) -> list[str]:
|
||
lines = []
|
||
if meta.get("host_warnings"):
|
||
lines += ["", "## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
if meta.get("aborted"):
|
||
lines += ["", f"**Прогон прерван:** {meta['aborted']}"]
|
||
return lines
|
||
|
||
|
||
# --- rollout --------------------------------------------------------------------------
|
||
|
||
|
||
def run_rollout(profile: LoadProfile, label: str) -> Path:
|
||
workdir = temp_workdir()
|
||
folder = new_label_dir("rollout", label)
|
||
(folder / "raw").mkdir()
|
||
started_mono, started_wall = time.monotonic(), time.time()
|
||
# на пике — реплик на одну больше
|
||
pre = preflight(
|
||
profile.limits.total_cpus(profile.limits.api_replicas + 1),
|
||
profile.limits.total_memory_gib(profile.limits.api_replicas + 1),
|
||
)
|
||
old = prepare_image(label)
|
||
say("build the new image: working tree + expand revision 9001_rollout_probe")
|
||
context, head = probe_migration_context(workdir)
|
||
new_tag = f"{IMAGE_REPO}:bench-{label}-new"
|
||
build_image(new_tag, context)
|
||
new = image_info(new_tag)
|
||
docker("tag", old["tag"], SERVICE_TAG)
|
||
meta = {
|
||
"kind": "rollout",
|
||
"label": label,
|
||
"started": utc_now(),
|
||
"profile": dataclasses.asdict(profile),
|
||
"host": pre["host"],
|
||
"docker": pre["docker"],
|
||
"budget": pre["budget"],
|
||
"host_warnings": pre["warnings"],
|
||
"image": old,
|
||
"image_new": new,
|
||
"code": code_state(old),
|
||
"rollout": {"schema_head_before": head, "probe_revision": "9001"},
|
||
}
|
||
write_json(folder / "meta.json", meta)
|
||
stand = Stand(profile.limits, session_ttl=profile.session_ttl, workdir=workdir)
|
||
windows_rows, timeline_rows, resource_rows = [], [], []
|
||
try:
|
||
say("stand: postgres, valkey, migrate, 2 replicas on the old image, haproxy")
|
||
stand.start()
|
||
setup = stand.gen(
|
||
"setup",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--users",
|
||
str(profile.users),
|
||
"--read-messages",
|
||
str(profile.read_messages),
|
||
"--write-chats",
|
||
"1",
|
||
)
|
||
stand.maintain()
|
||
meta["setup"] = {k: v for k, v in setup.items() if k != "write_chats"}
|
||
meta["stand"] = stand.passport()
|
||
write_chat = setup["write_chats"][0]
|
||
before_ids = stand.api_ids()
|
||
roles = roles_of(stand)
|
||
rollout = meta["rollout"]
|
||
rollout["replicas_before"] = [c[:12] for c in before_ids]
|
||
rollout["images_before"] = sorted(
|
||
{
|
||
docker("inspect", "-f", "{{.Image}}", c).stdout.strip()
|
||
for c in before_ids
|
||
}
|
||
)
|
||
rollout["alembic_before"] = stand.psql(
|
||
"SELECT version_num FROM alembic_version;\n"
|
||
)
|
||
offset = clock_offset()
|
||
rollout["clock_offset_s"] = round(offset, 4)
|
||
stand.start_sampler("scan")
|
||
events_path = workdir / "rollout-events.log"
|
||
events = start_events(
|
||
events_path, "--filter", f"label=com.docker.compose.project={PROJECT}"
|
||
)
|
||
since = time.time()
|
||
duration = (
|
||
profile.rollout_before_s
|
||
+ profile.rollout_budget_s
|
||
+ profile.rollout_after_s
|
||
)
|
||
load = stand.gen_popen(
|
||
"constant",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--write-rate",
|
||
str(profile.write_rate),
|
||
"--read-rate",
|
||
str(profile.read_rate),
|
||
"--duration",
|
||
str(duration),
|
||
"--procs",
|
||
str(profile.gen_procs),
|
||
"--seed",
|
||
str(profile.seed),
|
||
"--key-prefix",
|
||
"rollout",
|
||
"--log",
|
||
"/tmp/rollout.log",
|
||
)
|
||
try:
|
||
time.sleep(profile.rollout_before_s + 0.3)
|
||
docker("tag", new_tag, SERVICE_TAG)
|
||
script_start = time.time() + offset
|
||
script = subprocess.run(
|
||
[str(ROOT / "scripts" / "rolling-update.sh")],
|
||
cwd=ROOT,
|
||
env={
|
||
**stand.script_env(),
|
||
"API_REPLICAS": str(profile.limits.api_replicas),
|
||
"ROLLOUT_BUILD": "0",
|
||
},
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=profile.rollout_budget_s + 60,
|
||
check=False,
|
||
)
|
||
script_end = time.time() + offset
|
||
out, err = load.communicate(timeout=duration + 60)
|
||
finally:
|
||
if load.poll() is None:
|
||
load.kill()
|
||
stop_events(events)
|
||
script_lines = [
|
||
line
|
||
for line in (script.stdout + script.stderr).splitlines()
|
||
if "rolling-update:" in line
|
||
]
|
||
(folder / "raw" / "rollout-script.log").write_text(
|
||
"\n".join(script_lines) + "\n", encoding="utf-8"
|
||
)
|
||
rollout["script_exit"] = script.returncode
|
||
rollout["script_seconds"] = round(script_end - script_start, 1)
|
||
if load.returncode != 0:
|
||
raise StandError(f"generator failed: {err.strip()[-500:]}")
|
||
if script.returncode != 0:
|
||
raise StandError(
|
||
f"rolling-update.sh exit {script.returncode}:\n"
|
||
+ "\n".join(script_lines[-10:])
|
||
)
|
||
summary = json.loads(out.strip().splitlines()[-1])
|
||
raw = collect_requests(stand, summary["logs"])
|
||
requests = analysis.parse_requests(raw)
|
||
retry = stand.gen("retry", "--fixture", FIXTURE, *summary["logs"])
|
||
rows, distinct = verify_writes(stand, write_chat, "rollout")
|
||
lb_text = lb_log_since(since)
|
||
(folder / "raw" / "rollout-lb.log").write_text(lb_text, encoding="utf-8")
|
||
(folder / "raw" / "rollout-events.log").write_text(
|
||
events_path.read_text(encoding="utf-8"), encoding="utf-8"
|
||
)
|
||
with gzip.open(
|
||
folder / "raw" / "rollout-requests.csv.gz", "wt", encoding="ascii"
|
||
) as fh:
|
||
fh.write(raw)
|
||
|
||
after_ids = stand.api_ids()
|
||
rollout["replicas_after"] = [c[:12] for c in after_ids]
|
||
rollout["images_after"] = sorted(
|
||
{docker("inspect", "-f", "{{.Image}}", c).stdout.strip() for c in after_ids}
|
||
)
|
||
rollout["alembic_after"] = stand.psql(
|
||
"SELECT version_num FROM alembic_version;\n"
|
||
)
|
||
rollout["probe_column"] = (
|
||
stand.psql(
|
||
"SELECT count(*) FROM information_schema.columns "
|
||
"WHERE table_name = 'messages' AND column_name = 'rollout_probe';\n"
|
||
)
|
||
== "1"
|
||
)
|
||
stand.wait_lb_up(profile.limits.api_replicas, timeout=30)
|
||
events_list = analysis.parse_events(events_path.read_text(encoding="utf-8"))
|
||
old_ids = {c[:12] for c in before_ids}
|
||
new_ids = {c[:12] for c in after_ids}
|
||
rollout["checks"] = {
|
||
"replicas_after": len(after_ids) == profile.limits.api_replicas,
|
||
"no_old_replica": not (old_ids & new_ids),
|
||
# образ контейнера — digest конфигурации (хранилище containerd), не
|
||
# Id тега: сравниваются контейнеры до и после между собой
|
||
"single_new_image": len(rollout["images_after"]) == 1
|
||
and rollout["images_after"] != rollout["images_before"],
|
||
"migrated": rollout["alembic_after"] == "9001",
|
||
"probe_column": rollout["probe_column"],
|
||
"writes_verified": rows == distinct == retry["keys"],
|
||
}
|
||
load_start = summary["start"] * 1000
|
||
load_end = load_start + duration * 1000
|
||
windows = {
|
||
"before": (load_start + 2000, script_start * 1000),
|
||
"during": (script_start * 1000, script_end * 1000),
|
||
"after": (script_end * 1000, load_end),
|
||
}
|
||
for window, (start, end) in windows.items():
|
||
stats = analysis.window_stats(requests, start, end)
|
||
windows_rows.append(
|
||
{
|
||
"window": window,
|
||
**{k: v for k, v in stats.items() if k != "error_kinds"},
|
||
"error_kinds": ";".join(
|
||
f"{k}:{v}" for k, v in sorted(stats["error_kinds"].items())
|
||
),
|
||
"ok_old": sum(
|
||
r.ok and r.instance in old_ids
|
||
for r in requests
|
||
if start <= r.scheduled < end
|
||
),
|
||
"ok_new": sum(
|
||
r.ok and r.instance in new_ids
|
||
for r in requests
|
||
if start <= r.scheduled < end
|
||
),
|
||
},
|
||
)
|
||
container_events = [
|
||
{
|
||
"t_s": round((t - script_start * 1000) / 1000, 2),
|
||
"action": a,
|
||
"container": c[:12],
|
||
# остальные контейнеры проекта за замену — миграция
|
||
"role": "old"
|
||
if c[:12] in old_ids
|
||
else "new"
|
||
if c[:12] in new_ids
|
||
else "migrate",
|
||
}
|
||
for t, a, c in events_list
|
||
if a in ("create", "start", "kill", "die", "stop", "destroy")
|
||
]
|
||
rollout["container_events"] = container_events
|
||
rollout["retry"] = retry
|
||
rollout["rows"] = rows
|
||
rollout["distinct_texts"] = distinct
|
||
rollout["lb_errors"] = analysis.lb_http_errors(lb_text)
|
||
timeline_rows = [
|
||
{"phase": "rollout", **row}
|
||
for row in analysis.timeline(
|
||
requests,
|
||
script_start * 1000,
|
||
load_start,
|
||
load_end,
|
||
{"old": old_ids, "new": new_ids},
|
||
)
|
||
]
|
||
ticks = stand.sampler_lines()
|
||
meta["sampler_ticks"] = len(ticks)
|
||
for cid in new_ids:
|
||
roles[cid] = "api"
|
||
# созданные за замену контейнеры, кроме реплик, — контейнер миграции
|
||
for _, action, container in events_list:
|
||
if action == "create" and container[:12] not in roles:
|
||
roles[container[:12]] = "migrate"
|
||
for window, (start, end) in windows.items():
|
||
usage = resources_by_role(
|
||
analysis.window_ticks(ticks, start / 1000, end / 1000), roles
|
||
)
|
||
ok_rps = next(w["ok_rps"] for w in windows_rows if w["window"] == window)
|
||
resource_rows.append(
|
||
{
|
||
"phase": "rollout",
|
||
"window": window,
|
||
**usage,
|
||
"api_cpu_us_per_req": _per_req(usage.get("api_cpu_cores"), ok_rps),
|
||
"db_cpu_us_per_req": _per_req(usage.get("db_cpu_cores"), ok_rps),
|
||
},
|
||
)
|
||
if not all(rollout["checks"].values()):
|
||
raise StandError(f"rollout checks failed: {rollout['checks']}")
|
||
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
|
||
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
|
||
say(f"aborted: {meta['aborted']}")
|
||
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
|
||
finally:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
docker("rmi", f"{IMAGE_REPO}:bench-{label}-new", check=False)
|
||
drop_quick_image(label)
|
||
meta["finished"] = utc_now()
|
||
finish_host_state(meta, started_mono, started_wall)
|
||
write_json(folder / "meta.json", meta)
|
||
write_csv(
|
||
folder / "windows.csv",
|
||
(
|
||
"window",
|
||
"seconds",
|
||
"requests",
|
||
"ok",
|
||
"ok_rps",
|
||
"ok_write_rps",
|
||
"ok_read_rps",
|
||
"errors",
|
||
"error_kinds",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"ok_old",
|
||
"ok_new",
|
||
),
|
||
windows_rows,
|
||
)
|
||
write_csv(folder / "timeline.csv", TIMELINE_COLUMNS, timeline_rows)
|
||
write_csv(folder / "resources.csv", RESOURCE_COLUMNS, resource_rows)
|
||
(folder / "summary.md").write_text(
|
||
rollout_summary(meta, windows_rows, resource_rows), encoding="utf-8"
|
||
)
|
||
say(f"done: {folder.relative_to(ROOT)}")
|
||
if meta.get("aborted"):
|
||
raise SystemExit(1)
|
||
return folder
|
||
|
||
|
||
def rollout_summary(meta: dict, windows: list[dict], resources: list[dict]) -> str:
|
||
profile, rollout = meta["profile"], meta.get("rollout", {})
|
||
lines = [
|
||
f"# Поочерёдная замена под нагрузкой: {meta['label']}",
|
||
"",
|
||
_passport_lines(meta),
|
||
"",
|
||
f"Нагрузка: {profile['write_rate']:.0f} записей с ключом и "
|
||
f"{profile['read_rate']:.0f} чтений в секунду через HAProxy стенда, "
|
||
"открытая модель. Две реплики на образе рабочего дерева; посреди "
|
||
"нагрузки `scripts/rolling-update.sh` (тот же, что `make rolling-update`) "
|
||
"меняет их на образ с expand-ревизией `9001_rollout_probe` (колонка "
|
||
"`messages.rollout_probe` NULL): миграция один раз, затем по одной "
|
||
"реплике — новая `UP` в LB, старая `drain` → `maint` → stop → rm.",
|
||
"",
|
||
f"- Скрипт: код выхода {rollout.get('script_exit')}, "
|
||
f"{rollout.get('script_seconds')} с; ревизия схемы "
|
||
f"{rollout.get('alembic_before')} → {rollout.get('alembic_after')}; колонка "
|
||
f"есть: {rollout.get('probe_column')}.",
|
||
f"- Реплики до: {', '.join(rollout.get('replicas_before', []))}; после: "
|
||
f"{', '.join(rollout.get('replicas_after', []))}.",
|
||
f"- Проверки: {rollout.get('checks')}.",
|
||
f"- Повтор неуспешных записей с тем же ключом: {rollout.get('retry')}; "
|
||
f"строк в чате записи {rollout.get('rows')}, разных текстов "
|
||
f"{rollout.get('distinct_texts')}.",
|
||
f"- Ответы самого HAProxy с ошибкой: {rollout.get('lb_errors') or 'нет'}.",
|
||
"",
|
||
"| Окно | с | Запросов | Успешных/с | Ошибок | Виды | p95, мс | max, мс | Ответов старых | Ответов новых |",
|
||
"|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for w in windows:
|
||
lines.append(
|
||
f"| {w['window']} | {w['seconds']} | {w['requests']} | {w['ok_rps']} | "
|
||
f"{w['errors']} | {w['error_kinds'] or '—'} | {_v(w['p95_ms'])} | "
|
||
f"{_v(w['max_ms'])} | {w['ok_old']} | {w['ok_new']} |"
|
||
)
|
||
lines += ["", "## Контейнеры (секунды от начала скрипта)", ""]
|
||
for event in rollout.get("container_events", []):
|
||
lines.append(
|
||
f"- {event['t_s']:+.2f} с {event['action']} {event['role']} {event['container']}"
|
||
)
|
||
if resources:
|
||
lines += [
|
||
"",
|
||
"## CPU и память по окнам",
|
||
"",
|
||
"| Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | ВМ занята | API, мкс/запрос |",
|
||
"|---|---|---|---|---|---|",
|
||
]
|
||
for r in resources:
|
||
lines.append(
|
||
f"| {r['window']} | {_v(r.get('api_cpu_cores'))} ({_v(r.get('api_cpu_max'))}) | "
|
||
f"{_v(r.get('db_cpu_cores'))} | {_v(r.get('lb_cpu_cores'))} | "
|
||
f"{_v(r.get('vm_busy_share'))} | {_v(r.get('api_cpu_us_per_req'))} |"
|
||
)
|
||
lines += _warning_lines(meta)
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
# --- scaling (трек 1) -----------------------------------------------------------------
|
||
|
||
|
||
def parse_config(name: str, base: Limits) -> Limits:
|
||
"""`4x1` — 4 реплики по 1 CPU; `-pg0.5` — PostgreSQL 0,5 CPU;
|
||
`-mc50` — max_connections 50. Память реплики — 512 МиБ на CPU."""
|
||
head, *options = name.split("-")
|
||
replicas, cpus = head.split("x")
|
||
limits = dataclasses.replace(
|
||
base,
|
||
api_replicas=int(replicas),
|
||
api_cpus=float(cpus),
|
||
api_memory=f"{int(512 * float(cpus))}m",
|
||
)
|
||
for option in options:
|
||
if option.startswith("pg"):
|
||
limits = dataclasses.replace(limits, db_cpus=float(option[2:]))
|
||
elif option.startswith("mc"):
|
||
limits = dataclasses.replace(limits, db_max_connections=int(option[2:]))
|
||
else:
|
||
raise SystemExit(f"unknown config option {option!r} in {name!r}")
|
||
return limits
|
||
|
||
|
||
def cell_plan(profile: ScalingProfile) -> list[dict]:
|
||
"""Ячейки по кругам: в каждом круге — все конфигурации в случайном
|
||
порядке (зерно профиля); одна конфигурация не идёт дважды подряд."""
|
||
plan, previous = [], None
|
||
for repeat in range(1, profile.repeats + 1):
|
||
order = list(profile.configs)
|
||
random.Random(profile.order_seed + repeat).shuffle(order)
|
||
if len(order) > 1 and order[0] == previous:
|
||
order[0], order[1] = order[1], order[0]
|
||
for config in order:
|
||
plan.append({"cell": len(plan) + 1, "config": config, "repeat": repeat})
|
||
previous = order[-1]
|
||
return plan
|
||
|
||
|
||
def run_scaling(profile: ScalingProfile, label: str, resume: bool) -> Path:
|
||
workdir = temp_workdir()
|
||
limits_all = {c: parse_config(c, profile.base) for c in profile.configs}
|
||
worst_cpu = max(limits.total_cpus() for limits in limits_all.values())
|
||
worst_mem = max(limits.total_memory_gib() for limits in limits_all.values())
|
||
folder = EVIDENCE / "scaling" / label
|
||
if resume:
|
||
if not folder.exists():
|
||
raise SystemExit(
|
||
f"nothing to resume: {folder.relative_to(ROOT)} is missing"
|
||
)
|
||
meta = json.loads((folder / "meta.json").read_text(encoding="utf-8"))
|
||
if meta["profile"] != json.loads(json.dumps(dataclasses.asdict(profile))):
|
||
raise SystemExit("resume: profile differs from the label passport")
|
||
pre = preflight(worst_cpu, worst_mem)
|
||
image = prepare_image(label)
|
||
if image["src_sha256"] != meta["image"]["src_sha256"]:
|
||
raise SystemExit(
|
||
"resume: code differs from the label passport (src_sha256)"
|
||
)
|
||
meta.setdefault("resumed", []).append(utc_now())
|
||
else:
|
||
folder = new_label_dir("scaling", label)
|
||
(folder / "raw").mkdir()
|
||
pre = preflight(worst_cpu, worst_mem)
|
||
image = prepare_image(label)
|
||
meta = {
|
||
"kind": "scaling",
|
||
"label": label,
|
||
"series": profile.series,
|
||
"started": utc_now(),
|
||
"profile": dataclasses.asdict(profile),
|
||
"configs": {
|
||
c: dataclasses.asdict(limits) for c, limits in limits_all.items()
|
||
},
|
||
"budget": {
|
||
c: {
|
||
"cpus": limits.total_cpus(),
|
||
"memory_gib": round(limits.total_memory_gib(), 2),
|
||
}
|
||
for c, limits in limits_all.items()
|
||
},
|
||
"vm": pre["budget"],
|
||
"host": pre["host"],
|
||
"docker": pre["docker"],
|
||
"host_warnings": pre["warnings"],
|
||
"image": image,
|
||
"code": code_state(image),
|
||
"thresholds": analysis.THRESHOLDS,
|
||
"plan": cell_plan(profile),
|
||
}
|
||
started_mono, started_wall = time.monotonic(), time.time()
|
||
write_json(folder / "meta.json", meta)
|
||
done_cells = {int(row["cell"]) for row in read_csv(folder / "searches.csv")}
|
||
pool = image["app_defaults"]
|
||
pool_cap = pool["pool_size"] + pool["pool_overflow"]
|
||
try:
|
||
for cell in meta["plan"]:
|
||
if cell["cell"] in done_cells:
|
||
continue
|
||
say(
|
||
f"cell {cell['cell']}/{len(meta['plan'])}: {cell['config']} "
|
||
f"repeat {cell['repeat']}"
|
||
)
|
||
scaling_cell(
|
||
profile, limits_all[cell["config"]], cell, folder, workdir, pool_cap
|
||
)
|
||
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
|
||
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
|
||
say(f"aborted: {meta['aborted']} (continue: --resume)")
|
||
drop_quick_image(label)
|
||
meta["finished"] = utc_now()
|
||
finish_host_state(meta, started_mono, started_wall)
|
||
write_json(folder / "meta.json", meta)
|
||
searches = read_csv(folder / "searches.csv")
|
||
runs = read_csv(folder / "runs.csv")
|
||
(folder / "summary.md").write_text(
|
||
scaling_summary(meta, searches, runs), encoding="utf-8"
|
||
)
|
||
if searches:
|
||
plots.render_label(folder)
|
||
say(f"done: {folder.relative_to(ROOT)}")
|
||
if meta.get("aborted"):
|
||
raise SystemExit(1)
|
||
return folder
|
||
|
||
|
||
def scaling_cell(
|
||
profile: ScalingProfile,
|
||
limits: Limits,
|
||
cell: dict,
|
||
folder: Path,
|
||
workdir: Path,
|
||
pool_cap: int,
|
||
) -> None:
|
||
started = time.monotonic()
|
||
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
|
||
limit_dict = dataclasses.asdict(limits)
|
||
try:
|
||
stand.start()
|
||
setup = stand.gen(
|
||
"setup",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--users",
|
||
str(profile.users),
|
||
"--read-messages",
|
||
str(profile.read_messages),
|
||
"--write-chats",
|
||
str(profile.write_chats),
|
||
"--fill-clients",
|
||
str(profile.fill_clients),
|
||
)
|
||
stand.maintain()
|
||
if cell["cell"] == 1 or not (folder / "stand.json").exists():
|
||
write_json(folder / "stand.json", stand.passport())
|
||
roles = roles_of(stand)
|
||
replica_addrs = {stand.address(c) for c in stand.api_ids()}
|
||
stand.start_sampler("list", full_ids(stand))
|
||
search = analysis.Search(
|
||
fixed=profile.fixed_rates,
|
||
base=profile.ladder_base,
|
||
factor=profile.ladder_factor,
|
||
start=profile.start_per_replica * limits.api_replicas,
|
||
bisections=profile.bisections,
|
||
cap=profile.rate_cap,
|
||
)
|
||
previous_rate, fresh = 0.0, True
|
||
step_no = 0
|
||
rows: list[dict] = []
|
||
while (step := search.next_rate()) is not None:
|
||
rate, phase = step
|
||
step_no += 1
|
||
warmup = profile.warmup_first_s if fresh else profile.warmup_ramp_s
|
||
rate_from = 0.0 if fresh else previous_rate
|
||
result = stand.gen(
|
||
"step",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--rate",
|
||
str(rate),
|
||
"--rate-from",
|
||
str(rate_from),
|
||
"--warmup",
|
||
str(warmup),
|
||
"--window",
|
||
str(profile.window_s),
|
||
"--mix",
|
||
profile.mix,
|
||
"--procs",
|
||
str(profile.gen_procs),
|
||
"--seed",
|
||
str(profile.order_seed + cell["cell"] * 100 + step_no),
|
||
"--timeout",
|
||
str(profile.request_timeout_s),
|
||
"--key-prefix",
|
||
f"c{cell['cell']}s{step_no}",
|
||
"--abort-error-share",
|
||
str(profile.abort_error_share),
|
||
"--abort-p95-ms",
|
||
str(profile.abort_p95_ms),
|
||
timeout=profile.warmup_first_s + profile.window_s + 120,
|
||
)
|
||
ticks = analysis.window_ticks(
|
||
stand.sampler_lines(), result["window_start"], result["window_end"]
|
||
)
|
||
row = step_row(
|
||
profile, cell, step_no, phase, result, ticks, roles, replica_addrs
|
||
)
|
||
row["slo_pass"] = bool(
|
||
not result["abort"]
|
||
and result["p95_ms"] is not None
|
||
and result["p95_ms"] <= profile.slo_p95_ms
|
||
and result["fail_share"] <= profile.slo_fail_share
|
||
)
|
||
row["signals"] = ";".join(analysis.signals(row, limit_dict, pool_cap))
|
||
append_csv(folder / "runs.csv", RUN_COLUMNS, row)
|
||
for sample in step_samples(
|
||
cell, step_no, rate, ticks, roles, result["window_start"]
|
||
):
|
||
append_csv(folder / "samples.csv", SAMPLE_COLUMNS, sample)
|
||
(folder / "raw" / f"c{cell['cell']:02d}-s{step_no:02d}.json").write_text(
|
||
json.dumps(result, ensure_ascii=False), encoding="utf-8"
|
||
)
|
||
rows.append(row)
|
||
say(
|
||
f" step {step_no} {phase} {rate:.0f}/s: p95 {result['p95_ms']} ms, "
|
||
f"fail {result['fail_share']:.4f}, {'pass' if row['slo_pass'] else 'FAIL'} "
|
||
f"[{row['signals']}]"
|
||
)
|
||
search.record(rate, row["slo_pass"], phase)
|
||
previous_rate = rate
|
||
fresh = not row["slo_pass"]
|
||
if fresh:
|
||
time.sleep(profile.pause_after_fail_s)
|
||
stand.gen("ready", "--timeout", "60")
|
||
r_star = search.r_star()
|
||
first_fail = search.first_fail()
|
||
fail_row = next(
|
||
(
|
||
r
|
||
for r in rows
|
||
if r["phase"] != "fixed" and float(r["rate"]) == first_fail
|
||
),
|
||
None,
|
||
)
|
||
star_row = next(
|
||
(r for r in rows if r["phase"] != "fixed" and float(r["rate"]) == r_star),
|
||
None,
|
||
)
|
||
flags = []
|
||
if fail_row and "generator" in fail_row["signals"].split(";"):
|
||
flags.append("generator_bound")
|
||
if r_star is not None and r_star >= profile.rate_cap:
|
||
flags.append("cap")
|
||
down = [name for name in (DB, CACHE, LB) if not _running(name)]
|
||
if down:
|
||
flags.append("stand_down:" + ",".join(down))
|
||
append_csv(
|
||
folder / "searches.csv",
|
||
SEARCH_COLUMNS,
|
||
{
|
||
"series": profile.series,
|
||
"config": cell["config"],
|
||
"replicas": limits.api_replicas,
|
||
"api_cpus": limits.api_cpus,
|
||
"db_cpus": limits.db_cpus,
|
||
"repeat": cell["repeat"],
|
||
"cell": cell["cell"],
|
||
"r_star": r_star,
|
||
"first_fail": first_fail,
|
||
"p95_at_r_star": star_row["p95_ms"] if star_row else None,
|
||
"signals_at_fail": fail_row["signals"] if fail_row else "",
|
||
"generator_bound": "generator_bound" in flags,
|
||
"steps": step_no,
|
||
"fill_rps": setup["fill_rps"],
|
||
"duration_s": round(time.monotonic() - started, 1),
|
||
"flags": ";".join(flags),
|
||
},
|
||
)
|
||
say(f" R* = {r_star} (first fail {first_fail})")
|
||
finally:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
|
||
|
||
def _running(name: str) -> bool:
|
||
done = docker(
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}} {{.RestartCount}} {{.State.OOMKilled}}",
|
||
name,
|
||
check=False,
|
||
)
|
||
return done.stdout.strip() == "true 0 false"
|
||
|
||
|
||
def step_row(
|
||
profile: ScalingProfile,
|
||
cell: dict,
|
||
step_no: int,
|
||
phase: str,
|
||
result: dict,
|
||
ticks: list[dict],
|
||
roles: dict[str, str],
|
||
replica_addrs: set[str],
|
||
) -> dict:
|
||
usage = resources_by_role(ticks, roles)
|
||
pg = analysis.pg_stats(ticks, replica_addrs)
|
||
instances = result.get("instances") or {}
|
||
total = sum(instances.values())
|
||
row = {
|
||
"series": profile.series,
|
||
"config": cell["config"],
|
||
"repeat": cell["repeat"],
|
||
"cell": cell["cell"],
|
||
"step": step_no,
|
||
"phase": phase,
|
||
**{
|
||
key: result.get(key)
|
||
for key in (
|
||
"rate",
|
||
"rate_from",
|
||
"warmup_s",
|
||
"window_s",
|
||
"scheduled",
|
||
"ok",
|
||
"errors",
|
||
"short_pages",
|
||
"dropped",
|
||
"unsent",
|
||
"fail_share",
|
||
"achieved_rps",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"service_p95_ms",
|
||
"lag_p99_ms",
|
||
"abort",
|
||
)
|
||
},
|
||
**usage,
|
||
**pg,
|
||
"lb_retries": analysis.lb_delta(ticks, "wretr"),
|
||
"lb_redispatches": analysis.lb_delta(ticks, "wredis"),
|
||
"lb_conn_errors": analysis.lb_delta(ticks, "econ"),
|
||
"instances": len(instances),
|
||
"instance_share_min": round(min(instances.values()) / total, 3)
|
||
if total
|
||
else None,
|
||
"error_kinds": ";".join(
|
||
f"{k}:{v}" for k, v in sorted((result.get("error_kinds") or {}).items())
|
||
),
|
||
}
|
||
if row.get("gen_cpu_cores") is None:
|
||
# нет тиков cgroup генератора: его собственное время процессов
|
||
row["gen_cpu_cores"] = round(
|
||
result["gen_cpu_s"] / (result["warmup_s"] + result["window_s"]), 3
|
||
)
|
||
row["api_cpu_us_per_req"] = _per_req(
|
||
row.get("api_cpu_cores"), result.get("achieved_rps")
|
||
)
|
||
row["db_cpu_us_per_req"] = _per_req(
|
||
row.get("db_cpu_cores"), result.get("achieved_rps")
|
||
)
|
||
return row
|
||
|
||
|
||
def step_samples(
|
||
cell: dict,
|
||
step_no: int,
|
||
rate: float,
|
||
ticks: list[dict],
|
||
roles: dict[str, str],
|
||
window_start: float,
|
||
) -> list[dict]:
|
||
"""Посекундные интервалы окна: CPU, троттлинг, память каждого контейнера."""
|
||
out = []
|
||
for a, b in zip(ticks, ticks[1:], strict=False):
|
||
seconds = b["ts"] - a["ts"]
|
||
if seconds <= 0:
|
||
continue
|
||
for cid, role in roles.items():
|
||
x, y = a["cg"].get(cid), b["cg"].get(cid)
|
||
if not x or not y or y[0] < x[0]:
|
||
continue
|
||
periods = y[1] - x[1]
|
||
out.append(
|
||
{
|
||
"config": cell["config"],
|
||
"repeat": cell["repeat"],
|
||
"step": step_no,
|
||
"rate": rate,
|
||
"t": round(b["ts"] - window_start, 2),
|
||
"role": role,
|
||
"container": cid,
|
||
"cpu_cores": round((y[0] - x[0]) / seconds / 1e6, 3),
|
||
"throttled_share": round((y[2] - x[2]) / periods, 3)
|
||
if periods
|
||
else 0.0,
|
||
"memory_mib": round(y[4] / 2**20, 1),
|
||
}
|
||
)
|
||
return out
|
||
|
||
|
||
def _f(value) -> float | None:
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def scaling_summary(meta: dict, searches: list[dict], runs: list[dict]) -> str:
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Серия трека 1 «{meta['series']}»: {meta['label']}",
|
||
"",
|
||
_passport_lines(meta),
|
||
"",
|
||
f"Открытая модель через HAProxy стенда, смесь read/write/me "
|
||
f"{profile['mix']} (страница истории читаемого чата из "
|
||
f"{profile['read_messages']} сообщений, отправка с новым ключом в один из "
|
||
f"{profile['write_chats']} чатов, профиль). Ступень: прогрев "
|
||
f"{profile['warmup_first_s']:g} с с нуля или {profile['warmup_ramp_s']:g} с "
|
||
f"подъёма, окно {profile['window_s']:g} с. Фиксированные уровни "
|
||
f"{', '.join(f'{r:g}' for r in profile['fixed_rates'])}/с, затем поиск R*: "
|
||
f"лестница {profile['ladder_base']:g} × {profile['ladder_factor']:g}^i от "
|
||
f"{profile['start_per_replica']:g} × реплик, {profile['bisections']} деления. "
|
||
f"SLO: p95 успешных ≤ {profile['slo_p95_ms']:g} мс и (ошибки + неполные "
|
||
f"страницы + недосланные) ≤ {profile['slo_fail_share'] * 100:g} %. "
|
||
f"Ячейка — конфигурация × повтор ({profile['repeats']}), свежий стенд.",
|
||
"",
|
||
"| Конфигурация | Реплик × CPU | PostgreSQL CPU | R*, медиана (min–max) | Эффективность к 1 × 1 | p95 на "
|
||
+ " / ".join(f"{r:g}" for r in profile["fixed_rates"])
|
||
+ " /с, мс | API мкс/запрос на "
|
||
+ f"{profile['fixed_rates'][-1]:g}/с | PostgreSQL мкс/запрос | Признаки на первой нарушившей |",
|
||
"|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
by_config: dict[str, list[dict]] = {}
|
||
for row in searches:
|
||
by_config.setdefault(row["config"], []).append(row)
|
||
base = by_config.get("1x1")
|
||
base_r = statistics.median(_f(r["r_star"]) or 0 for r in base) if base else None
|
||
for config in profile["configs"]:
|
||
rows = by_config.get(config, [])
|
||
if not rows:
|
||
lines.append(f"| {config} | — | — | не снята | — | — | — | — | — |")
|
||
continue
|
||
values = [_f(r["r_star"]) or 0 for r in rows]
|
||
median = statistics.median(values)
|
||
replicas = int(rows[0]["replicas"])
|
||
efficiency = (
|
||
f"{median / (replicas * base_r):.2f}" if base_r and config != "1x1" else "—"
|
||
)
|
||
fixed = []
|
||
for rate in profile["fixed_rates"]:
|
||
p95 = [
|
||
_f(r["p95_ms"])
|
||
for r in runs
|
||
if r["config"] == config
|
||
and r["phase"] == "fixed"
|
||
and _f(r["rate"]) == rate
|
||
]
|
||
p95 = [v for v in p95 if v is not None]
|
||
fixed.append(f"{statistics.median(p95):g}" if p95 else "—")
|
||
last_fixed = [
|
||
r
|
||
for r in runs
|
||
if r["config"] == config
|
||
and r["phase"] == "fixed"
|
||
and _f(r["rate"]) == profile["fixed_rates"][-1]
|
||
]
|
||
api_us = [
|
||
_f(r["api_cpu_us_per_req"])
|
||
for r in last_fixed
|
||
if _f(r["api_cpu_us_per_req"])
|
||
]
|
||
db_us = [
|
||
_f(r["db_cpu_us_per_req"]) for r in last_fixed if _f(r["db_cpu_us_per_req"])
|
||
]
|
||
signals: dict[str, int] = {}
|
||
for r in rows:
|
||
for s in (r["signals_at_fail"] or "").split(";"):
|
||
if s:
|
||
signals[s] = signals.get(s, 0) + 1
|
||
flags = sorted({f for r in rows for f in (r["flags"] or "").split(";") if f})
|
||
api_text = f"{statistics.median(api_us):g}" if api_us else "—"
|
||
db_text = f"{statistics.median(db_us):g}" if db_us else "—"
|
||
signal_text = ", ".join(f"{k} ×{v}" for k, v in sorted(signals.items())) or "—"
|
||
if flags:
|
||
signal_text += f" ⚠ {', '.join(flags)}"
|
||
lines.append(
|
||
f"| {config} | {replicas} × {_f(rows[0]['api_cpus']):g} | "
|
||
f"{_f(rows[0]['db_cpus']):g} | {median:g} ({min(values):g}–{max(values):g}) | "
|
||
f"{efficiency} | {' / '.join(fixed)} | {api_text} | {db_text} | {signal_text} |"
|
||
)
|
||
lines += [
|
||
"",
|
||
"Признаки (пороги — `THRESHOLDS` в `bench/analysis.py`, заданы до замеров): "
|
||
"`app` — CPU реплики у лимита или троттлинг; `db_cpu` — CPU PostgreSQL у "
|
||
"лимита; `db_lock` — ожидание блокировок; `pool` — соединений реплики у "
|
||
"предела пула; `churn` — переоткрытие соединений (новые сессии PostgreSQL "
|
||
"за окно); `lb` — CPU HAProxy; `host` — ВМ Docker занята; `generator` — "
|
||
"генератор у своего лимита или отстаёт от расписания (R* — нижняя граница).",
|
||
"",
|
||
"Файлы: `runs.csv` — строка на ступень, `searches.csv` — строка на ячейку, "
|
||
"`samples.csv` — посекундно по контейнерам в окнах ступеней, "
|
||
"`p95-by-rate.svg`, `cpu-by-component.svg`, `raw/` — итоги генератора.",
|
||
]
|
||
lines += _warning_lines(meta)
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
# --- uploads (лаба 4) -----------------------------------------------------------------
|
||
|
||
|
||
def variant_override(path: Path, env: tuple[tuple[str, str], ...]) -> Path:
|
||
"""Override окружения реплик варианта: третий `-f` у compose стенда.
|
||
|
||
Окружение варианта — не `BENCH_*` основного override: тогда у прочих
|
||
профилей окружение реплик (`stand.app_env`) не меняется.
|
||
"""
|
||
lines = [
|
||
"# окружение реплик варианта профиля uploads (bench/run.py)",
|
||
"services:",
|
||
" api:",
|
||
" environment:",
|
||
*(f' {name}: "{value}"' for name, value in env),
|
||
]
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def wait_process_ready(container: str, timeout: float = 60) -> None:
|
||
"""Процесс реплики сам отвечает на /health/ready (команда HEALTHCHECK
|
||
образа через `docker exec`): новый контейнер может получить прежний IP,
|
||
и слот LB ещё `UP` от прежнего."""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
done = docker("exec", container, "python", "-m", "src.healthcheck", check=False)
|
||
if done.returncode == 0:
|
||
return
|
||
time.sleep(0.3)
|
||
raise StandError(f"replica {container[:12]} is not ready in {timeout:.0f}s")
|
||
|
||
|
||
def container_state(container: str) -> dict:
|
||
raw = docker("inspect", "-f", "{{json .State}}", container, check=False)
|
||
if raw.returncode != 0:
|
||
return {}
|
||
state = json.loads(raw.stdout)
|
||
return {
|
||
"oom_killed": state.get("OOMKilled"),
|
||
"exit_code": state.get("ExitCode"),
|
||
"running": state.get("Running"),
|
||
}
|
||
|
||
|
||
def full_id(name: str) -> str:
|
||
return docker("inspect", "-f", "{{.Id}}", name).stdout.strip()
|
||
|
||
|
||
def run_uploads(profile: UploadsProfile, label: str) -> Path:
|
||
workdir = temp_workdir()
|
||
folder = new_label_dir("uploads", label)
|
||
(folder / "raw").mkdir()
|
||
started_mono, started_wall = time.monotonic(), time.time()
|
||
limits = profile.limits()
|
||
pre = preflight(limits.total_cpus(), limits.total_memory_gib())
|
||
image = prepare_image(label)
|
||
plan = uploads.plan_runs(
|
||
profile.variants,
|
||
[name for name, _ in profile.sizes],
|
||
profile.clients,
|
||
profile.repeats,
|
||
profile.seed,
|
||
)
|
||
meta = {
|
||
"kind": "uploads",
|
||
"label": label,
|
||
"started": utc_now(),
|
||
"profile": dataclasses.asdict(profile),
|
||
"host": pre["host"],
|
||
"docker": pre["docker"],
|
||
"budget": pre["budget"],
|
||
"host_warnings": pre["warnings"],
|
||
"image": image,
|
||
"code": code_state(image),
|
||
"plan": plan,
|
||
}
|
||
write_json(folder / "meta.json", meta)
|
||
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
|
||
override = variant_override(
|
||
workdir / "variant-unlimited.compose.yml", profile.unlimited_env
|
||
)
|
||
rows: list[dict] = []
|
||
try:
|
||
say("stand: postgres, valkey, s3, migrate, 1 replica, haproxy")
|
||
stand.start(with_s3=True)
|
||
setup = stand.gen(
|
||
"setup",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--users",
|
||
"1",
|
||
"--read-messages",
|
||
"0",
|
||
"--write-chats",
|
||
"1",
|
||
)
|
||
meta["setup"] = {
|
||
k: v for k, v in setup.items() if k not in ("write_chats", "owner")
|
||
}
|
||
meta["bodies"] = {}
|
||
for name, size in profile.sizes:
|
||
body = stand.gen(
|
||
"body",
|
||
"--size",
|
||
str(size),
|
||
"--seed",
|
||
str(profile.seed),
|
||
"--out",
|
||
f"/tmp/body-{name}.png",
|
||
)
|
||
meta["bodies"][name] = {
|
||
key: body[key] for key in ("bytes", "width", "height", "sha256")
|
||
}
|
||
meta["stand"] = stand.passport()
|
||
meta["app_env"] = {}
|
||
write_json(folder / "meta.json", meta)
|
||
for item in plan:
|
||
say(f"run {item['order']}/{len(plan)} {item['run']}")
|
||
row = uploads_run(stand, profile, item, meta, folder, override)
|
||
rows.append(row)
|
||
append_csv(folder / "runs.csv", uploads.RUN_COLUMNS, row)
|
||
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
|
||
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
|
||
say(f"aborted: {meta['aborted']}")
|
||
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
|
||
finally:
|
||
stand.stop_sampler()
|
||
stand.extra_files = []
|
||
stand.stop()
|
||
drop_quick_image(label)
|
||
meta["finished"] = utc_now()
|
||
finish_host_state(meta, started_mono, started_wall)
|
||
write_json(folder / "meta.json", meta)
|
||
write_csv(folder / "runs.csv", uploads.RUN_COLUMNS, rows)
|
||
(folder / "summary.md").write_text(
|
||
uploads.render_summary(meta, rows), encoding="utf-8"
|
||
)
|
||
say(f"done: {folder.relative_to(ROOT)}")
|
||
if meta.get("aborted"):
|
||
raise SystemExit(1)
|
||
return folder
|
||
|
||
|
||
def uploads_run(
|
||
stand: Stand,
|
||
profile: UploadsProfile,
|
||
item: dict,
|
||
meta: dict,
|
||
folder: Path,
|
||
override: Path,
|
||
) -> dict:
|
||
"""Прогон: новый S3 на пустом volume, новая реплика с окружением
|
||
варианта, разогревающая загрузка, сброс пика, C клиентов окна, пик."""
|
||
stand.reset_s3()
|
||
stand.extra_files = [override] if item["variant"] == "unlimited" else []
|
||
stand.compose("up", "-d", "--no-deps", "--no-build", "--force-recreate", "api")
|
||
api = stand.api_ids()[0]
|
||
wait_process_ready(api)
|
||
# слот LB по адресу новой реплики: прежний мог ещё числиться UP
|
||
if stand.wait_slot(stand.address(api), "UP", timeout=30) is None:
|
||
raise StandError(f"{item['run']}: LB does not see the new replica UP")
|
||
if not stand.gen("ready", "--timeout", "60")["ready"]:
|
||
raise StandError("API is not ready through the LB:\n" + stand.app_tail())
|
||
body = f"/tmp/body-{item['size']}.png"
|
||
# разогрев: первое соединение S3 и первые страницы кода — до сброса пика
|
||
warm = stand.gen(
|
||
"uploads",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--body",
|
||
body,
|
||
"--clients",
|
||
"1",
|
||
"--warmup",
|
||
"0",
|
||
"--window",
|
||
"0.5",
|
||
)
|
||
if not warm["ok"]:
|
||
raise StandError(f"{item['run']}: warm-up upload failed: {warm['statuses']}")
|
||
if item["variant"] not in meta["app_env"]:
|
||
env = json.loads(docker("inspect", "-f", "{{json .Config.Env}}", api).stdout)
|
||
meta["app_env"][item["variant"]] = dict(
|
||
sorted(e.split("=", 1) for e in env if e.startswith("UPLOAD_"))
|
||
)
|
||
stand.start_peak_sampler(
|
||
{"api": api, "s3": full_id(S3), "lb": full_id(LB)}, profile.sample_interval_s
|
||
)
|
||
swap_before = swapouts()
|
||
try:
|
||
result = stand.gen(
|
||
"uploads",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--body",
|
||
body,
|
||
"--clients",
|
||
str(item["clients"]),
|
||
"--warmup",
|
||
str(profile.warmup_s),
|
||
"--window",
|
||
str(profile.window_s),
|
||
"--timeout",
|
||
str(profile.upload_timeout_s),
|
||
timeout=profile.warmup_s + profile.window_s + profile.upload_timeout_s + 60,
|
||
)
|
||
finally:
|
||
lines = stand.stop_peak_sampler()
|
||
swap_after = swapouts()
|
||
state = container_state(api)
|
||
(folder / "raw" / f"{item['run']}-gen.json").write_text(
|
||
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
(folder / "raw" / f"{item['run']}-peak.jsonl").write_text(
|
||
"".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8"
|
||
)
|
||
delta = (
|
||
swap_after - swap_before
|
||
if swap_before is not None and swap_after is not None
|
||
else None
|
||
)
|
||
row = uploads.run_row(
|
||
item,
|
||
body_bytes=meta["bodies"][item["size"]]["bytes"],
|
||
peaks=uploads.parse_peak(lines),
|
||
result=result,
|
||
state=state,
|
||
swapouts=delta,
|
||
)
|
||
say(
|
||
f" peak {row['peak_mib']} MiB, accepted {row['ok']}, 429 {row['rejected_429']}, "
|
||
f"5xx {row['errors_5xx']}, {row['uploads_per_s']} uploads/s {row['flags']}"
|
||
)
|
||
return row
|
||
|
||
|
||
# --- reconcile (лаба 4, трек 2) -------------------------------------------------------
|
||
|
||
|
||
def fill_stage(
|
||
stand: Stand,
|
||
profile: ReconcileProfile,
|
||
*,
|
||
start: int,
|
||
end: int,
|
||
size: int,
|
||
missing: int,
|
||
chat: str,
|
||
owner: str,
|
||
) -> dict:
|
||
"""Набор мимо API: строки `COPY` в PostgreSQL стенда и объекты PUT
|
||
загрузчиком на образе стенда. Номера [start, end) — классы по модели;
|
||
`missing` > 0 — все объекты согласованы, плюс столько строк без
|
||
объекта (номера [end, end + missing))."""
|
||
created = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 86400))
|
||
# пространство ключей S3 приложения — имя базы (src/s3.py)
|
||
namespace = bench_env("POSTGRES_DB")
|
||
force = reconcile.FORCE_OK if missing else None
|
||
common = {
|
||
"seed": profile.seed,
|
||
"size": size,
|
||
"namespace": namespace,
|
||
"chat_id": chat,
|
||
"owner_id": owner,
|
||
"created_at": created,
|
||
}
|
||
data = reconcile.rows_csv(start=start, end=end, force=force, **common)
|
||
if missing:
|
||
data += reconcile.rows_csv(
|
||
start=end, end=end + missing, force=reconcile.FORCE_MISSING, **common
|
||
)
|
||
started = time.monotonic()
|
||
columns = ", ".join(reconcile.COPY_COLUMNS)
|
||
stand.psql(
|
||
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv);\n{data}\\.\n"
|
||
)
|
||
copy_s = time.monotonic() - started
|
||
# ключи пользователя API — окружение docker run без значения в argv
|
||
keys = {
|
||
"S3_ACCESS_KEY_ID": stand.env["S3_API_ACCESS_KEY"],
|
||
"S3_SECRET_ACCESS_KEY": stand.env["S3_API_SECRET_KEY"],
|
||
}
|
||
sh(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
FILL,
|
||
"--network",
|
||
NET,
|
||
"--no-healthcheck",
|
||
"--cpus",
|
||
str(profile.fill_cpus),
|
||
"--memory",
|
||
profile.fill_memory,
|
||
"--memory-swap",
|
||
profile.fill_memory,
|
||
"-e",
|
||
"S3_ACCESS_KEY_ID",
|
||
"-e",
|
||
"S3_SECRET_ACCESS_KEY",
|
||
"--entrypoint",
|
||
"python",
|
||
SERVICE_TAG,
|
||
"-c",
|
||
"import time; time.sleep(86400)",
|
||
],
|
||
env={**stand.env, **keys},
|
||
)
|
||
try:
|
||
for name in ("reconcile.py", "fill_objects.py"):
|
||
docker("cp", str(BENCH / name), f"{FILL}:/tmp/{name}")
|
||
args = [
|
||
"--endpoint",
|
||
"http://s3:8333",
|
||
"--bucket",
|
||
bench_env("S3_BUCKET"),
|
||
"--namespace",
|
||
namespace,
|
||
"--seed",
|
||
str(profile.seed),
|
||
"--start",
|
||
str(start),
|
||
"--end",
|
||
str(end),
|
||
"--size",
|
||
str(size),
|
||
"--processes",
|
||
str(profile.fill_processes),
|
||
"--threads",
|
||
str(profile.fill_threads),
|
||
]
|
||
if force:
|
||
args += ["--force", force]
|
||
done = docker(
|
||
"exec", FILL, "python", "/tmp/fill_objects.py", *args, timeout=3600
|
||
)
|
||
finally:
|
||
docker("rm", "-f", FILL, check=False)
|
||
put = json.loads(done.stdout.strip().splitlines()[-1])
|
||
return {
|
||
"start": start,
|
||
"end": end,
|
||
"missing": missing,
|
||
"rows_copy_s": round(copy_s, 2),
|
||
**put,
|
||
"last_put_at": time.time(),
|
||
}
|
||
|
||
|
||
def run_reconcile(profile: ReconcileProfile, label: str) -> Path:
|
||
workdir = temp_workdir()
|
||
folder = new_label_dir("reconcile", label)
|
||
(folder / "raw").mkdir()
|
||
started_mono, started_wall = time.monotonic(), time.time()
|
||
limits = profile.limits()
|
||
# на пике: уборщик управляемого опыта или загрузчик, не оба сразу
|
||
peak_extra = max(profile.fast_cpus, profile.fill_cpus) - limits.cleaner_cpus
|
||
pre = preflight(
|
||
limits.total_cpus() + peak_extra,
|
||
limits.total_memory_gib() + gib(profile.fill_memory),
|
||
)
|
||
image = prepare_image(label)
|
||
plan = reconcile.plan_passes(
|
||
[(stage[0], stage[4]) for stage in profile.stages],
|
||
profile.repeats,
|
||
profile.seed,
|
||
)
|
||
stages = {
|
||
stage[0]: {"n": stage[1], "size": stage[2], "missing": stage[5]}
|
||
for stage in profile.stages
|
||
}
|
||
models = {
|
||
stage[0]: (
|
||
reconcile.model(0, clean=stage[1], missing=stage[5])
|
||
if stage[5]
|
||
else reconcile.model(stage[1])
|
||
)
|
||
for stage in profile.stages
|
||
}
|
||
default_rate = image["app_defaults"]["cleaner"]["reconcile_rate"]
|
||
meta = {
|
||
"kind": "reconcile",
|
||
"label": label,
|
||
"started": utc_now(),
|
||
"profile": dataclasses.asdict(profile),
|
||
"host": pre["host"],
|
||
"docker": pre["docker"],
|
||
"budget": pre["budget"],
|
||
"host_warnings": pre["warnings"],
|
||
"image": image,
|
||
"code": code_state(image),
|
||
"reconcile": {
|
||
"plan": plan,
|
||
"models": {k: dataclasses.asdict(m) for k, m in models.items()},
|
||
"default_rate": default_rate,
|
||
"fills": [],
|
||
},
|
||
}
|
||
write_json(folder / "meta.json", meta)
|
||
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
|
||
passes: list[dict] = []
|
||
windows: list[dict] = []
|
||
number = 0
|
||
try:
|
||
setup: dict = {}
|
||
filled = 0
|
||
for name, n, size, fresh, cells, missing in profile.stages:
|
||
if fresh:
|
||
if setup:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
say(f"stage {name}: fresh stand with s3, 2 replicas")
|
||
stand.start(with_s3=True)
|
||
setup = stand.gen(
|
||
"setup",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--users",
|
||
str(profile.users),
|
||
"--read-messages",
|
||
"200",
|
||
"--write-chats",
|
||
"1",
|
||
)
|
||
if "stand" not in meta:
|
||
meta["stand"] = stand.passport()
|
||
stand.start_sampler("scan", interval=profile.sample_interval_s)
|
||
filled = 0
|
||
say(f"stage {name}: fill [{filled}, {n}) x {size} B, missing {missing}")
|
||
fill = fill_stage(
|
||
stand,
|
||
profile,
|
||
start=filled,
|
||
end=n,
|
||
size=size,
|
||
missing=missing,
|
||
chat=setup["read_chat"],
|
||
owner=setup["owner"],
|
||
)
|
||
meta["reconcile"]["fills"].append({"stage": name, **fill})
|
||
say(f" {fill['objects']} objects, {fill['put_per_s']} put/s")
|
||
filled = n
|
||
stand.maintain()
|
||
if any(reconcile.CELLS[c].get("load") for c in cells):
|
||
ids = reconcile.downloads(profile.seed, n, profile.load_downloads)
|
||
path = stand.workdir / "downloads.json"
|
||
path.write_text(json.dumps(ids), encoding="ascii")
|
||
docker("cp", str(path), f"{GEN}:/tmp/downloads.json")
|
||
# объекты этапа старше grace: иначе они «свежие» и не кандидаты
|
||
wait = fill["last_put_at"] + profile.grace_s + 2 - time.time()
|
||
if wait > 0:
|
||
time.sleep(wait)
|
||
write_json(folder / "meta.json", meta)
|
||
for item in [p for p in plan if p["stage"] == name]:
|
||
number += 1
|
||
say(f"pass {item['order']}/{len(plan)} {item['pass']}")
|
||
row, rows = reconcile_pass(
|
||
stand,
|
||
profile,
|
||
item,
|
||
n=n,
|
||
size=size,
|
||
expected=models[name],
|
||
default_rate=default_rate,
|
||
number=number,
|
||
folder=folder,
|
||
)
|
||
passes.append(row)
|
||
windows += rows
|
||
append_csv(folder / "passes.csv", reconcile.PASS_COLUMNS, row)
|
||
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
|
||
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
|
||
say(f"aborted: {meta['aborted']}")
|
||
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
|
||
finally:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
drop_quick_image(label)
|
||
meta["finished"] = utc_now()
|
||
finish_host_state(meta, started_mono, started_wall)
|
||
write_json(folder / "meta.json", meta)
|
||
write_csv(folder / "passes.csv", reconcile.PASS_COLUMNS, passes)
|
||
write_csv(folder / "windows.csv", reconcile.WINDOW_COLUMNS, windows)
|
||
verdicts = [
|
||
reconcile.verdict_h1(stages, passes),
|
||
reconcile.verdict_h2(stages, passes),
|
||
reconcile.verdict_h3(stages, passes, models),
|
||
reconcile.verdict_h4(passes, windows),
|
||
reconcile.verdict_h5(passes),
|
||
]
|
||
meta_profile = {
|
||
**meta["profile"],
|
||
"default_rate": default_rate,
|
||
"page_size": image["app_defaults"]["cleaner"]["reconcile_page"],
|
||
}
|
||
(folder / "summary.md").write_text(
|
||
reconcile.render_summary(
|
||
{**meta, "profile": meta_profile}, passes, windows, verdicts
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
say(f"done: {folder.relative_to(ROOT)}")
|
||
if meta.get("aborted"):
|
||
raise SystemExit(1)
|
||
return folder
|
||
|
||
|
||
def reconcile_pass(
|
||
stand: Stand,
|
||
profile: ReconcileProfile,
|
||
item: dict,
|
||
*,
|
||
n: int,
|
||
size: int,
|
||
expected: reconcile.Model,
|
||
default_rate: float,
|
||
number: int,
|
||
folder: Path,
|
||
) -> tuple[dict, list[dict]]:
|
||
"""Разовый проход уборщика `docker compose run` с лимитом CPU ячейки;
|
||
у ячеек load-* — под фоновой нагрузкой с тихими окнами до и после."""
|
||
cell = reconcile.CELLS[item["cell"]]
|
||
cpus = profile.fast_cpus if cell["cpu"] == "fast" else profile.cleaner_cpus
|
||
rate = default_rate if cell["rate"] == "default" else float(cell["rate"])
|
||
args = ["reconcile", "--strategy", cell["strategy"]]
|
||
if cell.get("deep"):
|
||
args.append("--deep")
|
||
if cell["rate"] != "default":
|
||
args += ["--rate", f"{rate:g}"]
|
||
if cell.get("apply"):
|
||
args += ["--apply", "--max-changes", str(profile.refuse_max_changes)]
|
||
name = f"{RECONCILE_PREFIX}{number}"
|
||
load = None
|
||
if cell.get("load"):
|
||
docker("exec", GEN, "rm", "-f", "/tmp/load.stop")
|
||
load = stand.gen_popen(
|
||
"constant",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--write-rate",
|
||
str(profile.load_write_rate),
|
||
"--read-rate",
|
||
str(profile.load_read_rate),
|
||
"--download-rate",
|
||
str(profile.load_download_rate),
|
||
"--downloads",
|
||
"/tmp/downloads.json",
|
||
"--duration",
|
||
str(2 * profile.quiet_s + profile.pass_budget_s),
|
||
"--procs",
|
||
str(profile.load_procs),
|
||
"--seed",
|
||
str(profile.seed),
|
||
"--key-prefix",
|
||
item["pass"],
|
||
"--log",
|
||
f"/tmp/{item['pass']}.log",
|
||
"--stop-file",
|
||
"/tmp/load.stop",
|
||
)
|
||
time.sleep(profile.quiet_s + 0.3)
|
||
try:
|
||
stand.compose(
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
name,
|
||
"--no-deps",
|
||
"-e",
|
||
f"CLEANER_GRACE={profile.grace_s}",
|
||
"cleaner",
|
||
*args,
|
||
env={"BENCH_CLEANER_CPUS": str(cpus)},
|
||
)
|
||
waited = docker("wait", name, timeout=profile.pass_budget_s + 60)
|
||
exit_code = int(waited.stdout.strip())
|
||
logs = docker("logs", name, check=False)
|
||
log_text = logs.stdout + logs.stderr
|
||
nano = docker("inspect", "-f", "{{.HostConfig.NanoCpus}}", name).stdout
|
||
finally:
|
||
docker("rm", "-f", name, check=False)
|
||
(folder / "raw" / f"{item['pass']}-cleaner.log").write_text(
|
||
log_text, encoding="utf-8"
|
||
)
|
||
fields = reconcile.parse_finished(log_text)
|
||
window_rows: list[dict] = []
|
||
load_flags: list[str] = []
|
||
if load is not None:
|
||
try:
|
||
time.sleep(profile.quiet_s)
|
||
docker("exec", GEN, "touch", "/tmp/load.stop")
|
||
out, err = load.communicate(timeout=profile.pass_budget_s + 120)
|
||
finally:
|
||
if load.poll() is None:
|
||
load.kill()
|
||
if load.returncode != 0:
|
||
raise StandError(f"generator failed in {item['pass']}: {err[-500:]}")
|
||
summary = json.loads(out.strip().splitlines()[-1])
|
||
if summary["dropped"]:
|
||
load_flags.append("bg_dropped")
|
||
raw = collect_requests(stand, summary["logs"])
|
||
with gzip.open(
|
||
folder / "raw" / f"{item['pass']}-requests.csv.gz", "wt", encoding="ascii"
|
||
) as fh:
|
||
fh.write(raw)
|
||
requests = analysis.parse_requests(raw)
|
||
if fields and requests:
|
||
end = fields["ts"] * 1000
|
||
begin = end - (fields["took_ms"] or 0)
|
||
bounds = {
|
||
"before": (summary["start"] * 1000 + 2000, begin),
|
||
"pass": (begin, end),
|
||
"after": (end, requests[-1].scheduled + 1),
|
||
}
|
||
window_rows = reconcile.window_rows(
|
||
item, requests, bounds, analysis.window_stats
|
||
)
|
||
if not fields:
|
||
raise StandError(
|
||
f"{item['pass']}: no 'reconcile pass finished' line, exit {exit_code}:\n"
|
||
+ "\n".join(log_text.splitlines()[-15:])
|
||
)
|
||
end = fields["ts"]
|
||
begin = end - (fields["took_ms"] or 0) / 1000
|
||
ticks = analysis.window_ticks(stand.sampler_lines(), begin, end)
|
||
usage = {}
|
||
for role, ids in (
|
||
("s3", [full_id(S3)]),
|
||
("db", [full_id(DB)]),
|
||
("api", stand.api_ids()),
|
||
):
|
||
values = [analysis.cgroup_usage(ticks, cid) for cid in ids]
|
||
values = [v for v in values if v]
|
||
if values:
|
||
usage[f"{role}_cpu_cores"] = round(sum(v["cpu_cores"] for v in values), 3)
|
||
problems = reconcile.found_problems(fields, expected, deep=bool(cell.get("deep")))
|
||
row = reconcile.pass_row(
|
||
item,
|
||
n=n,
|
||
size=size,
|
||
cell=cell,
|
||
cleaner_cpus=cpus,
|
||
rate=rate,
|
||
fields=fields,
|
||
exit_code=exit_code,
|
||
nano_cpus=int(nano.strip() or 0) or None,
|
||
usage=usage,
|
||
problems=problems,
|
||
extra_flags=load_flags,
|
||
)
|
||
say(
|
||
f" {row['took_ms']} ms, cpu {row['cpu_ms']} ms, s3 list/head/get "
|
||
f"{row['s3_list']}/{row['s3_head']}/{row['s3_get']}, db {row['db_queries']}"
|
||
+ (f", flags {row['flags']}" if row["flags"] else "")
|
||
)
|
||
if problems:
|
||
raise StandError(f"{item['pass']}: found differs from the model: {problems}")
|
||
return row, window_rows
|
||
|
||
|
||
# --- queue (лаба 5, трек 1) ------------------------------------------------------------
|
||
|
||
|
||
class TimelinePoller(threading.Thread):
|
||
"""Шкала ячейки: состояние заданий одним SELECT раз в интервал (часы
|
||
PostgreSQL) с меткой текущей фазы. Отдельный поток: основной в это
|
||
время ждёт генератор и worker."""
|
||
|
||
def __init__(self, stand: Stand, interval: float) -> None:
|
||
super().__init__(daemon=True)
|
||
self.stand = stand
|
||
self.interval = interval
|
||
self.phase = ""
|
||
self.samples: list[dict] = []
|
||
self.failures = 0
|
||
# не `_stop`: так называется внутренний метод threading.Thread
|
||
self._halt = threading.Event()
|
||
|
||
def run(self) -> None:
|
||
next_tick = time.monotonic()
|
||
while not self._halt.is_set():
|
||
phase = self.phase
|
||
try:
|
||
sample = queue.parse_sample(self.stand.psql(queue.TIMELINE_SQL))
|
||
except StandError:
|
||
sample = None
|
||
if sample is None:
|
||
self.failures += 1
|
||
else:
|
||
self.samples.append({**sample, "phase": phase})
|
||
next_tick += self.interval
|
||
self._halt.wait(max(0.0, next_tick - time.monotonic()))
|
||
|
||
def finish(self) -> list[dict]:
|
||
self._halt.set()
|
||
self.join(timeout=10)
|
||
return self.samples
|
||
|
||
|
||
def server_now(stand: Stand) -> float:
|
||
"""Часы PostgreSQL стенда (ВМ Docker): отметки фаз в тех же часах, что
|
||
`created_at` вложений, журнал worker и генератор."""
|
||
return float(stand.psql(queue.NOW_SQL))
|
||
|
||
|
||
def pipeline_state(stand: Stand) -> dict:
|
||
sample = queue.parse_sample(stand.psql(queue.TIMELINE_SQL))
|
||
if sample is None:
|
||
raise StandError("cannot read the attachment_jobs state")
|
||
return sample
|
||
|
||
|
||
def worker_logs() -> str:
|
||
done = docker("logs", WORKER, check=False)
|
||
return done.stdout + done.stderr
|
||
|
||
|
||
def pipeline_tail(lines: int = 40) -> str:
|
||
done = docker("logs", "--tail", str(lines), WORKER, check=False)
|
||
return done.stdout + done.stderr
|
||
|
||
|
||
def _empty(state: dict) -> bool:
|
||
return state["unfinished"] == 0
|
||
|
||
|
||
def wait_state(
|
||
stand: Stand, what: str, done: Callable[[dict], bool], timeout: float
|
||
) -> dict:
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
state = pipeline_state(stand)
|
||
if done(state):
|
||
return state
|
||
if time.monotonic() > deadline:
|
||
raise StandError(
|
||
f"{what}: not reached in {timeout:.0f}s, state {state}\n"
|
||
+ pipeline_tail()
|
||
)
|
||
time.sleep(0.5)
|
||
|
||
|
||
def wait_consuming(before: int, timeout: float = 90) -> float:
|
||
"""Ждать в журнале worker строку `broker consuming` номер `before + 1`;
|
||
её время (часы ВМ) — начало работы потребителя."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
consuming = queue.parse_worker_log(worker_logs()).consuming
|
||
if len(consuming) > before:
|
||
return consuming[before]
|
||
if time.monotonic() > deadline:
|
||
raise StandError(
|
||
f"worker does not consume in {timeout:.0f}s:\n" + pipeline_tail()
|
||
)
|
||
time.sleep(0.3)
|
||
|
||
|
||
def queue_gen(
|
||
stand: Stand,
|
||
profile: QueueProfile,
|
||
folder: Path,
|
||
item: dict,
|
||
phase: str,
|
||
*,
|
||
rate: float,
|
||
duration: float,
|
||
count: int = 0,
|
||
) -> tuple[dict, list[queue.Upload]]:
|
||
"""Генератор `queue` фазы; итог и журнал загрузок — в `raw/`."""
|
||
if rate <= 0:
|
||
raise StandError(f"{item['cell']}: {phase} rate {rate} is not positive")
|
||
log = f"/tmp/{item['cell']}-{phase}.log"
|
||
args = [
|
||
"queue",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--body",
|
||
f"/tmp/body-{item['size_class']}.png",
|
||
"--rate",
|
||
f"{rate:.6g}",
|
||
"--duration",
|
||
f"{duration:.6g}",
|
||
"--log",
|
||
log,
|
||
"--timeout",
|
||
f"{profile.upload_timeout_s:g}",
|
||
]
|
||
if count:
|
||
args += ["--count", str(count)]
|
||
result = stand.gen(*args, timeout=duration + 2 * profile.upload_timeout_s + 60)
|
||
text = stand.gen_file(log)
|
||
docker("exec", GEN, "rm", "-f", log, check=False)
|
||
base = folder / "raw" / f"{item['cell']}-{phase}"
|
||
base.with_name(base.name + "-gen.json").write_text(
|
||
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||
)
|
||
base.with_name(base.name + "-gen.csv").write_text(text, encoding="ascii")
|
||
return result, queue.parse_gen_log(text)
|
||
|
||
|
||
def run_queue(profile: QueueProfile, label: str) -> Path:
|
||
fixed = dict(profile.fixed_rates)
|
||
for name, _, _, _, count in profile.classes:
|
||
# тела фазы ёмкости все незавершённые: при N ≥ предела часть получит 429
|
||
if count >= profile.backlog_limit:
|
||
raise SystemExit(f"class {name}: drain count {count} >= backlog limit")
|
||
if fixed.get(name, 0) <= 0:
|
||
raise SystemExit(f"class {name}: no fixed rate in the profile")
|
||
workdir = temp_workdir()
|
||
folder = new_label_dir("queue", label)
|
||
(folder / "raw").mkdir()
|
||
started_mono, started_wall = time.monotonic(), time.time()
|
||
limits = profile.limits()
|
||
pre = preflight(limits.total_cpus(), limits.total_memory_gib())
|
||
image = prepare_image(label)
|
||
plan = queue.plan_cells(
|
||
profile.configs, [c[0] for c in profile.classes], profile.repeats, profile.seed
|
||
)
|
||
meta = {
|
||
"kind": "queue",
|
||
"label": label,
|
||
"started": utc_now(),
|
||
"profile": dataclasses.asdict(profile),
|
||
"host": pre["host"],
|
||
"docker": pre["docker"],
|
||
"budget": pre["budget"],
|
||
"host_warnings": pre["warnings"],
|
||
"image": image,
|
||
"code": code_state(image),
|
||
"plan": plan,
|
||
# граница Q4: предел незавершённых пропускает загрузки, которые уже
|
||
# прошли проверку, — не больше мест загрузок всех реплик
|
||
"upload_slack": profile.api_replicas
|
||
* image["app_defaults"]["uploads"]["max_concurrent"],
|
||
"bodies": {},
|
||
"app_env": {},
|
||
"cells": {},
|
||
}
|
||
write_json(folder / "meta.json", meta)
|
||
stand = Stand(
|
||
limits,
|
||
session_ttl=profile.session_ttl,
|
||
workdir=workdir,
|
||
upload_max_unfinished=profile.backlog_limit,
|
||
)
|
||
rows: list[dict] = []
|
||
timeline: list[dict] = []
|
||
jobs: list[dict] = []
|
||
try:
|
||
for repeat in range(1, profile.repeats + 1):
|
||
if repeat > 1:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
say(
|
||
f"repeat {repeat}: stand with s3, rabbitmq, relay, worker, "
|
||
f"{profile.api_replicas} replicas"
|
||
)
|
||
stand.start(with_s3=True, with_pipeline=True)
|
||
stand.gen(
|
||
"setup",
|
||
"--fixture",
|
||
FIXTURE,
|
||
"--users",
|
||
"1",
|
||
"--read-messages",
|
||
"0",
|
||
"--write-chats",
|
||
"1",
|
||
)
|
||
for index, (name, kind, width, height, _) in enumerate(profile.classes):
|
||
args = ["body", "--width", str(width), "--height", str(height)]
|
||
args += [
|
||
"--seed",
|
||
str(profile.seed + index),
|
||
"--out",
|
||
f"/tmp/body-{name}.png",
|
||
]
|
||
if kind == "flat":
|
||
args.append("--flat")
|
||
body = stand.gen(*args, timeout=600)
|
||
if body["bytes"] > ATTACHMENT_MAX_BYTES:
|
||
raise StandError(f"body {name}: {body['bytes']} B > upload limit")
|
||
known = meta["bodies"].setdefault(
|
||
name,
|
||
{
|
||
k: body[k]
|
||
for k in ("kind", "width", "height", "bytes", "sha256")
|
||
},
|
||
)
|
||
if known["sha256"] != body["sha256"]:
|
||
raise StandError(f"body {name} differs between repeats")
|
||
if repeat == 1:
|
||
meta["stand"] = stand.passport()
|
||
write_json(folder / "meta.json", meta)
|
||
stand.start_sampler("scan", interval=profile.sample_interval_s)
|
||
for item in [p for p in plan if p["repeat"] == repeat]:
|
||
say(f"cell {item['order']}/{len(plan)} {item['cell']}")
|
||
cell, cell_timeline, cell_jobs = queue_cell(
|
||
stand, profile, item, meta, folder
|
||
)
|
||
rows += cell
|
||
timeline += cell_timeline
|
||
jobs += cell_jobs
|
||
for row in cell:
|
||
append_csv(folder / "cells.csv", queue.CELL_COLUMNS, row)
|
||
write_json(folder / "meta.json", meta)
|
||
with gzip.open(
|
||
folder / "raw" / f"sampler-r{repeat}.jsonl.gz", "wt", encoding="ascii"
|
||
) as fh:
|
||
for line in stand.sampler_lines():
|
||
fh.write(json.dumps(line, separators=(",", ":")) + "\n")
|
||
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
|
||
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
|
||
say(f"aborted: {meta['aborted']}")
|
||
tail = stand.compose(
|
||
"logs", "--tail", "80", "relay", "worker", "rabbitmq", check=False
|
||
)
|
||
(folder / "app-tail.log").write_text(
|
||
stand.app_tail(80) + tail.stdout + tail.stderr, encoding="utf-8"
|
||
)
|
||
finally:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
drop_quick_image(label)
|
||
meta["finished"] = utc_now()
|
||
finish_host_state(meta, started_mono, started_wall)
|
||
write_json(folder / "meta.json", meta)
|
||
write_csv(folder / "cells.csv", queue.CELL_COLUMNS, rows)
|
||
write_csv(folder / "timeline.csv", queue.TIMELINE_COLUMNS, timeline)
|
||
with gzip.open(folder / "jobs.csv.gz", "wt", newline="", encoding="utf-8") as fh:
|
||
writer = csv.DictWriter(fh, fieldnames=queue.JOB_COLUMNS, extrasaction="ignore")
|
||
writer.writeheader()
|
||
writer.writerows(jobs)
|
||
(folder / "summary.md").write_text(
|
||
queue.render_summary(meta, rows), encoding="utf-8"
|
||
)
|
||
say(f"done: {folder.relative_to(ROOT)}")
|
||
if meta.get("aborted"):
|
||
raise SystemExit(1)
|
||
return folder
|
||
|
||
|
||
def queue_cell(
|
||
stand: Stand, profile: QueueProfile, item: dict, meta: dict, folder: Path
|
||
) -> tuple[list[dict], list[dict], list[dict]]:
|
||
"""Ячейка: новый worker с настройкой ячейки, затем фазы ёмкости, ровной
|
||
нагрузки, общей интенсивности класса, перегрузки и восстановления;
|
||
между фазами очередь пустеет."""
|
||
name = item["cell"]
|
||
drain_n = {c[0]: c[4] for c in profile.classes}[item["size_class"]]
|
||
fill_rate = dict(profile.fill_rates)[item["size_class"]]
|
||
env = {
|
||
"BENCH_WORKER_CONCURRENCY": str(item["concurrency"]),
|
||
"BENCH_WORKER_PREFETCH": str(item["prefetch"]),
|
||
"BENCH_WORKER_RENDER_MEMORY_MB": str(profile.render_memory_mb),
|
||
}
|
||
stand.compose(
|
||
"up", "-d", "--no-deps", "--no-build", "--force-recreate", "worker", env=env
|
||
)
|
||
wait_consuming(0)
|
||
wait_state(
|
||
stand, f"{name}: empty queue before the cell", _empty, profile.settle_timeout_s
|
||
)
|
||
if item["config"] not in meta["app_env"]:
|
||
names = ("WORKER_", "RENDER_", "JOB_", "DB_POOL_", "LOG_")
|
||
values = json.loads(
|
||
docker("inspect", "-f", "{{json .Config.Env}}", WORKER).stdout
|
||
)
|
||
meta["app_env"][item["config"]] = dict(
|
||
sorted(v.split("=", 1) for v in values if v.startswith(names))
|
||
)
|
||
worker_id = full_id(WORKER)
|
||
marks: dict[str, float] = {}
|
||
gens: dict[str, dict] = {}
|
||
uploads_by: dict[str, list[queue.Upload]] = {}
|
||
poller = TimelinePoller(stand, profile.sample_interval_s)
|
||
# A — ёмкость: N тел ждут в брокере, worker стартует на готовую очередь
|
||
poller.phase = "drain"
|
||
poller.start()
|
||
try:
|
||
marks["drain"] = server_now(stand)
|
||
docker("stop", WORKER)
|
||
gens["drain"], uploads_by["drain"] = queue_gen(
|
||
stand,
|
||
profile,
|
||
folder,
|
||
item,
|
||
"drain",
|
||
rate=fill_rate,
|
||
duration=drain_n / fill_rate + 1,
|
||
count=drain_n,
|
||
)
|
||
wait_state(
|
||
stand, f"{name}: relay published the fill", lambda s: s["outbox"] == 0, 60
|
||
)
|
||
before = len(queue.parse_worker_log(worker_logs()).consuming)
|
||
marks["worker_start"] = server_now(stand)
|
||
docker("start", WORKER)
|
||
consuming_ts = wait_consuming(before)
|
||
wait_state(stand, f"{name}: drain", _empty, profile.settle_timeout_s)
|
||
fill_jobs = queue.parse_jobs(stand.psql(queue.jobs_sql(marks["drain"])))
|
||
throughput = queue.drain_throughput(fill_jobs, consuming_ts)
|
||
if not throughput:
|
||
raise StandError(
|
||
f"{name}: no ready job after the drain:\n" + pipeline_tail()
|
||
)
|
||
say(f" drain {drain_n}: {throughput:.3f} ready/s")
|
||
# B — ровная нагрузка, затем ждём, пока принятое станет ready
|
||
poller.phase = "steady"
|
||
marks["steady"] = server_now(stand)
|
||
gens["steady"], uploads_by["steady"] = queue_gen(
|
||
stand,
|
||
profile,
|
||
folder,
|
||
item,
|
||
"steady",
|
||
rate=profile.steady_factor * throughput,
|
||
duration=profile.window_s,
|
||
)
|
||
wait_state(stand, f"{name}: steady settle", _empty, profile.settle_timeout_s)
|
||
# B' — общая интенсивность класса: задержки настроек при равной
|
||
# нагрузке, а не при равной доле своей ёмкости
|
||
poller.phase = "fixed"
|
||
marks["fixed"] = server_now(stand)
|
||
gens["fixed"], uploads_by["fixed"] = queue_gen(
|
||
stand,
|
||
profile,
|
||
folder,
|
||
item,
|
||
"fixed",
|
||
rate=dict(profile.fixed_rates)[item["size_class"]],
|
||
duration=profile.window_s,
|
||
)
|
||
wait_state(stand, f"{name}: fixed settle", _empty, profile.settle_timeout_s)
|
||
# C — перегрузка: очередь упирается в предел, лишнее получает 429;
|
||
# окно не короче, чем нужно очереди, чтобы дойти до предела
|
||
poller.phase = "overload"
|
||
marks["overload"] = server_now(stand)
|
||
gens["overload"], uploads_by["overload"] = queue_gen(
|
||
stand,
|
||
profile,
|
||
folder,
|
||
item,
|
||
"overload",
|
||
rate=profile.overload_factor * throughput,
|
||
duration=queue.overload_window(
|
||
profile.window_s,
|
||
profile.backlog_limit,
|
||
throughput,
|
||
fill=profile.overload_fill,
|
||
cap=profile.overload_max_s,
|
||
),
|
||
)
|
||
# D — опустошение после перегрузки и снова ровная нагрузка
|
||
poller.phase = "recovery"
|
||
marks["recovery"] = server_now(stand)
|
||
wait_state(stand, f"{name}: recovery drain", _empty, profile.settle_timeout_s)
|
||
gens["recovery"], uploads_by["recovery"] = queue_gen(
|
||
stand,
|
||
profile,
|
||
folder,
|
||
item,
|
||
"recovery",
|
||
rate=profile.steady_factor * throughput,
|
||
duration=profile.window_s / 2,
|
||
)
|
||
wait_state(stand, f"{name}: recovery settle", _empty, profile.settle_timeout_s)
|
||
marks["end"] = server_now(stand)
|
||
finally:
|
||
samples = poller.finish()
|
||
# worker новый на ячейку, перезапуск в фазе A заводит cgroup заново:
|
||
# memory.peak — пик ячейки с начала фазы ёмкости
|
||
peak = stand.cgroup_value(worker_id, "memory.peak")
|
||
log_text = worker_logs()
|
||
(folder / "raw" / f"{name}-worker.log").write_text(log_text, encoding="utf-8")
|
||
data = queue.CellData(
|
||
item=item,
|
||
body_bytes=meta["bodies"][item["size_class"]]["bytes"],
|
||
marks=marks,
|
||
consuming_ts=consuming_ts,
|
||
gens=gens,
|
||
uploads=uploads_by,
|
||
jobs=queue.parse_jobs(stand.psql(queue.jobs_sql(marks["drain"]))),
|
||
log=queue.parse_worker_log(log_text),
|
||
ticks=queue.worker_ticks(stand.sampler_lines(), worker_id[:12]),
|
||
samples=samples,
|
||
peak_bytes=peak,
|
||
backlog_limit=profile.backlog_limit,
|
||
)
|
||
rows = queue.cell_rows(data)
|
||
meta["cells"][name] = {
|
||
"worker_id": worker_id,
|
||
"marks": marks,
|
||
"consuming_ts": consuming_ts,
|
||
"memory_peak_bytes": peak,
|
||
"timeline_samples": len(samples),
|
||
"timeline_failures": poller.failures,
|
||
}
|
||
by_phase = {row["phase"]: row for row in rows}
|
||
over, rec = by_phase["overload"], by_phase["recovery"]
|
||
say(
|
||
f" steady p95 {by_phase['steady']['ttr_p95_ms']} ms, fixed p95 "
|
||
f"{by_phase['fixed']['ttr_p95_ms']} ms, overload {over['window_s']} s: 429 "
|
||
f"{over['rejected_429']}/{over['sent']} (after limit "
|
||
f"{over['share_429_saturated']}), max unfinished {over['max_unfinished']}, "
|
||
f"recovery {rec['recovery_s']} s, worker peak {rows[0]['worker_peak_mib']} MiB"
|
||
+ (
|
||
f", flags {' '.join(r['flags'] for r in rows if r['flags'])}"
|
||
if any(r["flags"] for r in rows)
|
||
else ""
|
||
)
|
||
)
|
||
return rows, queue.timeline_rows(data), queue.job_rows(data)
|
||
|
||
|
||
# --- CLI ------------------------------------------------------------------------------
|
||
|
||
|
||
def main() -> None:
|
||
root = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
root.add_argument(
|
||
"profile",
|
||
choices=(
|
||
*PROFILES,
|
||
*UPLOADS_PROFILES,
|
||
*RECONCILE_PROFILES,
|
||
*QUEUE_PROFILES,
|
||
"scaling",
|
||
),
|
||
)
|
||
root.add_argument("--label")
|
||
root.add_argument("--series", choices=sorted(SERIES))
|
||
# ячейки серии вместо профильных: у ab — «до/после»; у остальных — только
|
||
# если конфигурация не помещается в ВМ Docker (пишется в паспорт метки)
|
||
root.add_argument("--configs", help="ячейки серии: '4x1-pg0.5 4x1'")
|
||
root.add_argument("--resume", action="store_true")
|
||
args = root.parse_args()
|
||
with bench_lock():
|
||
if args.profile == "scaling":
|
||
if not args.series:
|
||
raise SystemExit("scaling needs --series")
|
||
profile = SERIES[args.series]
|
||
if args.configs:
|
||
profile = dataclasses.replace(
|
||
profile, configs=tuple(args.configs.split())
|
||
)
|
||
quick = args.series == "quick"
|
||
label = args.label or (
|
||
quick_label("scaling") if quick else f"scaling-{args.series}"
|
||
)
|
||
run_scaling(profile, label, args.resume)
|
||
return
|
||
kind = args.profile.removesuffix("-quick")
|
||
if args.profile.endswith("quick"):
|
||
label = args.label or quick_label(kind)
|
||
else:
|
||
label = args.label or "baseline"
|
||
if kind == "queue":
|
||
run_queue(QUEUE_PROFILES[args.profile], label)
|
||
elif kind == "uploads":
|
||
run_uploads(UPLOADS_PROFILES[args.profile], label)
|
||
elif kind == "reconcile":
|
||
run_reconcile(RECONCILE_PROFILES[args.profile], label)
|
||
elif kind == "failover":
|
||
run_failover(PROFILES[args.profile], label)
|
||
else:
|
||
run_rollout(PROFILES[args.profile], label)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|