- upload: delete the stored object when ready fails before COMMIT or the pending row is gone (503 instead of 500); unknown COMMIT outcome keeps it - cleaner service (python -m src.cleaner): stale pending rows older than grace (900 s, must exceed 17 s live-upload bound) under FOR UPDATE SKIP LOCKED, object before row, own S3 user with read/delete only - shared session_scope for API and cleaner; early_stop process name - partial index on pending attachments, CREATE INDEX CONCURRENTLY - s3-bootstrap: cleaner user, policy and checks; make env_setup appends missing template variables to an existing .env - failure checks: cleaner on the failure stand, id-list hook actions, five new and three updated tests; driver upload-consistency - sync upload-consistency deltas into main specs (storage-consistency) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5991 lines
242 KiB
Python
5991 lines
242 KiB
Python
"""Нагрузочный стенд: матрица размеров истории и конкуренции.
|
||
|
||
Запуск — `make bench` / `make bench_quick`, профиль отказа реплик —
|
||
`make bench_failover` / `make bench_failover_quick`, поочерёдной замены —
|
||
`make bench_rollout` / `make bench_rollout_quick`, серии трека 1 —
|
||
`make bench_scaling SERIES=…` / `make bench_scaling_quick`. Стенд —
|
||
HAProxy, N реплик API, PostgreSQL и Valkey: проект compose `mtusi-bench` из
|
||
`docker-compose.yml` с override `bench/bench.compose.yml`. Методика —
|
||
design changes `load-bench-harness`, `bench-postgres`, `graceful-failover`,
|
||
`rolling-update` и `scaling-bench`, инварианты — `.claude/rules/bench.md`.
|
||
Только stdlib:
|
||
приложение не импортируется, его нагружает k6 по HTTP через LB; к
|
||
PostgreSQL стенда оркестратор ходит только для обслуживания и измерений
|
||
(VACUUM, размеры, планы запроса, сверка записей).
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import fcntl
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import secrets
|
||
import shutil
|
||
import signal
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable
|
||
from dataclasses import asdict, dataclass, replace
|
||
from datetime import UTC, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import failover
|
||
import plots
|
||
import resources
|
||
import rollout
|
||
import scaling
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
K6_IMAGE = (
|
||
"grafana/k6:2.3.0@sha256:"
|
||
"9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34"
|
||
)
|
||
APP_IMAGE = "mtusi-messenger:bench"
|
||
# профиль поочерёдной замены: образ прежнего commit (git archive), с него
|
||
# стартуют реплики; тег снимается при уборке
|
||
FROM_IMAGE = "mtusi-messenger:bench-from"
|
||
ROLLOUT_SCRIPT = ROOT / "scripts" / "rolling-update.sh"
|
||
PROJECT = "mtusi-bench"
|
||
NETWORK = "mtusi-bench"
|
||
# реплики api без постоянного имени: их список — docker compose ps
|
||
APP_SERVICE = "api"
|
||
# готовность процесса реплики изнутри контейнера — команда HEALTHCHECK
|
||
# образа (Dockerfile); у реплик стенда healthcheck выключен
|
||
APP_READY_PROBE = (
|
||
"import os, urllib.request; urllib.request.urlopen("
|
||
"'http://127.0.0.1:%s/health/ready' % os.environ.get('APP__PORT', "
|
||
"'8080'), timeout=2)"
|
||
)
|
||
LB = "mtusi-bench-haproxy"
|
||
DB = "mtusi-bench-postgres"
|
||
CACHE = "mtusi-bench-valkey"
|
||
K6 = "mtusi-bench-k6"
|
||
# сэмплер серий трека 1: образ PostgreSQL стенда, --cgroupns host
|
||
# (решение 7 design change scaling-bench)
|
||
SAMPLER = "mtusi-bench-sampler"
|
||
SAMPLER_SCRIPT = ROOT / "bench" / "sampler.sh"
|
||
# один прогон стенда любого профиля: flock держится до выхода процесса
|
||
LOCK_FILE = ROOT / "bench" / ".bench.lock"
|
||
# строки журналов окна с 5xx и отказами (`raw/<ступень>-errors.log`):
|
||
# PostgreSQL — отказ и FATAL; API — ответ 5xx или запись error; HAProxy —
|
||
# поле статуса 5xx или ответ без сервера. Подстроки «53300» и « 5»
|
||
# ловили метки времени и request_id (verify S4)
|
||
ERRORS_PG_RE = re.compile(r"too many clients|FATAL")
|
||
ERRORS_API_RE = re.compile(r'"status": 5\d\d\b|"level": "error"')
|
||
ERRORS_LB_RE = re.compile(r"\s5\d\d\s|<NOSRV>")
|
||
# длинная строка журнала: начало и конец (класс исключения — в конце)
|
||
ERRORS_HEAD = 300
|
||
ERRORS_TAIL = 300
|
||
# второе хранилище этапа восстановления: контейнер, volume и имя в сети
|
||
# стенда (по нему API переключается на восстановленную базу)
|
||
RESTORE_DB = "mtusi-bench-postgres-restore"
|
||
RESTORE_VOLUME = "mtusi-bench_restore"
|
||
RESTORE_HOST = "postgres-restore"
|
||
# таблицы данных приложения: сверка числа строк после восстановления
|
||
DATA_TABLES = ("users", "chats", "chat_members", "sessions", "messages")
|
||
# контейнеры стенда с постоянным именем по ролям: лимиты, паспорт; реплики
|
||
# api — отдельно (Stand.replicas)
|
||
CONTAINERS = {"postgres": DB, "valkey": CACHE, "haproxy": LB}
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "bench" / "bench.compose.yml",
|
||
)
|
||
# подстановки compose вместо .env рабочего стенда
|
||
ENV_FILE = ROOT / "bench" / "bench.env"
|
||
# что из окружения хоста нужно самому docker compose; остальное (в том
|
||
# числе POSTGRES_* из shell) в стенд не попадает
|
||
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "DOCKER_HOST", "DOCKER_CONFIG")
|
||
# части имён переменных окружения API, которые в паспорт (stand.app_env) не
|
||
# пишутся: пароли Postgres и Valkey, секрет и ключ доступа S3
|
||
SECRET_ENV_MARKS = ("PASSWORD", "SECRET", "ACCESS_KEY")
|
||
# статистика HAProxy изнутри контейнера LB (как make lb_stats): порт 8404
|
||
# наружу не опубликован, curl в образе нет, bash есть
|
||
LB_STATS = (
|
||
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
|
||
'printf "GET /stats;csv HTTP/1.0\\r\\n\\r\\n" >&3 && cat <&3'
|
||
)
|
||
SUCCESS_METRIC = "http_req_duration{expected_response:true}"
|
||
# предупреждение о VM: лимиты памяти стенда + этот запас больше памяти VM
|
||
VM_RESERVE_GIB = 1.5
|
||
# предупреждение о хосте: реальное время прогона больше монотонного, с
|
||
SLEEP_WARN_S = 5
|
||
# psql в контейнере Postgres от пользователя образа, без пароля; SQL — stdin
|
||
PSQL = (
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
|
||
)
|
||
# настройки Postgres в паспорте: от них зависят план и стоимость записи
|
||
PG_SETTINGS = (
|
||
"shared_buffers",
|
||
"effective_cache_size",
|
||
"work_mem",
|
||
"maintenance_work_mem",
|
||
"max_connections",
|
||
"synchronous_commit",
|
||
"fsync",
|
||
"wal_level",
|
||
"checkpoint_timeout",
|
||
"max_wal_size",
|
||
"random_page_cost",
|
||
"jit",
|
||
"autovacuum",
|
||
"plan_cache_mode",
|
||
"shared_preload_libraries",
|
||
)
|
||
# auto_explain на этапе плана; вне его log_min_duration = -1 (выключен)
|
||
EXPLAIN_SETTINGS = {
|
||
"auto_explain.log_min_duration": "0",
|
||
"auto_explain.log_analyze": "on",
|
||
"auto_explain.log_buffers": "on",
|
||
"auto_explain.log_timing": "on",
|
||
"auto_explain.log_format": "text",
|
||
}
|
||
# пауза после pg_reload_conf(): SIGHUP доходит до соединений пула API
|
||
RELOAD_PAUSE_S = 1.0
|
||
# индекс, который использовал план приложения
|
||
INDEX_NODE_RE = re.compile(
|
||
r"Index Only Scan(?: Backward)? using (\w+)"
|
||
r"|Index Scan(?: Backward)? using (\w+)"
|
||
r"|Bitmap Index Scan on (\w+)",
|
||
)
|
||
# условие курсора в запросе приложения: сравнение кортежей
|
||
OR_CURSOR_RE = re.compile(
|
||
r"\(messages\.created_at, messages\.id\) > "
|
||
r"\((\$\d+::[A-Z][A-Z ]*[A-Z]), (\$\d+::UUID)\)",
|
||
)
|
||
# длина текста сообщения по контракту (contracts/openapi.json, text
|
||
# maxLength): проба записи проверяет, что такое сообщение вставляется
|
||
MAX_TEXT_LENGTH = 4000
|
||
# сообщение максимальной длины из несжимаемых символов CJK (~12 КБ UTF-8,
|
||
# md5 номера символа — детерминировано): вставка в DO, итог — NOTICE
|
||
LONG_TEXT_PROBE = f"""DO $$
|
||
BEGIN
|
||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||
SELECT gen_random_uuid(), chat_id, sender_id,
|
||
(SELECT string_agg(
|
||
chr(19968 + ('x' || substr(md5(i::text), 1, 4))::bit(16)::int % 20000),
|
||
'')
|
||
FROM generate_series(1, {MAX_TEXT_LENGTH}) i),
|
||
now() + interval '1 hour'
|
||
FROM probe_src LIMIT 1;
|
||
RAISE NOTICE 'long-text: ok';
|
||
EXCEPTION WHEN others THEN
|
||
RAISE NOTICE 'long-text: %', SQLERRM;
|
||
END $$;
|
||
"""
|
||
|
||
RUN_FIELDS = [
|
||
"run",
|
||
"scenario",
|
||
"size",
|
||
"concurrency",
|
||
"repeat",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_start_mib",
|
||
"rss_end_mib",
|
||
"rss_max_mib",
|
||
"mem_cgroup_end_mib",
|
||
"store_messages_start",
|
||
"store_messages_end",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"check_failed",
|
||
# лаба 2: новые столбцы — только в конец
|
||
"db_cpu_cores",
|
||
# лаба 3, серии трека 1 (решение 8 design change scaling-bench); у
|
||
# строк матрицы пусто
|
||
*scaling.SCALING_FIELDS,
|
||
]
|
||
PREFILL_FIELDS = [
|
||
"size",
|
||
"chunk",
|
||
"messages_before",
|
||
"messages_after",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_end_mib",
|
||
"mem_cgroup_end_mib",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"db_cpu_cores",
|
||
# серии трека 1: ячейка и попытка порции
|
||
"cell",
|
||
"attempt",
|
||
]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Profile:
|
||
name: str
|
||
sizes: tuple[int, ...]
|
||
concurrency: tuple[int, ...]
|
||
repeats: int
|
||
read_warmup_s: int
|
||
read_measure_s: int
|
||
write_warmup: int
|
||
write_measure: int
|
||
# порция ~30 с при ~850 сообщ/с на Postgres: прогресс наполнения
|
||
# печатается после порции, а спек требует строку не реже раза в минуту
|
||
prefill_chunk: int = 25_000
|
||
prefill_vus: int = 32
|
||
text_length: int = 64
|
||
page_limit: int = 50
|
||
walk_limit: int = 100
|
||
cursor_pool: int = 1000
|
||
members: int = 10
|
||
seed: int = 20260930
|
||
# зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно
|
||
# те страницы, с которых начнётся замер
|
||
warmup_seed_shift: int = 1_000_003
|
||
generator_cpu_limit: float = 4.0
|
||
cpus: str = "1"
|
||
memory: str = "2g"
|
||
# лаба 2: лимиты хранилищ (swap = память) и срок сессии стенда; идут в
|
||
# compose переменными BENCH_* (bench/bench.compose.yml). Решение 2
|
||
# design change bench-postgres
|
||
db_cpus: str = "4"
|
||
db_memory: str = "3g"
|
||
cache_cpus: str = "2"
|
||
cache_memory: str = "512m"
|
||
session_ttl_seconds: int = 21600
|
||
# трек 2: фоновые чаты уровнями (число чатов, сообщений в чате), то же
|
||
# на каждом размере; варианты индекса истории (имя, столбцы) для
|
||
# планов и стоимости записи; пробы записи — строк и повторов.
|
||
# Решения 1, 4, 5 design change track2-query-plans
|
||
background: tuple[tuple[int, int], ...] = ()
|
||
index_variants: tuple[tuple[str, str], ...] = (
|
||
("chat_only", "(chat_id)"),
|
||
("covering", "(chat_id, created_at, id) INCLUDE (sender_id, text)"),
|
||
)
|
||
write_probe_rows: int = 20_000
|
||
write_probe_repeats: int = 3
|
||
# трек 2, часть 2: backup и восстановление в новое хранилище в конце
|
||
# размера; pg_restore -j, запись через HTTP во время backup. Решение 1
|
||
# design change track2-backup-restore
|
||
restore: bool = False
|
||
restore_jobs: int = 4
|
||
restore_write_load: int = 5_000
|
||
restore_write_vus: int = 10
|
||
# лаба 3: реплики api за HAProxy (лимиты cpus/memory — на каждую) и
|
||
# лимиты LB; в compose — BENCH_API_REPLICAS и BENCH_LB_*. Решение 6
|
||
# design change graceful-failover
|
||
replicas: int = 1
|
||
lb_cpus: str = "1"
|
||
lb_memory: str = "256m"
|
||
# профиль отказа реплик (решение 7): режимы фаз, постоянная
|
||
# интенсивность записи и чтения в секунду, окна фазы (до действия,
|
||
# отказ до возврата, после возврата), порог восстановления, VU k6 на
|
||
# сценарий: заранее созданные и предел. Зависшая реплика держит
|
||
# половину запросов до timeout server LB (10 с): 50/с × 10 с = 500 VU
|
||
# на сценарий. Без запаса k6 создаёт VU на ходу и пропускает итерации
|
||
# (самопроверка: 352 из 2 000 за окно pause), и интенсивность перестаёт
|
||
# быть постоянной
|
||
failover_modes: tuple[str, ...] = ()
|
||
failover_write_rate: int = 0
|
||
failover_read_rate: int = 0
|
||
failover_before_s: int = 0
|
||
failover_down_s: int = 0
|
||
failover_after_s: int = 0
|
||
failover_success: float = 0.99
|
||
failover_pre_vus: int = 600
|
||
failover_max_vus: int = 1500
|
||
# профиль поочерёдной замены (решение 8 design change rolling-update):
|
||
# commit прежнего образа, окна — до запуска scripts/rolling-update.sh,
|
||
# бюджет замены и после него. Постоянная нагрузка, порог доли успешных
|
||
# и VU k6 — поля failover_* выше: тот же k6/failover.js
|
||
rollout_from: str = ""
|
||
rollout_before_s: int = 0
|
||
rollout_budget_s: int = 0
|
||
rollout_after_s: int = 0
|
||
# профили уровня «4» (failover, rollout): сэмплер в режиме scan на всё
|
||
# время нагрузки — CPU и память по компонентам и окнам. Признак
|
||
# наблюдения, нагрузку и окна не меняет; лимиты и интервал сэмплера —
|
||
# поля sampler_* ниже (design change profile-resources, решение 2)
|
||
sample_resources: bool = False
|
||
# серии трека 1 (решения 1–6, 14 design change scaling-bench): имя
|
||
# серии, конфигурации (реплик, CPU и память реплики, max_connections
|
||
# PostgreSQL), чаты записи, смесь, фиксированные уровни, лестница и
|
||
# поиск R*, окна ступени, SLO, досрочный выход, VU k6 на запрос/с,
|
||
# лимиты генератора и сэмплера, зерно порядка ячеек. У прочих
|
||
# профилей — пусто: их поведение не меняется. max_connections — 100,
|
||
# как у образа; у ячейки серии — из конфигурации
|
||
db_max_connections: int = 100
|
||
series: str = ""
|
||
# у серии db — пятый элемент: CPU PostgreSQL конфигурации
|
||
configs: tuple[tuple[Any, ...], ...] = ()
|
||
write_chats: int = 1
|
||
mix: tuple[tuple[str, float], ...] = ()
|
||
fixed_rps: tuple[int, ...] = ()
|
||
ladder_base: int = 100
|
||
step_factor: float = 1.25
|
||
start_rps_per_replica: int = 300
|
||
bisect_steps: int = 2
|
||
max_steps: int = 12
|
||
first_warmup_s: int = 15
|
||
step_warmup_s: int = 5
|
||
step_measure_s: int = 20
|
||
cooldown_s: int = 10
|
||
slo_p95_ms: float = 200
|
||
slo_error_ratio: float = 0.01
|
||
abort_error_ratio: float = 0.5
|
||
abort_p95_ms: float = 2000
|
||
request_timeout_s: int = 15
|
||
prevus_per_rps: float = 0.05
|
||
vus_per_rps: float = 0.2
|
||
generator_cpus: str = ""
|
||
generator_memory: str = ""
|
||
sampler_cpus: str = "0.25"
|
||
sampler_memory: str = "128m"
|
||
sample_interval_s: float = 1.0
|
||
order_seed: int = 20261011
|
||
|
||
@property
|
||
def is_series(self) -> bool:
|
||
return bool(self.series)
|
||
|
||
@property
|
||
def background_total(self) -> int:
|
||
return sum(count * size for count, size in self.background)
|
||
|
||
@property
|
||
def is_failover(self) -> bool:
|
||
return bool(self.failover_modes)
|
||
|
||
@property
|
||
def is_rollout(self) -> bool:
|
||
return bool(self.rollout_from)
|
||
|
||
|
||
PROFILES = {
|
||
"full": Profile(
|
||
name="full",
|
||
# 500k вместо 1M лабы 1: прогон ~30 мин вместо ~45, а диапазон
|
||
# 10k → 500k (×50) уже показывает, растёт ли страница с историей
|
||
sizes=(10_000, 100_000, 500_000),
|
||
concurrency=(1, 10, 50),
|
||
repeats=3,
|
||
read_warmup_s=5,
|
||
read_measure_s=20,
|
||
write_warmup=500,
|
||
write_measure=5000,
|
||
# 100 000 фоновых сообщений: в таблице ≥110 000 на любом размере
|
||
background=((10, 5_000), (100, 400), (1_000, 10)),
|
||
),
|
||
"quick": Profile(
|
||
name="quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(1, 4, 16),
|
||
repeats=1,
|
||
read_warmup_s=2,
|
||
read_measure_s=5,
|
||
write_warmup=100,
|
||
write_measure=1000,
|
||
background=((2, 1_000), (10, 100), (50, 10)),
|
||
write_probe_rows=5_000,
|
||
),
|
||
# backup и восстановление на данных полной матрицы, без прогонов
|
||
"restore": Profile(
|
||
name="restore",
|
||
sizes=(10_000, 100_000, 500_000),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
background=((10, 5_000), (100, 400), (1_000, 10)),
|
||
restore=True,
|
||
),
|
||
"restore_quick": Profile(
|
||
name="restore_quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
background=((2, 1_000), (10, 100), (50, 10)),
|
||
write_probe_rows=5_000,
|
||
restore=True,
|
||
restore_write_load=1_000,
|
||
),
|
||
# отказ реплик под нагрузкой (уровень 4): две реплики с лимитами
|
||
# рабочего стенда (docker-compose.yml), 100 записей и 100 чтений в
|
||
# секунду, каждая реплика в каждом режиме. Решение 7 design change
|
||
# graceful-failover
|
||
"failover": Profile(
|
||
name="failover",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_modes=("stop", "kill", "pause"),
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
failover_before_s=15,
|
||
failover_down_s=20,
|
||
failover_after_s=15,
|
||
sample_resources=True,
|
||
),
|
||
"failover_quick": Profile(
|
||
name="failover_quick",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_modes=("stop", "kill"),
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
failover_before_s=5,
|
||
failover_down_s=10,
|
||
failover_after_s=8,
|
||
sample_resources=True,
|
||
),
|
||
# поочерёдная замена под нагрузкой (уровень 4): та же нагрузка и
|
||
# лимиты, что у failover; реплики стартуют на образе e1161f7 —
|
||
# последнего commit лабы 4 до change attachments (без таблиц вложений
|
||
# и с прежним ограничением текста) — и заменяются на образ текущего
|
||
# checkout. Решение 8 design change rolling-update, commit — решение
|
||
# 14 design change attachments; правило выбора — bench.md
|
||
"rollout": Profile(
|
||
name="rollout",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
rollout_from="e1161f7",
|
||
rollout_before_s=15,
|
||
rollout_budget_s=90,
|
||
rollout_after_s=15,
|
||
sample_resources=True,
|
||
),
|
||
"rollout_quick": Profile(
|
||
name="rollout_quick",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
rollout_from="e1161f7",
|
||
rollout_before_s=5,
|
||
rollout_budget_s=60,
|
||
rollout_after_s=8,
|
||
sample_resources=True,
|
||
),
|
||
}
|
||
|
||
|
||
def scaling_profile(
|
||
name: str,
|
||
series: str,
|
||
configs: tuple[tuple[Any, ...], ...],
|
||
**fields: Any,
|
||
) -> Profile:
|
||
"""Профиль серии трека 1: общие поля всех серий в одном месте, чтобы
|
||
методика серий не расходилась (решения 1–3, 6, 14 design change
|
||
`scaling-bench`). Лимиты хранилищ и LB — как у рабочего стенда, k6 —
|
||
2 CPU / 1 ГиБ, сэмплер — 0,25 / 128 МиБ; поля реплик (`replicas`,
|
||
`cpus`, `memory`) заменяет конфигурация ячейки."""
|
||
common: dict[str, Any] = {
|
||
"name": name,
|
||
"sizes": (10_000,),
|
||
"concurrency": (),
|
||
"repeats": 3,
|
||
"read_warmup_s": 0,
|
||
"read_measure_s": 0,
|
||
"write_warmup": 0,
|
||
"write_measure": 0,
|
||
"members": 10,
|
||
"cpus": "1",
|
||
"memory": "512m",
|
||
"db_cpus": "2",
|
||
"db_memory": "1g",
|
||
"cache_cpus": "0.5",
|
||
"cache_memory": "128m",
|
||
"lb_cpus": "1",
|
||
"lb_memory": "256m",
|
||
"generator_cpu_limit": 1.8,
|
||
"series": series,
|
||
"configs": configs,
|
||
"write_chats": 10,
|
||
"mix": (("read", 0.7), ("write", 0.2), ("me", 0.1)),
|
||
"fixed_rps": (250, 500),
|
||
"generator_cpus": "2",
|
||
"generator_memory": "1g",
|
||
# все VU — заранее, как предел (найдено в apply): с 0,05 k6 создавал
|
||
# VU на ходу при росте задержек и пропускал итерации у R* при
|
||
# свободном пределе — недосланные шли в ошибки SLO от генератора
|
||
"prevus_per_rps": 0.2,
|
||
}
|
||
return Profile(**{**common, **fields})
|
||
|
||
|
||
B_CONFIGS = tuple((k, "1", "512m", 100) for k in range(1, 7))
|
||
# серия db: CPU PostgreSQL у 1, 2 и 4 реплик и контроль 4 × 1 при 2 CPU
|
||
DB_CPUS_TIGHT = "0.5"
|
||
DB_CONFIGS = (
|
||
(1, "1", "512m", 100, DB_CPUS_TIGHT),
|
||
(2, "1", "512m", 100, DB_CPUS_TIGHT),
|
||
(4, "1", "512m", 100, DB_CPUS_TIGHT),
|
||
(4, "1", "512m", 100, "2"),
|
||
)
|
||
PROFILES.update(
|
||
{
|
||
# фиксированный бюджет приложения 4 CPU / 2 ГиБ
|
||
"scaling_a": scaling_profile(
|
||
"scaling_a",
|
||
"a",
|
||
((1, "4", "2g", 100), (2, "2", "1g", 100), (4, "1", "512m", 100)),
|
||
),
|
||
# рост: 1…6 реплик по 1 CPU / 512 МиБ (решение 1: до 6 — бюджет ВМ)
|
||
"scaling_b": scaling_profile("scaling_b", "b", B_CONFIGS),
|
||
# граница пула: 4 реплики при max_connections 100 и 50
|
||
"scaling_pool": scaling_profile(
|
||
"scaling_pool",
|
||
"pool",
|
||
((4, "1", "512m", 100), (4, "1", "512m", 50)),
|
||
),
|
||
# «до/после» и A/A: 1 × 1 и 6 × 1 серии B (решение 11)
|
||
"scaling_ab": scaling_profile(
|
||
"scaling_ab",
|
||
"ab",
|
||
((1, "1", "512m", 100), (6, "1", "512m", 100)),
|
||
),
|
||
# самопроверка: две конфигурации, второй предел соединений
|
||
"scaling_quick": scaling_profile(
|
||
"scaling_quick",
|
||
"quick",
|
||
((1, "1", "512m", 100), (2, "1", "512m", 50)),
|
||
sizes=(2_000,),
|
||
repeats=1,
|
||
fixed_rps=(250,),
|
||
first_warmup_s=8,
|
||
step_warmup_s=3,
|
||
step_measure_s=8,
|
||
bisect_steps=1,
|
||
max_steps=6,
|
||
),
|
||
# насыщение PostgreSQL управляемым опытом (verify S2, указание
|
||
# пользователя 2026-10-11): 1, 2 и 4 реплики при узком лимите CPU
|
||
# PostgreSQL и контроль — 4 реплики при прежних 2 CPU. Пятый
|
||
# элемент конфигурации — CPU PostgreSQL; лимит — по калибровке
|
||
# (design, «Найдено в verify»)
|
||
"scaling_db": scaling_profile(
|
||
"scaling_db",
|
||
"db",
|
||
DB_CONFIGS,
|
||
),
|
||
# подмножество серии db для «до/после» change scaling-improvement
|
||
# (решение пользователя 2026-10-11 сократить замеры): основная
|
||
# ячейка 4 × 1 при 0,5 CPU и контроль 4 × 1 при 2 CPU, те же поля и
|
||
# 3 повтора. Имя серии и состав конфигураций к методике сравнения
|
||
# не относятся: метки db4 сравнимы с метками db по общим ячейкам
|
||
"scaling_db4": scaling_profile(
|
||
"scaling_db4",
|
||
"db4",
|
||
DB_CONFIGS[2:4],
|
||
),
|
||
# самопроверка серии db и калибровка лимита: 2 и 4 реплики при
|
||
# узком лимите, один повтор, короткие окна
|
||
"scaling_db_quick": scaling_profile(
|
||
"scaling_db_quick",
|
||
"db_quick",
|
||
DB_CONFIGS[1:3],
|
||
sizes=(2_000,),
|
||
repeats=1,
|
||
fixed_rps=(250,),
|
||
first_warmup_s=8,
|
||
step_warmup_s=3,
|
||
step_measure_s=8,
|
||
bisect_steps=1,
|
||
max_steps=6,
|
||
),
|
||
},
|
||
)
|
||
|
||
|
||
def cell_profile(profile: Profile, cell: dict[str, Any]) -> Profile:
|
||
"""Профиль ячейки: реплики, max_connections и (серия db) CPU
|
||
PostgreSQL из конфигурации."""
|
||
return replace(
|
||
profile,
|
||
replicas=cell["replicas"],
|
||
cpus=cell["replica_cpus"],
|
||
memory=cell["replica_memory"],
|
||
db_max_connections=cell["db_max_connections"],
|
||
db_cpus=cell.get("db_cpus") or profile.db_cpus,
|
||
)
|
||
|
||
|
||
def budget(profile: Profile, vm_cpus: int, vm_memory: int) -> dict[str, Any]:
|
||
"""Сумма лимитов CPU и памяти стенда и генератора по конфигурациям
|
||
серии (решение 1): реплики, PostgreSQL, LB, Valkey, k6, сэмплер."""
|
||
result = {}
|
||
for config in profile.configs:
|
||
parts = scaling.unpack_config(config)
|
||
replicas, cpus, memory, _, db_cpus = parts
|
||
key = scaling.config_key(*parts)
|
||
cpu = replicas * float(cpus) + sum(
|
||
float(value)
|
||
for value in (
|
||
db_cpus or profile.db_cpus,
|
||
profile.lb_cpus,
|
||
profile.cache_cpus,
|
||
profile.generator_cpus or "0",
|
||
profile.sampler_cpus,
|
||
)
|
||
)
|
||
mem = replicas * size_bytes(memory) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.lb_memory,
|
||
profile.cache_memory,
|
||
profile.generator_memory or "0",
|
||
profile.sampler_memory,
|
||
)
|
||
)
|
||
result[key] = {
|
||
"cpus": round(cpu, 2),
|
||
"memory_gib": round(mem / 2**30, 2),
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_memory / 2**30, 2),
|
||
}
|
||
return result
|
||
|
||
|
||
def level4_budget(
|
||
profile: Profile,
|
||
vm_cpus: int,
|
||
vm_memory: int,
|
||
) -> dict[str, Any]:
|
||
"""Сумма лимитов CPU и памяти профиля уровня «4» на пике (решение 8
|
||
design change `profile-resources`): реплики (у `rollout` — на одну
|
||
больше: на пике замены работает третья), PostgreSQL, LB, Valkey,
|
||
сэмплер; генератор — только если у него есть лимит. Миграция идёт до
|
||
третьей реплики и пик не поднимает."""
|
||
peak = profile.replicas + (1 if profile.is_rollout else 0)
|
||
cpu = peak * float(profile.cpus) + sum(
|
||
float(value)
|
||
for value in (
|
||
profile.db_cpus,
|
||
profile.lb_cpus,
|
||
profile.cache_cpus,
|
||
profile.generator_cpus or "0",
|
||
profile.sampler_cpus,
|
||
)
|
||
)
|
||
mem = peak * size_bytes(profile.memory) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.lb_memory,
|
||
profile.cache_memory,
|
||
profile.generator_memory or "0",
|
||
profile.sampler_memory,
|
||
)
|
||
)
|
||
return {
|
||
"cpus": round(cpu, 2),
|
||
"memory_gib": round(mem / 2**30, 2),
|
||
"peak_replicas": peak,
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_memory / 2**30, 2),
|
||
"generator": (
|
||
f"{profile.generator_cpus} CPU / {profile.generator_memory}"
|
||
if profile.generator_cpus
|
||
else "no limit"
|
||
),
|
||
}
|
||
|
||
|
||
class BenchError(Exception):
|
||
"""Стенд не может продолжать: сообщение для человека."""
|
||
|
||
|
||
# ---------------------------------------------------------------- утилиты
|
||
|
||
|
||
def sh(*args: str, check: bool = True) -> str:
|
||
"""Команда без shell; stdout без хвостовых пробелов.
|
||
|
||
`LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод
|
||
разбирается регулярками с точкой.
|
||
"""
|
||
proc = subprocess.run(
|
||
args,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
env={**os.environ, "LC_ALL": "C"},
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"{' '.join(args[:4])}…: exit {proc.returncode}\n"
|
||
f"{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
|
||
def stop_stream(proc: subprocess.Popen[bytes]) -> None:
|
||
"""Остановить поток `docker events`: SIGTERM, 10 с на выход, затем
|
||
SIGKILL — зависший CLI не держит уборку стенда."""
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=10)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
proc.wait()
|
||
|
||
|
||
def human(n: int) -> str:
|
||
"""10000 → 10k, 1000000 → 1M."""
|
||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||
return f"{n // 1_000_000}M"
|
||
if n >= 1000 and n % 1000 == 0:
|
||
return f"{n // 1000}k"
|
||
return str(n)
|
||
|
||
|
||
def fmt_left(seconds: float) -> str:
|
||
seconds = max(0, int(seconds))
|
||
hours, rest = divmod(seconds, 3600)
|
||
minutes, secs = divmod(rest, 60)
|
||
if hours:
|
||
return f"{hours}h{minutes:02d}m"
|
||
if minutes:
|
||
return f"{minutes}m{secs:02d}s"
|
||
return f"{secs}s"
|
||
|
||
|
||
def mib(kib: float | None) -> float | None:
|
||
return None if kib is None else round(kib / 1024, 1)
|
||
|
||
|
||
def mib_bytes(value: int | None) -> float | None:
|
||
return None if value is None else round(value / 2**20, 1)
|
||
|
||
|
||
def size_bytes(text: str) -> int:
|
||
"""Лимит памяти docker (`512m`, `2g`) в байтах."""
|
||
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kmg]?)b?", text.strip().lower())
|
||
if not match:
|
||
raise BenchError(f"bad memory limit {text!r}: use 512m or 2g")
|
||
power = {"": 0, "k": 1, "m": 2, "g": 3}[match.group(2)]
|
||
return int(float(match.group(1)) * 1024**power)
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 0) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 3) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def chat_sizes(profile: Profile, size: int) -> list[int]:
|
||
"""Размеры чатов наполнения: читаемый, затем фоновые по уровням."""
|
||
return [size] + [
|
||
chat_size
|
||
for count, chat_size in profile.background
|
||
for _ in range(count)
|
||
]
|
||
|
||
|
||
def fill_schedule(sizes: list[int]) -> list[int]:
|
||
"""Номер чата для каждого сообщения наполнения.
|
||
|
||
Чат с s сообщениями получает позиции (k + 0,5) / s, k = 0…s−1; все
|
||
позиции сортируются, при равенстве — по номеру чата. Сообщения
|
||
каждого чата равномерно размазаны по наполнению, порядок
|
||
детерминирован (решение 2 design change `track2-query-plans`).
|
||
"""
|
||
slots = sorted(
|
||
((k + 0.5) / size, chat)
|
||
for chat, size in enumerate(sizes)
|
||
for k in range(size)
|
||
)
|
||
return [chat for _, chat in slots]
|
||
|
||
|
||
def or_cursor(text: str) -> str:
|
||
"""Запрос приложения с условием курсора через OR вместо кортежей.
|
||
|
||
Остальное — список столбцов, ORDER BY, LIMIT — как у приложения. Нет
|
||
шаблона — приложение изменило запрос, тихой копии SQL не делаем.
|
||
"""
|
||
new, count = OR_CURSOR_RE.subn(
|
||
r"(messages.created_at > \1 OR "
|
||
r"(messages.created_at = \1 AND messages.id > \2))",
|
||
text,
|
||
)
|
||
if count != 1:
|
||
raise BenchError(
|
||
"history query has no (created_at, id) row comparison: the app "
|
||
"query changed, update OR_CURSOR_RE",
|
||
)
|
||
return new
|
||
|
||
|
||
# ------------------------------------------------------------ хост и паспорт
|
||
|
||
|
||
def swapouts() -> int | None:
|
||
"""Счётчик выгрузок страниц в swap (`vm_stat`, macOS); иначе None."""
|
||
if platform.system() != "Darwin":
|
||
return None
|
||
match = re.search(r"Swapouts:\s+(\d+)", sh("vm_stat", check=False))
|
||
return int(match.group(1)) if match else None
|
||
|
||
|
||
def host_state() -> dict[str, Any]:
|
||
"""Swap, выгрузки в swap, свободная память и питание (macOS); на
|
||
другой ОС — null. Занятый swap — не помеха; помеха — прирост
|
||
`swapouts` за прогон (решение 12 design change `scaling-bench`)."""
|
||
state: dict[str, Any] = {
|
||
"swap_used_mib": None,
|
||
"swap_total_mib": None,
|
||
"swapouts": None,
|
||
"memory_free_percent": None,
|
||
"power": None,
|
||
"battery_percent": None,
|
||
}
|
||
if platform.system() != "Darwin":
|
||
return state
|
||
state["swapouts"] = swapouts()
|
||
swap = sh("sysctl", "-n", "vm.swapusage", check=False)
|
||
for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")):
|
||
match = re.search(rf"{key} = ([\d.,]+)M", swap)
|
||
if match:
|
||
state[name] = float(match.group(1).replace(",", "."))
|
||
pressure = sh("memory_pressure", check=False)
|
||
match = re.search(r"free percentage: (\d+)%", pressure)
|
||
if match:
|
||
state["memory_free_percent"] = int(match.group(1))
|
||
batt = sh("pmset", "-g", "batt", check=False)
|
||
match = re.search(r"drawing from '([^']+)'", batt)
|
||
if match:
|
||
state["power"] = match.group(1)
|
||
match = re.search(r"(\d+)%;", batt)
|
||
if match:
|
||
state["battery_percent"] = int(match.group(1))
|
||
return state
|
||
|
||
|
||
def host_warnings(state: dict[str, Any]) -> list[str]:
|
||
warnings = []
|
||
if state["power"] == "Battery Power":
|
||
warnings.append("host on battery power: CPU may be throttled")
|
||
return warnings
|
||
|
||
|
||
def swapouts_delta(start: dict[str, Any], end: dict[str, Any]) -> int | None:
|
||
"""Прирост выгрузок в swap за прогон: None, если одного из счётчиков
|
||
нет; иначе разность, в том числе 0 (счётчик 0 после перезагрузки —
|
||
тоже значение, design change `fix-known-issues`, решение 13)."""
|
||
before, after = start.get("swapouts"), end.get("swapouts")
|
||
if before is None or after is None:
|
||
return None
|
||
return after - before
|
||
|
||
|
||
def swap_warning(start: dict[str, Any], end: dict[str, Any]) -> str | None:
|
||
"""Выгрузка в swap за прогон: счётчик вырос — предупреждение."""
|
||
delta = swapouts_delta(start, end)
|
||
if delta is None or delta <= 0:
|
||
return None
|
||
return f"host swapped out {delta} pages during the run: rerun"
|
||
|
||
|
||
def vm_warnings(profile: Profile) -> list[str]:
|
||
"""Лимиты памяти стенда не помещаются в VM docker с запасом.
|
||
|
||
Тогда лимит перестаёт быть гарантией: контейнеры делят память VM, и
|
||
метки разных дней несравнимы. У серий — наибольшая конфигурация с
|
||
генератором и сэмплером.
|
||
"""
|
||
total = sh("docker", "info", "--format", "{{.MemTotal}}", check=False)
|
||
if not total.isdigit():
|
||
return []
|
||
replicas = [(profile.replicas, profile.memory)]
|
||
if profile.is_series:
|
||
replicas = [
|
||
(k, memory)
|
||
for k, _, memory, *_ in map(scaling.unpack_config, profile.configs)
|
||
]
|
||
elif profile.is_rollout and profile.sample_resources:
|
||
# на пике замены работает третья реплика (design change
|
||
# profile-resources, решение 8)
|
||
replicas = [(profile.replicas + 1, profile.memory)]
|
||
extra: tuple[str, ...] = ()
|
||
if profile.is_series:
|
||
extra = (profile.generator_memory, profile.sampler_memory)
|
||
elif profile.sample_resources:
|
||
extra = (profile.sampler_memory,)
|
||
limits = max(k * size_bytes(memory) for k, memory in replicas) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.cache_memory,
|
||
profile.lb_memory,
|
||
*extra,
|
||
)
|
||
if value
|
||
)
|
||
budget = int(total) - int(VM_RESERVE_GIB * 2**30)
|
||
if limits <= budget:
|
||
return []
|
||
return [
|
||
f"memory limits {limits / 2**30:.1f} GiB + {VM_RESERVE_GIB} GiB "
|
||
f"reserve exceed docker VM memory {int(total) / 2**30:.1f} GiB: "
|
||
"limits are not guaranteed",
|
||
]
|
||
|
||
|
||
def git_info(exclude: str | None = None) -> dict[str, Any]:
|
||
"""Commit и признак незакоммиченных изменений; `exclude` — каталог
|
||
своей метки при продолжении серии: её частичные результаты — не
|
||
изменение кода (решение 5 design change `scaling-bench`)."""
|
||
commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD")
|
||
args = ["git", "-C", str(ROOT), "status", "--porcelain"]
|
||
if exclude:
|
||
args += ["--", ".", f":!{exclude}"]
|
||
status = sh(*args)
|
||
return {"commit": commit, "dirty": bool(status)}
|
||
|
||
|
||
def static_passport(profile: Profile) -> dict[str, Any]:
|
||
docker = json.loads(sh("docker", "info", "--format", "{{json .}}"))
|
||
versions = sh(
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
"-c",
|
||
"import importlib.metadata as m, json, platform; print(json.dumps("
|
||
"{'python': platform.python_version(), **{p: m.version(p) for p in "
|
||
"('fastapi', 'starlette', 'pydantic', 'uvicorn', 'sqlalchemy', "
|
||
"'asyncpg', 'redis')}}))",
|
||
)
|
||
k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version")
|
||
host: dict[str, Any] = {"os": platform.system(), "machine": None}
|
||
if platform.system() == "Darwin":
|
||
host["os"] = f"macOS {platform.mac_ver()[0]}"
|
||
host["machine"] = sh(
|
||
"sysctl",
|
||
"-n",
|
||
"machdep.cpu.brand_string",
|
||
check=False,
|
||
)
|
||
memsize = sh("sysctl", "-n", "hw.memsize", check=False)
|
||
host["memory_gib"] = (
|
||
round(int(memsize) / 2**30, 1) if memsize.isdigit() else None
|
||
)
|
||
# Apple silicon: производительные и энергоэффективные ядра; vCPU ВМ
|
||
# к ним не привязаны (решение 12 design change scaling-bench)
|
||
for key, level in (
|
||
("performance_cores", "hw.perflevel0.physicalcpu"),
|
||
("efficiency_cores", "hw.perflevel1.physicalcpu"),
|
||
):
|
||
value = sh("sysctl", "-n", level, check=False)
|
||
host[key] = int(value) if value.isdigit() else None
|
||
return {
|
||
"app_image": {
|
||
"tag": APP_IMAGE,
|
||
"id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE),
|
||
"versions": json.loads(versions),
|
||
},
|
||
"generator": {"image": K6_IMAGE, "version": k6_version},
|
||
"docker": {
|
||
"server": docker.get("ServerVersion"),
|
||
"vm_os": docker.get("OperatingSystem"),
|
||
"vm_cpus": docker.get("NCPU"),
|
||
"vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1),
|
||
"cgroup": docker.get("CgroupVersion"),
|
||
"logging_driver": docker.get("LoggingDriver"),
|
||
},
|
||
"host": host,
|
||
"profile": asdict(profile),
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------------------ стенд
|
||
|
||
|
||
class Watcher:
|
||
"""Раз в секунду, пока идёт запуск k6: VmRSS реплик (сумма), живы ли
|
||
они, PostgreSQL и Valkey.
|
||
|
||
Реплика или хранилище умерли (OOM, kill) — k6 останавливается: иначе
|
||
каждый его запрос ждал бы таймаута, и наполнение тянулось бы часами.
|
||
"""
|
||
|
||
def __init__(self, replicas: list[str], *, rss: bool = True) -> None:
|
||
self.replicas = replicas
|
||
self.rss = rss
|
||
self.samples: list[int] = []
|
||
self.app_died = False
|
||
self.stand_died: list[str] = []
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(
|
||
target=self._loop if rss else self._light_loop,
|
||
daemon=True,
|
||
)
|
||
|
||
def _halt(self) -> None:
|
||
sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False)
|
||
|
||
def _light_loop(self) -> None:
|
||
"""Серии трека 1: один `docker inspect` реплик, хранилищ и LB в
|
||
секунду, без `docker exec` в реплики (решение 7 design change
|
||
`scaling-bench`): процесс `grep` попадал бы в cgroup реплики."""
|
||
while not self._stop.wait(1.0):
|
||
states = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Name}} {{.State.Running}}",
|
||
*self.replicas,
|
||
DB,
|
||
CACHE,
|
||
LB,
|
||
check=False,
|
||
)
|
||
running = dict(
|
||
line.split() for line in states.splitlines() if line
|
||
)
|
||
names = [*self.replicas, DB, CACHE, LB]
|
||
dead = [n for n in names if running.get(f"/{n}") != "true"]
|
||
if not dead:
|
||
continue
|
||
if any(name in self.replicas for name in dead):
|
||
self.app_died = True
|
||
else:
|
||
self.stand_died = dead
|
||
self._halt()
|
||
return
|
||
|
||
def _loop(self) -> None:
|
||
while not self._stop.wait(1.0):
|
||
states = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Name}} {{.State.Running}}",
|
||
DB,
|
||
CACHE,
|
||
check=False,
|
||
)
|
||
dead = [
|
||
name.lstrip("/")
|
||
for name, running in (
|
||
line.split() for line in states.splitlines() if line
|
||
)
|
||
if running != "true"
|
||
]
|
||
if dead:
|
||
self.stand_died = dead
|
||
self._halt()
|
||
return
|
||
total = 0
|
||
for replica in self.replicas:
|
||
out = sh(
|
||
"docker",
|
||
"exec",
|
||
replica,
|
||
"grep",
|
||
"^VmRSS:",
|
||
"/proc/1/status",
|
||
check=False,
|
||
)
|
||
match = re.search(r"(\d+) kB", out)
|
||
if match:
|
||
total += int(match.group(1))
|
||
continue
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
replica,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
self.app_died = True
|
||
self._halt()
|
||
return
|
||
total = 0
|
||
break
|
||
if total:
|
||
self.samples.append(total)
|
||
|
||
def __enter__(self) -> "Watcher":
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *_: object) -> None:
|
||
self._stop.set()
|
||
self._thread.join()
|
||
|
||
|
||
def compose_env(profile: Profile) -> dict[str, str]:
|
||
"""Окружение `docker compose` стенда, собранное с нуля.
|
||
|
||
Пароли хранилищ — новые на каждый прогон: живут только в этом словаре
|
||
и в контейнерах стенда, в результаты не пишутся. Лимиты и срок сессии
|
||
— из профиля (`BENCH_*` в `bench/bench.compose.yml`).
|
||
|
||
Секреты S3 compose требует при разборе файла (`:?` в
|
||
`docker-compose.yml`), хотя профили лаб 2–3 сервер S3 не запускают:
|
||
реплики API получают ключи, но к S3 не обращаются (design change
|
||
`s3-storage`, решение 9). С уборщиком (design change
|
||
`upload-consistency`, решение 9) — ещё ключи его пользователя: сервис
|
||
`cleaner` описан, но профили его не запускают.
|
||
"""
|
||
env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
|
||
return {
|
||
**env,
|
||
"LC_ALL": "C",
|
||
"POSTGRES_PASSWORD": secrets.token_urlsafe(24),
|
||
"VALKEY_PASSWORD": secrets.token_urlsafe(24),
|
||
"S3_ROOT_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_ROOT_SECRET_KEY": secrets.token_hex(20),
|
||
"S3_JWT_SIGNING_KEY": secrets.token_hex(32),
|
||
"S3_API_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_API_SECRET_KEY": secrets.token_hex(20),
|
||
"S3_CLEANER_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_CLEANER_SECRET_KEY": secrets.token_hex(20),
|
||
"BENCH_APP_CPUS": profile.cpus,
|
||
"BENCH_APP_MEMORY": profile.memory,
|
||
"BENCH_DB_CPUS": profile.db_cpus,
|
||
"BENCH_DB_MEMORY": profile.db_memory,
|
||
"BENCH_CACHE_CPUS": profile.cache_cpus,
|
||
"BENCH_CACHE_MEMORY": profile.cache_memory,
|
||
"BENCH_SESSION_TTL_SECONDS": str(profile.session_ttl_seconds),
|
||
"BENCH_API_REPLICAS": str(profile.replicas),
|
||
"BENCH_LB_CPUS": profile.lb_cpus,
|
||
"BENCH_LB_MEMORY": profile.lb_memory,
|
||
"BENCH_DB_MAX_CONNECTIONS": str(profile.db_max_connections),
|
||
}
|
||
|
||
|
||
def leftovers() -> dict[str, list[str]]:
|
||
"""Контейнеры, volume и сеть стенда, оставшиеся от другого прогона.
|
||
|
||
Ключ — подкоманда docker, которой их удалить.
|
||
"""
|
||
project = f"label=com.docker.compose.project={PROJECT}"
|
||
names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split()
|
||
containers = sh(
|
||
"docker",
|
||
"ps",
|
||
"-a",
|
||
"--filter",
|
||
project,
|
||
"--format",
|
||
"{{.Names}}",
|
||
).split()
|
||
networks = sh("docker", "network", "ls", "--format", "{{.Name}}").split()
|
||
extra = [name for name in (K6, SAMPLER) if name in names]
|
||
return {
|
||
"rm -f": containers + [n for n in extra if n not in containers],
|
||
"volume rm": sh(
|
||
"docker",
|
||
"volume",
|
||
"ls",
|
||
"-q",
|
||
"--filter",
|
||
project,
|
||
).split(),
|
||
"network rm": [NETWORK] if NETWORK in networks else [],
|
||
}
|
||
|
||
|
||
class Stand:
|
||
"""Проект compose `mtusi-bench` и контейнер k6 на время одного размера."""
|
||
|
||
def __init__(self, profile: Profile, label_dir: Path) -> None:
|
||
self.profile = profile
|
||
self.label_dir = label_dir
|
||
self.work_dir = label_dir / ".work"
|
||
self.env = compose_env(profile)
|
||
|
||
def compose(self, *args: str, check: bool = True) -> str:
|
||
"""docker compose стенда; без `-p mtusi-bench` не вызывается."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
PROJECT,
|
||
"--env-file",
|
||
str(ENV_FILE),
|
||
*files,
|
||
*args,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"docker compose {' '.join(args[:3])}: exit "
|
||
f"{proc.returncode}\n{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
def check_free(self) -> None:
|
||
found = {cmd: names for cmd, names in leftovers().items() if names}
|
||
if found:
|
||
fix = "; ".join(
|
||
f"docker {cmd} {' '.join(names)}"
|
||
for cmd, names in found.items()
|
||
)
|
||
raise BenchError(
|
||
"bench stand leftovers exist: another bench is running or a "
|
||
f"crashed one left them; remove with: {fix}",
|
||
)
|
||
|
||
def remove_leftovers(self, say: Callable[[str], None]) -> None:
|
||
"""Серии трека 1: блокировка взята, другого прогона нет, остатки
|
||
устарели — удаляются с печатью каждого объекта (решение 5 design
|
||
change `scaling-bench`). Прочие профили по-прежнему отказывают."""
|
||
found = leftovers()
|
||
for name in found["rm -f"]:
|
||
say(f"removing stale bench container {name}")
|
||
sh("docker", "rm", "-f", name, check=False)
|
||
for name in found["volume rm"]:
|
||
say(f"removing stale bench volume {name}")
|
||
sh("docker", "volume", "rm", name, check=False)
|
||
for name in found["network rm"]:
|
||
say(f"removing stale bench network {name}")
|
||
sh("docker", "network", "rm", name, check=False)
|
||
self.check_free()
|
||
|
||
def build_image(self) -> str:
|
||
sh(
|
||
"docker",
|
||
"build",
|
||
"-q",
|
||
"-t",
|
||
APP_IMAGE,
|
||
str(ROOT),
|
||
)
|
||
return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE)
|
||
|
||
def build_from(self, commit: str) -> str:
|
||
"""Образ прежнего commit тем же `Dockerfile` из файлов commit
|
||
(`git archive`, `.dockerignore` commit действует): рабочее дерево,
|
||
индекс и HEAD не меняются (решение 6 design change
|
||
`rolling-update`). Возвращает ID образа `FROM_IMAGE`."""
|
||
archive = subprocess.Popen(
|
||
["git", "-C", str(ROOT), "archive", "--format=tar", commit],
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
)
|
||
build = subprocess.run(
|
||
["docker", "build", "-q", "-t", FROM_IMAGE, "-"],
|
||
stdin=archive.stdout,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if archive.stdout is not None:
|
||
archive.stdout.close()
|
||
archive.wait()
|
||
if archive.returncode != 0 or build.returncode != 0:
|
||
raise BenchError(
|
||
f"image of commit {commit} not built: "
|
||
f"{(build.stdout + build.stderr)[-1500:]}",
|
||
)
|
||
return sh("docker", "image", "inspect", "-f", "{{.Id}}", FROM_IMAGE)
|
||
|
||
def script_env(self) -> dict[str, str]:
|
||
"""Окружение `scripts/rolling-update.sh`: тот же проект compose, что
|
||
у `compose()` — `-p`, оба `-f` и `--env-file` переменными docker
|
||
compose; пароли и `BENCH_*` — из окружения стенда."""
|
||
return {
|
||
**self.env,
|
||
"COMPOSE_PROJECT_NAME": PROJECT,
|
||
"COMPOSE_FILE": os.pathsep.join(str(p) for p in COMPOSE_FILES),
|
||
"COMPOSE_ENV_FILES": str(ENV_FILE),
|
||
"API_REPLICAS": str(self.profile.replicas),
|
||
}
|
||
|
||
def project_events(self, path: Path) -> subprocess.Popen[bytes]:
|
||
"""Поток событий контейнеров проекта (часы ВМ) в файл `path`:
|
||
создание, старт, сигнал, выход, удаление; формат —
|
||
`rollout.parse_container_events`.
|
||
|
||
Поток, а не `docker events --since` после прогона: демон хранит
|
||
только последние 256 событий, а `docker exec` скрипта замены и
|
||
оркестратора (по три события на вызов) вытесняют их за минуту
|
||
(самопроверка: событий миграции и старых реплик уже не было)."""
|
||
args = [
|
||
"docker",
|
||
"events",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
"--filter",
|
||
"type=container",
|
||
]
|
||
# exec_start — начало подготовки замены: первое обращение скрипта к
|
||
# LB (design change profile-resources, решение 4); exec проверок
|
||
# здоровья контейнеров отсекает rollout.script_start
|
||
for action in ("create", "start", "kill", "die", "destroy"):
|
||
args += ["--filter", f"event={action}"]
|
||
args += ["--filter", "event=exec_start"]
|
||
args += [
|
||
"--format",
|
||
"{{.TimeNano}}|{{.Action}}|{{.Actor.ID}}|"
|
||
'{{index .Actor.Attributes "name"}}|'
|
||
'{{index .Actor.Attributes "signal"}}',
|
||
]
|
||
with path.open("w") as out:
|
||
return subprocess.Popen(
|
||
args,
|
||
stdout=out,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
|
||
def container_events(
|
||
self,
|
||
container: str,
|
||
path: Path,
|
||
) -> subprocess.Popen[bytes]:
|
||
"""Поток событий Docker одного контейнера (часы ВМ) в файл `path`;
|
||
формат — `failover.parse_events`.
|
||
|
||
Поток, запущенный до нагрузки фазы, а не `docker events --since`
|
||
после неё: демон хранит только последние 256 событий всех
|
||
контейнеров, а проверки здоровья и `docker exec` оркестратора (по
|
||
три события на вызов) вытесняют их (design change
|
||
`fix-known-issues`, решение 9)."""
|
||
with path.open("w") as out:
|
||
return subprocess.Popen(
|
||
[
|
||
"docker",
|
||
"events",
|
||
"--filter",
|
||
f"container={container}",
|
||
"--format",
|
||
"{{.TimeNano}} {{.Action}} "
|
||
'{{index .Actor.Attributes "signal"}}',
|
||
],
|
||
stdout=out,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
|
||
def app_containers(self) -> list[dict[str, Any]]:
|
||
"""Реплики `api` (и остановленные): имя, `X-Instance-Id`, тег и
|
||
digest конфигурации образа. В хранилище containerd у тега и у
|
||
контейнера разные digest (индекс и конфигурация), поэтому
|
||
сравниваются контейнеры между собой, а не с тегом."""
|
||
rows = []
|
||
for name in self.replicas(stopped=True):
|
||
full, tag, image, running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Id}}|{{.Config.Image}}|{{.Image}}|{{.State.Running}}",
|
||
name,
|
||
).split("|")
|
||
rows.append(
|
||
{
|
||
"name": name,
|
||
"instance": full[:12],
|
||
"tag": tag,
|
||
"image": image,
|
||
"running": running == "true",
|
||
},
|
||
)
|
||
return rows
|
||
|
||
def start(self, say: Callable[[str], None]) -> None:
|
||
"""Свежий стенд: пустая база на новом volume, миграции, реплики API
|
||
и LB, k6."""
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
self.compose("up", "-d", "--wait", "postgres", "valkey")
|
||
say("postgres and valkey healthy")
|
||
self.compose("run", "--rm", "migrate")
|
||
say("migrations applied")
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-build",
|
||
APP_SERVICE,
|
||
"haproxy",
|
||
)
|
||
# генератор серий трека 1 — с лимитом без swap (решение 6 design
|
||
# change scaling-bench); у прочих профилей — без лимита
|
||
limits = (
|
||
[
|
||
"--cpus",
|
||
self.profile.generator_cpus,
|
||
"--memory",
|
||
self.profile.generator_memory,
|
||
"--memory-swap",
|
||
self.profile.generator_memory,
|
||
]
|
||
if self.profile.generator_cpus
|
||
else []
|
||
)
|
||
sh(
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
K6,
|
||
*limits,
|
||
"--network",
|
||
NETWORK,
|
||
"-v",
|
||
f"{ROOT / 'bench' / 'k6'}:/scripts:ro",
|
||
"-v",
|
||
f"{self.label_dir}:/out",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"sleep",
|
||
K6_IMAGE,
|
||
"infinity",
|
||
)
|
||
self.wait_ready()
|
||
say(f"{self.profile.replicas} api replicas UP behind haproxy")
|
||
|
||
def replicas(self, *, stopped: bool = False) -> list[str]:
|
||
"""Имена контейнеров реплик api по порядку; `stopped` — и
|
||
остановленные."""
|
||
args = ["ps", "--format", "{{.Name}}"]
|
||
if stopped:
|
||
args.append("-a")
|
||
names = self.compose(*args, APP_SERVICE).split()
|
||
return sorted(names, key=lambda name: int(name.rsplit("-", 1)[1]))
|
||
|
||
def address(self, name: str) -> str:
|
||
"""Адрес контейнера в сети стенда (его получает слот LB из DNS)."""
|
||
return sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
|
||
name,
|
||
)
|
||
|
||
def lb_slots(self) -> list[dict[str, str]]:
|
||
"""Занятые слоты backend `api` HAProxy: слот, адрес, состояние."""
|
||
csv_text = sh("docker", "exec", LB, "bash", "-c", LB_STATS)
|
||
return failover.parse_lb_stats(csv_text)
|
||
|
||
def wait_ready(self, timeout: float = 60) -> None:
|
||
"""Все `profile.replicas` реплик работают, и LB видит их `UP`.
|
||
|
||
Готовность — по проверкам LB (`/health/ready` дважды подряд), а не
|
||
запросом изнутри реплики: k6 ходит через LB. Слот считается по
|
||
адресу реплики: после пересоздания реплик старые адреса LB ещё
|
||
некоторое время держит.
|
||
"""
|
||
deadline = time.monotonic() + timeout
|
||
expected = self.profile.replicas
|
||
while time.monotonic() < deadline:
|
||
for name in self.replicas(stopped=True):
|
||
state = self.state(name)
|
||
if not state["running"]:
|
||
reason = "oom" if state["oom"] else "exited"
|
||
raise BenchError(f"{name} died during start ({reason})")
|
||
running = self.replicas()
|
||
if len(running) == expected:
|
||
addresses = {self.address(name) for name in running}
|
||
up = {
|
||
slot["addr"].rsplit(":", 1)[0]
|
||
for slot in self.lb_slots()
|
||
if slot["status"] == "UP"
|
||
}
|
||
if addresses <= up:
|
||
return
|
||
time.sleep(0.25)
|
||
raise BenchError(
|
||
f"{expected} api replicas not UP behind haproxy in {timeout:.0f}s",
|
||
)
|
||
|
||
def stop(self) -> bool:
|
||
"""k6, контейнеры, сеть и volume стенда; True — ничего не осталось.
|
||
|
||
k6 и хранилище восстановления — первыми: подключённый к сети
|
||
контейнер не из compose не дал бы compose удалить сеть. Volume
|
||
восстановления compose не знает, он удаляется отдельно.
|
||
"""
|
||
subprocess.run(
|
||
["docker", "rm", "-f", K6, SAMPLER, RESTORE_DB],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
self.compose("down", "-v", "--remove-orphans", check=False)
|
||
subprocess.run(
|
||
["docker", "volume", "rm", RESTORE_VOLUME],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
# тег образа прежнего commit (профиль замены); сам образ остаётся
|
||
# в кэше сборки
|
||
subprocess.run(
|
||
["docker", "image", "rm", FROM_IMAGE],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
return not any(leftovers().values())
|
||
|
||
def start_restore(self) -> None:
|
||
"""Новое хранилище: контейнер PostgreSQL на новом volume.
|
||
|
||
Образ — тот же, что у исходного контейнера (пин без копии в коде),
|
||
лимиты — как у Postgres стенда, пароль — `--env` без значения из
|
||
окружения процесса. Метка проекта — чтобы `leftovers()` видел
|
||
остатки (решение 3 design change `track2-backup-restore`).
|
||
"""
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
env = dict(
|
||
item.split("=", 1)
|
||
for item in json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", DB),
|
||
)
|
||
)
|
||
label = f"com.docker.compose.project={PROJECT}"
|
||
sh("docker", "volume", "create", "--label", label, RESTORE_VOLUME)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
RESTORE_DB,
|
||
"--label",
|
||
label,
|
||
"--network",
|
||
NETWORK,
|
||
"--network-alias",
|
||
RESTORE_HOST,
|
||
"--env",
|
||
"POSTGRES_PASSWORD",
|
||
"-e",
|
||
f"POSTGRES_USER={env['POSTGRES_USER']}",
|
||
"-e",
|
||
f"POSTGRES_DB={env['POSTGRES_DB']}",
|
||
"--cpus",
|
||
self.profile.db_cpus,
|
||
"--memory",
|
||
self.profile.db_memory,
|
||
"--memory-swap",
|
||
self.profile.db_memory,
|
||
"-v",
|
||
f"{RESTORE_VOLUME}:/var/lib/postgresql",
|
||
image,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"restore storage did not start: {proc.stderr}")
|
||
# по TCP, как healthcheck compose: при инициализации образ
|
||
# поднимает временный сервер только на сокете
|
||
deadline = time.monotonic() + 60
|
||
while time.monotonic() < deadline:
|
||
ready = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
RESTORE_DB,
|
||
"sh",
|
||
"-c",
|
||
'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" '
|
||
'-d "$POSTGRES_DB"',
|
||
],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if ready.returncode == 0:
|
||
return
|
||
time.sleep(0.25)
|
||
raise BenchError(f"{RESTORE_DB} not ready in 60s")
|
||
|
||
def limits(self) -> dict[str, Any]:
|
||
"""Лимиты контейнеров стенда, прочитанные у запущенных контейнеров.
|
||
|
||
`api` — первая реплика (у всех одно определение сервиса),
|
||
`api_replicas` — сколько их; лимиты различаются — отказ.
|
||
"""
|
||
replicas = self.replicas()
|
||
roles = {APP_SERVICE: replicas[0], **CONTAINERS, "k6": K6}
|
||
limits: dict[str, Any] = {}
|
||
for role, name in roles.items():
|
||
limits[role] = self.container_limits(name)
|
||
for name in replicas[1:]:
|
||
if self.container_limits(name) != limits[APP_SERVICE]:
|
||
raise BenchError(f"{name}: limits differ from {replicas[0]}")
|
||
limits["api_replicas"] = len(replicas)
|
||
# предел соединений — параметр конфигурации серии (решение 1
|
||
# design change scaling-bench), у прочих профилей 100
|
||
limits["max_connections"] = int(self.psql("SHOW max_connections;\n"))
|
||
if limits["max_connections"] != self.profile.db_max_connections:
|
||
raise BenchError(
|
||
f"PostgreSQL max_connections {limits['max_connections']}, "
|
||
f"expected {self.profile.db_max_connections}",
|
||
)
|
||
return limits
|
||
|
||
def start_sampler(self, *, scan: bool = False) -> None:
|
||
"""Сэмплер ячейки серии (решение 7 design change `scaling-bench`):
|
||
образ PostgreSQL стенда, `--cgroupns host`, свои лимиты; за 5 с
|
||
должны появиться строки каждого контейнера, ВМ, LB и PostgreSQL.
|
||
|
||
`scan` — профили уровня «4» (решения 1, 2 design change
|
||
`profile-resources`): режим обхода cgroup, контейнер без сети, без
|
||
метки проекта и без паролей; за 5 с — строки работающих
|
||
контейнеров проекта, k6, самого сэмплера и ВМ.
|
||
"""
|
||
if scan:
|
||
self.start_scan_sampler()
|
||
return
|
||
p = self.profile
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
env = dict(
|
||
item.split("=", 1)
|
||
for item in json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", DB),
|
||
)
|
||
)
|
||
targets = [
|
||
f"{APP_SERVICE}={self.full_id(name)}" for name in self.replicas()
|
||
]
|
||
targets += [
|
||
f"{role}={self.full_id(name)}"
|
||
for role, name in (*CONTAINERS.items(), ("k6", K6))
|
||
]
|
||
for name in ("samples.log", "pg-samples.log"):
|
||
(self.work_dir / name).unlink(missing_ok=True)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--label",
|
||
f"com.docker.compose.project={PROJECT}",
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
NETWORK,
|
||
"--cpus",
|
||
p.sampler_cpus,
|
||
"--memory",
|
||
p.sampler_memory,
|
||
"--memory-swap",
|
||
p.sampler_memory,
|
||
"--env",
|
||
"POSTGRES_PASSWORD",
|
||
"-e",
|
||
f"POSTGRES_USER={env['POSTGRES_USER']}",
|
||
"-e",
|
||
f"POSTGRES_DB={env['POSTGRES_DB']}",
|
||
"-v",
|
||
f"{SAMPLER_SCRIPT}:/sampler.sh:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"bash",
|
||
image,
|
||
"/sampler.sh",
|
||
f"{p.sample_interval_s:g}",
|
||
*targets,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"sampler did not start: {proc.stderr}")
|
||
expected = {f"cg {t.split('=')[0]}" for t in targets}
|
||
deadline = time.monotonic() + 5
|
||
problem = "no samples"
|
||
while time.monotonic() < deadline:
|
||
time.sleep(0.5)
|
||
samples = scaling.parse_samples(*self.sampler_logs())
|
||
if samples.missing:
|
||
problem = f"missing: {', '.join(samples.missing)}"
|
||
break
|
||
ids = {ident for _, ident in samples.cg}
|
||
want = {t.split("=")[1][:12] for t in targets}
|
||
if (
|
||
want <= ids
|
||
and samples.vm
|
||
and samples.lb
|
||
and samples.db
|
||
and samples.wal
|
||
):
|
||
return
|
||
problem = (
|
||
f"cgroups {len(ids)}/{len(want)}, vm {len(samples.vm)}, lb "
|
||
f"{len(samples.lb)}, postgres {len(samples.db)}; expected "
|
||
f"{sorted(expected)}"
|
||
)
|
||
tail = sh("docker", "logs", "--tail", "5", SAMPLER, check=False)
|
||
raise BenchError(f"sampler: {problem}\n{tail}")
|
||
|
||
def start_scan_sampler(self) -> None:
|
||
"""Сэмплер в режиме `scan`: `--network none` — к приложению,
|
||
хранилищам и LB он обратиться не может; без `--label` проекта —
|
||
иначе `docker compose up` скрипта замены видел бы его сиротой
|
||
проекта; без `POSTGRES_*` — пароль ему не нужен. `leftovers()` и
|
||
`stop()` находят его по имени."""
|
||
p = self.profile
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
(self.work_dir / "samples.log").unlink(missing_ok=True)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
"none",
|
||
"--cpus",
|
||
p.sampler_cpus,
|
||
"--memory",
|
||
p.sampler_memory,
|
||
"--memory-swap",
|
||
p.sampler_memory,
|
||
"-v",
|
||
f"{SAMPLER_SCRIPT}:/sampler.sh:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"bash",
|
||
image,
|
||
"/sampler.sh",
|
||
f"{p.sample_interval_s:g}",
|
||
"scan",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"sampler did not start: {proc.stderr}")
|
||
want = {
|
||
ident[:12]
|
||
for ident in sh(
|
||
"docker",
|
||
"ps",
|
||
"-q",
|
||
"--no-trunc",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
).split()
|
||
}
|
||
want |= {self.full_id(name)[:12] for name in (K6, SAMPLER)}
|
||
deadline = time.monotonic() + 5
|
||
problem = "no samples"
|
||
while time.monotonic() < deadline:
|
||
time.sleep(0.5)
|
||
samples = scaling.parse_samples(self.sampler_logs()[0])
|
||
ids = {ident for _, ident in samples.cg}
|
||
if want <= ids and samples.vm:
|
||
return
|
||
problem = (
|
||
f"cgroups seen {len(want & ids)}/{len(want)} (missing "
|
||
f"{sorted(want - ids)}), vm {len(samples.vm)}"
|
||
)
|
||
tail = sh("docker", "logs", "--tail", "5", SAMPLER, check=False)
|
||
raise BenchError(f"sampler: {problem}\n{tail}")
|
||
|
||
def container_map(self) -> dict[str, dict[str, str]]:
|
||
"""id12 → имя и роль контейнера (решение 1 design change
|
||
`profile-resources`): контейнеры проекта (и удалённые не
|
||
попадут — их добавляет поток событий) по метке сервиса compose,
|
||
k6 и сэмплер — по имени."""
|
||
found: dict[str, dict[str, str]] = {}
|
||
listing = sh(
|
||
"docker",
|
||
"ps",
|
||
"-a",
|
||
"--no-trunc",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
"--format",
|
||
'{{.ID}}|{{.Names}}|{{.Label "com.docker.compose.service"}}',
|
||
)
|
||
for line in listing.splitlines():
|
||
ident, name, service = (line.split("|") + ["", ""])[:3]
|
||
if ident:
|
||
found[ident[:12]] = {"name": name, "role": service}
|
||
for name, role in ((K6, "k6"), (SAMPLER, "sampler")):
|
||
ident = sh("docker", "inspect", "-f", "{{.Id}}", name, check=False)
|
||
if re.fullmatch(r"[0-9a-f]{64}", ident):
|
||
found[ident[:12]] = {"name": name, "role": role}
|
||
return found
|
||
|
||
def healthcheck_actions(self) -> set[str]:
|
||
"""Действия `exec_start` проверок здоровья контейнеров проекта, как
|
||
их пишет поток событий: их не путать с обращением скрипта замены к
|
||
стенду (найдено в apply change `profile-resources`)."""
|
||
actions = set()
|
||
names = sh(
|
||
"docker",
|
||
"ps",
|
||
"-q",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
).split()
|
||
for name in names:
|
||
test = json.loads(
|
||
sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{json .Config.Healthcheck}}",
|
||
name,
|
||
check=False,
|
||
)
|
||
or "null",
|
||
)
|
||
command = (test or {}).get("Test") or []
|
||
if command[:1] == ["CMD-SHELL"]:
|
||
actions.add(f"exec_start: /bin/sh -c {' '.join(command[1:])}")
|
||
elif command[:1] == ["CMD"]:
|
||
actions.add(f"exec_start: {' '.join(command[1:])}")
|
||
return actions
|
||
|
||
def sampler_logs(self) -> tuple[str, str]:
|
||
texts = []
|
||
for name in ("samples.log", "pg-samples.log"):
|
||
path = self.work_dir / name
|
||
texts.append(
|
||
path.read_text(encoding="utf-8", errors="replace")
|
||
if path.exists()
|
||
else "",
|
||
)
|
||
return texts[0], texts[1]
|
||
|
||
def stop_sampler(self) -> None:
|
||
subprocess.run(
|
||
["docker", "rm", "-f", SAMPLER],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
def full_id(self, name: str) -> str:
|
||
return sh("docker", "inspect", "-f", "{{.Id}}", name)
|
||
|
||
def container_limits(self, name: str) -> dict[str, Any]:
|
||
host = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .HostConfig}}", name),
|
||
)
|
||
config = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config}}", name),
|
||
)
|
||
return {
|
||
"nano_cpus": host.get("NanoCpus"),
|
||
"memory": host.get("Memory"),
|
||
"memory_swap": host.get("MemorySwap"),
|
||
"published_ports": host.get("PortBindings") or {},
|
||
"healthcheck": (config.get("Healthcheck") or {}).get("Test"),
|
||
"log_driver": (host.get("LogConfig") or {}).get("Type"),
|
||
}
|
||
|
||
def app_state(self) -> dict[str, Any]:
|
||
"""Состояние приложения в целом: все реплики работают, ни одна не
|
||
убита по памяти и не перезапускалась."""
|
||
names = self.replicas(stopped=True)
|
||
states = [self.state(name) for name in names]
|
||
return {
|
||
"running": len(states) == self.profile.replicas
|
||
and all(state["running"] for state in states),
|
||
"oom": any(state["oom"] for state in states),
|
||
"restarts": sum(state["restarts"] for state in states),
|
||
}
|
||
|
||
def state(self, name: str) -> dict[str, Any]:
|
||
state = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .State}}", name),
|
||
)
|
||
restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", name)
|
||
return {
|
||
"running": state.get("Running"),
|
||
"oom": state.get("OOMKilled"),
|
||
"restarts": int(restarts or 0),
|
||
}
|
||
|
||
def down_storages(self) -> list[str]:
|
||
"""Хранилища, которые за прогон умирали, убивались или рестартовали."""
|
||
down = []
|
||
for name in (DB, CACHE):
|
||
state = self.state(name)
|
||
if not state["running"] or state["oom"] or state["restarts"]:
|
||
down.append(name)
|
||
return down
|
||
|
||
def psql(
|
||
self,
|
||
sql: str,
|
||
*,
|
||
check: bool = True,
|
||
container: str = DB,
|
||
) -> str:
|
||
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля.
|
||
|
||
`container` — исходное хранилище или хранилище восстановления.
|
||
"""
|
||
proc = subprocess.run(
|
||
["docker", "exec", "-i", container, "sh", "-c", PSQL],
|
||
input=sql,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(f"psql: exit {proc.returncode}\n{proc.stderr}")
|
||
return proc.stdout.strip()
|
||
|
||
def maintain(self) -> float:
|
||
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды.
|
||
|
||
Статистика и состояние таблиц одинаковы у всех меток, а не
|
||
зависят от того, когда сработал autovacuum (решение 4).
|
||
"""
|
||
started = time.monotonic()
|
||
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
|
||
return time.monotonic() - started
|
||
|
||
def storage(self) -> dict[str, Any]:
|
||
"""Объём таблицы сообщений и её индексов, число строк."""
|
||
row = self.psql(
|
||
"SELECT pg_table_size('messages'), pg_indexes_size('messages'), "
|
||
"count(*) FROM messages;\n",
|
||
)
|
||
table, indexes, count = (int(value) for value in row.split("|"))
|
||
return {
|
||
"messages_rows": count,
|
||
"messages_table_bytes": table,
|
||
"messages_indexes_bytes": indexes,
|
||
"db_bytes_per_message": (
|
||
round((table + indexes) / count) if count else None
|
||
),
|
||
}
|
||
|
||
def chat_counts(self) -> dict[str, int]:
|
||
"""Число сообщений по чатам (проверка распределения наполнения)."""
|
||
rows = self.psql(
|
||
"SELECT chat_id, count(*) FROM messages GROUP BY chat_id;\n",
|
||
)
|
||
return {
|
||
chat: int(count)
|
||
for chat, count in (line.split("|") for line in rows.splitlines())
|
||
}
|
||
|
||
def indexes(self, container: str = DB) -> list[str]:
|
||
"""Определения индексов messages: до и после этапов с вариантами."""
|
||
return self.psql(
|
||
"SELECT indexdef FROM pg_indexes WHERE tablename = 'messages' "
|
||
"ORDER BY indexname;\n",
|
||
container=container,
|
||
).splitlines()
|
||
|
||
def message_count(self) -> int:
|
||
return int(self.psql("SELECT count(*) FROM messages;\n"))
|
||
|
||
def write_probe(
|
||
self,
|
||
app_index: str,
|
||
columns: str | None,
|
||
) -> dict[str, Any]:
|
||
"""Одна проба записи при индексе истории `columns` (None — без него).
|
||
|
||
Всё в транзакции, которая откатывается: индекс приложения
|
||
удаляется, конфигурация строится заново, `CHECKPOINT`, вставка
|
||
порции во все чаты в пропорции наполнения и сообщение максимальной
|
||
длины (решение 5 design change `track2-query-plans`).
|
||
"""
|
||
p = self.profile
|
||
create = (
|
||
f"CREATE INDEX bench_probe ON messages {columns};\n"
|
||
if columns
|
||
else ""
|
||
)
|
||
size = "pg_relation_size('bench_probe')" if columns else "0"
|
||
proc = subprocess.run(
|
||
["docker", "exec", "-i", DB, "sh", "-c", PSQL],
|
||
input=(
|
||
"SET lock_timeout = '5s';\n"
|
||
"BEGIN;\n"
|
||
"CREATE TEMP TABLE probe_src ON COMMIT DROP AS "
|
||
"SELECT chat_id, sender_id, "
|
||
"row_number() OVER (ORDER BY created_at, id) AS g "
|
||
"FROM (SELECT chat_id, sender_id, created_at, id FROM messages "
|
||
f"ORDER BY created_at, id LIMIT {p.write_probe_rows}) s;\n"
|
||
f'DROP INDEX "{app_index}";\n'
|
||
f"{create}"
|
||
f"SELECT {size}, (SELECT count(*) FROM messages), "
|
||
"(SELECT count(*) FROM probe_src);\n"
|
||
"CHECKPOINT;\n"
|
||
"EXPLAIN (ANALYZE, BUFFERS, WAL) INSERT INTO messages "
|
||
"(id, chat_id, sender_id, text, created_at) "
|
||
"SELECT gen_random_uuid(), chat_id, sender_id, "
|
||
f"rpad('probe-' || g, {p.text_length}, 'x'), "
|
||
"now() + g * interval '1 microsecond' FROM probe_src;\n"
|
||
f"{LONG_TEXT_PROBE}"
|
||
"ROLLBACK;\n"
|
||
),
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"write probe: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
head, _, plan = proc.stdout.strip().partition("\n")
|
||
index_bytes, rows_before, rows = (int(v) for v in head.split("|"))
|
||
wal = re.search(r"WAL: ([^\n]*)", plan)
|
||
|
||
def wal_field(name: str) -> int:
|
||
# EXPLAIN не печатает нулевые поля WAL
|
||
match = (
|
||
re.search(rf"\b{name}=(\d+)", wal.group(1)) if wal else None
|
||
)
|
||
return int(match.group(1)) if match else 0
|
||
|
||
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
|
||
long_text = re.search(r"long-text: ([^\n]*)", proc.stderr)
|
||
if execution is None or long_text is None:
|
||
raise BenchError(f"write probe: unexpected output\n{proc.stdout}")
|
||
return {
|
||
"index_bytes": index_bytes,
|
||
"rows_before": rows_before,
|
||
"rows": rows,
|
||
"time_ms": float(execution.group(1)),
|
||
"wal_records": wal_field("records"),
|
||
"wal_fpi": wal_field("fpi"),
|
||
"wal_bytes": wal_field("bytes"),
|
||
"long_text": long_text.group(1).strip(),
|
||
}
|
||
|
||
def db_log(self) -> str:
|
||
"""Журнал Postgres стенда (stderr контейнера): только в памяти."""
|
||
proc = subprocess.run(
|
||
["docker", "logs", DB],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def passport(self) -> dict[str, Any]:
|
||
"""Хранилища и окружение API для паспорта; паролей здесь нет."""
|
||
settings = self.psql(
|
||
"SELECT name, current_setting(name) FROM pg_settings "
|
||
"WHERE name = ANY(string_to_array("
|
||
f"'{','.join(PG_SETTINGS)}', ',')) ORDER BY name;\n",
|
||
)
|
||
info = sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" INFO server',
|
||
)
|
||
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.M)
|
||
memory = sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" CONFIG GET "maxmemory*"',
|
||
).splitlines()
|
||
env = json.loads(
|
||
sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{json .Config.Env}}",
|
||
self.replicas()[0],
|
||
),
|
||
)
|
||
# без секретов: пароли хранилищ, секрет и ключ доступа S3
|
||
app_env = dict(
|
||
sorted(
|
||
item.split("=", 1)
|
||
for item in env
|
||
if re.match(r"[A-Z0-9_]+__[A-Z0-9_]+=", item)
|
||
and not any(
|
||
mark in item.split("=", 1)[0] for mark in SECRET_ENV_MARKS
|
||
)
|
||
),
|
||
)
|
||
|
||
def image(name: str) -> dict[str, str]:
|
||
return {
|
||
"image": sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Config.Image}}",
|
||
name,
|
||
),
|
||
"id": sh("docker", "inspect", "-f", "{{.Image}}", name),
|
||
}
|
||
|
||
return {
|
||
"postgres": {
|
||
**image(DB),
|
||
"version": self.psql("SHOW server_version;\n"),
|
||
"settings": dict(
|
||
line.split("|", 1) for line in settings.splitlines()
|
||
),
|
||
},
|
||
"valkey": {
|
||
**image(CACHE),
|
||
"version": valkey_version.group(1) if valkey_version else None,
|
||
**dict(zip(memory[::2], memory[1::2], strict=True)),
|
||
},
|
||
# конфигурация LB — infra/haproxy/haproxy.cfg рабочего стенда
|
||
"haproxy": {
|
||
**image(LB),
|
||
"version": sh("docker", "exec", LB, "haproxy", "-v")
|
||
.splitlines()[0]
|
||
.strip(),
|
||
},
|
||
"app_env": app_env,
|
||
}
|
||
|
||
def snapshot(self) -> dict[str, Any]:
|
||
"""CPU и память приложения, CPU PostgreSQL и генератора.
|
||
|
||
Приложение — сумма по репликам (`VmHWM` — максимум): при одной
|
||
реплике — числа её процесса, как до LB. Если реплика уже мертва
|
||
(OOM), поля приложения — None: снимок нужен и для записи
|
||
проваленного прогона.
|
||
"""
|
||
replicas = [
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
name,
|
||
"sh",
|
||
"-c",
|
||
"cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; "
|
||
"grep -E '^(VmRSS|VmHWM):' /proc/1/status",
|
||
check=False,
|
||
)
|
||
for name in self.replicas(stopped=True)
|
||
]
|
||
k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat")
|
||
db = sh(
|
||
"docker",
|
||
"exec",
|
||
DB,
|
||
"cat",
|
||
"/sys/fs/cgroup/cpu.stat",
|
||
check=False,
|
||
)
|
||
|
||
def number(pattern: str, text: str) -> int | None:
|
||
match = re.search(pattern, text, re.MULTILINE)
|
||
return int(match.group(1)) if match else None
|
||
|
||
def total(pattern: str) -> int | None:
|
||
values = [number(pattern, app) for app in replicas]
|
||
if not values or None in values:
|
||
return None
|
||
return sum(v for v in values if v is not None)
|
||
|
||
hwm = [number(r"^VmHWM:\s+(\d+) kB", app) for app in replicas]
|
||
return {
|
||
"app_usec": total(r"^usage_usec (\d+)$"),
|
||
"mem_bytes": total(r"^(\d+)$"),
|
||
"rss_kib": total(r"^VmRSS:\s+(\d+) kB"),
|
||
"hwm_kib": (
|
||
None if not hwm or None in hwm else max(v or 0 for v in hwm)
|
||
),
|
||
"k6_usec": number(r"^usage_usec (\d+)$", k6),
|
||
"db_usec": number(r"^usage_usec (\d+)$", db),
|
||
}
|
||
|
||
def k6_args(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
*extra: str,
|
||
) -> list[str]:
|
||
"""Команда `k6 run` сценария в контейнере k6 стенда; `extra` —
|
||
дополнительные флаги k6."""
|
||
common = {
|
||
"TEXT_LENGTH": self.profile.text_length,
|
||
"PAGE_LIMIT": self.profile.page_limit,
|
||
}
|
||
flags = [
|
||
item
|
||
for key, value in {**common, **env}.items()
|
||
for item in ("-e", f"{key}={value}")
|
||
]
|
||
return [
|
||
"docker",
|
||
"exec",
|
||
K6,
|
||
"k6",
|
||
"run",
|
||
"--quiet",
|
||
"--no-color",
|
||
"--log-format",
|
||
"raw",
|
||
*extra,
|
||
*flags,
|
||
f"/scripts/{script}",
|
||
]
|
||
|
||
def k6(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
*,
|
||
check: bool = True,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
proc = subprocess.run(
|
||
self.k6_args(script, env),
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(k6_failure(script, proc))
|
||
return proc
|
||
|
||
def k6_start(self, script: str, env: dict[str, Any]) -> subprocess.Popen:
|
||
"""Сценарий k6 в фоне: запись во время backup."""
|
||
return subprocess.Popen(
|
||
self.k6_args(script, env),
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
)
|
||
|
||
def k6_console(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
name: str,
|
||
) -> subprocess.Popen:
|
||
"""Сценарий k6 в фоне; `console.log` k6 пишет сам в файл `name`
|
||
рабочего каталога, прочий вывод — в `<name>.out`.
|
||
|
||
Строка на каждый запрос фазы отказа переполнила бы трубу, пока
|
||
оркестратор ждёт момента действия. А через вывод `docker exec`
|
||
в самопроверке пропали две строки из 4 601 (вероятно, склеились
|
||
строки разных VU), поэтому файл пишет k6, а не docker.
|
||
"""
|
||
with (self.work_dir / f"{name}.out").open("w") as out:
|
||
return subprocess.Popen(
|
||
self.k6_args(
|
||
script,
|
||
env,
|
||
"--console-output",
|
||
f"/work/{name}",
|
||
),
|
||
stdout=out,
|
||
stderr=subprocess.STDOUT,
|
||
)
|
||
|
||
def pg_dump(self, path: Path) -> None:
|
||
"""Логический backup исходной базы (формат custom) в файл хоста."""
|
||
with path.open("wb") as file:
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
DB,
|
||
"sh",
|
||
"-c",
|
||
'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc',
|
||
],
|
||
stdout=file,
|
||
stderr=subprocess.PIPE,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"pg_dump: exit {proc.returncode}\n{proc.stderr}")
|
||
|
||
def pg_restore(self, path: Path) -> tuple[float, float]:
|
||
"""Backup в хранилище восстановления: копия файла и pg_restore -j.
|
||
|
||
Возвращает секунды копирования и восстановления.
|
||
"""
|
||
started = time.monotonic()
|
||
sh("docker", "cp", str(path), f"{RESTORE_DB}:/tmp/backup.dump")
|
||
copied = time.monotonic()
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
RESTORE_DB,
|
||
"sh",
|
||
"-c",
|
||
'exec pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
||
f"-j {self.profile.restore_jobs} --no-owner --no-privileges "
|
||
"--exit-on-error /tmp/backup.dump",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"pg_restore: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
return copied - started, time.monotonic() - copied
|
||
|
||
def switch_api(self, host: str) -> None:
|
||
"""Реплики API заново, с адресом базы `host`: `POSTGRES_HOST`
|
||
перекрывает значение из `bench/bench.env` для `x-db-env`.
|
||
|
||
Готовы — когда процесс каждой новой реплики сам отвечает на
|
||
`/health/ready`, и только затем LB видит их `UP`. Пересозданный
|
||
контейнер получает тот же IP, а LB ещё держит слот этого адреса
|
||
`UP` от прежнего: `wait_ready` сразу после `up` вернулся бы до
|
||
старта процесса, и первые запросы получали бы 503 `<NOSRV>`, когда
|
||
LB поймает отказ соединения (design change `attachments`,
|
||
«Уточнения apply», п. 9). После готовности процесса проверки LB
|
||
успешны: слот, помеченный `DOWN` до неё, `wait_ready` дождётся
|
||
снова `UP`, а не помеченный — уже ведёт на готовый процесс.
|
||
"""
|
||
env = self.env
|
||
self.env = {**env, "POSTGRES_HOST": host}
|
||
try:
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--force-recreate",
|
||
APP_SERVICE,
|
||
)
|
||
finally:
|
||
self.env = env
|
||
self.wait_processes_ready()
|
||
self.wait_ready()
|
||
|
||
def wait_processes_ready(self, timeout: float = 60) -> None:
|
||
"""Процесс каждой реплики отвечает на свой `/health/ready` изнутри
|
||
контейнера — та же команда, что healthcheck образа (у стенда он
|
||
выключен). Только docker CLI, без сети хоста."""
|
||
deadline = time.monotonic() + timeout
|
||
pending = set(self.replicas())
|
||
while pending and time.monotonic() < deadline:
|
||
for name in sorted(pending):
|
||
state = self.state(name)
|
||
if not state["running"]:
|
||
reason = "oom" if state["oom"] else "exited"
|
||
raise BenchError(f"{name} died during start ({reason})")
|
||
probe = subprocess.run(
|
||
["docker", "exec", name, "python", "-c", APP_READY_PROBE],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if probe.returncode == 0:
|
||
pending.discard(name)
|
||
if pending:
|
||
time.sleep(0.25)
|
||
if pending:
|
||
raise BenchError(
|
||
f"api replicas {sorted(pending)} not ready in {timeout:.0f}s",
|
||
)
|
||
|
||
def logs(self, container: str, since: float) -> str:
|
||
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
|
||
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
|
||
proc = subprocess.run(
|
||
["docker", "logs", "-t", "--since", f"{since:.3f}", container],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def table_counts(self, container: str = DB) -> dict[str, int]:
|
||
"""Число строк таблиц данных (сверка восстановления)."""
|
||
sql = " UNION ALL ".join(
|
||
f"SELECT '{table}', count(*) FROM {table}" for table in DATA_TABLES
|
||
)
|
||
rows = self.psql(f"{sql};\n", container=container)
|
||
return {
|
||
table: int(count)
|
||
for table, count in (line.split("|") for line in rows.splitlines())
|
||
}
|
||
|
||
def message_rows(self, container: str = DB) -> dict[str, tuple[str, str]]:
|
||
"""Сообщения базы: id → (chat_id, md5 всех полей)."""
|
||
rows = self.psql(
|
||
"SELECT id, chat_id, md5(concat_ws('|', chat_id, sender_id, "
|
||
"text, created_at)) FROM messages;\n",
|
||
container=container,
|
||
)
|
||
return {
|
||
message: (chat, digest)
|
||
for message, chat, digest in (
|
||
line.split("|") for line in rows.splitlines()
|
||
)
|
||
}
|
||
|
||
def work_json(self, name: str) -> Any:
|
||
return json.loads((self.work_dir / name).read_text(encoding="utf-8"))
|
||
|
||
|
||
# -------------------------------------------------------------- измерение
|
||
|
||
|
||
def parse_summary(path: Path) -> dict[str, Any]:
|
||
"""Итог k6: успехи, ошибки, RPS и перцентили успешных запросов.
|
||
|
||
Нет файла (k6 остановлен сторожем до итога) — нулевой итог.
|
||
"""
|
||
if not path.exists():
|
||
return {
|
||
"duration_s": 0,
|
||
"requests_ok": 0,
|
||
"requests_failed": 0,
|
||
"check_failed": 0,
|
||
"rps": None,
|
||
"error_ratio": None,
|
||
"p50_ms": None,
|
||
"p95_ms": None,
|
||
"p99_ms": None,
|
||
"max_ms": None,
|
||
}
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
metrics = data["metrics"]
|
||
failed = metrics.get("http_req_failed", {}).get("values", {})
|
||
ok = int(failed.get("fails", 0))
|
||
bad = int(failed.get("passes", 0))
|
||
checks = metrics.get("checks", {}).get("values", {})
|
||
success = metrics.get(SUCCESS_METRIC, {}).get("values", {})
|
||
duration = data["state"]["testRunDurationMs"] / 1000
|
||
return {
|
||
"duration_s": round(duration, 3),
|
||
"requests_ok": ok,
|
||
"requests_failed": bad,
|
||
"check_failed": int(checks.get("fails", 0)),
|
||
"rps": round(ok / duration, 1) if duration else None,
|
||
"error_ratio": round(bad / (ok + bad), 6) if ok + bad else None,
|
||
"p50_ms": rounded(success.get("p(50)")),
|
||
"p95_ms": rounded(success.get("p(95)")),
|
||
"p99_ms": rounded(success.get("p(99)")),
|
||
"max_ms": rounded(success.get("max")),
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class Measured:
|
||
summary: dict[str, Any]
|
||
before: dict[str, Any]
|
||
after: dict[str, Any]
|
||
rss_max_kib: int | None
|
||
|
||
def cores(self, key: str) -> float | None:
|
||
"""Среднее число занятых ядер за прогон по счётчику cgroup."""
|
||
duration = self.summary["duration_s"]
|
||
start, end = self.before[key], self.after[key]
|
||
if not duration or start is None or end is None:
|
||
return None
|
||
return round((end - start) / 1e6 / duration, 3)
|
||
|
||
@property
|
||
def app_cpu(self) -> float | None:
|
||
return self.cores("app_usec")
|
||
|
||
@property
|
||
def generator_cpu(self) -> float | None:
|
||
return self.cores("k6_usec")
|
||
|
||
@property
|
||
def db_cpu(self) -> float | None:
|
||
return self.cores("db_usec")
|
||
|
||
|
||
def shorten(line: str) -> str:
|
||
"""Строка журнала в файл ошибок: длинная — начало, «…» и конец."""
|
||
if len(line) <= ERRORS_HEAD + ERRORS_TAIL:
|
||
return line
|
||
return f"{line[:ERRORS_HEAD]} … {line[-ERRORS_TAIL:]}"
|
||
|
||
|
||
def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str:
|
||
tail = (proc.stdout + proc.stderr)[-3000:]
|
||
return f"k6 {script} failed ({proc.returncode}):\n{tail}"
|
||
|
||
|
||
def measure(
|
||
stand: Stand,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
raw: Path,
|
||
) -> Measured:
|
||
"""Один запуск k6 со снимками до и после, опросом RSS и сторожем."""
|
||
before = stand.snapshot()
|
||
with Watcher(stand.replicas()) as watcher:
|
||
proc = stand.k6(script, env, check=False)
|
||
# если приложение или хранилище умерло, k6 остановлен сторожем: прогон
|
||
# пишется с пометкой, а не падает здесь
|
||
died = watcher.app_died or watcher.stand_died
|
||
if proc.returncode != 0 and not died:
|
||
raise BenchError(k6_failure(script, proc))
|
||
after = stand.snapshot()
|
||
values = [
|
||
v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v
|
||
]
|
||
return Measured(
|
||
summary=parse_summary(raw),
|
||
before=before,
|
||
after=after,
|
||
rss_max_kib=max(values) if values else None,
|
||
)
|
||
|
||
|
||
def run_flags(
|
||
profile: Profile,
|
||
measured: Measured,
|
||
app: dict[str, Any],
|
||
storages_down: list[str],
|
||
) -> list[str]:
|
||
flags = []
|
||
if app["oom"]:
|
||
flags.append("oom")
|
||
if not app["running"] or app["restarts"]:
|
||
flags.append("app_down")
|
||
if storages_down:
|
||
flags.append("stand_down")
|
||
summary = measured.summary
|
||
if summary["requests_failed"] or summary["check_failed"]:
|
||
flags.append("errors")
|
||
cpu = measured.generator_cpu
|
||
if cpu is not None and cpu >= profile.generator_cpu_limit:
|
||
flags.append("generator")
|
||
return flags
|
||
|
||
|
||
# ----------------------------------------------------------- план запроса
|
||
|
||
|
||
def plan_entries(log: str, marker: str) -> list[str]:
|
||
"""Записи auto_explain между маркерами этапа плана.
|
||
|
||
Запись начинается строкой `LOG: duration: … plan:` и продолжается
|
||
строками с табуляцией; префикс времени и pid отбрасывается.
|
||
"""
|
||
lines = log.splitlines()
|
||
bounds = [
|
||
next(
|
||
(i for i, line in enumerate(lines) if f"{marker} {edge}" in line),
|
||
None,
|
||
)
|
||
for edge in ("start", "end")
|
||
]
|
||
if None in bounds:
|
||
return []
|
||
entries: list[list[str]] = []
|
||
current: list[str] | None = None
|
||
for line in lines[bounds[0] + 1 : bounds[1]]:
|
||
if line.startswith("\t"):
|
||
if current is not None:
|
||
current.append(line[1:])
|
||
continue
|
||
current = None
|
||
head = line.partition("LOG: ")[2].rstrip()
|
||
if head.startswith("duration:") and head.endswith("plan:"):
|
||
current = [head]
|
||
entries.append(current)
|
||
return ["\n".join(entry) for entry in entries]
|
||
|
||
|
||
def is_history(entry: str) -> bool:
|
||
"""Запись — запрос страницы истории (читает таблицу messages).
|
||
|
||
Остальные записи окна (проверка сессии с хэшем токена в параметрах,
|
||
служебные запросы) в результаты не попадают.
|
||
"""
|
||
text = entry.partition("Query Parameters:")[0]
|
||
return "Query Text:" in text and "FROM messages" in text
|
||
|
||
|
||
def plan_parts(entry: str) -> dict[str, Any] | None:
|
||
"""Текст запроса, параметры и тело плана записи auto_explain."""
|
||
lines = entry.splitlines()
|
||
starts = [i for i, line in enumerate(lines) if line.startswith("Query ")]
|
||
if len(starts) != 2 or not lines[starts[1]].startswith("Query Param"):
|
||
return None
|
||
text = "\n".join(lines[starts[0] : starts[1]])
|
||
params = lines[starts[1]].partition(": ")[2]
|
||
return {
|
||
"text": text.removeprefix("Query Text: "),
|
||
"params": [
|
||
value.replace("''", "'")
|
||
for value in re.findall(r"\$\d+ = '((?:[^']|'')*)'", params)
|
||
],
|
||
"plan": "\n".join(lines[starts[1] + 1 :]),
|
||
}
|
||
|
||
|
||
def plan_summary(plan: str, time_ms: float | None) -> dict[str, Any]:
|
||
"""Узел доступа к messages, буферы верхнего узла, строки, время."""
|
||
node = re.search(
|
||
r"((?:Parallel )?(?:Seq Scan|Index Only Scan|Index Scan|"
|
||
r"Bitmap Heap Scan)(?: Backward)?(?: using \w+)? on messages)",
|
||
plan,
|
||
)
|
||
buffers = re.search(r"Buffers: ([^\n]*)", plan)
|
||
hit = re.search(r"shared hit=(\d+)", buffers.group(1)) if buffers else None
|
||
read = re.search(r"read=(\d+)", buffers.group(1)) if buffers else None
|
||
rows = re.search(r"\(actual time=[\d.]+\.\.[\d.]+ rows=([\d.]+)", plan)
|
||
if time_ms is None:
|
||
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
|
||
time_ms = float(execution.group(1)) if execution else None
|
||
return {
|
||
"node": node.group(1) if node else None,
|
||
"shared_hit": int(hit.group(1)) if hit else 0,
|
||
"shared_read": int(read.group(1)) if read else 0,
|
||
"rows": float(rows.group(1)) if rows else None,
|
||
"time_ms": time_ms,
|
||
# параметры в условиях плана — generic plan, литералы — custom
|
||
"kind": "generic" if re.search(r"\$\d", plan) else "custom",
|
||
}
|
||
|
||
|
||
def entry_duration(entry: str) -> float | None:
|
||
match = re.match(r"duration: ([\d.]+) ms", entry)
|
||
return float(match.group(1)) if match else None
|
||
|
||
|
||
def index_name(plan: str) -> str | None:
|
||
match = INDEX_NODE_RE.search(plan)
|
||
return (
|
||
next((name for name in match.groups() if name), None)
|
||
if match
|
||
else None
|
||
)
|
||
|
||
|
||
def render_explain(
|
||
size: int,
|
||
git: dict[str, Any],
|
||
profile: Profile,
|
||
pages: list[dict[str, Any]],
|
||
entries: list[str],
|
||
compared: list[tuple[dict[str, Any], dict[str, str]]],
|
||
) -> str:
|
||
"""Файл `explain-<размер>.txt`: планы приложения и варианты psql."""
|
||
tiers = ", ".join(
|
||
f"{count} × {human(chat_size)}"
|
||
for count, chat_size in profile.background
|
||
)
|
||
lines = [
|
||
f"# План запроса истории: размер {human(size)}",
|
||
"",
|
||
f"Commit {git['commit']}"
|
||
+ (" + незакоммиченные изменения" if git["dirty"] else ""),
|
||
f"Читаемый чат — {size} сообщений; фоновые чаты: {tiers or 'нет'}.",
|
||
"Страница — `limit` профиля; «после» — сколько сообщений чата до",
|
||
"страницы.",
|
||
"",
|
||
"## Планы приложения",
|
||
"",
|
||
"auto_explain (ANALYZE, BUFFERS, TIMING): запрос, параметры и план,",
|
||
"с которыми его выполнило приложение (prepared statement asyncpg).",
|
||
"",
|
||
]
|
||
for page, entry in zip(pages, entries, strict=True):
|
||
lines += [
|
||
f"### {page['chat']}: {page['page']}, после {page['after']}",
|
||
"",
|
||
entry,
|
||
"",
|
||
]
|
||
variants = ", ".join(
|
||
f"`{name}` {columns}" for name, columns in profile.index_variants
|
||
)
|
||
lines += [
|
||
"## Страницы сравнения в psql: варианты",
|
||
"",
|
||
"Тот же текст запроса и те же параметры, что у плана приложения этой",
|
||
"страницы, в одной транзакции, которая откатывается: индексы базы",
|
||
"после неё те же. Каждый вариант выполнен дважды, здесь — второе",
|
||
"выполнение. `psql` — индекс приложения, custom plan; `generic` — он",
|
||
"же с force_generic_plan; `or-cursor` — условие курсора через OR;",
|
||
"`no-index` — без индекса приложения; варианты индекса вместо него:",
|
||
f"{variants}.",
|
||
"",
|
||
]
|
||
for page, plans in compared:
|
||
lines += [
|
||
f"### {page['chat']}: {page['page']}, после {page['after']}",
|
||
"",
|
||
]
|
||
if not plans:
|
||
lines += [
|
||
"План приложения не использовал индекс: вариантов нет.",
|
||
"",
|
||
]
|
||
for name, plan in plans.items():
|
||
lines += [f"#### {name}", "", plan, ""]
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ------------------------------------------------------------ результаты
|
||
|
||
|
||
class Results:
|
||
"""Файлы метки: runs.csv, prefill.csv, meta.json, progress.json; у
|
||
профиля отказа — ещё phases.csv и timeline.csv."""
|
||
|
||
def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None:
|
||
self.dir = label_dir
|
||
self.meta = meta
|
||
self.runs: list[dict[str, Any]] = []
|
||
self.prefill: list[dict[str, Any]] = []
|
||
self.sizes: list[dict[str, Any]] = []
|
||
self.phases: list[dict[str, Any]] = []
|
||
self.timeline: list[dict[str, Any]] = []
|
||
self.rollout: dict[str, Any] | None = None
|
||
self.replaced: list[dict[str, Any]] = []
|
||
# профили уровня «4»: строки resources.csv для сводки (design
|
||
# change profile-resources, решение 7)
|
||
self.resources: list[dict[str, Any]] = []
|
||
# продолжение серии дописывает в существующий каталог метки
|
||
(label_dir / "raw").mkdir(parents=True, exist_ok=True)
|
||
self.write_meta()
|
||
|
||
def add_step(self, row: dict[str, Any]) -> None:
|
||
"""Ступень серии трека 1 — строка `runs.csv`."""
|
||
self.append("runs.csv", RUN_FIELDS, row)
|
||
|
||
def add_search(self, row: dict[str, Any]) -> None:
|
||
"""Ячейка серии — строка `searches.csv`: она же признак готовой
|
||
ячейки при продолжении."""
|
||
self.append("searches.csv", scaling.SEARCH_FIELDS, row)
|
||
|
||
def add_samples(self, rows: list[dict[str, Any]]) -> None:
|
||
"""Посекундные отсчёты окна замера — `samples.csv`."""
|
||
path = self.dir / "samples.csv"
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=scaling.SAMPLE_FIELDS,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
for row in rows:
|
||
writer.writerow(
|
||
{key: row.get(key) for key in scaling.SAMPLE_FIELDS},
|
||
)
|
||
|
||
def add_resources(self, result: resources.Result) -> None:
|
||
"""Ресурсы фазы или замены: `resources.csv` (по окнам) и
|
||
`samples.csv` (посекундно) профилей уровня «4»."""
|
||
self.resources += result.rows
|
||
for name, fields, rows in (
|
||
("resources.csv", resources.RESOURCE_FIELDS, result.rows),
|
||
("samples.csv", resources.SAMPLE_FIELDS, result.samples),
|
||
):
|
||
path = self.dir / name
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=fields,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
for row in rows:
|
||
writer.writerow({key: row.get(key) for key in fields})
|
||
|
||
def append(
|
||
self,
|
||
name: str,
|
||
fields: list[str],
|
||
row: dict[str, Any],
|
||
) -> None:
|
||
path = self.dir / name
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=fields,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
writer.writerow({key: row.get(key) for key in fields})
|
||
file.flush()
|
||
|
||
def add_run(self, row: dict[str, Any]) -> None:
|
||
self.runs.append(row)
|
||
self.append("runs.csv", RUN_FIELDS, row)
|
||
|
||
def add_prefill(self, row: dict[str, Any]) -> None:
|
||
self.prefill.append(row)
|
||
self.append("prefill.csv", PREFILL_FIELDS, row)
|
||
|
||
def add_phase(
|
||
self,
|
||
row: dict[str, Any],
|
||
timeline: list[dict[str, Any]],
|
||
) -> None:
|
||
self.phases.append(row)
|
||
self.append("phases.csv", failover.PHASE_FIELDS, row)
|
||
for second in timeline:
|
||
self.timeline.append(second)
|
||
self.append("timeline.csv", failover.TIMELINE_FIELDS, second)
|
||
|
||
def add_rollout(
|
||
self,
|
||
row: dict[str, Any],
|
||
replaced: list[dict[str, Any]],
|
||
timeline: list[dict[str, Any]],
|
||
) -> None:
|
||
"""Профиль замены: строка прогона, шаги по репликам, шкала."""
|
||
self.rollout = row
|
||
self.replaced = replaced
|
||
self.timeline = timeline
|
||
self.append("rollout.csv", rollout.ROLLOUT_FIELDS, row)
|
||
for step in replaced:
|
||
self.append("replicas.csv", rollout.REPLICA_FIELDS, step)
|
||
for second in timeline:
|
||
self.append("timeline.csv", rollout.TIMELINE_FIELDS, second)
|
||
|
||
def write_meta(self) -> None:
|
||
self.meta["sizes"] = self.sizes
|
||
text = json.dumps(self.meta, ensure_ascii=False, indent=2)
|
||
(self.dir / "meta.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_progress(self, progress: dict[str, Any]) -> None:
|
||
progress["updated_at"] = datetime.now(UTC).isoformat(
|
||
timespec="seconds",
|
||
)
|
||
text = json.dumps(progress, ensure_ascii=False, indent=2)
|
||
(self.dir / "progress.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_summary(self) -> None:
|
||
if self.meta["profile"].get("series"):
|
||
# серия: сводка и графики из таблиц метки — в них и строки
|
||
# прежних запусков при продолжении (решения 8, 13)
|
||
text = scaling.render_summary(
|
||
self.meta,
|
||
scaling.read_csv(self.dir / "runs.csv"),
|
||
scaling.read_csv(self.dir / "searches.csv"),
|
||
)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
plots.render_series(self.dir)
|
||
return
|
||
if self.meta["profile"].get("rollout_from"):
|
||
text = rollout.render_summary(
|
||
self.meta,
|
||
self.rollout,
|
||
self.replaced,
|
||
self.timeline,
|
||
self.resources or None,
|
||
)
|
||
elif self.meta["profile"].get("failover_modes"):
|
||
text = failover.render_summary(
|
||
self.meta,
|
||
self.phases,
|
||
self.timeline,
|
||
self.resources or None,
|
||
)
|
||
else:
|
||
text = build_summary(
|
||
self.meta,
|
||
self.runs,
|
||
self.prefill,
|
||
self.sizes,
|
||
)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
|
||
|
||
def cell(values: list[float | None], digits: int = 1) -> str:
|
||
"""Медиана и min–max повторов; один повтор — просто значение."""
|
||
present = [v for v in values if v is not None]
|
||
if not present:
|
||
return "—"
|
||
median = statistics.median(present)
|
||
if len(present) == 1:
|
||
return f"{median:.{digits}f}"
|
||
return (
|
||
f"{median:.{digits}f} ({min(present):.{digits}f}–"
|
||
f"{max(present):.{digits}f})"
|
||
)
|
||
|
||
|
||
def build_summary(
|
||
meta: dict[str, Any],
|
||
runs: list[dict[str, Any]],
|
||
prefill: list[dict[str, Any]],
|
||
sizes: list[dict[str, Any]],
|
||
) -> str:
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Нагрузочный замер: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{meta['status']}**"
|
||
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
|
||
f"- Commit: `{meta['git']['commit']}`"
|
||
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
|
||
f"- Профиль `{profile['name']}`: размеры "
|
||
f"{', '.join(human(s) for s in profile['sizes'])}; клиенты "
|
||
f"{', '.join(map(str, profile['concurrency']))}; повторов "
|
||
f"{profile['repeats']}; чтение {profile['read_measure_s']} с после "
|
||
f"{profile['read_warmup_s']} с прогрева; запись "
|
||
f"{profile['write_measure']} после {profile['write_warmup']}",
|
||
f"- Лимиты без swap: API {profile['cpus']} CPU / "
|
||
f"{profile['memory']} × {profile.get('replicas', 1)} реплик за "
|
||
f"HAProxy {profile.get('lb_cpus', '—')} CPU / "
|
||
f"{profile.get('lb_memory', '—')}, PostgreSQL "
|
||
f"{profile['db_cpus']} CPU / {profile['db_memory']}, Valkey "
|
||
f"{profile['cache_cpus']} CPU / {profile['cache_memory']}; CPU и "
|
||
"RSS API — сумма по репликам; значения — медиана (min–max) по "
|
||
"повторам",
|
||
"- Посторонние контейнеры в VM во время замера: "
|
||
+ (", ".join(meta.get("other_containers") or []) or "нет"),
|
||
"",
|
||
]
|
||
for scenario in ("read", "write"):
|
||
rows = [r for r in runs if r["scenario"] == scenario]
|
||
if not rows:
|
||
continue
|
||
lines += [
|
||
f"## Сценарий `{scenario}`",
|
||
"",
|
||
"| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | "
|
||
"max, мс | CPU API, ядер | CPU БД, ядер | RSS конец, МиБ | "
|
||
"Сообщений в начале | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
cells = sorted({(r["size"], r["concurrency"]) for r in rows})
|
||
for size, conc in cells:
|
||
group = [
|
||
r
|
||
for r in rows
|
||
if r["size"] == size and r["concurrency"] == conc
|
||
]
|
||
flags = sorted({f for r in group for f in r["flags"].split()})
|
||
lines.append(
|
||
f"| {human(size)} | {conc} | "
|
||
f"{cell([r['rps'] for r in group], 0)} | "
|
||
f"{cell([r['p50_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p95_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p99_ms'] for r in group], 2)} | "
|
||
f"{cell([r['max_ms'] for r in group], 1)} | "
|
||
f"{cell([r['app_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['db_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['rss_end_mib'] for r in group], 1)} | "
|
||
f"{cell([r['store_messages_start'] for r in group], 0)} | "
|
||
f"{' '.join(flags) or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if sizes:
|
||
lines += [
|
||
"## Память и наполнение по размерам",
|
||
"",
|
||
"Объём `messages` — после `VACUUM (ANALYZE)`, до прогонов: в "
|
||
"таблице N сообщений читаемого чата и сообщения фоновых чатов "
|
||
"(`Сообщений`).",
|
||
"",
|
||
"| Размер | Сообщений | RSS API до, МиБ | RSS API после, МиБ | "
|
||
"Байт RSS на сообщение | Таблица, МиБ | Индексы, МиБ | "
|
||
"Байт в БД на сообщение | Наполнение, сообщ/с | "
|
||
"Курсоров в пуле | Пик RSS за размер, МиБ |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
lines.append(
|
||
f"| {human(info['size'])} | "
|
||
f"{info.get('messages_total', info['size'])} | "
|
||
f"{info['rss_before_mib']} | "
|
||
f"{info['rss_after_mib']} | {info['bytes_per_message']} | "
|
||
f"{mib_bytes(info.get('messages_table_bytes'))} | "
|
||
f"{mib_bytes(info.get('messages_indexes_bytes'))} | "
|
||
f"{info.get('db_bytes_per_message', '—')} | "
|
||
f"{info['prefill_rate']} | {info['cursors']} | "
|
||
f"{info.get('rss_peak_mib', '—')} |",
|
||
)
|
||
lines.append("")
|
||
lines += [
|
||
"## Планы запроса истории",
|
||
"",
|
||
"Страница (`limit` профиля) читаемого чата и первого чата "
|
||
"каждого фонового уровня. `app` — план запроса приложения "
|
||
"(auto_explain). Остальное — тот же запрос с теми же "
|
||
"параметрами в psql, в откатываемой транзакции: `psql` — "
|
||
"индекс приложения, `generic` — он же с generic plan, "
|
||
"`or-cursor` — условие курсора через OR, `no-index` — без "
|
||
"индекса, дальше — варианты индекса профиля. Буферы — верхнего "
|
||
"узла плана, файлы — `explain-<размер>.txt`.",
|
||
"",
|
||
"| Размер | Чат | Страница | После сообщений | Вариант | Узел | "
|
||
"shared hit | read | Строк | Время, мс | План |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
for plan in info.get("plans", []):
|
||
lines.append(
|
||
f"| {human(info['size'])} | {plan.get('chat', 'read')} | "
|
||
f"{plan['page']} | {plan['after']} | {plan['source']} | "
|
||
f"{plan['node'] or '—'} | {plan['shared_hit']} | "
|
||
f"{plan['shared_read']} | {fmt(plan['rows'])} | "
|
||
f"{fmt(plan['time_ms'], 3)} | {plan['kind']} |",
|
||
)
|
||
lines.append("")
|
||
costs = [
|
||
(info, row) for info in sizes for row in info.get("write_cost", [])
|
||
]
|
||
if costs:
|
||
lines += [
|
||
"## Стоимость записи по вариантам индекса",
|
||
"",
|
||
"Вставка порции сообщений во все чаты в пропорции наполнения "
|
||
"одной командой, в транзакции, которая откатывается; индекс "
|
||
"истории построен заново, перед вставкой `CHECKPOINT`. "
|
||
"Значения — медиана (min–max) повторов. «4000 символов» — "
|
||
"вставка сообщения максимальной длины из несжимаемого текста.",
|
||
"",
|
||
"| Размер | Вариант | Строк вставки | Индекс, МиБ | "
|
||
"Байт индекса на сообщение | мкс на сообщение | "
|
||
"WAL байт на сообщение | FPI на 1000 | 4000 символов |",
|
||
"|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info, row in costs:
|
||
lines.append(
|
||
f"| {human(info['size'])} | {row['variant']} | "
|
||
f"{row['rows']} | {mib_bytes(row['index_bytes'])} | "
|
||
f"{row['index_bytes_per_message']} | "
|
||
f"{cell(row['insert_us_per_message'], 2)} | "
|
||
f"{cell(row['wal_bytes_per_message'], 0)} | "
|
||
f"{cell(row['fpi_per_1000'], 1)} | {row['long_text']} |",
|
||
)
|
||
lines.append("")
|
||
restores = [info for info in sizes if info.get("restore")]
|
||
if restores:
|
||
lines += [
|
||
"## Backup и восстановление",
|
||
"",
|
||
"`pg_dump -Fc` исходной базы во время записи через HTTP, "
|
||
"затем активность после backup (пользователь, сессия, чат, "
|
||
"сообщение), `pg_restore -j` в новый контейнер PostgreSQL "
|
||
"на новом volume, `ANALYZE`, сверка обеих баз (число строк, "
|
||
"md5 каждого сообщения, индексы, ревизия схемы), "
|
||
"переключение API и проверки через HTTP до и после "
|
||
"`FLUSHALL`. RTO — от запуска нового хранилища до "
|
||
"готовности API, без сверки. Сверка прошла на каждом "
|
||
"размере: иначе команда завершилась бы ошибкой.",
|
||
"",
|
||
"| Размер | До backup | Исходная в конце | Восстановлено | "
|
||
"Потеряно | Записей во время backup (в backup) | Backup, МиБ | "
|
||
"Байт backup на сообщение | dump, с | restore, с | "
|
||
"ANALYZE, с | API, с | RTO, с | Сверка, с | Чат после "
|
||
"backup: GET / POST до FLUSHALL, GET после | Сессия после "
|
||
"backup |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"
|
||
"---|---|",
|
||
]
|
||
for info in restores:
|
||
r = info["restore"]
|
||
lines.append(
|
||
f"| {human(info['size'])} | {r['messages_before_backup']} "
|
||
f"| {r['messages_source_end']} | {r['messages_restored']} "
|
||
f"| {r['messages_lost']} | {r['load_written']} "
|
||
f"({r['load_in_backup']}) | "
|
||
f"{mib_bytes(r['backup_bytes'])} | "
|
||
f"{r['backup_bytes_per_message']} | {r['dump_seconds']} | "
|
||
f"{r['restore_seconds']} | {r['analyze_seconds']} | "
|
||
f"{r['api_ready_seconds']} | {r['rto_seconds']} | "
|
||
f"{r['verify_seconds']} | {r['stale_chat_get']} / "
|
||
f"{r['stale_chat_post']}, {r['flushed_chat_get']} | "
|
||
f"{r['late_session']} |",
|
||
)
|
||
lines.append("")
|
||
if prefill:
|
||
largest = max(r["size"] for r in prefill)
|
||
curve = [r for r in prefill if r["size"] == largest]
|
||
lines += [
|
||
f"## Кривая наполнения {human(largest)}: запись при росте "
|
||
f"хранилища, {profile['prefill_vus']} клиента",
|
||
"",
|
||
"| Порция | Сообщений после | RPS | p50, мс | p95, мс | "
|
||
"p99, мс | max, мс | CPU API, ядер | CPU БД, ядер | RSS, МиБ | "
|
||
"Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in curve:
|
||
lines.append(
|
||
f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | "
|
||
f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | "
|
||
f"{r['max_ms']} | {r['app_cpu_cores']} | "
|
||
f"{r['db_cpu_cores']} | {r['rss_end_mib']} "
|
||
f"| {r['flags'] or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if meta.get("host_warnings"):
|
||
lines += ["## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ---------------------------------------------------------------- прогон
|
||
|
||
|
||
class Bench:
|
||
def __init__(
|
||
self,
|
||
profile: Profile,
|
||
label: str,
|
||
results: Results,
|
||
) -> None:
|
||
self.profile = profile
|
||
self.label = label
|
||
self.results = results
|
||
self.stand = Stand(profile, results.dir)
|
||
self.started = time.monotonic()
|
||
self.runs_total = (
|
||
len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats
|
||
)
|
||
if profile.is_failover:
|
||
# прогон профиля отказа — фаза: режим × реплика
|
||
self.runs_total = len(profile.failover_modes) * profile.replicas
|
||
if profile.is_rollout:
|
||
# профиль замены — один прогон: замена всех реплик под нагрузкой
|
||
self.runs_total = 1
|
||
if profile.is_series:
|
||
# серия трека 1 — ячейки: конфигурация × повтор
|
||
self.runs_total = len(profile.configs) * profile.repeats
|
||
self.runs_done = 0
|
||
# серии: ячейка и попытка текущей ступени, длительности ячеек,
|
||
# номер ступени в метке (столбец run)
|
||
self.cell: dict[str, Any] | None = None
|
||
self.cell_index = 0
|
||
self.attempt = 0
|
||
self.cell_seconds: list[float] = []
|
||
self.cell_started = time.monotonic()
|
||
self.step_no = 0
|
||
self.resume = False
|
||
self.stage = "start"
|
||
self.size: int | None = None
|
||
# скорости стенда для оценки оставшегося времени
|
||
self.prefill_rate = 1500.0
|
||
self.write_rps: dict[int, float] = {}
|
||
self.run_overhead = 3.0
|
||
self.done_sizes = 0
|
||
self.prefilled = 0
|
||
# текущий размер наполнен и курсоры собраны
|
||
self.prepared = False
|
||
# индекс плана приложения: от него варианты и пробы записи
|
||
self.app_index: str | None = None
|
||
# ресурсы профилей уровня «4» (design change profile-resources):
|
||
# контейнеры (id12 → имя и роль сервиса), моменты запуска (id12 →
|
||
# секунды часов ВМ) и ID вне стенда за весь прогон
|
||
self.containers: dict[str, dict[str, str]] = {}
|
||
self.starts: dict[str, list[float]] = {}
|
||
self.frozen: dict[str, list[tuple[float, float]]] = {}
|
||
self.unmapped: set[str] = set()
|
||
|
||
# ------------------------------------------------------------ вывод
|
||
|
||
def say(self, text: str) -> None:
|
||
elapsed = int(time.monotonic() - self.started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
def eta(self) -> float:
|
||
p = self.profile
|
||
if p.is_series:
|
||
# ячейка ≈ 5–6 мин (решение 5), затем среднее снятых
|
||
per_cell = (
|
||
statistics.mean(self.cell_seconds)
|
||
if self.cell_seconds
|
||
else 330
|
||
)
|
||
current = 0.0
|
||
if self.cell is not None:
|
||
current = min(
|
||
per_cell,
|
||
time.monotonic() - self.cell_started,
|
||
)
|
||
left = (self.runs_total - self.runs_done) * per_cell
|
||
return max(0.0, left - current)
|
||
if p.is_rollout:
|
||
# нагрузка, сбор журналов, повтор и сверка
|
||
left = p.rollout_before_s + p.rollout_budget_s + p.rollout_after_s
|
||
left = 0 if self.runs_done else left + 15
|
||
return left + (0 if self.prepared else 60)
|
||
if p.is_failover:
|
||
# фаза: окна, сбор журналов, ожидание реплик и сверка
|
||
phase = p.failover_before_s + p.failover_down_s
|
||
phase += p.failover_after_s + 15
|
||
left = (self.runs_total - self.runs_done) * phase
|
||
return left + (0 if self.prepared else 60)
|
||
left = 0.0
|
||
for index, size in enumerate(p.sizes):
|
||
current = index == self.done_sizes
|
||
if index < self.done_sizes:
|
||
continue
|
||
# пробы записи в конце размера: сборка индексов и вставки;
|
||
# backup, восстановление и сверка
|
||
left += 60 + (30 if p.restore else 0)
|
||
if current and self.prepared:
|
||
continue
|
||
filled = self.prefilled if current else 0
|
||
total = size + p.background_total
|
||
left += (total - filled) / self.prefill_rate + size / 100 / 500
|
||
# стенд, миграции, обслуживание БД, план и его варианты
|
||
left += 60
|
||
per_size = len(p.concurrency) * p.repeats
|
||
reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes)
|
||
done_in_size = self.runs_done - self.done_sizes * per_size * 2
|
||
reads_left -= min(done_in_size, per_size)
|
||
writes_left -= max(0, done_in_size - per_size)
|
||
read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead
|
||
left += reads_left * read_time
|
||
rps = (
|
||
statistics.mean(self.write_rps.values())
|
||
if self.write_rps
|
||
else 1000
|
||
)
|
||
write_time = (p.write_warmup + p.write_measure) / rps
|
||
left += writes_left * (write_time + self.run_overhead)
|
||
return left
|
||
|
||
def progress(self, last: dict[str, Any] | None = None) -> None:
|
||
state = {
|
||
"label": self.label,
|
||
"profile": self.profile.name,
|
||
"stage": self.stage,
|
||
"size": self.size,
|
||
"runs_done": self.runs_done,
|
||
"runs_total": self.runs_total,
|
||
"elapsed_s": round(time.monotonic() - self.started),
|
||
"eta_s": round(self.eta()),
|
||
"warnings": self.results.meta.get("host_warnings", []),
|
||
}
|
||
if self.profile.is_series:
|
||
state["cell"] = None if self.cell is None else self.cell["cell"]
|
||
state["attempt"] = self.attempt
|
||
state["cells_done"] = self.runs_done
|
||
state["cells_total"] = self.runs_total
|
||
if last is not None:
|
||
state["last_run"] = last
|
||
self.results.write_progress(state)
|
||
|
||
# --------------------------------------------------------- этапы
|
||
|
||
def run(self) -> None:
|
||
if self.profile.is_series:
|
||
self.run_series()
|
||
return
|
||
if self.profile.is_rollout:
|
||
self.run_rollout()
|
||
return
|
||
if self.profile.is_failover:
|
||
self.run_failover()
|
||
return
|
||
for index, size in enumerate(self.profile.sizes, start=1):
|
||
self.size = size
|
||
self.prefilled = 0
|
||
self.prepared = False
|
||
# индекс плана приложения ищется заново на каждом размере
|
||
self.app_index = None
|
||
self.run_size(index, size)
|
||
self.done_sizes += 1
|
||
|
||
def run_size(self, index: int, size: int) -> None:
|
||
stand = self.stand
|
||
name = human(size)
|
||
self.stage = f"size {name}: start"
|
||
self.say(f"size {name} ({index}/{len(self.profile.sizes)}): start")
|
||
stand.start(lambda text: self.say(f"size {name}: {text}"))
|
||
limits = stand.limits()
|
||
self.results.meta.setdefault("limits", limits)
|
||
if limits != self.results.meta["limits"]:
|
||
raise BenchError(f"limits changed between sizes: {limits}")
|
||
if "stand" not in self.results.meta:
|
||
self.results.meta["stand"] = stand.passport()
|
||
self.results.write_meta()
|
||
rss_start = stand.snapshot()["rss_kib"]
|
||
self.say(f"size {name}: app started, RSS {mib(rss_start)} MiB")
|
||
stand.k6(
|
||
"prepare.js",
|
||
{
|
||
"MEMBERS": self.profile.members,
|
||
"BACKGROUND": ",".join(
|
||
f"{count}x{chat_size}"
|
||
for count, chat_size in self.profile.background
|
||
),
|
||
},
|
||
)
|
||
fixture = stand.work_json("fixture.json")
|
||
sizes = chat_sizes(self.profile, size)
|
||
if [size] + [c["size"] for c in fixture["bgChats"]] != sizes:
|
||
raise BenchError(
|
||
f"size {name}: background chats differ from profile",
|
||
)
|
||
total = sum(sizes)
|
||
(stand.work_dir / "schedule.json").write_text(
|
||
json.dumps(fill_schedule(sizes)),
|
||
encoding="utf-8",
|
||
)
|
||
self.say(
|
||
f"size {name}: {len(fixture['bgChats'])} background chats, "
|
||
f"{total} messages to prefill",
|
||
)
|
||
self.progress()
|
||
rss_before = stand.snapshot()["rss_kib"]
|
||
prefill_seconds = self.run_prefill(size, total)
|
||
after = stand.snapshot()
|
||
rss_after = after["rss_kib"]
|
||
self.stage = f"size {name}: maintenance"
|
||
self.progress()
|
||
seconds = stand.maintain()
|
||
storage = stand.storage()
|
||
if storage["messages_rows"] != total:
|
||
raise BenchError(
|
||
f"messages table has {storage['messages_rows']} rows after "
|
||
f"prefill, expected {total}",
|
||
)
|
||
self.check_distribution(size, fixture)
|
||
self.say(
|
||
f"size {name}: VACUUM (ANALYZE) and CHECKPOINT in {seconds:.1f}s; "
|
||
f"messages {mib_bytes(storage['messages_table_bytes'])} MiB + "
|
||
f"indexes {mib_bytes(storage['messages_indexes_bytes'])} MiB, "
|
||
f"{storage['db_bytes_per_message']} B/msg",
|
||
)
|
||
self.stage = f"size {name}: cursors"
|
||
self.progress()
|
||
stand.k6(
|
||
"cursors.js",
|
||
{
|
||
"WALK_LIMIT": self.profile.walk_limit,
|
||
"POOL_SIZE": self.profile.cursor_pool,
|
||
},
|
||
)
|
||
pool = stand.work_json("cursors.json")
|
||
if pool["walked"] != size:
|
||
raise BenchError(
|
||
f"read chat has {pool['walked']} messages, expected {size}",
|
||
)
|
||
self.say(f"size {name}: cursors {len(pool['cursors'])}")
|
||
self.stage = f"size {name}: query plan"
|
||
self.progress()
|
||
plans = self.explain(size, pool, fixture)
|
||
info = {
|
||
"size": size,
|
||
"messages_total": total,
|
||
"rss_before_mib": mib(rss_before),
|
||
"rss_after_mib": mib(rss_after),
|
||
"bytes_per_message": round(
|
||
(rss_after - rss_before) * 1024 / total,
|
||
),
|
||
"prefill_seconds": round(prefill_seconds, 1),
|
||
"prefill_rate": round(total / prefill_seconds),
|
||
"cursors": len(pool["cursors"]),
|
||
"maintenance_seconds": round(seconds, 1),
|
||
**storage,
|
||
"plans": plans,
|
||
}
|
||
self.results.sizes.append(info)
|
||
self.results.write_meta()
|
||
self.prepared = True
|
||
self.say(
|
||
f"size {name}: prefill done, app RSS {info['rss_after_mib']} "
|
||
f"MiB (+{info['bytes_per_message']} B/msg)",
|
||
)
|
||
store = total
|
||
for scenario in ("read", "write"):
|
||
for conc in self.profile.concurrency:
|
||
for repeat in range(1, self.profile.repeats + 1):
|
||
store = self.run_one(scenario, size, conc, repeat, store)
|
||
if scenario == "read":
|
||
self.check_read_chat(size)
|
||
info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"])
|
||
self.stage = f"size {name}: write cost"
|
||
self.progress()
|
||
info["write_cost"] = self.write_cost(size)
|
||
self.results.write_meta()
|
||
if self.profile.restore:
|
||
self.stage = f"size {name}: backup and restore"
|
||
self.progress()
|
||
info["restore"] = self.backup_restore(size, fixture, store)
|
||
self.results.write_meta()
|
||
if not stand.stop():
|
||
raise BenchError(f"stand of size {name} was not removed")
|
||
|
||
def check_read_chat(self, size: int) -> None:
|
||
"""После блока чтения в читаемом чате по-прежнему ровно N.
|
||
|
||
Новое сообщение встало бы в конец истории, поэтому достаточно
|
||
дочитать хвост от последнего курсора пула.
|
||
"""
|
||
self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit})
|
||
tail = self.stand.work_json("tail.json")
|
||
total = tail["from_after"] + tail["tail"]
|
||
if total != size:
|
||
raise BenchError(
|
||
f"read chat has {total} messages after the read block, "
|
||
f"expected {size}",
|
||
)
|
||
self.say(f"size {human(size)}: read chat unchanged, {total} messages")
|
||
|
||
def check_distribution(self, size: int, fixture: dict[str, Any]) -> None:
|
||
"""После наполнения у каждого чата ровно его размер по профилю."""
|
||
expected = {
|
||
fixture["readChatId"]: size,
|
||
fixture["writeChatId"]: 0,
|
||
**{chat["id"]: chat["size"] for chat in fixture["bgChats"]},
|
||
}
|
||
counts = self.stand.chat_counts()
|
||
wrong = [
|
||
f"{counts.get(chat, 0)} instead of {want}"
|
||
for chat, want in expected.items()
|
||
if counts.get(chat, 0) != want
|
||
]
|
||
wrong += [
|
||
f"unknown chat with {counts[c]}"
|
||
for c in counts.keys() - expected.keys()
|
||
]
|
||
if wrong:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages per chat differ from the "
|
||
f"profile in {len(wrong)} chats: {', '.join(wrong[:5])}",
|
||
)
|
||
sizes = sorted(expected.values())
|
||
self.say(
|
||
f"size {human(size)}: {len(expected) - 1} chats with "
|
||
f"{sizes[1]}–{sizes[-1]} messages, distribution matches profile",
|
||
)
|
||
|
||
def plan_pages(
|
||
self,
|
||
size: int,
|
||
pool: dict[str, Any],
|
||
fixture: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Страницы этапа плана: читаемый чат и первый чат каждого уровня.
|
||
|
||
Читаемый — из пула курсоров, фоновые — проходом `plan_cursors.js`
|
||
до включения auto_explain. Первая, средняя и последняя полная
|
||
страница; у чата без полных страниц после первой — только первая.
|
||
Страница сравнения — средняя, иначе первая (решение 3 design
|
||
change `track2-query-plans`).
|
||
"""
|
||
p = self.profile
|
||
representatives = []
|
||
offset = 0
|
||
for count, _ in p.background:
|
||
representatives.append(fixture["bgChats"][offset])
|
||
offset += count
|
||
walks: dict[str, Any] = {}
|
||
if representatives:
|
||
(self.stand.work_dir / "plan_chats.json").write_text(
|
||
json.dumps(representatives),
|
||
encoding="utf-8",
|
||
)
|
||
self.stand.k6("plan_cursors.js", {"WALK_LIMIT": p.walk_limit})
|
||
walks = {
|
||
walk["id"]: walk
|
||
for walk in self.stand.work_json("plan_walks.json")
|
||
}
|
||
chats = [
|
||
(
|
||
f"read {human(size)}",
|
||
fixture["readChatId"],
|
||
size,
|
||
[
|
||
{"after": after, "cursor": cursor}
|
||
for after, cursor in zip(
|
||
pool["afters"],
|
||
pool["cursors"],
|
||
strict=True,
|
||
)
|
||
],
|
||
),
|
||
]
|
||
for chat in representatives:
|
||
walk = walks[chat["id"]]
|
||
if walk["walked"] != chat["size"]:
|
||
raise BenchError(
|
||
f"size {human(size)}: background chat has "
|
||
f"{walk['walked']} messages, expected {chat['size']}",
|
||
)
|
||
chats.append(
|
||
(
|
||
f"bg {human(chat['size'])}",
|
||
chat["id"],
|
||
chat["size"],
|
||
[
|
||
position
|
||
for position in walk["positions"]
|
||
if chat["size"] - position["after"] >= p.page_limit
|
||
],
|
||
),
|
||
)
|
||
pages = []
|
||
for label, chat_id, chat_size, positions in chats:
|
||
common = {
|
||
"chat": label,
|
||
"chat_id": chat_id,
|
||
"chat_size": chat_size,
|
||
}
|
||
pages.append(
|
||
{
|
||
**common,
|
||
"page": "first",
|
||
"after": 0,
|
||
"cursor": None,
|
||
"items": min(p.page_limit, chat_size),
|
||
"compare": not positions,
|
||
},
|
||
)
|
||
if not positions:
|
||
continue
|
||
for page, position in (
|
||
("middle", positions[len(positions) // 2]),
|
||
("last", positions[-1]),
|
||
):
|
||
pages.append(
|
||
{
|
||
**common,
|
||
"page": page,
|
||
"after": position["after"],
|
||
"cursor": position["cursor"],
|
||
"items": p.page_limit,
|
||
"compare": page == "middle",
|
||
},
|
||
)
|
||
return pages
|
||
|
||
def explain(
|
||
self,
|
||
size: int,
|
||
pool: dict[str, Any],
|
||
fixture: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Этап плана: EXPLAIN (ANALYZE, BUFFERS) запросов истории приложения.
|
||
|
||
auto_explain включается между маркерами, `explain.js` читает
|
||
страницы из `plan_pages.json`; планы берутся из журнала Postgres.
|
||
Затем страница сравнения каждого чата в psql: варианты индекса и
|
||
формы запроса в откатываемой транзакции. Решение 5 design change
|
||
`bench-postgres`, решения 3 и 4 design change `track2-query-plans`.
|
||
"""
|
||
stand = self.stand
|
||
pages = self.plan_pages(size, pool, fixture)
|
||
(stand.work_dir / "plan_pages.json").write_text(
|
||
json.dumps(pages),
|
||
encoding="utf-8",
|
||
)
|
||
indexes = stand.indexes()
|
||
marker = f"bench-explain {uuid.uuid4().hex}"
|
||
enable = "".join(
|
||
f"ALTER SYSTEM SET {key} = '{value}';\n"
|
||
for key, value in EXPLAIN_SETTINGS.items()
|
||
)
|
||
disable = "".join(
|
||
f"ALTER SYSTEM RESET {key};\n" for key in EXPLAIN_SETTINGS
|
||
)
|
||
stand.psql(
|
||
f"DO $$ BEGIN RAISE LOG '{marker} start'; END $$;\n"
|
||
f"{enable}SELECT pg_reload_conf();\n",
|
||
)
|
||
time.sleep(RELOAD_PAUSE_S)
|
||
try:
|
||
stand.k6("explain.js", {})
|
||
finally:
|
||
stand.psql(
|
||
f"{disable}SELECT pg_reload_conf();\n"
|
||
f"DO $$ BEGIN RAISE LOG '{marker} end'; END $$;\n",
|
||
)
|
||
# соединения пула API должны получить выключение до прогонов
|
||
time.sleep(RELOAD_PAUSE_S)
|
||
entries = [
|
||
entry
|
||
for entry in plan_entries(stand.db_log(), marker)
|
||
if is_history(entry)
|
||
]
|
||
if len(entries) != len(pages):
|
||
raise BenchError(
|
||
f"size {human(size)}: expected {len(pages)} history "
|
||
f"query plans in the PostgreSQL log, found {len(entries)}",
|
||
)
|
||
plans = []
|
||
compared = []
|
||
for page, entry in zip(pages, entries, strict=True):
|
||
parts = plan_parts(entry) or {"plan": entry}
|
||
where = {
|
||
"chat": page["chat"],
|
||
"chat_size": page["chat_size"],
|
||
"page": page["page"],
|
||
"after": page["after"],
|
||
}
|
||
plans.append(
|
||
{
|
||
**where,
|
||
"source": "app",
|
||
**plan_summary(parts["plan"], entry_duration(entry)),
|
||
},
|
||
)
|
||
if self.app_index is None:
|
||
self.app_index = index_name(parts["plan"])
|
||
if not page["compare"]:
|
||
continue
|
||
variants = self.explain_variants(entry, page["cursor"] is not None)
|
||
compared.append((page, variants))
|
||
for variant, plan in variants.items():
|
||
summary = plan_summary(plan, None)
|
||
plans.append({**where, "source": variant, **summary})
|
||
self.say(
|
||
f"size {human(size)}: {page['chat']} {page['page']} "
|
||
f"{variant}: {summary['node']}, "
|
||
f"hit={summary['shared_hit']} read={summary['shared_read']}"
|
||
f", {summary['time_ms']} ms",
|
||
)
|
||
if stand.indexes() != indexes:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages indexes changed by the plan "
|
||
"stage",
|
||
)
|
||
text = render_explain(
|
||
size,
|
||
self.results.meta["git"],
|
||
self.profile,
|
||
pages,
|
||
entries,
|
||
compared,
|
||
)
|
||
path = self.results.dir / f"explain-{human(size)}.txt"
|
||
path.write_text(text, encoding="utf-8")
|
||
return plans
|
||
|
||
def explain_variants(self, entry: str, has_cursor: bool) -> dict[str, str]:
|
||
"""Тот же запрос и параметры в psql: варианты индекса и запроса.
|
||
|
||
Всё в транзакции, которая откатывается; нагрузки в этот момент нет.
|
||
Каждый вариант выполняется дважды, в результат идёт второе
|
||
выполнение: только что построенный индекс ещё не в shared_buffers.
|
||
План приложения без индекса — вариантов нет (решение 4 design
|
||
change `track2-query-plans`).
|
||
"""
|
||
parts = plan_parts(entry)
|
||
index = index_name(parts["plan"]) if parts else None
|
||
if parts is None or index is None:
|
||
return {}
|
||
args = ", ".join(
|
||
"'" + value.replace("'", "''") + "'" for value in parts["params"]
|
||
)
|
||
|
||
def twice(name: str, statement: str) -> str:
|
||
run = f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE {statement}({args});\n"
|
||
return f"\\echo @@warmup\n{run}\\echo @@{name}\n{run}"
|
||
|
||
script = (
|
||
"SET lock_timeout = '5s';\n"
|
||
"SET plan_cache_mode = force_custom_plan;\n"
|
||
"BEGIN;\n"
|
||
f"PREPARE page AS {parts['text']};\n"
|
||
+ twice("psql", "page")
|
||
+ "SET LOCAL plan_cache_mode = force_generic_plan;\n"
|
||
+ twice("generic", "page")
|
||
+ "SET LOCAL plan_cache_mode = force_custom_plan;\n"
|
||
)
|
||
if has_cursor:
|
||
script += f"PREPARE page_or AS {or_cursor(parts['text'])};\n"
|
||
script += twice("or-cursor", "page_or")
|
||
script += f'DROP INDEX "{index}";\n' + twice("no-index", "page")
|
||
for name, columns in self.profile.index_variants:
|
||
script += (
|
||
f"CREATE INDEX bench_variant ON messages {columns};\n"
|
||
+ twice(name, "page")
|
||
+ "DROP INDEX bench_variant;\n"
|
||
)
|
||
script += "ROLLBACK;\n"
|
||
sections: dict[str, list[str]] = {}
|
||
current: list[str] = []
|
||
for line in self.stand.psql(script).splitlines():
|
||
if line.startswith("@@"):
|
||
current = sections.setdefault(line[2:], [])
|
||
current.clear()
|
||
continue
|
||
current.append(line)
|
||
sections.pop("warmup", None)
|
||
return {
|
||
name: "\n".join(lines).strip() for name, lines in sections.items()
|
||
}
|
||
|
||
def write_cost(self, size: int) -> list[dict[str, Any]]:
|
||
"""Стоимость записи по вариантам индекса истории.
|
||
|
||
Конфигурации `app` (индекс приложения, построенный заново), варианты
|
||
профиля и `none`; повторы со сдвигом порядка, `VACUUM` после каждой
|
||
пробы. Медиана и разброс — в сводке (решение 5 design change
|
||
`track2-query-plans`).
|
||
"""
|
||
stand = self.stand
|
||
p = self.profile
|
||
if self.app_index is None:
|
||
raise BenchError(
|
||
f"size {human(size)}: the app history plan used no index",
|
||
)
|
||
definition = stand.psql(
|
||
f"SELECT pg_get_indexdef('{self.app_index}'::regclass);\n",
|
||
)
|
||
columns = definition.partition(" USING btree ")[2]
|
||
if not columns:
|
||
raise BenchError(f"unexpected index definition: {definition}")
|
||
configs: list[tuple[str, str | None]] = [
|
||
("app", columns),
|
||
*p.index_variants,
|
||
("none", None),
|
||
]
|
||
before = (stand.indexes(), stand.message_count())
|
||
samples: dict[str, list[dict[str, Any]]] = {n: [] for n, _ in configs}
|
||
for repeat in range(p.write_probe_repeats):
|
||
shift = repeat % len(configs)
|
||
for name, config in configs[shift:] + configs[:shift]:
|
||
samples[name].append(stand.write_probe(self.app_index, config))
|
||
stand.psql("VACUUM messages;\n")
|
||
if (stand.indexes(), stand.message_count()) != before:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages indexes or rows changed by the "
|
||
"write probes",
|
||
)
|
||
result = []
|
||
for name, config in configs:
|
||
runs = samples[name]
|
||
index_bytes = int(
|
||
statistics.median(r["index_bytes"] for r in runs),
|
||
)
|
||
row = {
|
||
"variant": name,
|
||
"columns": config,
|
||
"rows": runs[0]["rows"],
|
||
"messages": runs[0]["rows_before"],
|
||
"index_bytes": index_bytes,
|
||
"index_bytes_per_message": round(
|
||
index_bytes / runs[0]["rows_before"],
|
||
1,
|
||
),
|
||
"insert_us_per_message": [
|
||
round(r["time_ms"] * 1000 / r["rows"], 2) for r in runs
|
||
],
|
||
"wal_bytes_per_message": [
|
||
round(r["wal_bytes"] / r["rows"], 1) for r in runs
|
||
],
|
||
"fpi_per_1000": [
|
||
round(r["wal_fpi"] * 1000 / r["rows"], 1) for r in runs
|
||
],
|
||
"long_text": runs[0]["long_text"],
|
||
}
|
||
result.append(row)
|
||
self.say(
|
||
f"size {human(size)}: write cost {name}: "
|
||
f"{cell(row['insert_us_per_message'], 2)} us/msg, "
|
||
f"{cell(row['wal_bytes_per_message'], 0)} WAL B/msg, index "
|
||
f"{row['index_bytes_per_message']} B/msg, 4000 chars: "
|
||
f"{row['long_text']}",
|
||
)
|
||
return result
|
||
|
||
def backup_restore(
|
||
self,
|
||
size: int,
|
||
fixture: dict[str, Any],
|
||
store: int,
|
||
) -> dict[str, Any]:
|
||
"""Backup под записью, восстановление в новое хранилище, сверка.
|
||
|
||
Порядок и проверки — решения 2–5 design change
|
||
`track2-backup-restore`. Расхождение — BenchError с размером;
|
||
хранилище восстановления убирает `Stand.stop()`.
|
||
"""
|
||
stand = self.stand
|
||
p = self.profile
|
||
name = human(size)
|
||
rows_before = stand.message_rows()
|
||
counts_before = stand.table_counts()
|
||
# запись через HTTP во время backup: dump начинается, когда она идёт
|
||
load = stand.k6_start(
|
||
"write.js",
|
||
{
|
||
"VUS": p.restore_write_vus,
|
||
"COUNT": p.restore_write_load,
|
||
"OFFSET": store,
|
||
},
|
||
)
|
||
dump = stand.work_dir / "backup.dump"
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while stand.message_count() <= counts_before["messages"]:
|
||
if load.poll() is not None or time.monotonic() > deadline:
|
||
raise BenchError(
|
||
f"size {name}: write load did not start before backup",
|
||
)
|
||
time.sleep(0.05)
|
||
started = time.monotonic()
|
||
stand.pg_dump(dump)
|
||
dump_seconds = time.monotonic() - started
|
||
at_dump_end = stand.message_count()
|
||
finally:
|
||
out, err = load.communicate()
|
||
if load.returncode != 0:
|
||
raise BenchError(
|
||
k6_failure(
|
||
"write.js",
|
||
subprocess.CompletedProcess([], load.returncode, out, err),
|
||
),
|
||
)
|
||
written = parse_summary(stand.work_dir / "last.json")
|
||
# backup не должен мешать записи: k6 выходит с 0 и при ошибках
|
||
if (
|
||
written["requests_failed"]
|
||
or written["requests_ok"] != p.restore_write_load
|
||
):
|
||
raise BenchError(
|
||
f"size {name}: writes during backup: "
|
||
f"{written['requests_ok']} of {p.restore_write_load} ok, "
|
||
f"{written['requests_failed']} failed",
|
||
)
|
||
dump_bytes = dump.stat().st_size
|
||
counts_loaded = stand.table_counts()
|
||
self.say(
|
||
f"size {name}: backup {mib_bytes(dump_bytes)} MiB in "
|
||
f"{dump_seconds:.2f}s during {written['requests_ok']} writes "
|
||
f"({written['requests_failed']} failed)",
|
||
)
|
||
stand.k6("restore_check.js", {"PHASE": "source"})
|
||
source_check = stand.work_json("check-source.json")
|
||
stand.k6("after_backup.js", {})
|
||
after = stand.work_json("after.json")
|
||
rows_source = stand.message_rows()
|
||
|
||
# восстановление: RTO — от запуска нового хранилища до готовности API
|
||
started = time.monotonic()
|
||
stand.start_restore()
|
||
storage_seconds = time.monotonic() - started
|
||
copy_seconds, restore_seconds = stand.pg_restore(dump)
|
||
mark = time.monotonic()
|
||
stand.psql("ANALYZE;\n", container=RESTORE_DB)
|
||
analyze_seconds = time.monotonic() - mark
|
||
db_seconds = time.monotonic() - started
|
||
|
||
mark = time.monotonic()
|
||
rows_restored = stand.message_rows(container=RESTORE_DB)
|
||
counts_restored = stand.table_counts(container=RESTORE_DB)
|
||
problems = []
|
||
mismatched = sum(
|
||
1
|
||
for message, row in rows_restored.items()
|
||
if rows_source.get(message) != row
|
||
)
|
||
if mismatched:
|
||
problems.append(f"{mismatched} restored messages differ")
|
||
missing_before = len(rows_before.keys() - rows_restored.keys())
|
||
if missing_before:
|
||
problems.append(
|
||
f"{missing_before} messages from before backup lost",
|
||
)
|
||
lost = rows_source.keys() - rows_restored.keys()
|
||
allowed = {fixture["writeChatId"], after["chatId"]}
|
||
foreign = sum(1 for m in lost if rows_source[m][0] not in allowed)
|
||
if foreign:
|
||
problems.append(f"{foreign} lost messages outside written chats")
|
||
for table in DATA_TABLES[:-1]:
|
||
if counts_restored[table] != counts_loaded[table]:
|
||
problems.append(
|
||
f"{table}: {counts_restored[table]} restored, "
|
||
f"{counts_loaded[table]} at backup",
|
||
)
|
||
if stand.indexes(container=RESTORE_DB) != stand.indexes():
|
||
problems.append("messages indexes differ")
|
||
version = "SELECT version_num FROM alembic_version;\n"
|
||
if stand.psql(version, container=RESTORE_DB) != stand.psql(version):
|
||
problems.append("alembic revision differs")
|
||
verify_seconds = time.monotonic() - mark
|
||
|
||
mark = time.monotonic()
|
||
stand.switch_api(RESTORE_HOST)
|
||
api_seconds = time.monotonic() - mark
|
||
stand.k6("restore_check.js", {"PHASE": "stale"})
|
||
stale = stand.work_json("check-stale.json")
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" FLUSHALL',
|
||
)
|
||
stand.k6("restore_check.js", {"PHASE": "restored"})
|
||
restored = stand.work_json("check-restored.json")
|
||
for phase, check in (
|
||
("source", source_check),
|
||
("stale", stale),
|
||
("restored", restored),
|
||
):
|
||
if check["reader_me"] != 200:
|
||
problems.append(
|
||
f"{phase}: reader session {check['reader_me']}",
|
||
)
|
||
if stale["late_me"] != 401:
|
||
problems.append(f"session after backup: {stale['late_me']}")
|
||
if restored["after_chat_get"] != 404:
|
||
problems.append(
|
||
f"chat after backup after FLUSHALL: {restored['after_chat_get']}",
|
||
)
|
||
for phase, check in (("stale", stale), ("restored", restored)):
|
||
differ = [
|
||
f"{a['chat']} {a['page']}"
|
||
for a, b in zip(
|
||
check["pages"],
|
||
source_check["pages"],
|
||
strict=True,
|
||
)
|
||
if a["status"] != 200 or a["items"] != b["items"]
|
||
]
|
||
if differ:
|
||
problems.append(f"{phase}: pages differ: {', '.join(differ)}")
|
||
if problems:
|
||
raise BenchError(
|
||
f"size {name}: restore check failed: {'; '.join(problems)}",
|
||
)
|
||
result = {
|
||
"messages_before_backup": counts_before["messages"],
|
||
"messages_at_dump_end": at_dump_end,
|
||
"messages_source_end": len(rows_source),
|
||
"messages_restored": len(rows_restored),
|
||
"messages_lost": len(lost),
|
||
"lost_after_backup_chat": sum(
|
||
1 for m in lost if rows_source[m][0] == after["chatId"]
|
||
),
|
||
"load_written": written["requests_ok"],
|
||
"load_failed": written["requests_failed"],
|
||
"load_in_backup": len(rows_restored) - counts_before["messages"],
|
||
"tables_restored": counts_restored,
|
||
"backup_bytes": dump_bytes,
|
||
"backup_bytes_per_message": round(
|
||
dump_bytes / len(rows_restored),
|
||
1,
|
||
),
|
||
"dump_seconds": round(dump_seconds, 2),
|
||
"storage_seconds": round(storage_seconds, 2),
|
||
"copy_seconds": round(copy_seconds, 2),
|
||
"restore_seconds": round(restore_seconds, 2),
|
||
"analyze_seconds": round(analyze_seconds, 2),
|
||
"verify_seconds": round(verify_seconds, 2),
|
||
"api_ready_seconds": round(api_seconds, 2),
|
||
"rto_seconds": round(db_seconds + api_seconds, 2),
|
||
"pages_checked": len(source_check["pages"]),
|
||
"stale_chat_get": stale["after_chat_get"],
|
||
"stale_chat_post": stale["after_chat_post"],
|
||
"flushed_chat_get": restored["after_chat_get"],
|
||
"late_session": stale["late_me"],
|
||
}
|
||
self.say(
|
||
f"size {name}: restored {len(rows_restored)} of "
|
||
f"{len(rows_source)} messages ({len(lost)} lost after the "
|
||
f"snapshot), restore {restore_seconds:.2f}s, ANALYZE "
|
||
f"{analyze_seconds:.2f}s, API {api_seconds:.2f}s, RTO "
|
||
f"{result['rto_seconds']}s; stale chat GET "
|
||
f"{result['stale_chat_get']} POST {result['stale_chat_post']}, "
|
||
f"after FLUSHALL {result['flushed_chat_get']}",
|
||
)
|
||
return result
|
||
|
||
def run_prefill(self, size: int, total: int) -> float:
|
||
"""Наполнение всех чатов размера по расписанию, порциями."""
|
||
chunk_size = self.profile.prefill_chunk
|
||
total_seconds = 0.0
|
||
for chunk, offset in enumerate(range(0, total, chunk_size), start=1):
|
||
count = min(chunk_size, total - offset)
|
||
name = f"prefill-{human(size)}-{chunk:02d}"
|
||
if self.cell is not None:
|
||
# серии: своя порция у каждой ячейки и попытки
|
||
name = f"{self.cell['cell']}-a{self.attempt}-{name}"
|
||
self.stage = f"size {human(size)}: prefill"
|
||
measured = measure(
|
||
self.stand,
|
||
"prefill.js",
|
||
{
|
||
"COUNT": count,
|
||
"OFFSET": offset,
|
||
"VUS": self.profile.prefill_vus,
|
||
"NAME": name,
|
||
},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
down = self.stand.down_storages()
|
||
flags = run_flags(
|
||
self.profile,
|
||
measured,
|
||
self.stand.app_state(),
|
||
down,
|
||
)
|
||
self.results.add_prefill(
|
||
{
|
||
**summary,
|
||
"size": size,
|
||
"chunk": chunk,
|
||
"messages_before": offset,
|
||
"messages_after": offset + summary["requests_ok"],
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"mem_cgroup_end_mib": mib_bytes(
|
||
measured.after["mem_bytes"],
|
||
),
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
"db_cpu_cores": measured.db_cpu,
|
||
"cell": None if self.cell is None else self.cell["cell"],
|
||
"attempt": self.attempt or None,
|
||
},
|
||
)
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during prefill chunk "
|
||
f"{chunk}: matrix stopped",
|
||
)
|
||
if summary["requests_ok"] != count:
|
||
raise BenchError(
|
||
f"prefill chunk {chunk}: {summary['requests_ok']} of "
|
||
f"{count} messages confirmed ({' '.join(flags)})",
|
||
)
|
||
total_seconds += summary["duration_s"]
|
||
self.prefilled = offset + count
|
||
self.prefill_rate = self.prefilled / total_seconds
|
||
self.say(
|
||
f"size {human(size)}: prefill {self.prefilled}/{total} "
|
||
f"({self.prefilled * 100 // total}%), "
|
||
f"{self.prefill_rate:.0f} msg/s, "
|
||
f"~{fmt_left((total - self.prefilled) / self.prefill_rate)} "
|
||
"left",
|
||
)
|
||
self.progress()
|
||
return total_seconds
|
||
|
||
def run_one(
|
||
self,
|
||
scenario: str,
|
||
size: int,
|
||
conc: int,
|
||
repeat: int,
|
||
store: int,
|
||
) -> int:
|
||
"""Прогрев и измеряемый прогон; возвращает сообщений в хранилище."""
|
||
p = self.profile
|
||
number = self.runs_done + 1
|
||
name = f"{scenario}-{human(size)}-c{conc}-r{repeat}"
|
||
self.stage = f"run {number}/{self.runs_total} {name}"
|
||
self.progress()
|
||
wall = time.monotonic()
|
||
if scenario == "read":
|
||
warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"}
|
||
self.stand.k6(
|
||
"read.js",
|
||
{**warm, "SEED": p.seed + p.warmup_seed_shift},
|
||
)
|
||
env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"}
|
||
env["SEED"] = p.seed
|
||
else:
|
||
self.stand.k6(
|
||
"write.js",
|
||
{"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store},
|
||
)
|
||
warm_ok = parse_summary(self.stand.work_dir / "last.json")
|
||
store += warm_ok["requests_ok"]
|
||
env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store}
|
||
measured = measure(
|
||
self.stand,
|
||
f"{scenario}.js",
|
||
{**env, "NAME": name},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
app = self.stand.app_state()
|
||
down = self.stand.down_storages()
|
||
flags = run_flags(p, measured, app, down)
|
||
end = store + (summary["requests_ok"] if scenario == "write" else 0)
|
||
row = {
|
||
**summary,
|
||
"run": number,
|
||
"scenario": scenario,
|
||
"size": size,
|
||
"concurrency": conc,
|
||
"repeat": repeat,
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_start_mib": mib(measured.before["rss_kib"]),
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"rss_max_mib": mib(measured.rss_max_kib),
|
||
"mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]),
|
||
"store_messages_start": store,
|
||
"store_messages_end": end,
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
"db_cpu_cores": measured.db_cpu,
|
||
}
|
||
self.results.add_run(row)
|
||
self.runs_done += 1
|
||
if scenario == "write" and summary["rps"]:
|
||
self.write_rps[conc] = summary["rps"]
|
||
overhead = time.monotonic() - wall - summary["duration_s"]
|
||
if scenario == "read":
|
||
overhead -= p.read_warmup_s
|
||
self.run_overhead = max(1.0, overhead)
|
||
errors = summary["requests_failed"] + summary["check_failed"]
|
||
self.say(
|
||
f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} "
|
||
f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps "
|
||
f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} "
|
||
f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} "
|
||
f"db {measured.db_cpu} rss {row['rss_end_mib']} MiB "
|
||
f"err {errors} "
|
||
f"~{fmt_left(self.eta())} left"
|
||
+ (f" [{' '.join(flags)}]" if flags else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if "oom" in flags or "app_down" in flags:
|
||
raise BenchError(
|
||
f"api replica is down after {name} ({' '.join(flags)}): "
|
||
"matrix stopped",
|
||
)
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during {name}: matrix stopped",
|
||
)
|
||
return end
|
||
|
||
# ----------------------------------------------------- профиль отказа
|
||
|
||
def prepare_load(self, tag: str) -> dict[str, Any]:
|
||
"""Стенд и данные постоянной нагрузки (профили отказа и замены):
|
||
реплики за LB, паспорт, пользователи и чаты, читаемый чат,
|
||
курсоры; возвращает фикстуру."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
size = p.sizes[0]
|
||
self.size = size
|
||
name = human(size)
|
||
self.stage = "start"
|
||
self.say(
|
||
f"{tag}: start, {stand.profile.replicas} replicas behind haproxy",
|
||
)
|
||
stand.start(lambda text: self.say(f"{tag}: {text}"))
|
||
limits = stand.limits()
|
||
if self.cell is None:
|
||
self.results.meta["limits"] = limits
|
||
self.results.meta["stand"] = stand.passport()
|
||
else:
|
||
# серия: лимиты по конфигурациям, одинаковые во всех повторах;
|
||
# max_connections — в лимитах конфигурации, не в паспорте
|
||
known = self.results.meta.setdefault("limits", {})
|
||
config = self.cell["config"]
|
||
# лимиты сэмплера добавляет run_cell после его запуска
|
||
earlier = known.get(config, {})
|
||
sampler = earlier.get("sampler")
|
||
stand_only = {k: v for k, v in earlier.items() if k != "sampler"}
|
||
if config in known and stand_only != limits:
|
||
raise BenchError(
|
||
f"{config}: limits differ from the earlier repeat: "
|
||
f"{limits}",
|
||
)
|
||
known[config] = (
|
||
limits if sampler is None else {**limits, "sampler": sampler}
|
||
)
|
||
if "stand" not in self.results.meta:
|
||
passport = stand.passport()
|
||
passport["postgres"]["settings"].pop("max_connections", None)
|
||
self.results.meta["stand"] = passport
|
||
self.results.write_meta()
|
||
stand.k6(
|
||
"prepare.js",
|
||
{
|
||
"MEMBERS": p.members,
|
||
"BACKGROUND": "",
|
||
"WRITE_CHATS": p.write_chats,
|
||
},
|
||
)
|
||
fixture = stand.work_json("fixture.json")
|
||
(stand.work_dir / "schedule.json").write_text(
|
||
json.dumps(fill_schedule([size])),
|
||
encoding="utf-8",
|
||
)
|
||
self.stage = "prefill"
|
||
self.progress()
|
||
self.run_prefill(size, size)
|
||
stand.maintain()
|
||
stand.k6(
|
||
"cursors.js",
|
||
{"WALK_LIMIT": p.walk_limit, "POOL_SIZE": p.cursor_pool},
|
||
)
|
||
pool = stand.work_json("cursors.json")
|
||
if pool["walked"] != size:
|
||
raise BenchError(
|
||
f"read chat has {pool['walked']} messages, expected {size}",
|
||
)
|
||
self.say(f"{tag}: read chat {name}, {len(pool['cursors'])} cursors")
|
||
self.prepared = True
|
||
return fixture
|
||
|
||
def k6_requests(
|
||
self,
|
||
name: str,
|
||
log_name: str,
|
||
) -> tuple[list[failover.Request], dict[str, Any]]:
|
||
"""Строки о запросах прогона k6 и его итог. Строк каждого вида
|
||
столько же, сколько итераций сценария, иначе шкала и повтор
|
||
неуспешных записей неполны — отказ."""
|
||
output = (self.stand.work_dir / log_name).read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
requests = failover.parse_requests(output)
|
||
k6_data = json.loads(
|
||
(self.results.dir / "raw" / f"{name}-k6.json").read_text(
|
||
encoding="utf-8",
|
||
),
|
||
)
|
||
for kind, scenario in (("w", "write"), ("r", "read")):
|
||
iterations = (
|
||
k6_data["metrics"]
|
||
.get(f"iterations{{scenario:{scenario}}}", {})
|
||
.get("values", {})
|
||
.get("count")
|
||
)
|
||
lines = sum(1 for r in requests if r.kind == kind)
|
||
if iterations != lines:
|
||
raise BenchError(
|
||
f"{name}: {lines} k6 lines of {scenario}, "
|
||
f"{iterations} iterations; unparsed: "
|
||
f"{failover.unparsed(output)[:3]}",
|
||
)
|
||
return requests, k6_data
|
||
|
||
# ------------------------------------- ресурсы профилей уровня «4»
|
||
|
||
def start_observer(self, tag: str) -> None:
|
||
"""Сэмплер в режиме `scan` на всё время нагрузки (решения 1, 2
|
||
design change `profile-resources`): лимиты сэмплера, прочитанные у
|
||
контейнера, и карта контейнеров — в паспорт."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
stand.start_sampler(scan=True)
|
||
meta = self.results.meta
|
||
meta["limits"]["sampler"] = stand.container_limits(SAMPLER)
|
||
self.containers.update(stand.container_map())
|
||
info = meta.setdefault("resources", {})
|
||
info.update(
|
||
{
|
||
"mode": "scan",
|
||
"interval_s": p.sample_interval_s,
|
||
"containers": self.containers,
|
||
"unmapped": 0,
|
||
},
|
||
)
|
||
self.results.write_meta()
|
||
self.say(
|
||
f"{tag}: sampler started (scan, {p.sampler_cpus} CPU / "
|
||
f"{p.sampler_memory}, no network), {len(self.containers)} "
|
||
"containers",
|
||
)
|
||
|
||
def check_observer(self, name: str, first: float, last: float) -> None:
|
||
"""Сторож (решение 2): сэмплер работает, и в нагрузке соседние
|
||
тики ВМ не дальше `GAP_LIMIT_S`. Иначе — отказ с фазой: метка без
|
||
ресурсов цели не годится."""
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
SAMPLER,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
raise BenchError(
|
||
f"{name}: sampler {SAMPLER} stopped during the load: "
|
||
"resources not measured",
|
||
)
|
||
samples = scaling.parse_samples(self.stand.sampler_logs()[0])
|
||
gap = resources.max_gap(samples, first, last)
|
||
if gap > resources.GAP_LIMIT_S:
|
||
raise BenchError(
|
||
f"{name}: no sampler ticks for {gap:.1f}s during the load "
|
||
f"(limit {resources.GAP_LIMIT_S:g}s): resources not measured",
|
||
)
|
||
|
||
def observe(
|
||
self,
|
||
result: resources.Result,
|
||
name: str,
|
||
profile: str,
|
||
) -> None:
|
||
"""Строки ресурсов — в файлы метки, ID вне стенда — в паспорт,
|
||
строка ресурсов — в терминал."""
|
||
self.unmapped |= set(result.unmapped)
|
||
self.results.meta["resources"]["unmapped"] = len(self.unmapped)
|
||
self.results.add_resources(result)
|
||
self.results.write_meta()
|
||
self.say(f"{name}: {resources.terminal_line(result.rows, profile)}")
|
||
|
||
def run_failover(self) -> None:
|
||
"""Профиль отказа реплик (решение 7 design change
|
||
`graceful-failover`): стенд, читаемый чат и курсоры, затем фазы —
|
||
каждая реплика в каждом режиме. С `sample_resources` — сэмплер от
|
||
первой фазы до конца последней (design change
|
||
`profile-resources`)."""
|
||
p = self.profile
|
||
fixture = self.prepare_load("failover")
|
||
if p.sample_resources:
|
||
self.start_observer("failover")
|
||
run_id = secrets.token_hex(4)
|
||
writes = 0
|
||
number = 0
|
||
for mode in p.failover_modes:
|
||
for index in range(p.replicas):
|
||
number += 1
|
||
writes = self.run_phase(
|
||
number,
|
||
mode,
|
||
index,
|
||
run_id=run_id,
|
||
fixture=fixture,
|
||
writes=writes,
|
||
)
|
||
if p.sample_resources:
|
||
self.stand.stop_sampler()
|
||
|
||
def run_phase(
|
||
self,
|
||
number: int,
|
||
mode: str,
|
||
index: int,
|
||
*,
|
||
run_id: str,
|
||
fixture: dict[str, Any],
|
||
writes: int,
|
||
) -> int:
|
||
"""Фаза: нагрузка, действие над репликой `index`, возврат, разбор,
|
||
повтор неуспешных записей и сверка; возвращает число отправленных
|
||
за прогон записей (разных ключей)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
phase = f"{number:02d}-{mode}-r{index + 1}"
|
||
self.stage = f"phase {number}/{self.runs_total} {phase}"
|
||
self.progress()
|
||
stand.wait_ready()
|
||
target = stand.replicas()[index]
|
||
target_id = sh("docker", "inspect", "-f", "{{.Id}}", target)
|
||
instance = target_id[:12]
|
||
address = stand.address(target)
|
||
slot = next(
|
||
(
|
||
s["slot"]
|
||
for s in stand.lb_slots()
|
||
if s["addr"].rsplit(":", 1)[0] == address
|
||
),
|
||
None,
|
||
)
|
||
duration = p.failover_before_s + p.failover_down_s + p.failover_after_s
|
||
log_name = f"{phase}.log"
|
||
# события цели — потоком с начала фазы, не из буфера демона после
|
||
# неё (решение 9 design change fix-known-issues). Без --since:
|
||
# подписка CLI — доли секунды, а действие — через запуск k6 и
|
||
# failover_before_s (≥ 5 с); --since потребовал бы часов хоста
|
||
# против часов ВМ, а опоздавшая подписка — громкий отказ ниже
|
||
events_path = stand.work_dir / f"{phase}-events.log"
|
||
stream = stand.container_events(target_id, events_path)
|
||
try:
|
||
started = time.time()
|
||
load = stand.k6_console(
|
||
"failover.js",
|
||
{
|
||
"RUN": run_id,
|
||
"PHASE": number,
|
||
"DURATION": f"{duration}s",
|
||
"WRITE_RATE": p.failover_write_rate,
|
||
"READ_RATE": p.failover_read_rate,
|
||
"PRE_VUS": p.failover_pre_vus,
|
||
"MAX_VUS": p.failover_max_vus,
|
||
"SEED": p.seed + number,
|
||
"NAME": f"{phase}-k6",
|
||
},
|
||
log_name,
|
||
)
|
||
try:
|
||
time.sleep(p.failover_before_s)
|
||
acted = time.monotonic()
|
||
command = {
|
||
"stop": ("stop", "--timeout", "9"),
|
||
"kill": ("kill",),
|
||
"pause": ("pause",),
|
||
}[mode]
|
||
sh("docker", *command, target_id)
|
||
self.say(f"phase {phase}: {mode} {instance} ({slot})")
|
||
time.sleep(
|
||
max(0.0, acted + p.failover_down_s - time.monotonic()),
|
||
)
|
||
restore = ("unpause",) if mode == "pause" else ("start",)
|
||
sh("docker", *restore, target_id)
|
||
self.say(f"phase {phase}: {restore[0]} {instance}")
|
||
load.wait(timeout=duration + 120)
|
||
finally:
|
||
if load.poll() is None:
|
||
load.kill()
|
||
# реплика не должна остаться остановленной или замороженной
|
||
state = stand.state(target_id)
|
||
paused = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Paused}}",
|
||
target_id,
|
||
check=False,
|
||
)
|
||
if paused == "true":
|
||
sh("docker", "unpause", target_id, check=False)
|
||
elif not state["running"]:
|
||
sh("docker", "start", target_id, check=False)
|
||
if load.returncode != 0:
|
||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)[-3000:]
|
||
raise BenchError(
|
||
f"k6 failover.js failed ({load.returncode}):\n{tail}",
|
||
)
|
||
try:
|
||
stand.wait_ready(timeout=30)
|
||
except BenchError as exc:
|
||
raise BenchError(
|
||
f"phase {phase}: replica not back: {exc}",
|
||
) from exc
|
||
finally:
|
||
stop_stream(stream)
|
||
down = stand.down_storages()
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during phase {phase}",
|
||
)
|
||
events = failover.parse_events(
|
||
events_path.read_text(encoding="utf-8", errors="replace"),
|
||
)
|
||
action = next(
|
||
(e for e in events if e.action == failover.ACTIONS[mode]),
|
||
None,
|
||
)
|
||
back = next(
|
||
(
|
||
e
|
||
for e in events
|
||
if e.action == failover.RESTORES[mode]
|
||
and action is not None
|
||
and e.time > action.time
|
||
),
|
||
None,
|
||
)
|
||
if action is None or back is None:
|
||
raise BenchError(
|
||
f"phase {phase}: docker events of {instance} not found",
|
||
)
|
||
lb_log = stand.logs(LB, started - 2)
|
||
lb_events = failover.parse_lb_log(lb_log)
|
||
(self.results.dir / "raw" / f"{phase}-lb.log").write_text(
|
||
failover.lb_lines(lb_log) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
address_back = stand.address(target)
|
||
slot_back = next(
|
||
(
|
||
s["slot"]
|
||
for s in stand.lb_slots()
|
||
if s["addr"].rsplit(":", 1)[0] == address_back
|
||
),
|
||
None,
|
||
)
|
||
requests, k6_data = self.k6_requests(phase, log_name)
|
||
if p.sample_resources:
|
||
self.phase_resources(
|
||
phase,
|
||
mode,
|
||
requests,
|
||
instance=instance,
|
||
action=action.time,
|
||
back=back.time,
|
||
)
|
||
summary = parse_summary(self.results.dir / "raw" / f"{phase}-k6.json")
|
||
timeline = failover.build_timeline(
|
||
requests,
|
||
action.time,
|
||
phase=phase,
|
||
mode=mode,
|
||
replica=index + 1,
|
||
)
|
||
row = failover.analyze(
|
||
requests,
|
||
timeline,
|
||
lb_events,
|
||
mode=mode,
|
||
instance=instance,
|
||
t0=action.time,
|
||
restore=back.time,
|
||
slot=slot,
|
||
slot_back=slot_back,
|
||
threshold=p.failover_success,
|
||
)
|
||
# неподтверждённые записи — повтор с тем же ключом и телом до 201
|
||
failed = sorted({r.n for r in requests if r.kind == "w" and not r.ok})
|
||
(stand.work_dir / "retry.json").write_text(
|
||
json.dumps({"items": failed}),
|
||
encoding="utf-8",
|
||
)
|
||
stand.k6("retry.js", {"RUN": run_id, "PHASE": number})
|
||
retried = stand.work_json("retry-result.json")
|
||
if retried["failed"]:
|
||
raise BenchError(
|
||
f"phase {phase}: retries with the same key failed: "
|
||
f"{retried['failed'][:5]}",
|
||
)
|
||
writes += row["writes"]
|
||
messages, texts = (
|
||
int(value)
|
||
for value in stand.psql(
|
||
"SELECT count(*), count(DISTINCT text) FROM messages "
|
||
f"WHERE chat_id = '{fixture['writeChatId']}';\n",
|
||
).split("|")
|
||
)
|
||
dropped = (
|
||
k6_data["metrics"]
|
||
.get("dropped_iterations", {})
|
||
.get("values", {})
|
||
.get("count", 0)
|
||
)
|
||
if dropped:
|
||
# k6 не выдал заданную интенсивность: не хватило VU
|
||
row["flags"] = " ".join(filter(None, [row["flags"], "dropped"]))
|
||
row.update(
|
||
{
|
||
"phase": phase,
|
||
"mode": mode,
|
||
"replica": index + 1,
|
||
"instance": instance,
|
||
"slot": slot,
|
||
"action_at": failover.iso(action.time),
|
||
"signal": action.signal or None,
|
||
"restore_at": failover.iso(back.time),
|
||
"slot_back": slot_back,
|
||
"retried_ok": retried["ok"],
|
||
"write_chat_messages": messages,
|
||
"distinct_texts": texts,
|
||
"pg_idle_tx_terminated": failover.count_idle_terminations(
|
||
stand.logs(DB, started - 2),
|
||
),
|
||
"dropped_iterations": int(dropped),
|
||
},
|
||
)
|
||
self.results.add_phase(row, timeline)
|
||
self.runs_done += 1
|
||
self.say(
|
||
f"phase {phase}: LB out {row['lb_out_s']} s "
|
||
f"({row['lb_out_state']}, bound {row['bound_s']} s), errors "
|
||
f"{row['errors_down']} [{row['codes_down']}], recovery "
|
||
f"{row['recovery_s']} s, back in LB {row['lb_up_s']} s, errors "
|
||
f"after {row['errors_after']}, {summary['requests_ok']} ok of "
|
||
f"{row['requests']}; retried {retried['ok']}, messages "
|
||
f"{messages} = keys {writes}"
|
||
+ (f" [{row['flags']}]" if row["flags"] else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if messages != writes or texts != writes:
|
||
raise BenchError(
|
||
f"phase {phase}: write chat has {messages} messages and "
|
||
f"{texts} distinct texts, {writes} keys sent",
|
||
)
|
||
return writes
|
||
|
||
def phase_resources(
|
||
self,
|
||
phase: str,
|
||
mode: str,
|
||
requests: list[failover.Request],
|
||
*,
|
||
instance: str,
|
||
action: float,
|
||
back: float,
|
||
) -> None:
|
||
"""Ресурсы фазы по окнам (решения 4, 5 design change
|
||
`profile-resources`): разгон — 2 с с первого запроса, до действия,
|
||
окно отказа до возврата, после возврата до последнего запроса.
|
||
`docker start` после `stop` / `kill` — новая cgroup, её счёт идёт
|
||
с момента возврата; `pause` … `unpause` — заморозка: CPU
|
||
граничных интервалов — вне её."""
|
||
first = min(r.start for r in requests)
|
||
last = max(r.start for r in requests)
|
||
self.check_observer(f"phase {phase}", first, last)
|
||
if mode == "pause":
|
||
self.frozen.setdefault(instance, []).append((action, back))
|
||
else:
|
||
self.starts.setdefault(instance, []).append(back)
|
||
self.containers.update(self.stand.container_map())
|
||
roles = {}
|
||
for ident, item in self.containers.items():
|
||
role = item["role"]
|
||
if role == APP_SERVICE:
|
||
role = "api-target" if ident == instance else "api-other"
|
||
roles[ident] = role
|
||
windows = [
|
||
(resources.WARMUP, first, first + resources.WARMUP_S),
|
||
("before", first + resources.WARMUP_S, action),
|
||
("down", action, back),
|
||
("after", back, last),
|
||
]
|
||
result = resources.analyze(
|
||
scaling.parse_samples(self.stand.sampler_logs()[0]),
|
||
roles,
|
||
windows,
|
||
ref=action,
|
||
ok_starts=[r.start for r in requests if r.ok],
|
||
phase=phase,
|
||
mode=mode,
|
||
profile="failover",
|
||
starts=self.starts,
|
||
frozen=self.frozen,
|
||
)
|
||
self.observe(result, f"phase {phase}", "failover")
|
||
|
||
# ------------------------------------------- профиль поочерёдной замены
|
||
|
||
def run_rollout(self) -> None:
|
||
"""Профиль поочерёдной замены (решение 8 design change
|
||
`rolling-update`): стенд на образе прежнего commit, постоянная
|
||
нагрузка, посреди неё `scripts/rolling-update.sh` против проекта
|
||
стенда, затем разбор, повтор неуспешных записей и сверка."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
info: dict[str, Any] = self.results.meta.setdefault("rollout", {})
|
||
self.stage = "old image"
|
||
commit = sh(
|
||
"git",
|
||
"-C",
|
||
str(ROOT),
|
||
"rev-parse",
|
||
"--short",
|
||
f"{p.rollout_from}^{{commit}}",
|
||
)
|
||
info["from_commit"] = commit
|
||
info["new_image"] = self.results.meta["app_image"]["id"]
|
||
info["old_image"] = stand.build_from(commit)
|
||
# реплики и migrate стенда берут тег APP_IMAGE: на старте — образ
|
||
# прежнего commit; скрипт пересоберёт тег из текущего checkout
|
||
sh("docker", "tag", FROM_IMAGE, APP_IMAGE)
|
||
self.say(
|
||
f"rollout: image of {commit} built ({info['old_image'][:19]}…)",
|
||
)
|
||
fixture = self.prepare_load("rollout")
|
||
info["containers_before"] = stand.app_containers()
|
||
info["revision_before"] = stand.psql(
|
||
"SELECT version_num FROM alembic_version;\n",
|
||
)
|
||
self.results.write_meta()
|
||
if p.sample_resources:
|
||
# до потока событий и k6: один create/start сэмплера — до нагрузки
|
||
self.start_observer("rollout")
|
||
old = {c["instance"] for c in info["containers_before"]}
|
||
run_id = secrets.token_hex(4)
|
||
name = "rollout"
|
||
log_name = f"{name}.log"
|
||
self.stage = "rollout"
|
||
self.progress()
|
||
duration = p.rollout_before_s + p.rollout_budget_s + p.rollout_after_s
|
||
started = time.time()
|
||
events_path = stand.work_dir / "events.log"
|
||
events = stand.project_events(events_path)
|
||
script_out = stand.work_dir / "rollout-script.out"
|
||
script_s = None
|
||
script: subprocess.Popen[bytes] | None = None
|
||
load: subprocess.Popen[bytes] | None = None
|
||
try:
|
||
# k6 — внутри try: при его сбое finally останавливает поток
|
||
load = stand.k6_console(
|
||
"failover.js",
|
||
{
|
||
"RUN": run_id,
|
||
"PHASE": 1,
|
||
"DURATION": f"{duration}s",
|
||
"WRITE_RATE": p.failover_write_rate,
|
||
"READ_RATE": p.failover_read_rate,
|
||
"PRE_VUS": p.failover_pre_vus,
|
||
"MAX_VUS": p.failover_max_vus,
|
||
"SEED": p.seed + 1,
|
||
"NAME": f"{name}-k6",
|
||
},
|
||
log_name,
|
||
)
|
||
time.sleep(p.rollout_before_s)
|
||
self.say("rollout: scripts/rolling-update.sh started")
|
||
began = time.monotonic()
|
||
with script_out.open("w") as out:
|
||
script = subprocess.Popen(
|
||
["sh", str(ROLLOUT_SCRIPT)],
|
||
env=stand.script_env(),
|
||
stdout=out,
|
||
stderr=subprocess.STDOUT,
|
||
)
|
||
try:
|
||
script.wait(timeout=p.rollout_budget_s + 120)
|
||
except subprocess.TimeoutExpired:
|
||
# SIGTERM: скрипт доводит текущий шаг и выходит (143)
|
||
script.terminate()
|
||
script.wait(timeout=60)
|
||
script_s = round(time.monotonic() - began, 1)
|
||
self.say(
|
||
f"rollout: script exited {script.returncode} in {script_s}s",
|
||
)
|
||
load.wait(timeout=duration + 120)
|
||
finally:
|
||
if load is not None and load.poll() is None:
|
||
load.kill()
|
||
# Ctrl+C достаётся и скрипту (та же группа процессов): он
|
||
# доводит текущий шаг; стенд удаляется только после него
|
||
if script is not None and script.poll() is None:
|
||
try:
|
||
script.wait(timeout=60)
|
||
except subprocess.TimeoutExpired:
|
||
script.kill()
|
||
stop_stream(events)
|
||
text = (
|
||
script_out.read_text(encoding="utf-8", errors="replace")
|
||
if script_out.exists()
|
||
else ""
|
||
)
|
||
(self.results.dir / "raw" / "rollout-script.log").write_text(
|
||
rollout.script_lines(text) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
exit_code = None if script is None else script.returncode
|
||
if exit_code != 0:
|
||
raise BenchError(
|
||
f"rolling-update.sh exited {exit_code}: "
|
||
f"{rollout.script_error(text) or 'see raw/rollout-script.log'}",
|
||
)
|
||
if load.returncode != 0:
|
||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)[-3000:]
|
||
raise BenchError(
|
||
f"k6 failover.js failed ({load.returncode}):\n{tail}",
|
||
)
|
||
stand.wait_ready(timeout=30)
|
||
down = stand.down_storages()
|
||
if down:
|
||
raise BenchError(f"{', '.join(down)} went down during rollout")
|
||
containers = rollout.parse_container_events(
|
||
events_path.read_text(encoding="utf-8", errors="replace"),
|
||
)
|
||
lb_log = stand.logs(LB, started - 2)
|
||
(self.results.dir / "raw" / f"{name}-lb.log").write_text(
|
||
failover.lb_lines(lb_log) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
slots = rollout.parse_slot_events(lb_log)
|
||
info["containers_after"] = stand.app_containers()
|
||
info["revision_after"] = stand.psql(
|
||
"SELECT version_num FROM alembic_version;\n",
|
||
)
|
||
info["tag_after"] = sh(
|
||
"docker",
|
||
"image",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Id}}",
|
||
APP_IMAGE,
|
||
)
|
||
self.results.write_meta()
|
||
after = info["containers_after"]
|
||
new = {c["instance"] for c in after}
|
||
# итог замены: API_REPLICAS работающих реплик, ни одной прежней,
|
||
# все на одном образе, отличном от прежнего (решение 8)
|
||
if len(after) != p.replicas or not all(c["running"] for c in after):
|
||
raise BenchError(
|
||
f"{len(after)} api containers after rollout, expected "
|
||
f"{p.replicas} running",
|
||
)
|
||
if old & new:
|
||
raise BenchError(f"old replicas still running: {old & new}")
|
||
images_before = {c["image"] for c in info["containers_before"]}
|
||
images_after = {c["image"] for c in after}
|
||
if len(images_after) != 1 or images_after & images_before:
|
||
raise BenchError(
|
||
f"replicas after rollout run images {images_after}, before "
|
||
f"{images_before}",
|
||
)
|
||
requests, k6_data = self.k6_requests(name, log_name)
|
||
t0 = rollout.start_time(containers)
|
||
steps = rollout.parse_steps(text)
|
||
end_event = (
|
||
rollout.container_event(
|
||
containers,
|
||
steps[-1].get("old_id"),
|
||
"destroy",
|
||
)
|
||
if steps
|
||
else None
|
||
)
|
||
if t0 is None or end_event is None or len(steps) != p.replicas:
|
||
raise BenchError(
|
||
f"rollout steps not found: {len(steps)} steps in the script "
|
||
"log, docker events of the migration or the last old replica "
|
||
"missing",
|
||
)
|
||
replaced = rollout.replica_rows(steps, containers, slots, requests, t0)
|
||
if p.sample_resources:
|
||
self.rollout_resources(
|
||
requests,
|
||
containers,
|
||
t0=t0,
|
||
end=end_event.time,
|
||
old=old,
|
||
new=new,
|
||
)
|
||
self.stand.stop_sampler()
|
||
timeline = rollout.build_timeline(requests, t0, old, new)
|
||
row = rollout.analyze(
|
||
requests,
|
||
timeline,
|
||
replaced,
|
||
t0=t0,
|
||
end=end_event.time,
|
||
old=old,
|
||
new=new,
|
||
threshold=p.failover_success,
|
||
)
|
||
# неподтверждённые записи — повтор с тем же ключом и телом до 201
|
||
failed = sorted({r.n for r in requests if r.kind == "w" and not r.ok})
|
||
(stand.work_dir / "retry.json").write_text(
|
||
json.dumps({"items": failed}),
|
||
encoding="utf-8",
|
||
)
|
||
stand.k6("retry.js", {"RUN": run_id, "PHASE": 1})
|
||
retried = stand.work_json("retry-result.json")
|
||
if retried["failed"]:
|
||
raise BenchError(
|
||
f"rollout: retries with the same key failed: "
|
||
f"{retried['failed'][:5]}",
|
||
)
|
||
messages, texts = (
|
||
int(value)
|
||
for value in stand.psql(
|
||
"SELECT count(*), count(DISTINCT text) FROM messages "
|
||
f"WHERE chat_id = '{fixture['writeChatId']}';\n",
|
||
).split("|")
|
||
)
|
||
dropped = (
|
||
k6_data["metrics"]
|
||
.get("dropped_iterations", {})
|
||
.get("values", {})
|
||
.get("count", 0)
|
||
)
|
||
flags = row["flags"].split()
|
||
if dropped:
|
||
flags.append("dropped")
|
||
if script_s is not None and script_s > p.rollout_budget_s:
|
||
flags.append("over_budget")
|
||
row.update(
|
||
{
|
||
"label": self.label,
|
||
"from_commit": commit,
|
||
"replicas": p.replicas,
|
||
"revision_before": info["revision_before"],
|
||
"revision_after": info["revision_after"],
|
||
"build_s": rollout.seconds(rollout.BUILD_RE, text),
|
||
"migrate_s": rollout.seconds(rollout.MIGRATE_RE, text),
|
||
"script_exit": exit_code,
|
||
"retried_ok": retried["ok"],
|
||
"write_chat_messages": messages,
|
||
"distinct_texts": texts,
|
||
"dropped_iterations": int(dropped),
|
||
"flags": " ".join(flags),
|
||
},
|
||
)
|
||
self.results.add_rollout(row, replaced, timeline)
|
||
self.runs_done += 1
|
||
self.say(
|
||
f"rollout: {row['rollout_s']} s, errors before / during / after "
|
||
f"{row['errors_before']} / {row['errors_during']} "
|
||
f"[{row['codes_during']}] / {row['errors_after']}, min success "
|
||
f"{row['min_success']}, old after drain {row['old_after_drain']}, "
|
||
f"revision {info['revision_before']} -> {info['revision_after']}; "
|
||
f"retried {retried['ok']}, messages {messages} = keys "
|
||
f"{row['writes']}"
|
||
+ (f" [{row['flags']}]" if row["flags"] else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if messages != row["writes"] or texts != row["writes"]:
|
||
raise BenchError(
|
||
f"rollout: write chat has {messages} messages and {texts} "
|
||
f"distinct texts, {row['writes']} keys sent",
|
||
)
|
||
|
||
def rollout_resources(
|
||
self,
|
||
requests: list[failover.Request],
|
||
containers: list[rollout.ContainerEvent],
|
||
*,
|
||
t0: float,
|
||
end: float,
|
||
old: set[str],
|
||
new: set[str],
|
||
) -> None:
|
||
"""Ресурсы замены по окнам (решения 4, 5 design change
|
||
`profile-resources`): до — от конца разгона до первого обращения
|
||
скрипта к стенду, подготовка — до создания контейнера миграции,
|
||
замена — до удаления последней старой реплики, после — до
|
||
последнего запроса. Роли — по `X-Instance-Id` до и после, по
|
||
сервису compose и по `create` из потока событий проекта
|
||
(контейнеры, удалённые до конца прогона)."""
|
||
first = min(r.start for r in requests)
|
||
last = max(r.start for r in requests)
|
||
self.check_observer("rollout", first, last)
|
||
self.containers.update(self.stand.container_map())
|
||
for event in containers:
|
||
ident = event.id[:12]
|
||
if event.action != "create" or ident in self.containers:
|
||
continue
|
||
role = ""
|
||
if "-migrate-" in event.name:
|
||
role = "migrate"
|
||
elif re.search(rf"-{APP_SERVICE}-\d+$", event.name):
|
||
role = APP_SERVICE
|
||
self.containers[ident] = {"name": event.name, "role": role}
|
||
for event in containers:
|
||
if event.action == "start":
|
||
self.starts.setdefault(event.id[:12], []).append(event.time)
|
||
roles = {}
|
||
for ident, item in self.containers.items():
|
||
role = item["role"]
|
||
if ident in old:
|
||
role = "api-old"
|
||
elif ident in new:
|
||
role = "api-new"
|
||
roles[ident] = role
|
||
info = self.results.meta["rollout"]
|
||
prepare = rollout.script_start(
|
||
containers,
|
||
first,
|
||
self.stand.healthcheck_actions(),
|
||
)
|
||
info["script_start_at"] = (
|
||
failover.iso(prepare) if prepare is not None else None
|
||
)
|
||
if prepare is None or prepare > t0:
|
||
prepare = t0
|
||
windows = [
|
||
(resources.WARMUP, first, first + resources.WARMUP_S),
|
||
("before", first + resources.WARMUP_S, prepare),
|
||
("prepare", prepare, t0),
|
||
("during", t0, end),
|
||
("after", end, last),
|
||
]
|
||
result = resources.analyze(
|
||
scaling.parse_samples(self.stand.sampler_logs()[0]),
|
||
roles,
|
||
windows,
|
||
ref=t0,
|
||
ok_starts=[r.start for r in requests if r.ok],
|
||
phase="rollout",
|
||
mode="rollout",
|
||
profile="rollout",
|
||
starts=self.starts,
|
||
)
|
||
self.observe(result, "rollout", "rollout")
|
||
|
||
# ---------------------------------------------- серии трека 1
|
||
|
||
def run_series(self) -> None:
|
||
"""Серия трека 1 (решения 4–5 design change `scaling-bench`): план
|
||
ячеек кругами — в паспорт до первой ячейки; при продолжении
|
||
готовые ячейки (строка в `searches.csv`) пропускаются, прерванная
|
||
снимается заново с попыткой +1."""
|
||
p = self.profile
|
||
meta = self.results.meta
|
||
if "plan" not in meta:
|
||
meta["plan"] = scaling.plan_cells(
|
||
[tuple(c) for c in p.configs],
|
||
p.repeats,
|
||
p.order_seed,
|
||
)
|
||
self.results.write_meta()
|
||
plan = meta["plan"]
|
||
directory = self.results.dir
|
||
done = {
|
||
s["cell"] for s in scaling.read_csv(directory / "searches.csv")
|
||
}
|
||
runs = scaling.read_csv(directory / "runs.csv")
|
||
attempts: dict[str, int] = {}
|
||
for row in runs + scaling.read_csv(directory / "prefill.csv"):
|
||
if row.get("cell") and row.get("attempt"):
|
||
attempts[row["cell"]] = max(
|
||
attempts.get(row["cell"], 0),
|
||
int(row["attempt"]),
|
||
)
|
||
self.step_no = max(
|
||
(int(r["run"]) for r in runs if r["run"]),
|
||
default=0,
|
||
)
|
||
self.runs_done = len(done)
|
||
if done:
|
||
self.say(
|
||
f"resume: {len(done)}/{len(plan)} cells done, "
|
||
f"{len(plan) - len(done)} left",
|
||
)
|
||
for index, cell in enumerate(plan, start=1):
|
||
if cell["cell"] in done:
|
||
continue
|
||
self.run_cell(index, cell, attempts.get(cell["cell"], 0) + 1)
|
||
self.runs_done += 1
|
||
|
||
def run_cell(
|
||
self,
|
||
index: int,
|
||
cell: dict[str, Any],
|
||
attempt: int,
|
||
) -> None:
|
||
"""Ячейка: свежий стенд конфигурации, данные, сэмплер,
|
||
фиксированные уровни и поиск R*, сверка записей и читаемого чата,
|
||
строка `searches.csv`, уборка стенда (решения 2–4)."""
|
||
p = self.profile
|
||
self.cell = cell
|
||
self.cell_index = index
|
||
self.attempt = attempt
|
||
self.cell_started = time.monotonic()
|
||
profile = cell_profile(p, cell)
|
||
self.stand = Stand(profile, self.results.dir)
|
||
stand = self.stand
|
||
host_start = host_state()
|
||
started_at = datetime.now(UTC).isoformat(timespec="seconds")
|
||
others = sorted(
|
||
name
|
||
for name in sh(
|
||
"docker",
|
||
"ps",
|
||
"--format",
|
||
"{{.Names}}",
|
||
check=False,
|
||
).split()
|
||
if not name.startswith(PROJECT)
|
||
)
|
||
tag = f"cell {index}/{self.runs_total} {cell['cell']}" + (
|
||
f" (attempt {attempt})" if attempt > 1 else ""
|
||
)
|
||
fixture = self.prepare_load(tag)
|
||
size = p.sizes[0]
|
||
stand.start_sampler()
|
||
self.say(f"{tag}: sampler started")
|
||
# лимиты сэмплера — тоже у запущенного контейнера, в лимитах
|
||
# конфигурации (спек «Лимиты конфигурации серии», verify S11)
|
||
sampler = stand.container_limits(SAMPLER)
|
||
known = self.results.meta["limits"][cell["config"]]
|
||
if known.setdefault("sampler", sampler) != sampler:
|
||
raise BenchError(
|
||
f"{cell['config']}: sampler limits differ from the earlier "
|
||
f"repeat: {sampler}",
|
||
)
|
||
replicas = stand.replicas()
|
||
instances = [stand.full_id(name)[:12] for name in replicas]
|
||
addresses = {
|
||
stand.address(name): ident
|
||
for name, ident in zip(replicas, instances, strict=True)
|
||
}
|
||
limits = scaling.Limits(
|
||
replicas=profile.replicas,
|
||
db_cpus=float(profile.db_cpus),
|
||
lb_cpus=float(profile.lb_cpus),
|
||
generator_cpus=float(profile.generator_cpus or 0),
|
||
generator_cpu_limit=profile.generator_cpu_limit,
|
||
vm_cpus=float(self.results.meta["docker"]["vm_cpus"] or 0),
|
||
slo_p95_ms=p.slo_p95_ms,
|
||
slo_error_ratio=p.slo_error_ratio,
|
||
)
|
||
search = scaling.Search(
|
||
replicas=profile.replicas,
|
||
fixed=p.fixed_rps,
|
||
base=p.ladder_base,
|
||
factor=p.step_factor,
|
||
per_replica=p.start_rps_per_replica,
|
||
bisect_steps=p.bisect_steps,
|
||
max_steps=p.max_steps,
|
||
)
|
||
steps: list[dict[str, Any]] = []
|
||
previous: int | None = None
|
||
number = 0
|
||
while (step := search.next_step()) is not None:
|
||
level, kind = step
|
||
number += 1
|
||
row = self.run_step(
|
||
cell,
|
||
number,
|
||
level,
|
||
kind,
|
||
previous=previous,
|
||
instances=instances,
|
||
addresses=addresses,
|
||
limits=limits,
|
||
)
|
||
steps.append(row)
|
||
search.record(level, kind, row["_outcome"])
|
||
if row["_outcome"] == "pass":
|
||
previous = level
|
||
continue
|
||
# после нарушения — пауза без нагрузки и готовность реплик;
|
||
# следующая ступень — с нуля и долгим прогревом
|
||
previous = None
|
||
if not search.done:
|
||
time.sleep(p.cooldown_s)
|
||
stand.wait_ready(timeout=30)
|
||
stand.stop_sampler()
|
||
# сверка ячейки: читаемый чат не изменился, сообщений в чатах
|
||
# записи — между подтверждёнными и отправленными
|
||
ids = ", ".join(f"'{chat}'" for chat in fixture["writeChatIds"])
|
||
read_count, write_count = (
|
||
int(value)
|
||
for value in stand.psql(
|
||
"SELECT count(*) FILTER (WHERE chat_id = "
|
||
f"'{fixture['readChatId']}'), count(*) FILTER (WHERE "
|
||
f"chat_id IN ({ids})) FROM messages;\n",
|
||
).split("|")
|
||
)
|
||
writes_ok = sum(r["_writes_ok"] for r in steps)
|
||
writes_sent = sum(r["_writes_sent"] for r in steps)
|
||
slack = sum(r["_slack"] for r in steps)
|
||
if read_count != size:
|
||
raise BenchError(
|
||
f"cell {cell['cell']}: read chat has {read_count} messages, "
|
||
f"expected {size}",
|
||
)
|
||
if not writes_ok <= write_count <= writes_sent + slack:
|
||
raise BenchError(
|
||
f"cell {cell['cell']}: write chats have {write_count} "
|
||
f"messages, {writes_ok} confirmed and {writes_sent} sent"
|
||
+ (f" (+{slack} in flight at abort)" if slack else ""),
|
||
)
|
||
host_end = host_state()
|
||
start_out, end_out = host_start["swapouts"], host_end["swapouts"]
|
||
swapped = (
|
||
end_out - start_out
|
||
if start_out is not None and end_out is not None
|
||
else 0
|
||
)
|
||
flags = list(search.flags)
|
||
if swapped or "Battery Power" in (
|
||
host_start["power"],
|
||
host_end["power"],
|
||
):
|
||
flags.append("host_swap")
|
||
fixed = {r["level"]: r for r in steps if r["step_kind"] == "fixed"}
|
||
fail = search.first_fail
|
||
fail_row = next(
|
||
(
|
||
r
|
||
for r in reversed(steps)
|
||
if r["level"] == fail and r["_outcome"] == "fail"
|
||
),
|
||
None,
|
||
)
|
||
row: dict[str, Any] = {
|
||
**{
|
||
key: cell[key]
|
||
for key in (
|
||
"cell",
|
||
"config",
|
||
"replicas",
|
||
"replica_cpus",
|
||
"replica_memory",
|
||
"db_max_connections",
|
||
"repeat",
|
||
"round",
|
||
"order",
|
||
)
|
||
},
|
||
"series": p.series,
|
||
"attempt": attempt,
|
||
"started_at": started_at,
|
||
"finished_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"status": search.status,
|
||
"r_star_rps": search.r_star,
|
||
"r_star_bound": (
|
||
"lower" if search.status == "generator_bound" else "exact"
|
||
),
|
||
"first_fail_rps": fail,
|
||
"first_fail_reason": fail_row["fail_reason"] if fail_row else None,
|
||
"steps": len(steps),
|
||
"resolution": rounded(search.resolution, 4),
|
||
"app_cpu_second_max": max(
|
||
(
|
||
r["app_cpu_second_max"]
|
||
for r in steps
|
||
if r["app_cpu_second_max"] is not None
|
||
),
|
||
default=None,
|
||
),
|
||
"writes_ok": writes_ok,
|
||
"writes_sent": writes_sent,
|
||
"write_chat_messages": write_count,
|
||
"read_chat_messages": read_count,
|
||
"host_power": host_end["power"],
|
||
"host_memory_free_percent": host_end["memory_free_percent"],
|
||
"host_swapouts": swapped or 0,
|
||
"other_containers": " ".join(others),
|
||
"flags": " ".join(flags),
|
||
}
|
||
for number, level in enumerate(p.fixed_rps[:2], start=1):
|
||
item = fixed.get(level)
|
||
if item is None:
|
||
continue
|
||
row[f"fixed_rps_{number}"] = level
|
||
for q in ("p50", "p95", "p99"):
|
||
row[f"{q}_fixed_{number}_ms"] = item[f"{q}_ms"]
|
||
row[f"pass_fixed_{number}"] = item["slo_pass"]
|
||
second = fixed.get(p.fixed_rps[1]) if len(p.fixed_rps) > 1 else None
|
||
if second is not None:
|
||
row["app_us_per_request_fixed_2"] = second["app_us_per_request"]
|
||
row["app_throttled_ratio_fixed_2"] = second["app_throttled_ratio"]
|
||
if not stand.stop():
|
||
raise BenchError(f"stand of cell {cell['cell']} was not removed")
|
||
self.results.add_search(row)
|
||
self.results.meta.setdefault("cells", {})[cell["cell"]] = {
|
||
"attempt": attempt,
|
||
"status": search.status,
|
||
"r_star_rps": search.r_star,
|
||
"host": {"start": host_start, "end": host_end},
|
||
"other_containers": others,
|
||
}
|
||
self.results.write_meta()
|
||
self.cell_seconds.append(time.monotonic() - self.cell_started)
|
||
self.say(
|
||
f"{tag}: R* {search.r_star} rps ({search.status}), first fail "
|
||
f"{fail} {row['first_fail_reason'] or ''}, resolution "
|
||
f"{row['resolution']}, {len(steps)} steps; writes {writes_ok} ok "
|
||
f"/ {write_count} stored / {writes_sent} sent"
|
||
+ (f" [{row['flags']}]" if row["flags"] else ""),
|
||
)
|
||
self.cell = None
|
||
|
||
def run_step(
|
||
self,
|
||
cell: dict[str, Any],
|
||
number: int,
|
||
level: int,
|
||
kind: str,
|
||
*,
|
||
previous: int | None,
|
||
instances: list[str],
|
||
addresses: dict[str, str],
|
||
limits: scaling.Limits,
|
||
) -> dict[str, Any]:
|
||
"""Ступень: один запуск k6 `mixed.js` под облегчённым сторожем,
|
||
разбор итога, окна сэмплера и журнала PostgreSQL, строки
|
||
`runs.csv` и `samples.csv` (решения 3, 7, 8)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
warmup = p.step_warmup_s if previous else p.first_warmup_s
|
||
self.step_no += 1
|
||
name = f"{cell['cell']}-a{self.attempt}-s{number:02d}"
|
||
self.stage = f"cell {cell['cell']} step {number} {kind} {level}"
|
||
self.progress()
|
||
before = stand.snapshot()
|
||
store_start = stand.message_count()
|
||
swap_before = swapouts()
|
||
env = {
|
||
"LEVEL": level,
|
||
"START": previous or 0,
|
||
"MIX": ",".join(f"{n}:{s:g}" for n, s in p.mix),
|
||
"WARMUP_S": warmup,
|
||
"MEASURE_S": p.step_measure_s,
|
||
"TIMEOUT": p.request_timeout_s,
|
||
"PRE_VUS_PER_RPS": p.prevus_per_rps,
|
||
"MAX_VUS_PER_RPS": p.vus_per_rps,
|
||
"SEED": p.seed + 1000 * cell["repeat"] + number,
|
||
"STEP": self.step_no,
|
||
"INSTANCES": ",".join(instances),
|
||
"ABORT_ERRORS": p.abort_error_ratio,
|
||
"ABORT_P95_MS": p.abort_p95_ms,
|
||
"NAME": name,
|
||
}
|
||
with Watcher(stand.replicas(), rss=False) as watcher:
|
||
proc = stand.k6("mixed.js", env, check=False)
|
||
died = watcher.app_died or bool(watcher.stand_died)
|
||
# 99 — пороги k6 с abortOnFail: перегрузка, ступень нарушила SLO
|
||
aborted = proc.returncode == 99
|
||
raw = self.results.dir / "raw" / f"{name}.json"
|
||
if (proc.returncode not in (0, 99) and not died) or not raw.exists():
|
||
raise BenchError(k6_failure("mixed.js", proc))
|
||
after = stand.snapshot()
|
||
store_end = stand.message_count()
|
||
swap_after = swapouts()
|
||
data = json.loads(raw.read_text(encoding="utf-8"))
|
||
k6row = scaling.parse_k6(
|
||
data,
|
||
level=level,
|
||
mix=list(p.mix),
|
||
measure_s=p.step_measure_s,
|
||
instances=instances,
|
||
)
|
||
t0 = k6row["t0_ms"]
|
||
w0 = int((t0 + warmup * 1000) * 1e6)
|
||
w1 = int(
|
||
min(
|
||
w0 + p.step_measure_s * 1e9,
|
||
(t0 + k6row["run_ms"]) * 1e6,
|
||
),
|
||
)
|
||
agg, seconds = scaling.analyze_samples(
|
||
scaling.parse_samples(*stand.sampler_logs()),
|
||
w0,
|
||
w1,
|
||
addresses=addresses,
|
||
)
|
||
pg_log = self.window_log(DB, w0, w1)
|
||
too_many = scaling.count_too_many(pg_log)
|
||
rps = k6row["rps"]
|
||
app = stand.app_state()
|
||
down = stand.down_storages()
|
||
swapped = (
|
||
swap_after - swap_before
|
||
if swap_after is not None and swap_before is not None
|
||
else None
|
||
)
|
||
row: dict[str, Any] = {
|
||
**k6row,
|
||
**agg,
|
||
"run": self.step_no,
|
||
"scenario": "mixed",
|
||
"size": p.sizes[0],
|
||
"repeat": cell["repeat"],
|
||
"rss_start_mib": mib(before["rss_kib"]),
|
||
"rss_end_mib": mib(after["rss_kib"]),
|
||
"rss_max_mib": mib(
|
||
max(
|
||
(v for v in (before["rss_kib"], after["rss_kib"]) if v),
|
||
default=None,
|
||
),
|
||
),
|
||
"mem_cgroup_end_mib": mib_bytes(after["mem_bytes"]),
|
||
"store_messages_start": store_start,
|
||
"store_messages_end": store_end,
|
||
"series": p.series,
|
||
"cell": cell["cell"],
|
||
"attempt": self.attempt,
|
||
"config": cell["config"],
|
||
"replicas": cell["replicas"],
|
||
"replica_cpus": cell["replica_cpus"],
|
||
"replica_memory": cell["replica_memory"],
|
||
"db_max_connections": cell["db_max_connections"],
|
||
"step": number,
|
||
"step_kind": kind,
|
||
"level": level,
|
||
"db_too_many_clients": too_many,
|
||
"app_us_per_request": scaling.per_request(
|
||
agg.get("app_cpu_cores"),
|
||
rps,
|
||
),
|
||
"db_us_per_request": scaling.per_request(
|
||
agg.get("db_cpu_cores"),
|
||
rps,
|
||
),
|
||
"host_swapouts": swapped,
|
||
}
|
||
outcome, reason = scaling.verdict(row, limits, aborted=aborted)
|
||
found = scaling.signs(row, limits)
|
||
flags = []
|
||
if app["oom"]:
|
||
flags.append("oom")
|
||
if not app["running"] or app["restarts"] or watcher.app_died:
|
||
flags.append("app_down")
|
||
if down or watcher.stand_died:
|
||
flags.append("stand_down")
|
||
if row["requests_failed"] or row["check_failed"]:
|
||
flags.append("errors")
|
||
if outcome == "fail":
|
||
flags.append("slo")
|
||
if aborted:
|
||
flags.append("abort")
|
||
if outcome == "generator":
|
||
flags.append("generator")
|
||
if swapped:
|
||
flags.append("host_swap")
|
||
row.update(
|
||
{
|
||
"slo_pass": None
|
||
if outcome == "generator"
|
||
else outcome == "pass",
|
||
"fail_reason": reason or None,
|
||
"signs": " ".join(found),
|
||
"flags": " ".join(flags),
|
||
},
|
||
)
|
||
self.results.add_step(row)
|
||
self.results.add_samples(
|
||
[
|
||
{
|
||
**second,
|
||
"cell": cell["cell"],
|
||
"attempt": self.attempt,
|
||
"step": number,
|
||
}
|
||
for second in seconds
|
||
],
|
||
)
|
||
codes = row["error_codes"]
|
||
if too_many or any(c.startswith("5") for c in codes.split()):
|
||
self.save_errors(name, w0, w1, pg_log, codes)
|
||
self.say(
|
||
f"cell {self.cell_index}/{self.runs_total} {cell['cell']} step "
|
||
f"{number:>2} {kind:<6} {level:>5} rps: "
|
||
f"{'PASS' if outcome == 'pass' else outcome.upper()}"
|
||
+ (f" ({reason})" if reason else "")
|
||
+ f" ok {fmt(rps)}/s p95 {fmt(row['p95_ms'], 1)} p99 "
|
||
f"{fmt(row['p99_ms'], 1)} ms err {fmt((row['slo_error_ratio'] or 0) * 100, 2)}%"
|
||
f" app {fmt(row.get('app_cpu_cores'), 2)} db "
|
||
f"{fmt(row.get('db_cpu_cores'), 2)} k6 "
|
||
f"{fmt(row.get('generator_cpu_cores'), 2)} cores"
|
||
+ (f" [{' '.join(found)}]" if found else "")
|
||
+ (f" codes {codes}" if codes else "")
|
||
+ f" ~{fmt_left(self.eta())} left",
|
||
)
|
||
self.progress(last={k: row.get(k) for k in RUN_FIELDS})
|
||
if "app_down" in flags or "oom" in flags or "stand_down" in flags:
|
||
raise BenchError(
|
||
f"stand down after step {name} ({' '.join(flags)}): series "
|
||
"stopped; continue with RESUME=1 after analysis",
|
||
)
|
||
row["_outcome"] = outcome
|
||
row["_writes_sent"] = k6row["write_sent_total"]
|
||
row["_writes_ok"] = (
|
||
k6row["write_sent_total"] - k6row["write_failed_total"]
|
||
)
|
||
# прерванные досрочно запросы k6 не учитывает, а сервер мог их
|
||
# выполнить: допуск сверки — выделенные VU ступени
|
||
row["_slack"] = k6row["vus_max"] if aborted else 0
|
||
return row
|
||
|
||
def window_log(self, container: str, w0: int, w1: int) -> str:
|
||
"""Журнал контейнера за окно замера (часы ВМ, нс)."""
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"logs",
|
||
"-t",
|
||
"--since",
|
||
f"{w0 / 1e9:.3f}",
|
||
"--until",
|
||
f"{w1 / 1e9:.3f}",
|
||
container,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def save_errors(
|
||
self,
|
||
name: str,
|
||
w0: int,
|
||
w1: int,
|
||
pg_log: str,
|
||
codes: str,
|
||
) -> None:
|
||
"""Доказательства 5xx и отказов «too many clients» ступени (H3):
|
||
строки PostgreSQL, API и LB окна, без секретов — строки со словом
|
||
`password` отбрасываются. Длинная строка (traceback) сокращается
|
||
до начала и конца: класс исключения — в конце (verify S4)."""
|
||
|
||
def pick(text: str, pattern: re.Pattern[str], limit: int) -> list[str]:
|
||
lines = [
|
||
shorten(line)
|
||
for line in text.splitlines()
|
||
if pattern.search(line) and "password" not in line.lower()
|
||
]
|
||
return lines[:limit]
|
||
|
||
out = [
|
||
f"# {name}: коды ошибок окна k6 — {codes or 'нет'}",
|
||
"",
|
||
"## PostgreSQL",
|
||
*pick(pg_log, ERRORS_PG_RE, 10),
|
||
"",
|
||
"## API",
|
||
]
|
||
for replica in self.stand.replicas():
|
||
out += pick(self.window_log(replica, w0, w1), ERRORS_API_RE, 10)
|
||
out += [
|
||
"",
|
||
"## HAProxy",
|
||
*pick(self.window_log(LB, w0, w1), ERRORS_LB_RE, 10),
|
||
]
|
||
path = self.results.dir / "raw" / f"{name}-errors.log"
|
||
path.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||
|
||
|
||
# ------------------------------------------------------------------ CLI
|
||
|
||
|
||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--profile", choices=sorted(PROFILES), required=True)
|
||
parser.add_argument("--label", required=True)
|
||
# отладка стенда: переопределения пишутся в паспорт как overrides
|
||
parser.add_argument("--sizes", help="через запятую, например 120000")
|
||
parser.add_argument("--concurrency", help="через запятую")
|
||
parser.add_argument("--repeats", type=int)
|
||
parser.add_argument("--memory", help="лимит памяти, например 64m")
|
||
parser.add_argument(
|
||
"--modes",
|
||
help="режимы профиля отказа через запятую: stop,kill,pause",
|
||
)
|
||
parser.add_argument(
|
||
"--from",
|
||
dest="rollout_from",
|
||
help="commit прежнего образа профиля замены",
|
||
)
|
||
parser.add_argument(
|
||
"--resume",
|
||
action="store_true",
|
||
help="серии трека 1: продолжить прерванную метку",
|
||
)
|
||
parser.add_argument(
|
||
"--fixed",
|
||
help="серии трека 1, отладка: фиксированные уровни через запятую",
|
||
)
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def acquire_lock() -> int | None:
|
||
"""Блокировка стенда: один прогон любого профиля (решение 5 design
|
||
change `scaling-bench`). ОС снимает её при выходе, и при `kill -9`
|
||
тоже. Занята — None."""
|
||
fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
|
||
try:
|
||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except BlockingIOError:
|
||
os.close(fd)
|
||
return None
|
||
return fd
|
||
|
||
|
||
def resume_problems(
|
||
label_dir: Path,
|
||
profile: Profile,
|
||
git: dict[str, Any],
|
||
) -> list[str]:
|
||
"""Условия продолжения серии (решение 5): метка есть и не завершена,
|
||
тот же профиль, тот же commit, дерево чистое без своей метки."""
|
||
path = label_dir / "meta.json"
|
||
if not path.is_file():
|
||
return [f"{path.relative_to(ROOT)} not found: nothing to resume"]
|
||
meta = json.loads(path.read_text(encoding="utf-8"))
|
||
found = []
|
||
if meta.get("status") == "complete":
|
||
found.append("label is complete: nothing to resume")
|
||
name = meta.get("profile", {}).get("name")
|
||
if name != profile.name:
|
||
found.append(f"label profile {name}, requested {profile.name}")
|
||
commit = meta.get("git", {}).get("commit")
|
||
if commit != git["commit"]:
|
||
found.append(f"label commit {commit}, HEAD {git['commit']}")
|
||
if git["dirty"]:
|
||
found.append("working tree has uncommitted changes")
|
||
if meta.get("overrides"):
|
||
found.append(f"label has debug overrides {meta['overrides']}")
|
||
return found
|
||
|
||
|
||
def rollout_commit_problem(commit: str) -> str | None:
|
||
"""Commit прежнего образа профиля замены есть в истории; иначе —
|
||
сообщение с commit и подменой `--from` (design change `attachments`,
|
||
решение 14)."""
|
||
found = subprocess.run(
|
||
[
|
||
"git",
|
||
"-C",
|
||
str(ROOT),
|
||
"rev-parse",
|
||
"--verify",
|
||
"--quiet",
|
||
f"{commit}^{{commit}}",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if found.returncode == 0:
|
||
return None
|
||
return (
|
||
f"rollout commit {commit!r} is not in this repository history: "
|
||
"update Profile.rollout_from (.claude/rules/bench.md) or pass "
|
||
"--from <commit> for debugging"
|
||
)
|
||
|
||
|
||
def apply_overrides(
|
||
profile: Profile,
|
||
args: argparse.Namespace,
|
||
) -> tuple[Profile, dict[str, Any]]:
|
||
overrides: dict[str, Any] = {}
|
||
if args.sizes:
|
||
overrides["sizes"] = tuple(int(x) for x in args.sizes.split(","))
|
||
if args.concurrency:
|
||
overrides["concurrency"] = tuple(
|
||
int(x) for x in args.concurrency.split(",")
|
||
)
|
||
if args.repeats:
|
||
overrides["repeats"] = args.repeats
|
||
if args.memory:
|
||
overrides["memory"] = args.memory
|
||
if args.modes:
|
||
modes = tuple(args.modes.split(","))
|
||
if not profile.is_failover or set(modes) - set(failover.BOUNDS):
|
||
raise SystemExit(f"--modes {args.modes}: failover profiles only")
|
||
overrides["failover_modes"] = modes
|
||
if args.rollout_from:
|
||
if not profile.is_rollout:
|
||
raise SystemExit(
|
||
f"--from {args.rollout_from}: rollout profiles only",
|
||
)
|
||
overrides["rollout_from"] = args.rollout_from
|
||
if args.fixed:
|
||
if not profile.is_series:
|
||
raise SystemExit(f"--fixed {args.fixed}: track 1 series only")
|
||
overrides["fixed_rps"] = tuple(int(x) for x in args.fixed.split(","))
|
||
return replace(profile, **overrides), overrides
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
args = parse_args(argv)
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label):
|
||
print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True)
|
||
return 2
|
||
# один прогон стенда любого профиля: второй — отказ до стенда
|
||
lock = acquire_lock()
|
||
if lock is None:
|
||
print(
|
||
"another bench run of the stand is in progress "
|
||
"(bench/.bench.lock is held): wait for it to finish",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
profile, overrides = apply_overrides(PROFILES[args.profile], args)
|
||
if args.resume and not profile.is_series:
|
||
print("--resume: track 1 series profiles only", flush=True)
|
||
return 2
|
||
if profile.is_rollout:
|
||
# commit методики — до стенда, а не на шестой секунде сборки
|
||
# (спек load-benchmark, «Прежний commit профиля замены»)
|
||
problem = rollout_commit_problem(profile.rollout_from)
|
||
if problem:
|
||
print(problem, flush=True)
|
||
return 2
|
||
# профиль отказа реплик — доказательство уровня 4, рядом с прогонами
|
||
# make test-failover (решение 7 design change graceful-failover)
|
||
kind = "failover" if profile.is_failover else "bench"
|
||
if profile.is_rollout:
|
||
# поочерёдная замена — доказательство уровня 4 (решение 8 design
|
||
# change rolling-update)
|
||
kind = "rollout"
|
||
label_dir = ROOT / "evidence" / kind / args.label
|
||
relative = label_dir.relative_to(ROOT)
|
||
if label_dir.exists() and not args.resume:
|
||
print(f"label directory {relative} already exists", flush=True)
|
||
return 2
|
||
# чистота дерева серии — без своей метки: её частичные результаты при
|
||
# продолжении не изменение кода (решение 5 design change scaling-bench)
|
||
git = git_info(exclude=str(relative) if profile.is_series else None)
|
||
if args.resume:
|
||
problems = resume_problems(label_dir, profile, git)
|
||
if problems:
|
||
print(
|
||
f"cannot resume {relative}:\n " + "\n ".join(problems),
|
||
flush=True,
|
||
)
|
||
return 2
|
||
state = host_state()
|
||
try:
|
||
warnings = host_warnings(state) + vm_warnings(profile)
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 2
|
||
plan_budget: dict[str, Any] | None = None
|
||
peak_budget: dict[str, Any] | None = None
|
||
if profile.is_series or profile.sample_resources:
|
||
info = sh(
|
||
"docker",
|
||
"info",
|
||
"--format",
|
||
"{{.NCPU}} {{.MemTotal}}",
|
||
check=False,
|
||
).split()
|
||
vm_cpus, vm_memory = (
|
||
(int(v) for v in info) if len(info) == 2 else (0, 0)
|
||
)
|
||
if profile.sample_resources:
|
||
# профили уровня «4» с сэмплером: сумма лимитов CPU на пике не
|
||
# больше CPU ВМ (design change profile-resources, решение 8)
|
||
peak_budget = level4_budget(profile, vm_cpus, vm_memory)
|
||
if peak_budget["cpus"] > vm_cpus:
|
||
print(
|
||
f"CPU limits at peak ({peak_budget['peak_replicas']} "
|
||
f"replicas, sampler) {peak_budget['cpus']:g} CPU exceed "
|
||
f"docker VM CPUs ({vm_cpus})",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
if profile.is_series:
|
||
# сумма лимитов CPU конфигурации не больше CPU ВМ (решение 1)
|
||
plan_budget = budget(profile, vm_cpus, vm_memory)
|
||
over = [
|
||
f"{key}: {item['cpus']:g} CPU"
|
||
for key, item in plan_budget.items()
|
||
if item["cpus"] > vm_cpus
|
||
]
|
||
if over:
|
||
print(
|
||
f"CPU limits exceed docker VM CPUs ({vm_cpus}): "
|
||
+ ", ".join(over),
|
||
flush=True,
|
||
)
|
||
return 2
|
||
command = " ".join(["uv run python bench/run.py", *argv])
|
||
others = sorted(
|
||
sh("docker", "ps", "--format", "{{.Names}}", check=False).split(),
|
||
)
|
||
if args.resume:
|
||
meta = json.loads(
|
||
(label_dir / "meta.json").read_text(encoding="utf-8"),
|
||
)
|
||
meta["status"] = "incomplete"
|
||
meta["reason"] = "running"
|
||
meta["host_warnings"] = meta.get("host_warnings", []) + warnings
|
||
meta.setdefault("resumes", []).append(
|
||
{
|
||
"at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"command": command,
|
||
"git": git,
|
||
"host_state": state,
|
||
"other_containers": others,
|
||
},
|
||
)
|
||
else:
|
||
meta = {
|
||
"label": args.label,
|
||
"status": "incomplete",
|
||
"reason": "running",
|
||
"started_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"command": command,
|
||
"git": git,
|
||
"overrides": overrides,
|
||
"host_state": {"start": state},
|
||
"host_warnings": warnings,
|
||
# чужие контейнеры в той же VM: стенд их не трогает, но они
|
||
# шумят
|
||
"other_containers": others,
|
||
}
|
||
if plan_budget is not None:
|
||
meta["budget"] = plan_budget
|
||
if peak_budget is not None:
|
||
meta["resources"] = {"budget": peak_budget}
|
||
stand_check = Stand(profile, label_dir)
|
||
started = time.monotonic()
|
||
# на macOS монотонные часы во сне стоят, а реальные идут: разница —
|
||
# сколько хост проспал посреди прогона (решение 9 design change
|
||
# track2-backup-restore)
|
||
started_wall = time.time()
|
||
|
||
def say(text: str) -> None:
|
||
elapsed = int(time.monotonic() - started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
for warning in warnings:
|
||
say(f"WARNING: {warning}")
|
||
if overrides:
|
||
say(f"WARNING: profile overrides {overrides}: debug run")
|
||
if git["dirty"]:
|
||
say("NOTE: working tree has uncommitted changes (dirty: true)")
|
||
try:
|
||
if profile.is_series:
|
||
# блокировка взята: остатки — от прерванного прогона, серия
|
||
# убирает их сама (решение 5 design change scaling-bench)
|
||
stand_check.remove_leftovers(say)
|
||
else:
|
||
stand_check.check_free()
|
||
image = stand_check.build_image()
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 1
|
||
say(f"image {APP_IMAGE} built ({image[:19]}…), compose project {PROJECT}")
|
||
if args.resume:
|
||
# ID индекса образа в хранилище containerd меняется при каждой
|
||
# сборке и на том же коде (аттестации); код один — проверено по
|
||
# commit и чистому дереву выше
|
||
meta["resumes"][-1]["app_image_id"] = image
|
||
else:
|
||
meta.update(static_passport(profile))
|
||
results = Results(label_dir, meta)
|
||
bench = Bench(profile, args.label, results)
|
||
bench.started = started
|
||
bench.resume = args.resume
|
||
code = 0
|
||
|
||
def terminate(*_: object) -> None:
|
||
raise KeyboardInterrupt
|
||
|
||
signal.signal(signal.SIGTERM, terminate)
|
||
try:
|
||
bench.run()
|
||
meta["status"] = "complete"
|
||
meta.pop("reason", None)
|
||
except KeyboardInterrupt:
|
||
meta["reason"] = "interrupted"
|
||
say("interrupted: partial results kept")
|
||
code = 130
|
||
except BenchError as exc:
|
||
meta["reason"] = str(exc).splitlines()[0]
|
||
say(f"bench failed: {exc}")
|
||
code = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку стенда
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if not bench.stand.stop():
|
||
say(f"cleanup incomplete: {leftovers()}")
|
||
code = code or 1
|
||
# fixture.json с токенами — только здесь, в результаты не попадает
|
||
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
|
||
meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds")
|
||
end_state = host_state()
|
||
meta["host_state"]["end"] = end_state
|
||
# выгрузка в swap за этот запуск (у продолжения — от его начала)
|
||
swapped = swap_warning(state, end_state)
|
||
if swapped is not None:
|
||
meta["host_warnings"].append(swapped)
|
||
say(f"WARNING: {swapped}")
|
||
delta = swapouts_delta(state, end_state)
|
||
if delta is not None:
|
||
meta["host_state"]["swapouts_delta"] = delta
|
||
slept = (time.time() - started_wall) - (time.monotonic() - started)
|
||
meta["host_state"]["slept_seconds"] = round(max(0.0, slept))
|
||
if slept > SLEEP_WARN_S:
|
||
warning = f"host slept {slept:.0f}s during the run: rerun"
|
||
meta["host_warnings"].append(warning)
|
||
say(f"WARNING: {warning}")
|
||
results.write_meta()
|
||
results.write_summary()
|
||
bench.stage = "done" if code == 0 else meta.get("reason", "stopped")
|
||
bench.progress()
|
||
say(
|
||
f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, "
|
||
f"results in {relative}",
|
||
)
|
||
return code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|