k6 --console-output appends, so a run killed before cleanup left <run>.warm.k6 in .work and the resumed warmup counted 24 of 12. Remove the console file before each k6 run and a stale .work at start; ignore evidence/**/.work/. A telemetry storage that exits during the lookup (Tempo in metrics-r3: name no longer resolvable) now fails the run with the container state, OOMKilled flag and its log tail saved to raw/; Tempo requests of the lookup are limited to 3 at a time and memory peaks after the lookup are recorded. Raise the light classes of the telemetry cost load by 15 % (read 126.5, write 23, me 34.5, card 34.5 per second): on a quiet host `full` kept the replicas at 28-29.5 % CPU, below the 30 % calibration bound. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10046 lines
403 KiB
Python
10046 lines
403 KiB
Python
"""Нагрузочный стенд: матрица размеров истории и конкуренции.
|
||
|
||
Запуск — `make bench` / `make bench_quick`, профиль отказа реплик —
|
||
`make bench_failover` / `make bench_failover_quick`, поочерёдной замены —
|
||
`make bench_rollout` / `make bench_rollout_quick`, серии трека 1 —
|
||
`make bench_scaling SERIES=…` / `make bench_scaling_quick`. Стенд —
|
||
HAProxy, N реплик API, PostgreSQL и Valkey: проект compose `mtusi-bench` из
|
||
`docker-compose.yml` с override `bench/bench.compose.yml`. Методика —
|
||
design changes `load-bench-harness`, `bench-postgres`, `graceful-failover`,
|
||
`rolling-update` и `scaling-bench`, инварианты — `.claude/rules/bench.md`.
|
||
Только stdlib:
|
||
приложение не импортируется, его нагружает k6 по HTTP через LB; к
|
||
PostgreSQL стенда оркестратор ходит только для обслуживания и измерений
|
||
(VACUUM, размеры, планы запроса, сверка записей).
|
||
"""
|
||
|
||
import argparse
|
||
import csv
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import platform
|
||
import random
|
||
import re
|
||
import secrets
|
||
import shutil
|
||
import signal
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable
|
||
from dataclasses import asdict, dataclass, replace
|
||
from datetime import UTC, datetime, timedelta
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import failover
|
||
import overload
|
||
import plots
|
||
import reconcile
|
||
import resources
|
||
import rollout
|
||
import scaling
|
||
import telemetry_cost
|
||
import uploads
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
K6_IMAGE = (
|
||
"grafana/k6:2.3.0@sha256:"
|
||
"9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34"
|
||
)
|
||
APP_IMAGE = "mtusi-messenger:bench"
|
||
# профиль поочерёдной замены: образ прежнего commit (git archive), с него
|
||
# стартуют реплики; тег снимается при уборке
|
||
FROM_IMAGE = "mtusi-messenger:bench-from"
|
||
ROLLOUT_SCRIPT = ROOT / "scripts" / "rolling-update.sh"
|
||
PROJECT = "mtusi-bench"
|
||
NETWORK = "mtusi-bench"
|
||
# реплики api без постоянного имени: их список — docker compose ps
|
||
APP_SERVICE = "api"
|
||
# готовность процесса реплики изнутри контейнера — команда HEALTHCHECK
|
||
# образа (Dockerfile); у реплик стенда healthcheck выключен
|
||
APP_READY_PROBE = (
|
||
"import os, urllib.request; urllib.request.urlopen("
|
||
"'http://127.0.0.1:%s/health/ready' % os.environ.get('APP__PORT', "
|
||
"'8080'), timeout=2)"
|
||
)
|
||
LB = "mtusi-bench-haproxy"
|
||
DB = "mtusi-bench-postgres"
|
||
CACHE = "mtusi-bench-valkey"
|
||
# S3 стенда (container_name в bench.compose.yml) и его volume: запускает
|
||
# только профиль uploads, volume пересоздаётся перед каждым прогоном
|
||
S3 = "mtusi-bench-s3"
|
||
S3_VOLUME = f"{PROJECT}_s3_data"
|
||
# брокер стенда описан, ни один профиль его не запускает: реплика к нему не
|
||
# обращается (design change pipeline-reliability, решение 16); start_broker —
|
||
# для профиля, который будет мерить обработку (трек 1)
|
||
BROKER_SERVICE = "rabbitmq"
|
||
# образ контейнера в паспорте: манифест платформы из метки compose, у контейнера
|
||
# без неё — `.Image`. Само `.Image` при том же манифесте различается: оно
|
||
# совпадает с индексом тега на момент создания (`docker.md`, «Обновление кода
|
||
# на стенде»; design change pipeline-reliability, решение 16)
|
||
CONTAINER_IMAGE = (
|
||
'{{with index .Config.Labels "com.docker.compose.image"}}{{.}}'
|
||
"{{else}}{{.Image}}{{end}}"
|
||
)
|
||
# окружение реплики по варианту профиля uploads: override поверх
|
||
# bench.compose.yml в рабочем каталоге метки
|
||
VARIANT_COMPOSE = "variant.compose.yml"
|
||
K6 = "mtusi-bench-k6"
|
||
# профиль сверки (design change track2-reconcile-cost, решения 3 и 8):
|
||
# загрузчик объектов на образе стенда, разовые проходы уборщика
|
||
# `<префикс><номер>`, override S3 с портом метрик шлюза и override образа
|
||
# уборщика «до» — в рабочем каталоге метки
|
||
FILL = "mtusi-bench-fill"
|
||
RECONCILE_PASS = "mtusi-bench-reconcile-"
|
||
S3_METRICS_COMPOSE = "s3-metrics.compose.yml"
|
||
BEFORE_CLEANER_COMPOSE = "cleaner-before.compose.yml"
|
||
# профиль очереди (design change track1-queue-overload): постоянные контейнеры
|
||
# конвейера (`container_name` в bench.compose.yml), второй worker блоков w2-*,
|
||
# генератор файлов классов, override с плагином брокера и политикой worker
|
||
BROKER = "mtusi-bench-rabbitmq"
|
||
RELAY = "mtusi-bench-relay"
|
||
WORKER = "mtusi-bench-worker"
|
||
WORKER_2 = "mtusi-bench-worker-2"
|
||
CLEANER = "mtusi-bench-cleaner"
|
||
IMAGES = "mtusi-bench-images"
|
||
QUEUE_COMPOSE = ROOT / "bench" / "queue.compose.yml"
|
||
# сколько ждать нуля незавершённых после разогрева, с
|
||
QUEUE_WARM_WAIT_S = 120
|
||
# телеметрия стенда (design change track1-telemetry-cost, решения 1, 9, 13):
|
||
# override включённого SDK, сервисы телеметрии в порядке старта (Prometheus с
|
||
# healthcheck — первым), их контейнеры и volume, сеть хранилищ, контейнеры
|
||
# выборки и размера volume
|
||
TELEMETRY_COMPOSE = ROOT / "bench" / "telemetry.compose.yml"
|
||
TELEMETRY_SERVICES = ("prometheus", "otel-collector", "loki", "tempo")
|
||
TELEMETRY_CONTAINERS = {
|
||
service: f"mtusi-bench-{service}" for service in TELEMETRY_SERVICES
|
||
}
|
||
TELEMETRY_VOLUMES = {
|
||
name: f"{PROJECT}_{name}_data" for name in telemetry_cost.DISKS
|
||
}
|
||
TELEMETRY_NETWORK = "mtusi-bench-telemetry"
|
||
LOOKUP = "mtusi-bench-lookup"
|
||
DU = "mtusi-bench-du"
|
||
LOOKUP_SCRIPT = "/bench/telemetry_lookup.py"
|
||
# строк журнала погибшего сервиса телеметрии в raw/ (до уборки стенда)
|
||
TELEMETRY_LOG_TAIL = 200
|
||
MIB = 1024 * 1024
|
||
# профили, которые принимают выключатель телеметрии (решение 13)
|
||
TELEMETRY_SWITCH_PROFILES = (
|
||
"uploads",
|
||
"uploads_quick",
|
||
"queue",
|
||
"queue_quick",
|
||
)
|
||
# лимиты сервисов телеметрии: поле Profile → сервис основного compose
|
||
TELEMETRY_LIMIT_FIELDS = {
|
||
"otel-collector": ("collector_cpus", "collector_memory"),
|
||
"prometheus": ("prometheus_cpus", "prometheus_memory"),
|
||
"loki": ("loki_cpus", "loki_memory"),
|
||
"tempo": ("tempo_cpus", "tempo_memory"),
|
||
}
|
||
# сэмплер серий трека 1: образ PostgreSQL стенда, --cgroupns host
|
||
# (решение 7 design change scaling-bench)
|
||
SAMPLER = "mtusi-bench-sampler"
|
||
SAMPLER_SCRIPT = ROOT / "bench" / "sampler.sh"
|
||
# один прогон стенда любого профиля: flock держится до выхода процесса
|
||
LOCK_FILE = ROOT / "bench" / ".bench.lock"
|
||
# строки журналов окна с 5xx и отказами (`raw/<ступень>-errors.log`):
|
||
# PostgreSQL — отказ и FATAL; API — ответ 5xx или запись error; HAProxy —
|
||
# поле статуса 5xx или ответ без сервера. Подстроки «53300» и « 5»
|
||
# ловили метки времени и request_id (verify S4)
|
||
ERRORS_PG_RE = re.compile(r"too many clients|FATAL")
|
||
ERRORS_API_RE = re.compile(r'"status": 5\d\d\b|"level": "error"')
|
||
ERRORS_LB_RE = re.compile(r"\s5\d\d\s|<NOSRV>")
|
||
# длинная строка журнала: начало и конец (класс исключения — в конце)
|
||
ERRORS_HEAD = 300
|
||
ERRORS_TAIL = 300
|
||
# второе хранилище этапа восстановления: контейнер, volume и имя в сети
|
||
# стенда (по нему API переключается на восстановленную базу)
|
||
RESTORE_DB = "mtusi-bench-postgres-restore"
|
||
RESTORE_VOLUME = "mtusi-bench_restore"
|
||
RESTORE_HOST = "postgres-restore"
|
||
# таблицы данных приложения: сверка числа строк после восстановления
|
||
DATA_TABLES = ("users", "chats", "chat_members", "sessions", "messages")
|
||
# контейнеры стенда с постоянным именем по ролям: лимиты, паспорт; реплики
|
||
# api — отдельно (Stand.replicas)
|
||
CONTAINERS = {"postgres": DB, "valkey": CACHE, "haproxy": LB}
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "bench" / "bench.compose.yml",
|
||
)
|
||
# подстановки compose вместо .env рабочего стенда
|
||
ENV_FILE = ROOT / "bench" / "bench.env"
|
||
# что из окружения хоста нужно самому docker compose; остальное (в том
|
||
# числе POSTGRES_* из shell) в стенд не попадает
|
||
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "DOCKER_HOST", "DOCKER_CONFIG")
|
||
# части имён переменных окружения API, которые в паспорт (stand.app_env) не
|
||
# пишутся: пароли Postgres и Valkey (PASS — и пароль брокера, если реплика его
|
||
# когда-нибудь получит), секреты и ключи доступа S3
|
||
SECRET_ENV_MARKS = ("PASS", "SECRET", "ACCESS_KEY")
|
||
# статистика HAProxy изнутри контейнера LB (как make lb_stats): порт 8404
|
||
# наружу не опубликован, curl в образе нет, bash есть
|
||
LB_STATS = (
|
||
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
|
||
'printf "GET /stats;csv HTTP/1.0\\r\\n\\r\\n" >&3 && cat <&3'
|
||
)
|
||
SUCCESS_METRIC = "http_req_duration{expected_response:true}"
|
||
# предупреждение о VM: лимиты памяти стенда + этот запас больше памяти VM
|
||
VM_RESERVE_GIB = 1.5
|
||
# предупреждение о хосте: реальное время прогона больше монотонного, с
|
||
SLEEP_WARN_S = 5
|
||
# psql в контейнере Postgres от пользователя образа, без пароля; SQL — stdin
|
||
PSQL = (
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq -v ON_ERROR_STOP=1'
|
||
)
|
||
# настройки Postgres в паспорте: от них зависят план и стоимость записи
|
||
PG_SETTINGS = (
|
||
"shared_buffers",
|
||
"effective_cache_size",
|
||
"work_mem",
|
||
"maintenance_work_mem",
|
||
"max_connections",
|
||
"synchronous_commit",
|
||
"fsync",
|
||
"wal_level",
|
||
"checkpoint_timeout",
|
||
"max_wal_size",
|
||
"random_page_cost",
|
||
"jit",
|
||
"autovacuum",
|
||
"plan_cache_mode",
|
||
"shared_preload_libraries",
|
||
)
|
||
# auto_explain на этапе плана; вне его log_min_duration = -1 (выключен)
|
||
EXPLAIN_SETTINGS = {
|
||
"auto_explain.log_min_duration": "0",
|
||
"auto_explain.log_analyze": "on",
|
||
"auto_explain.log_buffers": "on",
|
||
"auto_explain.log_timing": "on",
|
||
"auto_explain.log_format": "text",
|
||
}
|
||
# пауза после pg_reload_conf(): SIGHUP доходит до соединений пула API
|
||
RELOAD_PAUSE_S = 1.0
|
||
# индекс, который использовал план приложения
|
||
INDEX_NODE_RE = re.compile(
|
||
r"Index Only Scan(?: Backward)? using (\w+)"
|
||
r"|Index Scan(?: Backward)? using (\w+)"
|
||
r"|Bitmap Index Scan on (\w+)",
|
||
)
|
||
# условие курсора в запросе приложения: сравнение кортежей
|
||
OR_CURSOR_RE = re.compile(
|
||
r"\(messages\.created_at, messages\.id\) > "
|
||
r"\((\$\d+::[A-Z][A-Z ]*[A-Z]), (\$\d+::UUID)\)",
|
||
)
|
||
# длина текста сообщения по контракту (contracts/openapi.json, text
|
||
# maxLength): проба записи проверяет, что такое сообщение вставляется
|
||
MAX_TEXT_LENGTH = 4000
|
||
# сообщение максимальной длины из несжимаемых символов CJK (~12 КБ UTF-8,
|
||
# md5 номера символа — детерминировано): вставка в DO, итог — NOTICE
|
||
LONG_TEXT_PROBE = f"""DO $$
|
||
BEGIN
|
||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||
SELECT gen_random_uuid(), chat_id, sender_id,
|
||
(SELECT string_agg(
|
||
chr(19968 + ('x' || substr(md5(i::text), 1, 4))::bit(16)::int % 20000),
|
||
'')
|
||
FROM generate_series(1, {MAX_TEXT_LENGTH}) i),
|
||
now() + interval '1 hour'
|
||
FROM probe_src LIMIT 1;
|
||
RAISE NOTICE 'long-text: ok';
|
||
EXCEPTION WHEN others THEN
|
||
RAISE NOTICE 'long-text: %', SQLERRM;
|
||
END $$;
|
||
"""
|
||
|
||
RUN_FIELDS = [
|
||
"run",
|
||
"scenario",
|
||
"size",
|
||
"concurrency",
|
||
"repeat",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_start_mib",
|
||
"rss_end_mib",
|
||
"rss_max_mib",
|
||
"mem_cgroup_end_mib",
|
||
"store_messages_start",
|
||
"store_messages_end",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"check_failed",
|
||
# лаба 2: новые столбцы — только в конец
|
||
"db_cpu_cores",
|
||
# лаба 3, серии трека 1 (решение 8 design change scaling-bench); у
|
||
# строк матрицы пусто
|
||
*scaling.SCALING_FIELDS,
|
||
]
|
||
PREFILL_FIELDS = [
|
||
"size",
|
||
"chunk",
|
||
"messages_before",
|
||
"messages_after",
|
||
"duration_s",
|
||
"requests_ok",
|
||
"requests_failed",
|
||
"rps",
|
||
"error_ratio",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"p99_ms",
|
||
"max_ms",
|
||
"app_cpu_cores",
|
||
"rss_end_mib",
|
||
"mem_cgroup_end_mib",
|
||
"generator_cpu_cores",
|
||
"flags",
|
||
"db_cpu_cores",
|
||
# серии трека 1: ячейка и попытка порции
|
||
"cell",
|
||
"attempt",
|
||
]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Profile:
|
||
name: str
|
||
sizes: tuple[int, ...]
|
||
concurrency: tuple[int, ...]
|
||
repeats: int
|
||
read_warmup_s: int
|
||
read_measure_s: int
|
||
write_warmup: int
|
||
write_measure: int
|
||
# порция ~30 с при ~850 сообщ/с на Postgres: прогресс наполнения
|
||
# печатается после порции, а спек требует строку не реже раза в минуту
|
||
prefill_chunk: int = 25_000
|
||
prefill_vus: int = 32
|
||
text_length: int = 64
|
||
page_limit: int = 50
|
||
walk_limit: int = 100
|
||
cursor_pool: int = 1000
|
||
members: int = 10
|
||
seed: int = 20260930
|
||
# зерно прогрева = seed + сдвиг: прогрев не должен запрашивать ровно
|
||
# те страницы, с которых начнётся замер
|
||
warmup_seed_shift: int = 1_000_003
|
||
generator_cpu_limit: float = 4.0
|
||
cpus: str = "1"
|
||
memory: str = "2g"
|
||
# лаба 2: лимиты хранилищ (swap = память) и срок сессии стенда; идут в
|
||
# compose переменными BENCH_* (bench/bench.compose.yml). Решение 2
|
||
# design change bench-postgres
|
||
db_cpus: str = "4"
|
||
db_memory: str = "3g"
|
||
cache_cpus: str = "2"
|
||
cache_memory: str = "512m"
|
||
session_ttl_seconds: int = 21600
|
||
# трек 2: фоновые чаты уровнями (число чатов, сообщений в чате), то же
|
||
# на каждом размере; варианты индекса истории (имя, столбцы) для
|
||
# планов и стоимости записи; пробы записи — строк и повторов.
|
||
# Решения 1, 4, 5 design change track2-query-plans
|
||
background: tuple[tuple[int, int], ...] = ()
|
||
index_variants: tuple[tuple[str, str], ...] = (
|
||
("chat_only", "(chat_id)"),
|
||
("covering", "(chat_id, created_at, id) INCLUDE (sender_id, text)"),
|
||
)
|
||
write_probe_rows: int = 20_000
|
||
write_probe_repeats: int = 3
|
||
# трек 2, часть 2: backup и восстановление в новое хранилище в конце
|
||
# размера; pg_restore -j, запись через HTTP во время backup. Решение 1
|
||
# design change track2-backup-restore
|
||
restore: bool = False
|
||
restore_jobs: int = 4
|
||
restore_write_load: int = 5_000
|
||
restore_write_vus: int = 10
|
||
# лаба 3: реплики api за HAProxy (лимиты cpus/memory — на каждую) и
|
||
# лимиты LB; в compose — BENCH_API_REPLICAS и BENCH_LB_*. Решение 6
|
||
# design change graceful-failover
|
||
replicas: int = 1
|
||
lb_cpus: str = "1"
|
||
lb_memory: str = "256m"
|
||
# профиль отказа реплик (решение 7): режимы фаз, постоянная
|
||
# интенсивность записи и чтения в секунду, окна фазы (до действия,
|
||
# отказ до возврата, после возврата), порог восстановления, VU k6 на
|
||
# сценарий: заранее созданные и предел. Зависшая реплика держит
|
||
# половину запросов до timeout server LB (10 с): 50/с × 10 с = 500 VU
|
||
# на сценарий. Без запаса k6 создаёт VU на ходу и пропускает итерации
|
||
# (самопроверка: 352 из 2 000 за окно pause), и интенсивность перестаёт
|
||
# быть постоянной
|
||
failover_modes: tuple[str, ...] = ()
|
||
failover_write_rate: int = 0
|
||
failover_read_rate: int = 0
|
||
failover_before_s: int = 0
|
||
failover_down_s: int = 0
|
||
failover_after_s: int = 0
|
||
failover_success: float = 0.99
|
||
failover_pre_vus: int = 600
|
||
failover_max_vus: int = 1500
|
||
# профиль поочерёдной замены (решение 8 design change rolling-update):
|
||
# commit прежнего образа, окна — до запуска scripts/rolling-update.sh,
|
||
# бюджет замены и после него. Постоянная нагрузка, порог доли успешных
|
||
# и VU k6 — поля failover_* выше: тот же k6/failover.js
|
||
rollout_from: str = ""
|
||
rollout_before_s: int = 0
|
||
rollout_budget_s: int = 0
|
||
rollout_after_s: int = 0
|
||
# профили уровня «4» (failover, rollout): сэмплер в режиме scan на всё
|
||
# время нагрузки — CPU и память по компонентам и окнам. Признак
|
||
# наблюдения, нагрузку и окна не меняет; лимиты и интервал сэмплера —
|
||
# поля sampler_* ниже (design change profile-resources, решение 2)
|
||
sample_resources: bool = False
|
||
# профиль загрузок (design change upload-memory, решение 10): варианты
|
||
# кода (bench/uploads.py, VARIANTS), commit образа «до», тела — имя и
|
||
# высота PNG из шума шириной 1024, число одновременных клиентов, разгон
|
||
# и окно прогона, окружение варианта «без лимита», интервал сэмплера
|
||
# peak. Лимиты S3 — у всех профилей (compose требует их при разборе),
|
||
# но S3 запускает только этот профиль. Код успеха варианта (design change
|
||
# async-attachments, решение 16): 202; у образа commit до перехода 4 → 5
|
||
# был 201, а «до» лабы 6 — commit переноса, код лабы 5 — тоже 202
|
||
uploads_variants: tuple[str, ...] = ()
|
||
uploads_before: str = ""
|
||
uploads_success_codes: tuple[tuple[str, int], ...] = (
|
||
("before", 202),
|
||
("unlimited", 202),
|
||
("limited", 202),
|
||
)
|
||
uploads_sizes: tuple[tuple[str, int], ...] = ()
|
||
uploads_clients: tuple[int, ...] = ()
|
||
uploads_ramp_s: int = 3
|
||
uploads_window_s: int = 20
|
||
uploads_unlimited_env: tuple[tuple[str, str], ...] = (
|
||
("UPLOADS__MAX_CONCURRENT", "1000"),
|
||
("UPLOADS__MAX_WAITING", "0"),
|
||
)
|
||
uploads_sample_interval_s: float = 0.2
|
||
s3_cpus: str = "1"
|
||
s3_memory: str = "512m"
|
||
# брокер, relay и worker: лимиты рабочего стенда без swap — у всех профилей
|
||
# (compose требует их при разборе); не запускает их ни один профиль: стенд
|
||
# меряет реплику, задания ждут в PostgreSQL (design change
|
||
# pipeline-reliability, решение 16)
|
||
broker_cpus: str = "1"
|
||
broker_memory: str = "512m"
|
||
relay_cpus: str = "0.25"
|
||
relay_memory: str = "256m"
|
||
worker_cpus: str = "1"
|
||
worker_memory: str = "1g"
|
||
# уборщик (сервис cleaner): лимиты рабочего стенда без swap — у всех
|
||
# профилей (compose требует их при разборе), запускает его только
|
||
# профиль сверки (design change track2-reconcile-cost, решение 1)
|
||
cleaner_cpus: str = "0.25"
|
||
cleaner_memory: str = "256m"
|
||
# профиль сверки (design change track2-reconcile-cost, решения 1–3,
|
||
# 7–10): этапы — (имя, свежий стенд, объектов всего, размер объекта,
|
||
# ячейки bench/reconcile.py CELLS); отказ по пределу — commit образа
|
||
# «до», согласованных объектов, кандидатов по шагам и предел
|
||
# изменений; grace проходов и запас после последнего PUT; CPU уборщика
|
||
# управляемого опыта; фоновая нагрузка (сценарий → запросов в
|
||
# секунду), скачиваемых вложений, тихие окна; загрузчик объектов;
|
||
# интервал сэмплера; стадии гипотез для сводки
|
||
reconcile_stages: tuple[tuple[Any, ...], ...] = ()
|
||
reconcile_before: str = ""
|
||
reconcile_s4_clean: int = 0
|
||
reconcile_s4_candidates: tuple[int, ...] = ()
|
||
reconcile_s4_max_changes: int = 100
|
||
reconcile_grace_s: int = 30
|
||
reconcile_fresh_wait_s: int = 2
|
||
reconcile_fast_cpus: str = "1"
|
||
reconcile_default_rate: float = 2000
|
||
reconcile_background: tuple[tuple[str, int], ...] = (
|
||
("download", 300),
|
||
("write", 50),
|
||
("read", 50),
|
||
)
|
||
reconcile_downloads: int = 5000
|
||
# предел VU сценария фоновой нагрузки (MAX_VUS background.js): им же
|
||
# ограничено число итераций, прерванных остановкой SIGINT
|
||
reconcile_bg_max_vus: int = 600
|
||
reconcile_quiet_s: int = 30
|
||
reconcile_fill_processes: int = 3
|
||
reconcile_fill_threads: int = 24
|
||
reconcile_sample_interval_s: float = 0.5
|
||
reconcile_metrics_port: int = 9327
|
||
reconcile_h1_stages: tuple[str, ...] = ()
|
||
reconcile_h2_stages: tuple[str, ...] = ()
|
||
reconcile_h3_stages: tuple[str, ...] = ()
|
||
reconcile_s4_base: str = ""
|
||
# профиль очереди (design change track1-queue-overload, решения 5, 6, 8, 14):
|
||
# конфигурации — (блок, класс, конкуренция, prefetch, низкая λ, высокая λ,
|
||
# окно over, предел незавершённых или None, действие); окна; файлы классов от
|
||
# зерна; VU заранее; контрольный поток; разогрев; выборка SHA-256; предел по
|
||
# умолчанию для вердиктов
|
||
queue_configs: tuple[tuple[Any, ...], ...] = ()
|
||
queue_before_s: int = 10
|
||
queue_after_s: int = 10
|
||
queue_recovery_max_s: int = 420
|
||
queue_image_seed: int = 20261101
|
||
queue_low_vus: int = 4
|
||
queue_control: tuple[tuple[str, int], ...] = (
|
||
("download", 5),
|
||
("write", 10),
|
||
("read", 10),
|
||
)
|
||
queue_control_max_vus: int = 60
|
||
queue_warm: int = 12
|
||
queue_sample: int = 10
|
||
queue_default_limit: int = 250
|
||
# выключатель телеметрии профилей загрузок и очереди (design change
|
||
# track1-telemetry-cost, решение 13): `on` — сервисы телеметрии и SDK с
|
||
# настройками рабочего стенда; не отладочное переопределение
|
||
telemetry: str = "off"
|
||
# лимиты сервисов телеметрии — рабочего стенда (сверка с основным
|
||
# docker-compose.yml до стенда, решение 1); контейнер выборки
|
||
collector_cpus: str = "0.5"
|
||
collector_memory: str = "256m"
|
||
prometheus_cpus: str = "0.5"
|
||
prometheus_memory: str = "384m"
|
||
loki_cpus: str = "0.5"
|
||
loki_memory: str = "512m"
|
||
tempo_cpus: str = "0.5"
|
||
tempo_memory: str = "640m"
|
||
lookup_cpus: str = "0.5"
|
||
lookup_memory: str = "256m"
|
||
# профиль цены телеметрии (решения 2, 8, 9, 14): варианты, интенсивности
|
||
# классов (в секунду), окна (разогрев, чистое, отказов), моменты
|
||
# блокировок от начала окна отказов и их длительность, порог медленного
|
||
# запроса, доля head, интервал metrics, выборка GET /users/me, ожидание
|
||
# незавершённых заданий. Лёгкие классы — +15 % к калибровке (110, 20, 30,
|
||
# 30): на тихом хосте `full` метки baseline дал 28–29,5 % CPU реплики при
|
||
# нижней границе 30 % (design, «Уточнения apply», п. 16)
|
||
telemetry_variants: tuple[str, ...] = ()
|
||
telemetry_rates: tuple[tuple[str, float], ...] = (
|
||
("read", 126.5),
|
||
("write", 23),
|
||
("me", 34.5),
|
||
("card", 34.5),
|
||
("download", 10),
|
||
("upload", 1),
|
||
("upload_bad", 0.1),
|
||
("broken", 0.2),
|
||
("register", 0.2),
|
||
)
|
||
telemetry_windows: tuple[int, int, int] = (30, 90, 30)
|
||
telemetry_locks: tuple[int, ...] = (5, 15, 25)
|
||
telemetry_lock_s: float = 0.6
|
||
telemetry_slow_ms: float = 250
|
||
telemetry_head_ratio: float = 0.1
|
||
telemetry_metrics_interval: float = 30
|
||
telemetry_sample_me: int = 200
|
||
telemetry_drain_s: int = 60
|
||
# серии трека 1 (решения 1–6, 14 design change scaling-bench): имя
|
||
# серии, конфигурации (реплик, CPU и память реплики, max_connections
|
||
# PostgreSQL), чаты записи, смесь, фиксированные уровни, лестница и
|
||
# поиск R*, окна ступени, SLO, досрочный выход, VU k6 на запрос/с,
|
||
# лимиты генератора и сэмплера, зерно порядка ячеек. У прочих
|
||
# профилей — пусто: их поведение не меняется. max_connections — 100,
|
||
# как у образа; у ячейки серии — из конфигурации
|
||
db_max_connections: int = 100
|
||
series: str = ""
|
||
# у серии db — пятый элемент: CPU PostgreSQL конфигурации
|
||
configs: tuple[tuple[Any, ...], ...] = ()
|
||
write_chats: int = 1
|
||
mix: tuple[tuple[str, float], ...] = ()
|
||
fixed_rps: tuple[int, ...] = ()
|
||
ladder_base: int = 100
|
||
step_factor: float = 1.25
|
||
start_rps_per_replica: int = 300
|
||
bisect_steps: int = 2
|
||
max_steps: int = 12
|
||
first_warmup_s: int = 15
|
||
step_warmup_s: int = 5
|
||
step_measure_s: int = 20
|
||
cooldown_s: int = 10
|
||
slo_p95_ms: float = 200
|
||
slo_error_ratio: float = 0.01
|
||
abort_error_ratio: float = 0.5
|
||
abort_p95_ms: float = 2000
|
||
request_timeout_s: int = 15
|
||
prevus_per_rps: float = 0.05
|
||
vus_per_rps: float = 0.2
|
||
generator_cpus: str = ""
|
||
generator_memory: str = ""
|
||
sampler_cpus: str = "0.25"
|
||
sampler_memory: str = "128m"
|
||
sample_interval_s: float = 1.0
|
||
order_seed: int = 20261011
|
||
|
||
@property
|
||
def is_series(self) -> bool:
|
||
return bool(self.series)
|
||
|
||
@property
|
||
def background_total(self) -> int:
|
||
return sum(count * size for count, size in self.background)
|
||
|
||
@property
|
||
def is_failover(self) -> bool:
|
||
return bool(self.failover_modes)
|
||
|
||
@property
|
||
def is_rollout(self) -> bool:
|
||
return bool(self.rollout_from)
|
||
|
||
@property
|
||
def is_uploads(self) -> bool:
|
||
return bool(self.uploads_variants)
|
||
|
||
@property
|
||
def is_reconcile(self) -> bool:
|
||
return bool(self.reconcile_stages)
|
||
|
||
@property
|
||
def is_queue(self) -> bool:
|
||
return bool(self.queue_configs)
|
||
|
||
@property
|
||
def is_telemetry_cost(self) -> bool:
|
||
return bool(self.telemetry_variants)
|
||
|
||
@property
|
||
def runs_pipeline(self) -> bool:
|
||
"""S3, брокер, relay, worker и уборщик: профили очереди и цены
|
||
телеметрии меряют обработку."""
|
||
return self.is_queue or self.is_telemetry_cost
|
||
|
||
@property
|
||
def telemetry_on(self) -> bool:
|
||
"""Выключатель профилей загрузок и очереди (у цены телеметрии режим
|
||
задаёт вариант прогона)."""
|
||
return self.telemetry == "on"
|
||
|
||
|
||
# интенсивности загрузок по классу: низкая и высокая, в секунду (решение 5)
|
||
QUEUE_RATES = {
|
||
"s": (1.5, 8),
|
||
"m": (1.5, 6),
|
||
"lj": (1.2, 6),
|
||
"lp": (0.5, 2.5),
|
||
}
|
||
# VU заранее окна over: тело — open на VU, создание на ходу теряет итерации
|
||
QUEUE_OVER_VUS = 20
|
||
QUEUE_ACCEPT_VUS = 80
|
||
|
||
|
||
def queue_config(
|
||
block: str,
|
||
klass: str,
|
||
concurrency: int,
|
||
prefetch: int,
|
||
*,
|
||
high: float | None = None,
|
||
over_s: int = 20,
|
||
limit: int | None = None,
|
||
action: str = "",
|
||
) -> tuple[Any, ...]:
|
||
low, default_high = QUEUE_RATES[klass]
|
||
return (
|
||
block,
|
||
klass,
|
||
concurrency,
|
||
prefetch,
|
||
low,
|
||
default_high if high is None else high,
|
||
over_s,
|
||
limit,
|
||
action,
|
||
)
|
||
|
||
|
||
def queue_matrix() -> tuple[tuple[Any, ...], ...]:
|
||
"""12 конфигураций решения 5: (2, 2) и (4, 4) на всех классах, (1, 1) на `m`
|
||
и `lp`, (2, 256) на `s` и `lp`."""
|
||
configs = []
|
||
for klass in QUEUE_RATES:
|
||
configs.append(queue_config("matrix", klass, 2, 2))
|
||
configs.append(queue_config("matrix", klass, 4, 4))
|
||
configs += [queue_config("matrix", k, 1, 1) for k in ("m", "lp")]
|
||
configs += [queue_config("matrix", k, 2, 256) for k in ("s", "lp")]
|
||
return tuple(configs)
|
||
|
||
|
||
def queue_blocks() -> tuple[tuple[Any, ...], ...]:
|
||
"""Блоки решения 8: семь конфигураций."""
|
||
return (
|
||
queue_config("gate-on", "s", 2, 2, high=20, over_s=25),
|
||
queue_config("gate-off", "s", 2, 2, high=20, over_s=25, limit=0),
|
||
queue_config("accept", "m", 2, 2, high=15, over_s=45),
|
||
queue_config("w2-p2", "s", 2, 2, action="second-worker"),
|
||
queue_config("w2-p256", "s", 2, 256, action="second-worker"),
|
||
queue_config(
|
||
"kill-worker",
|
||
"s",
|
||
2,
|
||
2,
|
||
over_s=25,
|
||
action="kill-worker",
|
||
),
|
||
queue_config(
|
||
"restart-broker",
|
||
"s",
|
||
2,
|
||
2,
|
||
over_s=25,
|
||
action="restart-broker",
|
||
),
|
||
)
|
||
|
||
|
||
def queue_quick_configs() -> tuple[tuple[Any, ...], ...]:
|
||
"""Быстрый профиль (решение 14): предел 40 и без него, `s` (2, 2), `lp`
|
||
(4, 4), `kill-worker`, `w2-p256`, окна over 10 с."""
|
||
return (
|
||
queue_config("matrix", "s", 2, 2, over_s=10),
|
||
queue_config("matrix", "lp", 4, 4, over_s=10),
|
||
queue_config("gate-on", "s", 2, 2, high=20, over_s=10, limit=40),
|
||
queue_config("gate-off", "s", 2, 2, high=20, over_s=10, limit=0),
|
||
queue_config(
|
||
"kill-worker",
|
||
"s",
|
||
2,
|
||
2,
|
||
over_s=10,
|
||
action="kill-worker",
|
||
),
|
||
queue_config(
|
||
"w2-p256",
|
||
"s",
|
||
2,
|
||
256,
|
||
over_s=10,
|
||
action="second-worker",
|
||
),
|
||
)
|
||
|
||
|
||
def queue_cell_configs(profile: Profile) -> list[dict[str, Any]]:
|
||
keys = (
|
||
"block",
|
||
"class",
|
||
"concurrency",
|
||
"prefetch",
|
||
"rate_low",
|
||
"rate_high",
|
||
"over_s",
|
||
"limit",
|
||
"action",
|
||
)
|
||
return [
|
||
{
|
||
**dict(zip(keys, item, strict=True)),
|
||
"before_s": profile.queue_before_s,
|
||
"after_s": profile.queue_after_s,
|
||
}
|
||
for item in profile.queue_configs
|
||
]
|
||
|
||
|
||
def queue_budget(
|
||
profile: Profile,
|
||
vm_cpus: int,
|
||
vm_memory: int,
|
||
) -> dict[str, Any]:
|
||
"""Сумма лимитов профиля очереди на пике (решение 10): реплики, хранилища,
|
||
брокер, relay, два worker, уборщик, сэмплер; генератор без лимита."""
|
||
cpu = profile.replicas * float(profile.cpus) + sum(
|
||
float(value)
|
||
for value in (
|
||
profile.db_cpus,
|
||
profile.lb_cpus,
|
||
profile.cache_cpus,
|
||
profile.s3_cpus,
|
||
profile.broker_cpus,
|
||
profile.relay_cpus,
|
||
profile.worker_cpus,
|
||
profile.worker_cpus,
|
||
profile.cleaner_cpus,
|
||
profile.sampler_cpus,
|
||
)
|
||
)
|
||
mem = profile.replicas * size_bytes(profile.memory) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.lb_memory,
|
||
profile.cache_memory,
|
||
profile.s3_memory,
|
||
profile.broker_memory,
|
||
profile.relay_memory,
|
||
profile.worker_memory,
|
||
profile.worker_memory,
|
||
profile.cleaner_memory,
|
||
profile.sampler_memory,
|
||
)
|
||
)
|
||
if profile.telemetry_on:
|
||
# выключатель телеметрии (design change track1-telemetry-cost,
|
||
# решение 13): сервисы телеметрии на пике
|
||
tel_cpu, tel_mem = telemetry_limits(profile)
|
||
cpu += tel_cpu
|
||
mem += tel_mem
|
||
return {
|
||
"cpus": round(cpu, 2),
|
||
"memory_gib": round(mem / 2**30, 2),
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_memory / 2**30, 2),
|
||
"generator": "no limit",
|
||
"telemetry": profile.telemetry,
|
||
}
|
||
|
||
|
||
def telemetry_limits(profile: Profile) -> tuple[float, int]:
|
||
"""Сумма лимитов CPU и памяти (байт) Collector и трёх хранилищ."""
|
||
cpu = sum(
|
||
float(getattr(profile, fields[0]))
|
||
for fields in TELEMETRY_LIMIT_FIELDS.values()
|
||
)
|
||
mem = sum(
|
||
size_bytes(getattr(profile, fields[1]))
|
||
for fields in TELEMETRY_LIMIT_FIELDS.values()
|
||
)
|
||
return cpu, mem
|
||
|
||
|
||
def telemetry_budget(
|
||
profile: Profile,
|
||
vm_cpus: int,
|
||
vm_memory: int,
|
||
) -> dict[str, Any]:
|
||
"""Сумма лимитов профиля цены телеметрии на пике (решение 14): стенд
|
||
очереди с одним worker, сервисы телеметрии и сэмплер; генератор без
|
||
лимита, выборка и размер volume — после нагрузки."""
|
||
tel_cpu, tel_mem = telemetry_limits(profile)
|
||
cpu = (
|
||
profile.replicas * float(profile.cpus)
|
||
+ tel_cpu
|
||
+ sum(
|
||
float(value)
|
||
for value in (
|
||
profile.db_cpus,
|
||
profile.lb_cpus,
|
||
profile.cache_cpus,
|
||
profile.s3_cpus,
|
||
profile.broker_cpus,
|
||
profile.relay_cpus,
|
||
profile.worker_cpus,
|
||
profile.cleaner_cpus,
|
||
profile.sampler_cpus,
|
||
)
|
||
)
|
||
)
|
||
mem = (
|
||
profile.replicas * size_bytes(profile.memory)
|
||
+ tel_mem
|
||
+ sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.lb_memory,
|
||
profile.cache_memory,
|
||
profile.s3_memory,
|
||
profile.broker_memory,
|
||
profile.relay_memory,
|
||
profile.worker_memory,
|
||
profile.cleaner_memory,
|
||
profile.sampler_memory,
|
||
)
|
||
)
|
||
)
|
||
return {
|
||
"cpus": round(cpu, 2),
|
||
"memory_gib": round(mem / 2**30, 2),
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_memory / 2**30, 2),
|
||
"generator": "no limit",
|
||
}
|
||
|
||
|
||
PROFILES = {
|
||
"full": Profile(
|
||
name="full",
|
||
# 500k вместо 1M лабы 1: прогон ~30 мин вместо ~45, а диапазон
|
||
# 10k → 500k (×50) уже показывает, растёт ли страница с историей
|
||
sizes=(10_000, 100_000, 500_000),
|
||
concurrency=(1, 10, 50),
|
||
repeats=3,
|
||
read_warmup_s=5,
|
||
read_measure_s=20,
|
||
write_warmup=500,
|
||
write_measure=5000,
|
||
# 100 000 фоновых сообщений: в таблице ≥110 000 на любом размере
|
||
background=((10, 5_000), (100, 400), (1_000, 10)),
|
||
),
|
||
"quick": Profile(
|
||
name="quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(1, 4, 16),
|
||
repeats=1,
|
||
read_warmup_s=2,
|
||
read_measure_s=5,
|
||
write_warmup=100,
|
||
write_measure=1000,
|
||
background=((2, 1_000), (10, 100), (50, 10)),
|
||
write_probe_rows=5_000,
|
||
),
|
||
# backup и восстановление на данных полной матрицы, без прогонов
|
||
"restore": Profile(
|
||
name="restore",
|
||
sizes=(10_000, 100_000, 500_000),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
background=((10, 5_000), (100, 400), (1_000, 10)),
|
||
restore=True,
|
||
),
|
||
"restore_quick": Profile(
|
||
name="restore_quick",
|
||
sizes=(1_000, 5_000, 10_000),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
background=((2, 1_000), (10, 100), (50, 10)),
|
||
write_probe_rows=5_000,
|
||
restore=True,
|
||
restore_write_load=1_000,
|
||
),
|
||
# отказ реплик под нагрузкой (уровень 4): две реплики с лимитами
|
||
# рабочего стенда (docker-compose.yml), 100 записей и 100 чтений в
|
||
# секунду, каждая реплика в каждом режиме. Решение 7 design change
|
||
# graceful-failover
|
||
"failover": Profile(
|
||
name="failover",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_modes=("stop", "kill", "pause"),
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
failover_before_s=15,
|
||
failover_down_s=20,
|
||
failover_after_s=15,
|
||
sample_resources=True,
|
||
),
|
||
"failover_quick": Profile(
|
||
name="failover_quick",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_modes=("stop", "kill"),
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
failover_before_s=5,
|
||
failover_down_s=10,
|
||
failover_after_s=8,
|
||
sample_resources=True,
|
||
),
|
||
# поочерёдная замена под нагрузкой (уровень 4): та же нагрузка и
|
||
# лимиты, что у failover; реплики стартуют на образе 9cc6c0c —
|
||
# commit переноса лабы 5 в лабу 6, до изменений лабы 6 — и заменяются
|
||
# на образ текущего checkout. Решение 8 design change rolling-update;
|
||
# правило выбора — bench.md
|
||
"rollout": Profile(
|
||
name="rollout",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
rollout_from="9cc6c0c",
|
||
rollout_before_s=15,
|
||
rollout_budget_s=90,
|
||
rollout_after_s=15,
|
||
sample_resources=True,
|
||
),
|
||
"rollout_quick": Profile(
|
||
name="rollout_quick",
|
||
sizes=(1_000,),
|
||
concurrency=(),
|
||
repeats=0,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
prefill_vus=8,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
failover_write_rate=100,
|
||
failover_read_rate=100,
|
||
rollout_from="9cc6c0c",
|
||
rollout_before_s=5,
|
||
rollout_budget_s=60,
|
||
rollout_after_s=8,
|
||
sample_resources=True,
|
||
),
|
||
# память реплики при одновременных загрузках (уровень 4 лабы 4): одна
|
||
# реплика с лимитами рабочего стенда без swap, S3 рабочего стенда; три
|
||
# варианта кода × тела 1 и 10 MiB × 1/10/20/40 клиентов × 3 повтора,
|
||
# на каждый прогон — новые volume S3 и реплика. «До» — 9cc6c0c, commit
|
||
# переноса лабы 5 в лабу 6: прежнего кода в истории лабы 6 нет, вариант
|
||
# показывает уже места загрузок, результат «до» — в отчёте лабы 4.
|
||
# Design change upload-memory, решение 10
|
||
"uploads": Profile(
|
||
name="uploads",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=3,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
uploads_variants=("before", "unlimited", "limited"),
|
||
uploads_before="9cc6c0c",
|
||
uploads_sizes=(("1MiB", 341), ("10MiB", 3409)),
|
||
uploads_clients=(1, 10, 20, 40),
|
||
),
|
||
"uploads_quick": Profile(
|
||
name="uploads_quick",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=1,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
uploads_variants=("unlimited", "limited"),
|
||
uploads_sizes=(("10MiB", 3409),),
|
||
uploads_clients=(1, 10, 40),
|
||
),
|
||
# стоимость сверки bucket и БД (трек 2 «5» лабы 4): LB, две реплики,
|
||
# хранилища и S3 с лимитами рабочего стенда, разовые проходы уборщика.
|
||
# Этапы A (1 MiB), B → C → D (4 KiB, набор растёт), затем отказ по
|
||
# пределу S4 на образе 9cc6c0c — commit переноса лабы 5 в лабу 6:
|
||
# прежнего кода в истории лабы 6 нет, результат «до» — в отчёте лабы 4 —
|
||
# и на checkout.
|
||
# Design change track2-reconcile-cost, решения 1–3, 7–10
|
||
"reconcile": Profile(
|
||
name="reconcile",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=3,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
reconcile_stages=(
|
||
("A", True, 1_000, 1024 * 1024, ("batch", "deep", "deep-fast")),
|
||
("B", True, 10_000, 4096, ("batch", "head", "deep")),
|
||
("C", False, 30_000, 4096, ("batch", "head")),
|
||
(
|
||
"D",
|
||
False,
|
||
100_000,
|
||
4096,
|
||
(
|
||
"batch",
|
||
"stand",
|
||
"stand-unlimited",
|
||
"fast-unlimited",
|
||
"fast-limited",
|
||
),
|
||
),
|
||
),
|
||
reconcile_before="9cc6c0c",
|
||
reconcile_s4_clean=10_000,
|
||
reconcile_s4_candidates=(1_000, 10_000),
|
||
reconcile_h1_stages=("B", "C", "D"),
|
||
reconcile_h2_stages=("B", "C"),
|
||
reconcile_h3_stages=("B", "A"),
|
||
reconcile_s4_base="B",
|
||
),
|
||
"reconcile_quick": Profile(
|
||
name="reconcile_quick",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=1,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
reconcile_stages=(
|
||
(
|
||
"Q",
|
||
True,
|
||
2_000,
|
||
4096,
|
||
(
|
||
"batch",
|
||
"head",
|
||
"deep",
|
||
"stand",
|
||
"stand-unlimited",
|
||
"fast-unlimited",
|
||
"fast-limited",
|
||
),
|
||
),
|
||
),
|
||
reconcile_before="9cc6c0c",
|
||
reconcile_s4_clean=2_000,
|
||
reconcile_s4_candidates=(300,),
|
||
reconcile_quiet_s=5,
|
||
reconcile_downloads=1_000,
|
||
reconcile_h1_stages=("Q",),
|
||
reconcile_h2_stages=("Q",),
|
||
reconcile_h3_stages=("Q", ""),
|
||
reconcile_s4_base="Q",
|
||
),
|
||
# очередь под перегрузкой (трек 1 «5» лабы 5): LB, две реплики, хранилища, S3,
|
||
# брокер, relay, worker и уборщик с лимитами рабочего стенда; настройки ×
|
||
# классы × 3 повтора и блоки. Design change track1-queue-overload, решения 5–8
|
||
"queue": Profile(
|
||
name="queue",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=3,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
queue_configs=(*queue_matrix(), *queue_blocks()),
|
||
),
|
||
"queue_quick": Profile(
|
||
name="queue_quick",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=1,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
queue_configs=queue_quick_configs(),
|
||
queue_before_s=5,
|
||
queue_after_s=5,
|
||
queue_default_limit=40,
|
||
),
|
||
# цена телеметрии (трек 1 «5» лабы 6): стенд профиля очереди и, в
|
||
# вариантах с телеметрией, Collector и хранилища рабочего стенда; семь
|
||
# вариантов × 3 повтора кругами, свежий стенд на прогон. Design change
|
||
# track1-telemetry-cost, решения 1, 2, 8, 14
|
||
"telemetry": Profile(
|
||
name="telemetry",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=3,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
telemetry_variants=telemetry_cost.ORDER,
|
||
),
|
||
# быстрый: все варианты, кроме копии A/A, по одному разу, короткие окна
|
||
"telemetry_quick": Profile(
|
||
name="telemetry_quick",
|
||
sizes=(),
|
||
concurrency=(),
|
||
repeats=1,
|
||
read_warmup_s=0,
|
||
read_measure_s=0,
|
||
write_warmup=0,
|
||
write_measure=0,
|
||
members=1,
|
||
cpus="1",
|
||
memory="512m",
|
||
db_cpus="2",
|
||
db_memory="1g",
|
||
cache_cpus="0.5",
|
||
cache_memory="128m",
|
||
replicas=2,
|
||
telemetry_variants=tuple(
|
||
v for v in telemetry_cost.ORDER if v != "full-aa"
|
||
),
|
||
telemetry_windows=(10, 30, 15),
|
||
telemetry_locks=(3, 9),
|
||
telemetry_sample_me=100,
|
||
),
|
||
}
|
||
|
||
|
||
def scaling_profile(
|
||
name: str,
|
||
series: str,
|
||
configs: tuple[tuple[Any, ...], ...],
|
||
**fields: Any,
|
||
) -> Profile:
|
||
"""Профиль серии трека 1: общие поля всех серий в одном месте, чтобы
|
||
методика серий не расходилась (решения 1–3, 6, 14 design change
|
||
`scaling-bench`). Лимиты хранилищ и LB — как у рабочего стенда, k6 —
|
||
2 CPU / 1 ГиБ, сэмплер — 0,25 / 128 МиБ; поля реплик (`replicas`,
|
||
`cpus`, `memory`) заменяет конфигурация ячейки."""
|
||
common: dict[str, Any] = {
|
||
"name": name,
|
||
"sizes": (10_000,),
|
||
"concurrency": (),
|
||
"repeats": 3,
|
||
"read_warmup_s": 0,
|
||
"read_measure_s": 0,
|
||
"write_warmup": 0,
|
||
"write_measure": 0,
|
||
"members": 10,
|
||
"cpus": "1",
|
||
"memory": "512m",
|
||
"db_cpus": "2",
|
||
"db_memory": "1g",
|
||
"cache_cpus": "0.5",
|
||
"cache_memory": "128m",
|
||
"lb_cpus": "1",
|
||
"lb_memory": "256m",
|
||
"generator_cpu_limit": 1.8,
|
||
"series": series,
|
||
"configs": configs,
|
||
"write_chats": 10,
|
||
"mix": (("read", 0.7), ("write", 0.2), ("me", 0.1)),
|
||
"fixed_rps": (250, 500),
|
||
"generator_cpus": "2",
|
||
"generator_memory": "1g",
|
||
# все VU — заранее, как предел (найдено в apply): с 0,05 k6 создавал
|
||
# VU на ходу при росте задержек и пропускал итерации у R* при
|
||
# свободном пределе — недосланные шли в ошибки SLO от генератора
|
||
"prevus_per_rps": 0.2,
|
||
}
|
||
return Profile(**{**common, **fields})
|
||
|
||
|
||
B_CONFIGS = tuple((k, "1", "512m", 100) for k in range(1, 7))
|
||
# серия db: CPU PostgreSQL у 1, 2 и 4 реплик и контроль 4 × 1 при 2 CPU
|
||
DB_CPUS_TIGHT = "0.5"
|
||
DB_CONFIGS = (
|
||
(1, "1", "512m", 100, DB_CPUS_TIGHT),
|
||
(2, "1", "512m", 100, DB_CPUS_TIGHT),
|
||
(4, "1", "512m", 100, DB_CPUS_TIGHT),
|
||
(4, "1", "512m", 100, "2"),
|
||
)
|
||
PROFILES.update(
|
||
{
|
||
# фиксированный бюджет приложения 4 CPU / 2 ГиБ
|
||
"scaling_a": scaling_profile(
|
||
"scaling_a",
|
||
"a",
|
||
((1, "4", "2g", 100), (2, "2", "1g", 100), (4, "1", "512m", 100)),
|
||
),
|
||
# рост: 1…6 реплик по 1 CPU / 512 МиБ (решение 1: до 6 — бюджет ВМ)
|
||
"scaling_b": scaling_profile("scaling_b", "b", B_CONFIGS),
|
||
# граница пула: 4 реплики при max_connections 100 и 50
|
||
"scaling_pool": scaling_profile(
|
||
"scaling_pool",
|
||
"pool",
|
||
((4, "1", "512m", 100), (4, "1", "512m", 50)),
|
||
),
|
||
# «до/после» и A/A: 1 × 1 и 6 × 1 серии B (решение 11)
|
||
"scaling_ab": scaling_profile(
|
||
"scaling_ab",
|
||
"ab",
|
||
((1, "1", "512m", 100), (6, "1", "512m", 100)),
|
||
),
|
||
# самопроверка: две конфигурации, второй предел соединений
|
||
"scaling_quick": scaling_profile(
|
||
"scaling_quick",
|
||
"quick",
|
||
((1, "1", "512m", 100), (2, "1", "512m", 50)),
|
||
sizes=(2_000,),
|
||
repeats=1,
|
||
fixed_rps=(250,),
|
||
first_warmup_s=8,
|
||
step_warmup_s=3,
|
||
step_measure_s=8,
|
||
bisect_steps=1,
|
||
max_steps=6,
|
||
),
|
||
# насыщение PostgreSQL управляемым опытом (verify S2, указание
|
||
# пользователя 2026-10-11): 1, 2 и 4 реплики при узком лимите CPU
|
||
# PostgreSQL и контроль — 4 реплики при прежних 2 CPU. Пятый
|
||
# элемент конфигурации — CPU PostgreSQL; лимит — по калибровке
|
||
# (design, «Найдено в verify»)
|
||
"scaling_db": scaling_profile(
|
||
"scaling_db",
|
||
"db",
|
||
DB_CONFIGS,
|
||
),
|
||
# подмножество серии db для «до/после» change scaling-improvement
|
||
# (решение пользователя 2026-10-11 сократить замеры): основная
|
||
# ячейка 4 × 1 при 0,5 CPU и контроль 4 × 1 при 2 CPU, те же поля и
|
||
# 3 повтора. Имя серии и состав конфигураций к методике сравнения
|
||
# не относятся: метки db4 сравнимы с метками db по общим ячейкам
|
||
"scaling_db4": scaling_profile(
|
||
"scaling_db4",
|
||
"db4",
|
||
DB_CONFIGS[2:4],
|
||
),
|
||
# самопроверка серии db и калибровка лимита: 2 и 4 реплики при
|
||
# узком лимите, один повтор, короткие окна
|
||
"scaling_db_quick": scaling_profile(
|
||
"scaling_db_quick",
|
||
"db_quick",
|
||
DB_CONFIGS[1:3],
|
||
sizes=(2_000,),
|
||
repeats=1,
|
||
fixed_rps=(250,),
|
||
first_warmup_s=8,
|
||
step_warmup_s=3,
|
||
step_measure_s=8,
|
||
bisect_steps=1,
|
||
max_steps=6,
|
||
),
|
||
},
|
||
)
|
||
|
||
|
||
def cell_profile(profile: Profile, cell: dict[str, Any]) -> Profile:
|
||
"""Профиль ячейки: реплики, max_connections и (серия db) CPU
|
||
PostgreSQL из конфигурации."""
|
||
return replace(
|
||
profile,
|
||
replicas=cell["replicas"],
|
||
cpus=cell["replica_cpus"],
|
||
memory=cell["replica_memory"],
|
||
db_max_connections=cell["db_max_connections"],
|
||
db_cpus=cell.get("db_cpus") or profile.db_cpus,
|
||
)
|
||
|
||
|
||
def budget(profile: Profile, vm_cpus: int, vm_memory: int) -> dict[str, Any]:
|
||
"""Сумма лимитов CPU и памяти стенда и генератора по конфигурациям
|
||
серии (решение 1): реплики, PostgreSQL, LB, Valkey, k6, сэмплер."""
|
||
result = {}
|
||
for config in profile.configs:
|
||
parts = scaling.unpack_config(config)
|
||
replicas, cpus, memory, _, db_cpus = parts
|
||
key = scaling.config_key(*parts)
|
||
cpu = replicas * float(cpus) + sum(
|
||
float(value)
|
||
for value in (
|
||
db_cpus or profile.db_cpus,
|
||
profile.lb_cpus,
|
||
profile.cache_cpus,
|
||
profile.generator_cpus or "0",
|
||
profile.sampler_cpus,
|
||
)
|
||
)
|
||
mem = replicas * size_bytes(memory) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.lb_memory,
|
||
profile.cache_memory,
|
||
profile.generator_memory or "0",
|
||
profile.sampler_memory,
|
||
)
|
||
)
|
||
result[key] = {
|
||
"cpus": round(cpu, 2),
|
||
"memory_gib": round(mem / 2**30, 2),
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_memory / 2**30, 2),
|
||
}
|
||
return result
|
||
|
||
|
||
def level4_budget(
|
||
profile: Profile,
|
||
vm_cpus: int,
|
||
vm_memory: int,
|
||
) -> dict[str, Any]:
|
||
"""Сумма лимитов CPU и памяти профиля уровня «4» на пике (решение 8
|
||
design change `profile-resources`): реплики (у `rollout` — на одну
|
||
больше: на пике замены работает третья), PostgreSQL, LB, Valkey,
|
||
сэмплер; генератор — только если у него есть лимит. Миграция идёт до
|
||
третьей реплики и пик не поднимает."""
|
||
peak = profile.replicas + (1 if profile.is_rollout else 0)
|
||
cpu = peak * float(profile.cpus) + sum(
|
||
float(value)
|
||
for value in (
|
||
profile.db_cpus,
|
||
profile.lb_cpus,
|
||
profile.cache_cpus,
|
||
profile.generator_cpus or "0",
|
||
profile.sampler_cpus,
|
||
)
|
||
)
|
||
mem = peak * size_bytes(profile.memory) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.lb_memory,
|
||
profile.cache_memory,
|
||
profile.generator_memory or "0",
|
||
profile.sampler_memory,
|
||
)
|
||
)
|
||
return {
|
||
"cpus": round(cpu, 2),
|
||
"memory_gib": round(mem / 2**30, 2),
|
||
"peak_replicas": peak,
|
||
"vm_cpus": vm_cpus,
|
||
"vm_memory_gib": round(vm_memory / 2**30, 2),
|
||
"generator": (
|
||
f"{profile.generator_cpus} CPU / {profile.generator_memory}"
|
||
if profile.generator_cpus
|
||
else "no limit"
|
||
),
|
||
}
|
||
|
||
|
||
class BenchError(Exception):
|
||
"""Стенд не может продолжать: сообщение для человека."""
|
||
|
||
|
||
# ---------------------------------------------------------------- утилиты
|
||
|
||
|
||
def sh(*args: str, check: bool = True) -> str:
|
||
"""Команда без shell; stdout без хвостовых пробелов.
|
||
|
||
`LC_ALL=C`: при русской локали sysctl печатает `0,00M`, а вывод
|
||
разбирается регулярками с точкой.
|
||
"""
|
||
proc = subprocess.run(
|
||
args,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
env={**os.environ, "LC_ALL": "C"},
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"{' '.join(args[:4])}…: exit {proc.returncode}\n"
|
||
f"{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
|
||
def stop_stream(proc: subprocess.Popen[bytes]) -> None:
|
||
"""Остановить поток `docker events`: SIGTERM, 10 с на выход, затем
|
||
SIGKILL — зависший CLI не держит уборку стенда."""
|
||
proc.terminate()
|
||
try:
|
||
proc.wait(timeout=10)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill()
|
||
proc.wait()
|
||
|
||
|
||
def hashlib_sha256(path: Path) -> str:
|
||
"""SHA-256 файла конфигурации для паспорта."""
|
||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
||
|
||
def human(n: int) -> str:
|
||
"""10000 → 10k, 1000000 → 1M."""
|
||
if n >= 1_000_000 and n % 1_000_000 == 0:
|
||
return f"{n // 1_000_000}M"
|
||
if n >= 1000 and n % 1000 == 0:
|
||
return f"{n // 1000}k"
|
||
return str(n)
|
||
|
||
|
||
def fmt_left(seconds: float) -> str:
|
||
seconds = max(0, int(seconds))
|
||
hours, rest = divmod(seconds, 3600)
|
||
minutes, secs = divmod(rest, 60)
|
||
if hours:
|
||
return f"{hours}h{minutes:02d}m"
|
||
if minutes:
|
||
return f"{minutes}m{secs:02d}s"
|
||
return f"{secs}s"
|
||
|
||
|
||
def mib(kib: float | None) -> float | None:
|
||
return None if kib is None else round(kib / 1024, 1)
|
||
|
||
|
||
def mib_bytes(value: int | None) -> float | None:
|
||
return None if value is None else round(value / 2**20, 1)
|
||
|
||
|
||
def size_bytes(text: str) -> int:
|
||
"""Лимит памяти docker (`512m`, `2g`) в байтах."""
|
||
match = re.fullmatch(r"(\d+(?:\.\d+)?)([kmg]?)b?", text.strip().lower())
|
||
if not match:
|
||
raise BenchError(f"bad memory limit {text!r}: use 512m or 2g")
|
||
power = {"": 0, "k": 1, "m": 2, "g": 3}[match.group(2)]
|
||
return int(float(match.group(1)) * 1024**power)
|
||
|
||
|
||
def fmt(value: float | None, digits: int = 0) -> str:
|
||
return "—" if value is None else f"{value:.{digits}f}"
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 3) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def chat_sizes(profile: Profile, size: int) -> list[int]:
|
||
"""Размеры чатов наполнения: читаемый, затем фоновые по уровням."""
|
||
return [size] + [
|
||
chat_size
|
||
for count, chat_size in profile.background
|
||
for _ in range(count)
|
||
]
|
||
|
||
|
||
def fill_schedule(sizes: list[int]) -> list[int]:
|
||
"""Номер чата для каждого сообщения наполнения.
|
||
|
||
Чат с s сообщениями получает позиции (k + 0,5) / s, k = 0…s−1; все
|
||
позиции сортируются, при равенстве — по номеру чата. Сообщения
|
||
каждого чата равномерно размазаны по наполнению, порядок
|
||
детерминирован (решение 2 design change `track2-query-plans`).
|
||
"""
|
||
slots = sorted(
|
||
((k + 0.5) / size, chat)
|
||
for chat, size in enumerate(sizes)
|
||
for k in range(size)
|
||
)
|
||
return [chat for _, chat in slots]
|
||
|
||
|
||
def or_cursor(text: str) -> str:
|
||
"""Запрос приложения с условием курсора через OR вместо кортежей.
|
||
|
||
Остальное — список столбцов, ORDER BY, LIMIT — как у приложения. Нет
|
||
шаблона — приложение изменило запрос, тихой копии SQL не делаем.
|
||
"""
|
||
new, count = OR_CURSOR_RE.subn(
|
||
r"(messages.created_at > \1 OR "
|
||
r"(messages.created_at = \1 AND messages.id > \2))",
|
||
text,
|
||
)
|
||
if count != 1:
|
||
raise BenchError(
|
||
"history query has no (created_at, id) row comparison: the app "
|
||
"query changed, update OR_CURSOR_RE",
|
||
)
|
||
return new
|
||
|
||
|
||
# ------------------------------------------------------------ хост и паспорт
|
||
|
||
|
||
def swapouts() -> int | None:
|
||
"""Счётчик выгрузок страниц в swap (`vm_stat`, macOS); иначе None."""
|
||
if platform.system() != "Darwin":
|
||
return None
|
||
match = re.search(r"Swapouts:\s+(\d+)", sh("vm_stat", check=False))
|
||
return int(match.group(1)) if match else None
|
||
|
||
|
||
def host_state() -> dict[str, Any]:
|
||
"""Swap, выгрузки в swap, свободная память и питание (macOS); на
|
||
другой ОС — null. Занятый swap — не помеха; помеха — прирост
|
||
`swapouts` за прогон (решение 12 design change `scaling-bench`)."""
|
||
state: dict[str, Any] = {
|
||
"swap_used_mib": None,
|
||
"swap_total_mib": None,
|
||
"swapouts": None,
|
||
"memory_free_percent": None,
|
||
"power": None,
|
||
"battery_percent": None,
|
||
}
|
||
if platform.system() != "Darwin":
|
||
return state
|
||
state["swapouts"] = swapouts()
|
||
swap = sh("sysctl", "-n", "vm.swapusage", check=False)
|
||
for key, name in (("total", "swap_total_mib"), ("used", "swap_used_mib")):
|
||
match = re.search(rf"{key} = ([\d.,]+)M", swap)
|
||
if match:
|
||
state[name] = float(match.group(1).replace(",", "."))
|
||
pressure = sh("memory_pressure", check=False)
|
||
match = re.search(r"free percentage: (\d+)%", pressure)
|
||
if match:
|
||
state["memory_free_percent"] = int(match.group(1))
|
||
batt = sh("pmset", "-g", "batt", check=False)
|
||
match = re.search(r"drawing from '([^']+)'", batt)
|
||
if match:
|
||
state["power"] = match.group(1)
|
||
match = re.search(r"(\d+)%;", batt)
|
||
if match:
|
||
state["battery_percent"] = int(match.group(1))
|
||
return state
|
||
|
||
|
||
def host_warnings(state: dict[str, Any]) -> list[str]:
|
||
warnings = []
|
||
if state["power"] == "Battery Power":
|
||
warnings.append("host on battery power: CPU may be throttled")
|
||
return warnings
|
||
|
||
|
||
def swapouts_delta(start: dict[str, Any], end: dict[str, Any]) -> int | None:
|
||
"""Прирост выгрузок в swap за прогон: None, если одного из счётчиков
|
||
нет; иначе разность, в том числе 0 (счётчик 0 после перезагрузки —
|
||
тоже значение, design change `fix-known-issues`, решение 13)."""
|
||
before, after = start.get("swapouts"), end.get("swapouts")
|
||
if before is None or after is None:
|
||
return None
|
||
return after - before
|
||
|
||
|
||
def swap_warning(start: dict[str, Any], end: dict[str, Any]) -> str | None:
|
||
"""Выгрузка в swap за прогон: счётчик вырос — предупреждение."""
|
||
delta = swapouts_delta(start, end)
|
||
if delta is None or delta <= 0:
|
||
return None
|
||
return f"host swapped out {delta} pages during the run: rerun"
|
||
|
||
|
||
def vm_warnings(profile: Profile) -> list[str]:
|
||
"""Лимиты памяти стенда не помещаются в VM docker с запасом.
|
||
|
||
Тогда лимит перестаёт быть гарантией: контейнеры делят память VM, и
|
||
метки разных дней несравнимы. У серий — наибольшая конфигурация с
|
||
генератором и сэмплером.
|
||
"""
|
||
total = sh("docker", "info", "--format", "{{.MemTotal}}", check=False)
|
||
if not total.isdigit():
|
||
return []
|
||
replicas = [(profile.replicas, profile.memory)]
|
||
if profile.is_series:
|
||
replicas = [
|
||
(k, memory)
|
||
for k, _, memory, *_ in map(scaling.unpack_config, profile.configs)
|
||
]
|
||
elif profile.is_rollout and profile.sample_resources:
|
||
# на пике замены работает третья реплика (design change
|
||
# profile-resources, решение 8)
|
||
replicas = [(profile.replicas + 1, profile.memory)]
|
||
extra: tuple[str, ...] = ()
|
||
if profile.is_series:
|
||
extra = (profile.generator_memory, profile.sampler_memory)
|
||
elif profile.sample_resources:
|
||
extra = (profile.sampler_memory,)
|
||
elif profile.is_uploads:
|
||
# профиль загрузок: S3 и сэмплер peak
|
||
extra = (
|
||
profile.s3_memory,
|
||
profile.sampler_memory,
|
||
)
|
||
elif profile.is_reconcile:
|
||
# профиль сверки: S3, разовый уборщик и сэмплер scan
|
||
extra = (
|
||
profile.s3_memory,
|
||
profile.cleaner_memory,
|
||
profile.sampler_memory,
|
||
)
|
||
elif profile.is_queue:
|
||
# профиль очереди: S3, брокер, relay, два worker, уборщик и сэмплер
|
||
extra = (
|
||
profile.s3_memory,
|
||
profile.broker_memory,
|
||
profile.relay_memory,
|
||
profile.worker_memory,
|
||
profile.worker_memory,
|
||
profile.cleaner_memory,
|
||
profile.sampler_memory,
|
||
)
|
||
elif profile.is_telemetry_cost:
|
||
# цена телеметрии: стенд очереди с одним worker, сервисы телеметрии
|
||
# и сэмплер (design change track1-telemetry-cost, решение 14)
|
||
extra = (
|
||
profile.s3_memory,
|
||
profile.broker_memory,
|
||
profile.relay_memory,
|
||
profile.worker_memory,
|
||
profile.cleaner_memory,
|
||
profile.sampler_memory,
|
||
*(
|
||
getattr(profile, fields[1])
|
||
for fields in TELEMETRY_LIMIT_FIELDS.values()
|
||
),
|
||
)
|
||
if profile.telemetry_on:
|
||
extra = (
|
||
*extra,
|
||
*(
|
||
getattr(profile, fields[1])
|
||
for fields in TELEMETRY_LIMIT_FIELDS.values()
|
||
),
|
||
)
|
||
limits = max(k * size_bytes(memory) for k, memory in replicas) + sum(
|
||
size_bytes(value)
|
||
for value in (
|
||
profile.db_memory,
|
||
profile.cache_memory,
|
||
profile.lb_memory,
|
||
*extra,
|
||
)
|
||
if value
|
||
)
|
||
budget = int(total) - int(VM_RESERVE_GIB * 2**30)
|
||
if limits <= budget:
|
||
return []
|
||
return [
|
||
f"memory limits {limits / 2**30:.1f} GiB + {VM_RESERVE_GIB} GiB "
|
||
f"reserve exceed docker VM memory {int(total) / 2**30:.1f} GiB: "
|
||
"limits are not guaranteed",
|
||
]
|
||
|
||
|
||
def git_info(exclude: str | None = None) -> dict[str, Any]:
|
||
"""Commit и признак незакоммиченных изменений; `exclude` — каталог
|
||
своей метки при продолжении серии: её частичные результаты — не
|
||
изменение кода (решение 5 design change `scaling-bench`)."""
|
||
commit = sh("git", "-C", str(ROOT), "rev-parse", "--short", "HEAD")
|
||
args = ["git", "-C", str(ROOT), "status", "--porcelain"]
|
||
if exclude:
|
||
args += ["--", ".", f":!{exclude}"]
|
||
status = sh(*args)
|
||
return {"commit": commit, "dirty": bool(status)}
|
||
|
||
|
||
def static_passport(profile: Profile) -> dict[str, Any]:
|
||
docker = json.loads(sh("docker", "info", "--format", "{{json .}}"))
|
||
versions = sh(
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
"-c",
|
||
"import importlib.metadata as m, json, platform; print(json.dumps("
|
||
"{'python': platform.python_version(), **{p: m.version(p) for p in "
|
||
"('fastapi', 'starlette', 'pydantic', 'uvicorn', 'sqlalchemy', "
|
||
"'asyncpg', 'redis')}}))",
|
||
)
|
||
k6_version = sh("docker", "run", "--rm", K6_IMAGE, "version")
|
||
host: dict[str, Any] = {"os": platform.system(), "machine": None}
|
||
if platform.system() == "Darwin":
|
||
host["os"] = f"macOS {platform.mac_ver()[0]}"
|
||
host["machine"] = sh(
|
||
"sysctl",
|
||
"-n",
|
||
"machdep.cpu.brand_string",
|
||
check=False,
|
||
)
|
||
memsize = sh("sysctl", "-n", "hw.memsize", check=False)
|
||
host["memory_gib"] = (
|
||
round(int(memsize) / 2**30, 1) if memsize.isdigit() else None
|
||
)
|
||
# Apple silicon: производительные и энергоэффективные ядра; vCPU ВМ
|
||
# к ним не привязаны (решение 12 design change scaling-bench)
|
||
for key, level in (
|
||
("performance_cores", "hw.perflevel0.physicalcpu"),
|
||
("efficiency_cores", "hw.perflevel1.physicalcpu"),
|
||
):
|
||
value = sh("sysctl", "-n", level, check=False)
|
||
host[key] = int(value) if value.isdigit() else None
|
||
return {
|
||
"app_image": {
|
||
"tag": APP_IMAGE,
|
||
"id": sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE),
|
||
"versions": json.loads(versions),
|
||
},
|
||
"generator": {"image": K6_IMAGE, "version": k6_version},
|
||
"docker": {
|
||
"server": docker.get("ServerVersion"),
|
||
"vm_os": docker.get("OperatingSystem"),
|
||
# ядро ВМ: сброс memory.peak через свой дескриптор — с 6.12
|
||
# (профиль uploads)
|
||
"kernel": docker.get("KernelVersion"),
|
||
"vm_cpus": docker.get("NCPU"),
|
||
"vm_memory_gib": round(docker.get("MemTotal", 0) / 2**30, 1),
|
||
"cgroup": docker.get("CgroupVersion"),
|
||
"logging_driver": docker.get("LoggingDriver"),
|
||
},
|
||
"host": host,
|
||
"profile": asdict(profile),
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------------------ стенд
|
||
|
||
|
||
class Watcher:
|
||
"""Раз в секунду, пока идёт запуск k6: VmRSS реплик (сумма), живы ли
|
||
они, PostgreSQL и Valkey.
|
||
|
||
Реплика или хранилище умерли (OOM, kill) — k6 останавливается: иначе
|
||
каждый его запрос ждал бы таймаута, и наполнение тянулось бы часами.
|
||
"""
|
||
|
||
def __init__(self, replicas: list[str], *, rss: bool = True) -> None:
|
||
self.replicas = replicas
|
||
self.rss = rss
|
||
self.samples: list[int] = []
|
||
self.app_died = False
|
||
self.stand_died: list[str] = []
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(
|
||
target=self._loop if rss else self._light_loop,
|
||
daemon=True,
|
||
)
|
||
|
||
def _halt(self) -> None:
|
||
sh("docker", "exec", K6, "pkill", "-INT", "k6", check=False)
|
||
|
||
def _light_loop(self) -> None:
|
||
"""Серии трека 1: один `docker inspect` реплик, хранилищ и LB в
|
||
секунду, без `docker exec` в реплики (решение 7 design change
|
||
`scaling-bench`): процесс `grep` попадал бы в cgroup реплики."""
|
||
while not self._stop.wait(1.0):
|
||
states = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Name}} {{.State.Running}}",
|
||
*self.replicas,
|
||
DB,
|
||
CACHE,
|
||
LB,
|
||
check=False,
|
||
)
|
||
running = dict(
|
||
line.split() for line in states.splitlines() if line
|
||
)
|
||
names = [*self.replicas, DB, CACHE, LB]
|
||
dead = [n for n in names if running.get(f"/{n}") != "true"]
|
||
if not dead:
|
||
continue
|
||
if any(name in self.replicas for name in dead):
|
||
self.app_died = True
|
||
else:
|
||
self.stand_died = dead
|
||
self._halt()
|
||
return
|
||
|
||
def _loop(self) -> None:
|
||
while not self._stop.wait(1.0):
|
||
states = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Name}} {{.State.Running}}",
|
||
DB,
|
||
CACHE,
|
||
check=False,
|
||
)
|
||
dead = [
|
||
name.lstrip("/")
|
||
for name, running in (
|
||
line.split() for line in states.splitlines() if line
|
||
)
|
||
if running != "true"
|
||
]
|
||
if dead:
|
||
self.stand_died = dead
|
||
self._halt()
|
||
return
|
||
total = 0
|
||
for replica in self.replicas:
|
||
out = sh(
|
||
"docker",
|
||
"exec",
|
||
replica,
|
||
"grep",
|
||
"^VmRSS:",
|
||
"/proc/1/status",
|
||
check=False,
|
||
)
|
||
match = re.search(r"(\d+) kB", out)
|
||
if match:
|
||
total += int(match.group(1))
|
||
continue
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
replica,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
self.app_died = True
|
||
self._halt()
|
||
return
|
||
total = 0
|
||
break
|
||
if total:
|
||
self.samples.append(total)
|
||
|
||
def __enter__(self) -> "Watcher":
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *_: object) -> None:
|
||
self._stop.set()
|
||
self._thread.join()
|
||
|
||
|
||
def compose_env(profile: Profile) -> dict[str, str]:
|
||
"""Окружение `docker compose` стенда, собранное с нуля.
|
||
|
||
Пароли хранилищ — новые на каждый прогон: живут только в этом словаре
|
||
и в контейнерах стенда, в результаты не пишутся. Лимиты и срок сессии
|
||
— из профиля (`BENCH_*` в `bench/bench.compose.yml`).
|
||
|
||
Секреты S3 compose требует при разборе файла (`:?` в
|
||
`docker-compose.yml`), хотя профили лаб 2–3 сервер S3 не запускают:
|
||
реплики API получают ключи, но к S3 не обращаются (design change
|
||
`s3-storage`, решение 9). С уборщиком (design change
|
||
`upload-consistency`, решение 9) — ещё ключи его пользователя: сервис
|
||
`cleaner` описан, но профили его не запускают. С брокером, relay и worker
|
||
(design change `async-attachments`, решение 16; `pipeline-reliability`,
|
||
решение 16) — пароль брокера и ключи пользователя worker: bootstrap S3
|
||
заводит его и в профилях с S3. Реплики пароля брокера не получают. С
|
||
телеметрией (design change `otel-telemetry`, решение 20) — пароль
|
||
администратора Grafana: сервисы телеметрии описаны, но ни один профиль их
|
||
не запускает, а реплики и worker стенда — с выключенной телеметрией.
|
||
"""
|
||
env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
|
||
return {
|
||
**env,
|
||
"LC_ALL": "C",
|
||
"POSTGRES_PASSWORD": secrets.token_urlsafe(24),
|
||
"VALKEY_PASSWORD": secrets.token_urlsafe(24),
|
||
"S3_ROOT_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_ROOT_SECRET_KEY": secrets.token_hex(20),
|
||
"S3_JWT_SIGNING_KEY": secrets.token_hex(32),
|
||
"S3_API_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_API_SECRET_KEY": secrets.token_hex(20),
|
||
"S3_CLEANER_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_CLEANER_SECRET_KEY": secrets.token_hex(20),
|
||
"S3_WORKER_ACCESS_KEY": secrets.token_hex(10),
|
||
"S3_WORKER_SECRET_KEY": secrets.token_hex(20),
|
||
"RABBITMQ_DEFAULT_PASS": secrets.token_urlsafe(24),
|
||
"GRAFANA_ADMIN_PASSWORD": secrets.token_hex(16),
|
||
"BENCH_APP_CPUS": profile.cpus,
|
||
"BENCH_APP_MEMORY": profile.memory,
|
||
"BENCH_DB_CPUS": profile.db_cpus,
|
||
"BENCH_DB_MEMORY": profile.db_memory,
|
||
"BENCH_CACHE_CPUS": profile.cache_cpus,
|
||
"BENCH_CACHE_MEMORY": profile.cache_memory,
|
||
"BENCH_SESSION_TTL_SECONDS": str(profile.session_ttl_seconds),
|
||
"BENCH_API_REPLICAS": str(profile.replicas),
|
||
"BENCH_LB_CPUS": profile.lb_cpus,
|
||
"BENCH_LB_MEMORY": profile.lb_memory,
|
||
"BENCH_DB_MAX_CONNECTIONS": str(profile.db_max_connections),
|
||
"BENCH_S3_CPUS": profile.s3_cpus,
|
||
"BENCH_S3_MEMORY": profile.s3_memory,
|
||
"BENCH_CLEANER_CPUS": profile.cleaner_cpus,
|
||
"BENCH_CLEANER_MEMORY": profile.cleaner_memory,
|
||
"BENCH_BROKER_CPUS": profile.broker_cpus,
|
||
"BENCH_BROKER_MEMORY": profile.broker_memory,
|
||
"BENCH_RELAY_CPUS": profile.relay_cpus,
|
||
"BENCH_RELAY_MEMORY": profile.relay_memory,
|
||
"BENCH_WORKER_CPUS": profile.worker_cpus,
|
||
"BENCH_WORKER_MEMORY": profile.worker_memory,
|
||
# сервисы телеметрии (design change track1-telemetry-cost, решение
|
||
# 1): compose требует лимиты при разборе у всех профилей
|
||
"BENCH_COLLECTOR_CPUS": profile.collector_cpus,
|
||
"BENCH_COLLECTOR_MEMORY": profile.collector_memory,
|
||
"BENCH_PROMETHEUS_CPUS": profile.prometheus_cpus,
|
||
"BENCH_PROMETHEUS_MEMORY": profile.prometheus_memory,
|
||
"BENCH_LOKI_CPUS": profile.loki_cpus,
|
||
"BENCH_LOKI_MEMORY": profile.loki_memory,
|
||
"BENCH_TEMPO_CPUS": profile.tempo_cpus,
|
||
"BENCH_TEMPO_MEMORY": profile.tempo_memory,
|
||
}
|
||
|
||
|
||
def leftovers() -> dict[str, list[str]]:
|
||
"""Контейнеры, volume и сеть стенда, оставшиеся от другого прогона.
|
||
|
||
Ключ — подкоманда docker, которой их удалить.
|
||
"""
|
||
project = f"label=com.docker.compose.project={PROJECT}"
|
||
names = sh("docker", "ps", "-a", "--format", "{{.Names}}").split()
|
||
containers = sh(
|
||
"docker",
|
||
"ps",
|
||
"-a",
|
||
"--filter",
|
||
project,
|
||
"--format",
|
||
"{{.Names}}",
|
||
).split()
|
||
networks = sh("docker", "network", "ls", "--format", "{{.Name}}").split()
|
||
extra = [
|
||
name
|
||
for name in (K6, SAMPLER, FILL, IMAGES, LOOKUP, DU)
|
||
if name in names
|
||
]
|
||
return {
|
||
"rm -f": containers + [n for n in extra if n not in containers],
|
||
"volume rm": sh(
|
||
"docker",
|
||
"volume",
|
||
"ls",
|
||
"-q",
|
||
"--filter",
|
||
project,
|
||
).split(),
|
||
"network rm": [
|
||
name for name in (NETWORK, TELEMETRY_NETWORK) if name in networks
|
||
],
|
||
}
|
||
|
||
|
||
class Stand:
|
||
"""Проект compose `mtusi-bench` и контейнер k6 на время одного размера."""
|
||
|
||
def __init__(self, profile: Profile, label_dir: Path) -> None:
|
||
self.profile = profile
|
||
self.label_dir = label_dir
|
||
self.work_dir = label_dir / ".work"
|
||
self.env = compose_env(profile)
|
||
# профиль uploads: override окружения реплики по варианту
|
||
self.extra_files: list[Path] = []
|
||
# включённая телеметрия (design change track1-telemetry-cost, решения
|
||
# 1, 13): выключатель профилей загрузок и очереди или вариант прогона
|
||
# профиля цены телеметрии; её override идёт между базой и своим
|
||
# override профиля, поэтому extra_files его не перезаписывает
|
||
self.telemetry = profile.telemetry_on
|
||
|
||
@property
|
||
def telemetry_files(self) -> list[Path]:
|
||
return [TELEMETRY_COMPOSE] if self.telemetry else []
|
||
|
||
def compose(self, *args: str, check: bool = True) -> str:
|
||
"""docker compose стенда; без `-p mtusi-bench` не вызывается. Список
|
||
`-f`: база, телеметрия, свой override профиля."""
|
||
files = [
|
||
arg
|
||
for path in (
|
||
*COMPOSE_FILES,
|
||
*self.telemetry_files,
|
||
*self.extra_files,
|
||
)
|
||
for arg in ("-f", str(path))
|
||
]
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
PROJECT,
|
||
"--env-file",
|
||
str(ENV_FILE),
|
||
*files,
|
||
*args,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(
|
||
f"docker compose {' '.join(args[:3])}: exit "
|
||
f"{proc.returncode}\n{(proc.stdout + proc.stderr)[-2000:]}",
|
||
)
|
||
return proc.stdout.strip()
|
||
|
||
def check_free(self) -> None:
|
||
found = {cmd: names for cmd, names in leftovers().items() if names}
|
||
if found:
|
||
fix = "; ".join(
|
||
f"docker {cmd} {' '.join(names)}"
|
||
for cmd, names in found.items()
|
||
)
|
||
raise BenchError(
|
||
"bench stand leftovers exist: another bench is running or a "
|
||
f"crashed one left them; remove with: {fix}",
|
||
)
|
||
|
||
def remove_leftovers(self, say: Callable[[str], None]) -> None:
|
||
"""Серии трека 1: блокировка взята, другого прогона нет, остатки
|
||
устарели — удаляются с печатью каждого объекта (решение 5 design
|
||
change `scaling-bench`). Прочие профили по-прежнему отказывают."""
|
||
found = leftovers()
|
||
for name in found["rm -f"]:
|
||
say(f"removing stale bench container {name}")
|
||
sh("docker", "rm", "-f", name, check=False)
|
||
for name in found["volume rm"]:
|
||
say(f"removing stale bench volume {name}")
|
||
sh("docker", "volume", "rm", name, check=False)
|
||
for name in found["network rm"]:
|
||
say(f"removing stale bench network {name}")
|
||
sh("docker", "network", "rm", name, check=False)
|
||
self.check_free()
|
||
|
||
def build_image(self) -> str:
|
||
sh(
|
||
"docker",
|
||
"build",
|
||
"-q",
|
||
"-t",
|
||
APP_IMAGE,
|
||
str(ROOT),
|
||
)
|
||
return sh("docker", "image", "inspect", "-f", "{{.Id}}", APP_IMAGE)
|
||
|
||
def build_from(self, commit: str) -> str:
|
||
"""Образ прежнего commit тем же `Dockerfile` из файлов commit
|
||
(`git archive`, `.dockerignore` commit действует): рабочее дерево,
|
||
индекс и HEAD не меняются (решение 6 design change
|
||
`rolling-update`). Возвращает ID образа `FROM_IMAGE`."""
|
||
archive = subprocess.Popen(
|
||
["git", "-C", str(ROOT), "archive", "--format=tar", commit],
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
)
|
||
build = subprocess.run(
|
||
["docker", "build", "-q", "-t", FROM_IMAGE, "-"],
|
||
stdin=archive.stdout,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if archive.stdout is not None:
|
||
archive.stdout.close()
|
||
archive.wait()
|
||
if archive.returncode != 0 or build.returncode != 0:
|
||
raise BenchError(
|
||
f"image of commit {commit} not built: "
|
||
f"{(build.stdout + build.stderr)[-1500:]}",
|
||
)
|
||
return sh("docker", "image", "inspect", "-f", "{{.Id}}", FROM_IMAGE)
|
||
|
||
def script_env(self) -> dict[str, str]:
|
||
"""Окружение `scripts/rolling-update.sh`: тот же проект compose, что
|
||
у `compose()` — `-p`, оба `-f` и `--env-file` переменными docker
|
||
compose; пароли и `BENCH_*` — из окружения стенда."""
|
||
return {
|
||
**self.env,
|
||
"COMPOSE_PROJECT_NAME": PROJECT,
|
||
"COMPOSE_FILE": os.pathsep.join(str(p) for p in COMPOSE_FILES),
|
||
"COMPOSE_ENV_FILES": str(ENV_FILE),
|
||
"API_REPLICAS": str(self.profile.replicas),
|
||
}
|
||
|
||
def project_events(self, path: Path) -> subprocess.Popen[bytes]:
|
||
"""Поток событий контейнеров проекта (часы ВМ) в файл `path`:
|
||
создание, старт, сигнал, выход, удаление; формат —
|
||
`rollout.parse_container_events`.
|
||
|
||
Поток, а не `docker events --since` после прогона: демон хранит
|
||
только последние 256 событий, а `docker exec` скрипта замены и
|
||
оркестратора (по три события на вызов) вытесняют их за минуту
|
||
(самопроверка: событий миграции и старых реплик уже не было)."""
|
||
args = [
|
||
"docker",
|
||
"events",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
"--filter",
|
||
"type=container",
|
||
]
|
||
# exec_start — начало подготовки замены: первое обращение скрипта к
|
||
# LB (design change profile-resources, решение 4); exec проверок
|
||
# здоровья контейнеров отсекает rollout.script_start
|
||
for action in ("create", "start", "kill", "die", "destroy"):
|
||
args += ["--filter", f"event={action}"]
|
||
args += ["--filter", "event=exec_start"]
|
||
args += [
|
||
"--format",
|
||
"{{.TimeNano}}|{{.Action}}|{{.Actor.ID}}|"
|
||
'{{index .Actor.Attributes "name"}}|'
|
||
'{{index .Actor.Attributes "signal"}}',
|
||
]
|
||
with path.open("w") as out:
|
||
return subprocess.Popen(
|
||
args,
|
||
stdout=out,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
|
||
def death_events(self, path: Path) -> subprocess.Popen[bytes]:
|
||
"""Поток завершений контейнеров проекта для профиля очереди: `oom`, `kill`
|
||
и `die` с кодом выхода, формат — `overload.parse_docker_events`. Поток, как
|
||
у `project_events`; причина перезапуска есть только здесь: `.State.OOMKilled`
|
||
новый старт сбрасывает, счётчик cgroup уходит вместе с жизнью контейнера."""
|
||
args = [
|
||
"docker",
|
||
"events",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
"--filter",
|
||
"type=container",
|
||
]
|
||
for action in ("oom", "kill", "die"):
|
||
args += ["--filter", f"event={action}"]
|
||
args += [
|
||
"--format",
|
||
'{{.TimeNano}}|{{.Action}}|{{index .Actor.Attributes "name"}}|'
|
||
'{{index .Actor.Attributes "exitCode"}}|'
|
||
'{{index .Actor.Attributes "signal"}}',
|
||
]
|
||
with path.open("w") as out:
|
||
return subprocess.Popen(
|
||
args,
|
||
stdout=out,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
|
||
def container_events(
|
||
self,
|
||
container: str,
|
||
path: Path,
|
||
) -> subprocess.Popen[bytes]:
|
||
"""Поток событий Docker одного контейнера (часы ВМ) в файл `path`;
|
||
формат — `failover.parse_events`.
|
||
|
||
Поток, запущенный до нагрузки фазы, а не `docker events --since`
|
||
после неё: демон хранит только последние 256 событий всех
|
||
контейнеров, а проверки здоровья и `docker exec` оркестратора (по
|
||
три события на вызов) вытесняют их (design change
|
||
`fix-known-issues`, решение 9)."""
|
||
with path.open("w") as out:
|
||
return subprocess.Popen(
|
||
[
|
||
"docker",
|
||
"events",
|
||
"--filter",
|
||
f"container={container}",
|
||
"--format",
|
||
"{{.TimeNano}} {{.Action}} "
|
||
'{{index .Actor.Attributes "signal"}}',
|
||
],
|
||
stdout=out,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
|
||
def app_containers(self) -> list[dict[str, Any]]:
|
||
"""Реплики `api` (и остановленные): имя, `X-Instance-Id`, тег и
|
||
образ по метке (`CONTAINER_IMAGE`). Сравниваются контейнеры между
|
||
собой, а не с тегом: у тега индекс образа, у метки — манифест
|
||
платформы."""
|
||
rows = []
|
||
for name in self.replicas(stopped=True):
|
||
full, tag, image, running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
f"{{{{.Id}}}}|{{{{.Config.Image}}}}|{CONTAINER_IMAGE}|"
|
||
"{{.State.Running}}",
|
||
name,
|
||
).split("|")
|
||
rows.append(
|
||
{
|
||
"name": name,
|
||
"instance": full[:12],
|
||
"tag": tag,
|
||
"image": image,
|
||
"running": running == "true",
|
||
},
|
||
)
|
||
return rows
|
||
|
||
def start(self, say: Callable[[str], None]) -> None:
|
||
"""Свежий стенд: пустая база на новом volume, миграции, реплики API
|
||
и LB, k6."""
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
self.compose("up", "-d", "--wait", "postgres", "valkey")
|
||
say("postgres and valkey healthy")
|
||
self.compose("run", "--rm", "migrate")
|
||
say("migrations applied")
|
||
if self.telemetry:
|
||
# телеметрия — раньше процессов приложения: первые пакеты SDK не
|
||
# уходят в незапущенный Collector (design change
|
||
# track1-telemetry-cost, решение 1)
|
||
self.start_telemetry()
|
||
say("telemetry services ready")
|
||
if self.profile.is_uploads:
|
||
# S3 — только профили загрузок (design change upload-memory,
|
||
# решение 10) и сверки; прочие профили его не запускают
|
||
self.start_s3()
|
||
say("s3 healthy, bootstrap done")
|
||
if self.profile.is_reconcile:
|
||
# профиль сверки: аргументы S3 рабочего стенда и порт метрик
|
||
# шлюза на loopback контейнера (design change
|
||
# track2-reconcile-cost, решение 8)
|
||
self.extra_files = [self.s3_metrics_override()]
|
||
self.start_s3()
|
||
say(
|
||
f"s3 healthy with metrics port "
|
||
f"{self.profile.reconcile_metrics_port}, bootstrap done",
|
||
)
|
||
if self.profile.runs_pipeline:
|
||
# профили очереди и цены телеметрии меряют обработку: S3 и брокер
|
||
# до реплик (решение 10 design change track1-queue-overload)
|
||
self.start_s3()
|
||
self.start_broker()
|
||
say("s3 and broker healthy, bootstrap done")
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-build",
|
||
APP_SERVICE,
|
||
"haproxy",
|
||
)
|
||
if self.profile.runs_pipeline:
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-build",
|
||
"relay",
|
||
"cleaner",
|
||
"worker",
|
||
)
|
||
# генератор серий трека 1 — с лимитом без swap (решение 6 design
|
||
# change scaling-bench); у прочих профилей — без лимита
|
||
limits = (
|
||
[
|
||
"--cpus",
|
||
self.profile.generator_cpus,
|
||
"--memory",
|
||
self.profile.generator_memory,
|
||
"--memory-swap",
|
||
self.profile.generator_memory,
|
||
]
|
||
if self.profile.generator_cpus
|
||
else []
|
||
)
|
||
sh(
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
K6,
|
||
*limits,
|
||
"--network",
|
||
NETWORK,
|
||
"-v",
|
||
f"{ROOT / 'bench' / 'k6'}:/scripts:ro",
|
||
"-v",
|
||
f"{self.label_dir}:/out",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"sleep",
|
||
K6_IMAGE,
|
||
"infinity",
|
||
)
|
||
self.wait_ready()
|
||
say(f"{self.profile.replicas} api replicas UP behind haproxy")
|
||
|
||
def start_s3(self) -> None:
|
||
"""S3 стенда и его bootstrap: `up --wait s3`, затем код выхода
|
||
bootstrap отдельным `up --exit-code-from` (`--wait` одноразовый
|
||
сервис считает готовым уже запущенным, как `make start_s3`)."""
|
||
self.compose("up", "-d", "--wait", "s3")
|
||
self.compose(
|
||
"up",
|
||
"--no-deps",
|
||
"--exit-code-from",
|
||
"s3-bootstrap",
|
||
"s3-bootstrap",
|
||
)
|
||
|
||
def start_broker(self) -> None:
|
||
"""Брокер стенда: `up --wait`, готовность — healthcheck сервиса."""
|
||
self.compose("up", "-d", "--wait", BROKER_SERVICE)
|
||
|
||
def start_telemetry(self) -> None:
|
||
"""Сервисы телеметрии рабочего стенда (решение 1): Prometheus `--wait`
|
||
(healthcheck), затем Collector, Loki и Tempo; готовность — `wait-ready`
|
||
контейнера выборки. Grafana не запускается: старт перечисляет
|
||
сервисы."""
|
||
self.compose("up", "-d", "--wait", "--no-deps", "prometheus")
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"otel-collector",
|
||
"loki",
|
||
"tempo",
|
||
)
|
||
proc = self.lookup("wait-ready", "--timeout", "90")
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"telemetry services not ready: {proc.stdout.strip()}",
|
||
)
|
||
|
||
def lookup(self, *args: str) -> subprocess.CompletedProcess[str]:
|
||
"""`bench/telemetry_lookup.py` в контейнере `mtusi-bench-lookup` на
|
||
образе стенда: сеть хранилищ, `bench/` на чтение, рабочий каталог,
|
||
свои лимиты; без меток проекта и паролей (решение 9)."""
|
||
p = self.profile
|
||
subprocess.run(
|
||
["docker", "rm", "-f", LOOKUP],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
return subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--name",
|
||
LOOKUP,
|
||
"--network",
|
||
TELEMETRY_NETWORK,
|
||
"--cpus",
|
||
p.lookup_cpus,
|
||
"--memory",
|
||
p.lookup_memory,
|
||
"--memory-swap",
|
||
p.lookup_memory,
|
||
"-v",
|
||
f"{ROOT / 'bench'}:/bench:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
LOOKUP_SCRIPT,
|
||
*args,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
|
||
def volume_sizes(self) -> dict[str, int]:
|
||
"""Размер volume хранилищ телеметрии (`du -sb`) — контейнер
|
||
`mtusi-bench-du` на образе PostgreSQL стенда, `--user 0`, без сети,
|
||
volume только на чтение (решение 9)."""
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
mounts = [
|
||
item
|
||
for name, volume in TELEMETRY_VOLUMES.items()
|
||
for item in ("-v", f"{volume}:/v/{name}:ro")
|
||
]
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--name",
|
||
DU,
|
||
"--user",
|
||
"0",
|
||
"--network",
|
||
"none",
|
||
*mounts,
|
||
"--entrypoint",
|
||
"sh",
|
||
image,
|
||
"-c",
|
||
"du -sb /v/*",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return telemetry_cost.parse_du(proc.stdout)
|
||
|
||
def telemetry_peaks(self) -> dict[str, float | None]:
|
||
"""`memory.peak` сервисов телеметрии в MiB после выборки — пик за
|
||
жизнь контейнера вместе с запросами выборки, запас до лимита
|
||
(design, «Уточнения apply», п. 16). Контейнер `mtusi-bench-du` на
|
||
образе PostgreSQL стенда, `--cgroupns host`, без сети; нет cgroup
|
||
(контейнер завершён) — `None`."""
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
ids = {
|
||
service: sh("docker", "inspect", "-f", "{{.Id}}", name)
|
||
for service, name in TELEMETRY_CONTAINERS.items()
|
||
}
|
||
script = "".join(
|
||
f"echo {service} $(cat /sys/fs/cgroup/docker/{cid}/memory.peak "
|
||
"2>/dev/null || echo -);"
|
||
for service, cid in ids.items()
|
||
)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--name",
|
||
DU,
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
"none",
|
||
"--entrypoint",
|
||
"sh",
|
||
image,
|
||
"-c",
|
||
script,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
peaks: dict[str, float | None] = dict.fromkeys(ids)
|
||
for line in proc.stdout.splitlines():
|
||
service, _, value = line.partition(" ")
|
||
if service in peaks and value.strip().isdigit():
|
||
peaks[service] = round(int(value) / MIB, 1)
|
||
return peaks
|
||
|
||
def telemetry_passport(self) -> dict[str, Any]:
|
||
"""Образы, лимиты и SHA-256 конфигураций сервисов телеметрии (решение
|
||
13; спек `load-benchmark`, «Сервисы телеметрии на стенде»)."""
|
||
images = {}
|
||
for service, name in TELEMETRY_CONTAINERS.items():
|
||
images[service] = {
|
||
"image": sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Config.Image}}",
|
||
name,
|
||
),
|
||
"id": sh("docker", "inspect", "-f", CONTAINER_IMAGE, name),
|
||
"limits": self.container_limits(name),
|
||
}
|
||
configs = {
|
||
path: hashlib_sha256(ROOT / path)
|
||
for path in (
|
||
telemetry_cost.COLLECTOR_CONFIG,
|
||
telemetry_cost.PROMETHEUS_CONFIG,
|
||
"infra/otel/prometheus/rules.yml",
|
||
"infra/otel/loki/loki.yaml",
|
||
"infra/otel/tempo/tempo.yaml",
|
||
"bench/telemetry.compose.yml",
|
||
)
|
||
}
|
||
return {"services": images, "configs": configs}
|
||
|
||
def s3_command(self) -> list[str]:
|
||
"""Аргументы сервера S3 по `docker compose config` стенда без
|
||
override метрик: те же, что у рабочего стенда."""
|
||
saved = self.extra_files
|
||
self.extra_files = []
|
||
try:
|
||
config = json.loads(self.compose("config", "--format", "json"))
|
||
finally:
|
||
self.extra_files = saved
|
||
return list(config["services"]["s3"]["command"])
|
||
|
||
def s3_metrics_override(self) -> Path:
|
||
"""`.work/s3-metrics.compose.yml`: `command` сервиса `s3` из
|
||
конфигурации стенда и `-metricsPort` (JSON — подмножество YAML).
|
||
Порт слушает `-metricsIp`, по умолчанию `-ip.bind` — loopback
|
||
контейнера (design change `track2-reconcile-cost`, решение 8)."""
|
||
command = [
|
||
*self.s3_command(),
|
||
f"-metricsPort={self.profile.reconcile_metrics_port}",
|
||
]
|
||
path = self.work_dir / S3_METRICS_COMPOSE
|
||
path.write_text(
|
||
json.dumps({"services": {"s3": {"command": command}}}) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
return path
|
||
|
||
def s3_metrics(self) -> dict[str, float]:
|
||
"""`SeaweedFS_s3_request_total` по типам изнутри контейнера S3."""
|
||
text = sh(
|
||
"docker",
|
||
"exec",
|
||
S3,
|
||
"wget",
|
||
"-qO-",
|
||
f"http://127.0.0.1:{self.profile.reconcile_metrics_port}/metrics",
|
||
)
|
||
return reconcile.parse_metrics(text)
|
||
|
||
def storage_names(self) -> tuple[str, str, str]:
|
||
"""Bucket, имя базы (namespace ключей объектов) и адрес S3 в сети
|
||
стенда — из окружения реплик по конфигурации compose."""
|
||
api_env = json.loads(self.compose("config", "--format", "json"))[
|
||
"services"
|
||
][APP_SERVICE]["environment"]
|
||
return (
|
||
api_env["S3__BUCKET"],
|
||
api_env["DATABASE__NAME"],
|
||
f"http://{api_env['S3__HOST']}:{api_env['S3__PORT']}",
|
||
)
|
||
|
||
def copy_rows(self, rows: str) -> None:
|
||
"""Метаданные набора профиля сверки — `\\copy attachments … FROM
|
||
STDIN` в базе стенда (данные стенда, схему не трогает; решение 3)."""
|
||
columns = ", ".join(reconcile.COPY_COLUMNS)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
"-i",
|
||
DB,
|
||
"sh",
|
||
"-c",
|
||
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -XAtq '
|
||
f'-v ON_ERROR_STOP=1 -c "\\copy attachments ({columns}) '
|
||
'FROM STDIN WITH (FORMAT csv)"',
|
||
],
|
||
input=rows,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"copy rows: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
|
||
def fill_objects(
|
||
self,
|
||
*,
|
||
seed: int,
|
||
start: int,
|
||
end: int,
|
||
size: int,
|
||
force: str | None,
|
||
) -> str:
|
||
"""Объекты набора прямым PUT в S3 стенда — `bench/fill_objects.py`
|
||
в контейнере `mtusi-bench-fill` на образе стенда, ключами
|
||
пользователя API из окружения (`-e <имя>` без значения в argv)."""
|
||
p = self.profile
|
||
env = {
|
||
**self.env,
|
||
"S3__ACCESS_KEY_ID": self.env["S3_API_ACCESS_KEY"],
|
||
"S3__SECRET_ACCESS_KEY": self.env["S3_API_SECRET_KEY"],
|
||
}
|
||
bucket, namespace, endpoint = self.storage_names()
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--name",
|
||
FILL,
|
||
"--network",
|
||
NETWORK,
|
||
"-e",
|
||
"S3__ACCESS_KEY_ID",
|
||
"-e",
|
||
"S3__SECRET_ACCESS_KEY",
|
||
"-v",
|
||
f"{ROOT / 'bench'}:/bench:ro",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
"/bench/fill_objects.py",
|
||
"--endpoint",
|
||
endpoint,
|
||
"--bucket",
|
||
bucket,
|
||
"--namespace",
|
||
namespace,
|
||
"--seed",
|
||
str(seed),
|
||
"--start",
|
||
str(start),
|
||
"--end",
|
||
str(end),
|
||
"--size",
|
||
str(size),
|
||
*(["--force", force] if force else []),
|
||
"--processes",
|
||
str(p.reconcile_fill_processes),
|
||
"--threads",
|
||
str(p.reconcile_fill_threads),
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=env,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"fill objects [{start}, {end}): exit {proc.returncode}\n"
|
||
f"{(proc.stdout + proc.stderr)[-1500:]}",
|
||
)
|
||
return proc.stdout.strip().splitlines()[-1]
|
||
|
||
def run_pass(
|
||
self,
|
||
name: str,
|
||
args: list[str],
|
||
*,
|
||
cpus: str,
|
||
before: Path | None,
|
||
grace_s: int,
|
||
timeout_s: float = 3600,
|
||
) -> dict[str, Any]:
|
||
"""Разовый проход уборщика `docker compose run -d --name <name>
|
||
--no-deps cleaner …` с лимитом CPU `cpus`, затем `docker wait`,
|
||
журнал, состояние и лимиты контейнера; контейнер удаляется при
|
||
любом исходе. `before` — override образа «до»."""
|
||
saved_env, saved_files = self.env, self.extra_files
|
||
self.env = {**self.env, "BENCH_CLEANER_CPUS": cpus}
|
||
if before is not None:
|
||
self.extra_files = [*self.extra_files, before]
|
||
try:
|
||
self.compose(
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
name,
|
||
"--no-deps",
|
||
"-e",
|
||
f"CLEANER__PENDING_GRACE_SECONDS={grace_s}",
|
||
"cleaner",
|
||
*args,
|
||
)
|
||
finally:
|
||
self.env, self.extra_files = saved_env, saved_files
|
||
try:
|
||
waited = subprocess.run(
|
||
["docker", "wait", name],
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout_s,
|
||
check=False,
|
||
)
|
||
code = int(waited.stdout.strip() or -1)
|
||
proc = subprocess.run(
|
||
["docker", "logs", name],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
started, finished, ident, image = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.StartedAt}}|{{.State.FinishedAt}}|{{.Id}}|"
|
||
"{{.Config.Image}}",
|
||
name,
|
||
).split("|")
|
||
return {
|
||
"exit_code": code,
|
||
"log": proc.stdout + proc.stderr,
|
||
"started_at": reconcile.timestamp(started),
|
||
"finished_at": reconcile.timestamp(finished),
|
||
"id": ident,
|
||
"image": image,
|
||
"limits": self.container_limits(name),
|
||
}
|
||
finally:
|
||
subprocess.run(
|
||
["docker", "rm", "-f", name],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
def recycle_uploads(self, image: str, env: dict[str, str]) -> None:
|
||
"""Перед прогоном профиля загрузок (решение 10): новый volume S3 с
|
||
bootstrap и новый контейнер реплики на образе варианта `image` с
|
||
окружением `env` (override `VARIANT_COMPOSE`), затем готовность
|
||
процесса и `UP` в LB.
|
||
|
||
S3 останавливается `kill`, а не плавно (~15 с): его volume всё равно
|
||
удаляется. Реплика — сначала удаляется: новый контейнер может
|
||
получить тот же адрес, и LB держал бы слот `UP` от прежнего (поэтому,
|
||
как в `switch_api`, сначала готовность процесса)."""
|
||
self.compose("rm", "-s", "-f", APP_SERVICE, check=False)
|
||
services = ("s3", "s3-bootstrap")
|
||
self.compose("kill", *services, check=False)
|
||
self.compose("rm", "-f", *services)
|
||
subprocess.run(
|
||
["docker", "volume", "rm", S3_VOLUME],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if S3_VOLUME in sh("docker", "volume", "ls", "-q").split():
|
||
raise BenchError(f"volume {S3_VOLUME} was not removed")
|
||
self.start_s3()
|
||
sh("docker", "tag", image, APP_IMAGE)
|
||
path = self.work_dir / VARIANT_COMPOSE
|
||
self.extra_files = []
|
||
if env:
|
||
lines = ["services:", " api:", " environment:"]
|
||
lines += [f' {key}: "{value}"' for key, value in env.items()]
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
self.extra_files = [path]
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-build",
|
||
"--force-recreate",
|
||
APP_SERVICE,
|
||
)
|
||
self.wait_processes_ready()
|
||
self.wait_ready()
|
||
|
||
def app_env(self, name: str) -> dict[str, str]:
|
||
"""Окружение приложения контейнера без секретов: параметры в
|
||
формате настроек (`SECTION__FIELD`), без паролей, секрета и ключа
|
||
доступа S3."""
|
||
env = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", name),
|
||
)
|
||
return dict(
|
||
sorted(
|
||
item.split("=", 1)
|
||
for item in env
|
||
if re.match(r"[A-Z0-9_]+__[A-Z0-9_]+=", item)
|
||
and not any(
|
||
mark in item.split("=", 1)[0] for mark in SECRET_ENV_MARKS
|
||
)
|
||
),
|
||
)
|
||
|
||
def tcp_rmem(self, name: str) -> str | None:
|
||
"""Буфер приёма сокетов контейнера (`net.ipv4.tcp_rmem`: мин.,
|
||
начальный, наибольший): у реплик — из `sysctls` сервиса `api`
|
||
(design change `upload-memory`, решение 6)."""
|
||
value = sh(
|
||
"docker",
|
||
"exec",
|
||
name,
|
||
"cat",
|
||
"/proc/sys/net/ipv4/tcp_rmem",
|
||
check=False,
|
||
)
|
||
return " ".join(value.split()) or None
|
||
|
||
def start_peak_sampler(self, name: str, targets: dict[str, str]) -> None:
|
||
"""Сэмплер прогона профиля загрузок в режиме `peak`: `--user 0`,
|
||
cgroupfs на запись (сброс `memory.peak` через свой дескриптор),
|
||
`--network none`, без метки проекта и паролей. Готов — когда для
|
||
каждой роли записан итог сброса и есть первая выборка."""
|
||
p = self.profile
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
for suffix in (".log", ".stop"):
|
||
(self.work_dir / f"{name}{suffix}").unlink(missing_ok=True)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--user",
|
||
"0",
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
"none",
|
||
"--cpus",
|
||
p.sampler_cpus,
|
||
"--memory",
|
||
p.sampler_memory,
|
||
"--memory-swap",
|
||
p.sampler_memory,
|
||
"-v",
|
||
"/sys/fs/cgroup:/sys/fs/cgroup:rw",
|
||
"-v",
|
||
f"{SAMPLER_SCRIPT}:/sampler.sh:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"bash",
|
||
image,
|
||
"/sampler.sh",
|
||
f"{p.uploads_sample_interval_s:g}",
|
||
"peak",
|
||
name,
|
||
*(f"{role}={ident}" for role, ident in targets.items()),
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"peak sampler did not start: {proc.stderr}")
|
||
path = self.work_dir / f"{name}.log"
|
||
deadline = time.monotonic() + 5
|
||
while time.monotonic() < deadline:
|
||
time.sleep(0.2)
|
||
text = path.read_text(encoding="utf-8") if path.exists() else ""
|
||
parsed = uploads.parse_peak(text)
|
||
if all(
|
||
role in parsed.roles and parsed.roles[role].samples
|
||
for role in targets
|
||
) and text.count("reset ") >= len(targets):
|
||
return
|
||
tail = sh("docker", "logs", "--tail", "5", SAMPLER, check=False)
|
||
raise BenchError(f"peak sampler: no samples in 5s\n{tail}")
|
||
|
||
def stop_peak_sampler(self, name: str) -> str:
|
||
"""Конец окна: файл остановки, сэмплер читает пики и выходит;
|
||
возвращает его строки и удаляет контейнер."""
|
||
(self.work_dir / f"{name}.stop").touch()
|
||
deadline = time.monotonic() + 15
|
||
while time.monotonic() < deadline:
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
SAMPLER,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
break
|
||
time.sleep(0.2)
|
||
self.stop_sampler()
|
||
path = self.work_dir / f"{name}.log"
|
||
return path.read_text(encoding="utf-8") if path.exists() else ""
|
||
|
||
def exit_state(self, name: str) -> dict[str, Any]:
|
||
"""Состояние контейнера после прогона: работает ли, убит ли из-за
|
||
памяти и код выхода."""
|
||
state = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .State}}", name),
|
||
)
|
||
return {
|
||
"running": bool(state.get("Running")),
|
||
"oom_killed": bool(state.get("OOMKilled")),
|
||
"exit_code": None
|
||
if state.get("Running")
|
||
else state.get("ExitCode"),
|
||
}
|
||
|
||
def replicas(self, *, stopped: bool = False) -> list[str]:
|
||
"""Имена контейнеров реплик api по порядку; `stopped` — и
|
||
остановленные."""
|
||
args = ["ps", "--format", "{{.Name}}"]
|
||
if stopped:
|
||
args.append("-a")
|
||
names = self.compose(*args, APP_SERVICE).split()
|
||
return sorted(names, key=lambda name: int(name.rsplit("-", 1)[1]))
|
||
|
||
def address(self, name: str) -> str:
|
||
"""Адрес контейнера в сети стенда (его получает слот LB из DNS)."""
|
||
return sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
|
||
name,
|
||
)
|
||
|
||
def lb_slots(self) -> list[dict[str, str]]:
|
||
"""Занятые слоты backend `api` HAProxy: слот, адрес, состояние."""
|
||
csv_text = sh("docker", "exec", LB, "bash", "-c", LB_STATS)
|
||
return failover.parse_lb_stats(csv_text)
|
||
|
||
def wait_ready(self, timeout: float = 60) -> None:
|
||
"""Все `profile.replicas` реплик работают, и LB видит их `UP`.
|
||
|
||
Готовность — по проверкам LB (`/health/ready` дважды подряд), а не
|
||
запросом изнутри реплики: k6 ходит через LB. Слот считается по
|
||
адресу реплики: после пересоздания реплик старые адреса LB ещё
|
||
некоторое время держит.
|
||
"""
|
||
deadline = time.monotonic() + timeout
|
||
expected = self.profile.replicas
|
||
while time.monotonic() < deadline:
|
||
for name in self.replicas(stopped=True):
|
||
state = self.state(name)
|
||
if not state["running"]:
|
||
reason = "oom" if state["oom"] else "exited"
|
||
raise BenchError(f"{name} died during start ({reason})")
|
||
running = self.replicas()
|
||
if len(running) == expected:
|
||
addresses = {self.address(name) for name in running}
|
||
up = {
|
||
slot["addr"].rsplit(":", 1)[0]
|
||
for slot in self.lb_slots()
|
||
if slot["status"] == "UP"
|
||
}
|
||
if addresses <= up:
|
||
return
|
||
time.sleep(0.25)
|
||
raise BenchError(
|
||
f"{expected} api replicas not UP behind haproxy in {timeout:.0f}s",
|
||
)
|
||
|
||
def stop(self) -> bool:
|
||
"""k6, контейнеры, сеть и volume стенда; True — ничего не осталось.
|
||
|
||
k6 и хранилище восстановления — первыми: подключённый к сети
|
||
контейнер не из compose не дал бы compose удалить сеть. Volume
|
||
восстановления compose не знает, он удаляется отдельно. У профиля
|
||
очереди перед `down` — `kill`: S3 иначе останавливается ~15 с, а его
|
||
volume всё равно уходит.
|
||
"""
|
||
# -v — анонимные volume образа (у сэмплера — /var/lib/postgresql);
|
||
# именованный volume восстановления удаляется ниже отдельно
|
||
subprocess.run(
|
||
[
|
||
"docker",
|
||
"rm",
|
||
"-f",
|
||
"-v",
|
||
K6,
|
||
SAMPLER,
|
||
RESTORE_DB,
|
||
FILL,
|
||
WORKER_2,
|
||
IMAGES,
|
||
LOOKUP,
|
||
DU,
|
||
],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
# разовые проходы уборщика профиля сверки (`compose run`)
|
||
passes = sh(
|
||
"docker",
|
||
"ps",
|
||
"-aq",
|
||
"--filter",
|
||
f"name={RECONCILE_PASS}",
|
||
check=False,
|
||
).split()
|
||
if passes:
|
||
subprocess.run(
|
||
["docker", "rm", "-f", *passes],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if self.profile.runs_pipeline:
|
||
self.compose("kill", check=False)
|
||
self.compose("down", "-v", "--remove-orphans", check=False)
|
||
subprocess.run(
|
||
["docker", "volume", "rm", RESTORE_VOLUME],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
# тег образа прежнего commit (профиль замены); сам образ остаётся
|
||
# в кэше сборки
|
||
subprocess.run(
|
||
["docker", "image", "rm", FROM_IMAGE],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
return not any(leftovers().values())
|
||
|
||
def start_restore(self) -> None:
|
||
"""Новое хранилище: контейнер PostgreSQL на новом volume.
|
||
|
||
Образ — тот же, что у исходного контейнера (пин без копии в коде),
|
||
лимиты — как у Postgres стенда, пароль — `--env` без значения из
|
||
окружения процесса. Метка проекта — чтобы `leftovers()` видел
|
||
остатки (решение 3 design change `track2-backup-restore`).
|
||
"""
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
env = dict(
|
||
item.split("=", 1)
|
||
for item in json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", DB),
|
||
)
|
||
)
|
||
label = f"com.docker.compose.project={PROJECT}"
|
||
sh("docker", "volume", "create", "--label", label, RESTORE_VOLUME)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
RESTORE_DB,
|
||
"--label",
|
||
label,
|
||
"--network",
|
||
NETWORK,
|
||
"--network-alias",
|
||
RESTORE_HOST,
|
||
"--env",
|
||
"POSTGRES_PASSWORD",
|
||
"-e",
|
||
f"POSTGRES_USER={env['POSTGRES_USER']}",
|
||
"-e",
|
||
f"POSTGRES_DB={env['POSTGRES_DB']}",
|
||
"--cpus",
|
||
self.profile.db_cpus,
|
||
"--memory",
|
||
self.profile.db_memory,
|
||
"--memory-swap",
|
||
self.profile.db_memory,
|
||
"-v",
|
||
f"{RESTORE_VOLUME}:/var/lib/postgresql",
|
||
image,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"restore storage did not start: {proc.stderr}")
|
||
# по TCP, как healthcheck compose: при инициализации образ
|
||
# поднимает временный сервер только на сокете
|
||
deadline = time.monotonic() + 60
|
||
while time.monotonic() < deadline:
|
||
ready = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
RESTORE_DB,
|
||
"sh",
|
||
"-c",
|
||
'pg_isready -q -h 127.0.0.1 -U "$POSTGRES_USER" '
|
||
'-d "$POSTGRES_DB"',
|
||
],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if ready.returncode == 0:
|
||
return
|
||
time.sleep(0.25)
|
||
raise BenchError(f"{RESTORE_DB} not ready in 60s")
|
||
|
||
def limits(self) -> dict[str, Any]:
|
||
"""Лимиты контейнеров стенда, прочитанные у запущенных контейнеров.
|
||
|
||
`api` — первая реплика (у всех одно определение сервиса),
|
||
`api_replicas` — сколько их; лимиты различаются — отказ.
|
||
"""
|
||
replicas = self.replicas()
|
||
roles = {APP_SERVICE: replicas[0], **CONTAINERS, "k6": K6}
|
||
if self.profile.is_uploads or self.profile.is_reconcile:
|
||
roles["s3"] = S3
|
||
if self.profile.runs_pipeline:
|
||
roles.update(
|
||
{
|
||
"s3": S3,
|
||
"rabbitmq": BROKER,
|
||
"relay": RELAY,
|
||
"worker": WORKER,
|
||
"cleaner": CLEANER,
|
||
},
|
||
)
|
||
if self.telemetry:
|
||
roles.update(TELEMETRY_CONTAINERS)
|
||
limits: dict[str, Any] = {}
|
||
for role, name in roles.items():
|
||
limits[role] = self.container_limits(name)
|
||
for name in replicas[1:]:
|
||
if self.container_limits(name) != limits[APP_SERVICE]:
|
||
raise BenchError(f"{name}: limits differ from {replicas[0]}")
|
||
limits["api_replicas"] = len(replicas)
|
||
# предел соединений — параметр конфигурации серии (решение 1
|
||
# design change scaling-bench), у прочих профилей 100
|
||
limits["max_connections"] = int(self.psql("SHOW max_connections;\n"))
|
||
if limits["max_connections"] != self.profile.db_max_connections:
|
||
raise BenchError(
|
||
f"PostgreSQL max_connections {limits['max_connections']}, "
|
||
f"expected {self.profile.db_max_connections}",
|
||
)
|
||
return limits
|
||
|
||
def start_sampler(self, *, scan: bool = False) -> None:
|
||
"""Сэмплер ячейки серии (решение 7 design change `scaling-bench`):
|
||
образ PostgreSQL стенда, `--cgroupns host`, свои лимиты; за 5 с
|
||
должны появиться строки каждого контейнера, ВМ, LB и PostgreSQL.
|
||
|
||
`scan` — профили уровня «4» (решения 1, 2 design change
|
||
`profile-resources`): режим обхода cgroup, контейнер без сети, без
|
||
метки проекта и без паролей; за 5 с — строки работающих
|
||
контейнеров проекта, k6, самого сэмплера и ВМ.
|
||
"""
|
||
if scan:
|
||
self.start_scan_sampler()
|
||
return
|
||
p = self.profile
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
env = dict(
|
||
item.split("=", 1)
|
||
for item in json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config.Env}}", DB),
|
||
)
|
||
)
|
||
targets = [
|
||
f"{APP_SERVICE}={self.full_id(name)}" for name in self.replicas()
|
||
]
|
||
targets += [
|
||
f"{role}={self.full_id(name)}"
|
||
for role, name in (*CONTAINERS.items(), ("k6", K6))
|
||
]
|
||
for name in ("samples.log", "pg-samples.log"):
|
||
(self.work_dir / name).unlink(missing_ok=True)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--label",
|
||
f"com.docker.compose.project={PROJECT}",
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
NETWORK,
|
||
"--cpus",
|
||
p.sampler_cpus,
|
||
"--memory",
|
||
p.sampler_memory,
|
||
"--memory-swap",
|
||
p.sampler_memory,
|
||
"--env",
|
||
"POSTGRES_PASSWORD",
|
||
"-e",
|
||
f"POSTGRES_USER={env['POSTGRES_USER']}",
|
||
"-e",
|
||
f"POSTGRES_DB={env['POSTGRES_DB']}",
|
||
"-v",
|
||
f"{SAMPLER_SCRIPT}:/sampler.sh:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"bash",
|
||
image,
|
||
"/sampler.sh",
|
||
f"{p.sample_interval_s:g}",
|
||
*targets,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=self.env,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"sampler did not start: {proc.stderr}")
|
||
expected = {f"cg {t.split('=')[0]}" for t in targets}
|
||
deadline = time.monotonic() + 5
|
||
problem = "no samples"
|
||
while time.monotonic() < deadline:
|
||
time.sleep(0.5)
|
||
samples = scaling.parse_samples(*self.sampler_logs())
|
||
if samples.missing:
|
||
problem = f"missing: {', '.join(samples.missing)}"
|
||
break
|
||
ids = {ident for _, ident in samples.cg}
|
||
want = {t.split("=")[1][:12] for t in targets}
|
||
if (
|
||
want <= ids
|
||
and samples.vm
|
||
and samples.lb
|
||
and samples.db
|
||
and samples.wal
|
||
):
|
||
return
|
||
problem = (
|
||
f"cgroups {len(ids)}/{len(want)}, vm {len(samples.vm)}, lb "
|
||
f"{len(samples.lb)}, postgres {len(samples.db)}; expected "
|
||
f"{sorted(expected)}"
|
||
)
|
||
tail = sh("docker", "logs", "--tail", "5", SAMPLER, check=False)
|
||
raise BenchError(f"sampler: {problem}\n{tail}")
|
||
|
||
def start_scan_sampler(self, mode: str = "scan") -> None:
|
||
"""Сэмплер в режиме `scan`: `--network none` — к приложению,
|
||
хранилищам и LB он обратиться не может; без `--label` проекта —
|
||
иначе `docker compose up` скрипта замены видел бы его сиротой
|
||
проекта; без `POSTGRES_*` — пароль ему не нужен. `leftovers()` и
|
||
`stop()` находят его по имени. `mode` `cost` — профиль цены
|
||
телеметрии: ещё строки `mp` (design change track1-telemetry-cost,
|
||
решение 10)."""
|
||
p = self.profile
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
(self.work_dir / "samples.log").unlink(missing_ok=True)
|
||
# проход сверки 10k длится ~3 с: интервала в 1 с мало (design
|
||
# change track2-reconcile-cost, решение 9)
|
||
interval = (
|
||
p.reconcile_sample_interval_s
|
||
if p.is_reconcile
|
||
else p.sample_interval_s
|
||
)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
"none",
|
||
"--cpus",
|
||
p.sampler_cpus,
|
||
"--memory",
|
||
p.sampler_memory,
|
||
"--memory-swap",
|
||
p.sampler_memory,
|
||
"-v",
|
||
f"{SAMPLER_SCRIPT}:/sampler.sh:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"bash",
|
||
image,
|
||
"/sampler.sh",
|
||
f"{interval:g}",
|
||
mode,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"sampler did not start: {proc.stderr}")
|
||
want = {
|
||
ident[:12]
|
||
for ident in sh(
|
||
"docker",
|
||
"ps",
|
||
"-q",
|
||
"--no-trunc",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
).split()
|
||
}
|
||
want |= {self.full_id(name)[:12] for name in (K6, SAMPLER)}
|
||
deadline = time.monotonic() + 5
|
||
problem = "no samples"
|
||
while time.monotonic() < deadline:
|
||
time.sleep(0.5)
|
||
samples = scaling.parse_samples(self.sampler_logs()[0])
|
||
ids = {ident for _, ident in samples.cg}
|
||
if want <= ids and samples.vm:
|
||
return
|
||
problem = (
|
||
f"cgroups seen {len(want & ids)}/{len(want)} (missing "
|
||
f"{sorted(want - ids)}), vm {len(samples.vm)}"
|
||
)
|
||
tail = sh("docker", "logs", "--tail", "5", SAMPLER, check=False)
|
||
raise BenchError(f"sampler: {problem}\n{tail}")
|
||
|
||
def make_images(self, seed: int) -> dict[str, Any]:
|
||
"""Файлы классов профиля очереди в `.work/` (решение 5): контейнер на
|
||
образе стенда без сети; ненулевой код — файл вышел за лимит контракта."""
|
||
self.work_dir.mkdir(parents=True, exist_ok=True)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--name",
|
||
IMAGES,
|
||
"--network",
|
||
"none",
|
||
"-v",
|
||
f"{ROOT / 'bench'}:/bench:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"python",
|
||
APP_IMAGE,
|
||
"/bench/make_images.py",
|
||
str(seed),
|
||
"/work",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"make_images: exit {proc.returncode}\n{proc.stderr[-2000:]}",
|
||
)
|
||
return json.loads(proc.stdout)
|
||
|
||
def queue_overrides(self, env: dict[str, dict[str, str]]) -> None:
|
||
"""Плагин брокера и политика worker (`queue.compose.yml`), окружение
|
||
ячейки — override `VARIANT_COMPOSE`, как у профиля загрузок: сервисы и
|
||
их переменные."""
|
||
path = self.work_dir / VARIANT_COMPOSE
|
||
lines = ["services:"]
|
||
for service, values in env.items():
|
||
lines += [f" {service}:", " environment:"]
|
||
lines += [
|
||
f' {key}: "{value}"' for key, value in values.items()
|
||
]
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
self.extra_files = [QUEUE_COMPOSE, path]
|
||
|
||
def start_queue_sampler(self, queue: str) -> None:
|
||
"""Сэмплер в режиме `queue`: сеть стенда (метрики брокера, статистика LB,
|
||
`psql \\watch`), `--cgroupns host`, свои лимиты; готов, когда видит все
|
||
контейнеры проекта, тик ВМ и первую строку заданий."""
|
||
p = self.profile
|
||
image = sh("docker", "inspect", "-f", "{{.Config.Image}}", DB)
|
||
for name in ("samples.log", "jobs.log"):
|
||
(self.work_dir / name).unlink(missing_ok=True)
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"-d",
|
||
"--name",
|
||
SAMPLER,
|
||
"--cgroupns",
|
||
"host",
|
||
"--network",
|
||
NETWORK,
|
||
"--cpus",
|
||
p.sampler_cpus,
|
||
"--memory",
|
||
p.sampler_memory,
|
||
"--memory-swap",
|
||
p.sampler_memory,
|
||
"--env",
|
||
"POSTGRES_PASSWORD",
|
||
"-e",
|
||
f"POSTGRES_USER={self.env_value('POSTGRES_USER')}",
|
||
"-e",
|
||
f"POSTGRES_DB={self.env_value('POSTGRES_DB')}",
|
||
"-v",
|
||
f"{SAMPLER_SCRIPT}:/sampler.sh:ro",
|
||
"-v",
|
||
f"{self.work_dir}:/work",
|
||
"--entrypoint",
|
||
"bash",
|
||
image,
|
||
"/sampler.sh",
|
||
f"{p.sample_interval_s:g}",
|
||
"queue",
|
||
queue,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env={
|
||
**os.environ,
|
||
"POSTGRES_PASSWORD": self.env["POSTGRES_PASSWORD"],
|
||
},
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"sampler did not start: {proc.stderr}")
|
||
deadline = time.monotonic() + 10
|
||
problem = "no samples"
|
||
while time.monotonic() < deadline:
|
||
time.sleep(0.5)
|
||
samples = scaling.parse_samples(self.sampler_logs()[0])
|
||
jobs = overload.parse_jobs(self.read_work("jobs.log"))
|
||
if samples.cg and samples.vm and jobs:
|
||
return
|
||
problem = (
|
||
f"cgroups {len(samples.cg)}, vm {len(samples.vm)}, "
|
||
f"jobs {len(jobs)}"
|
||
)
|
||
tail = sh("docker", "logs", "--tail", "5", SAMPLER, check=False)
|
||
raise BenchError(f"queue sampler: {problem}\n{tail}")
|
||
|
||
def env_value(self, name: str) -> str:
|
||
"""Значение подстановки compose из `bench.env` стенда (не секрет)."""
|
||
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
|
||
if line.startswith(f"{name}="):
|
||
return line.split("=", 1)[1].strip()
|
||
raise BenchError(f"{name} is not in {ENV_FILE.name}")
|
||
|
||
def read_work(self, name: str) -> str:
|
||
path = self.work_dir / name
|
||
if not path.exists():
|
||
return ""
|
||
return path.read_text(encoding="utf-8", errors="replace")
|
||
|
||
def container_map(self) -> dict[str, dict[str, str]]:
|
||
"""id12 → имя и роль контейнера (решение 1 design change
|
||
`profile-resources`): контейнеры проекта (и удалённые не
|
||
попадут — их добавляет поток событий) по метке сервиса compose,
|
||
k6 и сэмплер — по имени."""
|
||
found: dict[str, dict[str, str]] = {}
|
||
listing = sh(
|
||
"docker",
|
||
"ps",
|
||
"-a",
|
||
"--no-trunc",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
"--format",
|
||
'{{.ID}}|{{.Names}}|{{.Label "com.docker.compose.service"}}',
|
||
)
|
||
for line in listing.splitlines():
|
||
ident, name, service = (line.split("|") + ["", ""])[:3]
|
||
if ident:
|
||
found[ident[:12]] = {"name": name, "role": service}
|
||
for name, role in ((K6, "k6"), (SAMPLER, "sampler")):
|
||
ident = sh("docker", "inspect", "-f", "{{.Id}}", name, check=False)
|
||
if re.fullmatch(r"[0-9a-f]{64}", ident):
|
||
found[ident[:12]] = {"name": name, "role": role}
|
||
return found
|
||
|
||
def healthcheck_actions(self) -> set[str]:
|
||
"""Действия `exec_start` проверок здоровья контейнеров проекта, как
|
||
их пишет поток событий: их не путать с обращением скрипта замены к
|
||
стенду (найдено в apply change `profile-resources`)."""
|
||
actions = set()
|
||
names = sh(
|
||
"docker",
|
||
"ps",
|
||
"-q",
|
||
"--filter",
|
||
f"label=com.docker.compose.project={PROJECT}",
|
||
).split()
|
||
for name in names:
|
||
test = json.loads(
|
||
sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{json .Config.Healthcheck}}",
|
||
name,
|
||
check=False,
|
||
)
|
||
or "null",
|
||
)
|
||
command = (test or {}).get("Test") or []
|
||
if command[:1] == ["CMD-SHELL"]:
|
||
actions.add(f"exec_start: /bin/sh -c {' '.join(command[1:])}")
|
||
elif command[:1] == ["CMD"]:
|
||
actions.add(f"exec_start: {' '.join(command[1:])}")
|
||
return actions
|
||
|
||
def sampler_logs(self) -> tuple[str, str]:
|
||
texts = []
|
||
for name in ("samples.log", "pg-samples.log"):
|
||
path = self.work_dir / name
|
||
texts.append(
|
||
path.read_text(encoding="utf-8", errors="replace")
|
||
if path.exists()
|
||
else "",
|
||
)
|
||
return texts[0], texts[1]
|
||
|
||
def stop_sampler(self) -> None:
|
||
# -v: сэмплер — на образе PostgreSQL, у него volume образа
|
||
# /var/lib/postgresql; без -v каждый запуск оставлял бы анонимный
|
||
# volume (найдено в apply change upload-memory)
|
||
subprocess.run(
|
||
["docker", "rm", "-f", "-v", SAMPLER],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
|
||
def full_id(self, name: str) -> str:
|
||
return sh("docker", "inspect", "-f", "{{.Id}}", name)
|
||
|
||
def container_limits(self, name: str) -> dict[str, Any]:
|
||
host = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .HostConfig}}", name),
|
||
)
|
||
config = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .Config}}", name),
|
||
)
|
||
return {
|
||
"nano_cpus": host.get("NanoCpus"),
|
||
"memory": host.get("Memory"),
|
||
"memory_swap": host.get("MemorySwap"),
|
||
"published_ports": host.get("PortBindings") or {},
|
||
"healthcheck": (config.get("Healthcheck") or {}).get("Test"),
|
||
"log_driver": (host.get("LogConfig") or {}).get("Type"),
|
||
}
|
||
|
||
def app_state(self) -> dict[str, Any]:
|
||
"""Состояние приложения в целом: все реплики работают, ни одна не
|
||
убита по памяти и не перезапускалась."""
|
||
names = self.replicas(stopped=True)
|
||
states = [self.state(name) for name in names]
|
||
return {
|
||
"running": len(states) == self.profile.replicas
|
||
and all(state["running"] for state in states),
|
||
"oom": any(state["oom"] for state in states),
|
||
"restarts": sum(state["restarts"] for state in states),
|
||
}
|
||
|
||
def state(self, name: str) -> dict[str, Any]:
|
||
state = json.loads(
|
||
sh("docker", "inspect", "-f", "{{json .State}}", name),
|
||
)
|
||
restarts = sh("docker", "inspect", "-f", "{{.RestartCount}}", name)
|
||
return {
|
||
"running": state.get("Running"),
|
||
"oom": state.get("OOMKilled"),
|
||
"restarts": int(restarts or 0),
|
||
"exit_code": state.get("ExitCode"),
|
||
}
|
||
|
||
def down_storages(self) -> list[str]:
|
||
"""Хранилища, которые за прогон умирали, убивались или рестартовали."""
|
||
down = []
|
||
for name in (DB, CACHE):
|
||
state = self.state(name)
|
||
if not state["running"] or state["oom"] or state["restarts"]:
|
||
down.append(name)
|
||
return down
|
||
|
||
def psql(
|
||
self,
|
||
sql: str,
|
||
*,
|
||
check: bool = True,
|
||
container: str = DB,
|
||
) -> str:
|
||
"""SQL в базе стенда через stdin: пользователь контейнера, без пароля.
|
||
|
||
`container` — исходное хранилище или хранилище восстановления.
|
||
"""
|
||
proc = subprocess.run(
|
||
["docker", "exec", "-i", container, "sh", "-c", PSQL],
|
||
input=sql,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(f"psql: exit {proc.returncode}\n{proc.stderr}")
|
||
return proc.stdout.strip()
|
||
|
||
def maintain(self) -> float:
|
||
"""VACUUM (ANALYZE) и CHECKPOINT после наполнения; секунды.
|
||
|
||
Статистика и состояние таблиц одинаковы у всех меток, а не
|
||
зависят от того, когда сработал autovacuum (решение 4).
|
||
"""
|
||
started = time.monotonic()
|
||
self.psql("VACUUM (ANALYZE);\nCHECKPOINT;\n")
|
||
return time.monotonic() - started
|
||
|
||
def storage(self) -> dict[str, Any]:
|
||
"""Объём таблицы сообщений и её индексов, число строк."""
|
||
row = self.psql(
|
||
"SELECT pg_table_size('messages'), pg_indexes_size('messages'), "
|
||
"count(*) FROM messages;\n",
|
||
)
|
||
table, indexes, count = (int(value) for value in row.split("|"))
|
||
return {
|
||
"messages_rows": count,
|
||
"messages_table_bytes": table,
|
||
"messages_indexes_bytes": indexes,
|
||
"db_bytes_per_message": (
|
||
round((table + indexes) / count) if count else None
|
||
),
|
||
}
|
||
|
||
def chat_counts(self) -> dict[str, int]:
|
||
"""Число сообщений по чатам (проверка распределения наполнения)."""
|
||
rows = self.psql(
|
||
"SELECT chat_id, count(*) FROM messages GROUP BY chat_id;\n",
|
||
)
|
||
return {
|
||
chat: int(count)
|
||
for chat, count in (line.split("|") for line in rows.splitlines())
|
||
}
|
||
|
||
def indexes(self, container: str = DB) -> list[str]:
|
||
"""Определения индексов messages: до и после этапов с вариантами."""
|
||
return self.psql(
|
||
"SELECT indexdef FROM pg_indexes WHERE tablename = 'messages' "
|
||
"ORDER BY indexname;\n",
|
||
container=container,
|
||
).splitlines()
|
||
|
||
def message_count(self) -> int:
|
||
return int(self.psql("SELECT count(*) FROM messages;\n"))
|
||
|
||
def write_probe(
|
||
self,
|
||
app_index: str,
|
||
columns: str | None,
|
||
) -> dict[str, Any]:
|
||
"""Одна проба записи при индексе истории `columns` (None — без него).
|
||
|
||
Всё в транзакции, которая откатывается: индекс приложения
|
||
удаляется, конфигурация строится заново, `CHECKPOINT`, вставка
|
||
порции во все чаты в пропорции наполнения и сообщение максимальной
|
||
длины (решение 5 design change `track2-query-plans`).
|
||
"""
|
||
p = self.profile
|
||
create = (
|
||
f"CREATE INDEX bench_probe ON messages {columns};\n"
|
||
if columns
|
||
else ""
|
||
)
|
||
size = "pg_relation_size('bench_probe')" if columns else "0"
|
||
proc = subprocess.run(
|
||
["docker", "exec", "-i", DB, "sh", "-c", PSQL],
|
||
input=(
|
||
"SET lock_timeout = '5s';\n"
|
||
"BEGIN;\n"
|
||
"CREATE TEMP TABLE probe_src ON COMMIT DROP AS "
|
||
"SELECT chat_id, sender_id, "
|
||
"row_number() OVER (ORDER BY created_at, id) AS g "
|
||
"FROM (SELECT chat_id, sender_id, created_at, id FROM messages "
|
||
f"ORDER BY created_at, id LIMIT {p.write_probe_rows}) s;\n"
|
||
f'DROP INDEX "{app_index}";\n'
|
||
f"{create}"
|
||
f"SELECT {size}, (SELECT count(*) FROM messages), "
|
||
"(SELECT count(*) FROM probe_src);\n"
|
||
"CHECKPOINT;\n"
|
||
"EXPLAIN (ANALYZE, BUFFERS, WAL) INSERT INTO messages "
|
||
"(id, chat_id, sender_id, text, created_at) "
|
||
"SELECT gen_random_uuid(), chat_id, sender_id, "
|
||
f"rpad('probe-' || g, {p.text_length}, 'x'), "
|
||
"now() + g * interval '1 microsecond' FROM probe_src;\n"
|
||
f"{LONG_TEXT_PROBE}"
|
||
"ROLLBACK;\n"
|
||
),
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"write probe: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
head, _, plan = proc.stdout.strip().partition("\n")
|
||
index_bytes, rows_before, rows = (int(v) for v in head.split("|"))
|
||
wal = re.search(r"WAL: ([^\n]*)", plan)
|
||
|
||
def wal_field(name: str) -> int:
|
||
# EXPLAIN не печатает нулевые поля WAL
|
||
match = (
|
||
re.search(rf"\b{name}=(\d+)", wal.group(1)) if wal else None
|
||
)
|
||
return int(match.group(1)) if match else 0
|
||
|
||
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
|
||
long_text = re.search(r"long-text: ([^\n]*)", proc.stderr)
|
||
if execution is None or long_text is None:
|
||
raise BenchError(f"write probe: unexpected output\n{proc.stdout}")
|
||
return {
|
||
"index_bytes": index_bytes,
|
||
"rows_before": rows_before,
|
||
"rows": rows,
|
||
"time_ms": float(execution.group(1)),
|
||
"wal_records": wal_field("records"),
|
||
"wal_fpi": wal_field("fpi"),
|
||
"wal_bytes": wal_field("bytes"),
|
||
"long_text": long_text.group(1).strip(),
|
||
}
|
||
|
||
def db_log(self) -> str:
|
||
"""Журнал Postgres стенда (stderr контейнера): только в памяти."""
|
||
proc = subprocess.run(
|
||
["docker", "logs", DB],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def passport(self) -> dict[str, Any]:
|
||
"""Хранилища и окружение API для паспорта; паролей здесь нет."""
|
||
settings = self.psql(
|
||
"SELECT name, current_setting(name) FROM pg_settings "
|
||
"WHERE name = ANY(string_to_array("
|
||
f"'{','.join(PG_SETTINGS)}', ',')) ORDER BY name;\n",
|
||
)
|
||
info = sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" INFO server',
|
||
)
|
||
valkey_version = re.search(r"^valkey_version:(\S+)", info, re.M)
|
||
memory = sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" CONFIG GET "maxmemory*"',
|
||
).splitlines()
|
||
replica = self.replicas()[0]
|
||
# без секретов: пароли хранилищ, секрет и ключ доступа S3
|
||
app_env = self.app_env(replica)
|
||
|
||
def image(name: str) -> dict[str, str]:
|
||
return {
|
||
"image": sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Config.Image}}",
|
||
name,
|
||
),
|
||
"id": sh("docker", "inspect", "-f", CONTAINER_IMAGE, name),
|
||
}
|
||
|
||
extra: dict[str, Any] = {}
|
||
if (
|
||
self.profile.is_uploads
|
||
or self.profile.is_reconcile
|
||
or self.profile.runs_pipeline
|
||
):
|
||
# S3 — только у профилей загрузок, сверки, очереди и цены
|
||
# телеметрии; образ и аргументы — рабочего стенда (у сверки ещё
|
||
# порт метрик, meta.reconcile)
|
||
extra["s3"] = image(S3)
|
||
if self.profile.runs_pipeline:
|
||
# брокер запускают только профили очереди и цены телеметрии
|
||
extra["rabbitmq"] = image(BROKER)
|
||
# режим телеметрии стенда (design change track1-telemetry-cost,
|
||
# решение 13): у цены телеметрии — вариант прогона
|
||
extra["telemetry"] = {"mode": "on" if self.telemetry else "off"}
|
||
if self.telemetry:
|
||
extra["telemetry"].update(self.telemetry_passport())
|
||
return {
|
||
**extra,
|
||
# буфер приёма сокетов реплики: sysctls сервиса api (design
|
||
# change upload-memory, решение 6), одинаков у всех профилей
|
||
"api_tcp_rmem": self.tcp_rmem(replica),
|
||
"postgres": {
|
||
**image(DB),
|
||
"version": self.psql("SHOW server_version;\n"),
|
||
"settings": dict(
|
||
line.split("|", 1) for line in settings.splitlines()
|
||
),
|
||
},
|
||
"valkey": {
|
||
**image(CACHE),
|
||
"version": valkey_version.group(1) if valkey_version else None,
|
||
**dict(zip(memory[::2], memory[1::2], strict=True)),
|
||
},
|
||
# конфигурация LB — infra/haproxy/haproxy.cfg рабочего стенда
|
||
"haproxy": {
|
||
**image(LB),
|
||
"version": sh("docker", "exec", LB, "haproxy", "-v")
|
||
.splitlines()[0]
|
||
.strip(),
|
||
},
|
||
"app_env": app_env,
|
||
}
|
||
|
||
def snapshot(self) -> dict[str, Any]:
|
||
"""CPU и память приложения, CPU PostgreSQL и генератора.
|
||
|
||
Приложение — сумма по репликам (`VmHWM` — максимум): при одной
|
||
реплике — числа её процесса, как до LB. Если реплика уже мертва
|
||
(OOM), поля приложения — None: снимок нужен и для записи
|
||
проваленного прогона.
|
||
"""
|
||
replicas = [
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
name,
|
||
"sh",
|
||
"-c",
|
||
"cat /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.current; "
|
||
"grep -E '^(VmRSS|VmHWM):' /proc/1/status",
|
||
check=False,
|
||
)
|
||
for name in self.replicas(stopped=True)
|
||
]
|
||
k6 = sh("docker", "exec", K6, "cat", "/sys/fs/cgroup/cpu.stat")
|
||
db = sh(
|
||
"docker",
|
||
"exec",
|
||
DB,
|
||
"cat",
|
||
"/sys/fs/cgroup/cpu.stat",
|
||
check=False,
|
||
)
|
||
|
||
def number(pattern: str, text: str) -> int | None:
|
||
match = re.search(pattern, text, re.MULTILINE)
|
||
return int(match.group(1)) if match else None
|
||
|
||
def total(pattern: str) -> int | None:
|
||
values = [number(pattern, app) for app in replicas]
|
||
if not values or None in values:
|
||
return None
|
||
return sum(v for v in values if v is not None)
|
||
|
||
hwm = [number(r"^VmHWM:\s+(\d+) kB", app) for app in replicas]
|
||
return {
|
||
"app_usec": total(r"^usage_usec (\d+)$"),
|
||
"mem_bytes": total(r"^(\d+)$"),
|
||
"rss_kib": total(r"^VmRSS:\s+(\d+) kB"),
|
||
"hwm_kib": (
|
||
None if not hwm or None in hwm else max(v or 0 for v in hwm)
|
||
),
|
||
"k6_usec": number(r"^usage_usec (\d+)$", k6),
|
||
"db_usec": number(r"^usage_usec (\d+)$", db),
|
||
}
|
||
|
||
def k6_args(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
*extra: str,
|
||
) -> list[str]:
|
||
"""Команда `k6 run` сценария в контейнере k6 стенда; `extra` —
|
||
дополнительные флаги k6."""
|
||
common = {
|
||
"TEXT_LENGTH": self.profile.text_length,
|
||
"PAGE_LIMIT": self.profile.page_limit,
|
||
}
|
||
flags = [
|
||
item
|
||
for key, value in {**common, **env}.items()
|
||
for item in ("-e", f"{key}={value}")
|
||
]
|
||
return [
|
||
"docker",
|
||
"exec",
|
||
K6,
|
||
"k6",
|
||
"run",
|
||
"--quiet",
|
||
"--no-color",
|
||
"--log-format",
|
||
"raw",
|
||
*extra,
|
||
*flags,
|
||
f"/scripts/{script}",
|
||
]
|
||
|
||
def k6(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
*,
|
||
check: bool = True,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
proc = subprocess.run(
|
||
self.k6_args(script, env),
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check and proc.returncode != 0:
|
||
raise BenchError(k6_failure(script, proc))
|
||
return proc
|
||
|
||
def k6_start(self, script: str, env: dict[str, Any]) -> subprocess.Popen:
|
||
"""Сценарий k6 в фоне: запись во время backup."""
|
||
return subprocess.Popen(
|
||
self.k6_args(script, env),
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
)
|
||
|
||
def k6_console(
|
||
self,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
name: str,
|
||
) -> subprocess.Popen:
|
||
"""Сценарий k6 в фоне; `console.log` k6 пишет сам в файл `name`
|
||
рабочего каталога, прочий вывод — в `<name>.out`.
|
||
|
||
Строка на каждый запрос фазы отказа переполнила бы трубу, пока
|
||
оркестратор ждёт момента действия. А через вывод `docker exec`
|
||
в самопроверке пропали две строки из 4 601 (вероятно, склеились
|
||
строки разных VU), поэтому файл пишет k6, а не docker.
|
||
"""
|
||
# k6 дописывает в существующий файл консоли: файл запуска с тем же
|
||
# именем от убитого процесса удвоил бы строки (метка `baseline`
|
||
# профиля telemetry: разогрев 24 из 12)
|
||
(self.work_dir / name).unlink(missing_ok=True)
|
||
with (self.work_dir / f"{name}.out").open("w") as out:
|
||
return subprocess.Popen(
|
||
self.k6_args(
|
||
script,
|
||
env,
|
||
"--console-output",
|
||
f"/work/{name}",
|
||
),
|
||
stdout=out,
|
||
stderr=subprocess.STDOUT,
|
||
)
|
||
|
||
def pg_dump(self, path: Path) -> None:
|
||
"""Логический backup исходной базы (формат custom) в файл хоста."""
|
||
with path.open("wb") as file:
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
DB,
|
||
"sh",
|
||
"-c",
|
||
'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc',
|
||
],
|
||
stdout=file,
|
||
stderr=subprocess.PIPE,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(f"pg_dump: exit {proc.returncode}\n{proc.stderr}")
|
||
|
||
def pg_restore(self, path: Path) -> tuple[float, float]:
|
||
"""Backup в хранилище восстановления: копия файла и pg_restore -j.
|
||
|
||
Возвращает секунды копирования и восстановления.
|
||
"""
|
||
started = time.monotonic()
|
||
sh("docker", "cp", str(path), f"{RESTORE_DB}:/tmp/backup.dump")
|
||
copied = time.monotonic()
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"exec",
|
||
RESTORE_DB,
|
||
"sh",
|
||
"-c",
|
||
'exec pg_restore -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
||
f"-j {self.profile.restore_jobs} --no-owner --no-privileges "
|
||
"--exit-on-error /tmp/backup.dump",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
raise BenchError(
|
||
f"pg_restore: exit {proc.returncode}\n{proc.stderr}",
|
||
)
|
||
return copied - started, time.monotonic() - copied
|
||
|
||
def switch_api(self, host: str) -> None:
|
||
"""Реплики API заново, с адресом базы `host`: `POSTGRES_HOST`
|
||
перекрывает значение из `bench/bench.env` для `x-db-env`.
|
||
|
||
Готовы — когда процесс каждой новой реплики сам отвечает на
|
||
`/health/ready`, и только затем LB видит их `UP`. Пересозданный
|
||
контейнер получает тот же IP, а LB ещё держит слот этого адреса
|
||
`UP` от прежнего: `wait_ready` сразу после `up` вернулся бы до
|
||
старта процесса, и первые запросы получали бы 503 `<NOSRV>`, когда
|
||
LB поймает отказ соединения (design change `attachments`,
|
||
«Уточнения apply», п. 9). После готовности процесса проверки LB
|
||
успешны: слот, помеченный `DOWN` до неё, `wait_ready` дождётся
|
||
снова `UP`, а не помеченный — уже ведёт на готовый процесс.
|
||
"""
|
||
env = self.env
|
||
self.env = {**env, "POSTGRES_HOST": host}
|
||
try:
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--force-recreate",
|
||
APP_SERVICE,
|
||
)
|
||
finally:
|
||
self.env = env
|
||
self.wait_processes_ready()
|
||
self.wait_ready()
|
||
|
||
def wait_processes_ready(self, timeout: float = 60) -> None:
|
||
"""Процесс каждой реплики отвечает на свой `/health/ready` изнутри
|
||
контейнера — та же команда, что healthcheck образа (у стенда он
|
||
выключен). Только docker CLI, без сети хоста."""
|
||
deadline = time.monotonic() + timeout
|
||
pending = set(self.replicas())
|
||
while pending and time.monotonic() < deadline:
|
||
for name in sorted(pending):
|
||
state = self.state(name)
|
||
if not state["running"]:
|
||
reason = "oom" if state["oom"] else "exited"
|
||
raise BenchError(f"{name} died during start ({reason})")
|
||
probe = subprocess.run(
|
||
["docker", "exec", name, "python", "-c", APP_READY_PROBE],
|
||
capture_output=True,
|
||
check=False,
|
||
)
|
||
if probe.returncode == 0:
|
||
pending.discard(name)
|
||
if pending:
|
||
time.sleep(0.25)
|
||
if pending:
|
||
raise BenchError(
|
||
f"api replicas {sorted(pending)} not ready in {timeout:.0f}s",
|
||
)
|
||
|
||
def logs(self, container: str, since: float) -> str:
|
||
"""Журнал контейнера с отметками Docker (часы ВМ) с момента
|
||
`since`; HAProxy и PostgreSQL пишут в stdout и stderr."""
|
||
proc = subprocess.run(
|
||
["docker", "logs", "-t", "--since", f"{since:.3f}", container],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def table_counts(self, container: str = DB) -> dict[str, int]:
|
||
"""Число строк таблиц данных (сверка восстановления)."""
|
||
sql = " UNION ALL ".join(
|
||
f"SELECT '{table}', count(*) FROM {table}" for table in DATA_TABLES
|
||
)
|
||
rows = self.psql(f"{sql};\n", container=container)
|
||
return {
|
||
table: int(count)
|
||
for table, count in (line.split("|") for line in rows.splitlines())
|
||
}
|
||
|
||
def message_rows(self, container: str = DB) -> dict[str, tuple[str, str]]:
|
||
"""Сообщения базы: id → (chat_id, md5 всех полей)."""
|
||
rows = self.psql(
|
||
"SELECT id, chat_id, md5(concat_ws('|', chat_id, sender_id, "
|
||
"text, created_at)) FROM messages;\n",
|
||
container=container,
|
||
)
|
||
return {
|
||
message: (chat, digest)
|
||
for message, chat, digest in (
|
||
line.split("|") for line in rows.splitlines()
|
||
)
|
||
}
|
||
|
||
def work_json(self, name: str) -> Any:
|
||
return json.loads((self.work_dir / name).read_text(encoding="utf-8"))
|
||
|
||
|
||
# -------------------------------------------------------------- измерение
|
||
|
||
|
||
def parse_summary(path: Path) -> dict[str, Any]:
|
||
"""Итог k6: успехи, ошибки, RPS и перцентили успешных запросов.
|
||
|
||
Нет файла (k6 остановлен сторожем до итога) — нулевой итог.
|
||
"""
|
||
if not path.exists():
|
||
return {
|
||
"duration_s": 0,
|
||
"requests_ok": 0,
|
||
"requests_failed": 0,
|
||
"check_failed": 0,
|
||
"rps": None,
|
||
"error_ratio": None,
|
||
"p50_ms": None,
|
||
"p95_ms": None,
|
||
"p99_ms": None,
|
||
"max_ms": None,
|
||
}
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
metrics = data["metrics"]
|
||
failed = metrics.get("http_req_failed", {}).get("values", {})
|
||
ok = int(failed.get("fails", 0))
|
||
bad = int(failed.get("passes", 0))
|
||
checks = metrics.get("checks", {}).get("values", {})
|
||
success = metrics.get(SUCCESS_METRIC, {}).get("values", {})
|
||
duration = data["state"]["testRunDurationMs"] / 1000
|
||
return {
|
||
"duration_s": round(duration, 3),
|
||
"requests_ok": ok,
|
||
"requests_failed": bad,
|
||
"check_failed": int(checks.get("fails", 0)),
|
||
"rps": round(ok / duration, 1) if duration else None,
|
||
"error_ratio": round(bad / (ok + bad), 6) if ok + bad else None,
|
||
"p50_ms": rounded(success.get("p(50)")),
|
||
"p95_ms": rounded(success.get("p(95)")),
|
||
"p99_ms": rounded(success.get("p(99)")),
|
||
"max_ms": rounded(success.get("max")),
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class Measured:
|
||
summary: dict[str, Any]
|
||
before: dict[str, Any]
|
||
after: dict[str, Any]
|
||
rss_max_kib: int | None
|
||
|
||
def cores(self, key: str) -> float | None:
|
||
"""Среднее число занятых ядер за прогон по счётчику cgroup."""
|
||
duration = self.summary["duration_s"]
|
||
start, end = self.before[key], self.after[key]
|
||
if not duration or start is None or end is None:
|
||
return None
|
||
return round((end - start) / 1e6 / duration, 3)
|
||
|
||
@property
|
||
def app_cpu(self) -> float | None:
|
||
return self.cores("app_usec")
|
||
|
||
@property
|
||
def generator_cpu(self) -> float | None:
|
||
return self.cores("k6_usec")
|
||
|
||
@property
|
||
def db_cpu(self) -> float | None:
|
||
return self.cores("db_usec")
|
||
|
||
|
||
def shorten(line: str) -> str:
|
||
"""Строка журнала в файл ошибок: длинная — начало, «…» и конец."""
|
||
if len(line) <= ERRORS_HEAD + ERRORS_TAIL:
|
||
return line
|
||
return f"{line[:ERRORS_HEAD]} … {line[-ERRORS_TAIL:]}"
|
||
|
||
|
||
def k6_failure(script: str, proc: subprocess.CompletedProcess[str]) -> str:
|
||
tail = (proc.stdout + proc.stderr)[-3000:]
|
||
return f"k6 {script} failed ({proc.returncode}):\n{tail}"
|
||
|
||
|
||
def measure(
|
||
stand: Stand,
|
||
script: str,
|
||
env: dict[str, Any],
|
||
raw: Path,
|
||
) -> Measured:
|
||
"""Один запуск k6 со снимками до и после, опросом RSS и сторожем."""
|
||
before = stand.snapshot()
|
||
with Watcher(stand.replicas()) as watcher:
|
||
proc = stand.k6(script, env, check=False)
|
||
# если приложение или хранилище умерло, k6 остановлен сторожем: прогон
|
||
# пишется с пометкой, а не падает здесь
|
||
died = watcher.app_died or watcher.stand_died
|
||
if proc.returncode != 0 and not died:
|
||
raise BenchError(k6_failure(script, proc))
|
||
after = stand.snapshot()
|
||
values = [
|
||
v for v in (before["rss_kib"], *watcher.samples, after["rss_kib"]) if v
|
||
]
|
||
return Measured(
|
||
summary=parse_summary(raw),
|
||
before=before,
|
||
after=after,
|
||
rss_max_kib=max(values) if values else None,
|
||
)
|
||
|
||
|
||
def run_flags(
|
||
profile: Profile,
|
||
measured: Measured,
|
||
app: dict[str, Any],
|
||
storages_down: list[str],
|
||
) -> list[str]:
|
||
flags = []
|
||
if app["oom"]:
|
||
flags.append("oom")
|
||
if not app["running"] or app["restarts"]:
|
||
flags.append("app_down")
|
||
if storages_down:
|
||
flags.append("stand_down")
|
||
summary = measured.summary
|
||
if summary["requests_failed"] or summary["check_failed"]:
|
||
flags.append("errors")
|
||
cpu = measured.generator_cpu
|
||
if cpu is not None and cpu >= profile.generator_cpu_limit:
|
||
flags.append("generator")
|
||
return flags
|
||
|
||
|
||
# ----------------------------------------------------------- план запроса
|
||
|
||
|
||
def plan_entries(log: str, marker: str) -> list[str]:
|
||
"""Записи auto_explain между маркерами этапа плана.
|
||
|
||
Запись начинается строкой `LOG: duration: … plan:` и продолжается
|
||
строками с табуляцией; префикс времени и pid отбрасывается.
|
||
"""
|
||
lines = log.splitlines()
|
||
bounds = [
|
||
next(
|
||
(i for i, line in enumerate(lines) if f"{marker} {edge}" in line),
|
||
None,
|
||
)
|
||
for edge in ("start", "end")
|
||
]
|
||
if None in bounds:
|
||
return []
|
||
entries: list[list[str]] = []
|
||
current: list[str] | None = None
|
||
for line in lines[bounds[0] + 1 : bounds[1]]:
|
||
if line.startswith("\t"):
|
||
if current is not None:
|
||
current.append(line[1:])
|
||
continue
|
||
current = None
|
||
head = line.partition("LOG: ")[2].rstrip()
|
||
if head.startswith("duration:") and head.endswith("plan:"):
|
||
current = [head]
|
||
entries.append(current)
|
||
return ["\n".join(entry) for entry in entries]
|
||
|
||
|
||
def is_history(entry: str) -> bool:
|
||
"""Запись — запрос страницы истории (читает таблицу messages).
|
||
|
||
Остальные записи окна (проверка сессии с хэшем токена в параметрах,
|
||
служебные запросы) в результаты не попадают.
|
||
"""
|
||
text = entry.partition("Query Parameters:")[0]
|
||
return "Query Text:" in text and "FROM messages" in text
|
||
|
||
|
||
def plan_parts(entry: str) -> dict[str, Any] | None:
|
||
"""Текст запроса, параметры и тело плана записи auto_explain."""
|
||
lines = entry.splitlines()
|
||
starts = [i for i, line in enumerate(lines) if line.startswith("Query ")]
|
||
if len(starts) != 2 or not lines[starts[1]].startswith("Query Param"):
|
||
return None
|
||
text = "\n".join(lines[starts[0] : starts[1]])
|
||
params = lines[starts[1]].partition(": ")[2]
|
||
return {
|
||
"text": text.removeprefix("Query Text: "),
|
||
"params": [
|
||
value.replace("''", "'")
|
||
for value in re.findall(r"\$\d+ = '((?:[^']|'')*)'", params)
|
||
],
|
||
"plan": "\n".join(lines[starts[1] + 1 :]),
|
||
}
|
||
|
||
|
||
def plan_summary(plan: str, time_ms: float | None) -> dict[str, Any]:
|
||
"""Узел доступа к messages, буферы верхнего узла, строки, время."""
|
||
node = re.search(
|
||
r"((?:Parallel )?(?:Seq Scan|Index Only Scan|Index Scan|"
|
||
r"Bitmap Heap Scan)(?: Backward)?(?: using \w+)? on messages)",
|
||
plan,
|
||
)
|
||
buffers = re.search(r"Buffers: ([^\n]*)", plan)
|
||
hit = re.search(r"shared hit=(\d+)", buffers.group(1)) if buffers else None
|
||
read = re.search(r"read=(\d+)", buffers.group(1)) if buffers else None
|
||
rows = re.search(r"\(actual time=[\d.]+\.\.[\d.]+ rows=([\d.]+)", plan)
|
||
if time_ms is None:
|
||
execution = re.search(r"Execution Time: ([\d.]+) ms", plan)
|
||
time_ms = float(execution.group(1)) if execution else None
|
||
return {
|
||
"node": node.group(1) if node else None,
|
||
"shared_hit": int(hit.group(1)) if hit else 0,
|
||
"shared_read": int(read.group(1)) if read else 0,
|
||
"rows": float(rows.group(1)) if rows else None,
|
||
"time_ms": time_ms,
|
||
# параметры в условиях плана — generic plan, литералы — custom
|
||
"kind": "generic" if re.search(r"\$\d", plan) else "custom",
|
||
}
|
||
|
||
|
||
def entry_duration(entry: str) -> float | None:
|
||
match = re.match(r"duration: ([\d.]+) ms", entry)
|
||
return float(match.group(1)) if match else None
|
||
|
||
|
||
def index_name(plan: str) -> str | None:
|
||
match = INDEX_NODE_RE.search(plan)
|
||
return (
|
||
next((name for name in match.groups() if name), None)
|
||
if match
|
||
else None
|
||
)
|
||
|
||
|
||
def render_explain(
|
||
size: int,
|
||
git: dict[str, Any],
|
||
profile: Profile,
|
||
pages: list[dict[str, Any]],
|
||
entries: list[str],
|
||
compared: list[tuple[dict[str, Any], dict[str, str]]],
|
||
) -> str:
|
||
"""Файл `explain-<размер>.txt`: планы приложения и варианты psql."""
|
||
tiers = ", ".join(
|
||
f"{count} × {human(chat_size)}"
|
||
for count, chat_size in profile.background
|
||
)
|
||
lines = [
|
||
f"# План запроса истории: размер {human(size)}",
|
||
"",
|
||
f"Commit {git['commit']}"
|
||
+ (" + незакоммиченные изменения" if git["dirty"] else ""),
|
||
f"Читаемый чат — {size} сообщений; фоновые чаты: {tiers or 'нет'}.",
|
||
"Страница — `limit` профиля; «после» — сколько сообщений чата до",
|
||
"страницы.",
|
||
"",
|
||
"## Планы приложения",
|
||
"",
|
||
"auto_explain (ANALYZE, BUFFERS, TIMING): запрос, параметры и план,",
|
||
"с которыми его выполнило приложение (prepared statement asyncpg).",
|
||
"",
|
||
]
|
||
for page, entry in zip(pages, entries, strict=True):
|
||
lines += [
|
||
f"### {page['chat']}: {page['page']}, после {page['after']}",
|
||
"",
|
||
entry,
|
||
"",
|
||
]
|
||
variants = ", ".join(
|
||
f"`{name}` {columns}" for name, columns in profile.index_variants
|
||
)
|
||
lines += [
|
||
"## Страницы сравнения в psql: варианты",
|
||
"",
|
||
"Тот же текст запроса и те же параметры, что у плана приложения этой",
|
||
"страницы, в одной транзакции, которая откатывается: индексы базы",
|
||
"после неё те же. Каждый вариант выполнен дважды, здесь — второе",
|
||
"выполнение. `psql` — индекс приложения, custom plan; `generic` — он",
|
||
"же с force_generic_plan; `or-cursor` — условие курсора через OR;",
|
||
"`no-index` — без индекса приложения; варианты индекса вместо него:",
|
||
f"{variants}.",
|
||
"",
|
||
]
|
||
for page, plans in compared:
|
||
lines += [
|
||
f"### {page['chat']}: {page['page']}, после {page['after']}",
|
||
"",
|
||
]
|
||
if not plans:
|
||
lines += [
|
||
"План приложения не использовал индекс: вариантов нет.",
|
||
"",
|
||
]
|
||
for name, plan in plans.items():
|
||
lines += [f"#### {name}", "", plan, ""]
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ------------------------------------------------------------ результаты
|
||
|
||
|
||
class Results:
|
||
"""Файлы метки: runs.csv, prefill.csv, meta.json, progress.json; у
|
||
профиля отказа — ещё phases.csv и timeline.csv."""
|
||
|
||
def __init__(self, label_dir: Path, meta: dict[str, Any]) -> None:
|
||
self.dir = label_dir
|
||
self.meta = meta
|
||
self.runs: list[dict[str, Any]] = []
|
||
self.prefill: list[dict[str, Any]] = []
|
||
self.sizes: list[dict[str, Any]] = []
|
||
self.phases: list[dict[str, Any]] = []
|
||
self.timeline: list[dict[str, Any]] = []
|
||
self.rollout: dict[str, Any] | None = None
|
||
self.replaced: list[dict[str, Any]] = []
|
||
# профили уровня «4»: строки resources.csv для сводки (design
|
||
# change profile-resources, решение 7)
|
||
self.resources: list[dict[str, Any]] = []
|
||
# продолжение серии дописывает в существующий каталог метки
|
||
(label_dir / "raw").mkdir(parents=True, exist_ok=True)
|
||
self.write_meta()
|
||
|
||
def add_step(self, row: dict[str, Any]) -> None:
|
||
"""Ступень серии трека 1 — строка `runs.csv`."""
|
||
self.append("runs.csv", RUN_FIELDS, row)
|
||
|
||
def add_search(self, row: dict[str, Any]) -> None:
|
||
"""Ячейка серии — строка `searches.csv`: она же признак готовой
|
||
ячейки при продолжении."""
|
||
self.append("searches.csv", scaling.SEARCH_FIELDS, row)
|
||
|
||
def add_samples(self, rows: list[dict[str, Any]]) -> None:
|
||
"""Посекундные отсчёты окна замера — `samples.csv`."""
|
||
path = self.dir / "samples.csv"
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=scaling.SAMPLE_FIELDS,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
for row in rows:
|
||
writer.writerow(
|
||
{key: row.get(key) for key in scaling.SAMPLE_FIELDS},
|
||
)
|
||
|
||
def add_resources(self, result: resources.Result) -> None:
|
||
"""Ресурсы фазы или замены: `resources.csv` (по окнам) и
|
||
`samples.csv` (посекундно) профилей уровня «4»."""
|
||
self.resources += result.rows
|
||
for name, fields, rows in (
|
||
("resources.csv", resources.RESOURCE_FIELDS, result.rows),
|
||
("samples.csv", resources.SAMPLE_FIELDS, result.samples),
|
||
):
|
||
path = self.dir / name
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=fields,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
for row in rows:
|
||
writer.writerow({key: row.get(key) for key in fields})
|
||
|
||
def append(
|
||
self,
|
||
name: str,
|
||
fields: list[str],
|
||
row: dict[str, Any],
|
||
) -> None:
|
||
path = self.dir / name
|
||
new = not path.exists()
|
||
with path.open("a", newline="", encoding="utf-8") as file:
|
||
writer = csv.DictWriter(
|
||
file,
|
||
fieldnames=fields,
|
||
lineterminator="\n",
|
||
)
|
||
if new:
|
||
writer.writeheader()
|
||
writer.writerow({key: row.get(key) for key in fields})
|
||
file.flush()
|
||
|
||
def add_run(self, row: dict[str, Any]) -> None:
|
||
self.runs.append(row)
|
||
self.append("runs.csv", RUN_FIELDS, row)
|
||
|
||
def add_upload(self, row: dict[str, Any]) -> None:
|
||
"""Прогон профиля загрузок — строка `runs.csv` (столбцы
|
||
`uploads.RUN_FIELDS`)."""
|
||
self.runs.append(row)
|
||
self.append("runs.csv", uploads.RUN_FIELDS, row)
|
||
|
||
def add_prefill(self, row: dict[str, Any]) -> None:
|
||
self.prefill.append(row)
|
||
self.append("prefill.csv", PREFILL_FIELDS, row)
|
||
|
||
def add_pass(self, row: dict[str, Any]) -> None:
|
||
"""Проход профиля сверки — строка `passes.csv`."""
|
||
self.runs.append(row)
|
||
self.append("passes.csv", reconcile.PASS_FIELDS, row)
|
||
|
||
def add_cell(
|
||
self,
|
||
row: dict[str, Any],
|
||
timeline: list[dict[str, Any]],
|
||
) -> None:
|
||
"""Ячейка профиля очереди: шкала, затем строка `cells.csv` — она же
|
||
признак готовой ячейки при продолжении."""
|
||
for second in timeline:
|
||
self.append("timeline.csv", overload.TIMELINE_FIELDS, second)
|
||
self.append("cells.csv", overload.CELL_FIELDS, row)
|
||
|
||
def add_telemetry_run(
|
||
self,
|
||
row: dict[str, Any],
|
||
components: list[dict[str, Any]],
|
||
classes: list[dict[str, Any]],
|
||
) -> None:
|
||
"""Прогон профиля цены телеметрии: компоненты и классы, затем строка
|
||
`runs.csv` — она же признак готового прогона при продолжении."""
|
||
for item in components:
|
||
self.append(
|
||
"components.csv",
|
||
telemetry_cost.COMPONENT_FIELDS,
|
||
item,
|
||
)
|
||
for item in classes:
|
||
self.append("classes.csv", telemetry_cost.CLASS_FIELDS, item)
|
||
self.append("runs.csv", telemetry_cost.RUN_FIELDS, row)
|
||
|
||
def add_windows(self, rows: list[dict[str, Any]]) -> None:
|
||
"""Окна фоновой нагрузки прохода — строки `windows.csv`."""
|
||
for row in rows:
|
||
self.timeline.append(row)
|
||
self.append("windows.csv", reconcile.WINDOW_FIELDS, row)
|
||
|
||
def add_phase(
|
||
self,
|
||
row: dict[str, Any],
|
||
timeline: list[dict[str, Any]],
|
||
) -> None:
|
||
self.phases.append(row)
|
||
self.append("phases.csv", failover.PHASE_FIELDS, row)
|
||
for second in timeline:
|
||
self.timeline.append(second)
|
||
self.append("timeline.csv", failover.TIMELINE_FIELDS, second)
|
||
|
||
def add_rollout(
|
||
self,
|
||
row: dict[str, Any],
|
||
replaced: list[dict[str, Any]],
|
||
timeline: list[dict[str, Any]],
|
||
) -> None:
|
||
"""Профиль замены: строка прогона, шаги по репликам, шкала."""
|
||
self.rollout = row
|
||
self.replaced = replaced
|
||
self.timeline = timeline
|
||
self.append("rollout.csv", rollout.ROLLOUT_FIELDS, row)
|
||
for step in replaced:
|
||
self.append("replicas.csv", rollout.REPLICA_FIELDS, step)
|
||
for second in timeline:
|
||
self.append("timeline.csv", rollout.TIMELINE_FIELDS, second)
|
||
|
||
def write_meta(self) -> None:
|
||
self.meta["sizes"] = self.sizes
|
||
text = json.dumps(self.meta, ensure_ascii=False, indent=2)
|
||
(self.dir / "meta.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_progress(self, progress: dict[str, Any]) -> None:
|
||
progress["updated_at"] = datetime.now(UTC).isoformat(
|
||
timespec="seconds",
|
||
)
|
||
text = json.dumps(progress, ensure_ascii=False, indent=2)
|
||
(self.dir / "progress.json").write_text(text + "\n", encoding="utf-8")
|
||
|
||
def write_summary(self) -> None:
|
||
if self.meta["profile"].get("telemetry_variants"):
|
||
# цена телеметрии: сводка и графики из таблиц метки, в них и
|
||
# строки прежних запусков при продолжении (решение 12 design
|
||
# change track1-telemetry-cost)
|
||
rows = telemetry_cost.read_csv(self.dir / "runs.csv")
|
||
classes = telemetry_cost.read_csv(self.dir / "classes.csv")
|
||
comps = telemetry_cost.read_csv(self.dir / "components.csv")
|
||
text = telemetry_cost.render_summary(
|
||
self.meta,
|
||
rows,
|
||
classes,
|
||
comps,
|
||
)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
for name, svg in telemetry_cost.render_plots(
|
||
rows,
|
||
classes,
|
||
).items():
|
||
(self.dir / name).write_text(svg, encoding="utf-8")
|
||
return
|
||
if self.meta["profile"].get("series"):
|
||
# серия: сводка и графики из таблиц метки — в них и строки
|
||
# прежних запусков при продолжении (решения 8, 13)
|
||
text = scaling.render_summary(
|
||
self.meta,
|
||
scaling.read_csv(self.dir / "runs.csv"),
|
||
scaling.read_csv(self.dir / "searches.csv"),
|
||
)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
plots.render_series(self.dir)
|
||
return
|
||
if self.meta["profile"].get("queue_configs"):
|
||
# профиль очереди: сводка из таблиц метки, в них и строки прежних
|
||
# запусков при продолжении (решение 14 design change
|
||
# track1-queue-overload)
|
||
rows = overload.read_csv(self.dir / "cells.csv")
|
||
text = overload.render_summary(self.meta, rows)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
for name, svg in overload.render_plots(
|
||
rows,
|
||
overload.read_csv(self.dir / "timeline.csv"),
|
||
).items():
|
||
(self.dir / name).write_text(svg, encoding="utf-8")
|
||
return
|
||
if self.meta["profile"].get("reconcile_stages"):
|
||
# профиль сверки: сводка и графики (решение 13 design change
|
||
# track2-reconcile-cost)
|
||
text = reconcile.render_summary(
|
||
self.meta,
|
||
self.runs,
|
||
self.timeline,
|
||
)
|
||
for name, svg in reconcile.render_plots(
|
||
self.meta,
|
||
self.runs,
|
||
self.timeline,
|
||
).items():
|
||
(self.dir / name).write_text(svg, encoding="utf-8")
|
||
elif self.meta["profile"].get("uploads_variants"):
|
||
text = uploads.render_summary(self.meta, self.runs)
|
||
elif self.meta["profile"].get("rollout_from"):
|
||
text = rollout.render_summary(
|
||
self.meta,
|
||
self.rollout,
|
||
self.replaced,
|
||
self.timeline,
|
||
self.resources or None,
|
||
)
|
||
elif self.meta["profile"].get("failover_modes"):
|
||
text = failover.render_summary(
|
||
self.meta,
|
||
self.phases,
|
||
self.timeline,
|
||
self.resources or None,
|
||
)
|
||
else:
|
||
text = build_summary(
|
||
self.meta,
|
||
self.runs,
|
||
self.prefill,
|
||
self.sizes,
|
||
)
|
||
(self.dir / "summary.md").write_text(text, encoding="utf-8")
|
||
|
||
|
||
def cell(values: list[float | None], digits: int = 1) -> str:
|
||
"""Медиана и min–max повторов; один повтор — просто значение."""
|
||
present = [v for v in values if v is not None]
|
||
if not present:
|
||
return "—"
|
||
median = statistics.median(present)
|
||
if len(present) == 1:
|
||
return f"{median:.{digits}f}"
|
||
return (
|
||
f"{median:.{digits}f} ({min(present):.{digits}f}–"
|
||
f"{max(present):.{digits}f})"
|
||
)
|
||
|
||
|
||
def build_summary(
|
||
meta: dict[str, Any],
|
||
runs: list[dict[str, Any]],
|
||
prefill: list[dict[str, Any]],
|
||
sizes: list[dict[str, Any]],
|
||
) -> str:
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Нагрузочный замер: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{meta['status']}**"
|
||
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
|
||
f"- Commit: `{meta['git']['commit']}`"
|
||
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
|
||
f"- Профиль `{profile['name']}`: размеры "
|
||
f"{', '.join(human(s) for s in profile['sizes'])}; клиенты "
|
||
f"{', '.join(map(str, profile['concurrency']))}; повторов "
|
||
f"{profile['repeats']}; чтение {profile['read_measure_s']} с после "
|
||
f"{profile['read_warmup_s']} с прогрева; запись "
|
||
f"{profile['write_measure']} после {profile['write_warmup']}",
|
||
f"- Лимиты без swap: API {profile['cpus']} CPU / "
|
||
f"{profile['memory']} × {profile.get('replicas', 1)} реплик за "
|
||
f"HAProxy {profile.get('lb_cpus', '—')} CPU / "
|
||
f"{profile.get('lb_memory', '—')}, PostgreSQL "
|
||
f"{profile['db_cpus']} CPU / {profile['db_memory']}, Valkey "
|
||
f"{profile['cache_cpus']} CPU / {profile['cache_memory']}; CPU и "
|
||
"RSS API — сумма по репликам; значения — медиана (min–max) по "
|
||
"повторам",
|
||
"- Посторонние контейнеры в VM во время замера: "
|
||
+ (", ".join(meta.get("other_containers") or []) or "нет"),
|
||
"",
|
||
]
|
||
for scenario in ("read", "write"):
|
||
rows = [r for r in runs if r["scenario"] == scenario]
|
||
if not rows:
|
||
continue
|
||
lines += [
|
||
f"## Сценарий `{scenario}`",
|
||
"",
|
||
"| Размер | Клиенты | RPS | p50, мс | p95, мс | p99, мс | "
|
||
"max, мс | CPU API, ядер | CPU БД, ядер | RSS конец, МиБ | "
|
||
"Сообщений в начале | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
cells = sorted({(r["size"], r["concurrency"]) for r in rows})
|
||
for size, conc in cells:
|
||
group = [
|
||
r
|
||
for r in rows
|
||
if r["size"] == size and r["concurrency"] == conc
|
||
]
|
||
flags = sorted({f for r in group for f in r["flags"].split()})
|
||
lines.append(
|
||
f"| {human(size)} | {conc} | "
|
||
f"{cell([r['rps'] for r in group], 0)} | "
|
||
f"{cell([r['p50_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p95_ms'] for r in group], 2)} | "
|
||
f"{cell([r['p99_ms'] for r in group], 2)} | "
|
||
f"{cell([r['max_ms'] for r in group], 1)} | "
|
||
f"{cell([r['app_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['db_cpu_cores'] for r in group], 2)} | "
|
||
f"{cell([r['rss_end_mib'] for r in group], 1)} | "
|
||
f"{cell([r['store_messages_start'] for r in group], 0)} | "
|
||
f"{' '.join(flags) or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if sizes:
|
||
lines += [
|
||
"## Память и наполнение по размерам",
|
||
"",
|
||
"Объём `messages` — после `VACUUM (ANALYZE)`, до прогонов: в "
|
||
"таблице N сообщений читаемого чата и сообщения фоновых чатов "
|
||
"(`Сообщений`).",
|
||
"",
|
||
"| Размер | Сообщений | RSS API до, МиБ | RSS API после, МиБ | "
|
||
"Байт RSS на сообщение | Таблица, МиБ | Индексы, МиБ | "
|
||
"Байт в БД на сообщение | Наполнение, сообщ/с | "
|
||
"Курсоров в пуле | Пик RSS за размер, МиБ |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
lines.append(
|
||
f"| {human(info['size'])} | "
|
||
f"{info.get('messages_total', info['size'])} | "
|
||
f"{info['rss_before_mib']} | "
|
||
f"{info['rss_after_mib']} | {info['bytes_per_message']} | "
|
||
f"{mib_bytes(info.get('messages_table_bytes'))} | "
|
||
f"{mib_bytes(info.get('messages_indexes_bytes'))} | "
|
||
f"{info.get('db_bytes_per_message', '—')} | "
|
||
f"{info['prefill_rate']} | {info['cursors']} | "
|
||
f"{info.get('rss_peak_mib', '—')} |",
|
||
)
|
||
lines.append("")
|
||
lines += [
|
||
"## Планы запроса истории",
|
||
"",
|
||
"Страница (`limit` профиля) читаемого чата и первого чата "
|
||
"каждого фонового уровня. `app` — план запроса приложения "
|
||
"(auto_explain). Остальное — тот же запрос с теми же "
|
||
"параметрами в psql, в откатываемой транзакции: `psql` — "
|
||
"индекс приложения, `generic` — он же с generic plan, "
|
||
"`or-cursor` — условие курсора через OR, `no-index` — без "
|
||
"индекса, дальше — варианты индекса профиля. Буферы — верхнего "
|
||
"узла плана, файлы — `explain-<размер>.txt`.",
|
||
"",
|
||
"| Размер | Чат | Страница | После сообщений | Вариант | Узел | "
|
||
"shared hit | read | Строк | Время, мс | План |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info in sizes:
|
||
for plan in info.get("plans", []):
|
||
lines.append(
|
||
f"| {human(info['size'])} | {plan.get('chat', 'read')} | "
|
||
f"{plan['page']} | {plan['after']} | {plan['source']} | "
|
||
f"{plan['node'] or '—'} | {plan['shared_hit']} | "
|
||
f"{plan['shared_read']} | {fmt(plan['rows'])} | "
|
||
f"{fmt(plan['time_ms'], 3)} | {plan['kind']} |",
|
||
)
|
||
lines.append("")
|
||
costs = [
|
||
(info, row) for info in sizes for row in info.get("write_cost", [])
|
||
]
|
||
if costs:
|
||
lines += [
|
||
"## Стоимость записи по вариантам индекса",
|
||
"",
|
||
"Вставка порции сообщений во все чаты в пропорции наполнения "
|
||
"одной командой, в транзакции, которая откатывается; индекс "
|
||
"истории построен заново, перед вставкой `CHECKPOINT`. "
|
||
"Значения — медиана (min–max) повторов. «4000 символов» — "
|
||
"вставка сообщения максимальной длины из несжимаемого текста.",
|
||
"",
|
||
"| Размер | Вариант | Строк вставки | Индекс, МиБ | "
|
||
"Байт индекса на сообщение | мкс на сообщение | "
|
||
"WAL байт на сообщение | FPI на 1000 | 4000 символов |",
|
||
"|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for info, row in costs:
|
||
lines.append(
|
||
f"| {human(info['size'])} | {row['variant']} | "
|
||
f"{row['rows']} | {mib_bytes(row['index_bytes'])} | "
|
||
f"{row['index_bytes_per_message']} | "
|
||
f"{cell(row['insert_us_per_message'], 2)} | "
|
||
f"{cell(row['wal_bytes_per_message'], 0)} | "
|
||
f"{cell(row['fpi_per_1000'], 1)} | {row['long_text']} |",
|
||
)
|
||
lines.append("")
|
||
restores = [info for info in sizes if info.get("restore")]
|
||
if restores:
|
||
lines += [
|
||
"## Backup и восстановление",
|
||
"",
|
||
"`pg_dump -Fc` исходной базы во время записи через HTTP, "
|
||
"затем активность после backup (пользователь, сессия, чат, "
|
||
"сообщение), `pg_restore -j` в новый контейнер PostgreSQL "
|
||
"на новом volume, `ANALYZE`, сверка обеих баз (число строк, "
|
||
"md5 каждого сообщения, индексы, ревизия схемы), "
|
||
"переключение API и проверки через HTTP до и после "
|
||
"`FLUSHALL`. RTO — от запуска нового хранилища до "
|
||
"готовности API, без сверки. Сверка прошла на каждом "
|
||
"размере: иначе команда завершилась бы ошибкой.",
|
||
"",
|
||
"| Размер | До backup | Исходная в конце | Восстановлено | "
|
||
"Потеряно | Записей во время backup (в backup) | Backup, МиБ | "
|
||
"Байт backup на сообщение | dump, с | restore, с | "
|
||
"ANALYZE, с | API, с | RTO, с | Сверка, с | Чат после "
|
||
"backup: GET / POST до FLUSHALL, GET после | Сессия после "
|
||
"backup |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"
|
||
"---|---|",
|
||
]
|
||
for info in restores:
|
||
r = info["restore"]
|
||
lines.append(
|
||
f"| {human(info['size'])} | {r['messages_before_backup']} "
|
||
f"| {r['messages_source_end']} | {r['messages_restored']} "
|
||
f"| {r['messages_lost']} | {r['load_written']} "
|
||
f"({r['load_in_backup']}) | "
|
||
f"{mib_bytes(r['backup_bytes'])} | "
|
||
f"{r['backup_bytes_per_message']} | {r['dump_seconds']} | "
|
||
f"{r['restore_seconds']} | {r['analyze_seconds']} | "
|
||
f"{r['api_ready_seconds']} | {r['rto_seconds']} | "
|
||
f"{r['verify_seconds']} | {r['stale_chat_get']} / "
|
||
f"{r['stale_chat_post']}, {r['flushed_chat_get']} | "
|
||
f"{r['late_session']} |",
|
||
)
|
||
lines.append("")
|
||
if prefill:
|
||
largest = max(r["size"] for r in prefill)
|
||
curve = [r for r in prefill if r["size"] == largest]
|
||
lines += [
|
||
f"## Кривая наполнения {human(largest)}: запись при росте "
|
||
f"хранилища, {profile['prefill_vus']} клиента",
|
||
"",
|
||
"| Порция | Сообщений после | RPS | p50, мс | p95, мс | "
|
||
"p99, мс | max, мс | CPU API, ядер | CPU БД, ядер | RSS, МиБ | "
|
||
"Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in curve:
|
||
lines.append(
|
||
f"| {r['chunk']} | {r['messages_after']} | {fmt(r['rps'])} | "
|
||
f"{r['p50_ms']} | {r['p95_ms']} | {r['p99_ms']} | "
|
||
f"{r['max_ms']} | {r['app_cpu_cores']} | "
|
||
f"{r['db_cpu_cores']} | {r['rss_end_mib']} "
|
||
f"| {r['flags'] or '—'} |",
|
||
)
|
||
lines.append("")
|
||
if meta.get("host_warnings"):
|
||
lines += ["## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ---------------------------------------------------------------- прогон
|
||
|
||
|
||
def reconcile_plan(profile: Profile) -> list[dict[str, Any]]:
|
||
"""Проходы профиля сверки: у этапа — ячейки без нагрузки, затем блок
|
||
под нагрузкой; отказ по пределу — шагами числа кандидатов `S4-<D>`.
|
||
Повторы — кругами внутри группы (решение 1)."""
|
||
groups: list[tuple[str, tuple[str, ...]]] = []
|
||
for stage, _fresh, _n, _size, cells in profile.reconcile_stages:
|
||
quiet = tuple(c for c in cells if not reconcile.CELLS[c].get("load"))
|
||
loaded = tuple(c for c in cells if reconcile.CELLS[c].get("load"))
|
||
groups += [(stage, group) for group in (quiet, loaded) if group]
|
||
for count in profile.reconcile_s4_candidates:
|
||
groups.append((f"S4-{count}", ("before", "after")))
|
||
plan = reconcile.plan_passes(groups, profile.repeats, profile.order_seed)
|
||
for entry in plan:
|
||
entry["load"] = bool(reconcile.CELLS[entry["cell"]].get("load"))
|
||
return plan
|
||
|
||
|
||
class Bench:
|
||
def __init__(
|
||
self,
|
||
profile: Profile,
|
||
label: str,
|
||
results: Results,
|
||
) -> None:
|
||
self.profile = profile
|
||
self.label = label
|
||
self.results = results
|
||
self.stand = Stand(profile, results.dir)
|
||
self.started = time.monotonic()
|
||
self.runs_total = (
|
||
len(profile.sizes) * len(profile.concurrency) * 2 * profile.repeats
|
||
)
|
||
if profile.is_failover:
|
||
# прогон профиля отказа — фаза: режим × реплика
|
||
self.runs_total = len(profile.failover_modes) * profile.replicas
|
||
if profile.is_rollout:
|
||
# профиль замены — один прогон: замена всех реплик под нагрузкой
|
||
self.runs_total = 1
|
||
if profile.is_series:
|
||
# серия трека 1 — ячейки: конфигурация × повтор
|
||
self.runs_total = len(profile.configs) * profile.repeats
|
||
if profile.is_uploads:
|
||
# профиль загрузок — вариант × тело × клиенты × повтор
|
||
self.runs_total = (
|
||
len(profile.uploads_variants)
|
||
* len(profile.uploads_sizes)
|
||
* len(profile.uploads_clients)
|
||
* profile.repeats
|
||
)
|
||
if profile.is_reconcile:
|
||
# профиль сверки — проходы: этапы и S4, по повторам
|
||
self.runs_total = len(reconcile_plan(profile))
|
||
if profile.is_queue:
|
||
# профиль очереди — ячейки: конфигурация × повтор
|
||
self.runs_total = len(profile.queue_configs) * profile.repeats
|
||
if profile.is_telemetry_cost:
|
||
# цена телеметрии — прогоны: вариант × повтор
|
||
self.runs_total = len(profile.telemetry_variants) * profile.repeats
|
||
# профиль сверки: длительности проходов для оценки остатка и
|
||
# фикстура стенда (читаемый чат и его владелец — чат набора)
|
||
self.pass_seconds: list[float] = []
|
||
self.fixture: dict[str, Any] = {}
|
||
self.runs_done = 0
|
||
# серии: ячейка и попытка текущей ступени, длительности ячеек,
|
||
# номер ступени в метке (столбец run)
|
||
self.cell: dict[str, Any] | None = None
|
||
self.cell_index = 0
|
||
self.attempt = 0
|
||
self.cell_seconds: list[float] = []
|
||
# профиль очереди: поток завершений контейнеров текущей ячейки
|
||
self.queue_events: subprocess.Popen[bytes] | None = None
|
||
self.cell_started = time.monotonic()
|
||
self.step_no = 0
|
||
self.resume = False
|
||
self.stage = "start"
|
||
self.size: int | None = None
|
||
# скорости стенда для оценки оставшегося времени
|
||
self.prefill_rate = 1500.0
|
||
self.write_rps: dict[int, float] = {}
|
||
self.run_overhead = 3.0
|
||
self.done_sizes = 0
|
||
self.prefilled = 0
|
||
# текущий размер наполнен и курсоры собраны
|
||
self.prepared = False
|
||
# индекс плана приложения: от него варианты и пробы записи
|
||
self.app_index: str | None = None
|
||
# ресурсы профилей уровня «4» (design change profile-resources):
|
||
# контейнеры (id12 → имя и роль сервиса), моменты запуска (id12 →
|
||
# секунды часов ВМ) и ID вне стенда за весь прогон
|
||
self.containers: dict[str, dict[str, str]] = {}
|
||
self.starts: dict[str, list[float]] = {}
|
||
self.frozen: dict[str, list[tuple[float, float]]] = {}
|
||
self.unmapped: set[str] = set()
|
||
|
||
# ------------------------------------------------------------ вывод
|
||
|
||
def say(self, text: str) -> None:
|
||
elapsed = int(time.monotonic() - self.started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
def eta(self) -> float:
|
||
p = self.profile
|
||
if p.is_telemetry_cost:
|
||
# прогон: стенд ~60 с, окна, ожидание и выборка ~2 мин
|
||
# (решение 14), затем среднее снятых
|
||
per_run = (
|
||
statistics.mean(self.cell_seconds)
|
||
if self.cell_seconds
|
||
else sum(p.telemetry_windows) + 150
|
||
)
|
||
return (self.runs_total - self.runs_done) * per_run
|
||
if p.is_queue:
|
||
# ячейка ≈ 2 мин (решение 14), затем среднее снятых
|
||
per_cell = (
|
||
statistics.mean(self.cell_seconds)
|
||
if self.cell_seconds
|
||
else 120
|
||
)
|
||
return (self.runs_total - self.runs_done) * per_cell
|
||
if p.is_reconcile:
|
||
# проход: старт контейнера и сам проход (среднее снятых), тихие
|
||
# окна проходов под нагрузкой не считаются
|
||
per_pass = (
|
||
statistics.mean(self.pass_seconds) if self.pass_seconds else 20
|
||
)
|
||
return (self.runs_total - self.runs_done) * per_pass
|
||
if p.is_uploads:
|
||
# прогон: смена S3 и реплики (~15 с), разгон, окно, остаток
|
||
per_run = 20 + p.uploads_ramp_s + p.uploads_window_s
|
||
left = (self.runs_total - self.runs_done) * per_run
|
||
return left + (0 if self.prepared else 90)
|
||
if p.is_series:
|
||
# ячейка ≈ 5–6 мин (решение 5), затем среднее снятых
|
||
per_cell = (
|
||
statistics.mean(self.cell_seconds)
|
||
if self.cell_seconds
|
||
else 330
|
||
)
|
||
current = 0.0
|
||
if self.cell is not None:
|
||
current = min(
|
||
per_cell,
|
||
time.monotonic() - self.cell_started,
|
||
)
|
||
left = (self.runs_total - self.runs_done) * per_cell
|
||
return max(0.0, left - current)
|
||
if p.is_rollout:
|
||
# нагрузка, сбор журналов, повтор и сверка
|
||
left = p.rollout_before_s + p.rollout_budget_s + p.rollout_after_s
|
||
left = 0 if self.runs_done else left + 15
|
||
return left + (0 if self.prepared else 60)
|
||
if p.is_failover:
|
||
# фаза: окна, сбор журналов, ожидание реплик и сверка
|
||
phase = p.failover_before_s + p.failover_down_s
|
||
phase += p.failover_after_s + 15
|
||
left = (self.runs_total - self.runs_done) * phase
|
||
return left + (0 if self.prepared else 60)
|
||
left = 0.0
|
||
for index, size in enumerate(p.sizes):
|
||
current = index == self.done_sizes
|
||
if index < self.done_sizes:
|
||
continue
|
||
# пробы записи в конце размера: сборка индексов и вставки;
|
||
# backup, восстановление и сверка
|
||
left += 60 + (30 if p.restore else 0)
|
||
if current and self.prepared:
|
||
continue
|
||
filled = self.prefilled if current else 0
|
||
total = size + p.background_total
|
||
left += (total - filled) / self.prefill_rate + size / 100 / 500
|
||
# стенд, миграции, обслуживание БД, план и его варианты
|
||
left += 60
|
||
per_size = len(p.concurrency) * p.repeats
|
||
reads_left = writes_left = per_size * (len(p.sizes) - self.done_sizes)
|
||
done_in_size = self.runs_done - self.done_sizes * per_size * 2
|
||
reads_left -= min(done_in_size, per_size)
|
||
writes_left -= max(0, done_in_size - per_size)
|
||
read_time = p.read_warmup_s + p.read_measure_s + self.run_overhead
|
||
left += reads_left * read_time
|
||
rps = (
|
||
statistics.mean(self.write_rps.values())
|
||
if self.write_rps
|
||
else 1000
|
||
)
|
||
write_time = (p.write_warmup + p.write_measure) / rps
|
||
left += writes_left * (write_time + self.run_overhead)
|
||
return left
|
||
|
||
def progress(self, last: dict[str, Any] | None = None) -> None:
|
||
state = {
|
||
"label": self.label,
|
||
"profile": self.profile.name,
|
||
"stage": self.stage,
|
||
"size": self.size,
|
||
"runs_done": self.runs_done,
|
||
"runs_total": self.runs_total,
|
||
"elapsed_s": round(time.monotonic() - self.started),
|
||
"eta_s": round(self.eta()),
|
||
"warnings": self.results.meta.get("host_warnings", []),
|
||
}
|
||
if self.profile.is_series:
|
||
state["cell"] = None if self.cell is None else self.cell["cell"]
|
||
state["attempt"] = self.attempt
|
||
state["cells_done"] = self.runs_done
|
||
state["cells_total"] = self.runs_total
|
||
if last is not None:
|
||
state["last_run"] = last
|
||
self.results.write_progress(state)
|
||
|
||
# --------------------------------------------------------- этапы
|
||
|
||
def run(self) -> None:
|
||
if self.profile.is_telemetry_cost:
|
||
self.run_telemetry()
|
||
return
|
||
if self.profile.is_queue:
|
||
self.run_queue()
|
||
return
|
||
if self.profile.is_reconcile:
|
||
self.run_reconcile()
|
||
return
|
||
if self.profile.is_uploads:
|
||
self.run_uploads()
|
||
return
|
||
if self.profile.is_series:
|
||
self.run_series()
|
||
return
|
||
if self.profile.is_rollout:
|
||
self.run_rollout()
|
||
return
|
||
if self.profile.is_failover:
|
||
self.run_failover()
|
||
return
|
||
for index, size in enumerate(self.profile.sizes, start=1):
|
||
self.size = size
|
||
self.prefilled = 0
|
||
self.prepared = False
|
||
# индекс плана приложения ищется заново на каждом размере
|
||
self.app_index = None
|
||
self.run_size(index, size)
|
||
self.done_sizes += 1
|
||
|
||
def run_size(self, index: int, size: int) -> None:
|
||
stand = self.stand
|
||
name = human(size)
|
||
self.stage = f"size {name}: start"
|
||
self.say(f"size {name} ({index}/{len(self.profile.sizes)}): start")
|
||
stand.start(lambda text: self.say(f"size {name}: {text}"))
|
||
limits = stand.limits()
|
||
self.results.meta.setdefault("limits", limits)
|
||
if limits != self.results.meta["limits"]:
|
||
raise BenchError(f"limits changed between sizes: {limits}")
|
||
if "stand" not in self.results.meta:
|
||
self.results.meta["stand"] = stand.passport()
|
||
self.results.write_meta()
|
||
rss_start = stand.snapshot()["rss_kib"]
|
||
self.say(f"size {name}: app started, RSS {mib(rss_start)} MiB")
|
||
stand.k6(
|
||
"prepare.js",
|
||
{
|
||
"MEMBERS": self.profile.members,
|
||
"BACKGROUND": ",".join(
|
||
f"{count}x{chat_size}"
|
||
for count, chat_size in self.profile.background
|
||
),
|
||
},
|
||
)
|
||
fixture = stand.work_json("fixture.json")
|
||
sizes = chat_sizes(self.profile, size)
|
||
if [size] + [c["size"] for c in fixture["bgChats"]] != sizes:
|
||
raise BenchError(
|
||
f"size {name}: background chats differ from profile",
|
||
)
|
||
total = sum(sizes)
|
||
(stand.work_dir / "schedule.json").write_text(
|
||
json.dumps(fill_schedule(sizes)),
|
||
encoding="utf-8",
|
||
)
|
||
self.say(
|
||
f"size {name}: {len(fixture['bgChats'])} background chats, "
|
||
f"{total} messages to prefill",
|
||
)
|
||
self.progress()
|
||
rss_before = stand.snapshot()["rss_kib"]
|
||
prefill_seconds = self.run_prefill(size, total)
|
||
after = stand.snapshot()
|
||
rss_after = after["rss_kib"]
|
||
self.stage = f"size {name}: maintenance"
|
||
self.progress()
|
||
seconds = stand.maintain()
|
||
storage = stand.storage()
|
||
if storage["messages_rows"] != total:
|
||
raise BenchError(
|
||
f"messages table has {storage['messages_rows']} rows after "
|
||
f"prefill, expected {total}",
|
||
)
|
||
self.check_distribution(size, fixture)
|
||
self.say(
|
||
f"size {name}: VACUUM (ANALYZE) and CHECKPOINT in {seconds:.1f}s; "
|
||
f"messages {mib_bytes(storage['messages_table_bytes'])} MiB + "
|
||
f"indexes {mib_bytes(storage['messages_indexes_bytes'])} MiB, "
|
||
f"{storage['db_bytes_per_message']} B/msg",
|
||
)
|
||
self.stage = f"size {name}: cursors"
|
||
self.progress()
|
||
stand.k6(
|
||
"cursors.js",
|
||
{
|
||
"WALK_LIMIT": self.profile.walk_limit,
|
||
"POOL_SIZE": self.profile.cursor_pool,
|
||
},
|
||
)
|
||
pool = stand.work_json("cursors.json")
|
||
if pool["walked"] != size:
|
||
raise BenchError(
|
||
f"read chat has {pool['walked']} messages, expected {size}",
|
||
)
|
||
self.say(f"size {name}: cursors {len(pool['cursors'])}")
|
||
self.stage = f"size {name}: query plan"
|
||
self.progress()
|
||
plans = self.explain(size, pool, fixture)
|
||
info = {
|
||
"size": size,
|
||
"messages_total": total,
|
||
"rss_before_mib": mib(rss_before),
|
||
"rss_after_mib": mib(rss_after),
|
||
"bytes_per_message": round(
|
||
(rss_after - rss_before) * 1024 / total,
|
||
),
|
||
"prefill_seconds": round(prefill_seconds, 1),
|
||
"prefill_rate": round(total / prefill_seconds),
|
||
"cursors": len(pool["cursors"]),
|
||
"maintenance_seconds": round(seconds, 1),
|
||
**storage,
|
||
"plans": plans,
|
||
}
|
||
self.results.sizes.append(info)
|
||
self.results.write_meta()
|
||
self.prepared = True
|
||
self.say(
|
||
f"size {name}: prefill done, app RSS {info['rss_after_mib']} "
|
||
f"MiB (+{info['bytes_per_message']} B/msg)",
|
||
)
|
||
store = total
|
||
for scenario in ("read", "write"):
|
||
for conc in self.profile.concurrency:
|
||
for repeat in range(1, self.profile.repeats + 1):
|
||
store = self.run_one(scenario, size, conc, repeat, store)
|
||
if scenario == "read":
|
||
self.check_read_chat(size)
|
||
info["rss_peak_mib"] = mib(stand.snapshot()["hwm_kib"])
|
||
self.stage = f"size {name}: write cost"
|
||
self.progress()
|
||
info["write_cost"] = self.write_cost(size)
|
||
self.results.write_meta()
|
||
if self.profile.restore:
|
||
self.stage = f"size {name}: backup and restore"
|
||
self.progress()
|
||
info["restore"] = self.backup_restore(size, fixture, store)
|
||
self.results.write_meta()
|
||
if not stand.stop():
|
||
raise BenchError(f"stand of size {name} was not removed")
|
||
|
||
def check_read_chat(self, size: int) -> None:
|
||
"""После блока чтения в читаемом чате по-прежнему ровно N.
|
||
|
||
Новое сообщение встало бы в конец истории, поэтому достаточно
|
||
дочитать хвост от последнего курсора пула.
|
||
"""
|
||
self.stand.k6("tail.js", {"WALK_LIMIT": self.profile.walk_limit})
|
||
tail = self.stand.work_json("tail.json")
|
||
total = tail["from_after"] + tail["tail"]
|
||
if total != size:
|
||
raise BenchError(
|
||
f"read chat has {total} messages after the read block, "
|
||
f"expected {size}",
|
||
)
|
||
self.say(f"size {human(size)}: read chat unchanged, {total} messages")
|
||
|
||
def check_distribution(self, size: int, fixture: dict[str, Any]) -> None:
|
||
"""После наполнения у каждого чата ровно его размер по профилю."""
|
||
expected = {
|
||
fixture["readChatId"]: size,
|
||
fixture["writeChatId"]: 0,
|
||
**{chat["id"]: chat["size"] for chat in fixture["bgChats"]},
|
||
}
|
||
counts = self.stand.chat_counts()
|
||
wrong = [
|
||
f"{counts.get(chat, 0)} instead of {want}"
|
||
for chat, want in expected.items()
|
||
if counts.get(chat, 0) != want
|
||
]
|
||
wrong += [
|
||
f"unknown chat with {counts[c]}"
|
||
for c in counts.keys() - expected.keys()
|
||
]
|
||
if wrong:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages per chat differ from the "
|
||
f"profile in {len(wrong)} chats: {', '.join(wrong[:5])}",
|
||
)
|
||
sizes = sorted(expected.values())
|
||
self.say(
|
||
f"size {human(size)}: {len(expected) - 1} chats with "
|
||
f"{sizes[1]}–{sizes[-1]} messages, distribution matches profile",
|
||
)
|
||
|
||
def plan_pages(
|
||
self,
|
||
size: int,
|
||
pool: dict[str, Any],
|
||
fixture: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Страницы этапа плана: читаемый чат и первый чат каждого уровня.
|
||
|
||
Читаемый — из пула курсоров, фоновые — проходом `plan_cursors.js`
|
||
до включения auto_explain. Первая, средняя и последняя полная
|
||
страница; у чата без полных страниц после первой — только первая.
|
||
Страница сравнения — средняя, иначе первая (решение 3 design
|
||
change `track2-query-plans`).
|
||
"""
|
||
p = self.profile
|
||
representatives = []
|
||
offset = 0
|
||
for count, _ in p.background:
|
||
representatives.append(fixture["bgChats"][offset])
|
||
offset += count
|
||
walks: dict[str, Any] = {}
|
||
if representatives:
|
||
(self.stand.work_dir / "plan_chats.json").write_text(
|
||
json.dumps(representatives),
|
||
encoding="utf-8",
|
||
)
|
||
self.stand.k6("plan_cursors.js", {"WALK_LIMIT": p.walk_limit})
|
||
walks = {
|
||
walk["id"]: walk
|
||
for walk in self.stand.work_json("plan_walks.json")
|
||
}
|
||
chats = [
|
||
(
|
||
f"read {human(size)}",
|
||
fixture["readChatId"],
|
||
size,
|
||
[
|
||
{"after": after, "cursor": cursor}
|
||
for after, cursor in zip(
|
||
pool["afters"],
|
||
pool["cursors"],
|
||
strict=True,
|
||
)
|
||
],
|
||
),
|
||
]
|
||
for chat in representatives:
|
||
walk = walks[chat["id"]]
|
||
if walk["walked"] != chat["size"]:
|
||
raise BenchError(
|
||
f"size {human(size)}: background chat has "
|
||
f"{walk['walked']} messages, expected {chat['size']}",
|
||
)
|
||
chats.append(
|
||
(
|
||
f"bg {human(chat['size'])}",
|
||
chat["id"],
|
||
chat["size"],
|
||
[
|
||
position
|
||
for position in walk["positions"]
|
||
if chat["size"] - position["after"] >= p.page_limit
|
||
],
|
||
),
|
||
)
|
||
pages = []
|
||
for label, chat_id, chat_size, positions in chats:
|
||
common = {
|
||
"chat": label,
|
||
"chat_id": chat_id,
|
||
"chat_size": chat_size,
|
||
}
|
||
pages.append(
|
||
{
|
||
**common,
|
||
"page": "first",
|
||
"after": 0,
|
||
"cursor": None,
|
||
"items": min(p.page_limit, chat_size),
|
||
"compare": not positions,
|
||
},
|
||
)
|
||
if not positions:
|
||
continue
|
||
for page, position in (
|
||
("middle", positions[len(positions) // 2]),
|
||
("last", positions[-1]),
|
||
):
|
||
pages.append(
|
||
{
|
||
**common,
|
||
"page": page,
|
||
"after": position["after"],
|
||
"cursor": position["cursor"],
|
||
"items": p.page_limit,
|
||
"compare": page == "middle",
|
||
},
|
||
)
|
||
return pages
|
||
|
||
def explain(
|
||
self,
|
||
size: int,
|
||
pool: dict[str, Any],
|
||
fixture: dict[str, Any],
|
||
) -> list[dict[str, Any]]:
|
||
"""Этап плана: EXPLAIN (ANALYZE, BUFFERS) запросов истории приложения.
|
||
|
||
auto_explain включается между маркерами, `explain.js` читает
|
||
страницы из `plan_pages.json`; планы берутся из журнала Postgres.
|
||
Затем страница сравнения каждого чата в psql: варианты индекса и
|
||
формы запроса в откатываемой транзакции. Решение 5 design change
|
||
`bench-postgres`, решения 3 и 4 design change `track2-query-plans`.
|
||
"""
|
||
stand = self.stand
|
||
pages = self.plan_pages(size, pool, fixture)
|
||
(stand.work_dir / "plan_pages.json").write_text(
|
||
json.dumps(pages),
|
||
encoding="utf-8",
|
||
)
|
||
indexes = stand.indexes()
|
||
marker = f"bench-explain {uuid.uuid4().hex}"
|
||
enable = "".join(
|
||
f"ALTER SYSTEM SET {key} = '{value}';\n"
|
||
for key, value in EXPLAIN_SETTINGS.items()
|
||
)
|
||
disable = "".join(
|
||
f"ALTER SYSTEM RESET {key};\n" for key in EXPLAIN_SETTINGS
|
||
)
|
||
stand.psql(
|
||
f"DO $$ BEGIN RAISE LOG '{marker} start'; END $$;\n"
|
||
f"{enable}SELECT pg_reload_conf();\n",
|
||
)
|
||
time.sleep(RELOAD_PAUSE_S)
|
||
try:
|
||
stand.k6("explain.js", {})
|
||
finally:
|
||
stand.psql(
|
||
f"{disable}SELECT pg_reload_conf();\n"
|
||
f"DO $$ BEGIN RAISE LOG '{marker} end'; END $$;\n",
|
||
)
|
||
# соединения пула API должны получить выключение до прогонов
|
||
time.sleep(RELOAD_PAUSE_S)
|
||
entries = [
|
||
entry
|
||
for entry in plan_entries(stand.db_log(), marker)
|
||
if is_history(entry)
|
||
]
|
||
if len(entries) != len(pages):
|
||
raise BenchError(
|
||
f"size {human(size)}: expected {len(pages)} history "
|
||
f"query plans in the PostgreSQL log, found {len(entries)}",
|
||
)
|
||
plans = []
|
||
compared = []
|
||
for page, entry in zip(pages, entries, strict=True):
|
||
parts = plan_parts(entry) or {"plan": entry}
|
||
where = {
|
||
"chat": page["chat"],
|
||
"chat_size": page["chat_size"],
|
||
"page": page["page"],
|
||
"after": page["after"],
|
||
}
|
||
plans.append(
|
||
{
|
||
**where,
|
||
"source": "app",
|
||
**plan_summary(parts["plan"], entry_duration(entry)),
|
||
},
|
||
)
|
||
if self.app_index is None:
|
||
self.app_index = index_name(parts["plan"])
|
||
if not page["compare"]:
|
||
continue
|
||
variants = self.explain_variants(entry, page["cursor"] is not None)
|
||
compared.append((page, variants))
|
||
for variant, plan in variants.items():
|
||
summary = plan_summary(plan, None)
|
||
plans.append({**where, "source": variant, **summary})
|
||
self.say(
|
||
f"size {human(size)}: {page['chat']} {page['page']} "
|
||
f"{variant}: {summary['node']}, "
|
||
f"hit={summary['shared_hit']} read={summary['shared_read']}"
|
||
f", {summary['time_ms']} ms",
|
||
)
|
||
if stand.indexes() != indexes:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages indexes changed by the plan "
|
||
"stage",
|
||
)
|
||
text = render_explain(
|
||
size,
|
||
self.results.meta["git"],
|
||
self.profile,
|
||
pages,
|
||
entries,
|
||
compared,
|
||
)
|
||
path = self.results.dir / f"explain-{human(size)}.txt"
|
||
path.write_text(text, encoding="utf-8")
|
||
return plans
|
||
|
||
def explain_variants(self, entry: str, has_cursor: bool) -> dict[str, str]:
|
||
"""Тот же запрос и параметры в psql: варианты индекса и запроса.
|
||
|
||
Всё в транзакции, которая откатывается; нагрузки в этот момент нет.
|
||
Каждый вариант выполняется дважды, в результат идёт второе
|
||
выполнение: только что построенный индекс ещё не в shared_buffers.
|
||
План приложения без индекса — вариантов нет (решение 4 design
|
||
change `track2-query-plans`).
|
||
"""
|
||
parts = plan_parts(entry)
|
||
index = index_name(parts["plan"]) if parts else None
|
||
if parts is None or index is None:
|
||
return {}
|
||
args = ", ".join(
|
||
"'" + value.replace("'", "''") + "'" for value in parts["params"]
|
||
)
|
||
|
||
def twice(name: str, statement: str) -> str:
|
||
run = f"EXPLAIN (ANALYZE, BUFFERS) EXECUTE {statement}({args});\n"
|
||
return f"\\echo @@warmup\n{run}\\echo @@{name}\n{run}"
|
||
|
||
script = (
|
||
"SET lock_timeout = '5s';\n"
|
||
"SET plan_cache_mode = force_custom_plan;\n"
|
||
"BEGIN;\n"
|
||
f"PREPARE page AS {parts['text']};\n"
|
||
+ twice("psql", "page")
|
||
+ "SET LOCAL plan_cache_mode = force_generic_plan;\n"
|
||
+ twice("generic", "page")
|
||
+ "SET LOCAL plan_cache_mode = force_custom_plan;\n"
|
||
)
|
||
if has_cursor:
|
||
script += f"PREPARE page_or AS {or_cursor(parts['text'])};\n"
|
||
script += twice("or-cursor", "page_or")
|
||
script += f'DROP INDEX "{index}";\n' + twice("no-index", "page")
|
||
for name, columns in self.profile.index_variants:
|
||
script += (
|
||
f"CREATE INDEX bench_variant ON messages {columns};\n"
|
||
+ twice(name, "page")
|
||
+ "DROP INDEX bench_variant;\n"
|
||
)
|
||
script += "ROLLBACK;\n"
|
||
sections: dict[str, list[str]] = {}
|
||
current: list[str] = []
|
||
for line in self.stand.psql(script).splitlines():
|
||
if line.startswith("@@"):
|
||
current = sections.setdefault(line[2:], [])
|
||
current.clear()
|
||
continue
|
||
current.append(line)
|
||
sections.pop("warmup", None)
|
||
return {
|
||
name: "\n".join(lines).strip() for name, lines in sections.items()
|
||
}
|
||
|
||
def write_cost(self, size: int) -> list[dict[str, Any]]:
|
||
"""Стоимость записи по вариантам индекса истории.
|
||
|
||
Конфигурации `app` (индекс приложения, построенный заново), варианты
|
||
профиля и `none`; повторы со сдвигом порядка, `VACUUM` после каждой
|
||
пробы. Медиана и разброс — в сводке (решение 5 design change
|
||
`track2-query-plans`).
|
||
"""
|
||
stand = self.stand
|
||
p = self.profile
|
||
if self.app_index is None:
|
||
raise BenchError(
|
||
f"size {human(size)}: the app history plan used no index",
|
||
)
|
||
definition = stand.psql(
|
||
f"SELECT pg_get_indexdef('{self.app_index}'::regclass);\n",
|
||
)
|
||
columns = definition.partition(" USING btree ")[2]
|
||
if not columns:
|
||
raise BenchError(f"unexpected index definition: {definition}")
|
||
configs: list[tuple[str, str | None]] = [
|
||
("app", columns),
|
||
*p.index_variants,
|
||
("none", None),
|
||
]
|
||
before = (stand.indexes(), stand.message_count())
|
||
samples: dict[str, list[dict[str, Any]]] = {n: [] for n, _ in configs}
|
||
for repeat in range(p.write_probe_repeats):
|
||
shift = repeat % len(configs)
|
||
for name, config in configs[shift:] + configs[:shift]:
|
||
samples[name].append(stand.write_probe(self.app_index, config))
|
||
stand.psql("VACUUM messages;\n")
|
||
if (stand.indexes(), stand.message_count()) != before:
|
||
raise BenchError(
|
||
f"size {human(size)}: messages indexes or rows changed by the "
|
||
"write probes",
|
||
)
|
||
result = []
|
||
for name, config in configs:
|
||
runs = samples[name]
|
||
index_bytes = int(
|
||
statistics.median(r["index_bytes"] for r in runs),
|
||
)
|
||
row = {
|
||
"variant": name,
|
||
"columns": config,
|
||
"rows": runs[0]["rows"],
|
||
"messages": runs[0]["rows_before"],
|
||
"index_bytes": index_bytes,
|
||
"index_bytes_per_message": round(
|
||
index_bytes / runs[0]["rows_before"],
|
||
1,
|
||
),
|
||
"insert_us_per_message": [
|
||
round(r["time_ms"] * 1000 / r["rows"], 2) for r in runs
|
||
],
|
||
"wal_bytes_per_message": [
|
||
round(r["wal_bytes"] / r["rows"], 1) for r in runs
|
||
],
|
||
"fpi_per_1000": [
|
||
round(r["wal_fpi"] * 1000 / r["rows"], 1) for r in runs
|
||
],
|
||
"long_text": runs[0]["long_text"],
|
||
}
|
||
result.append(row)
|
||
self.say(
|
||
f"size {human(size)}: write cost {name}: "
|
||
f"{cell(row['insert_us_per_message'], 2)} us/msg, "
|
||
f"{cell(row['wal_bytes_per_message'], 0)} WAL B/msg, index "
|
||
f"{row['index_bytes_per_message']} B/msg, 4000 chars: "
|
||
f"{row['long_text']}",
|
||
)
|
||
return result
|
||
|
||
def backup_restore(
|
||
self,
|
||
size: int,
|
||
fixture: dict[str, Any],
|
||
store: int,
|
||
) -> dict[str, Any]:
|
||
"""Backup под записью, восстановление в новое хранилище, сверка.
|
||
|
||
Порядок и проверки — решения 2–5 design change
|
||
`track2-backup-restore`. Расхождение — BenchError с размером;
|
||
хранилище восстановления убирает `Stand.stop()`.
|
||
"""
|
||
stand = self.stand
|
||
p = self.profile
|
||
name = human(size)
|
||
rows_before = stand.message_rows()
|
||
counts_before = stand.table_counts()
|
||
# запись через HTTP во время backup: dump начинается, когда она идёт
|
||
load = stand.k6_start(
|
||
"write.js",
|
||
{
|
||
"VUS": p.restore_write_vus,
|
||
"COUNT": p.restore_write_load,
|
||
"OFFSET": store,
|
||
},
|
||
)
|
||
dump = stand.work_dir / "backup.dump"
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while stand.message_count() <= counts_before["messages"]:
|
||
if load.poll() is not None or time.monotonic() > deadline:
|
||
raise BenchError(
|
||
f"size {name}: write load did not start before backup",
|
||
)
|
||
time.sleep(0.05)
|
||
started = time.monotonic()
|
||
stand.pg_dump(dump)
|
||
dump_seconds = time.monotonic() - started
|
||
at_dump_end = stand.message_count()
|
||
finally:
|
||
out, err = load.communicate()
|
||
if load.returncode != 0:
|
||
raise BenchError(
|
||
k6_failure(
|
||
"write.js",
|
||
subprocess.CompletedProcess([], load.returncode, out, err),
|
||
),
|
||
)
|
||
written = parse_summary(stand.work_dir / "last.json")
|
||
# backup не должен мешать записи: k6 выходит с 0 и при ошибках
|
||
if (
|
||
written["requests_failed"]
|
||
or written["requests_ok"] != p.restore_write_load
|
||
):
|
||
raise BenchError(
|
||
f"size {name}: writes during backup: "
|
||
f"{written['requests_ok']} of {p.restore_write_load} ok, "
|
||
f"{written['requests_failed']} failed",
|
||
)
|
||
dump_bytes = dump.stat().st_size
|
||
counts_loaded = stand.table_counts()
|
||
self.say(
|
||
f"size {name}: backup {mib_bytes(dump_bytes)} MiB in "
|
||
f"{dump_seconds:.2f}s during {written['requests_ok']} writes "
|
||
f"({written['requests_failed']} failed)",
|
||
)
|
||
stand.k6("restore_check.js", {"PHASE": "source"})
|
||
source_check = stand.work_json("check-source.json")
|
||
stand.k6("after_backup.js", {})
|
||
after = stand.work_json("after.json")
|
||
rows_source = stand.message_rows()
|
||
|
||
# восстановление: RTO — от запуска нового хранилища до готовности API
|
||
started = time.monotonic()
|
||
stand.start_restore()
|
||
storage_seconds = time.monotonic() - started
|
||
copy_seconds, restore_seconds = stand.pg_restore(dump)
|
||
mark = time.monotonic()
|
||
stand.psql("ANALYZE;\n", container=RESTORE_DB)
|
||
analyze_seconds = time.monotonic() - mark
|
||
db_seconds = time.monotonic() - started
|
||
|
||
mark = time.monotonic()
|
||
rows_restored = stand.message_rows(container=RESTORE_DB)
|
||
counts_restored = stand.table_counts(container=RESTORE_DB)
|
||
problems = []
|
||
mismatched = sum(
|
||
1
|
||
for message, row in rows_restored.items()
|
||
if rows_source.get(message) != row
|
||
)
|
||
if mismatched:
|
||
problems.append(f"{mismatched} restored messages differ")
|
||
missing_before = len(rows_before.keys() - rows_restored.keys())
|
||
if missing_before:
|
||
problems.append(
|
||
f"{missing_before} messages from before backup lost",
|
||
)
|
||
lost = rows_source.keys() - rows_restored.keys()
|
||
allowed = {fixture["writeChatId"], after["chatId"]}
|
||
foreign = sum(1 for m in lost if rows_source[m][0] not in allowed)
|
||
if foreign:
|
||
problems.append(f"{foreign} lost messages outside written chats")
|
||
for table in DATA_TABLES[:-1]:
|
||
if counts_restored[table] != counts_loaded[table]:
|
||
problems.append(
|
||
f"{table}: {counts_restored[table]} restored, "
|
||
f"{counts_loaded[table]} at backup",
|
||
)
|
||
if stand.indexes(container=RESTORE_DB) != stand.indexes():
|
||
problems.append("messages indexes differ")
|
||
version = "SELECT version_num FROM alembic_version;\n"
|
||
if stand.psql(version, container=RESTORE_DB) != stand.psql(version):
|
||
problems.append("alembic revision differs")
|
||
verify_seconds = time.monotonic() - mark
|
||
|
||
mark = time.monotonic()
|
||
stand.switch_api(RESTORE_HOST)
|
||
api_seconds = time.monotonic() - mark
|
||
stand.k6("restore_check.js", {"PHASE": "stale"})
|
||
stale = stand.work_json("check-stale.json")
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
CACHE,
|
||
"sh",
|
||
"-c",
|
||
'exec valkey-cli -p "$VALKEY_PORT" FLUSHALL',
|
||
)
|
||
stand.k6("restore_check.js", {"PHASE": "restored"})
|
||
restored = stand.work_json("check-restored.json")
|
||
for phase, check in (
|
||
("source", source_check),
|
||
("stale", stale),
|
||
("restored", restored),
|
||
):
|
||
if check["reader_me"] != 200:
|
||
problems.append(
|
||
f"{phase}: reader session {check['reader_me']}",
|
||
)
|
||
if stale["late_me"] != 401:
|
||
problems.append(f"session after backup: {stale['late_me']}")
|
||
if restored["after_chat_get"] != 404:
|
||
problems.append(
|
||
f"chat after backup after FLUSHALL: {restored['after_chat_get']}",
|
||
)
|
||
for phase, check in (("stale", stale), ("restored", restored)):
|
||
differ = [
|
||
f"{a['chat']} {a['page']}"
|
||
for a, b in zip(
|
||
check["pages"],
|
||
source_check["pages"],
|
||
strict=True,
|
||
)
|
||
if a["status"] != 200 or a["items"] != b["items"]
|
||
]
|
||
if differ:
|
||
problems.append(f"{phase}: pages differ: {', '.join(differ)}")
|
||
if problems:
|
||
raise BenchError(
|
||
f"size {name}: restore check failed: {'; '.join(problems)}",
|
||
)
|
||
result = {
|
||
"messages_before_backup": counts_before["messages"],
|
||
"messages_at_dump_end": at_dump_end,
|
||
"messages_source_end": len(rows_source),
|
||
"messages_restored": len(rows_restored),
|
||
"messages_lost": len(lost),
|
||
"lost_after_backup_chat": sum(
|
||
1 for m in lost if rows_source[m][0] == after["chatId"]
|
||
),
|
||
"load_written": written["requests_ok"],
|
||
"load_failed": written["requests_failed"],
|
||
"load_in_backup": len(rows_restored) - counts_before["messages"],
|
||
"tables_restored": counts_restored,
|
||
"backup_bytes": dump_bytes,
|
||
"backup_bytes_per_message": round(
|
||
dump_bytes / len(rows_restored),
|
||
1,
|
||
),
|
||
"dump_seconds": round(dump_seconds, 2),
|
||
"storage_seconds": round(storage_seconds, 2),
|
||
"copy_seconds": round(copy_seconds, 2),
|
||
"restore_seconds": round(restore_seconds, 2),
|
||
"analyze_seconds": round(analyze_seconds, 2),
|
||
"verify_seconds": round(verify_seconds, 2),
|
||
"api_ready_seconds": round(api_seconds, 2),
|
||
"rto_seconds": round(db_seconds + api_seconds, 2),
|
||
"pages_checked": len(source_check["pages"]),
|
||
"stale_chat_get": stale["after_chat_get"],
|
||
"stale_chat_post": stale["after_chat_post"],
|
||
"flushed_chat_get": restored["after_chat_get"],
|
||
"late_session": stale["late_me"],
|
||
}
|
||
self.say(
|
||
f"size {name}: restored {len(rows_restored)} of "
|
||
f"{len(rows_source)} messages ({len(lost)} lost after the "
|
||
f"snapshot), restore {restore_seconds:.2f}s, ANALYZE "
|
||
f"{analyze_seconds:.2f}s, API {api_seconds:.2f}s, RTO "
|
||
f"{result['rto_seconds']}s; stale chat GET "
|
||
f"{result['stale_chat_get']} POST {result['stale_chat_post']}, "
|
||
f"after FLUSHALL {result['flushed_chat_get']}",
|
||
)
|
||
return result
|
||
|
||
def run_prefill(self, size: int, total: int) -> float:
|
||
"""Наполнение всех чатов размера по расписанию, порциями."""
|
||
chunk_size = self.profile.prefill_chunk
|
||
total_seconds = 0.0
|
||
for chunk, offset in enumerate(range(0, total, chunk_size), start=1):
|
||
count = min(chunk_size, total - offset)
|
||
name = f"prefill-{human(size)}-{chunk:02d}"
|
||
if self.cell is not None:
|
||
# серии: своя порция у каждой ячейки и попытки
|
||
name = f"{self.cell['cell']}-a{self.attempt}-{name}"
|
||
self.stage = f"size {human(size)}: prefill"
|
||
measured = measure(
|
||
self.stand,
|
||
"prefill.js",
|
||
{
|
||
"COUNT": count,
|
||
"OFFSET": offset,
|
||
"VUS": self.profile.prefill_vus,
|
||
"NAME": name,
|
||
},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
down = self.stand.down_storages()
|
||
flags = run_flags(
|
||
self.profile,
|
||
measured,
|
||
self.stand.app_state(),
|
||
down,
|
||
)
|
||
self.results.add_prefill(
|
||
{
|
||
**summary,
|
||
"size": size,
|
||
"chunk": chunk,
|
||
"messages_before": offset,
|
||
"messages_after": offset + summary["requests_ok"],
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"mem_cgroup_end_mib": mib_bytes(
|
||
measured.after["mem_bytes"],
|
||
),
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
"db_cpu_cores": measured.db_cpu,
|
||
"cell": None if self.cell is None else self.cell["cell"],
|
||
"attempt": self.attempt or None,
|
||
},
|
||
)
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during prefill chunk "
|
||
f"{chunk}: matrix stopped",
|
||
)
|
||
if summary["requests_ok"] != count:
|
||
raise BenchError(
|
||
f"prefill chunk {chunk}: {summary['requests_ok']} of "
|
||
f"{count} messages confirmed ({' '.join(flags)})",
|
||
)
|
||
total_seconds += summary["duration_s"]
|
||
self.prefilled = offset + count
|
||
self.prefill_rate = self.prefilled / total_seconds
|
||
self.say(
|
||
f"size {human(size)}: prefill {self.prefilled}/{total} "
|
||
f"({self.prefilled * 100 // total}%), "
|
||
f"{self.prefill_rate:.0f} msg/s, "
|
||
f"~{fmt_left((total - self.prefilled) / self.prefill_rate)} "
|
||
"left",
|
||
)
|
||
self.progress()
|
||
return total_seconds
|
||
|
||
def run_one(
|
||
self,
|
||
scenario: str,
|
||
size: int,
|
||
conc: int,
|
||
repeat: int,
|
||
store: int,
|
||
) -> int:
|
||
"""Прогрев и измеряемый прогон; возвращает сообщений в хранилище."""
|
||
p = self.profile
|
||
number = self.runs_done + 1
|
||
name = f"{scenario}-{human(size)}-c{conc}-r{repeat}"
|
||
self.stage = f"run {number}/{self.runs_total} {name}"
|
||
self.progress()
|
||
wall = time.monotonic()
|
||
if scenario == "read":
|
||
warm = {"VUS": conc, "DURATION": f"{p.read_warmup_s}s"}
|
||
self.stand.k6(
|
||
"read.js",
|
||
{**warm, "SEED": p.seed + p.warmup_seed_shift},
|
||
)
|
||
env = {"VUS": conc, "DURATION": f"{p.read_measure_s}s"}
|
||
env["SEED"] = p.seed
|
||
else:
|
||
self.stand.k6(
|
||
"write.js",
|
||
{"VUS": conc, "COUNT": p.write_warmup, "OFFSET": store},
|
||
)
|
||
warm_ok = parse_summary(self.stand.work_dir / "last.json")
|
||
store += warm_ok["requests_ok"]
|
||
env = {"VUS": conc, "COUNT": p.write_measure, "OFFSET": store}
|
||
measured = measure(
|
||
self.stand,
|
||
f"{scenario}.js",
|
||
{**env, "NAME": name},
|
||
self.results.dir / "raw" / f"{name}.json",
|
||
)
|
||
summary = measured.summary
|
||
app = self.stand.app_state()
|
||
down = self.stand.down_storages()
|
||
flags = run_flags(p, measured, app, down)
|
||
end = store + (summary["requests_ok"] if scenario == "write" else 0)
|
||
row = {
|
||
**summary,
|
||
"run": number,
|
||
"scenario": scenario,
|
||
"size": size,
|
||
"concurrency": conc,
|
||
"repeat": repeat,
|
||
"app_cpu_cores": measured.app_cpu,
|
||
"rss_start_mib": mib(measured.before["rss_kib"]),
|
||
"rss_end_mib": mib(measured.after["rss_kib"]),
|
||
"rss_max_mib": mib(measured.rss_max_kib),
|
||
"mem_cgroup_end_mib": mib_bytes(measured.after["mem_bytes"]),
|
||
"store_messages_start": store,
|
||
"store_messages_end": end,
|
||
"generator_cpu_cores": measured.generator_cpu,
|
||
"flags": " ".join(flags),
|
||
"db_cpu_cores": measured.db_cpu,
|
||
}
|
||
self.results.add_run(row)
|
||
self.runs_done += 1
|
||
if scenario == "write" and summary["rps"]:
|
||
self.write_rps[conc] = summary["rps"]
|
||
overhead = time.monotonic() - wall - summary["duration_s"]
|
||
if scenario == "read":
|
||
overhead -= p.read_warmup_s
|
||
self.run_overhead = max(1.0, overhead)
|
||
errors = summary["requests_failed"] + summary["check_failed"]
|
||
self.say(
|
||
f"run {number}/{self.runs_total} {scenario:<5} {human(size):>4} "
|
||
f"c={conc:<2} r{repeat}: {fmt(summary['rps'])} rps "
|
||
f"p50 {summary['p50_ms']} p95 {summary['p95_ms']} "
|
||
f"p99 {summary['p99_ms']} ms cpu {measured.app_cpu} "
|
||
f"db {measured.db_cpu} rss {row['rss_end_mib']} MiB "
|
||
f"err {errors} "
|
||
f"~{fmt_left(self.eta())} left"
|
||
+ (f" [{' '.join(flags)}]" if flags else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if "oom" in flags or "app_down" in flags:
|
||
raise BenchError(
|
||
f"api replica is down after {name} ({' '.join(flags)}): "
|
||
"matrix stopped",
|
||
)
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during {name}: matrix stopped",
|
||
)
|
||
return end
|
||
|
||
# ----------------------------------------------------- профиль отказа
|
||
|
||
def prepare_load(self, tag: str) -> dict[str, Any]:
|
||
"""Стенд и данные постоянной нагрузки (профили отказа и замены):
|
||
реплики за LB, паспорт, пользователи и чаты, читаемый чат,
|
||
курсоры; возвращает фикстуру."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
size = p.sizes[0]
|
||
self.size = size
|
||
name = human(size)
|
||
self.stage = "start"
|
||
self.say(
|
||
f"{tag}: start, {stand.profile.replicas} replicas behind haproxy",
|
||
)
|
||
stand.start(lambda text: self.say(f"{tag}: {text}"))
|
||
limits = stand.limits()
|
||
if self.cell is None:
|
||
self.results.meta["limits"] = limits
|
||
self.results.meta["stand"] = stand.passport()
|
||
else:
|
||
# серия: лимиты по конфигурациям, одинаковые во всех повторах;
|
||
# max_connections — в лимитах конфигурации, не в паспорте
|
||
known = self.results.meta.setdefault("limits", {})
|
||
config = self.cell["config"]
|
||
# лимиты сэмплера добавляет run_cell после его запуска
|
||
earlier = known.get(config, {})
|
||
sampler = earlier.get("sampler")
|
||
stand_only = {k: v for k, v in earlier.items() if k != "sampler"}
|
||
if config in known and stand_only != limits:
|
||
raise BenchError(
|
||
f"{config}: limits differ from the earlier repeat: "
|
||
f"{limits}",
|
||
)
|
||
known[config] = (
|
||
limits if sampler is None else {**limits, "sampler": sampler}
|
||
)
|
||
if "stand" not in self.results.meta:
|
||
passport = stand.passport()
|
||
passport["postgres"]["settings"].pop("max_connections", None)
|
||
self.results.meta["stand"] = passport
|
||
self.results.write_meta()
|
||
stand.k6(
|
||
"prepare.js",
|
||
{
|
||
"MEMBERS": p.members,
|
||
"BACKGROUND": "",
|
||
"WRITE_CHATS": p.write_chats,
|
||
},
|
||
)
|
||
fixture = stand.work_json("fixture.json")
|
||
(stand.work_dir / "schedule.json").write_text(
|
||
json.dumps(fill_schedule([size])),
|
||
encoding="utf-8",
|
||
)
|
||
self.stage = "prefill"
|
||
self.progress()
|
||
self.run_prefill(size, size)
|
||
stand.maintain()
|
||
stand.k6(
|
||
"cursors.js",
|
||
{"WALK_LIMIT": p.walk_limit, "POOL_SIZE": p.cursor_pool},
|
||
)
|
||
pool = stand.work_json("cursors.json")
|
||
if pool["walked"] != size:
|
||
raise BenchError(
|
||
f"read chat has {pool['walked']} messages, expected {size}",
|
||
)
|
||
self.say(f"{tag}: read chat {name}, {len(pool['cursors'])} cursors")
|
||
self.prepared = True
|
||
return fixture
|
||
|
||
def k6_requests(
|
||
self,
|
||
name: str,
|
||
log_name: str,
|
||
) -> tuple[list[failover.Request], dict[str, Any]]:
|
||
"""Строки о запросах прогона k6 и его итог. Строк каждого вида
|
||
столько же, сколько итераций сценария, иначе шкала и повтор
|
||
неуспешных записей неполны — отказ."""
|
||
output = (self.stand.work_dir / log_name).read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
requests = failover.parse_requests(output)
|
||
k6_data = json.loads(
|
||
(self.results.dir / "raw" / f"{name}-k6.json").read_text(
|
||
encoding="utf-8",
|
||
),
|
||
)
|
||
for kind, scenario in (("w", "write"), ("r", "read")):
|
||
iterations = (
|
||
k6_data["metrics"]
|
||
.get(f"iterations{{scenario:{scenario}}}", {})
|
||
.get("values", {})
|
||
.get("count")
|
||
)
|
||
lines = sum(1 for r in requests if r.kind == kind)
|
||
if iterations != lines:
|
||
raise BenchError(
|
||
f"{name}: {lines} k6 lines of {scenario}, "
|
||
f"{iterations} iterations; unparsed: "
|
||
f"{failover.unparsed(output)[:3]}",
|
||
)
|
||
return requests, k6_data
|
||
|
||
# ------------------------------------- ресурсы профилей уровня «4»
|
||
|
||
def start_observer(self, tag: str) -> None:
|
||
"""Сэмплер в режиме `scan` на всё время нагрузки (решения 1, 2
|
||
design change `profile-resources`): лимиты сэмплера, прочитанные у
|
||
контейнера, и карта контейнеров — в паспорт."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
stand.start_sampler(scan=True)
|
||
meta = self.results.meta
|
||
meta["limits"]["sampler"] = stand.container_limits(SAMPLER)
|
||
self.containers.update(stand.container_map())
|
||
info = meta.setdefault("resources", {})
|
||
info.update(
|
||
{
|
||
"mode": "scan",
|
||
"interval_s": p.sample_interval_s,
|
||
"containers": self.containers,
|
||
"unmapped": 0,
|
||
},
|
||
)
|
||
self.results.write_meta()
|
||
self.say(
|
||
f"{tag}: sampler started (scan, {p.sampler_cpus} CPU / "
|
||
f"{p.sampler_memory}, no network), {len(self.containers)} "
|
||
"containers",
|
||
)
|
||
|
||
def check_observer(self, name: str, first: float, last: float) -> None:
|
||
"""Сторож (решение 2): сэмплер работает, и в нагрузке соседние
|
||
тики ВМ не дальше `GAP_LIMIT_S`. Иначе — отказ с фазой: метка без
|
||
ресурсов цели не годится."""
|
||
running = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
SAMPLER,
|
||
check=False,
|
||
)
|
||
if running != "true":
|
||
raise BenchError(
|
||
f"{name}: sampler {SAMPLER} stopped during the load: "
|
||
"resources not measured",
|
||
)
|
||
samples = scaling.parse_samples(self.stand.sampler_logs()[0])
|
||
gap = resources.max_gap(samples, first, last)
|
||
if gap > resources.GAP_LIMIT_S:
|
||
raise BenchError(
|
||
f"{name}: no sampler ticks for {gap:.1f}s during the load "
|
||
f"(limit {resources.GAP_LIMIT_S:g}s): resources not measured",
|
||
)
|
||
|
||
def observe(
|
||
self,
|
||
result: resources.Result,
|
||
name: str,
|
||
profile: str,
|
||
) -> None:
|
||
"""Строки ресурсов — в файлы метки, ID вне стенда — в паспорт,
|
||
строка ресурсов — в терминал."""
|
||
self.unmapped |= set(result.unmapped)
|
||
self.results.meta["resources"]["unmapped"] = len(self.unmapped)
|
||
self.results.add_resources(result)
|
||
self.results.write_meta()
|
||
self.say(f"{name}: {resources.terminal_line(result.rows, profile)}")
|
||
|
||
def run_failover(self) -> None:
|
||
"""Профиль отказа реплик (решение 7 design change
|
||
`graceful-failover`): стенд, читаемый чат и курсоры, затем фазы —
|
||
каждая реплика в каждом режиме. С `sample_resources` — сэмплер от
|
||
первой фазы до конца последней (design change
|
||
`profile-resources`)."""
|
||
p = self.profile
|
||
fixture = self.prepare_load("failover")
|
||
if p.sample_resources:
|
||
self.start_observer("failover")
|
||
run_id = secrets.token_hex(4)
|
||
writes = 0
|
||
number = 0
|
||
for mode in p.failover_modes:
|
||
for index in range(p.replicas):
|
||
number += 1
|
||
writes = self.run_phase(
|
||
number,
|
||
mode,
|
||
index,
|
||
run_id=run_id,
|
||
fixture=fixture,
|
||
writes=writes,
|
||
)
|
||
if p.sample_resources:
|
||
self.stand.stop_sampler()
|
||
|
||
def run_phase(
|
||
self,
|
||
number: int,
|
||
mode: str,
|
||
index: int,
|
||
*,
|
||
run_id: str,
|
||
fixture: dict[str, Any],
|
||
writes: int,
|
||
) -> int:
|
||
"""Фаза: нагрузка, действие над репликой `index`, возврат, разбор,
|
||
повтор неуспешных записей и сверка; возвращает число отправленных
|
||
за прогон записей (разных ключей)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
phase = f"{number:02d}-{mode}-r{index + 1}"
|
||
self.stage = f"phase {number}/{self.runs_total} {phase}"
|
||
self.progress()
|
||
stand.wait_ready()
|
||
target = stand.replicas()[index]
|
||
target_id = sh("docker", "inspect", "-f", "{{.Id}}", target)
|
||
instance = target_id[:12]
|
||
address = stand.address(target)
|
||
slot = next(
|
||
(
|
||
s["slot"]
|
||
for s in stand.lb_slots()
|
||
if s["addr"].rsplit(":", 1)[0] == address
|
||
),
|
||
None,
|
||
)
|
||
duration = p.failover_before_s + p.failover_down_s + p.failover_after_s
|
||
log_name = f"{phase}.log"
|
||
# события цели — потоком с начала фазы, не из буфера демона после
|
||
# неё (решение 9 design change fix-known-issues). Без --since:
|
||
# подписка CLI — доли секунды, а действие — через запуск k6 и
|
||
# failover_before_s (≥ 5 с); --since потребовал бы часов хоста
|
||
# против часов ВМ, а опоздавшая подписка — громкий отказ ниже
|
||
events_path = stand.work_dir / f"{phase}-events.log"
|
||
stream = stand.container_events(target_id, events_path)
|
||
try:
|
||
started = time.time()
|
||
load = stand.k6_console(
|
||
"failover.js",
|
||
{
|
||
"RUN": run_id,
|
||
"PHASE": number,
|
||
"DURATION": f"{duration}s",
|
||
"WRITE_RATE": p.failover_write_rate,
|
||
"READ_RATE": p.failover_read_rate,
|
||
"PRE_VUS": p.failover_pre_vus,
|
||
"MAX_VUS": p.failover_max_vus,
|
||
"SEED": p.seed + number,
|
||
"NAME": f"{phase}-k6",
|
||
},
|
||
log_name,
|
||
)
|
||
try:
|
||
time.sleep(p.failover_before_s)
|
||
acted = time.monotonic()
|
||
command = {
|
||
"stop": ("stop", "--timeout", "9"),
|
||
"kill": ("kill",),
|
||
"pause": ("pause",),
|
||
}[mode]
|
||
sh("docker", *command, target_id)
|
||
self.say(f"phase {phase}: {mode} {instance} ({slot})")
|
||
time.sleep(
|
||
max(0.0, acted + p.failover_down_s - time.monotonic()),
|
||
)
|
||
restore = ("unpause",) if mode == "pause" else ("start",)
|
||
sh("docker", *restore, target_id)
|
||
self.say(f"phase {phase}: {restore[0]} {instance}")
|
||
load.wait(timeout=duration + 120)
|
||
finally:
|
||
if load.poll() is None:
|
||
load.kill()
|
||
# реплика не должна остаться остановленной или замороженной
|
||
state = stand.state(target_id)
|
||
paused = sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Paused}}",
|
||
target_id,
|
||
check=False,
|
||
)
|
||
if paused == "true":
|
||
sh("docker", "unpause", target_id, check=False)
|
||
elif not state["running"]:
|
||
sh("docker", "start", target_id, check=False)
|
||
if load.returncode != 0:
|
||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)[-3000:]
|
||
raise BenchError(
|
||
f"k6 failover.js failed ({load.returncode}):\n{tail}",
|
||
)
|
||
try:
|
||
stand.wait_ready(timeout=30)
|
||
except BenchError as exc:
|
||
raise BenchError(
|
||
f"phase {phase}: replica not back: {exc}",
|
||
) from exc
|
||
finally:
|
||
stop_stream(stream)
|
||
down = stand.down_storages()
|
||
if down:
|
||
raise BenchError(
|
||
f"{', '.join(down)} went down during phase {phase}",
|
||
)
|
||
events = failover.parse_events(
|
||
events_path.read_text(encoding="utf-8", errors="replace"),
|
||
)
|
||
action = next(
|
||
(e for e in events if e.action == failover.ACTIONS[mode]),
|
||
None,
|
||
)
|
||
back = next(
|
||
(
|
||
e
|
||
for e in events
|
||
if e.action == failover.RESTORES[mode]
|
||
and action is not None
|
||
and e.time > action.time
|
||
),
|
||
None,
|
||
)
|
||
if action is None or back is None:
|
||
raise BenchError(
|
||
f"phase {phase}: docker events of {instance} not found",
|
||
)
|
||
lb_log = stand.logs(LB, started - 2)
|
||
lb_events = failover.parse_lb_log(lb_log)
|
||
(self.results.dir / "raw" / f"{phase}-lb.log").write_text(
|
||
failover.lb_lines(lb_log) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
address_back = stand.address(target)
|
||
slot_back = next(
|
||
(
|
||
s["slot"]
|
||
for s in stand.lb_slots()
|
||
if s["addr"].rsplit(":", 1)[0] == address_back
|
||
),
|
||
None,
|
||
)
|
||
requests, k6_data = self.k6_requests(phase, log_name)
|
||
if p.sample_resources:
|
||
self.phase_resources(
|
||
phase,
|
||
mode,
|
||
requests,
|
||
instance=instance,
|
||
action=action.time,
|
||
back=back.time,
|
||
)
|
||
summary = parse_summary(self.results.dir / "raw" / f"{phase}-k6.json")
|
||
timeline = failover.build_timeline(
|
||
requests,
|
||
action.time,
|
||
phase=phase,
|
||
mode=mode,
|
||
replica=index + 1,
|
||
)
|
||
row = failover.analyze(
|
||
requests,
|
||
timeline,
|
||
lb_events,
|
||
mode=mode,
|
||
instance=instance,
|
||
t0=action.time,
|
||
restore=back.time,
|
||
slot=slot,
|
||
slot_back=slot_back,
|
||
threshold=p.failover_success,
|
||
)
|
||
# неподтверждённые записи — повтор с тем же ключом и телом до 201
|
||
failed = sorted({r.n for r in requests if r.kind == "w" and not r.ok})
|
||
(stand.work_dir / "retry.json").write_text(
|
||
json.dumps({"items": failed}),
|
||
encoding="utf-8",
|
||
)
|
||
stand.k6("retry.js", {"RUN": run_id, "PHASE": number})
|
||
retried = stand.work_json("retry-result.json")
|
||
if retried["failed"]:
|
||
raise BenchError(
|
||
f"phase {phase}: retries with the same key failed: "
|
||
f"{retried['failed'][:5]}",
|
||
)
|
||
writes += row["writes"]
|
||
messages, texts = (
|
||
int(value)
|
||
for value in stand.psql(
|
||
"SELECT count(*), count(DISTINCT text) FROM messages "
|
||
f"WHERE chat_id = '{fixture['writeChatId']}';\n",
|
||
).split("|")
|
||
)
|
||
dropped = (
|
||
k6_data["metrics"]
|
||
.get("dropped_iterations", {})
|
||
.get("values", {})
|
||
.get("count", 0)
|
||
)
|
||
if dropped:
|
||
# k6 не выдал заданную интенсивность: не хватило VU
|
||
row["flags"] = " ".join(filter(None, [row["flags"], "dropped"]))
|
||
row.update(
|
||
{
|
||
"phase": phase,
|
||
"mode": mode,
|
||
"replica": index + 1,
|
||
"instance": instance,
|
||
"slot": slot,
|
||
"action_at": failover.iso(action.time),
|
||
"signal": action.signal or None,
|
||
"restore_at": failover.iso(back.time),
|
||
"slot_back": slot_back,
|
||
"retried_ok": retried["ok"],
|
||
"write_chat_messages": messages,
|
||
"distinct_texts": texts,
|
||
"pg_idle_tx_terminated": failover.count_idle_terminations(
|
||
stand.logs(DB, started - 2),
|
||
),
|
||
"dropped_iterations": int(dropped),
|
||
},
|
||
)
|
||
self.results.add_phase(row, timeline)
|
||
self.runs_done += 1
|
||
self.say(
|
||
f"phase {phase}: LB out {row['lb_out_s']} s "
|
||
f"({row['lb_out_state']}, bound {row['bound_s']} s), errors "
|
||
f"{row['errors_down']} [{row['codes_down']}], recovery "
|
||
f"{row['recovery_s']} s, back in LB {row['lb_up_s']} s, errors "
|
||
f"after {row['errors_after']}, {summary['requests_ok']} ok of "
|
||
f"{row['requests']}; retried {retried['ok']}, messages "
|
||
f"{messages} = keys {writes}"
|
||
+ (f" [{row['flags']}]" if row["flags"] else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if messages != writes or texts != writes:
|
||
raise BenchError(
|
||
f"phase {phase}: write chat has {messages} messages and "
|
||
f"{texts} distinct texts, {writes} keys sent",
|
||
)
|
||
return writes
|
||
|
||
def phase_resources(
|
||
self,
|
||
phase: str,
|
||
mode: str,
|
||
requests: list[failover.Request],
|
||
*,
|
||
instance: str,
|
||
action: float,
|
||
back: float,
|
||
) -> None:
|
||
"""Ресурсы фазы по окнам (решения 4, 5 design change
|
||
`profile-resources`): разгон — 2 с с первого запроса, до действия,
|
||
окно отказа до возврата, после возврата до последнего запроса.
|
||
`docker start` после `stop` / `kill` — новая cgroup, её счёт идёт
|
||
с момента возврата; `pause` … `unpause` — заморозка: CPU
|
||
граничных интервалов — вне её."""
|
||
first = min(r.start for r in requests)
|
||
last = max(r.start for r in requests)
|
||
self.check_observer(f"phase {phase}", first, last)
|
||
if mode == "pause":
|
||
self.frozen.setdefault(instance, []).append((action, back))
|
||
else:
|
||
self.starts.setdefault(instance, []).append(back)
|
||
self.containers.update(self.stand.container_map())
|
||
roles = {}
|
||
for ident, item in self.containers.items():
|
||
role = item["role"]
|
||
if role == APP_SERVICE:
|
||
role = "api-target" if ident == instance else "api-other"
|
||
roles[ident] = role
|
||
windows = [
|
||
(resources.WARMUP, first, first + resources.WARMUP_S),
|
||
("before", first + resources.WARMUP_S, action),
|
||
("down", action, back),
|
||
("after", back, last),
|
||
]
|
||
result = resources.analyze(
|
||
scaling.parse_samples(self.stand.sampler_logs()[0]),
|
||
roles,
|
||
windows,
|
||
ref=action,
|
||
ok_starts=[r.start for r in requests if r.ok],
|
||
phase=phase,
|
||
mode=mode,
|
||
profile="failover",
|
||
starts=self.starts,
|
||
frozen=self.frozen,
|
||
)
|
||
self.observe(result, f"phase {phase}", "failover")
|
||
|
||
# ------------------------------------------- профиль поочерёдной замены
|
||
|
||
def run_rollout(self) -> None:
|
||
"""Профиль поочерёдной замены (решение 8 design change
|
||
`rolling-update`): стенд на образе прежнего commit, постоянная
|
||
нагрузка, посреди неё `scripts/rolling-update.sh` против проекта
|
||
стенда, затем разбор, повтор неуспешных записей и сверка."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
info: dict[str, Any] = self.results.meta.setdefault("rollout", {})
|
||
self.stage = "old image"
|
||
commit = sh(
|
||
"git",
|
||
"-C",
|
||
str(ROOT),
|
||
"rev-parse",
|
||
"--short",
|
||
f"{p.rollout_from}^{{commit}}",
|
||
)
|
||
info["from_commit"] = commit
|
||
info["new_image"] = self.results.meta["app_image"]["id"]
|
||
info["old_image"] = stand.build_from(commit)
|
||
# реплики и migrate стенда берут тег APP_IMAGE: на старте — образ
|
||
# прежнего commit; скрипт пересоберёт тег из текущего checkout
|
||
sh("docker", "tag", FROM_IMAGE, APP_IMAGE)
|
||
self.say(
|
||
f"rollout: image of {commit} built ({info['old_image'][:19]}…)",
|
||
)
|
||
fixture = self.prepare_load("rollout")
|
||
info["containers_before"] = stand.app_containers()
|
||
info["revision_before"] = stand.psql(
|
||
"SELECT version_num FROM alembic_version;\n",
|
||
)
|
||
self.results.write_meta()
|
||
if p.sample_resources:
|
||
# до потока событий и k6: один create/start сэмплера — до нагрузки
|
||
self.start_observer("rollout")
|
||
old = {c["instance"] for c in info["containers_before"]}
|
||
run_id = secrets.token_hex(4)
|
||
name = "rollout"
|
||
log_name = f"{name}.log"
|
||
self.stage = "rollout"
|
||
self.progress()
|
||
duration = p.rollout_before_s + p.rollout_budget_s + p.rollout_after_s
|
||
started = time.time()
|
||
events_path = stand.work_dir / "events.log"
|
||
events = stand.project_events(events_path)
|
||
script_out = stand.work_dir / "rollout-script.out"
|
||
script_s = None
|
||
script: subprocess.Popen[bytes] | None = None
|
||
load: subprocess.Popen[bytes] | None = None
|
||
try:
|
||
# k6 — внутри try: при его сбое finally останавливает поток
|
||
load = stand.k6_console(
|
||
"failover.js",
|
||
{
|
||
"RUN": run_id,
|
||
"PHASE": 1,
|
||
"DURATION": f"{duration}s",
|
||
"WRITE_RATE": p.failover_write_rate,
|
||
"READ_RATE": p.failover_read_rate,
|
||
"PRE_VUS": p.failover_pre_vus,
|
||
"MAX_VUS": p.failover_max_vus,
|
||
"SEED": p.seed + 1,
|
||
"NAME": f"{name}-k6",
|
||
},
|
||
log_name,
|
||
)
|
||
time.sleep(p.rollout_before_s)
|
||
self.say("rollout: scripts/rolling-update.sh started")
|
||
began = time.monotonic()
|
||
with script_out.open("w") as out:
|
||
script = subprocess.Popen(
|
||
["sh", str(ROLLOUT_SCRIPT)],
|
||
env=stand.script_env(),
|
||
stdout=out,
|
||
stderr=subprocess.STDOUT,
|
||
)
|
||
try:
|
||
script.wait(timeout=p.rollout_budget_s + 120)
|
||
except subprocess.TimeoutExpired:
|
||
# SIGTERM: скрипт доводит текущий шаг и выходит (143)
|
||
script.terminate()
|
||
script.wait(timeout=60)
|
||
script_s = round(time.monotonic() - began, 1)
|
||
self.say(
|
||
f"rollout: script exited {script.returncode} in {script_s}s",
|
||
)
|
||
load.wait(timeout=duration + 120)
|
||
finally:
|
||
if load is not None and load.poll() is None:
|
||
load.kill()
|
||
# Ctrl+C достаётся и скрипту (та же группа процессов): он
|
||
# доводит текущий шаг; стенд удаляется только после него
|
||
if script is not None and script.poll() is None:
|
||
try:
|
||
script.wait(timeout=60)
|
||
except subprocess.TimeoutExpired:
|
||
script.kill()
|
||
stop_stream(events)
|
||
text = (
|
||
script_out.read_text(encoding="utf-8", errors="replace")
|
||
if script_out.exists()
|
||
else ""
|
||
)
|
||
(self.results.dir / "raw" / "rollout-script.log").write_text(
|
||
rollout.script_lines(text) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
exit_code = None if script is None else script.returncode
|
||
if exit_code != 0:
|
||
raise BenchError(
|
||
f"rolling-update.sh exited {exit_code}: "
|
||
f"{rollout.script_error(text) or 'see raw/rollout-script.log'}",
|
||
)
|
||
if load.returncode != 0:
|
||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)[-3000:]
|
||
raise BenchError(
|
||
f"k6 failover.js failed ({load.returncode}):\n{tail}",
|
||
)
|
||
stand.wait_ready(timeout=30)
|
||
down = stand.down_storages()
|
||
if down:
|
||
raise BenchError(f"{', '.join(down)} went down during rollout")
|
||
containers = rollout.parse_container_events(
|
||
events_path.read_text(encoding="utf-8", errors="replace"),
|
||
)
|
||
lb_log = stand.logs(LB, started - 2)
|
||
(self.results.dir / "raw" / f"{name}-lb.log").write_text(
|
||
failover.lb_lines(lb_log) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
slots = rollout.parse_slot_events(lb_log)
|
||
info["containers_after"] = stand.app_containers()
|
||
info["revision_after"] = stand.psql(
|
||
"SELECT version_num FROM alembic_version;\n",
|
||
)
|
||
info["tag_after"] = sh(
|
||
"docker",
|
||
"image",
|
||
"inspect",
|
||
"-f",
|
||
"{{.Id}}",
|
||
APP_IMAGE,
|
||
)
|
||
self.results.write_meta()
|
||
after = info["containers_after"]
|
||
new = {c["instance"] for c in after}
|
||
# итог замены: API_REPLICAS работающих реплик, ни одной прежней,
|
||
# все на одном образе, отличном от прежнего (решение 8)
|
||
if len(after) != p.replicas or not all(c["running"] for c in after):
|
||
raise BenchError(
|
||
f"{len(after)} api containers after rollout, expected "
|
||
f"{p.replicas} running",
|
||
)
|
||
if old & new:
|
||
raise BenchError(f"old replicas still running: {old & new}")
|
||
images_before = {c["image"] for c in info["containers_before"]}
|
||
images_after = {c["image"] for c in after}
|
||
if len(images_after) != 1 or images_after & images_before:
|
||
raise BenchError(
|
||
f"replicas after rollout run images {images_after}, before "
|
||
f"{images_before}",
|
||
)
|
||
requests, k6_data = self.k6_requests(name, log_name)
|
||
t0 = rollout.start_time(containers)
|
||
steps = rollout.parse_steps(text)
|
||
end_event = (
|
||
rollout.container_event(
|
||
containers,
|
||
steps[-1].get("old_id"),
|
||
"destroy",
|
||
)
|
||
if steps
|
||
else None
|
||
)
|
||
if t0 is None or end_event is None or len(steps) != p.replicas:
|
||
raise BenchError(
|
||
f"rollout steps not found: {len(steps)} steps in the script "
|
||
"log, docker events of the migration or the last old replica "
|
||
"missing",
|
||
)
|
||
replaced = rollout.replica_rows(steps, containers, slots, requests, t0)
|
||
if p.sample_resources:
|
||
self.rollout_resources(
|
||
requests,
|
||
containers,
|
||
t0=t0,
|
||
end=end_event.time,
|
||
old=old,
|
||
new=new,
|
||
)
|
||
self.stand.stop_sampler()
|
||
timeline = rollout.build_timeline(requests, t0, old, new)
|
||
row = rollout.analyze(
|
||
requests,
|
||
timeline,
|
||
replaced,
|
||
t0=t0,
|
||
end=end_event.time,
|
||
old=old,
|
||
new=new,
|
||
threshold=p.failover_success,
|
||
)
|
||
# неподтверждённые записи — повтор с тем же ключом и телом до 201
|
||
failed = sorted({r.n for r in requests if r.kind == "w" and not r.ok})
|
||
(stand.work_dir / "retry.json").write_text(
|
||
json.dumps({"items": failed}),
|
||
encoding="utf-8",
|
||
)
|
||
stand.k6("retry.js", {"RUN": run_id, "PHASE": 1})
|
||
retried = stand.work_json("retry-result.json")
|
||
if retried["failed"]:
|
||
raise BenchError(
|
||
f"rollout: retries with the same key failed: "
|
||
f"{retried['failed'][:5]}",
|
||
)
|
||
messages, texts = (
|
||
int(value)
|
||
for value in stand.psql(
|
||
"SELECT count(*), count(DISTINCT text) FROM messages "
|
||
f"WHERE chat_id = '{fixture['writeChatId']}';\n",
|
||
).split("|")
|
||
)
|
||
dropped = (
|
||
k6_data["metrics"]
|
||
.get("dropped_iterations", {})
|
||
.get("values", {})
|
||
.get("count", 0)
|
||
)
|
||
flags = row["flags"].split()
|
||
if dropped:
|
||
flags.append("dropped")
|
||
if script_s is not None and script_s > p.rollout_budget_s:
|
||
flags.append("over_budget")
|
||
row.update(
|
||
{
|
||
"label": self.label,
|
||
"from_commit": commit,
|
||
"replicas": p.replicas,
|
||
"revision_before": info["revision_before"],
|
||
"revision_after": info["revision_after"],
|
||
"build_s": rollout.seconds(rollout.BUILD_RE, text),
|
||
"migrate_s": rollout.seconds(rollout.MIGRATE_RE, text),
|
||
"script_exit": exit_code,
|
||
"retried_ok": retried["ok"],
|
||
"write_chat_messages": messages,
|
||
"distinct_texts": texts,
|
||
"dropped_iterations": int(dropped),
|
||
"flags": " ".join(flags),
|
||
},
|
||
)
|
||
self.results.add_rollout(row, replaced, timeline)
|
||
self.runs_done += 1
|
||
self.say(
|
||
f"rollout: {row['rollout_s']} s, errors before / during / after "
|
||
f"{row['errors_before']} / {row['errors_during']} "
|
||
f"[{row['codes_during']}] / {row['errors_after']}, min success "
|
||
f"{row['min_success']}, old after drain {row['old_after_drain']}, "
|
||
f"revision {info['revision_before']} -> {info['revision_after']}; "
|
||
f"retried {retried['ok']}, messages {messages} = keys "
|
||
f"{row['writes']}"
|
||
+ (f" [{row['flags']}]" if row["flags"] else ""),
|
||
)
|
||
self.progress(last=row)
|
||
if messages != row["writes"] or texts != row["writes"]:
|
||
raise BenchError(
|
||
f"rollout: write chat has {messages} messages and {texts} "
|
||
f"distinct texts, {row['writes']} keys sent",
|
||
)
|
||
|
||
def rollout_resources(
|
||
self,
|
||
requests: list[failover.Request],
|
||
containers: list[rollout.ContainerEvent],
|
||
*,
|
||
t0: float,
|
||
end: float,
|
||
old: set[str],
|
||
new: set[str],
|
||
) -> None:
|
||
"""Ресурсы замены по окнам (решения 4, 5 design change
|
||
`profile-resources`): до — от конца разгона до первого обращения
|
||
скрипта к стенду, подготовка — до создания контейнера миграции,
|
||
замена — до удаления последней старой реплики, после — до
|
||
последнего запроса. Роли — по `X-Instance-Id` до и после, по
|
||
сервису compose и по `create` из потока событий проекта
|
||
(контейнеры, удалённые до конца прогона)."""
|
||
first = min(r.start for r in requests)
|
||
last = max(r.start for r in requests)
|
||
self.check_observer("rollout", first, last)
|
||
self.containers.update(self.stand.container_map())
|
||
for event in containers:
|
||
ident = event.id[:12]
|
||
if event.action != "create" or ident in self.containers:
|
||
continue
|
||
role = ""
|
||
if "-migrate-" in event.name:
|
||
role = "migrate"
|
||
elif re.search(rf"-{APP_SERVICE}-\d+$", event.name):
|
||
role = APP_SERVICE
|
||
self.containers[ident] = {"name": event.name, "role": role}
|
||
for event in containers:
|
||
if event.action == "start":
|
||
self.starts.setdefault(event.id[:12], []).append(event.time)
|
||
roles = {}
|
||
for ident, item in self.containers.items():
|
||
role = item["role"]
|
||
if ident in old:
|
||
role = "api-old"
|
||
elif ident in new:
|
||
role = "api-new"
|
||
roles[ident] = role
|
||
info = self.results.meta["rollout"]
|
||
prepare = rollout.script_start(
|
||
containers,
|
||
first,
|
||
self.stand.healthcheck_actions(),
|
||
)
|
||
info["script_start_at"] = (
|
||
failover.iso(prepare) if prepare is not None else None
|
||
)
|
||
if prepare is None or prepare > t0:
|
||
prepare = t0
|
||
windows = [
|
||
(resources.WARMUP, first, first + resources.WARMUP_S),
|
||
("before", first + resources.WARMUP_S, prepare),
|
||
("prepare", prepare, t0),
|
||
("during", t0, end),
|
||
("after", end, last),
|
||
]
|
||
result = resources.analyze(
|
||
scaling.parse_samples(self.stand.sampler_logs()[0]),
|
||
roles,
|
||
windows,
|
||
ref=t0,
|
||
ok_starts=[r.start for r in requests if r.ok],
|
||
phase="rollout",
|
||
mode="rollout",
|
||
profile="rollout",
|
||
starts=self.starts,
|
||
)
|
||
self.observe(result, "rollout", "rollout")
|
||
|
||
# ---------------------------------------------- серии трека 1
|
||
|
||
# --------------------------------------------- профиль загрузок
|
||
|
||
def run_uploads(self) -> None:
|
||
"""Профиль загрузок (design change `upload-memory`, решение 10):
|
||
образы вариантов, тела, план кругами, стенд с S3 и брокером,
|
||
пользователь и чат, затем прогоны — на каждый новые volume S3 и
|
||
брокера и реплика."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
meta = self.results.meta
|
||
info: dict[str, Any] = meta.setdefault("uploads", {})
|
||
checkout = meta["app_image"]["id"]
|
||
images = {"unlimited": checkout, "limited": checkout}
|
||
if "before" in p.uploads_variants:
|
||
self.stage = "before image"
|
||
commit = sh(
|
||
"git",
|
||
"-C",
|
||
str(ROOT),
|
||
"rev-parse",
|
||
"--short",
|
||
f"{p.uploads_before}^{{commit}}",
|
||
)
|
||
info["before_commit"] = commit
|
||
images["before"] = stand.build_from(commit)
|
||
self.say(
|
||
f"uploads: image of {commit} built ({images['before'][:19]}…)",
|
||
)
|
||
info["images"] = {v: images[v] for v in p.uploads_variants}
|
||
codes = dict(p.uploads_success_codes)
|
||
info["success_codes"] = {v: codes[v] for v in p.uploads_variants}
|
||
bodies = uploads.build_bodies(p.uploads_sizes, p.seed)
|
||
heights = dict(p.uploads_sizes)
|
||
info["bodies"] = {
|
||
name: uploads.body_passport(name, heights[name], data)
|
||
for name, data in bodies.items()
|
||
}
|
||
stand.work_dir.mkdir(parents=True, exist_ok=True)
|
||
for name, data in bodies.items():
|
||
(stand.work_dir / f"body-{name}.png").write_bytes(data)
|
||
plan = uploads.plan_runs(
|
||
p.uploads_variants,
|
||
[name for name, _ in p.uploads_sizes],
|
||
p.uploads_clients,
|
||
p.repeats,
|
||
p.order_seed,
|
||
)
|
||
info["plan"] = plan
|
||
info["app_env"] = {}
|
||
self.runs_total = len(plan)
|
||
self.results.write_meta()
|
||
# стенд на образе checkout: миграции, пользователь и вход по Basic
|
||
# (Argon2id) — до первого окна; PostgreSQL и пользователь — на всю
|
||
# метку, а реплика окна — всегда новая
|
||
self.stage = "start"
|
||
stand.start(lambda text: self.say(f"uploads: {text}"))
|
||
meta["limits"] = stand.limits()
|
||
meta["stand"] = stand.passport()
|
||
self.results.write_meta()
|
||
stand.k6(
|
||
"prepare.js",
|
||
{"MEMBERS": p.members, "BACKGROUND": "", "WRITE_CHATS": 1},
|
||
)
|
||
self.prepared = True
|
||
self.say(
|
||
f"uploads: {len(plan)} runs planned "
|
||
f"({len(p.uploads_variants)} variants × "
|
||
f"{len(p.uploads_sizes)} bodies × {len(p.uploads_clients)} "
|
||
f"levels × {p.repeats})",
|
||
)
|
||
try:
|
||
for run in plan:
|
||
self.run_upload(run, images[run["variant"]], bodies)
|
||
self.runs_done += 1
|
||
finally:
|
||
# тег стенда — снова образ checkout
|
||
sh("docker", "tag", checkout, APP_IMAGE, check=False)
|
||
stand.extra_files = []
|
||
|
||
def run_upload(
|
||
self,
|
||
run: dict[str, Any],
|
||
image: str,
|
||
bodies: dict[str, bytes],
|
||
) -> None:
|
||
"""Один прогон: новые S3, брокер и реплика варианта, разогревающая
|
||
загрузка, сэмплер `peak` (сброс пика перед нагрузкой), k6
|
||
`constant-vus`, затем пики, состояние реплики и строка
|
||
`runs.csv`. Реплика, убитая из-за памяти, — пометка `oom`, а не
|
||
сбой команды."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
name = run["run"]
|
||
info = self.results.meta["uploads"]
|
||
self.stage = name
|
||
self.progress()
|
||
# worker нет, задания копятся: предел заданий мерил бы себя, а не память
|
||
# приёма (design change track1-queue-overload, решение 13). Образ «до»
|
||
# этой настройки не знает и её игнорирует
|
||
env = {
|
||
"UPLOADS__MAX_UNFINISHED_JOBS": "0",
|
||
**(
|
||
dict(p.uploads_unlimited_env)
|
||
if run["variant"] == "unlimited"
|
||
else {}
|
||
),
|
||
}
|
||
stand.recycle_uploads(image, env)
|
||
replica = stand.replicas()[0]
|
||
if run["variant"] not in info["app_env"]:
|
||
info["app_env"][run["variant"]] = stand.app_env(replica)
|
||
self.results.write_meta()
|
||
body = f"body-{run['size']}.png"
|
||
# разогрев: одна успешная загрузка до окна (соединения, модели
|
||
# botocore, соединение с брокером)
|
||
stand.k6("uploads.js", {"BODY": body, "WARMUP": 1})
|
||
targets = {
|
||
"api": stand.full_id(replica),
|
||
"s3": stand.full_id(S3),
|
||
"haproxy": stand.full_id(LB),
|
||
}
|
||
swap_before = swapouts()
|
||
stand.start_peak_sampler(name, targets)
|
||
log_name = f"{name}.log.k6"
|
||
load: subprocess.Popen[bytes] | None = None
|
||
peak_text = ""
|
||
try:
|
||
load = stand.k6_console(
|
||
"uploads.js",
|
||
{
|
||
"BODY": body,
|
||
"CLIENTS": run["clients"],
|
||
"RAMP_S": p.uploads_ramp_s,
|
||
"WINDOW_S": p.uploads_window_s,
|
||
"NAME": f"{name}-k6",
|
||
},
|
||
log_name,
|
||
)
|
||
load.wait(timeout=p.uploads_ramp_s + p.uploads_window_s + 180)
|
||
finally:
|
||
if load is not None and load.poll() is None:
|
||
load.kill()
|
||
peak_text = stand.stop_peak_sampler(name)
|
||
(self.results.dir / "raw" / f"{name}-peak.log").write_text(
|
||
peak_text,
|
||
encoding="utf-8",
|
||
)
|
||
if load.returncode != 0:
|
||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)[-3000:]
|
||
raise BenchError(
|
||
f"{name}: k6 uploads.js failed ({load.returncode}):\n{tail}",
|
||
)
|
||
swap_after = swapouts()
|
||
down = stand.down_storages()
|
||
if down:
|
||
raise BenchError(f"{name}: storage died: {', '.join(down)}")
|
||
state = stand.exit_state(replica)
|
||
s3_state = stand.exit_state(S3)
|
||
state["s3_down"] = not s3_state["running"] or s3_state["oom_killed"]
|
||
text = (stand.work_dir / log_name).read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)
|
||
t0, ups = uploads.parse_k6(text)
|
||
k6_data = json.loads(
|
||
(self.results.dir / "raw" / f"{name}-k6.json").read_text(
|
||
encoding="utf-8",
|
||
),
|
||
)
|
||
iterations = (
|
||
k6_data["metrics"]
|
||
.get("iterations{scenario:upload}", {})
|
||
.get("values", {})
|
||
.get("count")
|
||
)
|
||
if t0 is None or iterations != len(ups):
|
||
raise BenchError(
|
||
f"{name}: {len(ups)} k6 upload lines, {iterations} "
|
||
f"iterations, t0 {t0}",
|
||
)
|
||
stats = uploads.k6_stats(
|
||
ups,
|
||
t0,
|
||
p.uploads_ramp_s,
|
||
p.uploads_window_s,
|
||
success_code=info["success_codes"][run["variant"]],
|
||
)
|
||
swapped = (
|
||
None
|
||
if swap_before is None or swap_after is None
|
||
else swap_after - swap_before
|
||
)
|
||
row = uploads.run_row(
|
||
run,
|
||
len(bodies[run["size"]]),
|
||
uploads.parse_peak(peak_text),
|
||
stats,
|
||
state,
|
||
swapped,
|
||
)
|
||
self.results.add_upload(row)
|
||
fives = sum(stats[f"s{code}"] for code in (502, 503, 504))
|
||
self.say(
|
||
f"{name} ({run['order']}/{self.runs_total}): before "
|
||
f"{row['mem_before_mib']} MiB, peak {row['peak_mib']} MiB "
|
||
f"({row['peak_source']}), sock {row['sock_max_mib']} MiB, "
|
||
f"{stats['success_code']} "
|
||
f"{stats['s201']}, 429 {stats['s429']}, 5xx {fives}, conn "
|
||
f"{stats['s_conn_error']}, {stats['uploads_per_s']}/s, p95 "
|
||
f"{stats['p95_ms']} ms{' OOM' if 'oom' in row['flags'] else ''}",
|
||
)
|
||
self.progress(last={"run": name, "flags": row["flags"]})
|
||
|
||
# ------------------------------------------------ профиль очереди
|
||
|
||
def run_queue(self) -> None:
|
||
"""Профиль очереди (design change `track1-queue-overload`, решения
|
||
5–11, 14): файлы классов и план в паспорт до первой ячейки, ячейки на
|
||
свежем стенде; ячейка, потерявшая работу, — отказ всей метки в конце, но
|
||
все ячейки сняты."""
|
||
p = self.profile
|
||
meta = self.results.meta
|
||
plan = overload.plan_cells(
|
||
queue_cell_configs(p),
|
||
p.repeats,
|
||
p.order_seed,
|
||
)
|
||
self.runs_total = len(plan)
|
||
classes = self.stand.make_images(p.queue_image_seed)
|
||
self.say(
|
||
"queue: classes "
|
||
+ ", ".join(
|
||
f"{name} {item['bytes'] / 2**20:.2f} MiB"
|
||
for name, item in classes.items()
|
||
),
|
||
)
|
||
info: dict[str, Any] = meta.setdefault("queue", {})
|
||
info.update(
|
||
{
|
||
"classes": classes,
|
||
"image_seed": p.queue_image_seed,
|
||
"plan": plan,
|
||
"rates": {k: list(v) for k, v in QUEUE_RATES.items()},
|
||
"windows": {
|
||
"before_s": p.queue_before_s,
|
||
"after_s": p.queue_after_s,
|
||
"recovery_max_s": p.queue_recovery_max_s,
|
||
},
|
||
"control": dict(p.queue_control),
|
||
"default_limit": p.queue_default_limit,
|
||
},
|
||
)
|
||
info.setdefault("app_env", {})
|
||
self.results.write_meta()
|
||
done = {
|
||
row["cell"]
|
||
for row in overload.read_csv(self.results.dir / "cells.csv")
|
||
}
|
||
fatal = []
|
||
for row in overload.read_csv(self.results.dir / "cells.csv"):
|
||
if any(overload.flagged(row, f) for f in overload.FATAL_FLAGS):
|
||
fatal.append(row["cell"])
|
||
self.say(f"queue: {len(plan)} cells planned, {len(done)} already done")
|
||
self.runs_done = len(done)
|
||
for cell in plan:
|
||
if cell["cell"] in done:
|
||
continue
|
||
started = time.monotonic()
|
||
self.stage = cell["cell"]
|
||
self.progress()
|
||
try:
|
||
row = self.queue_cell(cell, classes)
|
||
finally:
|
||
self.queue_stop_events()
|
||
if not self.stand.stop():
|
||
self.say(f"cleanup incomplete: {leftovers()}")
|
||
if any(overload.flagged(row, f) for f in overload.FATAL_FLAGS):
|
||
fatal.append(cell["cell"])
|
||
self.cell_seconds.append(time.monotonic() - started)
|
||
self.progress(last={"cell": cell["cell"], "flags": row["flags"]})
|
||
if fatal:
|
||
raise BenchError(
|
||
"work was lost in cells: " + ", ".join(sorted(set(fatal))),
|
||
)
|
||
|
||
def queue_cell(
|
||
self,
|
||
cell: dict[str, Any],
|
||
classes: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
"""Ячейка: свежий стенд, пользователь и чат, разогрев, сэмплер, контрольный
|
||
поток, окна `before` и `over`, действие блока, разбор очереди, окно `after`,
|
||
обход метаданных, чтение SQL и журналов, строка `cells.csv`."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
name = cell["cell"]
|
||
self.say(f"{name} ({cell['order']}/{self.runs_total}): start")
|
||
env = {
|
||
"api": (
|
||
{}
|
||
if cell["limit"] is None
|
||
else {"UPLOADS__MAX_UNFINISHED_JOBS": str(cell["limit"])}
|
||
),
|
||
"worker": {
|
||
"WORKER__CONCURRENCY": str(cell["concurrency"]),
|
||
"WORKER__PREFETCH": str(cell["prefetch"]),
|
||
},
|
||
}
|
||
stand.queue_overrides({k: v for k, v in env.items() if v})
|
||
stand.start(lambda _: None)
|
||
info = self.results.meta["queue"]
|
||
limits = stand.limits()
|
||
known = self.results.meta.get("limits")
|
||
if known is None:
|
||
self.results.meta["limits"] = limits
|
||
self.results.meta["stand"] = stand.passport()
|
||
self.results.write_meta()
|
||
elif known != limits:
|
||
raise BenchError(f"{name}: limits differ from the first cell")
|
||
info["app_env"][cell["config"]] = {
|
||
"api": stand.app_env(stand.replicas()[0]),
|
||
"worker": stand.app_env(WORKER),
|
||
"relay": stand.app_env(RELAY),
|
||
}
|
||
stand.k6(
|
||
"prepare.js",
|
||
{"MEMBERS": p.members, "BACKGROUND": "", "WRITE_CHATS": 1},
|
||
)
|
||
self.queue_warm(cell, classes)
|
||
|
||
body = classes[cell["class"]]
|
||
vus_over = (
|
||
QUEUE_ACCEPT_VUS if cell["block"] == "accept" else QUEUE_OVER_VUS
|
||
)
|
||
base_env = {"BODY": body["file"], "MEDIA": body["media_type"]}
|
||
queue_name = f"{stand.env_value('POSTGRES_DB')}.attachment.process.v1"
|
||
stand.start_queue_sampler(queue_name)
|
||
swap_before = swapouts()
|
||
control = dict(p.queue_control)
|
||
background = stand.k6_console(
|
||
"background.js",
|
||
{
|
||
"DOWNLOAD_RATE": control["download"],
|
||
"WRITE_RATE": control["write"],
|
||
"READ_RATE": control["read"],
|
||
"DURATION": "6h",
|
||
"PRE_VUS": 10,
|
||
"MAX_VUS": p.queue_control_max_vus,
|
||
"NAME": f"{name}-bg",
|
||
},
|
||
f"{name}.bg.k6",
|
||
)
|
||
extra: dict[str, Any] = {}
|
||
action_at = None
|
||
self.queue_events = stand.death_events(
|
||
stand.work_dir / f"{name}.docker-events",
|
||
)
|
||
try:
|
||
time.sleep(2)
|
||
load_log = f"{name}.load.k6"
|
||
load = stand.k6_console(
|
||
"queue.js",
|
||
{
|
||
**base_env,
|
||
"WINDOWS": (
|
||
f"before:{cell['rate_low']}:{cell['before_s']}:"
|
||
f"{p.queue_low_vus},"
|
||
f"over:{cell['rate_high']}:{cell['over_s']}:{vus_over}"
|
||
),
|
||
"NAME": f"{name}-load",
|
||
},
|
||
load_log,
|
||
)
|
||
t0 = self.queue_wait_t0(load, load_log)
|
||
seen = time.monotonic()
|
||
over_start = cell["before_s"]
|
||
over_end = over_start + cell["over_s"]
|
||
action = cell["action"]
|
||
if action in ("kill-worker", "restart-broker"):
|
||
time.sleep(
|
||
max(
|
||
0.0,
|
||
seen
|
||
+ over_start
|
||
+ cell["over_s"] / 2
|
||
- time.monotonic(),
|
||
),
|
||
)
|
||
action_at = t0 + time.monotonic() - seen
|
||
if action == "kill-worker":
|
||
sh("docker", "kill", WORKER)
|
||
# демон считает это ручной остановкой: запуск — явный
|
||
sh("docker", "start", WORKER)
|
||
else:
|
||
stand.compose("restart", BROKER_SERVICE)
|
||
elif action == "second-worker":
|
||
time.sleep(max(0.0, seen + over_end - time.monotonic()))
|
||
action_at = t0 + time.monotonic() - seen
|
||
stand.compose(
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
WORKER_2,
|
||
"worker",
|
||
)
|
||
extra["second_started_at"] = action_at
|
||
load.wait(timeout=over_end + 180)
|
||
if load.returncode != 0:
|
||
raise BenchError(
|
||
f"{name}: k6 queue.js exit {load.returncode}:\n"
|
||
+ stand.read_work(f"{load_log}.out")[-3000:],
|
||
)
|
||
_, first_part = overload.parse_uploads(stand.read_work(load_log))
|
||
load_end = max(
|
||
(u.answered for u in first_part if u.window == "over"),
|
||
default=t0 + over_end,
|
||
)
|
||
self.queue_wait_zero(load_end, p.queue_recovery_max_s)
|
||
after_log = f"{name}.after.k6"
|
||
extra["after_start"] = time.time()
|
||
after = stand.k6_console(
|
||
"queue.js",
|
||
{
|
||
**base_env,
|
||
"WINDOWS": (
|
||
f"after:{cell['rate_low']}:{cell['after_s']}:"
|
||
f"{p.queue_low_vus}"
|
||
),
|
||
"NAME": f"{name}-after",
|
||
},
|
||
after_log,
|
||
)
|
||
after.wait(timeout=cell["after_s"] + 180)
|
||
if after.returncode != 0:
|
||
raise BenchError(
|
||
f"{name}: k6 after window exit {after.returncode}:\n"
|
||
+ stand.read_work(f"{after_log}.out")[-3000:],
|
||
)
|
||
extra["after_end"] = time.time()
|
||
after_t0, second_part = overload.parse_uploads(
|
||
stand.read_work(after_log),
|
||
)
|
||
if after_t0 is not None:
|
||
extra["after_start"] = after_t0
|
||
extra["after_end"] = after_t0 + cell["after_s"]
|
||
after_end = max(
|
||
(u.answered for u in second_part),
|
||
default=extra["after_end"],
|
||
)
|
||
self.queue_wait_zero(after_end, p.queue_recovery_max_s)
|
||
self.check_observer(name, t0, after_end)
|
||
finally:
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
K6,
|
||
"pkill",
|
||
"-INT",
|
||
"-f",
|
||
"background.js",
|
||
check=False,
|
||
)
|
||
try:
|
||
background.wait(timeout=60)
|
||
except subprocess.TimeoutExpired:
|
||
background.kill()
|
||
requests = reconcile.parse_background(stand.read_work(f"{name}.bg.k6"))
|
||
bg_data = json.loads(
|
||
(self.results.dir / "raw" / f"{name}-bg.json").read_text(
|
||
encoding="utf-8",
|
||
),
|
||
)
|
||
problems, _ = reconcile.line_problems(
|
||
requests,
|
||
bg_data,
|
||
max_vus=p.queue_control_max_vus,
|
||
)
|
||
if problems:
|
||
raise BenchError(f"{name}: " + "; ".join(problems))
|
||
uploads_seen = [*first_part, *second_part]
|
||
dropped = 0
|
||
for part, log, summary in (
|
||
(first_part, load_log, f"{name}-load"),
|
||
(second_part, after_log, f"{name}-after"),
|
||
):
|
||
data = json.loads(
|
||
(self.results.dir / "raw" / f"{summary}.json").read_text(
|
||
encoding="utf-8",
|
||
),
|
||
)
|
||
iterations = reconcile.summary_count(data, "iterations")
|
||
if iterations is None or iterations != len(part):
|
||
raise BenchError(
|
||
f"{name}: {len(part)} k6 lines in {log}, {iterations} "
|
||
"iterations",
|
||
)
|
||
dropped += reconcile.summary_count(data, "dropped_iterations") or 0
|
||
row, timeline = self.queue_collect(
|
||
cell,
|
||
t0,
|
||
uploads_seen,
|
||
requests,
|
||
dropped,
|
||
action_at,
|
||
swap_before,
|
||
extra,
|
||
)
|
||
self.results.add_cell(row, timeline)
|
||
# готовая строка — готовая ячейка: при прерывании сразу после неё счёт тот же
|
||
self.runs_done += 1
|
||
self.say(
|
||
f"{name}: accepted {row['accepted']}, 429 {row['s429']} (busy "
|
||
f"{row['busy_429']}, backlog {row['backlog_429']}), peak "
|
||
f"{row['peak_unfinished']}, ready/s {row['ready_s_over']} / "
|
||
f"{row['ready_s_recovery']}, recovery {row['recovery_s']} s, p95 "
|
||
f"{row['ready_over_p95_s']} s, memory.peak "
|
||
f"{row['memory_peak_mib']} MiB, oom {row['oom_kill']}, worker die "
|
||
f"oom/command/other {row['worker_die_oom']}/"
|
||
f"{row['worker_die_command']}/{row['worker_die_other']}, attempts>1 "
|
||
f"{row['attempts_over_one']}, lb up min {row['lb_up_min']}, flags "
|
||
f"[{row['flags']}]",
|
||
)
|
||
for path in stand.work_dir.glob(f"{name}.*"):
|
||
path.unlink(missing_ok=True)
|
||
return row
|
||
|
||
def queue_warm(
|
||
self,
|
||
cell: dict[str, Any],
|
||
classes: dict[str, Any],
|
||
) -> None:
|
||
"""Разогрев (решение 6): файлы `s` до `ready`; они же — готовые байты
|
||
контрольного потока."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
log = f"{cell['cell']}.warm.k6"
|
||
warm = stand.k6_console(
|
||
"queue.js",
|
||
{
|
||
"BODY": classes["s"]["file"],
|
||
"MEDIA": classes["s"]["media_type"],
|
||
"WARM": p.queue_warm,
|
||
"NAME": f"{cell['cell']}-warm",
|
||
},
|
||
log,
|
||
)
|
||
warm.wait(timeout=120)
|
||
_, seen = overload.parse_uploads(stand.read_work(log))
|
||
ids = [u.attachment_id for u in seen if u.status == 202]
|
||
if warm.returncode != 0 or len(ids) != p.queue_warm:
|
||
raise BenchError(
|
||
f"{cell['cell']}: warmup accepted {len(ids)} of {p.queue_warm} "
|
||
f"(k6 exit {warm.returncode}, statuses "
|
||
f"{sorted({u.status for u in seen})})",
|
||
)
|
||
deadline = time.monotonic() + QUEUE_WARM_WAIT_S
|
||
while True:
|
||
left = int(
|
||
stand.psql(
|
||
"SELECT count(*) FROM attachment_jobs "
|
||
"WHERE state IN ('queued', 'processing');\n",
|
||
),
|
||
)
|
||
if not left:
|
||
break
|
||
if time.monotonic() > deadline:
|
||
raise BenchError(f"{cell['cell']}: warmup left {left} jobs")
|
||
time.sleep(0.5)
|
||
(stand.work_dir / "downloads.json").write_text(
|
||
json.dumps(ids),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
def queue_wait_t0(
|
||
self,
|
||
load: subprocess.Popen[bytes],
|
||
log_name: str,
|
||
) -> float:
|
||
"""Момент начала окон: k6 создаёт VU с телом до `setup()`, это секунды."""
|
||
deadline = time.monotonic() + 120
|
||
while time.monotonic() < deadline:
|
||
t0, _ = overload.parse_uploads(self.stand.read_work(log_name))
|
||
if t0 is not None:
|
||
return t0
|
||
if load.poll() is not None:
|
||
break
|
||
time.sleep(0.05)
|
||
raise BenchError(
|
||
f"k6 queue.js has not started: exit {load.poll()}\n"
|
||
+ self.stand.read_work(f"{log_name}.out")[-3000:],
|
||
)
|
||
|
||
def queue_wait_zero(self, since: float, timeout: float) -> bool:
|
||
"""Ноль незавершённых по строкам сэмплера после момента `since`; срок
|
||
вышел — `False`: пометку `not_recovered` ставит разбор."""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
jobs = overload.parse_jobs(self.stand.read_work("jobs.log"))
|
||
if any(t.t >= since and t.unfinished == 0 for t in jobs):
|
||
return True
|
||
time.sleep(1)
|
||
return False
|
||
|
||
def queue_collect(
|
||
self,
|
||
cell: dict[str, Any],
|
||
t0: float,
|
||
uploads_seen: list[overload.Upload],
|
||
requests: list[Any],
|
||
dropped: int,
|
||
action_at: float | None,
|
||
swap_before: int | None,
|
||
extra: dict[str, Any],
|
||
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||
"""Обход метаданных, чтение SQL, журналов и счётчиков: строка ячейки."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
name = cell["cell"]
|
||
accepted = [u.attachment_id for u in uploads_seen if u.status == 202]
|
||
order = list(accepted)
|
||
random.Random(p.order_seed + cell["order"]).shuffle(order)
|
||
ids_file = f"{name}.ids.json"
|
||
(stand.work_dir / ids_file).write_text(
|
||
json.dumps(order),
|
||
encoding="utf-8",
|
||
)
|
||
sweep_log = f"{name}.sweep.k6"
|
||
sweep = stand.k6_console(
|
||
"queue_sweep.js",
|
||
{
|
||
"IDS": ids_file,
|
||
"SAMPLE": p.queue_sample,
|
||
"NAME": f"{name}-sweep",
|
||
},
|
||
sweep_log,
|
||
)
|
||
sweep.wait(timeout=620)
|
||
if sweep.returncode != 0:
|
||
raise BenchError(
|
||
f"{name}: k6 queue_sweep.js exit {sweep.returncode}:\n"
|
||
+ stand.read_work(f"{sweep_log}.out")[-3000:],
|
||
)
|
||
metas, checks = overload.parse_sweep(stand.read_work(sweep_log))
|
||
sql = json.loads(
|
||
stand.psql(
|
||
"SELECT json_build_object("
|
||
"'extra_jobs', (SELECT count(*) FROM (SELECT attachment_id "
|
||
"FROM attachment_jobs GROUP BY attachment_id, pipeline_version "
|
||
"HAVING count(*) > 1) x), "
|
||
"'bad_variants', (SELECT count(*) FROM attachments a "
|
||
"WHERE a.status = 'ready' AND (SELECT count(*) FROM "
|
||
"attachment_variants v WHERE v.attachment_id = a.id) <> 3), "
|
||
"'attempts_over_one', (SELECT count(*) FROM attachment_jobs "
|
||
"WHERE attempt > 1), "
|
||
"'failed', (SELECT coalesce(json_object_agg(k, n), '{}') FROM "
|
||
"(SELECT coalesce(a.error_code, '-') || '/' || "
|
||
"coalesce(j.last_error_code, '-') AS k, count(*) AS n "
|
||
"FROM attachment_jobs j JOIN attachments a ON "
|
||
"a.id = j.attachment_id WHERE j.state = 'failed' GROUP BY 1) s)"
|
||
");\n",
|
||
),
|
||
)
|
||
# в обратную сторону: вложение в базе, на которое никто не отвечал 202
|
||
db_ids = set(
|
||
json.loads(
|
||
stand.psql(
|
||
"SELECT coalesce(json_agg(id), '[]') FROM attachments;\n",
|
||
),
|
||
),
|
||
)
|
||
warm_ids = set(
|
||
json.loads(stand.read_work("downloads.json")),
|
||
)
|
||
worker_records = reconcile.parse_records(self.queue_log(WORKER))
|
||
started = overload.events(worker_records, "Started worker")
|
||
if not started:
|
||
raise BenchError(f"{name}: no «Started worker» record")
|
||
for record in started:
|
||
if (
|
||
record.get("concurrency") != cell["concurrency"]
|
||
or record.get("prefetch") != cell["prefetch"]
|
||
):
|
||
raise BenchError(
|
||
f"{name}: worker started with "
|
||
f"({record.get('concurrency')}, {record.get('prefetch')}), "
|
||
f"cell ({cell['concurrency']}, {cell['prefetch']})",
|
||
)
|
||
second_records = (
|
||
reconcile.parse_records(self.queue_log(WORKER_2))
|
||
if cell["action"] == "second-worker"
|
||
else []
|
||
)
|
||
relay_records = reconcile.parse_records(self.queue_log(RELAY))
|
||
api_records = [
|
||
record
|
||
for replica in stand.replicas()
|
||
for record in reconcile.parse_records(self.queue_log(replica))
|
||
]
|
||
samples = scaling.parse_samples(stand.sampler_logs()[0])
|
||
mapping = stand.container_map()
|
||
worker_ids = {
|
||
ident for ident, item in mapping.items() if item["name"] == WORKER
|
||
}
|
||
cg: dict[str, list[list[list[tuple[int, ...]]]]] = {}
|
||
for (_, ident), points in samples.cg.items():
|
||
role = mapping.get(ident, {}).get("role", "")
|
||
if not role:
|
||
continue
|
||
cg.setdefault(role, []).append(overload.split_lives(points))
|
||
text = stand.sampler_logs()[0]
|
||
mp_all = overload.parse_mp(text)
|
||
mp = {
|
||
"worker": [
|
||
life
|
||
for ident in worker_ids
|
||
for life in overload.split_lives(mp_all.get(ident, []))
|
||
],
|
||
}
|
||
# выключатель телеметрии: пик памяти сервисов телеметрии (design
|
||
# change track1-telemetry-cost, решение 13)
|
||
for role in overload.TELEMETRY_ROLES:
|
||
mp[role] = [
|
||
life
|
||
for ident, item in mapping.items()
|
||
if item["role"] == role
|
||
for life in overload.split_lives(mp_all.get(ident, []))
|
||
]
|
||
broker_id = next(
|
||
(i for i, item in mapping.items() if item["name"] == BROKER),
|
||
None,
|
||
)
|
||
broker_points = [
|
||
point
|
||
for key, points in samples.cg.items()
|
||
if key[1] == broker_id
|
||
for point in points
|
||
]
|
||
extra["broker_memory_mib"] = (
|
||
round(max(point[5] for point in broker_points) / 2**20, 1)
|
||
if broker_points
|
||
else None
|
||
)
|
||
extra["broker_refused"] = sum(
|
||
1
|
||
for record in relay_records
|
||
if record.get("event") == "Relay pass"
|
||
and record.get("unconfirmed")
|
||
)
|
||
mq, mq_missing = overload.parse_mq(text)
|
||
swap_after = swapouts()
|
||
swapped = (
|
||
None
|
||
if swap_before is None or swap_after is None
|
||
else swap_after - swap_before
|
||
)
|
||
hwm = sh(
|
||
"docker",
|
||
"exec",
|
||
WORKER,
|
||
"sh",
|
||
"-c",
|
||
"grep VmHWM /proc/1/status",
|
||
check=False,
|
||
).split()
|
||
restarts, relay_restarts = (
|
||
sh(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.RestartCount}}",
|
||
container,
|
||
check=False,
|
||
)
|
||
for container in (WORKER, RELAY)
|
||
)
|
||
self.queue_stop_events()
|
||
docker_events = overload.parse_docker_events(
|
||
stand.read_work(f"{name}.docker-events"),
|
||
)
|
||
died = overload.die_causes(docker_events, WORKER)["die"]
|
||
if restarts.isdigit() and int(restarts) > died:
|
||
raise BenchError(
|
||
f"{name}: worker restarted {restarts} times, docker events "
|
||
f"show {died} exits",
|
||
)
|
||
src = overload.Sources(
|
||
uploads=uploads_seen,
|
||
t0=t0,
|
||
metas=metas,
|
||
sha_checks=checks,
|
||
jobs=overload.parse_jobs(stand.read_work("jobs.log")),
|
||
mq=mq,
|
||
mq_missing=mq_missing,
|
||
cg=cg,
|
||
mp=mp,
|
||
control=requests,
|
||
worker_records=worker_records,
|
||
second_records=second_records,
|
||
relay_records=relay_records,
|
||
api_records=api_records,
|
||
sql=sql,
|
||
hwm_kib=int(hwm[1]) if len(hwm) >= 2 else None,
|
||
restarts=int(restarts) if restarts.isdigit() else 0,
|
||
dropped=dropped,
|
||
action_at=action_at,
|
||
swapouts=swapped,
|
||
lb_down=overload.lb_down(samples.lb, p.replicas),
|
||
storage_down=bool(stand.down_storages())
|
||
or not stand.exit_state(S3)["running"],
|
||
extra=extra,
|
||
docker_events=docker_events,
|
||
worker_name=WORKER,
|
||
relay_name=RELAY,
|
||
relay_restarts=(
|
||
int(relay_restarts) if relay_restarts.isdigit() else 0
|
||
),
|
||
lb_up_min=overload.lb_up_min(samples.lb),
|
||
db_ids=db_ids,
|
||
warm_ids=warm_ids,
|
||
)
|
||
stand.stop_sampler()
|
||
return overload.cell_row(cell, src), overload.timeline_rows(cell, src)
|
||
|
||
def queue_stop_events(self) -> None:
|
||
"""Поток событий ячейки: повторный вызов и уборка при отказе — без него."""
|
||
events, self.queue_events = self.queue_events, None
|
||
if events is None:
|
||
return
|
||
events.terminate()
|
||
try:
|
||
events.wait(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
events.kill()
|
||
events.wait()
|
||
|
||
def queue_log(self, container: str) -> str:
|
||
proc = subprocess.run(
|
||
["docker", "logs", container],
|
||
capture_output=True,
|
||
text=True,
|
||
errors="replace",
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
# ------------------------------------------- профиль цены телеметрии
|
||
|
||
def run_telemetry(self) -> None:
|
||
"""Профиль цены телеметрии (design change `track1-telemetry-cost`,
|
||
решения 1, 2, 8–12): файлы классов, таблица вариантов и план в паспорт
|
||
до первого прогона; прогон — свежий стенд; принятая работа, не
|
||
дошедшая до итога, — отказ метки в конце, но все прогоны сняты."""
|
||
p = self.profile
|
||
meta = self.results.meta
|
||
table = telemetry_cost.variants(
|
||
p.telemetry_head_ratio,
|
||
p.telemetry_metrics_interval,
|
||
)
|
||
plan = telemetry_cost.plan_runs(
|
||
p.telemetry_variants,
|
||
p.repeats,
|
||
p.order_seed,
|
||
)
|
||
self.runs_total = len(plan)
|
||
classes = self.stand.make_images(p.queue_image_seed)
|
||
info: dict[str, Any] = meta.setdefault("telemetry_cost", {})
|
||
info.update(
|
||
{
|
||
"variants": {
|
||
name: {
|
||
**table[name],
|
||
"sdk_env": telemetry_cost.sdk_env(table[name]),
|
||
"wait_after_s": telemetry_cost.wait_after(table[name]),
|
||
"config_sha256": {
|
||
path: hashlib_sha256(ROOT / path)
|
||
for path in (
|
||
table[name].get("collector_layer"),
|
||
table[name].get("prometheus"),
|
||
)
|
||
if path
|
||
},
|
||
}
|
||
for name in p.telemetry_variants
|
||
},
|
||
"plan": plan,
|
||
"rates": dict(p.telemetry_rates),
|
||
"windows": list(p.telemetry_windows),
|
||
"locks": {
|
||
"offsets_s": list(p.telemetry_locks),
|
||
"seconds": p.telemetry_lock_s,
|
||
"sql": "LOCK TABLE messages IN SHARE MODE",
|
||
},
|
||
"slow_ms": p.telemetry_slow_ms,
|
||
"classes": {
|
||
key: classes[key] for key in ("s", "bad") if key in classes
|
||
},
|
||
"warm_uploads": p.queue_warm,
|
||
},
|
||
)
|
||
info.setdefault("app_env", {})
|
||
info.setdefault("locks_at", {})
|
||
self.results.write_meta()
|
||
rows = telemetry_cost.read_csv(self.results.dir / "runs.csv")
|
||
done = {row["run"] for row in rows}
|
||
fatal = [
|
||
row["run"]
|
||
for row in rows
|
||
if any(
|
||
flag in (row.get("flags") or "").split()
|
||
for flag in telemetry_cost.FATAL_FLAGS
|
||
)
|
||
]
|
||
self.say(f"telemetry: {len(plan)} runs planned, {len(done)} done")
|
||
self.runs_done = len(done)
|
||
for run in plan:
|
||
if run["run"] in done:
|
||
continue
|
||
started = time.monotonic()
|
||
self.stage = run["run"]
|
||
self.progress()
|
||
try:
|
||
row = self.telemetry_run(run, table[run["variant"]], classes)
|
||
finally:
|
||
if not self.stand.stop():
|
||
self.say(f"cleanup incomplete: {leftovers()}")
|
||
if "lost" in row["flags"].split():
|
||
fatal.append(run["run"])
|
||
self.cell_seconds.append(time.monotonic() - started)
|
||
self.progress(last={"run": run["run"], "flags": row["flags"]})
|
||
if fatal:
|
||
raise BenchError(
|
||
"accepted uploads did not reach their outcome in runs: "
|
||
+ ", ".join(sorted(set(fatal))),
|
||
)
|
||
|
||
def telemetry_run(
|
||
self,
|
||
run: dict[str, Any],
|
||
variant: dict[str, Any],
|
||
classes: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
"""Прогон (решение 9): свежий стенд варианта, подготовка, разогрев,
|
||
вложение без объекта, сэмплер, k6 и блокировки окна отказов, ожидание
|
||
заданий, обход метаданных, ожидание отправок, проверка варианта,
|
||
выборка, размер volume, строки таблиц."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
name = run["run"]
|
||
info = self.results.meta["telemetry_cost"]
|
||
self.say(f"{name} ({run['order']}/{self.runs_total}): start")
|
||
path = stand.work_dir / VARIANT_COMPOSE
|
||
stand.work_dir.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(
|
||
telemetry_cost.variant_compose(variant, ROOT),
|
||
encoding="utf-8",
|
||
)
|
||
stand.telemetry = bool(variant.get("telemetry"))
|
||
stand.extra_files = [path]
|
||
stand.start(lambda _: None)
|
||
self.telemetry_passport(variant)
|
||
info["app_env"][run["variant"]] = {
|
||
"api": stand.app_env(stand.replicas()[0]),
|
||
"worker": stand.app_env(WORKER),
|
||
"relay": stand.app_env(RELAY),
|
||
}
|
||
stand.k6(
|
||
"prepare.js",
|
||
{"MEMBERS": p.members, "BACKGROUND": "", "WRITE_CHATS": 1},
|
||
)
|
||
self.queue_warm({"cell": name}, classes)
|
||
broken = self.telemetry_broken_row()
|
||
(stand.work_dir / "broken.json").write_text(
|
||
json.dumps(broken),
|
||
encoding="utf-8",
|
||
)
|
||
stand.start_scan_sampler("cost")
|
||
swap_before = swapouts()
|
||
windows = tuple(p.telemetry_windows)
|
||
total_s = sum(windows)
|
||
load_log = f"{name}.load.k6"
|
||
load = stand.k6_console(
|
||
"telemetry.js",
|
||
{
|
||
"RATES": ",".join(
|
||
f"{cls}:{rate:g}" for cls, rate in p.telemetry_rates
|
||
),
|
||
"DURATION": total_s,
|
||
"NAME": f"{name}-load",
|
||
},
|
||
load_log,
|
||
)
|
||
t0 = self.telemetry_wait_t0(load, load_log)
|
||
seen = time.monotonic()
|
||
locks: list[dict[str, float]] = []
|
||
locker = threading.Thread(
|
||
target=self.telemetry_locks,
|
||
args=(seen + windows[0] + windows[1], locks),
|
||
daemon=True,
|
||
)
|
||
locker.start()
|
||
try:
|
||
load.wait(timeout=total_s + 120)
|
||
except subprocess.TimeoutExpired as exc:
|
||
load.kill()
|
||
raise BenchError(
|
||
f"{name}: k6 telemetry.js did not finish",
|
||
) from exc
|
||
locker.join(timeout=30)
|
||
load_end = time.monotonic()
|
||
if load.returncode != 0:
|
||
raise BenchError(
|
||
f"{name}: k6 telemetry.js exit {load.returncode}:\n"
|
||
+ stand.read_work(f"{load_log}.out")[-3000:],
|
||
)
|
||
info["locks_at"][name] = locks
|
||
_, reqs = telemetry_cost.parse_k6(stand.read_work(load_log))
|
||
data = json.loads(
|
||
(self.results.dir / "raw" / f"{name}-load.json").read_text(
|
||
encoding="utf-8",
|
||
),
|
||
)
|
||
iterations = reconcile.summary_count(data, "iterations")
|
||
if iterations is None or iterations != len(reqs):
|
||
raise BenchError(
|
||
f"{name}: {len(reqs)} k6 lines in {load_log}, {iterations} "
|
||
"iterations",
|
||
)
|
||
dropped = reconcile.summary_count(data, "dropped_iterations") or 0
|
||
self.check_observer(name, t0, t0 + total_s)
|
||
drained = self.telemetry_drain()
|
||
metas = self.telemetry_sweep(name, reqs)
|
||
# обход метаданных — последний трафик прогона: его span и решения
|
||
# выбора трасс тоже должны дойти до хранилищ до выборки
|
||
load_end = time.monotonic()
|
||
stand.stop_sampler()
|
||
problems = telemetry_cost.startup_problems(
|
||
self.telemetry_startup_records(),
|
||
variant,
|
||
)
|
||
lookup = None
|
||
if variant.get("telemetry"):
|
||
time.sleep(
|
||
max(
|
||
0.0,
|
||
load_end
|
||
+ telemetry_cost.wait_after(variant)
|
||
- time.monotonic(),
|
||
),
|
||
)
|
||
self.telemetry_alive(name, "before the lookup")
|
||
try:
|
||
lookup = self.telemetry_lookup(name, reqs, t0)
|
||
except BenchError as exc:
|
||
# погибшее посреди выборки хранилище — понятный отказ с его
|
||
# состоянием и журналом, а не трассировка запроса
|
||
self.telemetry_alive(name, "during the lookup", cause=exc)
|
||
raise
|
||
(self.results.dir / "raw" / f"{name}-peaks.json").write_text(
|
||
json.dumps(stand.telemetry_peaks(), indent=2) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
problems += telemetry_cost.layer_problems(
|
||
lookup.get("layers", {}),
|
||
variant,
|
||
)
|
||
if problems:
|
||
raise BenchError(
|
||
f"{name}: variant {run['variant']} not applied: "
|
||
+ "; ".join(problems),
|
||
)
|
||
du = stand.volume_sizes() if variant.get("telemetry") else {}
|
||
swap_after = swapouts()
|
||
src = telemetry_cost.Sources(
|
||
run=run,
|
||
variant=variant,
|
||
windows=windows,
|
||
t0=t0,
|
||
reqs=reqs,
|
||
dropped=dropped,
|
||
sampler_text=stand.sampler_logs()[0],
|
||
containers=stand.container_map(),
|
||
metas=metas,
|
||
lookup=lookup,
|
||
du=du,
|
||
slow_ms=p.telemetry_slow_ms,
|
||
not_drained=not drained,
|
||
storage_down=self.telemetry_storage_down(),
|
||
oom=self.telemetry_oom(),
|
||
swapouts=(
|
||
None
|
||
if swap_before is None or swap_after is None
|
||
else swap_after - swap_before
|
||
),
|
||
)
|
||
comps = telemetry_cost.components(src)
|
||
row = telemetry_cost.run_row(src, comps)
|
||
self.results.add_telemetry_run(
|
||
row,
|
||
comps,
|
||
telemetry_cost.class_rows(src),
|
||
)
|
||
self.results.write_meta()
|
||
self.runs_done += 1
|
||
self.say(
|
||
f"{name}: light p50/p95/p99 {row['light_p50']}/"
|
||
f"{row['light_p95']}/{row['light_p99']} ms, replica CPU "
|
||
f"{row['replica_cpu_cores']} cores ({row['replica_cpu_ms_per_req']}"
|
||
f" ms/req), telemetry CPU {row['telemetry_cpu_cores']}, collector "
|
||
f"peak {row['otel_collector_peak_mib']} MiB, tempo spans/s "
|
||
f"{row['tempo_spans_per_s']}, ready {row['ready']}/"
|
||
f"{row['accepted']}, bad failed {row['bad_failed']}/"
|
||
f"{row['bad_accepted']}, slow writes {row['slow_writes']}, ttr p95 "
|
||
f"{row['ttr_p95_s']} s, flags [{row['flags']}]",
|
||
)
|
||
for item in stand.work_dir.glob(f"{name}.*"):
|
||
item.unlink(missing_ok=True)
|
||
return row
|
||
|
||
def telemetry_passport(self, variant: dict[str, Any]) -> None:
|
||
"""Лимиты и паспорт стенда: первый прогон каждого режима (без
|
||
телеметрии и с ней) записывает, следующие сверяют."""
|
||
stand = self.stand
|
||
meta = self.results.meta
|
||
mode = "on" if variant.get("telemetry") else "off"
|
||
limits = stand.limits()
|
||
known = meta.setdefault("limits_by_mode", {}).get(mode)
|
||
if known is None:
|
||
meta["limits_by_mode"][mode] = limits
|
||
passport = stand.passport()
|
||
# `stand` и `limits` — без телеметрии, если такой прогон есть;
|
||
# паспорт с телеметрией — отдельно
|
||
if mode == "off" or "stand" not in meta:
|
||
meta["stand"] = passport
|
||
meta["limits"] = limits
|
||
if mode == "on":
|
||
meta["stand_telemetry"] = passport
|
||
self.results.write_meta()
|
||
elif known != limits:
|
||
raise BenchError(
|
||
f"limits with telemetry {mode} differ from the first run",
|
||
)
|
||
|
||
def telemetry_broken_row(self) -> str:
|
||
"""Одна строка вложения `ready` без объекта (решение 8): `\\copy` в
|
||
базу свежего стенда, чат чтения и его владелец; скачивание — 500."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
fixture = stand.work_json("fixture.json")
|
||
_, namespace, _ = stand.storage_names()
|
||
stand.copy_rows(
|
||
reconcile.rows_csv(
|
||
seed=p.seed,
|
||
start=0,
|
||
end=1,
|
||
size=1024,
|
||
namespace=namespace,
|
||
chat_id=fixture["readChatId"],
|
||
owner_id=fixture["readerId"],
|
||
created_at=(datetime.now(UTC) - timedelta(days=1)).isoformat(),
|
||
force=reconcile.FORCE_MISSING,
|
||
),
|
||
)
|
||
return reconcile.object_id(p.seed, 0)
|
||
|
||
def telemetry_wait_t0(
|
||
self,
|
||
load: subprocess.Popen[bytes],
|
||
log_name: str,
|
||
) -> float:
|
||
deadline = time.monotonic() + 120
|
||
while time.monotonic() < deadline:
|
||
t0, _ = telemetry_cost.parse_k6(self.stand.read_work(log_name))
|
||
if t0 is not None:
|
||
return t0
|
||
if load.poll() is not None:
|
||
break
|
||
time.sleep(0.05)
|
||
raise BenchError(
|
||
f"k6 telemetry.js has not started: exit {load.poll()}\n"
|
||
+ self.stand.read_work(f"{log_name}.out")[-3000:],
|
||
)
|
||
|
||
def telemetry_locks(
|
||
self,
|
||
fault_start: float,
|
||
out: list[dict[str, float]],
|
||
) -> None:
|
||
"""Блокировки окна отказов (решение 8): в своей сессии psql, моменты
|
||
`clock_timestamp()` до захвата, после захвата и после снятия — часы
|
||
ВМ. Отказ psql — запись без моментов, разбор пометит."""
|
||
p = self.profile
|
||
sql = (
|
||
"SELECT 'start', extract(epoch FROM clock_timestamp());\n"
|
||
"BEGIN;\n"
|
||
"LOCK TABLE messages IN SHARE MODE;\n"
|
||
"SELECT 'locked', extract(epoch FROM clock_timestamp());\n"
|
||
f"SELECT pg_sleep({p.telemetry_lock_s:g});\n"
|
||
"COMMIT;\n"
|
||
"SELECT 'released', extract(epoch FROM clock_timestamp());\n"
|
||
)
|
||
for offset in p.telemetry_locks:
|
||
time.sleep(max(0.0, fault_start + offset - time.monotonic()))
|
||
text = self.stand.psql(sql, check=False)
|
||
moments = {
|
||
key: float(value)
|
||
for key, _, value in (
|
||
line.partition("|") for line in text.splitlines()
|
||
)
|
||
if key in ("start", "locked", "released") and value
|
||
}
|
||
out.append(moments)
|
||
|
||
def telemetry_drain(self) -> bool:
|
||
"""Незавершённых заданий 0 — не дольше `telemetry_drain_s`."""
|
||
deadline = time.monotonic() + self.profile.telemetry_drain_s
|
||
while True:
|
||
left = int(
|
||
self.stand.psql(
|
||
"SELECT count(*) FROM attachment_jobs "
|
||
"WHERE state IN ('queued', 'processing');\n",
|
||
),
|
||
)
|
||
if not left:
|
||
return True
|
||
if time.monotonic() > deadline:
|
||
return False
|
||
time.sleep(0.5)
|
||
|
||
def telemetry_sweep(
|
||
self,
|
||
name: str,
|
||
reqs: list[telemetry_cost.Req],
|
||
) -> dict[str, Any]:
|
||
"""Метаданные принятых загрузок (`queue_sweep.js`, без байтов)."""
|
||
stand = self.stand
|
||
ids = [
|
||
r.attachment_id
|
||
for r in reqs
|
||
if r.cls in ("upload", "upload_bad") and r.status == 202
|
||
]
|
||
ids_file = f"{name}.ids.json"
|
||
(stand.work_dir / ids_file).write_text(
|
||
json.dumps(ids),
|
||
encoding="utf-8",
|
||
)
|
||
log = f"{name}.sweep.k6"
|
||
sweep = stand.k6_console(
|
||
"queue_sweep.js",
|
||
{"IDS": ids_file, "SAMPLE": 0, "NAME": f"{name}-sweep"},
|
||
log,
|
||
)
|
||
sweep.wait(timeout=620)
|
||
if sweep.returncode != 0:
|
||
raise BenchError(
|
||
f"{name}: k6 queue_sweep.js exit {sweep.returncode}:\n"
|
||
+ stand.read_work(f"{log}.out")[-3000:],
|
||
)
|
||
metas, _ = overload.parse_sweep(stand.read_work(log))
|
||
return metas
|
||
|
||
def telemetry_startup_records(self) -> dict[str, list[dict[str, Any]]]:
|
||
"""Записи журналов реплик, worker и relay (записи о старте)."""
|
||
records = {
|
||
f"api:{replica}": reconcile.parse_records(self.queue_log(replica))
|
||
for replica in self.stand.replicas()
|
||
}
|
||
records["worker"] = reconcile.parse_records(self.queue_log(WORKER))
|
||
records["relay"] = reconcile.parse_records(self.queue_log(RELAY))
|
||
return records
|
||
|
||
def telemetry_lookup(
|
||
self,
|
||
name: str,
|
||
reqs: list[telemetry_cost.Req],
|
||
t0: float,
|
||
) -> dict[str, Any]:
|
||
"""Выборка из хранилищ (решение 9): события прогона → итог; предел
|
||
числа результатов — отказ."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
events = telemetry_cost.events_for_lookup(
|
||
reqs,
|
||
t0,
|
||
tuple(p.telemetry_windows),
|
||
slow_ms=p.telemetry_slow_ms,
|
||
sample_me=p.telemetry_sample_me,
|
||
seed=p.order_seed,
|
||
collected_at=time.time(),
|
||
)
|
||
(stand.work_dir / f"{name}.events.json").write_text(
|
||
json.dumps(events),
|
||
encoding="utf-8",
|
||
)
|
||
proc = stand.lookup(
|
||
"collect",
|
||
f"/work/{name}.events.json",
|
||
f"/work/{name}.lookup.json",
|
||
)
|
||
if proc.returncode != 0:
|
||
output = (proc.stdout + proc.stderr).strip()
|
||
# первая строка отказа идёт в `meta.reason` и `progress.json`:
|
||
# последняя строка вывода (исключение), а не середина стека
|
||
last = output.splitlines()[-1] if output else ""
|
||
raise BenchError(
|
||
f"{name}: telemetry lookup failed (exit {proc.returncode}): "
|
||
f"{last}\n" + output[-2000:],
|
||
)
|
||
text = stand.read_work(f"{name}.lookup.json")
|
||
(self.results.dir / "raw" / f"{name}-lookup.json").write_text(
|
||
text,
|
||
encoding="utf-8",
|
||
)
|
||
return json.loads(text)
|
||
|
||
def telemetry_alive(
|
||
self,
|
||
name: str,
|
||
when: str,
|
||
cause: BenchError | None = None,
|
||
) -> None:
|
||
"""Сервисы телеметрии работают, без OOM и перезапусков; иначе —
|
||
хвост журнала погибшего в `raw/<прогон>-<сервис>.log` (до уборки
|
||
стенда, которая удаляет контейнер) и отказ с его состоянием (design,
|
||
«Уточнения apply», п. 16)."""
|
||
dead = []
|
||
for service, container in TELEMETRY_CONTAINERS.items():
|
||
state = self.stand.state(container)
|
||
if state["running"] and not state["oom"] and not state["restarts"]:
|
||
continue
|
||
tail = self.queue_log(container).splitlines()[-TELEMETRY_LOG_TAIL:]
|
||
(self.results.dir / "raw" / f"{name}-{service}.log").write_text(
|
||
"\n".join(tail) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
dead.append(
|
||
f"{container} (running {state['running']}, exit "
|
||
f"{state['exit_code']}, OOMKilled {state['oom']}, restarts "
|
||
f"{state['restarts']})",
|
||
)
|
||
if dead:
|
||
raise BenchError(
|
||
f"{name}: telemetry storage down {when}: "
|
||
+ ", ".join(dead)
|
||
+ f"; log tail in raw/{name}-<service>.log",
|
||
) from cause
|
||
|
||
def telemetry_storage_down(self) -> bool:
|
||
"""Хранилище, брокер или сервис телеметрии завершался за прогон."""
|
||
names = [DB, CACHE, S3, BROKER]
|
||
if self.stand.telemetry:
|
||
names += list(TELEMETRY_CONTAINERS.values())
|
||
for name in names:
|
||
state = self.stand.state(name)
|
||
if not state["running"] or state["oom"] or state["restarts"]:
|
||
return True
|
||
return False
|
||
|
||
def telemetry_oom(self) -> bool:
|
||
return any(
|
||
self.stand.state(name)["oom"]
|
||
for name in (*self.stand.replicas(stopped=True), WORKER, RELAY)
|
||
)
|
||
|
||
# ------------------------------------------------ профиль сверки
|
||
|
||
def run_reconcile(self) -> None:
|
||
"""Профиль сверки (design change `track2-reconcile-cost`, решения
|
||
1–3, 7–10): план в паспорт до первого прохода; этапы — свежий или
|
||
тот же стенд, наполнение мимо API, проходы без нагрузки, блок под
|
||
нагрузкой; затем отказ по пределу на свежем стенде."""
|
||
p = self.profile
|
||
meta = self.results.meta
|
||
plan = reconcile_plan(p)
|
||
self.runs_total = len(plan)
|
||
commit = sh(
|
||
"git",
|
||
"-C",
|
||
str(ROOT),
|
||
"rev-parse",
|
||
"--short",
|
||
f"{p.reconcile_before}^{{commit}}",
|
||
)
|
||
info: dict[str, Any] = meta.setdefault("reconcile", {})
|
||
info.update(
|
||
{
|
||
"before_commit": commit,
|
||
"plan": plan,
|
||
"seed": p.seed,
|
||
"period": reconcile.PERIOD,
|
||
"classes": list(reconcile.CLASSES),
|
||
"stages": [
|
||
{
|
||
"stage": stage,
|
||
"fresh": fresh,
|
||
"objects_total": n,
|
||
"object_bytes": size,
|
||
"cells": list(cells),
|
||
"model": asdict(reconcile.model(n)),
|
||
}
|
||
for stage, fresh, n, size, cells in p.reconcile_stages
|
||
],
|
||
"object_bytes": {
|
||
stage: size for stage, _, _, size, _ in p.reconcile_stages
|
||
},
|
||
"s4_clean": p.reconcile_s4_clean,
|
||
"s4_candidates": list(p.reconcile_s4_candidates),
|
||
"s4_max_changes": p.reconcile_s4_max_changes,
|
||
"s4_base_stage": p.reconcile_s4_base,
|
||
"h1_stages": list(p.reconcile_h1_stages),
|
||
"h2_stages": list(p.reconcile_h2_stages),
|
||
"h3_small_stage": p.reconcile_h3_stages[0],
|
||
"h3_large_stage": p.reconcile_h3_stages[1],
|
||
"cleaner_cpus": {
|
||
"stand": p.cleaner_cpus,
|
||
"fast": p.reconcile_fast_cpus,
|
||
},
|
||
"grace_seconds": p.reconcile_grace_s,
|
||
"default_rate": p.reconcile_default_rate,
|
||
"background": {
|
||
**dict(p.reconcile_background),
|
||
"downloads": p.reconcile_downloads,
|
||
"quiet_s": p.reconcile_quiet_s,
|
||
},
|
||
"metrics_port": p.reconcile_metrics_port,
|
||
"images": {"checkout": meta["app_image"]["id"]},
|
||
},
|
||
)
|
||
self.results.write_meta()
|
||
self.say(
|
||
f"reconcile: {len(plan)} passes planned "
|
||
f"({len(p.reconcile_stages)} stages and "
|
||
f"{len(p.reconcile_s4_candidates)} limit steps × {p.repeats})",
|
||
)
|
||
filled = 0
|
||
for stage, fresh, n, size, _cells in p.reconcile_stages:
|
||
if fresh or not self.prepared:
|
||
self.reconcile_stand(stage)
|
||
filled = 0
|
||
self.reconcile_fill(stage, filled, n, size, force=None)
|
||
filled = n
|
||
expected = reconcile.model(n)
|
||
entries = [e for e in plan if e["stage"] == stage]
|
||
for entry in (e for e in entries if not e["load"]):
|
||
result = self.reconcile_pass(entry, expected, size)
|
||
self.results.add_pass(result["row"])
|
||
loaded = [e for e in entries if e["load"]]
|
||
if loaded:
|
||
self.reconcile_load(stage, loaded, expected, size, n)
|
||
if p.reconcile_s4_candidates:
|
||
self.reconcile_limit(plan)
|
||
|
||
def reconcile_stand(self, stage: str) -> None:
|
||
"""Свежий стенд этапа: прежний (если был) удаляется с volumes;
|
||
реплики за LB, S3 с портом метрик, пользователь и чат, сэмплер
|
||
`scan` на весь этап."""
|
||
stand = self.stand
|
||
if self.prepared:
|
||
stand.stop_sampler()
|
||
stand.stop()
|
||
self.stage = f"{stage} start"
|
||
self.progress()
|
||
stand.start(lambda text: self.say(f"{stage}: {text}"))
|
||
meta = self.results.meta
|
||
if "limits" not in meta:
|
||
meta["limits"] = stand.limits()
|
||
meta["stand"] = stand.passport()
|
||
meta["reconcile"]["s3_command"] = stand.s3_command() + [
|
||
f"-metricsPort={self.profile.reconcile_metrics_port}",
|
||
]
|
||
self.results.write_meta()
|
||
stand.k6(
|
||
"prepare.js",
|
||
{
|
||
"MEMBERS": self.profile.members,
|
||
"BACKGROUND": "",
|
||
"WRITE_CHATS": 1,
|
||
},
|
||
)
|
||
self.fixture = stand.work_json("fixture.json")
|
||
stand.start_scan_sampler()
|
||
self.prepared = True
|
||
|
||
def reconcile_fill(
|
||
self,
|
||
stage: str,
|
||
start: int,
|
||
end: int,
|
||
size: int,
|
||
*,
|
||
force: str | None,
|
||
objects: bool = True,
|
||
) -> None:
|
||
"""Наполнение мимо API: строки `\\copy`, объекты загрузчиком; затем
|
||
ожидание grace + запас после последнего PUT — объекты без строки
|
||
старше grace (решения 2 и 3)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
self.stage = f"{stage} fill"
|
||
self.progress()
|
||
_, namespace, _ = stand.storage_names()
|
||
created = (datetime.now(UTC) - timedelta(days=1)).isoformat()
|
||
started = time.monotonic()
|
||
rows = reconcile.rows_csv(
|
||
seed=p.seed,
|
||
start=start,
|
||
end=end,
|
||
size=size,
|
||
namespace=namespace,
|
||
chat_id=self.fixture["readChatId"],
|
||
owner_id=self.fixture["readerId"],
|
||
created_at=created,
|
||
force=force,
|
||
)
|
||
stand.copy_rows(rows)
|
||
line = "no objects"
|
||
if objects:
|
||
line = stand.fill_objects(
|
||
seed=p.seed,
|
||
start=start,
|
||
end=end,
|
||
size=size,
|
||
force=force,
|
||
)
|
||
self.say(
|
||
f"{stage}: rows [{start}, {end}) in "
|
||
f"{time.monotonic() - started:.1f} s, {line}",
|
||
)
|
||
if objects:
|
||
time.sleep(p.reconcile_grace_s + p.reconcile_fresh_wait_s)
|
||
|
||
def reconcile_args(self, cell: dict[str, Any]) -> list[str]:
|
||
"""Аргументы разового прохода ячейки: «до» — только `--apply
|
||
--max-changes` (прежний CLI других не знает)."""
|
||
p = self.profile
|
||
args = ["reconcile"]
|
||
if cell.get("apply"):
|
||
args += [
|
||
"--apply",
|
||
"--max-changes",
|
||
str(p.reconcile_s4_max_changes),
|
||
]
|
||
if cell.get("before"):
|
||
return args
|
||
args += ["--strategy", cell["strategy"]]
|
||
if cell["deep"]:
|
||
args.append("--deep")
|
||
if cell["rate"] != "default":
|
||
args += ["--max-objects-per-second", f"{cell['rate']:g}"]
|
||
return args
|
||
|
||
def reconcile_pass(
|
||
self,
|
||
entry: dict[str, Any],
|
||
expected: reconcile.Model,
|
||
size: int,
|
||
*,
|
||
found: dict[str, int] | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Один проход: метрики шлюза и `pg_stat_database` до и после,
|
||
разовый уборщик, проверка найденного по модели (иначе — отказ
|
||
команды), CPU и память за окно по сэмплеру; строка `passes.csv`
|
||
(добавляет вызывающий: у проходов под нагрузкой — после окон)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
name = entry["pass"]
|
||
cell = reconcile.CELLS[entry["cell"]]
|
||
cpus = (
|
||
p.cleaner_cpus if cell["cpu"] == "stand" else p.reconcile_fast_cpus
|
||
)
|
||
self.stage = name
|
||
self.progress()
|
||
gw0 = stand.s3_metrics()
|
||
pg0 = reconcile.parse_pg_stat(stand.psql(reconcile.PG_STAT_SQL))
|
||
before = (
|
||
stand.work_dir / BEFORE_CLEANER_COMPOSE
|
||
if cell.get("before")
|
||
else None
|
||
)
|
||
result = stand.run_pass(
|
||
f"{RECONCILE_PASS}{entry['order']}",
|
||
self.reconcile_args(cell),
|
||
cpus=cpus,
|
||
before=before,
|
||
grace_s=p.reconcile_grace_s,
|
||
)
|
||
# последний отсчёт сэмплера после выхода контейнера
|
||
time.sleep(2 * p.reconcile_sample_interval_s)
|
||
gw1 = stand.s3_metrics()
|
||
pg1 = reconcile.parse_pg_stat(stand.psql(reconcile.PG_STAT_SQL))
|
||
records = reconcile.parse_records(result["log"])
|
||
kept = reconcile.kept_records(records)
|
||
(self.results.dir / "raw" / f"{name}-cleaner.json").write_text(
|
||
json.dumps(
|
||
{**kept, "exit_code": result["exit_code"]},
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
)
|
||
+ "\n",
|
||
encoding="utf-8",
|
||
)
|
||
if result["exit_code"] not in (0, 3):
|
||
tail = "\n".join(result["log"].splitlines()[-10:])
|
||
raise BenchError(
|
||
f"{name}: cleaner exit {result['exit_code']}\n{tail}",
|
||
)
|
||
problems = reconcile.found_problems(
|
||
entry,
|
||
expected.found if found is None else found,
|
||
kept,
|
||
)
|
||
finished = kept["finished"] or {}
|
||
for field, want in (
|
||
("listed_objects", expected.objects),
|
||
("checked_rows", expected.ready_rows),
|
||
):
|
||
if finished.get(field) != want:
|
||
problems.append(
|
||
f"{name}: {field} {finished.get(field)}, model {want}",
|
||
)
|
||
if problems:
|
||
raise BenchError("; ".join(problems))
|
||
t1 = reconcile.timestamp(finished["timestamp"])
|
||
t0 = t1 - finished["duration_ms"] / 1000
|
||
samples = scaling.parse_samples(stand.sampler_logs()[0])
|
||
points = resources.points_by_id(samples)
|
||
|
||
def cpu(*names: str) -> float | None:
|
||
values = [
|
||
reconcile.window_cpu(
|
||
points.get(stand.full_id(n)[:12], []),
|
||
t0,
|
||
t1,
|
||
)
|
||
for n in names
|
||
]
|
||
if not values or None in values:
|
||
return None
|
||
return round(sum(v for v in values if v is not None), 4)
|
||
|
||
cleaner_points = points.get(result["id"][:12], [])
|
||
limits = self.results.meta["limits"].setdefault("cleaner", {})
|
||
limits.setdefault(cpus, result["limits"])
|
||
row = reconcile.pass_row(
|
||
entry,
|
||
cell=cell,
|
||
expected=expected,
|
||
object_bytes=size,
|
||
kept=kept,
|
||
exit_code=result["exit_code"],
|
||
gateway=reconcile.metrics_delta(gw0, gw1),
|
||
pg={key: pg1[key] - pg0[key] for key in pg0},
|
||
cpu={
|
||
"cleaner": reconcile.window_cpu(cleaner_points, t0, t1),
|
||
"s3": cpu(S3),
|
||
"postgres": cpu(DB),
|
||
"api": cpu(*stand.replicas()),
|
||
"haproxy": cpu(LB),
|
||
},
|
||
memory=reconcile.memory_start_peak(cleaner_points, t0),
|
||
cleaner_cpus=cpus,
|
||
rate=None if cell.get("before") else cell["rate"],
|
||
image="before" if cell.get("before") else "checkout",
|
||
)
|
||
self.runs_done += 1
|
||
self.pass_seconds.append(result["finished_at"] - result["started_at"])
|
||
s3_total = sum(
|
||
int(row.get(f"s3_{op}") or 0) for op in reconcile.S3_OPS
|
||
)
|
||
self.say(
|
||
f"{name} ({entry['order']}/{self.runs_total}): "
|
||
f"{finished['duration_ms'] / 1000:.2f} s, exit "
|
||
f"{result['exit_code']}, S3 {s3_total}, sessions "
|
||
f"{row['db_sessions']}, cleaner {row['cpu_cleaner']} cores, s3 "
|
||
f"{row['cpu_s3']}, memory {row['mem_start_mib']} → "
|
||
f"{row['mem_peak_mib']} MiB{' ' + row['flags'] if row['flags'] else ''}",
|
||
)
|
||
self.progress(last={"pass": name, "flags": row["flags"]})
|
||
return {
|
||
"row": row,
|
||
"started_at": result["started_at"],
|
||
"finished_at": result["finished_at"],
|
||
"t0": t0,
|
||
"t1": t1,
|
||
}
|
||
|
||
def reconcile_load(
|
||
self,
|
||
stage: str,
|
||
entries: list[dict[str, Any]],
|
||
expected: reconcile.Model,
|
||
size: int,
|
||
n: int,
|
||
) -> None:
|
||
"""Блок H4 (решение 10): один k6 `background.js` на блок, тихие окна
|
||
до первого прохода и после каждого; окна прохода — по часам ВМ
|
||
(старт и выход контейнера, итог прохода). k6 останавливается
|
||
SIGINT, итог он пишет; число строк сверяется с итерациями."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
quiet = p.reconcile_quiet_s
|
||
ids = reconcile.downloads(p.seed, n, p.reconcile_downloads)
|
||
(stand.work_dir / "downloads.json").write_text(
|
||
json.dumps(ids),
|
||
encoding="utf-8",
|
||
)
|
||
rates = dict(p.reconcile_background)
|
||
log_name = "background.log.k6"
|
||
load = stand.k6_console(
|
||
"background.js",
|
||
{
|
||
"DOWNLOAD_RATE": rates["download"],
|
||
"WRITE_RATE": rates["write"],
|
||
"READ_RATE": rates["read"],
|
||
"DURATION": "6h",
|
||
"MAX_VUS": p.reconcile_bg_max_vus,
|
||
"NAME": "h4-k6",
|
||
},
|
||
log_name,
|
||
)
|
||
done: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||
try:
|
||
self.say(f"{stage}: background load started, quiet {quiet} s")
|
||
# разгон k6 — вне первого тихого окна
|
||
time.sleep(quiet + 2)
|
||
for entry in entries:
|
||
done.append(
|
||
(entry, self.reconcile_pass(entry, expected, size)),
|
||
)
|
||
time.sleep(quiet)
|
||
finally:
|
||
sh(
|
||
"docker",
|
||
"exec",
|
||
K6,
|
||
"pkill",
|
||
"-INT",
|
||
"-f",
|
||
"background.js",
|
||
check=False,
|
||
)
|
||
try:
|
||
load.wait(timeout=60)
|
||
except subprocess.TimeoutExpired:
|
||
load.kill()
|
||
summary = self.results.dir / "raw" / "h4-k6.json"
|
||
if not summary.exists():
|
||
tail = (stand.work_dir / f"{log_name}.out").read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
)[-3000:]
|
||
raise BenchError(
|
||
f"{stage}: k6 background.js exit {load.returncode}, no "
|
||
f"summary:\n{tail}",
|
||
)
|
||
requests = reconcile.parse_background(
|
||
(stand.work_dir / log_name).read_text(
|
||
encoding="utf-8",
|
||
errors="replace",
|
||
),
|
||
)
|
||
k6_data = json.loads(summary.read_text(encoding="utf-8"))
|
||
problems, interrupted = reconcile.line_problems(
|
||
requests,
|
||
k6_data,
|
||
max_vus=p.reconcile_bg_max_vus,
|
||
)
|
||
self.results.meta["reconcile"].setdefault("bg_interrupted", {})[
|
||
stage
|
||
] = interrupted
|
||
# ошибки фона по всему итогу k6, не только 5xx окон (verify S9)
|
||
errors = reconcile.background_errors(k6_data)
|
||
self.results.meta["reconcile"].setdefault("bg_errors", {})[stage] = (
|
||
errors
|
||
)
|
||
block_flags = reconcile.background_flags(errors)
|
||
self.results.write_meta()
|
||
if problems:
|
||
raise BenchError(f"{stage}: " + "; ".join(problems))
|
||
for entry, result in done:
|
||
bounds = {
|
||
"before": (result["started_at"] - quiet, result["started_at"]),
|
||
"pass": (result["t0"], result["t1"]),
|
||
"after": (
|
||
result["finished_at"],
|
||
result["finished_at"] + quiet,
|
||
),
|
||
}
|
||
list_ms = result["row"].get("phase_list_ms")
|
||
if list_ms is not None:
|
||
bounds["list"] = (result["t0"], result["t0"] + list_ms / 1000)
|
||
rows, ratios = reconcile.windows_of(requests, bounds)
|
||
self.results.add_windows(
|
||
[
|
||
{
|
||
**{
|
||
key: entry[key]
|
||
for key in ("pass", "stage", "cell", "repeat")
|
||
},
|
||
**row,
|
||
}
|
||
for row in rows
|
||
],
|
||
)
|
||
pass_row = result["row"]
|
||
flags = [pass_row["flags"], *block_flags]
|
||
if any(value["s5xx"] for value in ratios.values()):
|
||
flags.append("bg_5xx")
|
||
pass_row["flags"] = " ".join(filter(None, flags))
|
||
self.results.add_pass(pass_row)
|
||
self.say(
|
||
f"{entry['pass']}: r "
|
||
+ ", ".join(
|
||
f"{scenario} {value['r']}"
|
||
for scenario, value in ratios.items()
|
||
)
|
||
+ ", a "
|
||
+ ", ".join(
|
||
f"{scenario} {value['a']}"
|
||
for scenario, value in ratios.items()
|
||
),
|
||
)
|
||
|
||
def reconcile_limit(self, plan: list[dict[str, Any]]) -> None:
|
||
"""Отказ по пределу (решение 7): свежий стенд, согласованные
|
||
объекты и строки, затем строки `ready` без объектов шагами числа
|
||
кандидатов; проходы «до» (образ commit методики, override образа
|
||
уборщика) и «после» (checkout)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
self.reconcile_stand("S4")
|
||
self.stage = "before image"
|
||
self.progress()
|
||
image = stand.build_from(
|
||
self.results.meta["reconcile"]["before_commit"],
|
||
)
|
||
self.results.meta["reconcile"]["images"]["before"] = image
|
||
self.results.write_meta()
|
||
(stand.work_dir / BEFORE_CLEANER_COMPOSE).write_text(
|
||
json.dumps({"services": {"cleaner": {"image": FROM_IMAGE}}})
|
||
+ "\n",
|
||
encoding="utf-8",
|
||
)
|
||
self.say(f"S4: image of {p.reconcile_before} built ({image[:19]}…)")
|
||
clean = p.reconcile_s4_clean
|
||
size = reconcile.S4_OBJECT_BYTES
|
||
# согласованные объекты: свежесть не важна, у каждого есть строка
|
||
stand.copy_rows(
|
||
reconcile.rows_csv(
|
||
seed=p.seed,
|
||
start=0,
|
||
end=clean,
|
||
size=size,
|
||
namespace=stand.storage_names()[1],
|
||
chat_id=self.fixture["readChatId"],
|
||
owner_id=self.fixture["readerId"],
|
||
created_at=(datetime.now(UTC) - timedelta(days=1)).isoformat(),
|
||
force=reconcile.FORCE_OK,
|
||
),
|
||
)
|
||
line = stand.fill_objects(
|
||
seed=p.seed,
|
||
start=0,
|
||
end=clean,
|
||
size=size,
|
||
force=reconcile.FORCE_OK,
|
||
)
|
||
self.say(f"S4: {clean} consistent rows and objects, {line}")
|
||
added = 0
|
||
for count in p.reconcile_s4_candidates:
|
||
self.reconcile_fill(
|
||
f"S4-{count}",
|
||
clean + added,
|
||
clean + count,
|
||
size,
|
||
force=reconcile.FORCE_MISSING,
|
||
objects=False,
|
||
)
|
||
added = count
|
||
expected = reconcile.model(0, clean=clean, missing=count)
|
||
stage = f"S4-{count}"
|
||
for entry in (e for e in plan if e["stage"] == stage):
|
||
# и «до», и «после» отказывают по пределу рано: commit «до» —
|
||
# перенос лабы 4, ранний отказ в нём уже есть (прежнего
|
||
# `7f4e626` в истории лабы 5 нет)
|
||
found = {
|
||
"missing_object": min(
|
||
count,
|
||
p.reconcile_s4_max_changes + 1,
|
||
),
|
||
}
|
||
result = self.reconcile_pass(
|
||
entry,
|
||
expected,
|
||
size,
|
||
found=found,
|
||
)
|
||
self.results.add_pass(result["row"])
|
||
|
||
def run_series(self) -> None:
|
||
"""Серия трека 1 (решения 4–5 design change `scaling-bench`): план
|
||
ячеек кругами — в паспорт до первой ячейки; при продолжении
|
||
готовые ячейки (строка в `searches.csv`) пропускаются, прерванная
|
||
снимается заново с попыткой +1."""
|
||
p = self.profile
|
||
meta = self.results.meta
|
||
if "plan" not in meta:
|
||
meta["plan"] = scaling.plan_cells(
|
||
[tuple(c) for c in p.configs],
|
||
p.repeats,
|
||
p.order_seed,
|
||
)
|
||
self.results.write_meta()
|
||
plan = meta["plan"]
|
||
directory = self.results.dir
|
||
done = {
|
||
s["cell"] for s in scaling.read_csv(directory / "searches.csv")
|
||
}
|
||
runs = scaling.read_csv(directory / "runs.csv")
|
||
attempts: dict[str, int] = {}
|
||
for row in runs + scaling.read_csv(directory / "prefill.csv"):
|
||
if row.get("cell") and row.get("attempt"):
|
||
attempts[row["cell"]] = max(
|
||
attempts.get(row["cell"], 0),
|
||
int(row["attempt"]),
|
||
)
|
||
self.step_no = max(
|
||
(int(r["run"]) for r in runs if r["run"]),
|
||
default=0,
|
||
)
|
||
self.runs_done = len(done)
|
||
if done:
|
||
self.say(
|
||
f"resume: {len(done)}/{len(plan)} cells done, "
|
||
f"{len(plan) - len(done)} left",
|
||
)
|
||
for index, cell in enumerate(plan, start=1):
|
||
if cell["cell"] in done:
|
||
continue
|
||
self.run_cell(index, cell, attempts.get(cell["cell"], 0) + 1)
|
||
self.runs_done += 1
|
||
|
||
def run_cell(
|
||
self,
|
||
index: int,
|
||
cell: dict[str, Any],
|
||
attempt: int,
|
||
) -> None:
|
||
"""Ячейка: свежий стенд конфигурации, данные, сэмплер,
|
||
фиксированные уровни и поиск R*, сверка записей и читаемого чата,
|
||
строка `searches.csv`, уборка стенда (решения 2–4)."""
|
||
p = self.profile
|
||
self.cell = cell
|
||
self.cell_index = index
|
||
self.attempt = attempt
|
||
self.cell_started = time.monotonic()
|
||
profile = cell_profile(p, cell)
|
||
self.stand = Stand(profile, self.results.dir)
|
||
stand = self.stand
|
||
host_start = host_state()
|
||
started_at = datetime.now(UTC).isoformat(timespec="seconds")
|
||
others = sorted(
|
||
name
|
||
for name in sh(
|
||
"docker",
|
||
"ps",
|
||
"--format",
|
||
"{{.Names}}",
|
||
check=False,
|
||
).split()
|
||
if not name.startswith(PROJECT)
|
||
)
|
||
tag = f"cell {index}/{self.runs_total} {cell['cell']}" + (
|
||
f" (attempt {attempt})" if attempt > 1 else ""
|
||
)
|
||
fixture = self.prepare_load(tag)
|
||
size = p.sizes[0]
|
||
stand.start_sampler()
|
||
self.say(f"{tag}: sampler started")
|
||
# лимиты сэмплера — тоже у запущенного контейнера, в лимитах
|
||
# конфигурации (спек «Лимиты конфигурации серии», verify S11)
|
||
sampler = stand.container_limits(SAMPLER)
|
||
known = self.results.meta["limits"][cell["config"]]
|
||
if known.setdefault("sampler", sampler) != sampler:
|
||
raise BenchError(
|
||
f"{cell['config']}: sampler limits differ from the earlier "
|
||
f"repeat: {sampler}",
|
||
)
|
||
replicas = stand.replicas()
|
||
instances = [stand.full_id(name)[:12] for name in replicas]
|
||
addresses = {
|
||
stand.address(name): ident
|
||
for name, ident in zip(replicas, instances, strict=True)
|
||
}
|
||
limits = scaling.Limits(
|
||
replicas=profile.replicas,
|
||
db_cpus=float(profile.db_cpus),
|
||
lb_cpus=float(profile.lb_cpus),
|
||
generator_cpus=float(profile.generator_cpus or 0),
|
||
generator_cpu_limit=profile.generator_cpu_limit,
|
||
vm_cpus=float(self.results.meta["docker"]["vm_cpus"] or 0),
|
||
slo_p95_ms=p.slo_p95_ms,
|
||
slo_error_ratio=p.slo_error_ratio,
|
||
)
|
||
search = scaling.Search(
|
||
replicas=profile.replicas,
|
||
fixed=p.fixed_rps,
|
||
base=p.ladder_base,
|
||
factor=p.step_factor,
|
||
per_replica=p.start_rps_per_replica,
|
||
bisect_steps=p.bisect_steps,
|
||
max_steps=p.max_steps,
|
||
)
|
||
steps: list[dict[str, Any]] = []
|
||
previous: int | None = None
|
||
number = 0
|
||
while (step := search.next_step()) is not None:
|
||
level, kind = step
|
||
number += 1
|
||
row = self.run_step(
|
||
cell,
|
||
number,
|
||
level,
|
||
kind,
|
||
previous=previous,
|
||
instances=instances,
|
||
addresses=addresses,
|
||
limits=limits,
|
||
)
|
||
steps.append(row)
|
||
search.record(level, kind, row["_outcome"])
|
||
if row["_outcome"] == "pass":
|
||
previous = level
|
||
continue
|
||
# после нарушения — пауза без нагрузки и готовность реплик;
|
||
# следующая ступень — с нуля и долгим прогревом
|
||
previous = None
|
||
if not search.done:
|
||
time.sleep(p.cooldown_s)
|
||
stand.wait_ready(timeout=30)
|
||
stand.stop_sampler()
|
||
# сверка ячейки: читаемый чат не изменился, сообщений в чатах
|
||
# записи — между подтверждёнными и отправленными
|
||
ids = ", ".join(f"'{chat}'" for chat in fixture["writeChatIds"])
|
||
read_count, write_count = (
|
||
int(value)
|
||
for value in stand.psql(
|
||
"SELECT count(*) FILTER (WHERE chat_id = "
|
||
f"'{fixture['readChatId']}'), count(*) FILTER (WHERE "
|
||
f"chat_id IN ({ids})) FROM messages;\n",
|
||
).split("|")
|
||
)
|
||
writes_ok = sum(r["_writes_ok"] for r in steps)
|
||
writes_sent = sum(r["_writes_sent"] for r in steps)
|
||
slack = sum(r["_slack"] for r in steps)
|
||
if read_count != size:
|
||
raise BenchError(
|
||
f"cell {cell['cell']}: read chat has {read_count} messages, "
|
||
f"expected {size}",
|
||
)
|
||
if not writes_ok <= write_count <= writes_sent + slack:
|
||
raise BenchError(
|
||
f"cell {cell['cell']}: write chats have {write_count} "
|
||
f"messages, {writes_ok} confirmed and {writes_sent} sent"
|
||
+ (f" (+{slack} in flight at abort)" if slack else ""),
|
||
)
|
||
host_end = host_state()
|
||
start_out, end_out = host_start["swapouts"], host_end["swapouts"]
|
||
swapped = (
|
||
end_out - start_out
|
||
if start_out is not None and end_out is not None
|
||
else 0
|
||
)
|
||
flags = list(search.flags)
|
||
if swapped or "Battery Power" in (
|
||
host_start["power"],
|
||
host_end["power"],
|
||
):
|
||
flags.append("host_swap")
|
||
fixed = {r["level"]: r for r in steps if r["step_kind"] == "fixed"}
|
||
fail = search.first_fail
|
||
fail_row = next(
|
||
(
|
||
r
|
||
for r in reversed(steps)
|
||
if r["level"] == fail and r["_outcome"] == "fail"
|
||
),
|
||
None,
|
||
)
|
||
row: dict[str, Any] = {
|
||
**{
|
||
key: cell[key]
|
||
for key in (
|
||
"cell",
|
||
"config",
|
||
"replicas",
|
||
"replica_cpus",
|
||
"replica_memory",
|
||
"db_max_connections",
|
||
"repeat",
|
||
"round",
|
||
"order",
|
||
)
|
||
},
|
||
"series": p.series,
|
||
"attempt": attempt,
|
||
"started_at": started_at,
|
||
"finished_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"status": search.status,
|
||
"r_star_rps": search.r_star,
|
||
"r_star_bound": (
|
||
"lower" if search.status == "generator_bound" else "exact"
|
||
),
|
||
"first_fail_rps": fail,
|
||
"first_fail_reason": fail_row["fail_reason"] if fail_row else None,
|
||
"steps": len(steps),
|
||
"resolution": rounded(search.resolution, 4),
|
||
"app_cpu_second_max": max(
|
||
(
|
||
r["app_cpu_second_max"]
|
||
for r in steps
|
||
if r["app_cpu_second_max"] is not None
|
||
),
|
||
default=None,
|
||
),
|
||
"writes_ok": writes_ok,
|
||
"writes_sent": writes_sent,
|
||
"write_chat_messages": write_count,
|
||
"read_chat_messages": read_count,
|
||
"host_power": host_end["power"],
|
||
"host_memory_free_percent": host_end["memory_free_percent"],
|
||
"host_swapouts": swapped or 0,
|
||
"other_containers": " ".join(others),
|
||
"flags": " ".join(flags),
|
||
}
|
||
for number, level in enumerate(p.fixed_rps[:2], start=1):
|
||
item = fixed.get(level)
|
||
if item is None:
|
||
continue
|
||
row[f"fixed_rps_{number}"] = level
|
||
for q in ("p50", "p95", "p99"):
|
||
row[f"{q}_fixed_{number}_ms"] = item[f"{q}_ms"]
|
||
row[f"pass_fixed_{number}"] = item["slo_pass"]
|
||
second = fixed.get(p.fixed_rps[1]) if len(p.fixed_rps) > 1 else None
|
||
if second is not None:
|
||
row["app_us_per_request_fixed_2"] = second["app_us_per_request"]
|
||
row["app_throttled_ratio_fixed_2"] = second["app_throttled_ratio"]
|
||
if not stand.stop():
|
||
raise BenchError(f"stand of cell {cell['cell']} was not removed")
|
||
self.results.add_search(row)
|
||
self.results.meta.setdefault("cells", {})[cell["cell"]] = {
|
||
"attempt": attempt,
|
||
"status": search.status,
|
||
"r_star_rps": search.r_star,
|
||
"host": {"start": host_start, "end": host_end},
|
||
"other_containers": others,
|
||
}
|
||
self.results.write_meta()
|
||
self.cell_seconds.append(time.monotonic() - self.cell_started)
|
||
self.say(
|
||
f"{tag}: R* {search.r_star} rps ({search.status}), first fail "
|
||
f"{fail} {row['first_fail_reason'] or ''}, resolution "
|
||
f"{row['resolution']}, {len(steps)} steps; writes {writes_ok} ok "
|
||
f"/ {write_count} stored / {writes_sent} sent"
|
||
+ (f" [{row['flags']}]" if row["flags"] else ""),
|
||
)
|
||
self.cell = None
|
||
|
||
def run_step(
|
||
self,
|
||
cell: dict[str, Any],
|
||
number: int,
|
||
level: int,
|
||
kind: str,
|
||
*,
|
||
previous: int | None,
|
||
instances: list[str],
|
||
addresses: dict[str, str],
|
||
limits: scaling.Limits,
|
||
) -> dict[str, Any]:
|
||
"""Ступень: один запуск k6 `mixed.js` под облегчённым сторожем,
|
||
разбор итога, окна сэмплера и журнала PostgreSQL, строки
|
||
`runs.csv` и `samples.csv` (решения 3, 7, 8)."""
|
||
p = self.profile
|
||
stand = self.stand
|
||
warmup = p.step_warmup_s if previous else p.first_warmup_s
|
||
self.step_no += 1
|
||
name = f"{cell['cell']}-a{self.attempt}-s{number:02d}"
|
||
self.stage = f"cell {cell['cell']} step {number} {kind} {level}"
|
||
self.progress()
|
||
before = stand.snapshot()
|
||
store_start = stand.message_count()
|
||
swap_before = swapouts()
|
||
env = {
|
||
"LEVEL": level,
|
||
"START": previous or 0,
|
||
"MIX": ",".join(f"{n}:{s:g}" for n, s in p.mix),
|
||
"WARMUP_S": warmup,
|
||
"MEASURE_S": p.step_measure_s,
|
||
"TIMEOUT": p.request_timeout_s,
|
||
"PRE_VUS_PER_RPS": p.prevus_per_rps,
|
||
"MAX_VUS_PER_RPS": p.vus_per_rps,
|
||
"SEED": p.seed + 1000 * cell["repeat"] + number,
|
||
"STEP": self.step_no,
|
||
"INSTANCES": ",".join(instances),
|
||
"ABORT_ERRORS": p.abort_error_ratio,
|
||
"ABORT_P95_MS": p.abort_p95_ms,
|
||
"NAME": name,
|
||
}
|
||
with Watcher(stand.replicas(), rss=False) as watcher:
|
||
proc = stand.k6("mixed.js", env, check=False)
|
||
died = watcher.app_died or bool(watcher.stand_died)
|
||
# 99 — пороги k6 с abortOnFail: перегрузка, ступень нарушила SLO
|
||
aborted = proc.returncode == 99
|
||
raw = self.results.dir / "raw" / f"{name}.json"
|
||
if (proc.returncode not in (0, 99) and not died) or not raw.exists():
|
||
raise BenchError(k6_failure("mixed.js", proc))
|
||
after = stand.snapshot()
|
||
store_end = stand.message_count()
|
||
swap_after = swapouts()
|
||
data = json.loads(raw.read_text(encoding="utf-8"))
|
||
k6row = scaling.parse_k6(
|
||
data,
|
||
level=level,
|
||
mix=list(p.mix),
|
||
measure_s=p.step_measure_s,
|
||
instances=instances,
|
||
)
|
||
t0 = k6row["t0_ms"]
|
||
w0 = int((t0 + warmup * 1000) * 1e6)
|
||
w1 = int(
|
||
min(
|
||
w0 + p.step_measure_s * 1e9,
|
||
(t0 + k6row["run_ms"]) * 1e6,
|
||
),
|
||
)
|
||
agg, seconds = scaling.analyze_samples(
|
||
scaling.parse_samples(*stand.sampler_logs()),
|
||
w0,
|
||
w1,
|
||
addresses=addresses,
|
||
)
|
||
pg_log = self.window_log(DB, w0, w1)
|
||
too_many = scaling.count_too_many(pg_log)
|
||
rps = k6row["rps"]
|
||
app = stand.app_state()
|
||
down = stand.down_storages()
|
||
swapped = (
|
||
swap_after - swap_before
|
||
if swap_after is not None and swap_before is not None
|
||
else None
|
||
)
|
||
row: dict[str, Any] = {
|
||
**k6row,
|
||
**agg,
|
||
"run": self.step_no,
|
||
"scenario": "mixed",
|
||
"size": p.sizes[0],
|
||
"repeat": cell["repeat"],
|
||
"rss_start_mib": mib(before["rss_kib"]),
|
||
"rss_end_mib": mib(after["rss_kib"]),
|
||
"rss_max_mib": mib(
|
||
max(
|
||
(v for v in (before["rss_kib"], after["rss_kib"]) if v),
|
||
default=None,
|
||
),
|
||
),
|
||
"mem_cgroup_end_mib": mib_bytes(after["mem_bytes"]),
|
||
"store_messages_start": store_start,
|
||
"store_messages_end": store_end,
|
||
"series": p.series,
|
||
"cell": cell["cell"],
|
||
"attempt": self.attempt,
|
||
"config": cell["config"],
|
||
"replicas": cell["replicas"],
|
||
"replica_cpus": cell["replica_cpus"],
|
||
"replica_memory": cell["replica_memory"],
|
||
"db_max_connections": cell["db_max_connections"],
|
||
"step": number,
|
||
"step_kind": kind,
|
||
"level": level,
|
||
"db_too_many_clients": too_many,
|
||
"app_us_per_request": scaling.per_request(
|
||
agg.get("app_cpu_cores"),
|
||
rps,
|
||
),
|
||
"db_us_per_request": scaling.per_request(
|
||
agg.get("db_cpu_cores"),
|
||
rps,
|
||
),
|
||
"host_swapouts": swapped,
|
||
}
|
||
outcome, reason = scaling.verdict(row, limits, aborted=aborted)
|
||
found = scaling.signs(row, limits)
|
||
flags = []
|
||
if app["oom"]:
|
||
flags.append("oom")
|
||
if not app["running"] or app["restarts"] or watcher.app_died:
|
||
flags.append("app_down")
|
||
if down or watcher.stand_died:
|
||
flags.append("stand_down")
|
||
if row["requests_failed"] or row["check_failed"]:
|
||
flags.append("errors")
|
||
if outcome == "fail":
|
||
flags.append("slo")
|
||
if aborted:
|
||
flags.append("abort")
|
||
if outcome == "generator":
|
||
flags.append("generator")
|
||
if swapped:
|
||
flags.append("host_swap")
|
||
row.update(
|
||
{
|
||
"slo_pass": None
|
||
if outcome == "generator"
|
||
else outcome == "pass",
|
||
"fail_reason": reason or None,
|
||
"signs": " ".join(found),
|
||
"flags": " ".join(flags),
|
||
},
|
||
)
|
||
self.results.add_step(row)
|
||
self.results.add_samples(
|
||
[
|
||
{
|
||
**second,
|
||
"cell": cell["cell"],
|
||
"attempt": self.attempt,
|
||
"step": number,
|
||
}
|
||
for second in seconds
|
||
],
|
||
)
|
||
codes = row["error_codes"]
|
||
if too_many or any(c.startswith("5") for c in codes.split()):
|
||
self.save_errors(name, w0, w1, pg_log, codes)
|
||
self.say(
|
||
f"cell {self.cell_index}/{self.runs_total} {cell['cell']} step "
|
||
f"{number:>2} {kind:<6} {level:>5} rps: "
|
||
f"{'PASS' if outcome == 'pass' else outcome.upper()}"
|
||
+ (f" ({reason})" if reason else "")
|
||
+ f" ok {fmt(rps)}/s p95 {fmt(row['p95_ms'], 1)} p99 "
|
||
f"{fmt(row['p99_ms'], 1)} ms err {fmt((row['slo_error_ratio'] or 0) * 100, 2)}%"
|
||
f" app {fmt(row.get('app_cpu_cores'), 2)} db "
|
||
f"{fmt(row.get('db_cpu_cores'), 2)} k6 "
|
||
f"{fmt(row.get('generator_cpu_cores'), 2)} cores"
|
||
+ (f" [{' '.join(found)}]" if found else "")
|
||
+ (f" codes {codes}" if codes else "")
|
||
+ f" ~{fmt_left(self.eta())} left",
|
||
)
|
||
self.progress(last={k: row.get(k) for k in RUN_FIELDS})
|
||
if "app_down" in flags or "oom" in flags or "stand_down" in flags:
|
||
raise BenchError(
|
||
f"stand down after step {name} ({' '.join(flags)}): series "
|
||
"stopped; continue with RESUME=1 after analysis",
|
||
)
|
||
row["_outcome"] = outcome
|
||
row["_writes_sent"] = k6row["write_sent_total"]
|
||
row["_writes_ok"] = (
|
||
k6row["write_sent_total"] - k6row["write_failed_total"]
|
||
)
|
||
# прерванные досрочно запросы k6 не учитывает, а сервер мог их
|
||
# выполнить: допуск сверки — выделенные VU ступени
|
||
row["_slack"] = k6row["vus_max"] if aborted else 0
|
||
return row
|
||
|
||
def window_log(self, container: str, w0: int, w1: int) -> str:
|
||
"""Журнал контейнера за окно замера (часы ВМ, нс)."""
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"logs",
|
||
"-t",
|
||
"--since",
|
||
f"{w0 / 1e9:.3f}",
|
||
"--until",
|
||
f"{w1 / 1e9:.3f}",
|
||
container,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return proc.stdout + proc.stderr
|
||
|
||
def save_errors(
|
||
self,
|
||
name: str,
|
||
w0: int,
|
||
w1: int,
|
||
pg_log: str,
|
||
codes: str,
|
||
) -> None:
|
||
"""Доказательства 5xx и отказов «too many clients» ступени (H3):
|
||
строки PostgreSQL, API и LB окна, без секретов — строки со словом
|
||
`password` отбрасываются. Длинная строка (traceback) сокращается
|
||
до начала и конца: класс исключения — в конце (verify S4)."""
|
||
|
||
def pick(text: str, pattern: re.Pattern[str], limit: int) -> list[str]:
|
||
lines = [
|
||
shorten(line)
|
||
for line in text.splitlines()
|
||
if pattern.search(line) and "password" not in line.lower()
|
||
]
|
||
return lines[:limit]
|
||
|
||
out = [
|
||
f"# {name}: коды ошибок окна k6 — {codes or 'нет'}",
|
||
"",
|
||
"## PostgreSQL",
|
||
*pick(pg_log, ERRORS_PG_RE, 10),
|
||
"",
|
||
"## API",
|
||
]
|
||
for replica in self.stand.replicas():
|
||
out += pick(self.window_log(replica, w0, w1), ERRORS_API_RE, 10)
|
||
out += [
|
||
"",
|
||
"## HAProxy",
|
||
*pick(self.window_log(LB, w0, w1), ERRORS_LB_RE, 10),
|
||
]
|
||
path = self.results.dir / "raw" / f"{name}-errors.log"
|
||
path.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||
|
||
|
||
# ------------------------------------------------------------------ CLI
|
||
|
||
|
||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--profile", choices=sorted(PROFILES), required=True)
|
||
parser.add_argument("--label", required=True)
|
||
# отладка стенда: переопределения пишутся в паспорт как overrides
|
||
parser.add_argument("--sizes", help="через запятую, например 120000")
|
||
parser.add_argument("--concurrency", help="через запятую")
|
||
parser.add_argument("--repeats", type=int)
|
||
parser.add_argument("--memory", help="лимит памяти, например 64m")
|
||
parser.add_argument(
|
||
"--modes",
|
||
help="режимы профиля отказа через запятую: stop,kill,pause",
|
||
)
|
||
parser.add_argument(
|
||
"--from",
|
||
dest="rollout_from",
|
||
help="commit прежнего образа профиля замены",
|
||
)
|
||
parser.add_argument(
|
||
"--resume",
|
||
action="store_true",
|
||
help="серии трека 1: продолжить прерванную метку",
|
||
)
|
||
parser.add_argument(
|
||
"--fixed",
|
||
help="серии трека 1, отладка: фиксированные уровни через запятую",
|
||
)
|
||
parser.add_argument(
|
||
"--variants",
|
||
help="цена телеметрии, отладка: варианты через запятую",
|
||
)
|
||
parser.add_argument(
|
||
"--telemetry",
|
||
choices=("on", "off"),
|
||
default="off",
|
||
help="выключатель телеметрии профилей загрузок и очереди",
|
||
)
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def acquire_lock() -> int | None:
|
||
"""Блокировка стенда: один прогон любого профиля (решение 5 design
|
||
change `scaling-bench`). ОС снимает её при выходе, и при `kill -9`
|
||
тоже. Занята — None."""
|
||
fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR, 0o644)
|
||
try:
|
||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except BlockingIOError:
|
||
os.close(fd)
|
||
return None
|
||
return fd
|
||
|
||
|
||
def resume_problems(
|
||
label_dir: Path,
|
||
profile: Profile,
|
||
git: dict[str, Any],
|
||
) -> list[str]:
|
||
"""Условия продолжения серии (решение 5): метка есть и не завершена,
|
||
тот же профиль, тот же commit, дерево чистое без своей метки."""
|
||
path = label_dir / "meta.json"
|
||
if not path.is_file():
|
||
return [f"{path.relative_to(ROOT)} not found: nothing to resume"]
|
||
meta = json.loads(path.read_text(encoding="utf-8"))
|
||
found = []
|
||
if meta.get("status") == "complete":
|
||
found.append("label is complete: nothing to resume")
|
||
name = meta.get("profile", {}).get("name")
|
||
if name != profile.name:
|
||
found.append(f"label profile {name}, requested {profile.name}")
|
||
commit = meta.get("git", {}).get("commit")
|
||
if commit != git["commit"]:
|
||
found.append(f"label commit {commit}, HEAD {git['commit']}")
|
||
if git["dirty"]:
|
||
found.append("working tree has uncommitted changes")
|
||
if meta.get("overrides"):
|
||
found.append(f"label has debug overrides {meta['overrides']}")
|
||
# выключатель телеметрии — методика, а не отладка: тот же режим (design
|
||
# change track1-telemetry-cost, решение 13)
|
||
mode = meta.get("profile", {}).get("telemetry", "off")
|
||
if mode != profile.telemetry:
|
||
found.append(
|
||
f"label telemetry mode {mode}, requested {profile.telemetry}",
|
||
)
|
||
return found
|
||
|
||
|
||
def rollout_commit_problem(
|
||
commit: str,
|
||
field: str = "rollout_from",
|
||
kind: str = "rollout",
|
||
) -> str | None:
|
||
"""Commit прежнего образа профиля замены (или варианта «до» профиля
|
||
загрузок) есть в истории; иначе — сообщение с commit и подменой
|
||
`--from` (design change `attachments`, решение 14)."""
|
||
found = subprocess.run(
|
||
[
|
||
"git",
|
||
"-C",
|
||
str(ROOT),
|
||
"rev-parse",
|
||
"--verify",
|
||
"--quiet",
|
||
f"{commit}^{{commit}}",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if found.returncode == 0:
|
||
return None
|
||
return (
|
||
f"{kind} commit {commit!r} is not in this repository history: "
|
||
f"update Profile.{field} (.claude/rules/bench.md) or pass "
|
||
"--from <commit> for debugging"
|
||
)
|
||
|
||
|
||
def apply_overrides(
|
||
profile: Profile,
|
||
args: argparse.Namespace,
|
||
) -> tuple[Profile, dict[str, Any]]:
|
||
overrides: dict[str, Any] = {}
|
||
if args.sizes:
|
||
overrides["sizes"] = tuple(int(x) for x in args.sizes.split(","))
|
||
if args.concurrency:
|
||
overrides["concurrency"] = tuple(
|
||
int(x) for x in args.concurrency.split(",")
|
||
)
|
||
if args.repeats:
|
||
overrides["repeats"] = args.repeats
|
||
if args.memory:
|
||
overrides["memory"] = args.memory
|
||
if args.modes:
|
||
modes = tuple(args.modes.split(","))
|
||
if not profile.is_failover or set(modes) - set(failover.BOUNDS):
|
||
raise SystemExit(f"--modes {args.modes}: failover profiles only")
|
||
overrides["failover_modes"] = modes
|
||
if args.rollout_from:
|
||
if profile.is_reconcile:
|
||
overrides["reconcile_before"] = args.rollout_from
|
||
elif profile.is_uploads:
|
||
overrides["uploads_before"] = args.rollout_from
|
||
elif not profile.is_rollout:
|
||
raise SystemExit(
|
||
f"--from {args.rollout_from}: rollout and uploads profiles "
|
||
"only",
|
||
)
|
||
else:
|
||
overrides["rollout_from"] = args.rollout_from
|
||
if args.concurrency and profile.is_uploads:
|
||
# профиль загрузок: число клиентов, а не уровни матрицы
|
||
overrides["uploads_clients"] = overrides.pop("concurrency")
|
||
if args.fixed:
|
||
if not profile.is_series:
|
||
raise SystemExit(f"--fixed {args.fixed}: track 1 series only")
|
||
overrides["fixed_rps"] = tuple(int(x) for x in args.fixed.split(","))
|
||
if args.variants:
|
||
names = tuple(args.variants.split(","))
|
||
if not profile.is_telemetry_cost or set(names) - set(
|
||
telemetry_cost.ORDER,
|
||
):
|
||
raise SystemExit(
|
||
f"--variants {args.variants}: telemetry profiles only, "
|
||
f"variants {', '.join(telemetry_cost.ORDER)}",
|
||
)
|
||
overrides["telemetry_variants"] = names
|
||
return replace(profile, **overrides), overrides
|
||
|
||
|
||
def telemetry_preflight(profile: Profile) -> list[str]:
|
||
"""Проверка до стенда (design change track1-telemetry-cost, решения 1, 6,
|
||
7): лимиты сервисов телеметрии в `Profile` — как в основном
|
||
`docker-compose.yml`; у профиля цены телеметрии ещё слои Collector
|
||
(`validate` закреплённого образа и сравнение `print-config` со слоем и без
|
||
него) и копия Prometheus (построчно вне блоков и `promtool check
|
||
config`). Пусто — проверки прошли."""
|
||
proc = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
PROJECT,
|
||
"--env-file",
|
||
str(ENV_FILE),
|
||
"-f",
|
||
str(COMPOSE_FILES[0]),
|
||
"config",
|
||
"--format",
|
||
"json",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
env=compose_env(profile),
|
||
check=False,
|
||
)
|
||
if proc.returncode != 0:
|
||
return [f"docker compose config: {proc.stderr.strip()[-500:]}"]
|
||
services = json.loads(proc.stdout)["services"]
|
||
problems = []
|
||
for service, (cpu_field, mem_field) in TELEMETRY_LIMIT_FIELDS.items():
|
||
limits = services[service]["deploy"]["resources"]["limits"]
|
||
want_cpu = float(getattr(profile, cpu_field))
|
||
want_mem = size_bytes(getattr(profile, mem_field))
|
||
if float(limits["cpus"]) != want_cpu or int(limits["memory"]) != (
|
||
want_mem
|
||
):
|
||
problems.append(
|
||
f"docker-compose.yml {service} limits {limits['cpus']} CPU / "
|
||
f"{limits['memory']} B, Profile.{cpu_field} / {mem_field} "
|
||
f"{want_cpu:g} / {want_mem} B",
|
||
)
|
||
if not profile.is_telemetry_cost:
|
||
return problems
|
||
table = telemetry_cost.variants(
|
||
profile.telemetry_head_ratio,
|
||
profile.telemetry_metrics_interval,
|
||
)
|
||
chosen = [table[name] for name in profile.telemetry_variants]
|
||
layers = sorted(
|
||
{
|
||
v["collector_layer"]
|
||
for v in chosen
|
||
if v.get("telemetry") and v["collector_layer"]
|
||
},
|
||
)
|
||
collector = services["otel-collector"]["image"]
|
||
base = ROOT / telemetry_cost.COLLECTOR_CONFIG
|
||
|
||
def otelcol(*args: str, layer: str | None = None) -> tuple[int, str]:
|
||
mounts = ["-v", f"{base}:/c/base.yaml:ro"]
|
||
configs = ["--config=/c/base.yaml"]
|
||
if layer:
|
||
mounts += ["-v", f"{ROOT / layer}:/c/layer.yaml:ro"]
|
||
configs.append("--config=/c/layer.yaml")
|
||
run = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--network",
|
||
"none",
|
||
*mounts,
|
||
collector,
|
||
*args,
|
||
*configs,
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return run.returncode, run.stdout if run.returncode == 0 else (
|
||
run.stdout + run.stderr
|
||
)
|
||
|
||
if layers:
|
||
code, text = otelcol("print-config", "--format=json")
|
||
if code != 0:
|
||
return [
|
||
*problems,
|
||
f"{base.name}: print-config failed: {text[-500:]}",
|
||
]
|
||
base_json = json.loads(text)
|
||
for layer in layers:
|
||
code, text = otelcol("validate", layer=layer)
|
||
if code != 0:
|
||
problems.append(f"{layer}: validate failed: {text[-500:]}")
|
||
continue
|
||
code, text = otelcol("print-config", "--format=json", layer=layer)
|
||
if code != 0:
|
||
problems.append(f"{layer}: print-config failed: {text[-500:]}")
|
||
continue
|
||
problems += [
|
||
f"{layer}: {item}"
|
||
for item in telemetry_cost.collector_layer_problems(
|
||
base_json,
|
||
json.loads(text),
|
||
)
|
||
]
|
||
leans = sorted(
|
||
{
|
||
v["prometheus"]
|
||
for v in chosen
|
||
if v.get("telemetry") and v["prometheus"]
|
||
},
|
||
)
|
||
working = (ROOT / telemetry_cost.PROMETHEUS_CONFIG).read_text(
|
||
encoding="utf-8",
|
||
)
|
||
rules = ROOT / "infra" / "otel" / "prometheus" / "rules.yml"
|
||
for lean in leans:
|
||
text = (ROOT / lean).read_text(encoding="utf-8")
|
||
problems += [
|
||
f"{lean}: {item}"
|
||
for item in telemetry_cost.lean_problems(text, working)
|
||
]
|
||
check = subprocess.run(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--network",
|
||
"none",
|
||
"-v",
|
||
f"{ROOT / lean}:/etc/prometheus/prometheus.yml:ro",
|
||
"-v",
|
||
f"{rules}:/etc/prometheus/rules.yml:ro",
|
||
"--entrypoint",
|
||
"promtool",
|
||
services["prometheus"]["image"],
|
||
"check",
|
||
"config",
|
||
"/etc/prometheus/prometheus.yml",
|
||
],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
if check.returncode != 0:
|
||
problems.append(
|
||
f"{lean}: promtool check config failed: "
|
||
f"{(check.stdout + check.stderr)[-500:]}",
|
||
)
|
||
return problems
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
args = parse_args(argv)
|
||
if not re.fullmatch(r"[A-Za-z0-9._-]+", args.label):
|
||
print(f"bad label {args.label!r}: use [A-Za-z0-9._-]", flush=True)
|
||
return 2
|
||
# один прогон стенда любого профиля: второй — отказ до стенда
|
||
lock = acquire_lock()
|
||
if lock is None:
|
||
print(
|
||
"another bench run of the stand is in progress "
|
||
"(bench/.bench.lock is held): wait for it to finish",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
profile, overrides = apply_overrides(PROFILES[args.profile], args)
|
||
if args.telemetry == "on":
|
||
# выключатель телеметрии — только профили загрузок и очереди (design
|
||
# change track1-telemetry-cost, решение 13); у цены телеметрии режим
|
||
# задают варианты. Не в overrides: это методика метки
|
||
if profile.name not in TELEMETRY_SWITCH_PROFILES:
|
||
print(
|
||
f"--telemetry on: profile {profile.name} does not accept the "
|
||
"telemetry switch; it is accepted by "
|
||
+ ", ".join(TELEMETRY_SWITCH_PROFILES),
|
||
flush=True,
|
||
)
|
||
return 2
|
||
profile = replace(profile, telemetry="on")
|
||
if args.resume and not (
|
||
profile.is_series or profile.is_queue or profile.is_telemetry_cost
|
||
):
|
||
print(
|
||
"--resume: track 1 series, queue and telemetry profiles only",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
if profile.is_telemetry_cost or profile.telemetry_on:
|
||
problems = telemetry_preflight(profile)
|
||
if problems:
|
||
print(
|
||
"telemetry configuration check failed:\n "
|
||
+ "\n ".join(problems),
|
||
flush=True,
|
||
)
|
||
return 2
|
||
if profile.is_rollout:
|
||
# commit методики — до стенда, а не на шестой секунде сборки
|
||
# (спек load-benchmark, «Прежний commit профиля замены»)
|
||
problem = rollout_commit_problem(profile.rollout_from)
|
||
if problem:
|
||
print(problem, flush=True)
|
||
return 2
|
||
if profile.is_uploads and "before" in profile.uploads_variants:
|
||
# commit варианта «до» — тоже до стенда (спек load-benchmark,
|
||
# «Профиль загрузок»)
|
||
problem = rollout_commit_problem(
|
||
profile.uploads_before,
|
||
"uploads_before",
|
||
"uploads «before»",
|
||
)
|
||
if problem:
|
||
print(problem, flush=True)
|
||
return 2
|
||
if profile.is_reconcile and profile.reconcile_s4_candidates:
|
||
# commit образа «до» отказа по пределу — до стенда (решение 7
|
||
# design change track2-reconcile-cost)
|
||
problem = rollout_commit_problem(
|
||
profile.reconcile_before,
|
||
"reconcile_before",
|
||
"reconcile «before»",
|
||
)
|
||
if problem:
|
||
print(problem, flush=True)
|
||
return 2
|
||
# профиль отказа реплик — доказательство уровня 4, рядом с прогонами
|
||
# make test-failover (решение 7 design change graceful-failover)
|
||
kind = "failover" if profile.is_failover else "bench"
|
||
if profile.is_rollout:
|
||
# поочерёдная замена — доказательство уровня 4 (решение 8 design
|
||
# change rolling-update)
|
||
kind = "rollout"
|
||
if profile.is_uploads:
|
||
# память загрузок — доказательство уровня 4 лабы 4 (design change
|
||
# upload-memory, решение 10)
|
||
kind = "uploads"
|
||
if profile.is_reconcile:
|
||
# стоимость сверки — рядом с evidence change track2-reconciler
|
||
# (решение 1 design change track2-reconcile-cost)
|
||
kind = "reconcile"
|
||
if profile.is_queue:
|
||
# очередь под перегрузкой: трек 1 лабы 5 (design change
|
||
# track1-queue-overload, решение 14)
|
||
kind = "queue"
|
||
if profile.is_telemetry_cost:
|
||
# цена телеметрии: трек 1 лабы 6 (design change track1-telemetry-cost,
|
||
# решение 12)
|
||
kind = "telemetry-cost"
|
||
label_dir = ROOT / "evidence" / kind / args.label
|
||
relative = label_dir.relative_to(ROOT)
|
||
if label_dir.exists() and not args.resume:
|
||
print(f"label directory {relative} already exists", flush=True)
|
||
return 2
|
||
# чистота дерева серии — без своей метки: её частичные результаты при
|
||
# продолжении не изменение кода (решение 5 design change scaling-bench)
|
||
resumable = (
|
||
profile.is_series or profile.is_queue or profile.is_telemetry_cost
|
||
)
|
||
git = git_info(exclude=str(relative) if resumable else None)
|
||
if args.resume:
|
||
problems = resume_problems(label_dir, profile, git)
|
||
if problems:
|
||
print(
|
||
f"cannot resume {relative}:\n " + "\n ".join(problems),
|
||
flush=True,
|
||
)
|
||
return 2
|
||
state = host_state()
|
||
try:
|
||
warnings = host_warnings(state) + vm_warnings(profile)
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 2
|
||
plan_budget: dict[str, Any] | None = None
|
||
peak_budget: dict[str, Any] | None = None
|
||
if (
|
||
profile.is_series
|
||
or profile.sample_resources
|
||
or profile.is_queue
|
||
or profile.is_telemetry_cost
|
||
):
|
||
info = sh(
|
||
"docker",
|
||
"info",
|
||
"--format",
|
||
"{{.NCPU}} {{.MemTotal}}",
|
||
check=False,
|
||
).split()
|
||
vm_cpus, vm_memory = (
|
||
(int(v) for v in info) if len(info) == 2 else (0, 0)
|
||
)
|
||
if profile.sample_resources:
|
||
# профили уровня «4» с сэмплером: сумма лимитов CPU на пике не
|
||
# больше CPU ВМ (design change profile-resources, решение 8)
|
||
peak_budget = level4_budget(profile, vm_cpus, vm_memory)
|
||
if peak_budget["cpus"] > vm_cpus:
|
||
print(
|
||
f"CPU limits at peak ({peak_budget['peak_replicas']} "
|
||
f"replicas, sampler) {peak_budget['cpus']:g} CPU exceed "
|
||
f"docker VM CPUs ({vm_cpus})",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
if profile.is_queue:
|
||
# профиль очереди: сумма лимитов с двумя worker (решение 10 design
|
||
# change track1-queue-overload)
|
||
peak_budget = queue_budget(profile, vm_cpus, vm_memory)
|
||
if peak_budget["cpus"] > vm_cpus:
|
||
if not profile.telemetry_on:
|
||
print(
|
||
f"CPU limits at peak {peak_budget['cpus']:g} CPU exceed "
|
||
f"docker VM CPUs ({vm_cpus})",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
# с телеметрией — предупреждение, а не отказ: лимиты CPU делят
|
||
# время, а ВМ уже отдаёт все ядра хоста (design change
|
||
# track1-telemetry-cost, решение 13)
|
||
warnings.append(
|
||
f"CPU limits at peak with telemetry {peak_budget['cpus']:g} "
|
||
f"CPU exceed docker VM CPUs ({vm_cpus}): limits share time",
|
||
)
|
||
if profile.is_telemetry_cost:
|
||
peak_budget = telemetry_budget(profile, vm_cpus, vm_memory)
|
||
if peak_budget["cpus"] > vm_cpus:
|
||
print(
|
||
f"CPU limits at peak {peak_budget['cpus']:g} CPU exceed docker "
|
||
f"VM CPUs ({vm_cpus})",
|
||
flush=True,
|
||
)
|
||
return 2
|
||
if profile.is_series:
|
||
# сумма лимитов CPU конфигурации не больше CPU ВМ (решение 1)
|
||
plan_budget = budget(profile, vm_cpus, vm_memory)
|
||
over = [
|
||
f"{key}: {item['cpus']:g} CPU"
|
||
for key, item in plan_budget.items()
|
||
if item["cpus"] > vm_cpus
|
||
]
|
||
if over:
|
||
print(
|
||
f"CPU limits exceed docker VM CPUs ({vm_cpus}): "
|
||
+ ", ".join(over),
|
||
flush=True,
|
||
)
|
||
return 2
|
||
command = " ".join(["uv run python bench/run.py", *argv])
|
||
others = sorted(
|
||
sh("docker", "ps", "--format", "{{.Names}}", check=False).split(),
|
||
)
|
||
if args.resume:
|
||
meta = json.loads(
|
||
(label_dir / "meta.json").read_text(encoding="utf-8"),
|
||
)
|
||
meta["status"] = "incomplete"
|
||
meta["reason"] = "running"
|
||
meta["host_warnings"] = meta.get("host_warnings", []) + warnings
|
||
meta.setdefault("resumes", []).append(
|
||
{
|
||
"at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"command": command,
|
||
"git": git,
|
||
"host_state": state,
|
||
"other_containers": others,
|
||
},
|
||
)
|
||
else:
|
||
meta = {
|
||
"label": args.label,
|
||
"status": "incomplete",
|
||
"reason": "running",
|
||
"started_at": datetime.now(UTC).isoformat(timespec="seconds"),
|
||
"command": command,
|
||
"git": git,
|
||
"overrides": overrides,
|
||
"host_state": {"start": state},
|
||
"host_warnings": warnings,
|
||
# чужие контейнеры в той же VM: стенд их не трогает, но они
|
||
# шумят
|
||
"other_containers": others,
|
||
}
|
||
if plan_budget is not None:
|
||
meta["budget"] = plan_budget
|
||
if peak_budget is not None:
|
||
meta["resources"] = {"budget": peak_budget}
|
||
stand_check = Stand(profile, label_dir)
|
||
started = time.monotonic()
|
||
# на macOS монотонные часы во сне стоят, а реальные идут: разница —
|
||
# сколько хост проспал посреди прогона (решение 9 design change
|
||
# track2-backup-restore)
|
||
started_wall = time.time()
|
||
|
||
def say(text: str) -> None:
|
||
elapsed = int(time.monotonic() - started)
|
||
stamp = time.strftime("%H:%M:%S", time.gmtime(elapsed))
|
||
print(f"[{stamp}] {text}", flush=True)
|
||
|
||
for warning in warnings:
|
||
say(f"WARNING: {warning}")
|
||
if overrides:
|
||
say(f"WARNING: profile overrides {overrides}: debug run")
|
||
if git["dirty"]:
|
||
say("NOTE: working tree has uncommitted changes (dirty: true)")
|
||
try:
|
||
if profile.is_series or (
|
||
(profile.is_queue or profile.is_telemetry_cost) and args.resume
|
||
):
|
||
# блокировка взята: остатки — от прерванного прогона, серия
|
||
# убирает их сама (решение 5 design change scaling-bench)
|
||
stand_check.remove_leftovers(say)
|
||
else:
|
||
stand_check.check_free()
|
||
image = stand_check.build_image()
|
||
except BenchError as exc:
|
||
print(f"bench failed: {exc}", flush=True)
|
||
return 1
|
||
say(f"image {APP_IMAGE} built ({image[:19]}…), compose project {PROJECT}")
|
||
if args.resume:
|
||
# ID индекса образа в хранилище containerd меняется при каждой
|
||
# сборке и на том же коде (аттестации); код один — проверено по
|
||
# commit и чистому дереву выше
|
||
meta["resumes"][-1]["app_image_id"] = image
|
||
else:
|
||
meta.update(static_passport(profile))
|
||
results = Results(label_dir, meta)
|
||
bench = Bench(profile, args.label, results)
|
||
bench.started = started
|
||
bench.resume = args.resume
|
||
if bench.stand.work_dir.exists():
|
||
# блокировка взята: `.work` остался от процесса, убитого до уборки
|
||
# (в нём файлы консоли k6 и fixture.json с токенами)
|
||
say(f"removing stale {bench.stand.work_dir.relative_to(ROOT)}")
|
||
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
|
||
code = 0
|
||
|
||
def terminate(*_: object) -> None:
|
||
raise KeyboardInterrupt
|
||
|
||
signal.signal(signal.SIGTERM, terminate)
|
||
try:
|
||
bench.run()
|
||
meta["status"] = "complete"
|
||
meta.pop("reason", None)
|
||
except KeyboardInterrupt:
|
||
meta["reason"] = "interrupted"
|
||
say("interrupted: partial results kept")
|
||
code = 130
|
||
except BenchError as exc:
|
||
meta["reason"] = str(exc).splitlines()[0]
|
||
say(f"bench failed: {exc}")
|
||
code = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку стенда
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if not bench.stand.stop():
|
||
say(f"cleanup incomplete: {leftovers()}")
|
||
code = code or 1
|
||
# fixture.json с токенами — только здесь, в результаты не попадает
|
||
shutil.rmtree(bench.stand.work_dir, ignore_errors=True)
|
||
meta["finished_at"] = datetime.now(UTC).isoformat(timespec="seconds")
|
||
end_state = host_state()
|
||
meta["host_state"]["end"] = end_state
|
||
# выгрузка в swap за этот запуск (у продолжения — от его начала)
|
||
swapped = swap_warning(state, end_state)
|
||
if swapped is not None:
|
||
meta["host_warnings"].append(swapped)
|
||
say(f"WARNING: {swapped}")
|
||
delta = swapouts_delta(state, end_state)
|
||
if delta is not None:
|
||
meta["host_state"]["swapouts_delta"] = delta
|
||
slept = (time.time() - started_wall) - (time.monotonic() - started)
|
||
meta["host_state"]["slept_seconds"] = round(max(0.0, slept))
|
||
if slept > SLEEP_WARN_S:
|
||
warning = f"host slept {slept:.0f}s during the run: rerun"
|
||
meta["host_warnings"].append(warning)
|
||
say(f"WARNING: {warning}")
|
||
results.write_meta()
|
||
results.write_summary()
|
||
bench.stage = "done" if code == 0 else meta.get("reason", "stopped")
|
||
bench.progress()
|
||
say(
|
||
f"{meta['status']}: {bench.runs_done}/{bench.runs_total} runs, "
|
||
f"results in {relative}",
|
||
)
|
||
return code
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|