Files
mtusi2026lab3/bench/run.py
T

1855 lines
70 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Оркестратор нагрузочного стенда лабы 3. Только stdlib.
uv run python bench/run.py failover [--label baseline] # уровень 4
uv run python bench/run.py failover-quick # проверка стенда
uv run python bench/run.py rollout [--label baseline] # уровень 4
uv run python bench/run.py rollout-quick
uv run python bench/run.py scaling --series a|b|db|ab|quick [--label L]
[--configs "4x1-pg0.5 4x1"] [--resume] # трек 1 уровня 5
Профили (поля — в `PROFILES`, целиком пишутся в `meta.json`):
- `failover` — стенд с лимитами рабочего (две реплики по 1 CPU / 512 МБ,
PostgreSQL 2 CPU / 1 ГБ, Valkey 0,5 / 128 МБ, HAProxy 1 / 256 МБ),
постоянная интенсивность записи с ключом и чтения; каждая реплика по
очереди в режимах `stop`, `kill`, `pause` с возвратом. Посекундная шкала,
время исключения из LB и возврата, ошибки, время восстановления, повтор
неуспешных записей с тем же ключом и сверка «без потерь и дублей»,
CPU и память компонентов по окнам фазы. Результат —
`evidence/failover/<метка>/`;
- `rollout` — тот же стенд и нагрузка, посреди неё
`scripts/rolling-update.sh` меняет обе реплики на новый образ с
expand-ревизией схемы (`9001_rollout_probe`, только в образе стенда).
Результат — `evidence/rollout/<метка>/`;
- `scaling` — серии трека 1 «Цена масштабирования»: конфигурации реплик
и лимитов, на каждую ячейку (конфигурация × повтор) свежий стенд,
открытая модель, фиксированные уровни и поиск R* по SLO, сэмплер
контейнеров, LB и PostgreSQL. Результат — `evidence/scaling/<метка>/`.
Стенд — чёрный ящик: запросы делает только генератор (`loadgen.py`) по
HTTP через HAProxy стенда; к PostgreSQL оркестратор обращается напрямую
только для обслуживания и сверки (VACUUM, CHECKPOINT, число строк,
ревизия схемы). Метки `quick-*` проверяют стенд и в отчёт не идут.
Правила — `bench/AGENTS.md`, порядок работы — `bench/RUNBOOK.md`.
"""
import argparse
import csv
import dataclasses
import gzip
import json
import random
import statistics
import subprocess
import time
from pathlib import Path
import analysis
import plots
from common import (
CACHE,
DB,
EVIDENCE,
FIXTURE,
GEN,
IMAGE_REPO,
LB,
PROJECT,
ROOT,
SAMPLER,
SERVICE_TAG,
Limits,
Stand,
StandError,
bench_lock,
build_image,
code_state,
docker,
finish_host_state,
image_info,
new_label_dir,
preflight,
probe_migration_context,
quick_label,
say,
temp_workdir,
utc_now,
write_json,
)
# лимиты рабочего стенда compose.yaml: профили уровня 4 отвечают, что видит
# клиент `make up` при отказе или замене реплики
WORKING = Limits(api_replicas=2, api_cpus=1.0, api_memory="512m")
@dataclasses.dataclass(frozen=True)
class LoadProfile:
"""Профили уровня 4 (`failover`, `rollout`). Часть методики."""
name: str
limits: Limits
users: int
read_messages: int
write_rate: float
read_rate: float
gen_procs: int
session_ttl: int
seed: int
# failover: режимы отказа и сколько реплик по очереди
modes: tuple[str, ...] = ()
targets: int = 0
before_s: float = 0.0
fail_s: float = 0.0
after_s: float = 0.0
# запас на возврат реплики в LB после восстановления
restore_s: float = 10.0
# rollout: окна до скрипта, бюджет замены и после
rollout_before_s: float = 0.0
rollout_budget_s: float = 0.0
rollout_after_s: float = 0.0
# худшее обнаружение отказа по формуле haproxy.cfg, с
bounds: tuple[tuple[str, float], ...] = (
("stop", 2.5),
("kill", 2.5),
("pause", 6.5),
)
@dataclasses.dataclass(frozen=True)
class ScalingProfile:
"""Серии трека 1. Часть методики: метки сравнимы при равном профиле."""
name: str
series: str
configs: tuple[str, ...]
repeats: int
users: int = 20
read_messages: int = 10_000
write_chats: int = 10
fill_clients: int = 16
# смесь read,write,me: история (70), отправка с ключом (20), профиль (10)
mix: str = "70,20,10"
fixed_rates: tuple[float, ...] = (250.0, 500.0)
ladder_base: float = 100.0
ladder_factor: float = 1.25
# поиск начинается не ниже этого × число реплик
start_per_replica: float = 300.0
bisections: int = 2
rate_cap: float = 8000.0
warmup_first_s: float = 10.0
warmup_ramp_s: float = 5.0
window_s: float = 15.0
pause_after_fail_s: float = 10.0
slo_p95_ms: float = 200.0
slo_fail_share: float = 0.01
abort_error_share: float = 0.5
abort_p95_ms: float = 2000.0
request_timeout_s: float = 10.0
gen_procs: int = 4
session_ttl: int = 21_600
order_seed: int = 20261014
# лимиты без реплик: PostgreSQL, Valkey, LB, генератор, сэмплер
base: Limits = Limits(api_replicas=1, api_cpus=1.0, api_memory="512m")
FAILOVER = LoadProfile(
name="failover",
limits=WORKING,
users=4,
read_messages=1_000,
write_rate=100.0,
read_rate=100.0,
gen_procs=2,
session_ttl=21_600,
seed=20261014,
modes=("stop", "kill", "pause"),
targets=2,
before_s=10.0,
fail_s=15.0,
after_s=10.0,
)
ROLLOUT = dataclasses.replace(
FAILOVER,
name="rollout",
modes=(),
targets=0,
rollout_before_s=10.0,
rollout_budget_s=90.0,
rollout_after_s=15.0,
)
PROFILES: dict[str, LoadProfile] = {
"failover": FAILOVER,
"failover-quick": dataclasses.replace(
FAILOVER,
name="failover-quick",
write_rate=50.0,
read_rate=50.0,
modes=("stop", "kill"),
targets=1,
before_s=5.0,
fail_s=8.0,
after_s=5.0,
),
"rollout": ROLLOUT,
"rollout-quick": dataclasses.replace(
ROLLOUT,
name="rollout-quick",
write_rate=50.0,
read_rate=50.0,
rollout_before_s=5.0,
rollout_budget_s=60.0,
rollout_after_s=5.0,
),
}
SERIES: dict[str, ScalingProfile] = {
# бюджет приложения 4 CPU / 2 ГиБ: один процесс, два, четыре
"a": ScalingProfile(
name="scaling", series="a", configs=("1x4", "2x2", "4x1"), repeats=3
),
# рост ресурсов: реплики по 1 CPU, PostgreSQL 2 CPU фиксирован
"b": ScalingProfile(
name="scaling", series="b", configs=("1x1", "2x1", "3x1", "4x1"), repeats=3
),
# управляемое насыщение PostgreSQL: 0,5 CPU при 1, 2, 4 репликах
"db": ScalingProfile(
name="scaling",
series="db",
configs=("1x1-pg0.5", "2x1-pg0.5", "4x1-pg0.5"),
repeats=3,
),
# «до/после» улучшения и A/A: ячейки задаёт --configs
"ab": ScalingProfile(
name="scaling", series="ab", configs=("4x1-pg0.5", "4x1"), repeats=3
),
# самопроверка серий: малые данные, короткие окна, один повтор
"quick": ScalingProfile(
name="scaling-quick",
series="quick",
configs=("1x1", "2x1"),
repeats=1,
users=5,
read_messages=1_000,
write_chats=3,
fixed_rates=(100.0, 200.0),
start_per_replica=150.0,
bisections=1,
warmup_first_s=4.0,
warmup_ramp_s=3.0,
window_s=6.0,
pause_after_fail_s=5.0,
),
}
RUN_COLUMNS = (
"series",
"config",
"repeat",
"cell",
"step",
"phase",
"rate",
"rate_from",
"warmup_s",
"window_s",
"scheduled",
"ok",
"errors",
"short_pages",
"dropped",
"unsent",
"fail_share",
"achieved_rps",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"service_p95_ms",
"lag_p99_ms",
"slo_pass",
"abort",
"api_cpu_cores",
"api_cpu_max",
"api_throttled_max",
"api_memory_max_mib",
"db_cpu_cores",
"db_throttled_share",
"lb_cpu_cores",
"cache_cpu_cores",
"gen_cpu_cores",
"vm_busy_share",
"pg_conn_max",
"pg_conn_replica_max",
"pg_active_mean",
"pg_lock_wait_mean",
"pg_new_sessions",
"pg_commits_per_s",
"lb_retries",
"lb_redispatches",
"lb_conn_errors",
"api_cpu_us_per_req",
"db_cpu_us_per_req",
"instances",
"instance_share_min",
"signals",
"error_kinds",
)
SEARCH_COLUMNS = (
"series",
"config",
"replicas",
"api_cpus",
"db_cpus",
"repeat",
"cell",
"r_star",
"first_fail",
"p95_at_r_star",
"signals_at_fail",
"generator_bound",
"steps",
"fill_rps",
"duration_s",
"flags",
)
SAMPLE_COLUMNS = (
"config",
"repeat",
"step",
"rate",
"t",
"role",
"container",
"cpu_cores",
"throttled_share",
"memory_mib",
)
# --- общие шаги ---------------------------------------------------------------------
def write_csv(path: Path, columns: tuple[str, ...], rows: list[dict]) -> None:
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore")
writer.writeheader()
writer.writerows(rows)
def append_csv(path: Path, columns: tuple[str, ...], row: dict) -> None:
new = not path.exists()
with path.open("a", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=columns, extrasaction="ignore")
if new:
writer.writeheader()
writer.writerow(row)
def read_csv(path: Path) -> list[dict]:
if not path.exists():
return []
with path.open(encoding="utf-8") as handle:
return list(csv.DictReader(handle))
def prepare_image(label: str) -> dict:
"""Образ из рабочего дерева под тегом метки; на него смотрит сервис api.
Образ полной метки остаётся в docker: по нему метку можно сверить
(`docker image inspect`); пробные `quick-*` убирает `drop_quick_image`.
"""
tag = f"{IMAGE_REPO}:bench-{label}"
say(f"build image {tag}")
build_image(tag)
docker("tag", tag, SERVICE_TAG)
return image_info(tag)
def drop_quick_image(label: str) -> None:
if label.startswith("quick-"):
docker("rmi", f"{IMAGE_REPO}:bench-{label}", check=False)
def clock_offset() -> float:
"""Часы ВМ Docker минус часы хоста, с: по контейнеру генератора.
Три замера, берётся с наименьшей задержкой вызова. Нужен только для
отметок, которые оркестратор ставит сам (начало и конец скрипта
замены); остальные отметки — часы ВМ.
"""
best = None
for _ in range(3):
before = time.time()
out = docker("exec", GEN, "python", "-c", "import time; print(time.time())")
after = time.time()
rtt = after - before
offset = float(out.stdout.strip()) - (before + after) / 2
if best is None or rtt < best[0]:
best = (rtt, offset)
return best[1]
def roles_of(stand: Stand) -> dict[str, str]:
"""id12 контейнера → роль: api, db, cache, lb, gen."""
roles = {container[:12]: "api" for container in stand.api_ids()}
for role, name in (
("db", DB),
("cache", CACHE),
("lb", LB),
("gen", GEN),
("sampler", SAMPLER),
):
done = docker("inspect", "-f", "{{.Id}}", name, check=False)
if done.returncode == 0:
roles[done.stdout.strip()[:12]] = role
return roles
def full_ids(stand: Stand) -> list[str]:
ids = list(stand.api_ids())
for name in (DB, CACHE, LB, GEN):
ids.append(docker("inspect", "-f", "{{.Id}}", name).stdout.strip())
return ids
def resources_by_role(ticks: list[dict], roles: dict[str, str]) -> dict:
"""CPU и память по ролям за окно; у api — сумма и максимум реплики."""
usage = {cid: analysis.cgroup_usage(ticks, cid) for cid in roles}
out: dict = {}
api = [u for cid, u in usage.items() if u and roles[cid] == "api"]
if api:
out["api_cpu_cores"] = round(sum(u["cpu_cores"] for u in api), 3)
out["api_cpu_max"] = max(u["cpu_cores"] for u in api)
out["api_throttled_max"] = max(u["throttled_share"] for u in api)
out["api_memory_max_mib"] = max(u["memory_mib"] for u in api)
for cid, role in roles.items():
if role == "api" or not usage.get(cid):
continue
out[f"{role}_cpu_cores"] = usage[cid]["cpu_cores"]
out[f"{role}_throttled_share"] = usage[cid]["throttled_share"]
out[f"{role}_memory_mib"] = usage[cid]["memory_mib"]
out["vm_busy_share"] = analysis.vm_busy(ticks)
return out
# --- failover -------------------------------------------------------------------------
def run_failover(profile: LoadProfile, label: str) -> Path:
workdir = temp_workdir()
folder = new_label_dir("failover", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
pre = preflight(profile.limits.total_cpus(), profile.limits.total_memory_gib())
image = prepare_image(label)
meta = {
"kind": "failover",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
}
write_json(folder / "meta.json", meta)
stand = Stand(profile.limits, session_ttl=profile.session_ttl, workdir=workdir)
phases, timeline_rows, resource_rows = [], [], []
try:
say("stand: postgres, valkey, migrate, 2 replicas, haproxy")
stand.start()
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
"1",
)
stand.maintain()
meta["setup"] = {k: v for k, v in setup.items() if k != "write_chats"}
meta["stand"] = stand.passport()
write_chat = setup["write_chats"][0]
targets = stand.api_ids()[: profile.targets]
roles = roles_of(stand)
stand.start_sampler("scan")
number = 0
for index, target in enumerate(targets, start=1):
for mode in profile.modes:
number += 1
name = f"p{number}-{mode}-r{index}"
say(f"phase {name}: target {target[:12]}")
phase, rows = failover_phase(
stand, profile, name, mode, target, write_chat, folder
)
phases.append(phase)
timeline_rows += rows
ticks = stand.sampler_lines()
roles.update(roles_of(stand))
for phase in phases:
for window, (start, end) in phase.pop("_windows").items():
usage = resources_by_role(
analysis.window_ticks(ticks, start / 1000, end / 1000), roles
)
ok_rps = phase[f"ok_rps_{window}"]
resource_rows.append(
{
"phase": phase["phase"],
"window": window,
**usage,
"api_cpu_us_per_req": _per_req(
usage.get("api_cpu_cores"), ok_rps
),
"db_cpu_us_per_req": _per_req(
usage.get("db_cpu_cores"), ok_rps
),
},
)
meta["sampler_ticks"] = len(ticks)
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
finally:
stand.stop_sampler()
stand.stop()
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(folder / "phases.csv", PHASE_COLUMNS, phases)
write_csv(folder / "timeline.csv", TIMELINE_COLUMNS, timeline_rows)
write_csv(folder / "resources.csv", RESOURCE_COLUMNS, resource_rows)
(folder / "summary.md").write_text(
failover_summary(meta, phases, resource_rows), encoding="utf-8"
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
PHASE_COLUMNS = (
"phase",
"mode",
"replica",
"target",
"slot",
"action_at_ms",
"restore_at_ms",
"detection_s",
"bound_s",
"over_bound",
"lb_return_s",
"errors_before",
"errors_fail",
"errors_after",
"error_kinds",
"first_error_s",
"last_error_s",
"recovery_s",
"ok_rps_before",
"ok_rps_fail",
"ok_rps_after",
"p95_before_ms",
"p95_fail_ms",
"p95_after_ms",
"max_fail_ms",
"target_after_down",
"writes",
"retried",
"recovered",
"rows",
"distinct_texts",
"verified",
"lb_errors",
)
TIMELINE_COLUMNS = (
"phase",
"second",
"sent",
"ok_write",
"ok_read",
"ok_target",
"ok_other",
"ok_old",
"ok_new",
"errors",
"error_kinds",
"p95_ms",
"max_ms",
)
RESOURCE_COLUMNS = (
"phase",
"window",
"api_cpu_cores",
"api_cpu_max",
"api_throttled_max",
"api_memory_max_mib",
"db_cpu_cores",
"db_memory_mib",
"lb_cpu_cores",
"cache_cpu_cores",
"gen_cpu_cores",
"vm_busy_share",
"api_cpu_us_per_req",
"db_cpu_us_per_req",
)
def _per_req(cores: float | None, rps: float | None) -> float | None:
if not cores or not rps:
return None
return round(cores * 1e6 / rps, 1)
def start_events(path: Path, *filters: str) -> subprocess.Popen:
"""Поток событий Docker в файл: буфер демона хранит лишь последние
события всех контейнеров, поэтому — поток с начала фазы."""
handle = path.open("w", encoding="utf-8")
process = subprocess.Popen(
["docker", "events", *filters, "--format", "{{json .}}"],
stdout=handle,
stderr=subprocess.DEVNULL,
cwd=ROOT,
)
process.handle = handle # type: ignore[attr-defined]
return process
def stop_events(process: subprocess.Popen) -> None:
time.sleep(0.5)
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.handle.close() # type: ignore[attr-defined]
def lb_log_since(since_wall: float) -> str:
"""Журнал HAProxy стенда с отметками ВМ (`docker logs -t`)."""
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(since_wall - 5))
done = docker("logs", "-t", "--since", stamp, LB, check=False)
return done.stdout + done.stderr
def collect_requests(stand: Stand, logs: list[str]) -> str:
return "".join(stand.gen_file(path) for path in logs)
def verify_writes(stand: Stand, chat: str, prefix: str) -> tuple[int, int]:
"""Строк и разных текстов в чате записи с ключами фазы."""
row = stand.psql(
"SELECT count(*), count(DISTINCT text) FROM messages "
f"WHERE chat_id = '{chat}' AND text LIKE '%{prefix}.%';\n",
)
rows, distinct = (int(value) for value in row.split("|"))
return rows, distinct
def failover_phase(
stand: Stand,
profile: LoadProfile,
name: str,
mode: str,
target: str,
write_chat: str,
folder: Path,
) -> tuple[dict, list[dict]]:
stand.wait_lb_up(profile.limits.api_replicas)
address = stand.address(target)
slot = next(s["slot"] for s in stand.lb_slots() if s["addr"] == address)
events_path = stand.workdir / f"{name}-events.log"
events = start_events(events_path, "--filter", f"container={target}")
since = time.time()
duration = profile.before_s + profile.fail_s + profile.restore_s + profile.after_s
log = f"/tmp/{name}.log"
load = stand.gen_popen(
"constant",
"--fixture",
FIXTURE,
"--write-rate",
str(profile.write_rate),
"--read-rate",
str(profile.read_rate),
"--duration",
str(duration),
"--procs",
str(profile.gen_procs),
"--seed",
str(profile.seed),
"--key-prefix",
name,
"--log",
log,
)
try:
time.sleep(profile.before_s + 0.3)
if mode == "stop":
# как scripts/stop-one.sh: graceful, срок 9 с
docker("stop", "--timeout", "9", target)
elif mode == "kill":
docker("kill", target)
else:
docker("pause", target)
time.sleep(profile.fail_s)
docker("unpause" if mode == "pause" else "start", target)
stand.wait_slot(address, "UP", timeout=profile.restore_s + 20)
out, err = load.communicate(timeout=duration + 60)
finally:
if load.poll() is None:
load.kill()
stop_events(events)
if load.returncode != 0:
raise StandError(f"generator failed in {name}: {err.strip()[-500:]}")
summary = json.loads(out.strip().splitlines()[-1])
raw = collect_requests(stand, summary["logs"])
requests = analysis.parse_requests(raw)
retry = stand.gen("retry", "--fixture", FIXTURE, *summary["logs"])
rows, distinct = verify_writes(stand, write_chat, name)
lb_text = lb_log_since(since)
events_list = analysis.parse_events(events_path.read_text(encoding="utf-8"))
(folder / "raw" / f"{name}-events.log").write_text(
events_path.read_text(encoding="utf-8"), encoding="utf-8"
)
(folder / "raw" / f"{name}-lb.log").write_text(lb_text, encoding="utf-8")
with gzip.open(
folder / "raw" / f"{name}-requests.csv.gz", "wt", encoding="ascii"
) as fh:
fh.write(raw)
action_names = {
"stop": ("kill", "stop", "die"),
"kill": ("kill", "die"),
"pause": ("pause",),
}
action_at = next(t for t, a, _ in events_list if a in action_names[mode])
restore_at = next(
t
for t, a, _ in events_list
if a == ("unpause" if mode == "pause" else "start") and t > action_at
)
lb_events = [e for e in analysis.parse_lb_log(lb_text) if e[1] == slot]
down = next(
(
t
for t, _, s in lb_events
if t >= action_at and s in ("is DOWN", "is going DOWN")
),
None,
)
up = next((t for t, _, s in lb_events if t >= restore_at and s == "is UP"), None)
load_start = summary["start"] * 1000
load_end = load_start + duration * 1000
windows = {
"before": (load_start + 2000, action_at),
"fail": (action_at, restore_at),
"after": (up or restore_at, load_end),
}
stats = {w: analysis.window_stats(requests, s, e) for w, (s, e) in windows.items()}
errors_all = [r for r in requests if not r.ok and r.scheduled >= load_start]
bound = dict(profile.bounds)[mode]
detection = round((down - action_at) / 1000, 2) if down else None
target_id = target[:12]
target_after = sum(
1
for r in requests
if r.ok
and r.instance == target_id
and down
and down + 1000 <= r.scheduled < restore_at
)
phase = {
"phase": name,
"mode": mode,
"replica": name.rsplit("-", 1)[1],
"target": target_id,
"slot": slot,
"action_at_ms": round(action_at, 1),
"restore_at_ms": round(restore_at, 1),
"detection_s": detection,
"bound_s": bound,
"over_bound": bool(detection is None or detection > bound),
"lb_return_s": round((up - restore_at) / 1000, 2) if up else None,
"errors_before": stats["before"]["errors"],
"errors_fail": stats["fail"]["errors"],
"errors_after": stats["after"]["errors"],
"error_kinds": ";".join(
f"{k}:{v}"
for k, v in sorted(
{
kind: sum(analysis.error_label(r) == kind for r in errors_all)
for kind in {analysis.error_label(r) for r in errors_all}
}.items()
)
),
"first_error_s": round((errors_all[0].scheduled - action_at) / 1000, 2)
if errors_all
else None,
"last_error_s": round((errors_all[-1].scheduled - action_at) / 1000, 2)
if errors_all
else None,
"recovery_s": analysis.recovery_seconds(
requests, action_at, restore_at, profile.write_rate + profile.read_rate
),
"ok_rps_before": stats["before"]["ok_rps"],
"ok_rps_fail": stats["fail"]["ok_rps"],
"ok_rps_after": stats["after"]["ok_rps"],
"p95_before_ms": stats["before"]["p95_ms"],
"p95_fail_ms": stats["fail"]["p95_ms"],
"p95_after_ms": stats["after"]["p95_ms"],
"max_fail_ms": stats["fail"]["max_ms"],
"target_after_down": target_after,
"writes": retry["keys"],
"retried": retry["failed"],
"recovered": retry["recovered"],
"rows": rows,
"distinct_texts": distinct,
"verified": rows == distinct == retry["keys"],
"lb_errors": ";".join(
f"{k}:{v}" for k, v in sorted(analysis.lb_http_errors(lb_text).items())
),
"_windows": windows,
}
if not phase["verified"]:
raise StandError(
f"{name}: {retry['keys']} keys sent, {rows} rows, {distinct} distinct texts"
)
groups = {
"target": {target_id},
"other": {s[:12] for s in stand.api_ids()} - {target_id},
}
rows_timeline = [
{"phase": name, **row}
for row in analysis.timeline(requests, action_at, load_start, load_end, groups)
]
return phase, rows_timeline
def failover_summary(meta: dict, phases: list[dict], resources: list[dict]) -> str:
profile = meta["profile"]
lines = [
f"# Отказ реплики под нагрузкой: {meta['label']}",
"",
_passport_lines(meta),
"",
f"Нагрузка: открытая модель, {profile['write_rate']:.0f} записей с "
f"`Idempotency-Key` и {profile['read_rate']:.0f} чтений страницы истории "
"в секунду через HAProxy стенда; две реплики с лимитами рабочего стенда. "
f"Фаза: {profile['before_s']:.0f} с до действия, действие, "
f"{profile['fail_s']:.0f} с отказа, возврат той же реплики, ещё "
f"{profile['after_s']:.0f} с.",
"",
"Отметки — часы ВМ Docker: действие и возврат — события Docker, "
"исключение и возврат в LB — журнал HAProxy, запросы — генератор "
"(момент по расписанию). Обнаружение — от действия до `DOWN` слота; "
"восстановление — от действия до начала секунды, после которой до "
"возврата нет ошибок и успешных ≥ 90 % поданной нагрузки.",
"",
"| Фаза | Режим | Реплика | Обнаружение, с (граница) | Возврат в LB, с | Ошибки до / отказ / после | Виды ошибок | Восстановление, с | Успешных/с до / отказ / после | p95, мс до / отказ / после | max в отказе, мс | Повтор: не прошло → прошло | Сверка |",
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
]
for p in phases:
lines.append(
f"| {p['phase']} | {p['mode']} | {p['replica']} ({p['target']}) | "
f"{_v(p['detection_s'])} ({p['bound_s']}){' ⚠' if p['over_bound'] else ''} | "
f"{_v(p['lb_return_s'])} | {p['errors_before']} / {p['errors_fail']} / "
f"{p['errors_after']} | {p['error_kinds'] or '—'} | {_v(p['recovery_s'])} | "
f"{p['ok_rps_before']} / {p['ok_rps_fail']} / {p['ok_rps_after']} | "
f"{_v(p['p95_before_ms'])} / {_v(p['p95_fail_ms'])} / {_v(p['p95_after_ms'])} | "
f"{_v(p['max_fail_ms'])} | {p['retried']} → {p['recovered']} | "
f"{'ok' if p['verified'] else 'нет'}: {p['writes']} ключей = {p['rows']} строк |"
)
if resources:
lines += [
"",
"## CPU и память по окнам фазы",
"",
"Сэмплер (cgroup каждого контейнера раз в секунду). Окна: до "
"действия (от первого запроса + 2 с), отказ (действие → возврат), "
"после (возврат в LB → конец нагрузки). CPU реплик — сумма; на "
"успешный запрос — только по окнам с нагрузкой.",
"",
"| Фаза | Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | Valkey | Генератор | ВМ занята | API, мкс/запрос | PostgreSQL, мкс/запрос |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for r in resources:
lines.append(
f"| {r['phase']} | {r['window']} | {_v(r.get('api_cpu_cores'))} "
f"({_v(r.get('api_cpu_max'))}) | {_v(r.get('db_cpu_cores'))} | "
f"{_v(r.get('lb_cpu_cores'))} | {_v(r.get('cache_cpu_cores'))} | "
f"{_v(r.get('gen_cpu_cores'))} | {_v(r.get('vm_busy_share'))} | "
f"{_v(r.get('api_cpu_us_per_req'))} | {_v(r.get('db_cpu_us_per_req'))} |"
)
lines += _warning_lines(meta)
return "\n".join(lines) + "\n"
def _v(value) -> str:
if value is None or value == "":
return "—"
if isinstance(value, float):
return f"{value:g}"
return str(value)
def _passport_lines(meta: dict) -> str:
code = meta.get("code", {})
commit = (code.get("git_commit") or "")[:12] or "git нет"
dock = meta.get("docker", {})
host = meta.get("host", {})
return (
f"Код: src_sha256 `{code.get('src_sha256', '')[:16]}`, commit {commit}. "
f"Хост: {host.get('cpu_model')}, {host.get('ram_gb')} ГиБ; ВМ Docker "
f"{dock.get('NCPU')} CPU, {round((dock.get('MemTotal') or 0) / 2**30, 1)} ГиБ, "
f"Docker {dock.get('ServerVersion')}, compose {dock.get('compose_version')}. "
f"Начало {meta.get('started')}, конец {meta.get('finished')}."
)
def _warning_lines(meta: dict) -> list[str]:
lines = []
if meta.get("host_warnings"):
lines += ["", "## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
if meta.get("aborted"):
lines += ["", f"**Прогон прерван:** {meta['aborted']}"]
return lines
# --- rollout --------------------------------------------------------------------------
def run_rollout(profile: LoadProfile, label: str) -> Path:
workdir = temp_workdir()
folder = new_label_dir("rollout", label)
(folder / "raw").mkdir()
started_mono, started_wall = time.monotonic(), time.time()
# на пике — реплик на одну больше
pre = preflight(
profile.limits.total_cpus(profile.limits.api_replicas + 1),
profile.limits.total_memory_gib(profile.limits.api_replicas + 1),
)
old = prepare_image(label)
say("build the new image: working tree + expand revision 9001_rollout_probe")
context, head = probe_migration_context(workdir)
new_tag = f"{IMAGE_REPO}:bench-{label}-new"
build_image(new_tag, context)
new = image_info(new_tag)
docker("tag", old["tag"], SERVICE_TAG)
meta = {
"kind": "rollout",
"label": label,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"host": pre["host"],
"docker": pre["docker"],
"budget": pre["budget"],
"host_warnings": pre["warnings"],
"image": old,
"image_new": new,
"code": code_state(old),
"rollout": {"schema_head_before": head, "probe_revision": "9001"},
}
write_json(folder / "meta.json", meta)
stand = Stand(profile.limits, session_ttl=profile.session_ttl, workdir=workdir)
windows_rows, timeline_rows, resource_rows = [], [], []
try:
say("stand: postgres, valkey, migrate, 2 replicas on the old image, haproxy")
stand.start()
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
"1",
)
stand.maintain()
meta["setup"] = {k: v for k, v in setup.items() if k != "write_chats"}
meta["stand"] = stand.passport()
write_chat = setup["write_chats"][0]
before_ids = stand.api_ids()
roles = roles_of(stand)
rollout = meta["rollout"]
rollout["replicas_before"] = [c[:12] for c in before_ids]
rollout["images_before"] = sorted(
{
docker("inspect", "-f", "{{.Image}}", c).stdout.strip()
for c in before_ids
}
)
rollout["alembic_before"] = stand.psql(
"SELECT version_num FROM alembic_version;\n"
)
offset = clock_offset()
rollout["clock_offset_s"] = round(offset, 4)
stand.start_sampler("scan")
events_path = workdir / "rollout-events.log"
events = start_events(
events_path, "--filter", f"label=com.docker.compose.project={PROJECT}"
)
since = time.time()
duration = (
profile.rollout_before_s
+ profile.rollout_budget_s
+ profile.rollout_after_s
)
load = stand.gen_popen(
"constant",
"--fixture",
FIXTURE,
"--write-rate",
str(profile.write_rate),
"--read-rate",
str(profile.read_rate),
"--duration",
str(duration),
"--procs",
str(profile.gen_procs),
"--seed",
str(profile.seed),
"--key-prefix",
"rollout",
"--log",
"/tmp/rollout.log",
)
try:
time.sleep(profile.rollout_before_s + 0.3)
docker("tag", new_tag, SERVICE_TAG)
script_start = time.time() + offset
script = subprocess.run(
[str(ROOT / "scripts" / "rolling-update.sh")],
cwd=ROOT,
env={
**stand.script_env(),
"API_REPLICAS": str(profile.limits.api_replicas),
"ROLLOUT_BUILD": "0",
},
capture_output=True,
text=True,
timeout=profile.rollout_budget_s + 60,
check=False,
)
script_end = time.time() + offset
out, err = load.communicate(timeout=duration + 60)
finally:
if load.poll() is None:
load.kill()
stop_events(events)
script_lines = [
line
for line in (script.stdout + script.stderr).splitlines()
if "rolling-update:" in line
]
(folder / "raw" / "rollout-script.log").write_text(
"\n".join(script_lines) + "\n", encoding="utf-8"
)
rollout["script_exit"] = script.returncode
rollout["script_seconds"] = round(script_end - script_start, 1)
if load.returncode != 0:
raise StandError(f"generator failed: {err.strip()[-500:]}")
if script.returncode != 0:
raise StandError(
f"rolling-update.sh exit {script.returncode}:\n"
+ "\n".join(script_lines[-10:])
)
summary = json.loads(out.strip().splitlines()[-1])
raw = collect_requests(stand, summary["logs"])
requests = analysis.parse_requests(raw)
retry = stand.gen("retry", "--fixture", FIXTURE, *summary["logs"])
rows, distinct = verify_writes(stand, write_chat, "rollout")
lb_text = lb_log_since(since)
(folder / "raw" / "rollout-lb.log").write_text(lb_text, encoding="utf-8")
(folder / "raw" / "rollout-events.log").write_text(
events_path.read_text(encoding="utf-8"), encoding="utf-8"
)
with gzip.open(
folder / "raw" / "rollout-requests.csv.gz", "wt", encoding="ascii"
) as fh:
fh.write(raw)
after_ids = stand.api_ids()
rollout["replicas_after"] = [c[:12] for c in after_ids]
rollout["images_after"] = sorted(
{docker("inspect", "-f", "{{.Image}}", c).stdout.strip() for c in after_ids}
)
rollout["alembic_after"] = stand.psql(
"SELECT version_num FROM alembic_version;\n"
)
rollout["probe_column"] = (
stand.psql(
"SELECT count(*) FROM information_schema.columns "
"WHERE table_name = 'messages' AND column_name = 'rollout_probe';\n"
)
== "1"
)
stand.wait_lb_up(profile.limits.api_replicas, timeout=30)
events_list = analysis.parse_events(events_path.read_text(encoding="utf-8"))
old_ids = {c[:12] for c in before_ids}
new_ids = {c[:12] for c in after_ids}
rollout["checks"] = {
"replicas_after": len(after_ids) == profile.limits.api_replicas,
"no_old_replica": not (old_ids & new_ids),
# образ контейнера — digest конфигурации (хранилище containerd), не
# Id тега: сравниваются контейнеры до и после между собой
"single_new_image": len(rollout["images_after"]) == 1
and rollout["images_after"] != rollout["images_before"],
"migrated": rollout["alembic_after"] == "9001",
"probe_column": rollout["probe_column"],
"writes_verified": rows == distinct == retry["keys"],
}
load_start = summary["start"] * 1000
load_end = load_start + duration * 1000
windows = {
"before": (load_start + 2000, script_start * 1000),
"during": (script_start * 1000, script_end * 1000),
"after": (script_end * 1000, load_end),
}
for window, (start, end) in windows.items():
stats = analysis.window_stats(requests, start, end)
windows_rows.append(
{
"window": window,
**{k: v for k, v in stats.items() if k != "error_kinds"},
"error_kinds": ";".join(
f"{k}:{v}" for k, v in sorted(stats["error_kinds"].items())
),
"ok_old": sum(
r.ok and r.instance in old_ids
for r in requests
if start <= r.scheduled < end
),
"ok_new": sum(
r.ok and r.instance in new_ids
for r in requests
if start <= r.scheduled < end
),
},
)
container_events = [
{
"t_s": round((t - script_start * 1000) / 1000, 2),
"action": a,
"container": c[:12],
# остальные контейнеры проекта за замену — миграция
"role": "old"
if c[:12] in old_ids
else "new"
if c[:12] in new_ids
else "migrate",
}
for t, a, c in events_list
if a in ("create", "start", "kill", "die", "stop", "destroy")
]
rollout["container_events"] = container_events
rollout["retry"] = retry
rollout["rows"] = rows
rollout["distinct_texts"] = distinct
rollout["lb_errors"] = analysis.lb_http_errors(lb_text)
timeline_rows = [
{"phase": "rollout", **row}
for row in analysis.timeline(
requests,
script_start * 1000,
load_start,
load_end,
{"old": old_ids, "new": new_ids},
)
]
ticks = stand.sampler_lines()
meta["sampler_ticks"] = len(ticks)
for cid in new_ids:
roles[cid] = "api"
# созданные за замену контейнеры, кроме реплик, — контейнер миграции
for _, action, container in events_list:
if action == "create" and container[:12] not in roles:
roles[container[:12]] = "migrate"
for window, (start, end) in windows.items():
usage = resources_by_role(
analysis.window_ticks(ticks, start / 1000, end / 1000), roles
)
ok_rps = next(w["ok_rps"] for w in windows_rows if w["window"] == window)
resource_rows.append(
{
"phase": "rollout",
"window": window,
**usage,
"api_cpu_us_per_req": _per_req(usage.get("api_cpu_cores"), ok_rps),
"db_cpu_us_per_req": _per_req(usage.get("db_cpu_cores"), ok_rps),
},
)
if not all(rollout["checks"].values()):
raise StandError(f"rollout checks failed: {rollout['checks']}")
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']}")
(folder / "app-tail.log").write_text(stand.app_tail(80), encoding="utf-8")
finally:
stand.stop_sampler()
stand.stop()
docker("rmi", f"{IMAGE_REPO}:bench-{label}-new", check=False)
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
write_csv(
folder / "windows.csv",
(
"window",
"seconds",
"requests",
"ok",
"ok_rps",
"ok_write_rps",
"ok_read_rps",
"errors",
"error_kinds",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"ok_old",
"ok_new",
),
windows_rows,
)
write_csv(folder / "timeline.csv", TIMELINE_COLUMNS, timeline_rows)
write_csv(folder / "resources.csv", RESOURCE_COLUMNS, resource_rows)
(folder / "summary.md").write_text(
rollout_summary(meta, windows_rows, resource_rows), encoding="utf-8"
)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def rollout_summary(meta: dict, windows: list[dict], resources: list[dict]) -> str:
profile, rollout = meta["profile"], meta.get("rollout", {})
lines = [
f"# Поочерёдная замена под нагрузкой: {meta['label']}",
"",
_passport_lines(meta),
"",
f"Нагрузка: {profile['write_rate']:.0f} записей с ключом и "
f"{profile['read_rate']:.0f} чтений в секунду через HAProxy стенда, "
"открытая модель. Две реплики на образе рабочего дерева; посреди "
"нагрузки `scripts/rolling-update.sh` (тот же, что `make rolling-update`) "
"меняет их на образ с expand-ревизией `9001_rollout_probe` (колонка "
"`messages.rollout_probe` NULL): миграция один раз, затем по одной "
"реплике — новая `UP` в LB, старая `drain` → `maint` → stop → rm.",
"",
f"- Скрипт: код выхода {rollout.get('script_exit')}, "
f"{rollout.get('script_seconds')} с; ревизия схемы "
f"{rollout.get('alembic_before')} → {rollout.get('alembic_after')}; колонка "
f"есть: {rollout.get('probe_column')}.",
f"- Реплики до: {', '.join(rollout.get('replicas_before', []))}; после: "
f"{', '.join(rollout.get('replicas_after', []))}.",
f"- Проверки: {rollout.get('checks')}.",
f"- Повтор неуспешных записей с тем же ключом: {rollout.get('retry')}; "
f"строк в чате записи {rollout.get('rows')}, разных текстов "
f"{rollout.get('distinct_texts')}.",
f"- Ответы самого HAProxy с ошибкой: {rollout.get('lb_errors') or 'нет'}.",
"",
"| Окно | с | Запросов | Успешных/с | Ошибок | Виды | p95, мс | max, мс | Ответов старых | Ответов новых |",
"|---|---|---|---|---|---|---|---|---|---|",
]
for w in windows:
lines.append(
f"| {w['window']} | {w['seconds']} | {w['requests']} | {w['ok_rps']} | "
f"{w['errors']} | {w['error_kinds'] or '—'} | {_v(w['p95_ms'])} | "
f"{_v(w['max_ms'])} | {w['ok_old']} | {w['ok_new']} |"
)
lines += ["", "## Контейнеры (секунды от начала скрипта)", ""]
for event in rollout.get("container_events", []):
lines.append(
f"- {event['t_s']:+.2f} с {event['action']} {event['role']} {event['container']}"
)
if resources:
lines += [
"",
"## CPU и память по окнам",
"",
"| Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | ВМ занята | API, мкс/запрос |",
"|---|---|---|---|---|---|",
]
for r in resources:
lines.append(
f"| {r['window']} | {_v(r.get('api_cpu_cores'))} ({_v(r.get('api_cpu_max'))}) | "
f"{_v(r.get('db_cpu_cores'))} | {_v(r.get('lb_cpu_cores'))} | "
f"{_v(r.get('vm_busy_share'))} | {_v(r.get('api_cpu_us_per_req'))} |"
)
lines += _warning_lines(meta)
return "\n".join(lines) + "\n"
# --- scaling (трек 1) -----------------------------------------------------------------
def parse_config(name: str, base: Limits) -> Limits:
"""`4x1` — 4 реплики по 1 CPU; `-pg0.5` — PostgreSQL 0,5 CPU;
`-mc50` — max_connections 50. Память реплики — 512 МиБ на CPU."""
head, *options = name.split("-")
replicas, cpus = head.split("x")
limits = dataclasses.replace(
base,
api_replicas=int(replicas),
api_cpus=float(cpus),
api_memory=f"{int(512 * float(cpus))}m",
)
for option in options:
if option.startswith("pg"):
limits = dataclasses.replace(limits, db_cpus=float(option[2:]))
elif option.startswith("mc"):
limits = dataclasses.replace(limits, db_max_connections=int(option[2:]))
else:
raise SystemExit(f"unknown config option {option!r} in {name!r}")
return limits
def cell_plan(profile: ScalingProfile) -> list[dict]:
"""Ячейки по кругам: в каждом круге — все конфигурации в случайном
порядке (зерно профиля); одна конфигурация не идёт дважды подряд."""
plan, previous = [], None
for repeat in range(1, profile.repeats + 1):
order = list(profile.configs)
random.Random(profile.order_seed + repeat).shuffle(order)
if len(order) > 1 and order[0] == previous:
order[0], order[1] = order[1], order[0]
for config in order:
plan.append({"cell": len(plan) + 1, "config": config, "repeat": repeat})
previous = order[-1]
return plan
def run_scaling(profile: ScalingProfile, label: str, resume: bool) -> Path:
workdir = temp_workdir()
limits_all = {c: parse_config(c, profile.base) for c in profile.configs}
worst_cpu = max(limits.total_cpus() for limits in limits_all.values())
worst_mem = max(limits.total_memory_gib() for limits in limits_all.values())
folder = EVIDENCE / "scaling" / label
if resume:
if not folder.exists():
raise SystemExit(
f"nothing to resume: {folder.relative_to(ROOT)} is missing"
)
meta = json.loads((folder / "meta.json").read_text(encoding="utf-8"))
if meta["profile"] != json.loads(json.dumps(dataclasses.asdict(profile))):
raise SystemExit("resume: profile differs from the label passport")
pre = preflight(worst_cpu, worst_mem)
image = prepare_image(label)
if image["src_sha256"] != meta["image"]["src_sha256"]:
raise SystemExit(
"resume: code differs from the label passport (src_sha256)"
)
meta.setdefault("resumed", []).append(utc_now())
else:
folder = new_label_dir("scaling", label)
(folder / "raw").mkdir()
pre = preflight(worst_cpu, worst_mem)
image = prepare_image(label)
meta = {
"kind": "scaling",
"label": label,
"series": profile.series,
"started": utc_now(),
"profile": dataclasses.asdict(profile),
"configs": {
c: dataclasses.asdict(limits) for c, limits in limits_all.items()
},
"budget": {
c: {
"cpus": limits.total_cpus(),
"memory_gib": round(limits.total_memory_gib(), 2),
}
for c, limits in limits_all.items()
},
"vm": pre["budget"],
"host": pre["host"],
"docker": pre["docker"],
"host_warnings": pre["warnings"],
"image": image,
"code": code_state(image),
"thresholds": analysis.THRESHOLDS,
"plan": cell_plan(profile),
}
started_mono, started_wall = time.monotonic(), time.time()
write_json(folder / "meta.json", meta)
done_cells = {int(row["cell"]) for row in read_csv(folder / "searches.csv")}
pool = image["app_defaults"]
pool_cap = pool["pool_size"] + pool["pool_overflow"]
try:
for cell in meta["plan"]:
if cell["cell"] in done_cells:
continue
say(
f"cell {cell['cell']}/{len(meta['plan'])}: {cell['config']} "
f"repeat {cell['repeat']}"
)
scaling_cell(
profile, limits_all[cell["config"]], cell, folder, workdir, pool_cap
)
except (StandError, subprocess.TimeoutExpired, KeyboardInterrupt) as exc:
meta["aborted"] = f"{type(exc).__name__}: {exc}"[:2000]
say(f"aborted: {meta['aborted']} (continue: --resume)")
drop_quick_image(label)
meta["finished"] = utc_now()
finish_host_state(meta, started_mono, started_wall)
write_json(folder / "meta.json", meta)
searches = read_csv(folder / "searches.csv")
runs = read_csv(folder / "runs.csv")
(folder / "summary.md").write_text(
scaling_summary(meta, searches, runs), encoding="utf-8"
)
if searches:
plots.render_label(folder)
say(f"done: {folder.relative_to(ROOT)}")
if meta.get("aborted"):
raise SystemExit(1)
return folder
def scaling_cell(
profile: ScalingProfile,
limits: Limits,
cell: dict,
folder: Path,
workdir: Path,
pool_cap: int,
) -> None:
started = time.monotonic()
stand = Stand(limits, session_ttl=profile.session_ttl, workdir=workdir)
limit_dict = dataclasses.asdict(limits)
try:
stand.start()
setup = stand.gen(
"setup",
"--fixture",
FIXTURE,
"--users",
str(profile.users),
"--read-messages",
str(profile.read_messages),
"--write-chats",
str(profile.write_chats),
"--fill-clients",
str(profile.fill_clients),
)
stand.maintain()
if cell["cell"] == 1 or not (folder / "stand.json").exists():
write_json(folder / "stand.json", stand.passport())
roles = roles_of(stand)
replica_addrs = {stand.address(c) for c in stand.api_ids()}
stand.start_sampler("list", full_ids(stand))
search = analysis.Search(
fixed=profile.fixed_rates,
base=profile.ladder_base,
factor=profile.ladder_factor,
start=profile.start_per_replica * limits.api_replicas,
bisections=profile.bisections,
cap=profile.rate_cap,
)
previous_rate, fresh = 0.0, True
step_no = 0
rows: list[dict] = []
while (step := search.next_rate()) is not None:
rate, phase = step
step_no += 1
warmup = profile.warmup_first_s if fresh else profile.warmup_ramp_s
rate_from = 0.0 if fresh else previous_rate
result = stand.gen(
"step",
"--fixture",
FIXTURE,
"--rate",
str(rate),
"--rate-from",
str(rate_from),
"--warmup",
str(warmup),
"--window",
str(profile.window_s),
"--mix",
profile.mix,
"--procs",
str(profile.gen_procs),
"--seed",
str(profile.order_seed + cell["cell"] * 100 + step_no),
"--timeout",
str(profile.request_timeout_s),
"--key-prefix",
f"c{cell['cell']}s{step_no}",
"--abort-error-share",
str(profile.abort_error_share),
"--abort-p95-ms",
str(profile.abort_p95_ms),
timeout=profile.warmup_first_s + profile.window_s + 120,
)
ticks = analysis.window_ticks(
stand.sampler_lines(), result["window_start"], result["window_end"]
)
row = step_row(
profile, cell, step_no, phase, result, ticks, roles, replica_addrs
)
row["slo_pass"] = bool(
not result["abort"]
and result["p95_ms"] is not None
and result["p95_ms"] <= profile.slo_p95_ms
and result["fail_share"] <= profile.slo_fail_share
)
row["signals"] = ";".join(analysis.signals(row, limit_dict, pool_cap))
append_csv(folder / "runs.csv", RUN_COLUMNS, row)
for sample in step_samples(
cell, step_no, rate, ticks, roles, result["window_start"]
):
append_csv(folder / "samples.csv", SAMPLE_COLUMNS, sample)
(folder / "raw" / f"c{cell['cell']:02d}-s{step_no:02d}.json").write_text(
json.dumps(result, ensure_ascii=False), encoding="utf-8"
)
rows.append(row)
say(
f" step {step_no} {phase} {rate:.0f}/s: p95 {result['p95_ms']} ms, "
f"fail {result['fail_share']:.4f}, {'pass' if row['slo_pass'] else 'FAIL'} "
f"[{row['signals']}]"
)
search.record(rate, row["slo_pass"], phase)
previous_rate = rate
fresh = not row["slo_pass"]
if fresh:
time.sleep(profile.pause_after_fail_s)
stand.gen("ready", "--timeout", "60")
r_star = search.r_star()
first_fail = search.first_fail()
fail_row = next(
(
r
for r in rows
if r["phase"] != "fixed" and float(r["rate"]) == first_fail
),
None,
)
star_row = next(
(r for r in rows if r["phase"] != "fixed" and float(r["rate"]) == r_star),
None,
)
flags = []
if fail_row and "generator" in fail_row["signals"].split(";"):
flags.append("generator_bound")
if r_star is not None and r_star >= profile.rate_cap:
flags.append("cap")
down = [name for name in (DB, CACHE, LB) if not _running(name)]
if down:
flags.append("stand_down:" + ",".join(down))
append_csv(
folder / "searches.csv",
SEARCH_COLUMNS,
{
"series": profile.series,
"config": cell["config"],
"replicas": limits.api_replicas,
"api_cpus": limits.api_cpus,
"db_cpus": limits.db_cpus,
"repeat": cell["repeat"],
"cell": cell["cell"],
"r_star": r_star,
"first_fail": first_fail,
"p95_at_r_star": star_row["p95_ms"] if star_row else None,
"signals_at_fail": fail_row["signals"] if fail_row else "",
"generator_bound": "generator_bound" in flags,
"steps": step_no,
"fill_rps": setup["fill_rps"],
"duration_s": round(time.monotonic() - started, 1),
"flags": ";".join(flags),
},
)
say(f" R* = {r_star} (first fail {first_fail})")
finally:
stand.stop_sampler()
stand.stop()
def _running(name: str) -> bool:
done = docker(
"inspect",
"-f",
"{{.State.Running}} {{.RestartCount}} {{.State.OOMKilled}}",
name,
check=False,
)
return done.stdout.strip() == "true 0 false"
def step_row(
profile: ScalingProfile,
cell: dict,
step_no: int,
phase: str,
result: dict,
ticks: list[dict],
roles: dict[str, str],
replica_addrs: set[str],
) -> dict:
usage = resources_by_role(ticks, roles)
pg = analysis.pg_stats(ticks, replica_addrs)
instances = result.get("instances") or {}
total = sum(instances.values())
row = {
"series": profile.series,
"config": cell["config"],
"repeat": cell["repeat"],
"cell": cell["cell"],
"step": step_no,
"phase": phase,
**{
key: result.get(key)
for key in (
"rate",
"rate_from",
"warmup_s",
"window_s",
"scheduled",
"ok",
"errors",
"short_pages",
"dropped",
"unsent",
"fail_share",
"achieved_rps",
"p50_ms",
"p95_ms",
"p99_ms",
"max_ms",
"service_p95_ms",
"lag_p99_ms",
"abort",
)
},
**usage,
**pg,
"lb_retries": analysis.lb_delta(ticks, "wretr"),
"lb_redispatches": analysis.lb_delta(ticks, "wredis"),
"lb_conn_errors": analysis.lb_delta(ticks, "econ"),
"instances": len(instances),
"instance_share_min": round(min(instances.values()) / total, 3)
if total
else None,
"error_kinds": ";".join(
f"{k}:{v}" for k, v in sorted((result.get("error_kinds") or {}).items())
),
}
if row.get("gen_cpu_cores") is None:
# нет тиков cgroup генератора: его собственное время процессов
row["gen_cpu_cores"] = round(
result["gen_cpu_s"] / (result["warmup_s"] + result["window_s"]), 3
)
row["api_cpu_us_per_req"] = _per_req(
row.get("api_cpu_cores"), result.get("achieved_rps")
)
row["db_cpu_us_per_req"] = _per_req(
row.get("db_cpu_cores"), result.get("achieved_rps")
)
return row
def step_samples(
cell: dict,
step_no: int,
rate: float,
ticks: list[dict],
roles: dict[str, str],
window_start: float,
) -> list[dict]:
"""Посекундные интервалы окна: CPU, троттлинг, память каждого контейнера."""
out = []
for a, b in zip(ticks, ticks[1:], strict=False):
seconds = b["ts"] - a["ts"]
if seconds <= 0:
continue
for cid, role in roles.items():
x, y = a["cg"].get(cid), b["cg"].get(cid)
if not x or not y or y[0] < x[0]:
continue
periods = y[1] - x[1]
out.append(
{
"config": cell["config"],
"repeat": cell["repeat"],
"step": step_no,
"rate": rate,
"t": round(b["ts"] - window_start, 2),
"role": role,
"container": cid,
"cpu_cores": round((y[0] - x[0]) / seconds / 1e6, 3),
"throttled_share": round((y[2] - x[2]) / periods, 3)
if periods
else 0.0,
"memory_mib": round(y[4] / 2**20, 1),
}
)
return out
def _f(value) -> float | None:
try:
return float(value)
except (TypeError, ValueError):
return None
def scaling_summary(meta: dict, searches: list[dict], runs: list[dict]) -> str:
profile = meta["profile"]
lines = [
f"# Серия трека 1 «{meta['series']}»: {meta['label']}",
"",
_passport_lines(meta),
"",
f"Открытая модель через HAProxy стенда, смесь read/write/me "
f"{profile['mix']} (страница истории читаемого чата из "
f"{profile['read_messages']} сообщений, отправка с новым ключом в один из "
f"{profile['write_chats']} чатов, профиль). Ступень: прогрев "
f"{profile['warmup_first_s']:g} с с нуля или {profile['warmup_ramp_s']:g} с "
f"подъёма, окно {profile['window_s']:g} с. Фиксированные уровни "
f"{', '.join(f'{r:g}' for r in profile['fixed_rates'])}/с, затем поиск R*: "
f"лестница {profile['ladder_base']:g} × {profile['ladder_factor']:g}^i от "
f"{profile['start_per_replica']:g} × реплик, {profile['bisections']} деления. "
f"SLO: p95 успешных ≤ {profile['slo_p95_ms']:g} мс и (ошибки + неполные "
f"страницы + недосланные) ≤ {profile['slo_fail_share'] * 100:g} %. "
f"Ячейка — конфигурация × повтор ({profile['repeats']}), свежий стенд.",
"",
"| Конфигурация | Реплик × CPU | PostgreSQL CPU | R*, медиана (min–max) | Эффективность к 1 × 1 | p95 на "
+ " / ".join(f"{r:g}" for r in profile["fixed_rates"])
+ " /с, мс | API мкс/запрос на "
+ f"{profile['fixed_rates'][-1]:g}/с | PostgreSQL мкс/запрос | Признаки на первой нарушившей |",
"|---|---|---|---|---|---|---|---|---|",
]
by_config: dict[str, list[dict]] = {}
for row in searches:
by_config.setdefault(row["config"], []).append(row)
base = by_config.get("1x1")
base_r = statistics.median(_f(r["r_star"]) or 0 for r in base) if base else None
for config in profile["configs"]:
rows = by_config.get(config, [])
if not rows:
lines.append(f"| {config} | — | — | не снята | — | — | — | — | — |")
continue
values = [_f(r["r_star"]) or 0 for r in rows]
median = statistics.median(values)
replicas = int(rows[0]["replicas"])
efficiency = (
f"{median / (replicas * base_r):.2f}" if base_r and config != "1x1" else "—"
)
fixed = []
for rate in profile["fixed_rates"]:
p95 = [
_f(r["p95_ms"])
for r in runs
if r["config"] == config
and r["phase"] == "fixed"
and _f(r["rate"]) == rate
]
p95 = [v for v in p95 if v is not None]
fixed.append(f"{statistics.median(p95):g}" if p95 else "—")
last_fixed = [
r
for r in runs
if r["config"] == config
and r["phase"] == "fixed"
and _f(r["rate"]) == profile["fixed_rates"][-1]
]
api_us = [
_f(r["api_cpu_us_per_req"])
for r in last_fixed
if _f(r["api_cpu_us_per_req"])
]
db_us = [
_f(r["db_cpu_us_per_req"]) for r in last_fixed if _f(r["db_cpu_us_per_req"])
]
signals: dict[str, int] = {}
for r in rows:
for s in (r["signals_at_fail"] or "").split(";"):
if s:
signals[s] = signals.get(s, 0) + 1
flags = sorted({f for r in rows for f in (r["flags"] or "").split(";") if f})
api_text = f"{statistics.median(api_us):g}" if api_us else "—"
db_text = f"{statistics.median(db_us):g}" if db_us else "—"
signal_text = ", ".join(f"{k} ×{v}" for k, v in sorted(signals.items())) or "—"
if flags:
signal_text += f" ⚠ {', '.join(flags)}"
lines.append(
f"| {config} | {replicas} × {_f(rows[0]['api_cpus']):g} | "
f"{_f(rows[0]['db_cpus']):g} | {median:g} ({min(values):g}–{max(values):g}) | "
f"{efficiency} | {' / '.join(fixed)} | {api_text} | {db_text} | {signal_text} |"
)
lines += [
"",
"Признаки (пороги — `THRESHOLDS` в `bench/analysis.py`, заданы до замеров): "
"`app` — CPU реплики у лимита или троттлинг; `db_cpu` — CPU PostgreSQL у "
"лимита; `db_lock` — ожидание блокировок; `pool` — соединений реплики у "
"предела пула; `churn` — переоткрытие соединений (новые сессии PostgreSQL "
"за окно); `lb` — CPU HAProxy; `host` — ВМ Docker занята; `generator` — "
"генератор у своего лимита или отстаёт от расписания (R* — нижняя граница).",
"",
"Файлы: `runs.csv` — строка на ступень, `searches.csv` — строка на ячейку, "
"`samples.csv` — посекундно по контейнерам в окнах ступеней, "
"`p95-by-rate.svg`, `cpu-by-component.svg`, `raw/` — итоги генератора.",
]
lines += _warning_lines(meta)
return "\n".join(lines) + "\n"
# --- CLI ------------------------------------------------------------------------------
def main() -> None:
root = argparse.ArgumentParser(description=__doc__.splitlines()[0])
root.add_argument(
"profile",
choices=("failover", "failover-quick", "rollout", "rollout-quick", "scaling"),
)
root.add_argument("--label")
root.add_argument("--series", choices=sorted(SERIES))
# ячейки серии вместо профильных: у ab — «до/после»; у остальных — только
# если конфигурация не помещается в ВМ Docker (пишется в паспорт метки)
root.add_argument("--configs", help="ячейки серии: '4x1-pg0.5 4x1'")
root.add_argument("--resume", action="store_true")
args = root.parse_args()
with bench_lock():
if args.profile == "scaling":
if not args.series:
raise SystemExit("scaling needs --series")
profile = SERIES[args.series]
if args.configs:
profile = dataclasses.replace(
profile, configs=tuple(args.configs.split())
)
quick = args.series == "quick"
label = args.label or (
quick_label("scaling") if quick else f"scaling-{args.series}"
)
run_scaling(profile, label, args.resume)
return
profile = PROFILES[args.profile]
kind = args.profile.removesuffix("-quick")
if args.profile.endswith("quick"):
label = args.label or quick_label(kind)
else:
label = args.label or "baseline"
if kind == "failover":
run_failover(profile, label)
else:
run_rollout(profile, label)
if __name__ == "__main__":
main()