Files
mtusi2026lab5/bench/failover.py
T
ricoromaandClaude Opus 5.5 04f859282d chore: port lab 4 solution into the lab 5 skeleton
Source: mtusi2026lab4-safronov, solution/bvt2301-safronov, 2546deb.
Contracts, public tests and evidence of lab 4 are not carried over.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 03:09:12 +03:00

607 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль отказа реплик: разбор и расчёт без docker.
Оркестрирует фазы `run.py` (профили `failover`, `failover_quick`), здесь —
чистые функции на stdlib: строки k6 о каждом запросе, статистика и
журнал HAProxy, события Docker, посекундная шкала, величины фазы и
`summary.md`. Определения величин — решение 5 design change
`graceful-failover`. Все отметки времени — часы ВМ Docker: `Date.now()` в
k6, отметки `docker logs -t` и `TimeNano` событий Docker.
"""
import csv
import io
import math
import re
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
import resources as res
# худшее время обнаружения по проверкам LB, с (infra/haproxy/haproxy.cfg:
# inter 1s, fastinter 500ms, fall 2, timeout check 2500ms, timeout connect
# 500ms): адрес пропал — inter + 0,5 + fastinter + 0,5; зависшая —
# inter + timeout check + fastinter + timeout check
BOUNDS = {"stop": 2.5, "kill": 2.5, "pause": 6.5}
# действие и возврат фазы — события Docker
ACTIONS = {"stop": "kill", "kill": "kill", "pause": "pause"}
RESTORES = {"stop": "start", "kill": "start", "pause": "unpause"}
PHASE_FIELDS = [
"phase",
"mode",
"replica",
"instance",
"slot",
"action_at",
"signal",
"lb_out_s",
"lb_out_state",
"lb_out_reason",
"bound_s",
"first_write_ok_s",
"first_read_ok_s",
"last_error_s",
"requests",
"errors_before",
"errors_down",
"codes_down",
"errors_after",
"codes_after",
"min_success",
"recovery_s",
"ok_rps_before",
"ok_rps_down_min",
"p95_before_ms",
"p95_down_max_ms",
"restore_at",
"lb_up_s",
"slot_back",
"instance_back_s",
"writes",
"writes_failed",
"retried_ok",
"write_chat_messages",
"distinct_texts",
"pg_idle_tx_terminated",
"dropped_iterations",
"flags",
]
TIMELINE_FIELDS = [
"phase",
"mode",
"replica",
"second",
"writes_ok",
"writes_failed",
"reads_ok",
"reads_failed",
"codes",
"success",
"p50_ms",
"p95_ms",
"instances",
]
# без привязки к началу и концу строки: две строки, склеенные без перевода
# строки, всё равно разбираются; число строк run.py сверяет с итогом k6
LINE_RE = re.compile(
r"fo,(?P<kind>[wr]),(?P<n>\d+),(?P<start>\d{13}),(?P<dur>-?\d+),"
r"(?P<status>-?\d+),(?P<instance>[A-Za-z0-9._-]*),(?P<ok>[01])",
)
def unparsed(text: str) -> list[str]:
"""Строки с `fo,`, которые разбор не взял: в сообщение об отказе."""
return [
line
for line in text.splitlines()
if "fo," in line and not LINE_RE.search(line)
]
LB_RE = re.compile(
r"^(?P<stamp>\S+) .*?Server api/(?P<slot>api-\d+) (?P<rest>.*)$",
)
IDLE_TX = "terminating connection due to idle-in-transaction timeout"
@dataclass(frozen=True)
class Request:
"""Один запрос фазы: строка `fo,…` вывода k6."""
kind: str
n: int
start: float
duration_ms: int
status: int
instance: str
ok: bool
@dataclass(frozen=True)
class LbEvent:
"""Смена состояния слота в журнале HAProxy."""
time: float
slot: str
state: str
reason: str
@dataclass(frozen=True)
class DockerEvent:
time: float
action: str
signal: str
def parse_requests(text: str) -> list[Request]:
return [
Request(
kind=match["kind"],
n=int(match["n"]),
start=int(match["start"]) / 1000,
duration_ms=max(0, int(match["dur"])),
status=int(match["status"]),
instance=match["instance"],
ok=match["ok"] == "1",
)
for match in LINE_RE.finditer(text)
]
def parse_stamp(stamp: str) -> float:
"""Отметка `docker logs -t` (RFC 3339, наносекунды) → секунды эпохи."""
head, _, frac = stamp.rstrip("Z").partition(".")
seconds = datetime.fromisoformat(head).replace(tzinfo=UTC).timestamp()
return seconds + (float(f"0.{frac}") if frac else 0.0)
def iso(seconds: float) -> str:
return datetime.fromtimestamp(seconds, UTC).isoformat(
timespec="milliseconds",
)
def parse_lb_stats(text: str) -> list[dict[str, str]]:
"""CSV статистики HAProxy → занятые слоты backend `api`."""
lines = [
line for line in text.splitlines() if line.startswith(("# ", "api,"))
]
if not lines:
return []
reader = csv.DictReader(io.StringIO("\n".join(lines).removeprefix("# ")))
return [
{
"slot": row["svname"],
"addr": row.get("addr") or "",
"status": row["status"],
"check_status": row.get("check_status") or "",
}
for row in reader
if row["pxname"] == "api"
and row["svname"] not in ("BACKEND", "FRONTEND")
and row.get("addr")
]
def parse_lb_log(text: str) -> list[LbEvent]:
"""Строки смены состояния слотов из журнала LB (`docker logs -t`).
HAProxy пишет каждое событие дважды (в журнал и предупреждением):
повтор с тем же слотом и состоянием в пределах миллисекунды
отбрасывается.
"""
events: list[LbEvent] = []
for line in text.splitlines():
match = LB_RE.match(line)
if not match:
continue
rest = match["rest"]
if "maintenance" in rest:
state = "MAINT"
elif rest.startswith("is DOWN,") or rest == "is DOWN.":
state = "DOWN"
elif rest.startswith("is UP"):
state = "UP"
else:
continue
reason = re.search(r"reason: ([^,]+)", rest)
info = re.search(r"maintenance \(([^)]*)\)", rest)
event = LbEvent(
time=parse_stamp(match["stamp"]),
slot=match["slot"],
state=state,
reason=(
reason.group(1)
if reason
else (info.group(1).strip() if info else "")
),
)
if any(
e.slot == event.slot
and e.state == event.state
and abs(e.time - event.time) < 0.001
for e in events
):
continue
events.append(event)
return sorted(events, key=lambda e: e.time)
def lb_lines(text: str) -> str:
"""Строки журнала LB о состоянии слотов — для `raw/<фаза>-lb.log`."""
return "\n".join(
line
for line in text.splitlines()
if "Server api/" in line and "[WARNING]" not in line
)
def parse_events(text: str) -> list[DockerEvent]:
events = []
for line in text.splitlines():
parts = line.split()
if len(parts) < 2 or not parts[0].isdigit():
continue
signal = parts[2] if len(parts) > 2 and parts[2].isdigit() else ""
events.append(DockerEvent(int(parts[0]) / 1e9, parts[1], signal))
return sorted(events, key=lambda e: e.time)
def count_idle_terminations(text: str) -> int:
"""Сколько соединений PostgreSQL прервал по простою в транзакции."""
return text.count(IDLE_TX)
def percentile(values: list[int], share: float) -> int | None:
"""Ближайший ранг: p95 из 100 значений — 95-е по возрастанию."""
if not values:
return None
ordered = sorted(values)
return ordered[max(0, math.ceil(share * len(ordered)) - 1)]
def code(request: Request) -> str:
"""Код ошибки: статус, 0 — ошибка соединения или таймаут клиента,
`200-short` — чтение без полной страницы."""
if request.status == 200 and not request.ok:
return "200-short"
return str(request.status)
def codes(requests: list[Request]) -> str:
counts: dict[str, int] = {}
for request in requests:
if not request.ok:
counts[code(request)] = counts.get(code(request), 0) + 1
return " ".join(f"{key}:{value}" for key, value in sorted(counts.items()))
def build_timeline(
requests: list[Request],
t0: float,
*,
phase: str,
mode: str,
replica: int,
) -> list[dict[str, Any]]:
"""Посекундная шкала от действия: секунда запроса — по отправке."""
buckets: dict[int, list[Request]] = {}
for request in requests:
buckets.setdefault(math.floor(request.start - t0), []).append(request)
rows = []
for second in range(min(buckets, default=0), max(buckets, default=-1) + 1):
group = buckets.get(second, [])
ok = [r for r in group if r.ok]
instances: dict[str, int] = {}
for request in ok:
instances[request.instance] = (
instances.get(request.instance, 0) + 1
)
durations = [r.duration_ms for r in ok]
rows.append(
{
"phase": phase,
"mode": mode,
"replica": replica,
"second": second,
"writes_ok": sum(1 for r in ok if r.kind == "w"),
"writes_failed": sum(
1 for r in group if r.kind == "w" and not r.ok
),
"reads_ok": sum(1 for r in ok if r.kind == "r"),
"reads_failed": sum(
1 for r in group if r.kind == "r" and not r.ok
),
"codes": codes(group),
"success": round(len(ok) / len(group), 4) if group else None,
"p50_ms": percentile(durations, 0.5),
"p95_ms": percentile(durations, 0.95),
"instances": " ".join(
f"{key}:{value}"
for key, value in sorted(instances.items())
),
},
)
return rows
def first_after(
events: list[LbEvent],
slot: str | None,
states: tuple[str, ...],
since: float,
) -> LbEvent | None:
return next(
(
e
for e in events
if e.slot == slot and e.state in states and e.time >= since
),
None,
)
def rounded(value: float | None, digits: int = 2) -> float | None:
return None if value is None else round(value, digits)
def analyze(
requests: list[Request],
timeline: list[dict[str, Any]],
lb_events: list[LbEvent],
*,
mode: str,
instance: str,
t0: float,
restore: float,
slot: str | None,
slot_back: str | None,
threshold: float,
) -> dict[str, Any]:
"""Величины фазы (решение 5): секунды от действия и от возврата.
До действия — запросы, завершённые до него. Запрос в полёте в момент
действия (на замороженной или убитой реплике) относится к окну
отказа, хотя в шкале он — в секунде отправки.
"""
def ended(request: Request) -> float:
return request.start + request.duration_ms / 1000
before = [r for r in requests if ended(r) < t0]
down = [r for r in requests if ended(r) >= t0 and r.start < restore]
after = [r for r in requests if r.start >= restore]
window = math.ceil(restore - t0)
down_rows = [row for row in timeline if 0 <= row["second"] < window]
before_rows = [row for row in timeline if row["second"] < 0][2:]
bad = [
row["second"]
for row in down_rows
if row["success"] is not None and row["success"] < threshold
]
out = first_after(lb_events, slot, ("DOWN", "MAINT"), t0)
up = first_after(lb_events, slot_back, ("UP",), restore)
back = next(
(
r
for r in sorted(after, key=lambda r: r.start)
if r.ok and r.instance == instance
),
None,
)
def first_ok(kind: str) -> float | None:
starts = [
r.start for r in down if r.kind == kind and r.ok and r.start >= t0
]
return rounded(min(starts) - t0) if starts else None
failed_down = [r for r in down if not r.ok]
p95_down = [
row["p95_ms"] for row in down_rows if row["p95_ms"] is not None
]
lb_out_s = rounded(out.time - t0) if out else None
flags = []
if out is None:
flags.append("no_lb_out")
elif lb_out_s is not None and lb_out_s > BOUNDS[mode]:
flags.append("over_bound")
if up is None:
flags.append("no_lb_up")
if [r for r in before if not r.ok]:
flags.append("errors_before")
return {
"lb_out_s": lb_out_s,
"lb_out_state": out.state if out else None,
"lb_out_reason": out.reason if out else None,
"bound_s": BOUNDS[mode],
"first_write_ok_s": first_ok("w"),
"first_read_ok_s": first_ok("r"),
"last_error_s": (
rounded(max(r.start for r in failed_down) - t0)
if failed_down
else None
),
"requests": len(requests),
"errors_before": sum(1 for r in before if not r.ok),
"errors_down": len(failed_down),
"codes_down": codes(down),
"errors_after": sum(1 for r in after if not r.ok),
"codes_after": codes(after),
"min_success": min(
(
row["success"]
for row in down_rows
if row["success"] is not None
),
default=None,
),
"recovery_s": max(bad) + 1 if bad else 0,
"ok_rps_before": (
round(
sum(row["writes_ok"] + row["reads_ok"] for row in before_rows)
/ len(before_rows),
1,
)
if before_rows
else None
),
"ok_rps_down_min": min(
(row["writes_ok"] + row["reads_ok"] for row in down_rows),
default=None,
),
"p95_before_ms": percentile(
[r.duration_ms for r in before if r.ok],
0.95,
),
"p95_down_max_ms": max(p95_down, default=None),
"lb_up_s": rounded(up.time - restore) if up else None,
"instance_back_s": rounded(back.start - restore) if back else None,
"writes": sum(1 for r in requests if r.kind == "w"),
"writes_failed": sum(
1 for r in requests if r.kind == "w" and not r.ok
),
"flags": " ".join(flags),
}
def fmt(value: Any) -> str:
if value is None or value == "":
return "—"
if isinstance(value, float):
return f"{value:g}"
return str(value)
def limits_line(profile: dict[str, Any]) -> str:
"""Строка лимитов шапки сводки; с сэмплером — и его лимит, и k6 без
лимита (design change `profile-resources`, решение 7)."""
line = (
f"- Лимиты без swap: API {profile['cpus']} CPU / "
f"{profile['memory']} на реплику, HAProxy {profile['lb_cpus']} CPU / "
f"{profile['lb_memory']}, PostgreSQL {profile['db_cpus']} CPU / "
f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / "
f"{profile['cache_memory']}"
)
if profile.get("sample_resources"):
line += (
f", сэмплер {profile['sampler_cpus']} CPU / "
f"{profile['sampler_memory']}, k6 без лимита"
)
return line
def render_summary(
meta: dict[str, Any],
phases: list[dict[str, Any]],
timeline: list[dict[str, Any]],
resources: list[dict[str, Any]] | None = None,
) -> str:
"""`summary.md` профиля отказа: условия, таблица фаз, шкалы вокруг
действия и возврата; с `resources` (строки `resources.csv`) — ещё
ресурсы по фазам и окнам."""
profile = meta["profile"]
lines = [
f"# Отказ реплик под нагрузкой: метка `{meta['label']}`",
"",
f"- Статус: **{meta['status']}**"
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
f"- Commit: `{meta['git']['commit']}`"
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
f"- Профиль `{profile['name']}`: {profile['replicas']} реплики за "
f"HAProxy; {profile['failover_write_rate']} записей с ключом и "
f"{profile['failover_read_rate']} чтений страницы истории в "
"секунду (k6, постоянная интенсивность); режимы "
f"{', '.join(profile['failover_modes'])}; окна фазы: "
f"{profile['failover_before_s']} с до действия, "
f"{profile['failover_down_s']} с до возврата, "
f"{profile['failover_after_s']} с после; порог восстановления "
f"{profile['failover_success']:.0%}",
limits_line(profile),
"- Посторонние контейнеры в VM во время замера: "
+ (", ".join(meta.get("other_containers") or []) or "нет"),
"",
"Время — секунды на часах ВМ Docker. Действие — событие Docker "
"(`kill` с сигналом 15 у `docker stop`, 9 у `docker kill`, "
"`pause`), исключение и возврат — строки журнала HAProxy, ответы — "
"отметки k6 при отправке. Граница — худший случай формулы проверок "
"LB для режима. Восстановление — конец последней секунды окна "
"отказа с долей успешных ниже порога (0 — таких секунд нет). "
"Сверка — после повтора неуспешных записей с тем же ключом: "
"сообщений в чате записи и разных текстов столько, сколько "
"отправлено разных ключей.",
"",
"| Фаза | Реплика | Исключение LB, с (граница) | Причина | "
"Первая запись / чтение, с | Ошибки в окне отказа | Мин. доля "
"успешных | Восстановление, с | Возврат в LB, с | Ошибки после "
"возврата | Повтор записей | Сообщений = ключей | PG: прервано "
"простоев | Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|",
]
for row in phases:
lines.append(
f"| {row['phase']} | {row['instance']} ({row['slot']}) | "
f"{fmt(row['lb_out_s'])} ({fmt(row['bound_s'])}) | "
f"{fmt(row['lb_out_state'])}: {fmt(row['lb_out_reason'])} | "
f"{fmt(row['first_write_ok_s'])} / {fmt(row['first_read_ok_s'])} "
f"| {row['errors_down']} {fmt(row['codes_down'])} | "
f"{fmt(row['min_success'])} | {row['recovery_s']} | "
f"{fmt(row['lb_up_s'])} | {row['errors_after']} | "
f"{row['writes_failed']} → {fmt(row['retried_ok'])} | "
f"{fmt(row['write_chat_messages'])} = "
f"{fmt(row['distinct_texts'])} | "
f"{fmt(row['pg_idle_tx_terminated'])} | {fmt(row['flags'])} |",
)
lines.append("")
if resources:
order = list(dict.fromkeys(r["phase"] for r in resources))
lines += res.render_failover(resources, order)
for row in phases:
rows = [r for r in timeline if r["phase"] == row["phase"]]
reach = max(
row["recovery_s"] or 0,
math.ceil(row["lb_out_s"] or 0),
)
down_s = profile["failover_down_s"]
shown = [
r
for r in rows
if -2 <= r["second"] <= reach + 2
or down_s - 1 <= r["second"] <= down_s + 3
]
lines += [
f"## Фаза {row['phase']}",
"",
f"Действие {row['action_at']} над `{row['instance']}` (слот "
f"{row['slot']}), возврат {row['restore_at']}; ответов "
f"{row['requests']}, до действия {fmt(row['ok_rps_before'])} "
f"успешных/с, p95 {fmt(row['p95_before_ms'])} мс. Полная шкала "
"— `timeline.csv`.",
"",
"| с от действия | Записи ok / ош | Чтения ok / ош | Коды ошибок "
"| Доля успешных | p50 / p95, мс | Ответы реплик |",
"|---|---|---|---|---|---|---|",
]
previous = None
for r in shown:
if previous is not None and r["second"] != previous + 1:
lines.append("| … | | | | | | |")
previous = r["second"]
lines.append(
f"| {r['second']} | {r['writes_ok']} / {r['writes_failed']} | "
f"{r['reads_ok']} / {r['reads_failed']} | {fmt(r['codes'])} | "
f"{fmt(r['success'])} | {fmt(r['p50_ms'])} / "
f"{fmt(r['p95_ms'])} | {fmt(r['instances'])} |",
)
lines.append("")
if resources:
lines += res.render_failover_phase(resources, row["phase"])
if meta.get("host_warnings"):
lines += ["## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
lines.append("")
return "\n".join(lines)