- carry the lab 3 solution over from mtusi2026lab3-safronov (solution/bvt2301-safronov, ce89eba): src, migrations, student-tests, bench, openspec, .claude, infra/haproxy and infra/valkey, own scripts, Dockerfile, compose, pyproject and uv lock - merge Makefile, .gitignore and .dockerignore (the lab 3 versions only add lines to the skeleton, which is the same in lab 4) and .env.example (lab 4 skeleton names listed in the header) - rename the compose project to mtusi-lab4 and the image to :lab4 so the stand does not reuse the lab 3 containers and volumes - update AGENTS.md for lab 4: current lab, origin commit, contract and test files of the new skeleton Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
607 lines
22 KiB
Python
607 lines
22 KiB
Python
"""Профиль отказа реплик: разбор и расчёт без docker.
|
||
|
||
Оркестрирует фазы `run.py` (профили `failover`, `failover_quick`), здесь —
|
||
чистые функции на stdlib: строки k6 о каждом запросе, статистика и
|
||
журнал HAProxy, события Docker, посекундная шкала, величины фазы и
|
||
`summary.md`. Определения величин — решение 5 design change
|
||
`graceful-failover`. Все отметки времени — часы ВМ Docker: `Date.now()` в
|
||
k6, отметки `docker logs -t` и `TimeNano` событий Docker.
|
||
"""
|
||
|
||
import csv
|
||
import io
|
||
import math
|
||
import re
|
||
from dataclasses import dataclass
|
||
from datetime import UTC, datetime
|
||
from typing import Any
|
||
|
||
import resources as res
|
||
|
||
# худшее время обнаружения по проверкам LB, с (infra/haproxy/haproxy.cfg:
|
||
# inter 1s, fastinter 500ms, fall 2, timeout check 2500ms, timeout connect
|
||
# 500ms): адрес пропал — inter + 0,5 + fastinter + 0,5; зависшая —
|
||
# inter + timeout check + fastinter + timeout check
|
||
BOUNDS = {"stop": 2.5, "kill": 2.5, "pause": 6.5}
|
||
# действие и возврат фазы — события Docker
|
||
ACTIONS = {"stop": "kill", "kill": "kill", "pause": "pause"}
|
||
RESTORES = {"stop": "start", "kill": "start", "pause": "unpause"}
|
||
|
||
PHASE_FIELDS = [
|
||
"phase",
|
||
"mode",
|
||
"replica",
|
||
"instance",
|
||
"slot",
|
||
"action_at",
|
||
"signal",
|
||
"lb_out_s",
|
||
"lb_out_state",
|
||
"lb_out_reason",
|
||
"bound_s",
|
||
"first_write_ok_s",
|
||
"first_read_ok_s",
|
||
"last_error_s",
|
||
"requests",
|
||
"errors_before",
|
||
"errors_down",
|
||
"codes_down",
|
||
"errors_after",
|
||
"codes_after",
|
||
"min_success",
|
||
"recovery_s",
|
||
"ok_rps_before",
|
||
"ok_rps_down_min",
|
||
"p95_before_ms",
|
||
"p95_down_max_ms",
|
||
"restore_at",
|
||
"lb_up_s",
|
||
"slot_back",
|
||
"instance_back_s",
|
||
"writes",
|
||
"writes_failed",
|
||
"retried_ok",
|
||
"write_chat_messages",
|
||
"distinct_texts",
|
||
"pg_idle_tx_terminated",
|
||
"dropped_iterations",
|
||
"flags",
|
||
]
|
||
TIMELINE_FIELDS = [
|
||
"phase",
|
||
"mode",
|
||
"replica",
|
||
"second",
|
||
"writes_ok",
|
||
"writes_failed",
|
||
"reads_ok",
|
||
"reads_failed",
|
||
"codes",
|
||
"success",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"instances",
|
||
]
|
||
|
||
# без привязки к началу и концу строки: две строки, склеенные без перевода
|
||
# строки, всё равно разбираются; число строк run.py сверяет с итогом k6
|
||
LINE_RE = re.compile(
|
||
r"fo,(?P<kind>[wr]),(?P<n>\d+),(?P<start>\d{13}),(?P<dur>-?\d+),"
|
||
r"(?P<status>-?\d+),(?P<instance>[A-Za-z0-9._-]*),(?P<ok>[01])",
|
||
)
|
||
|
||
|
||
def unparsed(text: str) -> list[str]:
|
||
"""Строки с `fo,`, которые разбор не взял: в сообщение об отказе."""
|
||
return [
|
||
line
|
||
for line in text.splitlines()
|
||
if "fo," in line and not LINE_RE.search(line)
|
||
]
|
||
|
||
|
||
LB_RE = re.compile(
|
||
r"^(?P<stamp>\S+) .*?Server api/(?P<slot>api-\d+) (?P<rest>.*)$",
|
||
)
|
||
IDLE_TX = "terminating connection due to idle-in-transaction timeout"
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Request:
|
||
"""Один запрос фазы: строка `fo,…` вывода k6."""
|
||
|
||
kind: str
|
||
n: int
|
||
start: float
|
||
duration_ms: int
|
||
status: int
|
||
instance: str
|
||
ok: bool
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class LbEvent:
|
||
"""Смена состояния слота в журнале HAProxy."""
|
||
|
||
time: float
|
||
slot: str
|
||
state: str
|
||
reason: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class DockerEvent:
|
||
time: float
|
||
action: str
|
||
signal: str
|
||
|
||
|
||
def parse_requests(text: str) -> list[Request]:
|
||
return [
|
||
Request(
|
||
kind=match["kind"],
|
||
n=int(match["n"]),
|
||
start=int(match["start"]) / 1000,
|
||
duration_ms=max(0, int(match["dur"])),
|
||
status=int(match["status"]),
|
||
instance=match["instance"],
|
||
ok=match["ok"] == "1",
|
||
)
|
||
for match in LINE_RE.finditer(text)
|
||
]
|
||
|
||
|
||
def parse_stamp(stamp: str) -> float:
|
||
"""Отметка `docker logs -t` (RFC 3339, наносекунды) → секунды эпохи."""
|
||
head, _, frac = stamp.rstrip("Z").partition(".")
|
||
seconds = datetime.fromisoformat(head).replace(tzinfo=UTC).timestamp()
|
||
return seconds + (float(f"0.{frac}") if frac else 0.0)
|
||
|
||
|
||
def iso(seconds: float) -> str:
|
||
return datetime.fromtimestamp(seconds, UTC).isoformat(
|
||
timespec="milliseconds",
|
||
)
|
||
|
||
|
||
def parse_lb_stats(text: str) -> list[dict[str, str]]:
|
||
"""CSV статистики HAProxy → занятые слоты backend `api`."""
|
||
lines = [
|
||
line for line in text.splitlines() if line.startswith(("# ", "api,"))
|
||
]
|
||
if not lines:
|
||
return []
|
||
reader = csv.DictReader(io.StringIO("\n".join(lines).removeprefix("# ")))
|
||
return [
|
||
{
|
||
"slot": row["svname"],
|
||
"addr": row.get("addr") or "",
|
||
"status": row["status"],
|
||
"check_status": row.get("check_status") or "",
|
||
}
|
||
for row in reader
|
||
if row["pxname"] == "api"
|
||
and row["svname"] not in ("BACKEND", "FRONTEND")
|
||
and row.get("addr")
|
||
]
|
||
|
||
|
||
def parse_lb_log(text: str) -> list[LbEvent]:
|
||
"""Строки смены состояния слотов из журнала LB (`docker logs -t`).
|
||
|
||
HAProxy пишет каждое событие дважды (в журнал и предупреждением):
|
||
повтор с тем же слотом и состоянием в пределах миллисекунды
|
||
отбрасывается.
|
||
"""
|
||
events: list[LbEvent] = []
|
||
for line in text.splitlines():
|
||
match = LB_RE.match(line)
|
||
if not match:
|
||
continue
|
||
rest = match["rest"]
|
||
if "maintenance" in rest:
|
||
state = "MAINT"
|
||
elif rest.startswith("is DOWN,") or rest == "is DOWN.":
|
||
state = "DOWN"
|
||
elif rest.startswith("is UP"):
|
||
state = "UP"
|
||
else:
|
||
continue
|
||
reason = re.search(r"reason: ([^,]+)", rest)
|
||
info = re.search(r"maintenance \(([^)]*)\)", rest)
|
||
event = LbEvent(
|
||
time=parse_stamp(match["stamp"]),
|
||
slot=match["slot"],
|
||
state=state,
|
||
reason=(
|
||
reason.group(1)
|
||
if reason
|
||
else (info.group(1).strip() if info else "")
|
||
),
|
||
)
|
||
if any(
|
||
e.slot == event.slot
|
||
and e.state == event.state
|
||
and abs(e.time - event.time) < 0.001
|
||
for e in events
|
||
):
|
||
continue
|
||
events.append(event)
|
||
return sorted(events, key=lambda e: e.time)
|
||
|
||
|
||
def lb_lines(text: str) -> str:
|
||
"""Строки журнала LB о состоянии слотов — для `raw/<фаза>-lb.log`."""
|
||
return "\n".join(
|
||
line
|
||
for line in text.splitlines()
|
||
if "Server api/" in line and "[WARNING]" not in line
|
||
)
|
||
|
||
|
||
def parse_events(text: str) -> list[DockerEvent]:
|
||
events = []
|
||
for line in text.splitlines():
|
||
parts = line.split()
|
||
if len(parts) < 2 or not parts[0].isdigit():
|
||
continue
|
||
signal = parts[2] if len(parts) > 2 and parts[2].isdigit() else ""
|
||
events.append(DockerEvent(int(parts[0]) / 1e9, parts[1], signal))
|
||
return sorted(events, key=lambda e: e.time)
|
||
|
||
|
||
def count_idle_terminations(text: str) -> int:
|
||
"""Сколько соединений PostgreSQL прервал по простою в транзакции."""
|
||
return text.count(IDLE_TX)
|
||
|
||
|
||
def percentile(values: list[int], share: float) -> int | None:
|
||
"""Ближайший ранг: p95 из 100 значений — 95-е по возрастанию."""
|
||
if not values:
|
||
return None
|
||
ordered = sorted(values)
|
||
return ordered[max(0, math.ceil(share * len(ordered)) - 1)]
|
||
|
||
|
||
def code(request: Request) -> str:
|
||
"""Код ошибки: статус, 0 — ошибка соединения или таймаут клиента,
|
||
`200-short` — чтение без полной страницы."""
|
||
if request.status == 200 and not request.ok:
|
||
return "200-short"
|
||
return str(request.status)
|
||
|
||
|
||
def codes(requests: list[Request]) -> str:
|
||
counts: dict[str, int] = {}
|
||
for request in requests:
|
||
if not request.ok:
|
||
counts[code(request)] = counts.get(code(request), 0) + 1
|
||
return " ".join(f"{key}:{value}" for key, value in sorted(counts.items()))
|
||
|
||
|
||
def build_timeline(
|
||
requests: list[Request],
|
||
t0: float,
|
||
*,
|
||
phase: str,
|
||
mode: str,
|
||
replica: int,
|
||
) -> list[dict[str, Any]]:
|
||
"""Посекундная шкала от действия: секунда запроса — по отправке."""
|
||
buckets: dict[int, list[Request]] = {}
|
||
for request in requests:
|
||
buckets.setdefault(math.floor(request.start - t0), []).append(request)
|
||
rows = []
|
||
for second in range(min(buckets, default=0), max(buckets, default=-1) + 1):
|
||
group = buckets.get(second, [])
|
||
ok = [r for r in group if r.ok]
|
||
instances: dict[str, int] = {}
|
||
for request in ok:
|
||
instances[request.instance] = (
|
||
instances.get(request.instance, 0) + 1
|
||
)
|
||
durations = [r.duration_ms for r in ok]
|
||
rows.append(
|
||
{
|
||
"phase": phase,
|
||
"mode": mode,
|
||
"replica": replica,
|
||
"second": second,
|
||
"writes_ok": sum(1 for r in ok if r.kind == "w"),
|
||
"writes_failed": sum(
|
||
1 for r in group if r.kind == "w" and not r.ok
|
||
),
|
||
"reads_ok": sum(1 for r in ok if r.kind == "r"),
|
||
"reads_failed": sum(
|
||
1 for r in group if r.kind == "r" and not r.ok
|
||
),
|
||
"codes": codes(group),
|
||
"success": round(len(ok) / len(group), 4) if group else None,
|
||
"p50_ms": percentile(durations, 0.5),
|
||
"p95_ms": percentile(durations, 0.95),
|
||
"instances": " ".join(
|
||
f"{key}:{value}"
|
||
for key, value in sorted(instances.items())
|
||
),
|
||
},
|
||
)
|
||
return rows
|
||
|
||
|
||
def first_after(
|
||
events: list[LbEvent],
|
||
slot: str | None,
|
||
states: tuple[str, ...],
|
||
since: float,
|
||
) -> LbEvent | None:
|
||
return next(
|
||
(
|
||
e
|
||
for e in events
|
||
if e.slot == slot and e.state in states and e.time >= since
|
||
),
|
||
None,
|
||
)
|
||
|
||
|
||
def rounded(value: float | None, digits: int = 2) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def analyze(
|
||
requests: list[Request],
|
||
timeline: list[dict[str, Any]],
|
||
lb_events: list[LbEvent],
|
||
*,
|
||
mode: str,
|
||
instance: str,
|
||
t0: float,
|
||
restore: float,
|
||
slot: str | None,
|
||
slot_back: str | None,
|
||
threshold: float,
|
||
) -> dict[str, Any]:
|
||
"""Величины фазы (решение 5): секунды от действия и от возврата.
|
||
|
||
До действия — запросы, завершённые до него. Запрос в полёте в момент
|
||
действия (на замороженной или убитой реплике) относится к окну
|
||
отказа, хотя в шкале он — в секунде отправки.
|
||
"""
|
||
|
||
def ended(request: Request) -> float:
|
||
return request.start + request.duration_ms / 1000
|
||
|
||
before = [r for r in requests if ended(r) < t0]
|
||
down = [r for r in requests if ended(r) >= t0 and r.start < restore]
|
||
after = [r for r in requests if r.start >= restore]
|
||
window = math.ceil(restore - t0)
|
||
down_rows = [row for row in timeline if 0 <= row["second"] < window]
|
||
before_rows = [row for row in timeline if row["second"] < 0][2:]
|
||
bad = [
|
||
row["second"]
|
||
for row in down_rows
|
||
if row["success"] is not None and row["success"] < threshold
|
||
]
|
||
out = first_after(lb_events, slot, ("DOWN", "MAINT"), t0)
|
||
up = first_after(lb_events, slot_back, ("UP",), restore)
|
||
back = next(
|
||
(
|
||
r
|
||
for r in sorted(after, key=lambda r: r.start)
|
||
if r.ok and r.instance == instance
|
||
),
|
||
None,
|
||
)
|
||
|
||
def first_ok(kind: str) -> float | None:
|
||
starts = [
|
||
r.start for r in down if r.kind == kind and r.ok and r.start >= t0
|
||
]
|
||
return rounded(min(starts) - t0) if starts else None
|
||
|
||
failed_down = [r for r in down if not r.ok]
|
||
p95_down = [
|
||
row["p95_ms"] for row in down_rows if row["p95_ms"] is not None
|
||
]
|
||
lb_out_s = rounded(out.time - t0) if out else None
|
||
flags = []
|
||
if out is None:
|
||
flags.append("no_lb_out")
|
||
elif lb_out_s is not None and lb_out_s > BOUNDS[mode]:
|
||
flags.append("over_bound")
|
||
if up is None:
|
||
flags.append("no_lb_up")
|
||
if [r for r in before if not r.ok]:
|
||
flags.append("errors_before")
|
||
return {
|
||
"lb_out_s": lb_out_s,
|
||
"lb_out_state": out.state if out else None,
|
||
"lb_out_reason": out.reason if out else None,
|
||
"bound_s": BOUNDS[mode],
|
||
"first_write_ok_s": first_ok("w"),
|
||
"first_read_ok_s": first_ok("r"),
|
||
"last_error_s": (
|
||
rounded(max(r.start for r in failed_down) - t0)
|
||
if failed_down
|
||
else None
|
||
),
|
||
"requests": len(requests),
|
||
"errors_before": sum(1 for r in before if not r.ok),
|
||
"errors_down": len(failed_down),
|
||
"codes_down": codes(down),
|
||
"errors_after": sum(1 for r in after if not r.ok),
|
||
"codes_after": codes(after),
|
||
"min_success": min(
|
||
(
|
||
row["success"]
|
||
for row in down_rows
|
||
if row["success"] is not None
|
||
),
|
||
default=None,
|
||
),
|
||
"recovery_s": max(bad) + 1 if bad else 0,
|
||
"ok_rps_before": (
|
||
round(
|
||
sum(row["writes_ok"] + row["reads_ok"] for row in before_rows)
|
||
/ len(before_rows),
|
||
1,
|
||
)
|
||
if before_rows
|
||
else None
|
||
),
|
||
"ok_rps_down_min": min(
|
||
(row["writes_ok"] + row["reads_ok"] for row in down_rows),
|
||
default=None,
|
||
),
|
||
"p95_before_ms": percentile(
|
||
[r.duration_ms for r in before if r.ok],
|
||
0.95,
|
||
),
|
||
"p95_down_max_ms": max(p95_down, default=None),
|
||
"lb_up_s": rounded(up.time - restore) if up else None,
|
||
"instance_back_s": rounded(back.start - restore) if back else None,
|
||
"writes": sum(1 for r in requests if r.kind == "w"),
|
||
"writes_failed": sum(
|
||
1 for r in requests if r.kind == "w" and not r.ok
|
||
),
|
||
"flags": " ".join(flags),
|
||
}
|
||
|
||
|
||
def fmt(value: Any) -> str:
|
||
if value is None or value == "":
|
||
return "—"
|
||
if isinstance(value, float):
|
||
return f"{value:g}"
|
||
return str(value)
|
||
|
||
|
||
def limits_line(profile: dict[str, Any]) -> str:
|
||
"""Строка лимитов шапки сводки; с сэмплером — и его лимит, и k6 без
|
||
лимита (design change `profile-resources`, решение 7)."""
|
||
line = (
|
||
f"- Лимиты без swap: API {profile['cpus']} CPU / "
|
||
f"{profile['memory']} на реплику, HAProxy {profile['lb_cpus']} CPU / "
|
||
f"{profile['lb_memory']}, PostgreSQL {profile['db_cpus']} CPU / "
|
||
f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / "
|
||
f"{profile['cache_memory']}"
|
||
)
|
||
if profile.get("sample_resources"):
|
||
line += (
|
||
f", сэмплер {profile['sampler_cpus']} CPU / "
|
||
f"{profile['sampler_memory']}, k6 без лимита"
|
||
)
|
||
return line
|
||
|
||
|
||
def render_summary(
|
||
meta: dict[str, Any],
|
||
phases: list[dict[str, Any]],
|
||
timeline: list[dict[str, Any]],
|
||
resources: list[dict[str, Any]] | None = None,
|
||
) -> str:
|
||
"""`summary.md` профиля отказа: условия, таблица фаз, шкалы вокруг
|
||
действия и возврата; с `resources` (строки `resources.csv`) — ещё
|
||
ресурсы по фазам и окнам."""
|
||
profile = meta["profile"]
|
||
lines = [
|
||
f"# Отказ реплик под нагрузкой: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{meta['status']}**"
|
||
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
|
||
f"- Commit: `{meta['git']['commit']}`"
|
||
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
|
||
f"- Профиль `{profile['name']}`: {profile['replicas']} реплики за "
|
||
f"HAProxy; {profile['failover_write_rate']} записей с ключом и "
|
||
f"{profile['failover_read_rate']} чтений страницы истории в "
|
||
"секунду (k6, постоянная интенсивность); режимы "
|
||
f"{', '.join(profile['failover_modes'])}; окна фазы: "
|
||
f"{profile['failover_before_s']} с до действия, "
|
||
f"{profile['failover_down_s']} с до возврата, "
|
||
f"{profile['failover_after_s']} с после; порог восстановления "
|
||
f"{profile['failover_success']:.0%}",
|
||
limits_line(profile),
|
||
"- Посторонние контейнеры в VM во время замера: "
|
||
+ (", ".join(meta.get("other_containers") or []) or "нет"),
|
||
"",
|
||
"Время — секунды на часах ВМ Docker. Действие — событие Docker "
|
||
"(`kill` с сигналом 15 у `docker stop`, 9 у `docker kill`, "
|
||
"`pause`), исключение и возврат — строки журнала HAProxy, ответы — "
|
||
"отметки k6 при отправке. Граница — худший случай формулы проверок "
|
||
"LB для режима. Восстановление — конец последней секунды окна "
|
||
"отказа с долей успешных ниже порога (0 — таких секунд нет). "
|
||
"Сверка — после повтора неуспешных записей с тем же ключом: "
|
||
"сообщений в чате записи и разных текстов столько, сколько "
|
||
"отправлено разных ключей.",
|
||
"",
|
||
"| Фаза | Реплика | Исключение LB, с (граница) | Причина | "
|
||
"Первая запись / чтение, с | Ошибки в окне отказа | Мин. доля "
|
||
"успешных | Восстановление, с | Возврат в LB, с | Ошибки после "
|
||
"возврата | Повтор записей | Сообщений = ключей | PG: прервано "
|
||
"простоев | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for row in phases:
|
||
lines.append(
|
||
f"| {row['phase']} | {row['instance']} ({row['slot']}) | "
|
||
f"{fmt(row['lb_out_s'])} ({fmt(row['bound_s'])}) | "
|
||
f"{fmt(row['lb_out_state'])}: {fmt(row['lb_out_reason'])} | "
|
||
f"{fmt(row['first_write_ok_s'])} / {fmt(row['first_read_ok_s'])} "
|
||
f"| {row['errors_down']} {fmt(row['codes_down'])} | "
|
||
f"{fmt(row['min_success'])} | {row['recovery_s']} | "
|
||
f"{fmt(row['lb_up_s'])} | {row['errors_after']} | "
|
||
f"{row['writes_failed']} → {fmt(row['retried_ok'])} | "
|
||
f"{fmt(row['write_chat_messages'])} = "
|
||
f"{fmt(row['distinct_texts'])} | "
|
||
f"{fmt(row['pg_idle_tx_terminated'])} | {fmt(row['flags'])} |",
|
||
)
|
||
lines.append("")
|
||
if resources:
|
||
order = list(dict.fromkeys(r["phase"] for r in resources))
|
||
lines += res.render_failover(resources, order)
|
||
for row in phases:
|
||
rows = [r for r in timeline if r["phase"] == row["phase"]]
|
||
reach = max(
|
||
row["recovery_s"] or 0,
|
||
math.ceil(row["lb_out_s"] or 0),
|
||
)
|
||
down_s = profile["failover_down_s"]
|
||
shown = [
|
||
r
|
||
for r in rows
|
||
if -2 <= r["second"] <= reach + 2
|
||
or down_s - 1 <= r["second"] <= down_s + 3
|
||
]
|
||
lines += [
|
||
f"## Фаза {row['phase']}",
|
||
"",
|
||
f"Действие {row['action_at']} над `{row['instance']}` (слот "
|
||
f"{row['slot']}), возврат {row['restore_at']}; ответов "
|
||
f"{row['requests']}, до действия {fmt(row['ok_rps_before'])} "
|
||
f"успешных/с, p95 {fmt(row['p95_before_ms'])} мс. Полная шкала "
|
||
"— `timeline.csv`.",
|
||
"",
|
||
"| с от действия | Записи ok / ош | Чтения ok / ош | Коды ошибок "
|
||
"| Доля успешных | p50 / p95, мс | Ответы реплик |",
|
||
"|---|---|---|---|---|---|---|",
|
||
]
|
||
previous = None
|
||
for r in shown:
|
||
if previous is not None and r["second"] != previous + 1:
|
||
lines.append("| … | | | | | | |")
|
||
previous = r["second"]
|
||
lines.append(
|
||
f"| {r['second']} | {r['writes_ok']} / {r['writes_failed']} | "
|
||
f"{r['reads_ok']} / {r['reads_failed']} | {fmt(r['codes'])} | "
|
||
f"{fmt(r['success'])} | {fmt(r['p50_ms'])} / "
|
||
f"{fmt(r['p95_ms'])} | {fmt(r['instances'])} |",
|
||
)
|
||
lines.append("")
|
||
if resources:
|
||
lines += res.render_failover_phase(resources, row["phase"])
|
||
if meta.get("host_warnings"):
|
||
lines += ["## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
lines.append("")
|
||
return "\n".join(lines)
|