"""Профиль отказа реплик: разбор и расчёт без docker. Оркестрирует фазы `run.py` (профили `failover`, `failover_quick`), здесь — чистые функции на stdlib: строки k6 о каждом запросе, статистика и журнал HAProxy, события Docker, посекундная шкала, величины фазы и `summary.md`. Определения величин — решение 5 design change `graceful-failover`. Все отметки времени — часы ВМ Docker: `Date.now()` в k6, отметки `docker logs -t` и `TimeNano` событий Docker. """ import csv import io import math import re from dataclasses import dataclass from datetime import UTC, datetime from typing import Any import resources as res # худшее время обнаружения по проверкам LB, с (infra/haproxy/haproxy.cfg: # inter 1s, fastinter 500ms, fall 2, timeout check 2500ms, timeout connect # 500ms): адрес пропал — inter + 0,5 + fastinter + 0,5; зависшая — # inter + timeout check + fastinter + timeout check BOUNDS = {"stop": 2.5, "kill": 2.5, "pause": 6.5} # действие и возврат фазы — события Docker ACTIONS = {"stop": "kill", "kill": "kill", "pause": "pause"} RESTORES = {"stop": "start", "kill": "start", "pause": "unpause"} PHASE_FIELDS = [ "phase", "mode", "replica", "instance", "slot", "action_at", "signal", "lb_out_s", "lb_out_state", "lb_out_reason", "bound_s", "first_write_ok_s", "first_read_ok_s", "last_error_s", "requests", "errors_before", "errors_down", "codes_down", "errors_after", "codes_after", "min_success", "recovery_s", "ok_rps_before", "ok_rps_down_min", "p95_before_ms", "p95_down_max_ms", "restore_at", "lb_up_s", "slot_back", "instance_back_s", "writes", "writes_failed", "retried_ok", "write_chat_messages", "distinct_texts", "pg_idle_tx_terminated", "dropped_iterations", "flags", ] TIMELINE_FIELDS = [ "phase", "mode", "replica", "second", "writes_ok", "writes_failed", "reads_ok", "reads_failed", "codes", "success", "p50_ms", "p95_ms", "instances", ] # без привязки к началу и концу строки: две строки, склеенные без перевода # строки, всё равно разбираются; число строк run.py сверяет с итогом k6 LINE_RE = re.compile( r"fo,(?P[wr]),(?P\d+),(?P\d{13}),(?P-?\d+)," r"(?P-?\d+),(?P[A-Za-z0-9._-]*),(?P[01])", ) def unparsed(text: str) -> list[str]: """Строки с `fo,`, которые разбор не взял: в сообщение об отказе.""" return [ line for line in text.splitlines() if "fo," in line and not LINE_RE.search(line) ] LB_RE = re.compile( r"^(?P\S+) .*?Server api/(?Papi-\d+) (?P.*)$", ) IDLE_TX = "terminating connection due to idle-in-transaction timeout" @dataclass(frozen=True) class Request: """Один запрос фазы: строка `fo,…` вывода k6.""" kind: str n: int start: float duration_ms: int status: int instance: str ok: bool @dataclass(frozen=True) class LbEvent: """Смена состояния слота в журнале HAProxy.""" time: float slot: str state: str reason: str @dataclass(frozen=True) class DockerEvent: time: float action: str signal: str def parse_requests(text: str) -> list[Request]: return [ Request( kind=match["kind"], n=int(match["n"]), start=int(match["start"]) / 1000, duration_ms=max(0, int(match["dur"])), status=int(match["status"]), instance=match["instance"], ok=match["ok"] == "1", ) for match in LINE_RE.finditer(text) ] def parse_stamp(stamp: str) -> float: """Отметка `docker logs -t` (RFC 3339, наносекунды) → секунды эпохи.""" head, _, frac = stamp.rstrip("Z").partition(".") seconds = datetime.fromisoformat(head).replace(tzinfo=UTC).timestamp() return seconds + (float(f"0.{frac}") if frac else 0.0) def iso(seconds: float) -> str: return datetime.fromtimestamp(seconds, UTC).isoformat( timespec="milliseconds", ) def parse_lb_stats(text: str) -> list[dict[str, str]]: """CSV статистики HAProxy → занятые слоты backend `api`.""" lines = [ line for line in text.splitlines() if line.startswith(("# ", "api,")) ] if not lines: return [] reader = csv.DictReader(io.StringIO("\n".join(lines).removeprefix("# "))) return [ { "slot": row["svname"], "addr": row.get("addr") or "", "status": row["status"], "check_status": row.get("check_status") or "", } for row in reader if row["pxname"] == "api" and row["svname"] not in ("BACKEND", "FRONTEND") and row.get("addr") ] def parse_lb_log(text: str) -> list[LbEvent]: """Строки смены состояния слотов из журнала LB (`docker logs -t`). HAProxy пишет каждое событие дважды (в журнал и предупреждением): повтор с тем же слотом и состоянием в пределах миллисекунды отбрасывается. """ events: list[LbEvent] = [] for line in text.splitlines(): match = LB_RE.match(line) if not match: continue rest = match["rest"] if "maintenance" in rest: state = "MAINT" elif rest.startswith("is DOWN,") or rest == "is DOWN.": state = "DOWN" elif rest.startswith("is UP"): state = "UP" else: continue reason = re.search(r"reason: ([^,]+)", rest) info = re.search(r"maintenance \(([^)]*)\)", rest) event = LbEvent( time=parse_stamp(match["stamp"]), slot=match["slot"], state=state, reason=( reason.group(1) if reason else (info.group(1).strip() if info else "") ), ) if any( e.slot == event.slot and e.state == event.state and abs(e.time - event.time) < 0.001 for e in events ): continue events.append(event) return sorted(events, key=lambda e: e.time) def lb_lines(text: str) -> str: """Строки журнала LB о состоянии слотов — для `raw/<фаза>-lb.log`.""" return "\n".join( line for line in text.splitlines() if "Server api/" in line and "[WARNING]" not in line ) def parse_events(text: str) -> list[DockerEvent]: events = [] for line in text.splitlines(): parts = line.split() if len(parts) < 2 or not parts[0].isdigit(): continue signal = parts[2] if len(parts) > 2 and parts[2].isdigit() else "" events.append(DockerEvent(int(parts[0]) / 1e9, parts[1], signal)) return sorted(events, key=lambda e: e.time) def count_idle_terminations(text: str) -> int: """Сколько соединений PostgreSQL прервал по простою в транзакции.""" return text.count(IDLE_TX) def percentile(values: list[int], share: float) -> int | None: """Ближайший ранг: p95 из 100 значений — 95-е по возрастанию.""" if not values: return None ordered = sorted(values) return ordered[max(0, math.ceil(share * len(ordered)) - 1)] def code(request: Request) -> str: """Код ошибки: статус, 0 — ошибка соединения или таймаут клиента, `200-short` — чтение без полной страницы.""" if request.status == 200 and not request.ok: return "200-short" return str(request.status) def codes(requests: list[Request]) -> str: counts: dict[str, int] = {} for request in requests: if not request.ok: counts[code(request)] = counts.get(code(request), 0) + 1 return " ".join(f"{key}:{value}" for key, value in sorted(counts.items())) def build_timeline( requests: list[Request], t0: float, *, phase: str, mode: str, replica: int, ) -> list[dict[str, Any]]: """Посекундная шкала от действия: секунда запроса — по отправке.""" buckets: dict[int, list[Request]] = {} for request in requests: buckets.setdefault(math.floor(request.start - t0), []).append(request) rows = [] for second in range(min(buckets, default=0), max(buckets, default=-1) + 1): group = buckets.get(second, []) ok = [r for r in group if r.ok] instances: dict[str, int] = {} for request in ok: instances[request.instance] = ( instances.get(request.instance, 0) + 1 ) durations = [r.duration_ms for r in ok] rows.append( { "phase": phase, "mode": mode, "replica": replica, "second": second, "writes_ok": sum(1 for r in ok if r.kind == "w"), "writes_failed": sum( 1 for r in group if r.kind == "w" and not r.ok ), "reads_ok": sum(1 for r in ok if r.kind == "r"), "reads_failed": sum( 1 for r in group if r.kind == "r" and not r.ok ), "codes": codes(group), "success": round(len(ok) / len(group), 4) if group else None, "p50_ms": percentile(durations, 0.5), "p95_ms": percentile(durations, 0.95), "instances": " ".join( f"{key}:{value}" for key, value in sorted(instances.items()) ), }, ) return rows def first_after( events: list[LbEvent], slot: str | None, states: tuple[str, ...], since: float, ) -> LbEvent | None: return next( ( e for e in events if e.slot == slot and e.state in states and e.time >= since ), None, ) def rounded(value: float | None, digits: int = 2) -> float | None: return None if value is None else round(value, digits) def analyze( requests: list[Request], timeline: list[dict[str, Any]], lb_events: list[LbEvent], *, mode: str, instance: str, t0: float, restore: float, slot: str | None, slot_back: str | None, threshold: float, ) -> dict[str, Any]: """Величины фазы (решение 5): секунды от действия и от возврата. До действия — запросы, завершённые до него. Запрос в полёте в момент действия (на замороженной или убитой реплике) относится к окну отказа, хотя в шкале он — в секунде отправки. """ def ended(request: Request) -> float: return request.start + request.duration_ms / 1000 before = [r for r in requests if ended(r) < t0] down = [r for r in requests if ended(r) >= t0 and r.start < restore] after = [r for r in requests if r.start >= restore] window = math.ceil(restore - t0) down_rows = [row for row in timeline if 0 <= row["second"] < window] before_rows = [row for row in timeline if row["second"] < 0][2:] bad = [ row["second"] for row in down_rows if row["success"] is not None and row["success"] < threshold ] out = first_after(lb_events, slot, ("DOWN", "MAINT"), t0) up = first_after(lb_events, slot_back, ("UP",), restore) back = next( ( r for r in sorted(after, key=lambda r: r.start) if r.ok and r.instance == instance ), None, ) def first_ok(kind: str) -> float | None: starts = [ r.start for r in down if r.kind == kind and r.ok and r.start >= t0 ] return rounded(min(starts) - t0) if starts else None failed_down = [r for r in down if not r.ok] p95_down = [ row["p95_ms"] for row in down_rows if row["p95_ms"] is not None ] lb_out_s = rounded(out.time - t0) if out else None flags = [] if out is None: flags.append("no_lb_out") elif lb_out_s is not None and lb_out_s > BOUNDS[mode]: flags.append("over_bound") if up is None: flags.append("no_lb_up") if [r for r in before if not r.ok]: flags.append("errors_before") return { "lb_out_s": lb_out_s, "lb_out_state": out.state if out else None, "lb_out_reason": out.reason if out else None, "bound_s": BOUNDS[mode], "first_write_ok_s": first_ok("w"), "first_read_ok_s": first_ok("r"), "last_error_s": ( rounded(max(r.start for r in failed_down) - t0) if failed_down else None ), "requests": len(requests), "errors_before": sum(1 for r in before if not r.ok), "errors_down": len(failed_down), "codes_down": codes(down), "errors_after": sum(1 for r in after if not r.ok), "codes_after": codes(after), "min_success": min( ( row["success"] for row in down_rows if row["success"] is not None ), default=None, ), "recovery_s": max(bad) + 1 if bad else 0, "ok_rps_before": ( round( sum(row["writes_ok"] + row["reads_ok"] for row in before_rows) / len(before_rows), 1, ) if before_rows else None ), "ok_rps_down_min": min( (row["writes_ok"] + row["reads_ok"] for row in down_rows), default=None, ), "p95_before_ms": percentile( [r.duration_ms for r in before if r.ok], 0.95, ), "p95_down_max_ms": max(p95_down, default=None), "lb_up_s": rounded(up.time - restore) if up else None, "instance_back_s": rounded(back.start - restore) if back else None, "writes": sum(1 for r in requests if r.kind == "w"), "writes_failed": sum( 1 for r in requests if r.kind == "w" and not r.ok ), "flags": " ".join(flags), } def fmt(value: Any) -> str: if value is None or value == "": return "—" if isinstance(value, float): return f"{value:g}" return str(value) def limits_line(profile: dict[str, Any]) -> str: """Строка лимитов шапки сводки; с сэмплером — и его лимит, и k6 без лимита (design change `profile-resources`, решение 7).""" line = ( f"- Лимиты без swap: API {profile['cpus']} CPU / " f"{profile['memory']} на реплику, HAProxy {profile['lb_cpus']} CPU / " f"{profile['lb_memory']}, PostgreSQL {profile['db_cpus']} CPU / " f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / " f"{profile['cache_memory']}" ) if profile.get("sample_resources"): line += ( f", сэмплер {profile['sampler_cpus']} CPU / " f"{profile['sampler_memory']}, k6 без лимита" ) return line def render_summary( meta: dict[str, Any], phases: list[dict[str, Any]], timeline: list[dict[str, Any]], resources: list[dict[str, Any]] | None = None, ) -> str: """`summary.md` профиля отказа: условия, таблица фаз, шкалы вокруг действия и возврата; с `resources` (строки `resources.csv`) — ещё ресурсы по фазам и окнам.""" profile = meta["profile"] lines = [ f"# Отказ реплик под нагрузкой: метка `{meta['label']}`", "", f"- Статус: **{meta['status']}**" + (f" ({meta['reason']})" if meta.get("reason") else ""), f"- Commit: `{meta['git']['commit']}`" + (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""), f"- Профиль `{profile['name']}`: {profile['replicas']} реплики за " f"HAProxy; {profile['failover_write_rate']} записей с ключом и " f"{profile['failover_read_rate']} чтений страницы истории в " "секунду (k6, постоянная интенсивность); режимы " f"{', '.join(profile['failover_modes'])}; окна фазы: " f"{profile['failover_before_s']} с до действия, " f"{profile['failover_down_s']} с до возврата, " f"{profile['failover_after_s']} с после; порог восстановления " f"{profile['failover_success']:.0%}", limits_line(profile), "- Посторонние контейнеры в VM во время замера: " + (", ".join(meta.get("other_containers") or []) or "нет"), "", "Время — секунды на часах ВМ Docker. Действие — событие Docker " "(`kill` с сигналом 15 у `docker stop`, 9 у `docker kill`, " "`pause`), исключение и возврат — строки журнала HAProxy, ответы — " "отметки k6 при отправке. Граница — худший случай формулы проверок " "LB для режима. Восстановление — конец последней секунды окна " "отказа с долей успешных ниже порога (0 — таких секунд нет). " "Сверка — после повтора неуспешных записей с тем же ключом: " "сообщений в чате записи и разных текстов столько, сколько " "отправлено разных ключей.", "", "| Фаза | Реплика | Исключение LB, с (граница) | Причина | " "Первая запись / чтение, с | Ошибки в окне отказа | Мин. доля " "успешных | Восстановление, с | Возврат в LB, с | Ошибки после " "возврата | Повтор записей | Сообщений = ключей | PG: прервано " "простоев | Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|", ] for row in phases: lines.append( f"| {row['phase']} | {row['instance']} ({row['slot']}) | " f"{fmt(row['lb_out_s'])} ({fmt(row['bound_s'])}) | " f"{fmt(row['lb_out_state'])}: {fmt(row['lb_out_reason'])} | " f"{fmt(row['first_write_ok_s'])} / {fmt(row['first_read_ok_s'])} " f"| {row['errors_down']} {fmt(row['codes_down'])} | " f"{fmt(row['min_success'])} | {row['recovery_s']} | " f"{fmt(row['lb_up_s'])} | {row['errors_after']} | " f"{row['writes_failed']} → {fmt(row['retried_ok'])} | " f"{fmt(row['write_chat_messages'])} = " f"{fmt(row['distinct_texts'])} | " f"{fmt(row['pg_idle_tx_terminated'])} | {fmt(row['flags'])} |", ) lines.append("") if resources: order = list(dict.fromkeys(r["phase"] for r in resources)) lines += res.render_failover(resources, order) for row in phases: rows = [r for r in timeline if r["phase"] == row["phase"]] reach = max( row["recovery_s"] or 0, math.ceil(row["lb_out_s"] or 0), ) down_s = profile["failover_down_s"] shown = [ r for r in rows if -2 <= r["second"] <= reach + 2 or down_s - 1 <= r["second"] <= down_s + 3 ] lines += [ f"## Фаза {row['phase']}", "", f"Действие {row['action_at']} над `{row['instance']}` (слот " f"{row['slot']}), возврат {row['restore_at']}; ответов " f"{row['requests']}, до действия {fmt(row['ok_rps_before'])} " f"успешных/с, p95 {fmt(row['p95_before_ms'])} мс. Полная шкала " "— `timeline.csv`.", "", "| с от действия | Записи ok / ош | Чтения ok / ош | Коды ошибок " "| Доля успешных | p50 / p95, мс | Ответы реплик |", "|---|---|---|---|---|---|---|", ] previous = None for r in shown: if previous is not None and r["second"] != previous + 1: lines.append("| … | | | | | | |") previous = r["second"] lines.append( f"| {r['second']} | {r['writes_ok']} / {r['writes_failed']} | " f"{r['reads_ok']} / {r['reads_failed']} | {fmt(r['codes'])} | " f"{fmt(r['success'])} | {fmt(r['p50_ms'])} / " f"{fmt(r['p95_ms'])} | {fmt(r['instances'])} |", ) lines.append("") if resources: lines += res.render_failover_phase(resources, row["phase"]) if meta.get("host_warnings"): lines += ["## Предупреждения о хосте", ""] lines += [f"- {w}" for w in meta["host_warnings"]] lines.append("") return "\n".join(lines)