Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e. Contracts, public tests and evidence of lab 5 are not carried over. The compose project, image and CI prefix are renamed to lab6; the first broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the OpenSpec changes of lab 6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
677 lines
25 KiB
Python
677 lines
25 KiB
Python
"""Профиль поочерёдной замены реплик: разбор и расчёт без docker.
|
||
|
||
Оркестрирует прогон `run.py` (профили `rollout`, `rollout_quick`), здесь —
|
||
чистые функции на stdlib: строки `scripts/rolling-update.sh`, события
|
||
Docker проекта, журнал HAProxy, посекундная шкала с ответами старых и
|
||
новых реплик, шаги по каждой заменённой реплике и `summary.md`. Строки
|
||
k6 о запросах, отметки времени и перцентили — из `failover.py`. Все
|
||
отметки — часы ВМ Docker. Решение 8 design change `rolling-update`.
|
||
"""
|
||
|
||
import math
|
||
import re
|
||
from dataclasses import dataclass
|
||
from typing import Any
|
||
|
||
import failover
|
||
import resources as res
|
||
|
||
ROLLOUT_FIELDS = [
|
||
"label",
|
||
"from_commit",
|
||
"replicas",
|
||
"revision_before",
|
||
"revision_after",
|
||
"build_s",
|
||
"migrate_s",
|
||
"rollout_s",
|
||
"script_exit",
|
||
"requests",
|
||
"errors_before",
|
||
"codes_before",
|
||
"errors_during",
|
||
"codes_during",
|
||
"errors_after",
|
||
"codes_after",
|
||
"min_success",
|
||
"seconds_below",
|
||
"ok_rps_before",
|
||
"ok_rps_during_min",
|
||
"p95_before_ms",
|
||
"p95_during_max_ms",
|
||
"first_new_s",
|
||
"last_old_s",
|
||
"old_after_drain",
|
||
"writes",
|
||
"writes_failed",
|
||
"retried_ok",
|
||
"write_chat_messages",
|
||
"distinct_texts",
|
||
"dropped_iterations",
|
||
"flags",
|
||
]
|
||
REPLICA_FIELDS = [
|
||
"step",
|
||
"new_instance",
|
||
"new_slot",
|
||
"old_instance",
|
||
"old_slot",
|
||
"new_created_s",
|
||
"new_up_s",
|
||
"ready_s",
|
||
"drain_s",
|
||
"drain_timeout",
|
||
"stop_s",
|
||
"old_removed_s",
|
||
"slot_ready_s",
|
||
"replace_s",
|
||
"errors",
|
||
"after_drain",
|
||
]
|
||
TIMELINE_FIELDS = [
|
||
"second",
|
||
"writes_ok",
|
||
"writes_failed",
|
||
"reads_ok",
|
||
"reads_failed",
|
||
"codes",
|
||
"success",
|
||
"p50_ms",
|
||
"p95_ms",
|
||
"old_ok",
|
||
"new_ok",
|
||
"instances",
|
||
]
|
||
|
||
SCRIPT_TAG = "rolling-update:"
|
||
NEW_UP_RE = re.compile(
|
||
r"replica (?P<n>\d+)/\d+: new (?P<id>[0-9a-f]{12}) addr (?P<addr>\S+) "
|
||
r"slot (?P<slot>api-\d+) UP",
|
||
)
|
||
DRAIN_RE = re.compile(
|
||
r"replica (?P<n>\d+)/\d+: drain old (?P<id>[0-9a-f]{12}) addr "
|
||
r"(?P<addr>\S+) slot (?P<slot>api-\d+)",
|
||
)
|
||
DRAIN_TIMEOUT_RE = re.compile(r"replica (?P<n>\d+)/\d+: drain timeout")
|
||
# запас на отметку отправки k6 против момента drain в журнале LB, с
|
||
DRAIN_MARGIN_S = 0.2
|
||
BUILD_RE = re.compile(r"build done in (\d+)s")
|
||
MIGRATE_RE = re.compile(r"migrate done in (\d+)s")
|
||
# состояния слота в журнале HAProxy: ручные (drain, maint, возврат из
|
||
# maint) и по проверкам и DNS
|
||
LB_STATES = (
|
||
("enters drain state", "DRAIN"),
|
||
("is going DOWN for maintenance (No IP", "DNS_MAINT"),
|
||
("is going DOWN for maintenance", "MAINT"),
|
||
("leaving forced maintenance", "READY"),
|
||
("is UP", "UP"),
|
||
("is DOWN", "DOWN"),
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ContainerEvent:
|
||
"""Событие Docker контейнера проекта: время, действие, полный ID."""
|
||
|
||
time: float
|
||
action: str
|
||
id: str
|
||
name: str
|
||
signal: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class SlotEvent:
|
||
time: float
|
||
slot: str
|
||
state: str
|
||
|
||
|
||
def script_lines(text: str) -> str:
|
||
"""Строки самого скрипта, без вывода сборки и миграции."""
|
||
return "\n".join(line for line in text.splitlines() if SCRIPT_TAG in line)
|
||
|
||
|
||
def script_error(text: str) -> str:
|
||
"""Последняя строка ошибки скрипта — в сообщение об отказе."""
|
||
errors = [
|
||
line
|
||
for line in text.splitlines()
|
||
if SCRIPT_TAG in line and "error:" in line
|
||
]
|
||
return errors[-1].split(SCRIPT_TAG, 1)[1].strip() if errors else ""
|
||
|
||
|
||
def parse_steps(text: str) -> list[dict[str, Any]]:
|
||
"""Шаги замены из строк скрипта: новая и старая реплика, их слоты."""
|
||
steps: dict[int, dict[str, Any]] = {}
|
||
for line in text.splitlines():
|
||
if match := NEW_UP_RE.search(line):
|
||
step = steps.setdefault(int(match["n"]), {})
|
||
step.update(
|
||
new_id=match["id"],
|
||
new_addr=match["addr"],
|
||
new_slot=match["slot"],
|
||
)
|
||
elif match := DRAIN_RE.search(line):
|
||
step = steps.setdefault(int(match["n"]), {})
|
||
step.update(
|
||
old_id=match["id"],
|
||
old_addr=match["addr"],
|
||
old_slot=match["slot"],
|
||
)
|
||
elif match := DRAIN_TIMEOUT_RE.search(line):
|
||
steps.setdefault(int(match["n"]), {})["drain_timeout"] = True
|
||
return [{"step": n, **steps[n]} for n in sorted(steps)]
|
||
|
||
|
||
def seconds(pattern: re.Pattern[str], text: str) -> int | None:
|
||
match = pattern.search(text)
|
||
return int(match.group(1)) if match else None
|
||
|
||
|
||
def parse_container_events(text: str) -> list[ContainerEvent]:
|
||
"""`docker events` в формате `время|действие|ID|имя|сигнал`.
|
||
|
||
Действие exec содержит команду, поэтому поля берутся с краёв.
|
||
"""
|
||
events = []
|
||
for line in text.splitlines():
|
||
parts = line.split("|")
|
||
if len(parts) < 5 or not parts[0].isdigit():
|
||
continue
|
||
events.append(
|
||
ContainerEvent(
|
||
time=int(parts[0]) / 1e9,
|
||
action="|".join(parts[1:-3]),
|
||
id=parts[-3],
|
||
name=parts[-2],
|
||
signal=parts[-1],
|
||
),
|
||
)
|
||
return sorted(events, key=lambda e: e.time)
|
||
|
||
|
||
def parse_slot_events(text: str) -> list[SlotEvent]:
|
||
"""Смена состояния слотов из журнала LB (`docker logs -t`); повтор
|
||
одной строки предупреждением в пределах миллисекунды отбрасывается."""
|
||
events: list[SlotEvent] = []
|
||
for line in text.splitlines():
|
||
match = failover.LB_RE.match(line)
|
||
if not match:
|
||
continue
|
||
state = next(
|
||
(name for marker, name in LB_STATES if marker in match["rest"]),
|
||
None,
|
||
)
|
||
if state is None:
|
||
continue
|
||
event = SlotEvent(
|
||
failover.parse_stamp(match["stamp"]),
|
||
match["slot"],
|
||
state,
|
||
)
|
||
if any(
|
||
e.slot == event.slot
|
||
and e.state == event.state
|
||
and abs(e.time - event.time) < 0.001
|
||
for e in events
|
||
):
|
||
continue
|
||
events.append(event)
|
||
return sorted(events, key=lambda e: e.time)
|
||
|
||
|
||
def container_event(
|
||
events: list[ContainerEvent],
|
||
short_id: str | None,
|
||
action: str,
|
||
since: float = 0.0,
|
||
) -> ContainerEvent | None:
|
||
if not short_id:
|
||
return None
|
||
return next(
|
||
(
|
||
e
|
||
for e in events
|
||
if e.id.startswith(short_id)
|
||
and e.action == action
|
||
and e.time >= since
|
||
),
|
||
None,
|
||
)
|
||
|
||
|
||
def slot_event(
|
||
events: list[SlotEvent],
|
||
slot: str | None,
|
||
state: str,
|
||
since: float | None,
|
||
) -> SlotEvent | None:
|
||
if slot is None or since is None:
|
||
return None
|
||
return next(
|
||
(
|
||
e
|
||
for e in events
|
||
if e.slot == slot and e.state == state and e.time >= since
|
||
),
|
||
None,
|
||
)
|
||
|
||
|
||
def start_time(events: list[ContainerEvent]) -> float | None:
|
||
"""Начало изменения стенда: создание контейнера миграции, без него —
|
||
первой новой реплики (миграции пропущены)."""
|
||
# контейнер сервиса migrate (scripts/rolling-update.sh пересоздаёт его
|
||
# на новом образе) или одноразовый migrate-run
|
||
migrate = next(
|
||
(e for e in events if e.action == "create" and "-migrate-" in e.name),
|
||
None,
|
||
)
|
||
if migrate is not None:
|
||
return migrate.time
|
||
create = next((e for e in events if e.action == "create"), None)
|
||
return create.time if create else None
|
||
|
||
|
||
def script_start(
|
||
events: list[ContainerEvent],
|
||
since: float,
|
||
ignore: frozenset[str] | set[str] = frozenset(),
|
||
) -> float | None:
|
||
"""Начало подготовки замены (решение 4 design change
|
||
`profile-resources`): первое `exec_start` в контейнер проекта не
|
||
раньше `since` (первого запроса) — первое обращение скрипта к LB.
|
||
Оркестратор во время нагрузки `docker exec` не делает, k6 — вне
|
||
проекта; `exec_start` проверок здоровья (`ignore` — их действия)
|
||
отбрасываются (найдено в apply: они тоже приходят каждые 5 с)."""
|
||
return next(
|
||
(
|
||
e.time
|
||
for e in events
|
||
if e.action.startswith("exec_start")
|
||
and e.action not in ignore
|
||
and e.time >= since
|
||
),
|
||
None,
|
||
)
|
||
|
||
|
||
def diff(later: Any, earlier: Any) -> float | None:
|
||
if later is None or earlier is None:
|
||
return None
|
||
return round(later.time - earlier.time, 2)
|
||
|
||
|
||
def since_t0(event: Any, t0: float) -> float | None:
|
||
return None if event is None else round(event.time - t0, 2)
|
||
|
||
|
||
def replica_rows(
|
||
steps: list[dict[str, Any]],
|
||
containers: list[ContainerEvent],
|
||
slots: list[SlotEvent],
|
||
requests: list[failover.Request],
|
||
t0: float,
|
||
) -> list[dict[str, Any]]:
|
||
"""Шаги каждой замены: от создания новой до возврата слота старой."""
|
||
rows = []
|
||
for step in steps:
|
||
created = container_event(containers, step.get("new_id"), "create")
|
||
started = container_event(containers, step.get("new_id"), "start")
|
||
up = slot_event(
|
||
slots,
|
||
step.get("new_slot"),
|
||
"UP",
|
||
started.time if started else None,
|
||
)
|
||
drain = slot_event(
|
||
slots,
|
||
step.get("old_slot"),
|
||
"DRAIN",
|
||
up.time if up else None,
|
||
)
|
||
maint = slot_event(
|
||
slots,
|
||
step.get("old_slot"),
|
||
"MAINT",
|
||
drain.time if drain else None,
|
||
)
|
||
kill = container_event(
|
||
containers,
|
||
step.get("old_id"),
|
||
"kill",
|
||
maint.time if maint else 0.0,
|
||
)
|
||
die = container_event(
|
||
containers,
|
||
step.get("old_id"),
|
||
"die",
|
||
kill.time if kill else 0.0,
|
||
)
|
||
destroy = container_event(containers, step.get("old_id"), "destroy")
|
||
ready = slot_event(
|
||
slots,
|
||
step.get("old_slot"),
|
||
"READY",
|
||
destroy.time if destroy else None,
|
||
)
|
||
window = [
|
||
r
|
||
for r in requests
|
||
if created is not None
|
||
and destroy is not None
|
||
and created.time <= r.start < destroy.time
|
||
]
|
||
# ответы старой реплики, отправленные после её вывода из LB: запрос
|
||
# мог уйти до команды drain на доли миллисекунды раньше отметки k6,
|
||
# поэтому запас DRAIN_MARGIN_S
|
||
after_drain = [
|
||
r
|
||
for r in requests
|
||
if drain is not None
|
||
and r.instance == step.get("old_id")
|
||
and r.start > drain.time + DRAIN_MARGIN_S
|
||
]
|
||
rows.append(
|
||
{
|
||
"step": step["step"],
|
||
"new_instance": step.get("new_id"),
|
||
"new_slot": step.get("new_slot"),
|
||
"old_instance": step.get("old_id"),
|
||
"old_slot": step.get("old_slot"),
|
||
"new_created_s": since_t0(created, t0),
|
||
"new_up_s": since_t0(up, t0),
|
||
"ready_s": diff(up, started),
|
||
"drain_s": diff(maint, drain),
|
||
"drain_timeout": bool(step.get("drain_timeout")),
|
||
"stop_s": diff(die, kill),
|
||
"old_removed_s": since_t0(destroy, t0),
|
||
"slot_ready_s": since_t0(ready, t0),
|
||
"replace_s": diff(destroy, created),
|
||
"errors": sum(1 for r in window if not r.ok),
|
||
"after_drain": len(after_drain),
|
||
},
|
||
)
|
||
return rows
|
||
|
||
|
||
def build_timeline(
|
||
requests: list[failover.Request],
|
||
t0: float,
|
||
old: set[str],
|
||
new: set[str],
|
||
) -> list[dict[str, Any]]:
|
||
"""Посекундная шкала от начала замены; секунда — по отправке.
|
||
|
||
`old` и `new` — `X-Instance-Id` (12 символов) реплик до и после.
|
||
"""
|
||
buckets: dict[int, list[failover.Request]] = {}
|
||
for request in requests:
|
||
buckets.setdefault(math.floor(request.start - t0), []).append(request)
|
||
rows = []
|
||
for second in range(min(buckets, default=0), max(buckets, default=-1) + 1):
|
||
group = buckets.get(second, [])
|
||
ok = [r for r in group if r.ok]
|
||
instances: dict[str, int] = {}
|
||
for request in ok:
|
||
instances[request.instance] = (
|
||
instances.get(request.instance, 0) + 1
|
||
)
|
||
durations = [r.duration_ms for r in ok]
|
||
rows.append(
|
||
{
|
||
"second": second,
|
||
"writes_ok": sum(1 for r in ok if r.kind == "w"),
|
||
"writes_failed": sum(
|
||
1 for r in group if r.kind == "w" and not r.ok
|
||
),
|
||
"reads_ok": sum(1 for r in ok if r.kind == "r"),
|
||
"reads_failed": sum(
|
||
1 for r in group if r.kind == "r" and not r.ok
|
||
),
|
||
"codes": failover.codes(group),
|
||
"success": round(len(ok) / len(group), 4) if group else None,
|
||
"p50_ms": failover.percentile(durations, 0.5),
|
||
"p95_ms": failover.percentile(durations, 0.95),
|
||
"old_ok": sum(1 for r in ok if r.instance in old),
|
||
"new_ok": sum(1 for r in ok if r.instance in new),
|
||
"instances": " ".join(
|
||
f"{key}:{value}"
|
||
for key, value in sorted(instances.items())
|
||
),
|
||
},
|
||
)
|
||
return rows
|
||
|
||
|
||
def analyze(
|
||
requests: list[failover.Request],
|
||
timeline: list[dict[str, Any]],
|
||
replicas: list[dict[str, Any]],
|
||
*,
|
||
t0: float,
|
||
end: float,
|
||
old: set[str],
|
||
new: set[str],
|
||
threshold: float,
|
||
) -> dict[str, Any]:
|
||
"""Величины прогона: окна до, во время и после замены (решение 8)."""
|
||
|
||
def ended(request: failover.Request) -> float:
|
||
return request.start + request.duration_ms / 1000
|
||
|
||
before = [r for r in requests if ended(r) < t0]
|
||
during = [r for r in requests if ended(r) >= t0 and r.start < end]
|
||
after = [r for r in requests if r.start >= end]
|
||
window = math.ceil(end - t0)
|
||
during_rows = [row for row in timeline if 0 <= row["second"] < window]
|
||
before_rows = [row for row in timeline if row["second"] < 0][2:]
|
||
# после вывода из LB старая реплика запросов получать не должна
|
||
old_after_drain = sum(r["after_drain"] for r in replicas)
|
||
new_ok = [r.start for r in requests if r.ok and r.instance in new]
|
||
old_ok = [r.start for r in requests if r.ok and r.instance in old]
|
||
p95_during = [
|
||
row["p95_ms"] for row in during_rows if row["p95_ms"] is not None
|
||
]
|
||
flags = []
|
||
if any(not r.ok for r in requests):
|
||
flags.append("errors")
|
||
if old_after_drain:
|
||
flags.append("old_after_drain")
|
||
return {
|
||
"rollout_s": round(end - t0, 2),
|
||
"requests": len(requests),
|
||
"errors_before": sum(1 for r in before if not r.ok),
|
||
"codes_before": failover.codes(before),
|
||
"errors_during": sum(1 for r in during if not r.ok),
|
||
"codes_during": failover.codes(during),
|
||
"errors_after": sum(1 for r in after if not r.ok),
|
||
"codes_after": failover.codes(after),
|
||
"min_success": min(
|
||
(
|
||
row["success"]
|
||
for row in during_rows
|
||
if row["success"] is not None
|
||
),
|
||
default=None,
|
||
),
|
||
"seconds_below": sum(
|
||
1
|
||
for row in during_rows
|
||
if row["success"] is not None and row["success"] < threshold
|
||
),
|
||
"ok_rps_before": (
|
||
round(
|
||
sum(row["writes_ok"] + row["reads_ok"] for row in before_rows)
|
||
/ len(before_rows),
|
||
1,
|
||
)
|
||
if before_rows
|
||
else None
|
||
),
|
||
"ok_rps_during_min": min(
|
||
(row["writes_ok"] + row["reads_ok"] for row in during_rows),
|
||
default=None,
|
||
),
|
||
"p95_before_ms": failover.percentile(
|
||
[r.duration_ms for r in before if r.ok],
|
||
0.95,
|
||
),
|
||
"p95_during_max_ms": max(p95_during, default=None),
|
||
"first_new_s": round(min(new_ok) - t0, 2) if new_ok else None,
|
||
"last_old_s": round(max(old_ok) - t0, 2) if old_ok else None,
|
||
"old_after_drain": old_after_drain,
|
||
"writes": sum(1 for r in requests if r.kind == "w"),
|
||
"writes_failed": sum(
|
||
1 for r in requests if r.kind == "w" and not r.ok
|
||
),
|
||
"flags": " ".join(flags),
|
||
}
|
||
|
||
|
||
fmt = failover.fmt
|
||
|
||
|
||
def render_summary(
|
||
meta: dict[str, Any],
|
||
row: dict[str, Any] | None,
|
||
replicas: list[dict[str, Any]],
|
||
timeline: list[dict[str, Any]],
|
||
resources: list[dict[str, Any]] | None = None,
|
||
) -> str:
|
||
"""`summary.md` профиля замены: условия, итог, шаги по репликам,
|
||
контейнеры до и после, шкала вокруг замены; с `resources` (строки
|
||
`resources.csv`) — ещё ресурсы по окнам замены."""
|
||
profile = meta["profile"]
|
||
info = meta.get("rollout", {})
|
||
lines = [
|
||
f"# Поочерёдная замена реплик под нагрузкой: метка `{meta['label']}`",
|
||
"",
|
||
f"- Статус: **{meta['status']}**"
|
||
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
|
||
f"- Commit: `{meta['git']['commit']}`"
|
||
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
|
||
f"- Профиль `{profile['name']}`: {profile['replicas']} реплики за "
|
||
f"HAProxy; замена с образа commit `{info.get('from_commit')}` на "
|
||
"образ текущего checkout (`scripts/rolling-update.sh`); "
|
||
f"{profile['failover_write_rate']} записей с ключом и "
|
||
f"{profile['failover_read_rate']} чтений страницы истории в "
|
||
"секунду (k6, постоянная интенсивность); окна: "
|
||
f"{profile['rollout_before_s']} с до запуска скрипта, бюджет "
|
||
f"замены {profile['rollout_budget_s']} с, ещё "
|
||
f"{profile['rollout_after_s']} с; порог доли успешных "
|
||
f"{profile['failover_success']:.0%}",
|
||
failover.limits_line(profile),
|
||
"- Посторонние контейнеры в VM во время замера: "
|
||
+ (", ".join(meta.get("other_containers") or []) or "нет"),
|
||
"",
|
||
"Время — секунды на часах ВМ Docker от начала замены (создание "
|
||
"контейнера миграции). Шаги — события Docker (создание и старт "
|
||
"новой реплики, SIGTERM, выход и удаление старой) и журнал HAProxy "
|
||
"(`UP` новой, `drain` и `maint` старой, возврат слота в `ready`). "
|
||
"Ответы — отметки k6 при отправке; «старые» и «новые» — по "
|
||
"`X-Instance-Id` контейнеров до и после замены. Окно замены — от "
|
||
"начала до удаления последней старой реплики. Сверка — после "
|
||
"повтора неуспешных записей с тем же ключом: сообщений в чате "
|
||
"записи и разных текстов столько, сколько отправлено ключей.",
|
||
"",
|
||
]
|
||
if row:
|
||
lines += [
|
||
"| Замена, с | Сборка / миграция, с | Ревизия до → после | "
|
||
"Ошибки до / во время / после | Мин. доля успешных | Секунд ниже "
|
||
"порога | Успешных в секунду: до / мин. во время | p95: до / "
|
||
"макс. во время, мс | Первый ответ новой / последний старой, с | "
|
||
"Ответов старых после дренажа | Повтор записей | "
|
||
"Сообщений = ключей | Пометки |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
f"| {fmt(row['rollout_s'])} | {fmt(row['build_s'])} / "
|
||
f"{fmt(row['migrate_s'])} | `{fmt(row['revision_before'])}` → "
|
||
f"`{fmt(row['revision_after'])}` | {row['errors_before']} / "
|
||
f"{row['errors_during']}"
|
||
+ (f" ({row['codes_during']})" if row["codes_during"] else "")
|
||
+ f" / {row['errors_after']} | {fmt(row['min_success'])} | "
|
||
f"{row['seconds_below']} | {fmt(row['ok_rps_before'])} / "
|
||
f"{fmt(row['ok_rps_during_min'])} | {fmt(row['p95_before_ms'])} / "
|
||
f"{fmt(row['p95_during_max_ms'])} | {fmt(row['first_new_s'])} / "
|
||
f"{fmt(row['last_old_s'])} | {row['old_after_drain']} | "
|
||
f"{row['writes_failed']} → {fmt(row['retried_ok'])} | "
|
||
f"{fmt(row['write_chat_messages'])} = "
|
||
f"{fmt(row['distinct_texts'])} | {fmt(row['flags'])} |",
|
||
"",
|
||
]
|
||
if replicas:
|
||
lines += [
|
||
"## Шаги по репликам",
|
||
"",
|
||
"| Шаг | Новая (слот) | Старая (слот) | Новая создана, с | Новая "
|
||
"`UP`, с | Старт → `UP`, с | Дренаж, с | Остановка, с | Старая "
|
||
"удалена, с | Слот `ready`, с | Замена целиком, с | Ошибок за "
|
||
"шаг | Ответов старой после дренажа |",
|
||
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in replicas:
|
||
drain = fmt(r["drain_s"]) + (
|
||
" (срок)" if r["drain_timeout"] else ""
|
||
)
|
||
lines.append(
|
||
f"| {r['step']} | `{fmt(r['new_instance'])}` "
|
||
f"({fmt(r['new_slot'])}) | `{fmt(r['old_instance'])}` "
|
||
f"({fmt(r['old_slot'])}) | {fmt(r['new_created_s'])} | "
|
||
f"{fmt(r['new_up_s'])} | {fmt(r['ready_s'])} | {drain} | "
|
||
f"{fmt(r['stop_s'])} | {fmt(r['old_removed_s'])} | "
|
||
f"{fmt(r['slot_ready_s'])} | {fmt(r['replace_s'])} | "
|
||
f"{r['errors']} | {r['after_drain']} |",
|
||
)
|
||
lines.append("")
|
||
if resources:
|
||
lines += res.render_rollout(resources)
|
||
for title, key in (
|
||
("до", "containers_before"),
|
||
("после", "containers_after"),
|
||
):
|
||
items = info.get(key) or []
|
||
if not items:
|
||
continue
|
||
lines += [
|
||
f"## Контейнеры `api` {title} замены",
|
||
"",
|
||
"| Контейнер | `X-Instance-Id` | Образ (тег) | Конфигурация образа |",
|
||
"|---|---|---|---|",
|
||
]
|
||
lines += [
|
||
f"| {c['name']} | `{c['instance']}` | `{c['tag']}` | "
|
||
f"`{c['image'][:19]}…` |"
|
||
for c in items
|
||
]
|
||
lines.append("")
|
||
if timeline:
|
||
end = math.ceil(row["rollout_s"]) if row else 0
|
||
shown = [r for r in timeline if -3 <= r["second"] <= end + 3]
|
||
lines += [
|
||
"## Шкала вокруг замены",
|
||
"",
|
||
"Полная шкала — `timeline.csv`.",
|
||
"",
|
||
"| с от начала | Записи ok / ош | Чтения ok / ош | Коды ошибок | "
|
||
"Доля успешных | p50 / p95, мс | Старые / новые ok | Ответы "
|
||
"реплик |",
|
||
"|---|---|---|---|---|---|---|---|",
|
||
]
|
||
for r in shown:
|
||
lines.append(
|
||
f"| {r['second']} | {r['writes_ok']} / {r['writes_failed']} | "
|
||
f"{r['reads_ok']} / {r['reads_failed']} | {fmt(r['codes'])} | "
|
||
f"{fmt(r['success'])} | {fmt(r['p50_ms'])} / "
|
||
f"{fmt(r['p95_ms'])} | {r['old_ok']} / {r['new_ok']} | "
|
||
f"{fmt(r['instances'])} |",
|
||
)
|
||
lines.append("")
|
||
if meta.get("host_warnings"):
|
||
lines += ["## Предупреждения о хосте", ""]
|
||
lines += [f"- {w}" for w in meta["host_warnings"]]
|
||
lines.append("")
|
||
return "\n".join(lines)
|