"""Профиль поочерёдной замены реплик: разбор и расчёт без docker. Оркестрирует прогон `run.py` (профили `rollout`, `rollout_quick`), здесь — чистые функции на stdlib: строки `scripts/rolling-update.sh`, события Docker проекта, журнал HAProxy, посекундная шкала с ответами старых и новых реплик, шаги по каждой заменённой реплике и `summary.md`. Строки k6 о запросах, отметки времени и перцентили — из `failover.py`. Все отметки — часы ВМ Docker. Решение 8 design change `rolling-update`. """ import math import re from dataclasses import dataclass from typing import Any import failover ROLLOUT_FIELDS = [ "label", "from_commit", "replicas", "revision_before", "revision_after", "build_s", "migrate_s", "rollout_s", "script_exit", "requests", "errors_before", "codes_before", "errors_during", "codes_during", "errors_after", "codes_after", "min_success", "seconds_below", "ok_rps_before", "ok_rps_during_min", "p95_before_ms", "p95_during_max_ms", "first_new_s", "last_old_s", "old_after_drain", "writes", "writes_failed", "retried_ok", "write_chat_messages", "distinct_texts", "dropped_iterations", "flags", ] REPLICA_FIELDS = [ "step", "new_instance", "new_slot", "old_instance", "old_slot", "new_created_s", "new_up_s", "ready_s", "drain_s", "drain_timeout", "stop_s", "old_removed_s", "slot_ready_s", "replace_s", "errors", "after_drain", ] TIMELINE_FIELDS = [ "second", "writes_ok", "writes_failed", "reads_ok", "reads_failed", "codes", "success", "p50_ms", "p95_ms", "old_ok", "new_ok", "instances", ] SCRIPT_TAG = "rolling-update:" NEW_UP_RE = re.compile( r"replica (?P\d+)/\d+: new (?P[0-9a-f]{12}) addr (?P\S+) " r"slot (?Papi-\d+) UP", ) DRAIN_RE = re.compile( r"replica (?P\d+)/\d+: drain old (?P[0-9a-f]{12}) addr " r"(?P\S+) slot (?Papi-\d+)", ) DRAIN_TIMEOUT_RE = re.compile(r"replica (?P\d+)/\d+: drain timeout") # запас на отметку отправки k6 против момента drain в журнале LB, с DRAIN_MARGIN_S = 0.2 BUILD_RE = re.compile(r"build done in (\d+)s") MIGRATE_RE = re.compile(r"migrate done in (\d+)s") # состояния слота в журнале HAProxy: ручные (drain, maint, возврат из # maint) и по проверкам и DNS LB_STATES = ( ("enters drain state", "DRAIN"), ("is going DOWN for maintenance (No IP", "DNS_MAINT"), ("is going DOWN for maintenance", "MAINT"), ("leaving forced maintenance", "READY"), ("is UP", "UP"), ("is DOWN", "DOWN"), ) @dataclass(frozen=True) class ContainerEvent: """Событие Docker контейнера проекта: время, действие, полный ID.""" time: float action: str id: str name: str signal: str @dataclass(frozen=True) class SlotEvent: time: float slot: str state: str def script_lines(text: str) -> str: """Строки самого скрипта, без вывода сборки и миграции.""" return "\n".join(line for line in text.splitlines() if SCRIPT_TAG in line) def script_error(text: str) -> str: """Последняя строка ошибки скрипта — в сообщение об отказе.""" errors = [ line for line in text.splitlines() if SCRIPT_TAG in line and "error:" in line ] return errors[-1].split(SCRIPT_TAG, 1)[1].strip() if errors else "" def parse_steps(text: str) -> list[dict[str, Any]]: """Шаги замены из строк скрипта: новая и старая реплика, их слоты.""" steps: dict[int, dict[str, Any]] = {} for line in text.splitlines(): if match := NEW_UP_RE.search(line): step = steps.setdefault(int(match["n"]), {}) step.update( new_id=match["id"], new_addr=match["addr"], new_slot=match["slot"], ) elif match := DRAIN_RE.search(line): step = steps.setdefault(int(match["n"]), {}) step.update( old_id=match["id"], old_addr=match["addr"], old_slot=match["slot"], ) elif match := DRAIN_TIMEOUT_RE.search(line): steps.setdefault(int(match["n"]), {})["drain_timeout"] = True return [{"step": n, **steps[n]} for n in sorted(steps)] def seconds(pattern: re.Pattern[str], text: str) -> int | None: match = pattern.search(text) return int(match.group(1)) if match else None def parse_container_events(text: str) -> list[ContainerEvent]: """`docker events` в формате `время|действие|ID|имя|сигнал`. Действие exec содержит команду, поэтому поля берутся с краёв. """ events = [] for line in text.splitlines(): parts = line.split("|") if len(parts) < 5 or not parts[0].isdigit(): continue events.append( ContainerEvent( time=int(parts[0]) / 1e9, action="|".join(parts[1:-3]), id=parts[-3], name=parts[-2], signal=parts[-1], ), ) return sorted(events, key=lambda e: e.time) def parse_slot_events(text: str) -> list[SlotEvent]: """Смена состояния слотов из журнала LB (`docker logs -t`); повтор одной строки предупреждением в пределах миллисекунды отбрасывается.""" events: list[SlotEvent] = [] for line in text.splitlines(): match = failover.LB_RE.match(line) if not match: continue state = next( (name for marker, name in LB_STATES if marker in match["rest"]), None, ) if state is None: continue event = SlotEvent( failover.parse_stamp(match["stamp"]), match["slot"], state, ) if any( e.slot == event.slot and e.state == event.state and abs(e.time - event.time) < 0.001 for e in events ): continue events.append(event) return sorted(events, key=lambda e: e.time) def container_event( events: list[ContainerEvent], short_id: str | None, action: str, since: float = 0.0, ) -> ContainerEvent | None: if not short_id: return None return next( ( e for e in events if e.id.startswith(short_id) and e.action == action and e.time >= since ), None, ) def slot_event( events: list[SlotEvent], slot: str | None, state: str, since: float | None, ) -> SlotEvent | None: if slot is None or since is None: return None return next( ( e for e in events if e.slot == slot and e.state == state and e.time >= since ), None, ) def start_time(events: list[ContainerEvent]) -> float | None: """Начало изменения стенда: создание контейнера миграции, без него — первой новой реплики (миграции пропущены).""" # контейнер сервиса migrate (scripts/rolling-update.sh пересоздаёт его # на новом образе) или одноразовый migrate-run migrate = next( (e for e in events if e.action == "create" and "-migrate-" in e.name), None, ) if migrate is not None: return migrate.time create = next((e for e in events if e.action == "create"), None) return create.time if create else None def diff(later: Any, earlier: Any) -> float | None: if later is None or earlier is None: return None return round(later.time - earlier.time, 2) def since_t0(event: Any, t0: float) -> float | None: return None if event is None else round(event.time - t0, 2) def replica_rows( steps: list[dict[str, Any]], containers: list[ContainerEvent], slots: list[SlotEvent], requests: list[failover.Request], t0: float, ) -> list[dict[str, Any]]: """Шаги каждой замены: от создания новой до возврата слота старой.""" rows = [] for step in steps: created = container_event(containers, step.get("new_id"), "create") started = container_event(containers, step.get("new_id"), "start") up = slot_event( slots, step.get("new_slot"), "UP", started.time if started else None, ) drain = slot_event( slots, step.get("old_slot"), "DRAIN", up.time if up else None, ) maint = slot_event( slots, step.get("old_slot"), "MAINT", drain.time if drain else None, ) kill = container_event( containers, step.get("old_id"), "kill", maint.time if maint else 0.0, ) die = container_event( containers, step.get("old_id"), "die", kill.time if kill else 0.0, ) destroy = container_event(containers, step.get("old_id"), "destroy") ready = slot_event( slots, step.get("old_slot"), "READY", destroy.time if destroy else None, ) window = [ r for r in requests if created is not None and destroy is not None and created.time <= r.start < destroy.time ] # ответы старой реплики, отправленные после её вывода из LB: запрос # мог уйти до команды drain на доли миллисекунды раньше отметки k6, # поэтому запас DRAIN_MARGIN_S after_drain = [ r for r in requests if drain is not None and r.instance == step.get("old_id") and r.start > drain.time + DRAIN_MARGIN_S ] rows.append( { "step": step["step"], "new_instance": step.get("new_id"), "new_slot": step.get("new_slot"), "old_instance": step.get("old_id"), "old_slot": step.get("old_slot"), "new_created_s": since_t0(created, t0), "new_up_s": since_t0(up, t0), "ready_s": diff(up, started), "drain_s": diff(maint, drain), "drain_timeout": bool(step.get("drain_timeout")), "stop_s": diff(die, kill), "old_removed_s": since_t0(destroy, t0), "slot_ready_s": since_t0(ready, t0), "replace_s": diff(destroy, created), "errors": sum(1 for r in window if not r.ok), "after_drain": len(after_drain), }, ) return rows def build_timeline( requests: list[failover.Request], t0: float, old: set[str], new: set[str], ) -> list[dict[str, Any]]: """Посекундная шкала от начала замены; секунда — по отправке. `old` и `new` — `X-Instance-Id` (12 символов) реплик до и после. """ buckets: dict[int, list[failover.Request]] = {} for request in requests: buckets.setdefault(math.floor(request.start - t0), []).append(request) rows = [] for second in range(min(buckets, default=0), max(buckets, default=-1) + 1): group = buckets.get(second, []) ok = [r for r in group if r.ok] instances: dict[str, int] = {} for request in ok: instances[request.instance] = ( instances.get(request.instance, 0) + 1 ) durations = [r.duration_ms for r in ok] rows.append( { "second": second, "writes_ok": sum(1 for r in ok if r.kind == "w"), "writes_failed": sum( 1 for r in group if r.kind == "w" and not r.ok ), "reads_ok": sum(1 for r in ok if r.kind == "r"), "reads_failed": sum( 1 for r in group if r.kind == "r" and not r.ok ), "codes": failover.codes(group), "success": round(len(ok) / len(group), 4) if group else None, "p50_ms": failover.percentile(durations, 0.5), "p95_ms": failover.percentile(durations, 0.95), "old_ok": sum(1 for r in ok if r.instance in old), "new_ok": sum(1 for r in ok if r.instance in new), "instances": " ".join( f"{key}:{value}" for key, value in sorted(instances.items()) ), }, ) return rows def analyze( requests: list[failover.Request], timeline: list[dict[str, Any]], replicas: list[dict[str, Any]], *, t0: float, end: float, old: set[str], new: set[str], threshold: float, ) -> dict[str, Any]: """Величины прогона: окна до, во время и после замены (решение 8).""" def ended(request: failover.Request) -> float: return request.start + request.duration_ms / 1000 before = [r for r in requests if ended(r) < t0] during = [r for r in requests if ended(r) >= t0 and r.start < end] after = [r for r in requests if r.start >= end] window = math.ceil(end - t0) during_rows = [row for row in timeline if 0 <= row["second"] < window] before_rows = [row for row in timeline if row["second"] < 0][2:] # после вывода из LB старая реплика запросов получать не должна old_after_drain = sum(r["after_drain"] for r in replicas) new_ok = [r.start for r in requests if r.ok and r.instance in new] old_ok = [r.start for r in requests if r.ok and r.instance in old] p95_during = [ row["p95_ms"] for row in during_rows if row["p95_ms"] is not None ] flags = [] if any(not r.ok for r in requests): flags.append("errors") if old_after_drain: flags.append("old_after_drain") return { "rollout_s": round(end - t0, 2), "requests": len(requests), "errors_before": sum(1 for r in before if not r.ok), "codes_before": failover.codes(before), "errors_during": sum(1 for r in during if not r.ok), "codes_during": failover.codes(during), "errors_after": sum(1 for r in after if not r.ok), "codes_after": failover.codes(after), "min_success": min( ( row["success"] for row in during_rows if row["success"] is not None ), default=None, ), "seconds_below": sum( 1 for row in during_rows if row["success"] is not None and row["success"] < threshold ), "ok_rps_before": ( round( sum(row["writes_ok"] + row["reads_ok"] for row in before_rows) / len(before_rows), 1, ) if before_rows else None ), "ok_rps_during_min": min( (row["writes_ok"] + row["reads_ok"] for row in during_rows), default=None, ), "p95_before_ms": failover.percentile( [r.duration_ms for r in before if r.ok], 0.95, ), "p95_during_max_ms": max(p95_during, default=None), "first_new_s": round(min(new_ok) - t0, 2) if new_ok else None, "last_old_s": round(max(old_ok) - t0, 2) if old_ok else None, "old_after_drain": old_after_drain, "writes": sum(1 for r in requests if r.kind == "w"), "writes_failed": sum( 1 for r in requests if r.kind == "w" and not r.ok ), "flags": " ".join(flags), } fmt = failover.fmt def render_summary( meta: dict[str, Any], row: dict[str, Any] | None, replicas: list[dict[str, Any]], timeline: list[dict[str, Any]], ) -> str: """`summary.md` профиля замены: условия, итог, шаги по репликам, контейнеры до и после, шкала вокруг замены.""" profile = meta["profile"] info = meta.get("rollout", {}) lines = [ f"# Поочерёдная замена реплик под нагрузкой: метка `{meta['label']}`", "", f"- Статус: **{meta['status']}**" + (f" ({meta['reason']})" if meta.get("reason") else ""), f"- Commit: `{meta['git']['commit']}`" + (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""), f"- Профиль `{profile['name']}`: {profile['replicas']} реплики за " f"HAProxy; замена с образа commit `{info.get('from_commit')}` на " "образ текущего checkout (`scripts/rolling-update.sh`); " f"{profile['failover_write_rate']} записей с ключом и " f"{profile['failover_read_rate']} чтений страницы истории в " "секунду (k6, постоянная интенсивность); окна: " f"{profile['rollout_before_s']} с до запуска скрипта, бюджет " f"замены {profile['rollout_budget_s']} с, ещё " f"{profile['rollout_after_s']} с; порог доли успешных " f"{profile['failover_success']:.0%}", f"- Лимиты без swap: API {profile['cpus']} CPU / " f"{profile['memory']} на реплику, HAProxy {profile['lb_cpus']} CPU / " f"{profile['lb_memory']}, PostgreSQL {profile['db_cpus']} CPU / " f"{profile['db_memory']}, Valkey {profile['cache_cpus']} CPU / " f"{profile['cache_memory']}", "- Посторонние контейнеры в VM во время замера: " + (", ".join(meta.get("other_containers") or []) or "нет"), "", "Время — секунды на часах ВМ Docker от начала замены (создание " "контейнера миграции). Шаги — события Docker (создание и старт " "новой реплики, SIGTERM, выход и удаление старой) и журнал HAProxy " "(`UP` новой, `drain` и `maint` старой, возврат слота в `ready`). " "Ответы — отметки k6 при отправке; «старые» и «новые» — по " "`X-Instance-Id` контейнеров до и после замены. Окно замены — от " "начала до удаления последней старой реплики. Сверка — после " "повтора неуспешных записей с тем же ключом: сообщений в чате " "записи и разных текстов столько, сколько отправлено ключей.", "", ] if row: lines += [ "| Замена, с | Сборка / миграция, с | Ревизия до → после | " "Ошибки до / во время / после | Мин. доля успешных | Секунд ниже " "порога | Успешных в секунду: до / мин. во время | p95: до / " "макс. во время, мс | Первый ответ новой / последний старой, с | " "Ответов старых после дренажа | Повтор записей | " "Сообщений = ключей | Пометки |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|", f"| {fmt(row['rollout_s'])} | {fmt(row['build_s'])} / " f"{fmt(row['migrate_s'])} | `{fmt(row['revision_before'])}` → " f"`{fmt(row['revision_after'])}` | {row['errors_before']} / " f"{row['errors_during']}" + (f" ({row['codes_during']})" if row["codes_during"] else "") + f" / {row['errors_after']} | {fmt(row['min_success'])} | " f"{row['seconds_below']} | {fmt(row['ok_rps_before'])} / " f"{fmt(row['ok_rps_during_min'])} | {fmt(row['p95_before_ms'])} / " f"{fmt(row['p95_during_max_ms'])} | {fmt(row['first_new_s'])} / " f"{fmt(row['last_old_s'])} | {row['old_after_drain']} | " f"{row['writes_failed']} → {fmt(row['retried_ok'])} | " f"{fmt(row['write_chat_messages'])} = " f"{fmt(row['distinct_texts'])} | {fmt(row['flags'])} |", "", ] if replicas: lines += [ "## Шаги по репликам", "", "| Шаг | Новая (слот) | Старая (слот) | Новая создана, с | Новая " "`UP`, с | Старт → `UP`, с | Дренаж, с | Остановка, с | Старая " "удалена, с | Слот `ready`, с | Замена целиком, с | Ошибок за " "шаг | Ответов старой после дренажа |", "|---|---|---|---|---|---|---|---|---|---|---|---|---|", ] for r in replicas: drain = fmt(r["drain_s"]) + ( " (срок)" if r["drain_timeout"] else "" ) lines.append( f"| {r['step']} | `{fmt(r['new_instance'])}` " f"({fmt(r['new_slot'])}) | `{fmt(r['old_instance'])}` " f"({fmt(r['old_slot'])}) | {fmt(r['new_created_s'])} | " f"{fmt(r['new_up_s'])} | {fmt(r['ready_s'])} | {drain} | " f"{fmt(r['stop_s'])} | {fmt(r['old_removed_s'])} | " f"{fmt(r['slot_ready_s'])} | {fmt(r['replace_s'])} | " f"{r['errors']} | {r['after_drain']} |", ) lines.append("") for title, key in ( ("до", "containers_before"), ("после", "containers_after"), ): items = info.get(key) or [] if not items: continue lines += [ f"## Контейнеры `api` {title} замены", "", "| Контейнер | `X-Instance-Id` | Образ (тег) | Конфигурация образа |", "|---|---|---|---|", ] lines += [ f"| {c['name']} | `{c['instance']}` | `{c['tag']}` | " f"`{c['image'][:19]}…` |" for c in items ] lines.append("") if timeline: end = math.ceil(row["rollout_s"]) if row else 0 shown = [r for r in timeline if -3 <= r["second"] <= end + 3] lines += [ "## Шкала вокруг замены", "", "Полная шкала — `timeline.csv`.", "", "| с от начала | Записи ok / ош | Чтения ok / ош | Коды ошибок | " "Доля успешных | p50 / p95, мс | Старые / новые ok | Ответы " "реплик |", "|---|---|---|---|---|---|---|---|", ] for r in shown: lines.append( f"| {r['second']} | {r['writes_ok']} / {r['writes_failed']} | " f"{r['reads_ok']} / {r['reads_failed']} | {fmt(r['codes'])} | " f"{fmt(r['success'])} | {fmt(r['p50_ms'])} / " f"{fmt(r['p95_ms'])} | {r['old_ok']} / {r['new_ok']} | " f"{fmt(r['instances'])} |", ) lines.append("") if meta.get("host_warnings"): lines += ["## Предупреждения о хосте", ""] lines += [f"- {w}" for w in meta["host_warnings"]] lines.append("") return "\n".join(lines)