Files
mtusi2026lab4/bench/rollout.py
T
ricoromaandClaude Opus 5.5 5d5d8f9d2a chore: carry lab 3 over
- carry the lab 3 solution over from mtusi2026lab3-safronov
  (solution/bvt2301-safronov, ce89eba): src, migrations, student-tests,
  bench, openspec, .claude, infra/haproxy and infra/valkey, own scripts,
  Dockerfile, compose, pyproject and uv lock
- merge Makefile, .gitignore and .dockerignore (the lab 3 versions only
  add lines to the skeleton, which is the same in lab 4) and
  .env.example (lab 4 skeleton names listed in the header)
- rename the compose project to mtusi-lab4 and the image to :lab4 so
  the stand does not reuse the lab 3 containers and volumes
- update AGENTS.md for lab 4: current lab, origin commit, contract and
  test files of the new skeleton

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 17:32:33 +03:00

677 lines
25 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Профиль поочерёдной замены реплик: разбор и расчёт без docker.
Оркестрирует прогон `run.py` (профили `rollout`, `rollout_quick`), здесь —
чистые функции на stdlib: строки `scripts/rolling-update.sh`, события
Docker проекта, журнал HAProxy, посекундная шкала с ответами старых и
новых реплик, шаги по каждой заменённой реплике и `summary.md`. Строки
k6 о запросах, отметки времени и перцентили — из `failover.py`. Все
отметки — часы ВМ Docker. Решение 8 design change `rolling-update`.
"""
import math
import re
from dataclasses import dataclass
from typing import Any
import failover
import resources as res
ROLLOUT_FIELDS = [
"label",
"from_commit",
"replicas",
"revision_before",
"revision_after",
"build_s",
"migrate_s",
"rollout_s",
"script_exit",
"requests",
"errors_before",
"codes_before",
"errors_during",
"codes_during",
"errors_after",
"codes_after",
"min_success",
"seconds_below",
"ok_rps_before",
"ok_rps_during_min",
"p95_before_ms",
"p95_during_max_ms",
"first_new_s",
"last_old_s",
"old_after_drain",
"writes",
"writes_failed",
"retried_ok",
"write_chat_messages",
"distinct_texts",
"dropped_iterations",
"flags",
]
REPLICA_FIELDS = [
"step",
"new_instance",
"new_slot",
"old_instance",
"old_slot",
"new_created_s",
"new_up_s",
"ready_s",
"drain_s",
"drain_timeout",
"stop_s",
"old_removed_s",
"slot_ready_s",
"replace_s",
"errors",
"after_drain",
]
TIMELINE_FIELDS = [
"second",
"writes_ok",
"writes_failed",
"reads_ok",
"reads_failed",
"codes",
"success",
"p50_ms",
"p95_ms",
"old_ok",
"new_ok",
"instances",
]
SCRIPT_TAG = "rolling-update:"
NEW_UP_RE = re.compile(
r"replica (?P<n>\d+)/\d+: new (?P<id>[0-9a-f]{12}) addr (?P<addr>\S+) "
r"slot (?P<slot>api-\d+) UP",
)
DRAIN_RE = re.compile(
r"replica (?P<n>\d+)/\d+: drain old (?P<id>[0-9a-f]{12}) addr "
r"(?P<addr>\S+) slot (?P<slot>api-\d+)",
)
DRAIN_TIMEOUT_RE = re.compile(r"replica (?P<n>\d+)/\d+: drain timeout")
# запас на отметку отправки k6 против момента drain в журнале LB, с
DRAIN_MARGIN_S = 0.2
BUILD_RE = re.compile(r"build done in (\d+)s")
MIGRATE_RE = re.compile(r"migrate done in (\d+)s")
# состояния слота в журнале HAProxy: ручные (drain, maint, возврат из
# maint) и по проверкам и DNS
LB_STATES = (
("enters drain state", "DRAIN"),
("is going DOWN for maintenance (No IP", "DNS_MAINT"),
("is going DOWN for maintenance", "MAINT"),
("leaving forced maintenance", "READY"),
("is UP", "UP"),
("is DOWN", "DOWN"),
)
@dataclass(frozen=True)
class ContainerEvent:
"""Событие Docker контейнера проекта: время, действие, полный ID."""
time: float
action: str
id: str
name: str
signal: str
@dataclass(frozen=True)
class SlotEvent:
time: float
slot: str
state: str
def script_lines(text: str) -> str:
"""Строки самого скрипта, без вывода сборки и миграции."""
return "\n".join(line for line in text.splitlines() if SCRIPT_TAG in line)
def script_error(text: str) -> str:
"""Последняя строка ошибки скрипта — в сообщение об отказе."""
errors = [
line
for line in text.splitlines()
if SCRIPT_TAG in line and "error:" in line
]
return errors[-1].split(SCRIPT_TAG, 1)[1].strip() if errors else ""
def parse_steps(text: str) -> list[dict[str, Any]]:
"""Шаги замены из строк скрипта: новая и старая реплика, их слоты."""
steps: dict[int, dict[str, Any]] = {}
for line in text.splitlines():
if match := NEW_UP_RE.search(line):
step = steps.setdefault(int(match["n"]), {})
step.update(
new_id=match["id"],
new_addr=match["addr"],
new_slot=match["slot"],
)
elif match := DRAIN_RE.search(line):
step = steps.setdefault(int(match["n"]), {})
step.update(
old_id=match["id"],
old_addr=match["addr"],
old_slot=match["slot"],
)
elif match := DRAIN_TIMEOUT_RE.search(line):
steps.setdefault(int(match["n"]), {})["drain_timeout"] = True
return [{"step": n, **steps[n]} for n in sorted(steps)]
def seconds(pattern: re.Pattern[str], text: str) -> int | None:
match = pattern.search(text)
return int(match.group(1)) if match else None
def parse_container_events(text: str) -> list[ContainerEvent]:
"""`docker events` в формате `время|действие|ID|имя|сигнал`.
Действие exec содержит команду, поэтому поля берутся с краёв.
"""
events = []
for line in text.splitlines():
parts = line.split("|")
if len(parts) < 5 or not parts[0].isdigit():
continue
events.append(
ContainerEvent(
time=int(parts[0]) / 1e9,
action="|".join(parts[1:-3]),
id=parts[-3],
name=parts[-2],
signal=parts[-1],
),
)
return sorted(events, key=lambda e: e.time)
def parse_slot_events(text: str) -> list[SlotEvent]:
"""Смена состояния слотов из журнала LB (`docker logs -t`); повтор
одной строки предупреждением в пределах миллисекунды отбрасывается."""
events: list[SlotEvent] = []
for line in text.splitlines():
match = failover.LB_RE.match(line)
if not match:
continue
state = next(
(name for marker, name in LB_STATES if marker in match["rest"]),
None,
)
if state is None:
continue
event = SlotEvent(
failover.parse_stamp(match["stamp"]),
match["slot"],
state,
)
if any(
e.slot == event.slot
and e.state == event.state
and abs(e.time - event.time) < 0.001
for e in events
):
continue
events.append(event)
return sorted(events, key=lambda e: e.time)
def container_event(
events: list[ContainerEvent],
short_id: str | None,
action: str,
since: float = 0.0,
) -> ContainerEvent | None:
if not short_id:
return None
return next(
(
e
for e in events
if e.id.startswith(short_id)
and e.action == action
and e.time >= since
),
None,
)
def slot_event(
events: list[SlotEvent],
slot: str | None,
state: str,
since: float | None,
) -> SlotEvent | None:
if slot is None or since is None:
return None
return next(
(
e
for e in events
if e.slot == slot and e.state == state and e.time >= since
),
None,
)
def start_time(events: list[ContainerEvent]) -> float | None:
"""Начало изменения стенда: создание контейнера миграции, без него —
первой новой реплики (миграции пропущены)."""
# контейнер сервиса migrate (scripts/rolling-update.sh пересоздаёт его
# на новом образе) или одноразовый migrate-run
migrate = next(
(e for e in events if e.action == "create" and "-migrate-" in e.name),
None,
)
if migrate is not None:
return migrate.time
create = next((e for e in events if e.action == "create"), None)
return create.time if create else None
def script_start(
events: list[ContainerEvent],
since: float,
ignore: frozenset[str] | set[str] = frozenset(),
) -> float | None:
"""Начало подготовки замены (решение 4 design change
`profile-resources`): первое `exec_start` в контейнер проекта не
раньше `since` (первого запроса) — первое обращение скрипта к LB.
Оркестратор во время нагрузки `docker exec` не делает, k6 — вне
проекта; `exec_start` проверок здоровья (`ignore` — их действия)
отбрасываются (найдено в apply: они тоже приходят каждые 5 с)."""
return next(
(
e.time
for e in events
if e.action.startswith("exec_start")
and e.action not in ignore
and e.time >= since
),
None,
)
def diff(later: Any, earlier: Any) -> float | None:
if later is None or earlier is None:
return None
return round(later.time - earlier.time, 2)
def since_t0(event: Any, t0: float) -> float | None:
return None if event is None else round(event.time - t0, 2)
def replica_rows(
steps: list[dict[str, Any]],
containers: list[ContainerEvent],
slots: list[SlotEvent],
requests: list[failover.Request],
t0: float,
) -> list[dict[str, Any]]:
"""Шаги каждой замены: от создания новой до возврата слота старой."""
rows = []
for step in steps:
created = container_event(containers, step.get("new_id"), "create")
started = container_event(containers, step.get("new_id"), "start")
up = slot_event(
slots,
step.get("new_slot"),
"UP",
started.time if started else None,
)
drain = slot_event(
slots,
step.get("old_slot"),
"DRAIN",
up.time if up else None,
)
maint = slot_event(
slots,
step.get("old_slot"),
"MAINT",
drain.time if drain else None,
)
kill = container_event(
containers,
step.get("old_id"),
"kill",
maint.time if maint else 0.0,
)
die = container_event(
containers,
step.get("old_id"),
"die",
kill.time if kill else 0.0,
)
destroy = container_event(containers, step.get("old_id"), "destroy")
ready = slot_event(
slots,
step.get("old_slot"),
"READY",
destroy.time if destroy else None,
)
window = [
r
for r in requests
if created is not None
and destroy is not None
and created.time <= r.start < destroy.time
]
# ответы старой реплики, отправленные после её вывода из LB: запрос
# мог уйти до команды drain на доли миллисекунды раньше отметки k6,
# поэтому запас DRAIN_MARGIN_S
after_drain = [
r
for r in requests
if drain is not None
and r.instance == step.get("old_id")
and r.start > drain.time + DRAIN_MARGIN_S
]
rows.append(
{
"step": step["step"],
"new_instance": step.get("new_id"),
"new_slot": step.get("new_slot"),
"old_instance": step.get("old_id"),
"old_slot": step.get("old_slot"),
"new_created_s": since_t0(created, t0),
"new_up_s": since_t0(up, t0),
"ready_s": diff(up, started),
"drain_s": diff(maint, drain),
"drain_timeout": bool(step.get("drain_timeout")),
"stop_s": diff(die, kill),
"old_removed_s": since_t0(destroy, t0),
"slot_ready_s": since_t0(ready, t0),
"replace_s": diff(destroy, created),
"errors": sum(1 for r in window if not r.ok),
"after_drain": len(after_drain),
},
)
return rows
def build_timeline(
requests: list[failover.Request],
t0: float,
old: set[str],
new: set[str],
) -> list[dict[str, Any]]:
"""Посекундная шкала от начала замены; секунда — по отправке.
`old` и `new` — `X-Instance-Id` (12 символов) реплик до и после.
"""
buckets: dict[int, list[failover.Request]] = {}
for request in requests:
buckets.setdefault(math.floor(request.start - t0), []).append(request)
rows = []
for second in range(min(buckets, default=0), max(buckets, default=-1) + 1):
group = buckets.get(second, [])
ok = [r for r in group if r.ok]
instances: dict[str, int] = {}
for request in ok:
instances[request.instance] = (
instances.get(request.instance, 0) + 1
)
durations = [r.duration_ms for r in ok]
rows.append(
{
"second": second,
"writes_ok": sum(1 for r in ok if r.kind == "w"),
"writes_failed": sum(
1 for r in group if r.kind == "w" and not r.ok
),
"reads_ok": sum(1 for r in ok if r.kind == "r"),
"reads_failed": sum(
1 for r in group if r.kind == "r" and not r.ok
),
"codes": failover.codes(group),
"success": round(len(ok) / len(group), 4) if group else None,
"p50_ms": failover.percentile(durations, 0.5),
"p95_ms": failover.percentile(durations, 0.95),
"old_ok": sum(1 for r in ok if r.instance in old),
"new_ok": sum(1 for r in ok if r.instance in new),
"instances": " ".join(
f"{key}:{value}"
for key, value in sorted(instances.items())
),
},
)
return rows
def analyze(
requests: list[failover.Request],
timeline: list[dict[str, Any]],
replicas: list[dict[str, Any]],
*,
t0: float,
end: float,
old: set[str],
new: set[str],
threshold: float,
) -> dict[str, Any]:
"""Величины прогона: окна до, во время и после замены (решение 8)."""
def ended(request: failover.Request) -> float:
return request.start + request.duration_ms / 1000
before = [r for r in requests if ended(r) < t0]
during = [r for r in requests if ended(r) >= t0 and r.start < end]
after = [r for r in requests if r.start >= end]
window = math.ceil(end - t0)
during_rows = [row for row in timeline if 0 <= row["second"] < window]
before_rows = [row for row in timeline if row["second"] < 0][2:]
# после вывода из LB старая реплика запросов получать не должна
old_after_drain = sum(r["after_drain"] for r in replicas)
new_ok = [r.start for r in requests if r.ok and r.instance in new]
old_ok = [r.start for r in requests if r.ok and r.instance in old]
p95_during = [
row["p95_ms"] for row in during_rows if row["p95_ms"] is not None
]
flags = []
if any(not r.ok for r in requests):
flags.append("errors")
if old_after_drain:
flags.append("old_after_drain")
return {
"rollout_s": round(end - t0, 2),
"requests": len(requests),
"errors_before": sum(1 for r in before if not r.ok),
"codes_before": failover.codes(before),
"errors_during": sum(1 for r in during if not r.ok),
"codes_during": failover.codes(during),
"errors_after": sum(1 for r in after if not r.ok),
"codes_after": failover.codes(after),
"min_success": min(
(
row["success"]
for row in during_rows
if row["success"] is not None
),
default=None,
),
"seconds_below": sum(
1
for row in during_rows
if row["success"] is not None and row["success"] < threshold
),
"ok_rps_before": (
round(
sum(row["writes_ok"] + row["reads_ok"] for row in before_rows)
/ len(before_rows),
1,
)
if before_rows
else None
),
"ok_rps_during_min": min(
(row["writes_ok"] + row["reads_ok"] for row in during_rows),
default=None,
),
"p95_before_ms": failover.percentile(
[r.duration_ms for r in before if r.ok],
0.95,
),
"p95_during_max_ms": max(p95_during, default=None),
"first_new_s": round(min(new_ok) - t0, 2) if new_ok else None,
"last_old_s": round(max(old_ok) - t0, 2) if old_ok else None,
"old_after_drain": old_after_drain,
"writes": sum(1 for r in requests if r.kind == "w"),
"writes_failed": sum(
1 for r in requests if r.kind == "w" and not r.ok
),
"flags": " ".join(flags),
}
fmt = failover.fmt
def render_summary(
meta: dict[str, Any],
row: dict[str, Any] | None,
replicas: list[dict[str, Any]],
timeline: list[dict[str, Any]],
resources: list[dict[str, Any]] | None = None,
) -> str:
"""`summary.md` профиля замены: условия, итог, шаги по репликам,
контейнеры до и после, шкала вокруг замены; с `resources` (строки
`resources.csv`) — ещё ресурсы по окнам замены."""
profile = meta["profile"]
info = meta.get("rollout", {})
lines = [
f"# Поочерёдная замена реплик под нагрузкой: метка `{meta['label']}`",
"",
f"- Статус: **{meta['status']}**"
+ (f" ({meta['reason']})" if meta.get("reason") else ""),
f"- Commit: `{meta['git']['commit']}`"
+ (" + незакоммиченные изменения" if meta["git"]["dirty"] else ""),
f"- Профиль `{profile['name']}`: {profile['replicas']} реплики за "
f"HAProxy; замена с образа commit `{info.get('from_commit')}` на "
"образ текущего checkout (`scripts/rolling-update.sh`); "
f"{profile['failover_write_rate']} записей с ключом и "
f"{profile['failover_read_rate']} чтений страницы истории в "
"секунду (k6, постоянная интенсивность); окна: "
f"{profile['rollout_before_s']} с до запуска скрипта, бюджет "
f"замены {profile['rollout_budget_s']} с, ещё "
f"{profile['rollout_after_s']} с; порог доли успешных "
f"{profile['failover_success']:.0%}",
failover.limits_line(profile),
"- Посторонние контейнеры в VM во время замера: "
+ (", ".join(meta.get("other_containers") or []) or "нет"),
"",
"Время — секунды на часах ВМ Docker от начала замены (создание "
"контейнера миграции). Шаги — события Docker (создание и старт "
"новой реплики, SIGTERM, выход и удаление старой) и журнал HAProxy "
"(`UP` новой, `drain` и `maint` старой, возврат слота в `ready`). "
"Ответы — отметки k6 при отправке; «старые» и «новые» — по "
"`X-Instance-Id` контейнеров до и после замены. Окно замены — от "
"начала до удаления последней старой реплики. Сверка — после "
"повтора неуспешных записей с тем же ключом: сообщений в чате "
"записи и разных текстов столько, сколько отправлено ключей.",
"",
]
if row:
lines += [
"| Замена, с | Сборка / миграция, с | Ревизия до → после | "
"Ошибки до / во время / после | Мин. доля успешных | Секунд ниже "
"порога | Успешных в секунду: до / мин. во время | p95: до / "
"макс. во время, мс | Первый ответ новой / последний старой, с | "
"Ответов старых после дренажа | Повтор записей | "
"Сообщений = ключей | Пометки |",
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
f"| {fmt(row['rollout_s'])} | {fmt(row['build_s'])} / "
f"{fmt(row['migrate_s'])} | `{fmt(row['revision_before'])}` → "
f"`{fmt(row['revision_after'])}` | {row['errors_before']} / "
f"{row['errors_during']}"
+ (f" ({row['codes_during']})" if row["codes_during"] else "")
+ f" / {row['errors_after']} | {fmt(row['min_success'])} | "
f"{row['seconds_below']} | {fmt(row['ok_rps_before'])} / "
f"{fmt(row['ok_rps_during_min'])} | {fmt(row['p95_before_ms'])} / "
f"{fmt(row['p95_during_max_ms'])} | {fmt(row['first_new_s'])} / "
f"{fmt(row['last_old_s'])} | {row['old_after_drain']} | "
f"{row['writes_failed']} → {fmt(row['retried_ok'])} | "
f"{fmt(row['write_chat_messages'])} = "
f"{fmt(row['distinct_texts'])} | {fmt(row['flags'])} |",
"",
]
if replicas:
lines += [
"## Шаги по репликам",
"",
"| Шаг | Новая (слот) | Старая (слот) | Новая создана, с | Новая "
"`UP`, с | Старт → `UP`, с | Дренаж, с | Остановка, с | Старая "
"удалена, с | Слот `ready`, с | Замена целиком, с | Ошибок за "
"шаг | Ответов старой после дренажа |",
"|---|---|---|---|---|---|---|---|---|---|---|---|---|",
]
for r in replicas:
drain = fmt(r["drain_s"]) + (
" (срок)" if r["drain_timeout"] else ""
)
lines.append(
f"| {r['step']} | `{fmt(r['new_instance'])}` "
f"({fmt(r['new_slot'])}) | `{fmt(r['old_instance'])}` "
f"({fmt(r['old_slot'])}) | {fmt(r['new_created_s'])} | "
f"{fmt(r['new_up_s'])} | {fmt(r['ready_s'])} | {drain} | "
f"{fmt(r['stop_s'])} | {fmt(r['old_removed_s'])} | "
f"{fmt(r['slot_ready_s'])} | {fmt(r['replace_s'])} | "
f"{r['errors']} | {r['after_drain']} |",
)
lines.append("")
if resources:
lines += res.render_rollout(resources)
for title, key in (
("до", "containers_before"),
("после", "containers_after"),
):
items = info.get(key) or []
if not items:
continue
lines += [
f"## Контейнеры `api` {title} замены",
"",
"| Контейнер | `X-Instance-Id` | Образ (тег) | Конфигурация образа |",
"|---|---|---|---|",
]
lines += [
f"| {c['name']} | `{c['instance']}` | `{c['tag']}` | "
f"`{c['image'][:19]}…` |"
for c in items
]
lines.append("")
if timeline:
end = math.ceil(row["rollout_s"]) if row else 0
shown = [r for r in timeline if -3 <= r["second"] <= end + 3]
lines += [
"## Шкала вокруг замены",
"",
"Полная шкала — `timeline.csv`.",
"",
"| с от начала | Записи ok / ош | Чтения ok / ош | Коды ошибок | "
"Доля успешных | p50 / p95, мс | Старые / новые ok | Ответы "
"реплик |",
"|---|---|---|---|---|---|---|---|",
]
for r in shown:
lines.append(
f"| {r['second']} | {r['writes_ok']} / {r['writes_failed']} | "
f"{r['reads_ok']} / {r['reads_failed']} | {fmt(r['codes'])} | "
f"{fmt(r['success'])} | {fmt(r['p50_ms'])} / "
f"{fmt(r['p95_ms'])} | {r['old_ok']} / {r['new_ok']} | "
f"{fmt(r['instances'])} |",
)
lines.append("")
if meta.get("host_warnings"):
lines += ["## Предупреждения о хосте", ""]
lines += [f"- {w}" for w in meta["host_warnings"]]
lines.append("")
return "\n".join(lines)