Files
mtusi2026lab4/bench/analysis.py
T

547 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Разбор результатов стенда без docker: журналы, окна, признаки, поиск R*.
Чистые функции над строками журналов генератора, событиями Docker,
журналом HAProxy и тиками сэмплера. Их зовёт `run.py`; самопроверка —
`uv run python bench/analysis.py --selftest`. Только stdlib.
Все отметки времени — часы ВМ Docker в миллисекундах эпохи: генератор
(`time.time()` в контейнере), события Docker (`timeNano`), журнал HAProxy
(`docker logs -t`), сэмплер. Часы хоста с ними не сравниваются.
"""
import dataclasses
import datetime
import json
import math
import re
import sys
# --- журнал запросов генератора (профили failover и rollout) ----------------
@dataclasses.dataclass(frozen=True)
class Request:
scheduled: float
sent: float | None
done: float | None
kind: str
status: int
instance: str
key: str
error: str
@property
def ok(self) -> bool:
return self.status == (201 if self.kind == "write" else 200)
@property
def latency(self) -> float | None:
"""Задержка от момента по расписанию до ответа, мс."""
return None if self.done is None else self.done - self.scheduled
def parse_requests(text: str) -> list[Request]:
out = []
for line in text.splitlines():
parts = line.split(",")
if len(parts) != 9 or parts[0] != "fo":
continue
out.append(
Request(
scheduled=float(parts[1]),
sent=float(parts[2]) if parts[2] else None,
done=float(parts[3]) if parts[3] else None,
kind=parts[4],
status=int(parts[5]),
instance=parts[6],
key=parts[7],
error=parts[8],
),
)
out.sort(key=lambda r: r.scheduled)
return out
def percentile(values: list[float], fraction: float) -> float | None:
if not values:
return None
ordered = sorted(values)
return ordered[max(1, math.ceil(fraction * len(ordered))) - 1]
def error_label(request: Request) -> str:
return request.error or str(request.status)
def window_stats(requests: list[Request], start: float, end: float) -> dict:
"""Запросы окна [start, end) по моменту расписания."""
inside = [r for r in requests if start <= r.scheduled < end]
seconds = max((end - start) / 1000, 1e-9)
ok = [r for r in inside if r.ok]
errors: dict[str, int] = {}
for request in inside:
if not request.ok:
errors[error_label(request)] = errors.get(error_label(request), 0) + 1
latencies = [r.latency for r in ok if r.latency is not None]
return {
"seconds": round(seconds, 2),
"requests": len(inside),
"ok": len(ok),
"ok_rps": round(len(ok) / seconds, 1),
"ok_write_rps": round(sum(r.kind == "write" for r in ok) / seconds, 1),
"ok_read_rps": round(sum(r.kind == "read" for r in ok) / seconds, 1),
"errors": sum(errors.values()),
"error_kinds": errors,
"p50_ms": _r(percentile(latencies, 0.5)),
"p95_ms": _r(percentile(latencies, 0.95)),
"p99_ms": _r(percentile(latencies, 0.99)),
"max_ms": _r(max(latencies) if latencies else None),
}
def _r(value: float | None, digits: int = 1) -> float | None:
return None if value is None else round(value, digits)
def timeline(
requests: list[Request],
origin: float,
start: float,
end: float,
groups: dict[str, set[str]] | None = None,
) -> list[dict]:
"""Посекундная шкала от `origin`: успешные по видам, ошибки, реплики.
`groups` — именованные наборы X-Instance-Id (цель и остальные, старые
и новые): у каждого — число успешных ответов в секунду.
"""
first = math.floor((start - origin) / 1000)
last = math.ceil((end - origin) / 1000)
rows = []
for second in range(first, last):
lo, hi = origin + second * 1000, origin + (second + 1) * 1000
inside = [r for r in requests if lo <= r.scheduled < hi]
ok = [r for r in inside if r.ok]
errors: dict[str, int] = {}
for request in inside:
if not request.ok:
errors[error_label(request)] = errors.get(error_label(request), 0) + 1
latencies = [r.latency for r in ok if r.latency is not None]
row = {
"second": second,
"sent": len(inside),
"ok_write": sum(r.kind == "write" for r in ok),
"ok_read": sum(r.kind == "read" for r in ok),
"errors": sum(errors.values()),
"error_kinds": ";".join(f"{k}:{v}" for k, v in sorted(errors.items())),
"p95_ms": _r(percentile(latencies, 0.95)),
"max_ms": _r(max(latencies) if latencies else None),
}
for name, ids in (groups or {}).items():
row[f"ok_{name}"] = sum(r.instance in ids for r in ok)
rows.append(row)
return rows
def recovery_seconds(
requests: list[Request],
action: float,
end: float,
offered_per_s: float,
) -> float | None:
"""Время восстановления, с: от действия до начала первой секунды, после
которой до `end` нет ни одной ошибки и успешных не меньше 90 % от
поданной нагрузки в каждой секунде. None — не восстановилось."""
rows = timeline(requests, action, action, end)
good = [
row["errors"] == 0 and row["ok_write"] + row["ok_read"] >= 0.9 * offered_per_s
for row in rows
]
for index in range(len(good)):
if all(good[index:]):
return float(rows[index]["second"])
return None
# --- события Docker и журнал HAProxy ------------------------------------------------
def parse_events(text: str) -> list[tuple[float, str, str]]:
"""`docker events --format '{{json .}}'`: (мс, действие, id контейнера)."""
events = []
for line in text.splitlines():
line = line.strip()
if not line:
continue
data = json.loads(line)
action = data.get("Action") or data.get("status") or ""
actor = (data.get("Actor") or {}).get("ID") or data.get("id") or ""
events.append((int(data["timeNano"]) / 1e6, action.split(":")[0], actor))
events.sort()
return events
_LB_STATE = re.compile(
r"Server api/(?P<slot>api-\d+)\b.*?(?P<state>is DOWN|is UP|is going DOWN"
r"|enters drain|enters maintenance|is leaving forced drain|is leaving maintenance"
r"|administratively READY)",
)
def parse_docker_time(stamp: str) -> float:
"""RFC 3339 с наносекундами из `docker logs -t` → мс эпохи."""
stamp = stamp.rstrip("Z")
main, _, fraction = stamp.partition(".")
base = datetime.datetime.fromisoformat(main).replace(tzinfo=datetime.UTC)
return base.timestamp() * 1000 + float(f"0.{fraction or 0}") * 1000
def parse_lb_log(text: str) -> list[tuple[float, str, str]]:
"""Смены состояния слотов: (мс, слот, состояние). Дубли строк HAProxy
(мастер и воркер пишут одну и ту же) схлопываются."""
seen = set()
out = []
for line in text.splitlines():
stamp, _, message = line.partition(" ")
match = _LB_STATE.search(message)
if not match:
continue
when = parse_docker_time(stamp)
item = (round(when / 50), match.group("slot"), match.group("state"))
if item in seen:
continue
seen.add(item)
out.append((when, match.group("slot"), match.group("state")))
out.sort()
return out
def lb_http_errors(text: str) -> dict[str, int]:
"""Ответы с ошибкой в журнале HAProxy (option dontlog-normal пишет
только их): статус → число."""
counts: dict[str, int] = {}
for line in text.splitlines():
match = re.search(r'\s(\d{3}) \d+ - - (\S{4}) .*"(GET|POST) ', line)
if match and match.group(1)[0] in "45":
counts[match.group(1)] = counts.get(match.group(1), 0) + 1
return counts
# --- сэмплер ---------------------------------------------------------------------------
def window_ticks(ticks: list[dict], start: float, end: float) -> list[dict]:
"""Тики окна [start, end] (секунды эпохи) плюс по одному снаружи:
приращения считаются по краям окна."""
inside = [t for t in ticks if start <= t["ts"] <= end]
before = [t for t in ticks if t["ts"] < start]
after = [t for t in ticks if t["ts"] > end]
return ([before[-1]] if before else []) + inside + ([after[0]] if after else [])
def cgroup_usage(ticks: list[dict], container: str) -> dict | None:
"""CPU (ядер), доля троттлинга и память контейнера по тикам окна.
Каждый соседний интервал считается отдельно: новая cgroup (контейнер
перезапущен) даёт отрицательное приращение — такой интервал
пропускается, а не обнуляет окно.
"""
points = [(t["ts"], t["cg"].get(container[:12])) for t in ticks]
points = [(ts, line) for ts, line in points if line]
if len(points) < 2:
return None
cpu_us = seconds = periods = throttled = 0.0
memory = 0
for (t0, a), (t1, b) in zip(points, points[1:], strict=False):
if b[0] < a[0]:
continue
cpu_us += b[0] - a[0]
seconds += t1 - t0
periods += b[1] - a[1]
throttled += b[2] - a[2]
memory = max(memory, a[4], b[4])
if seconds <= 0:
return None
return {
"cpu_cores": round(cpu_us / seconds / 1e6, 3),
"throttled_share": round(throttled / periods, 4) if periods else 0.0,
"memory_mib": round(memory / 2**20, 1),
}
def vm_busy(ticks: list[dict]) -> float | None:
if len(ticks) < 2:
return None
busy = ticks[-1]["vm"][0] - ticks[0]["vm"][0]
total = ticks[-1]["vm"][1] - ticks[0]["vm"][1]
return round(busy / total, 4) if total else None
def lb_delta(ticks: list[dict], field: str) -> int | None:
"""Приращение счётчика HAProxy по всем слотам за окно."""
with_lb = [t for t in ticks if t.get("lb")]
if len(with_lb) < 2:
return None
def total(tick: dict) -> int:
return sum(
row.get(field, 0) for name, row in tick["lb"].items() if name != "FRONTEND"
)
return total(with_lb[-1]) - total(with_lb[0])
def pg_stats(ticks: list[dict], replica_addrs: set[str]) -> dict:
"""Соединения приложения и счётчики базы за окно."""
with_pg = [t for t in ticks if t.get("pg")]
if not with_pg:
return {}
per_tick_total, per_replica_max, active, lock_waits = [], 0, [], []
for tick in with_pg:
by_addr: dict[str, int] = {}
busy = waiting = 0
for addr, state, wait, count in tick["pg"]["conn"]:
by_addr[addr] = by_addr.get(addr, 0) + count
if state == "active":
busy += count
if wait == "Lock":
waiting += count
per_tick_total.append(sum(by_addr.values()))
known = [
n for a, n in by_addr.items() if not replica_addrs or a in replica_addrs
]
per_replica_max = max([per_replica_max, *known])
active.append(busy)
lock_waits.append(waiting)
first, last = with_pg[0]["pg"]["db"], with_pg[-1]["pg"]["db"]
seconds = max(with_pg[-1]["ts"] - with_pg[0]["ts"], 1e-9)
return {
"pg_conn_max": max(per_tick_total),
"pg_conn_replica_max": per_replica_max,
"pg_active_mean": round(sum(active) / len(active), 2),
"pg_lock_wait_mean": round(sum(lock_waits) / len(lock_waits), 2),
"pg_new_sessions": last["sessions"] - first["sessions"],
"pg_commits_per_s": round(
(last["xact_commit"] - first["xact_commit"]) / seconds, 1
),
}
# --- признаки узкого места (серии трека 1) ------------------------------------------
# Пороги заданы до прогонов (REPORT.md, гипотезы трека 1). Сработавший
# признак на первой нарушившей ступени называет, во что упёрлась система
THRESHOLDS = {
# реплика: доля своего лимита CPU или заметный троттлинг CFS
"app_cpu_share": 0.85,
"app_throttled_share": 0.20,
# PostgreSQL: доля лимита CPU, средние ожидания блокировок
"db_cpu_share": 0.85,
"db_lock_wait_mean": 1.0,
# пул: соединений одной реплики не меньше доли её предела (размер + переполнение)
"pool_share": 0.90,
# переоткрытие соединений: новых сессий PostgreSQL за окно ступени
"churn_sessions": 20,
# LB: доля его лимита CPU
"lb_cpu_share": 0.85,
# ВМ Docker занята почти целиком: реплики ждут CPU хоста
"host_busy_share": 0.90,
# генератор: доля лимита CPU или отставание от расписания
"gen_cpu_share": 0.90,
"gen_lag_p99_ms": 50.0,
}
def signals(row: dict, limits: dict, pool_cap: int) -> list[str]:
"""Признаки узкого места ступени по агрегатам `row` (см. run.py)."""
found = []
if (row.get("api_cpu_max") or 0) >= THRESHOLDS["app_cpu_share"] * limits[
"api_cpus"
] or (row.get("api_throttled_max") or 0) >= THRESHOLDS["app_throttled_share"]:
found.append("app")
if (row.get("db_cpu_cores") or 0) >= THRESHOLDS["db_cpu_share"] * limits["db_cpus"]:
found.append("db_cpu")
if (row.get("pg_lock_wait_mean") or 0) >= THRESHOLDS["db_lock_wait_mean"]:
found.append("db_lock")
if (
pool_cap
and (row.get("pg_conn_replica_max") or 0) >= THRESHOLDS["pool_share"] * pool_cap
):
found.append("pool")
if (row.get("pg_new_sessions") or 0) >= THRESHOLDS["churn_sessions"]:
found.append("churn")
if (row.get("lb_cpu_cores") or 0) >= THRESHOLDS["lb_cpu_share"] * limits["lb_cpus"]:
found.append("lb")
if (row.get("vm_busy_share") or 0) >= THRESHOLDS["host_busy_share"]:
found.append("host")
if (
(row.get("gen_cpu_cores") or 0)
>= THRESHOLDS["gen_cpu_share"] * limits["gen_cpus"]
or (row.get("lag_p99_ms") or 0) >= THRESHOLDS["gen_lag_p99_ms"]
or (row.get("dropped") or 0) > 0
):
found.append("generator")
return found
# --- поиск R* ---------------------------------------------------------------------------
@dataclasses.dataclass
class Search:
"""Автомат поиска R*: наибольшей интенсивности, на которой ступень
выдерживает SLO.
Сначала фиксированные уровни (сравнение конфигураций на одной
нагрузке), затем лестница `base × factor^i`: начало — первая ступенька
не ниже `start`, подъём до первой нарушившей (если нарушена первая —
спуск до первой прошедшей), затем `bisections` делений интервала
«прошла — нарушила» по среднему геометрическому, с округлением до
целого. R* — наибольшая прошедшая ступень поиска.
"""
fixed: tuple[float, ...]
base: float
factor: float
start: float
bisections: int
cap: float
floor: float = 25.0
results: list[tuple[float, bool, str]] = dataclasses.field(default_factory=list)
def ladder(self, rate: float) -> float:
steps = math.ceil(math.log(rate / self.base, self.factor) - 1e-9)
return round(self.base * self.factor ** max(steps, 0))
def record(self, rate: float, passed: bool, phase: str) -> None:
self.results.append((rate, passed, phase))
def next_rate(self) -> tuple[float, str] | None:
done_fixed = [r for r in self.results if r[2] == "fixed"]
if len(done_fixed) < len(self.fixed):
return self.fixed[len(done_fixed)], "fixed"
search = [r for r in self.results if r[2] != "fixed"]
passed = [rate for rate, ok, _ in search if ok]
failed = [rate for rate, ok, _ in search if not ok]
if not search:
fixed_pass = [rate for rate, ok, _ in done_fixed if ok]
start = max([self.start, *fixed_pass])
return min(self.ladder(start), self.cap), "ladder"
phase = search[-1][2]
if phase == "ladder":
last_rate, last_ok, _ = search[-1]
if last_ok and not failed:
if last_rate >= self.cap:
return None
return min(round(last_rate * self.factor), self.cap), "ladder"
if not last_ok and not passed:
lower = round(last_rate / self.factor)
return (lower, "ladder") if lower >= self.floor else None
# интервал найден: деления
if not passed or not failed:
return None
bisections = sum(1 for r in search if r[2] == "bisect")
if bisections >= self.bisections:
return None
lo = max(passed)
hi = (
min(rate for rate in failed if rate > lo)
if any(r > lo for r in failed)
else None
)
if hi is None:
return None
middle = round(math.sqrt(lo * hi))
if middle <= lo or middle >= hi:
return None
return middle, "bisect"
def r_star(self) -> float | None:
passed = [rate for rate, ok, phase in self.results if ok and phase != "fixed"]
return max(passed) if passed else None
def first_fail(self) -> float | None:
r_star = self.r_star() or 0
failed = [
rate for rate, ok, phase in self.results if not ok and phase != "fixed"
]
above = [rate for rate in failed if rate > r_star]
return min(above) if above else None
# --- самопроверка ---------------------------------------------------------------------
def _selftest() -> None:
# поиск: подъём, нарушение, два деления
search = Search(
fixed=(250, 500), base=100, factor=1.25, start=600, bisections=2, cap=8000
)
capacity = 1000
plan = []
while (step := search.next_rate()) is not None:
rate, phase = step
plan.append((rate, phase))
search.record(rate, rate <= capacity, phase)
assert plan[:2] == [(250, "fixed"), (500, "fixed")], plan
# лестница 100 × 1,25^i: первая ступенька не ниже 600 — 745
assert plan[2] == (745, "ladder"), plan
assert search.r_star() is not None, plan
assert search.r_star() <= capacity, plan
assert search.first_fail() > capacity, plan
assert sum(phase == "bisect" for _, phase in plan) == 2, plan
# первая ступень поиска нарушена — спуск
search = Search(fixed=(), base=100, factor=1.25, start=300, bisections=1, cap=8000)
plan = []
while (step := search.next_rate()) is not None:
plan.append(step)
search.record(step[0], step[0] <= 200, step[1])
assert plan[0] == (305, "ladder"), plan
assert plan[1] == (244, "ladder"), plan
assert search.r_star() <= 200, plan
# окно и время восстановления
reqs = [
Request(
1000 + i * 10, 1000 + i * 10, 1005 + i * 10, "write", 201, "a", f"k{i}", ""
)
for i in range(500)
]
reqs[200] = dataclasses.replace(reqs[200], status=502)
stats = window_stats(reqs, 1000, 6000)
assert stats["errors"] == 1, stats
assert stats["ok"] == 499, stats
assert recovery_seconds(reqs, 1000, 6000, 100) == 3.0
rows = timeline(reqs, 1000, 1000, 6000, {"a": {"a"}})
assert rows[2]["errors"] == 1, rows[:3]
assert rows[0]["ok_a"] == 100, rows[:3]
# сэмплер: приращения, перезапуск контейнера
ticks = [
{"ts": 0.0, "cg": {"abc": [0, 0, 0, 0, 100]}, "vm": [0, 100]},
{"ts": 1.0, "cg": {"abc": [500_000, 10, 1, 0, 200]}, "vm": [50, 200]},
{"ts": 2.0, "cg": {"abc": [100_000, 2, 0, 0, 50]}, "vm": [60, 300]},
{"ts": 3.0, "cg": {"abc": [600_000, 12, 0, 0, 80]}, "vm": [160, 400]},
]
usage = cgroup_usage(ticks, "abc")
assert usage["cpu_cores"] == 0.5, usage
assert vm_busy(ticks) == 0.5333, vm_busy(ticks)
# журнал LB: дубли схлопываются
log = (
"2026-10-14T10:00:00.100000000Z Server api/api-1 is DOWN, reason: x\n"
"2026-10-14T10:00:00.101000000Z [WARNING] (8) : Server api/api-1 is DOWN, reason: x\n"
"2026-10-14T10:00:03.000000000Z Server api/api-1 is UP, reason: y\n"
)
parsed = parse_lb_log(log)
assert [p[2] for p in parsed] == ["is DOWN", "is UP"], parsed
print("analysis selftest: ok")
if __name__ == "__main__":
if sys.argv[1:] == ["--selftest"]:
_selftest()
else:
print("usage: python bench/analysis.py --selftest", file=sys.stderr)
sys.exit(2)