"""Разбор результатов стенда без docker: журналы, окна, признаки, поиск R*. Чистые функции над строками журналов генератора, событиями Docker, журналом HAProxy и тиками сэмплера. Их зовёт `run.py`; самопроверка — `uv run python bench/analysis.py --selftest`. Только stdlib. Все отметки времени — часы ВМ Docker в миллисекундах эпохи: генератор (`time.time()` в контейнере), события Docker (`timeNano`), журнал HAProxy (`docker logs -t`), сэмплер. Часы хоста с ними не сравниваются. """ import dataclasses import datetime import json import math import re import sys # --- журнал запросов генератора (профили failover и rollout) ---------------- @dataclasses.dataclass(frozen=True) class Request: scheduled: float sent: float | None done: float | None kind: str status: int instance: str key: str error: str @property def ok(self) -> bool: return self.status == (201 if self.kind == "write" else 200) @property def latency(self) -> float | None: """Задержка от момента по расписанию до ответа, мс.""" return None if self.done is None else self.done - self.scheduled def parse_requests(text: str) -> list[Request]: out = [] for line in text.splitlines(): parts = line.split(",") if len(parts) != 9 or parts[0] != "fo": continue out.append( Request( scheduled=float(parts[1]), sent=float(parts[2]) if parts[2] else None, done=float(parts[3]) if parts[3] else None, kind=parts[4], status=int(parts[5]), instance=parts[6], key=parts[7], error=parts[8], ), ) out.sort(key=lambda r: r.scheduled) return out def percentile(values: list[float], fraction: float) -> float | None: if not values: return None ordered = sorted(values) return ordered[max(1, math.ceil(fraction * len(ordered))) - 1] def error_label(request: Request) -> str: return request.error or str(request.status) def window_stats(requests: list[Request], start: float, end: float) -> dict: """Запросы окна [start, end) по моменту расписания.""" inside = [r for r in requests if start <= r.scheduled < end] seconds = max((end - start) / 1000, 1e-9) ok = [r for r in inside if r.ok] errors: dict[str, int] = {} for request in inside: if not request.ok: errors[error_label(request)] = errors.get(error_label(request), 0) + 1 latencies = [r.latency for r in ok if r.latency is not None] return { "seconds": round(seconds, 2), "requests": len(inside), "ok": len(ok), "ok_rps": round(len(ok) / seconds, 1), "ok_write_rps": round(sum(r.kind == "write" for r in ok) / seconds, 1), "ok_read_rps": round(sum(r.kind == "read" for r in ok) / seconds, 1), "errors": sum(errors.values()), "error_kinds": errors, "p50_ms": _r(percentile(latencies, 0.5)), "p95_ms": _r(percentile(latencies, 0.95)), "p99_ms": _r(percentile(latencies, 0.99)), "max_ms": _r(max(latencies) if latencies else None), } def _r(value: float | None, digits: int = 1) -> float | None: return None if value is None else round(value, digits) def timeline( requests: list[Request], origin: float, start: float, end: float, groups: dict[str, set[str]] | None = None, ) -> list[dict]: """Посекундная шкала от `origin`: успешные по видам, ошибки, реплики. `groups` — именованные наборы X-Instance-Id (цель и остальные, старые и новые): у каждого — число успешных ответов в секунду. """ first = math.floor((start - origin) / 1000) last = math.ceil((end - origin) / 1000) rows = [] for second in range(first, last): lo, hi = origin + second * 1000, origin + (second + 1) * 1000 inside = [r for r in requests if lo <= r.scheduled < hi] ok = [r for r in inside if r.ok] errors: dict[str, int] = {} for request in inside: if not request.ok: errors[error_label(request)] = errors.get(error_label(request), 0) + 1 latencies = [r.latency for r in ok if r.latency is not None] row = { "second": second, "sent": len(inside), "ok_write": sum(r.kind == "write" for r in ok), "ok_read": sum(r.kind == "read" for r in ok), "errors": sum(errors.values()), "error_kinds": ";".join(f"{k}:{v}" for k, v in sorted(errors.items())), "p95_ms": _r(percentile(latencies, 0.95)), "max_ms": _r(max(latencies) if latencies else None), } for name, ids in (groups or {}).items(): row[f"ok_{name}"] = sum(r.instance in ids for r in ok) rows.append(row) return rows def recovery_seconds( requests: list[Request], action: float, end: float, offered_per_s: float, ) -> float | None: """Время восстановления, с: от действия до начала первой секунды, после которой до `end` нет ни одной ошибки и успешных не меньше 90 % от поданной нагрузки в каждой секунде. None — не восстановилось.""" rows = timeline(requests, action, action, end) good = [ row["errors"] == 0 and row["ok_write"] + row["ok_read"] >= 0.9 * offered_per_s for row in rows ] for index in range(len(good)): if all(good[index:]): return float(rows[index]["second"]) return None # --- события Docker и журнал HAProxy ------------------------------------------------ def parse_events(text: str) -> list[tuple[float, str, str]]: """`docker events --format '{{json .}}'`: (мс, действие, id контейнера).""" events = [] for line in text.splitlines(): line = line.strip() if not line: continue data = json.loads(line) action = data.get("Action") or data.get("status") or "" actor = (data.get("Actor") or {}).get("ID") or data.get("id") or "" events.append((int(data["timeNano"]) / 1e6, action.split(":")[0], actor)) events.sort() return events _LB_STATE = re.compile( r"Server api/(?Papi-\d+)\b.*?(?Pis DOWN|is UP|is going DOWN" r"|enters drain|enters maintenance|is leaving forced drain|is leaving maintenance" r"|administratively READY)", ) def parse_docker_time(stamp: str) -> float: """RFC 3339 с наносекундами из `docker logs -t` → мс эпохи.""" stamp = stamp.rstrip("Z") main, _, fraction = stamp.partition(".") base = datetime.datetime.fromisoformat(main).replace(tzinfo=datetime.UTC) return base.timestamp() * 1000 + float(f"0.{fraction or 0}") * 1000 def parse_lb_log(text: str) -> list[tuple[float, str, str]]: """Смены состояния слотов: (мс, слот, состояние). Дубли строк HAProxy (мастер и воркер пишут одну и ту же) схлопываются.""" seen = set() out = [] for line in text.splitlines(): stamp, _, message = line.partition(" ") match = _LB_STATE.search(message) if not match: continue when = parse_docker_time(stamp) item = (round(when / 50), match.group("slot"), match.group("state")) if item in seen: continue seen.add(item) out.append((when, match.group("slot"), match.group("state"))) out.sort() return out def lb_http_errors(text: str) -> dict[str, int]: """Ответы с ошибкой в журнале HAProxy (option dontlog-normal пишет только их): статус → число.""" counts: dict[str, int] = {} for line in text.splitlines(): match = re.search(r'\s(\d{3}) \d+ - - (\S{4}) .*"(GET|POST) ', line) if match and match.group(1)[0] in "45": counts[match.group(1)] = counts.get(match.group(1), 0) + 1 return counts # --- сэмплер --------------------------------------------------------------------------- def window_ticks(ticks: list[dict], start: float, end: float) -> list[dict]: """Тики окна [start, end] (секунды эпохи) плюс по одному снаружи: приращения считаются по краям окна.""" inside = [t for t in ticks if start <= t["ts"] <= end] before = [t for t in ticks if t["ts"] < start] after = [t for t in ticks if t["ts"] > end] return ([before[-1]] if before else []) + inside + ([after[0]] if after else []) def cgroup_usage(ticks: list[dict], container: str) -> dict | None: """CPU (ядер), доля троттлинга и память контейнера по тикам окна. Каждый соседний интервал считается отдельно: новая cgroup (контейнер перезапущен) даёт отрицательное приращение — такой интервал пропускается, а не обнуляет окно. """ points = [(t["ts"], t["cg"].get(container[:12])) for t in ticks] points = [(ts, line) for ts, line in points if line] if len(points) < 2: return None cpu_us = seconds = periods = throttled = 0.0 memory = 0 for (t0, a), (t1, b) in zip(points, points[1:], strict=False): if b[0] < a[0]: continue cpu_us += b[0] - a[0] seconds += t1 - t0 periods += b[1] - a[1] throttled += b[2] - a[2] memory = max(memory, a[4], b[4]) if seconds <= 0: return None return { "cpu_cores": round(cpu_us / seconds / 1e6, 3), "throttled_share": round(throttled / periods, 4) if periods else 0.0, "memory_mib": round(memory / 2**20, 1), } def vm_busy(ticks: list[dict]) -> float | None: if len(ticks) < 2: return None busy = ticks[-1]["vm"][0] - ticks[0]["vm"][0] total = ticks[-1]["vm"][1] - ticks[0]["vm"][1] return round(busy / total, 4) if total else None def lb_delta(ticks: list[dict], field: str) -> int | None: """Приращение счётчика HAProxy по всем слотам за окно.""" with_lb = [t for t in ticks if t.get("lb")] if len(with_lb) < 2: return None def total(tick: dict) -> int: return sum( row.get(field, 0) for name, row in tick["lb"].items() if name != "FRONTEND" ) return total(with_lb[-1]) - total(with_lb[0]) def pg_stats(ticks: list[dict], replica_addrs: set[str]) -> dict: """Соединения приложения и счётчики базы за окно.""" with_pg = [t for t in ticks if t.get("pg")] if not with_pg: return {} per_tick_total, per_replica_max, active, lock_waits = [], 0, [], [] for tick in with_pg: by_addr: dict[str, int] = {} busy = waiting = 0 for addr, state, wait, count in tick["pg"]["conn"]: by_addr[addr] = by_addr.get(addr, 0) + count if state == "active": busy += count if wait == "Lock": waiting += count per_tick_total.append(sum(by_addr.values())) known = [ n for a, n in by_addr.items() if not replica_addrs or a in replica_addrs ] per_replica_max = max([per_replica_max, *known]) active.append(busy) lock_waits.append(waiting) first, last = with_pg[0]["pg"]["db"], with_pg[-1]["pg"]["db"] seconds = max(with_pg[-1]["ts"] - with_pg[0]["ts"], 1e-9) return { "pg_conn_max": max(per_tick_total), "pg_conn_replica_max": per_replica_max, "pg_active_mean": round(sum(active) / len(active), 2), "pg_lock_wait_mean": round(sum(lock_waits) / len(lock_waits), 2), "pg_new_sessions": last["sessions"] - first["sessions"], "pg_commits_per_s": round( (last["xact_commit"] - first["xact_commit"]) / seconds, 1 ), } # --- признаки узкого места (серии трека 1) ------------------------------------------ # Пороги заданы до прогонов (REPORT.md, гипотезы трека 1). Сработавший # признак на первой нарушившей ступени называет, во что упёрлась система THRESHOLDS = { # реплика: доля своего лимита CPU или заметный троттлинг CFS "app_cpu_share": 0.85, "app_throttled_share": 0.20, # PostgreSQL: доля лимита CPU, средние ожидания блокировок "db_cpu_share": 0.85, "db_lock_wait_mean": 1.0, # пул: соединений одной реплики не меньше доли её предела (размер + переполнение) "pool_share": 0.90, # переоткрытие соединений: новых сессий PostgreSQL за окно ступени "churn_sessions": 20, # LB: доля его лимита CPU "lb_cpu_share": 0.85, # ВМ Docker занята почти целиком: реплики ждут CPU хоста "host_busy_share": 0.90, # генератор: доля лимита CPU или отставание от расписания "gen_cpu_share": 0.90, "gen_lag_p99_ms": 50.0, } def signals(row: dict, limits: dict, pool_cap: int) -> list[str]: """Признаки узкого места ступени по агрегатам `row` (см. run.py).""" found = [] if (row.get("api_cpu_max") or 0) >= THRESHOLDS["app_cpu_share"] * limits[ "api_cpus" ] or (row.get("api_throttled_max") or 0) >= THRESHOLDS["app_throttled_share"]: found.append("app") if (row.get("db_cpu_cores") or 0) >= THRESHOLDS["db_cpu_share"] * limits["db_cpus"]: found.append("db_cpu") if (row.get("pg_lock_wait_mean") or 0) >= THRESHOLDS["db_lock_wait_mean"]: found.append("db_lock") if ( pool_cap and (row.get("pg_conn_replica_max") or 0) >= THRESHOLDS["pool_share"] * pool_cap ): found.append("pool") if (row.get("pg_new_sessions") or 0) >= THRESHOLDS["churn_sessions"]: found.append("churn") if (row.get("lb_cpu_cores") or 0) >= THRESHOLDS["lb_cpu_share"] * limits["lb_cpus"]: found.append("lb") if (row.get("vm_busy_share") or 0) >= THRESHOLDS["host_busy_share"]: found.append("host") if ( (row.get("gen_cpu_cores") or 0) >= THRESHOLDS["gen_cpu_share"] * limits["gen_cpus"] or (row.get("lag_p99_ms") or 0) >= THRESHOLDS["gen_lag_p99_ms"] or (row.get("dropped") or 0) > 0 ): found.append("generator") return found # --- поиск R* --------------------------------------------------------------------------- @dataclasses.dataclass class Search: """Автомат поиска R*: наибольшей интенсивности, на которой ступень выдерживает SLO. Сначала фиксированные уровни (сравнение конфигураций на одной нагрузке), затем лестница `base × factor^i`: начало — первая ступенька не ниже `start`, подъём до первой нарушившей (если нарушена первая — спуск до первой прошедшей), затем `bisections` делений интервала «прошла — нарушила» по среднему геометрическому, с округлением до целого. R* — наибольшая прошедшая ступень поиска. """ fixed: tuple[float, ...] base: float factor: float start: float bisections: int cap: float floor: float = 25.0 results: list[tuple[float, bool, str]] = dataclasses.field(default_factory=list) def ladder(self, rate: float) -> float: steps = math.ceil(math.log(rate / self.base, self.factor) - 1e-9) return round(self.base * self.factor ** max(steps, 0)) def record(self, rate: float, passed: bool, phase: str) -> None: self.results.append((rate, passed, phase)) def next_rate(self) -> tuple[float, str] | None: done_fixed = [r for r in self.results if r[2] == "fixed"] if len(done_fixed) < len(self.fixed): return self.fixed[len(done_fixed)], "fixed" search = [r for r in self.results if r[2] != "fixed"] passed = [rate for rate, ok, _ in search if ok] failed = [rate for rate, ok, _ in search if not ok] if not search: fixed_pass = [rate for rate, ok, _ in done_fixed if ok] start = max([self.start, *fixed_pass]) return min(self.ladder(start), self.cap), "ladder" phase = search[-1][2] if phase == "ladder": last_rate, last_ok, _ = search[-1] if last_ok and not failed: if last_rate >= self.cap: return None return min(round(last_rate * self.factor), self.cap), "ladder" if not last_ok and not passed: lower = round(last_rate / self.factor) return (lower, "ladder") if lower >= self.floor else None # интервал найден: деления if not passed or not failed: return None bisections = sum(1 for r in search if r[2] == "bisect") if bisections >= self.bisections: return None lo = max(passed) hi = ( min(rate for rate in failed if rate > lo) if any(r > lo for r in failed) else None ) if hi is None: return None middle = round(math.sqrt(lo * hi)) if middle <= lo or middle >= hi: return None return middle, "bisect" def r_star(self) -> float | None: passed = [rate for rate, ok, phase in self.results if ok and phase != "fixed"] return max(passed) if passed else None def first_fail(self) -> float | None: r_star = self.r_star() or 0 failed = [ rate for rate, ok, phase in self.results if not ok and phase != "fixed" ] above = [rate for rate in failed if rate > r_star] return min(above) if above else None # --- самопроверка --------------------------------------------------------------------- def _selftest() -> None: # поиск: подъём, нарушение, два деления search = Search( fixed=(250, 500), base=100, factor=1.25, start=600, bisections=2, cap=8000 ) capacity = 1000 plan = [] while (step := search.next_rate()) is not None: rate, phase = step plan.append((rate, phase)) search.record(rate, rate <= capacity, phase) assert plan[:2] == [(250, "fixed"), (500, "fixed")], plan # лестница 100 × 1,25^i: первая ступенька не ниже 600 — 745 assert plan[2] == (745, "ladder"), plan assert search.r_star() is not None, plan assert search.r_star() <= capacity, plan assert search.first_fail() > capacity, plan assert sum(phase == "bisect" for _, phase in plan) == 2, plan # первая ступень поиска нарушена — спуск search = Search(fixed=(), base=100, factor=1.25, start=300, bisections=1, cap=8000) plan = [] while (step := search.next_rate()) is not None: plan.append(step) search.record(step[0], step[0] <= 200, step[1]) assert plan[0] == (305, "ladder"), plan assert plan[1] == (244, "ladder"), plan assert search.r_star() <= 200, plan # окно и время восстановления reqs = [ Request( 1000 + i * 10, 1000 + i * 10, 1005 + i * 10, "write", 201, "a", f"k{i}", "" ) for i in range(500) ] reqs[200] = dataclasses.replace(reqs[200], status=502) stats = window_stats(reqs, 1000, 6000) assert stats["errors"] == 1, stats assert stats["ok"] == 499, stats assert recovery_seconds(reqs, 1000, 6000, 100) == 3.0 rows = timeline(reqs, 1000, 1000, 6000, {"a": {"a"}}) assert rows[2]["errors"] == 1, rows[:3] assert rows[0]["ok_a"] == 100, rows[:3] # сэмплер: приращения, перезапуск контейнера ticks = [ {"ts": 0.0, "cg": {"abc": [0, 0, 0, 0, 100]}, "vm": [0, 100]}, {"ts": 1.0, "cg": {"abc": [500_000, 10, 1, 0, 200]}, "vm": [50, 200]}, {"ts": 2.0, "cg": {"abc": [100_000, 2, 0, 0, 50]}, "vm": [60, 300]}, {"ts": 3.0, "cg": {"abc": [600_000, 12, 0, 0, 80]}, "vm": [160, 400]}, ] usage = cgroup_usage(ticks, "abc") assert usage["cpu_cores"] == 0.5, usage assert vm_busy(ticks) == 0.5333, vm_busy(ticks) # журнал LB: дубли схлопываются log = ( "2026-10-14T10:00:00.100000000Z Server api/api-1 is DOWN, reason: x\n" "2026-10-14T10:00:00.101000000Z [WARNING] (8) : Server api/api-1 is DOWN, reason: x\n" "2026-10-14T10:00:03.000000000Z Server api/api-1 is UP, reason: y\n" ) parsed = parse_lb_log(log) assert [p[2] for p in parsed] == ["is DOWN", "is UP"], parsed print("analysis selftest: ok") if __name__ == "__main__": if sys.argv[1:] == ["--selftest"]: _selftest() else: print("usage: python bench/analysis.py --selftest", file=sys.stderr) sys.exit(2)