547 lines
22 KiB
Python
547 lines
22 KiB
Python
"""Разбор результатов стенда без docker: журналы, окна, признаки, поиск R*.
|
||
|
||
Чистые функции над строками журналов генератора, событиями Docker,
|
||
журналом HAProxy и тиками сэмплера. Их зовёт `run.py`; самопроверка —
|
||
`uv run python bench/analysis.py --selftest`. Только stdlib.
|
||
|
||
Все отметки времени — часы ВМ Docker в миллисекундах эпохи: генератор
|
||
(`time.time()` в контейнере), события Docker (`timeNano`), журнал HAProxy
|
||
(`docker logs -t`), сэмплер. Часы хоста с ними не сравниваются.
|
||
"""
|
||
|
||
import dataclasses
|
||
import datetime
|
||
import json
|
||
import math
|
||
import re
|
||
import sys
|
||
|
||
# --- журнал запросов генератора (профили failover и rollout) ----------------
|
||
|
||
|
||
@dataclasses.dataclass(frozen=True)
|
||
class Request:
|
||
scheduled: float
|
||
sent: float | None
|
||
done: float | None
|
||
kind: str
|
||
status: int
|
||
instance: str
|
||
key: str
|
||
error: str
|
||
|
||
@property
|
||
def ok(self) -> bool:
|
||
return self.status == (201 if self.kind == "write" else 200)
|
||
|
||
@property
|
||
def latency(self) -> float | None:
|
||
"""Задержка от момента по расписанию до ответа, мс."""
|
||
return None if self.done is None else self.done - self.scheduled
|
||
|
||
|
||
def parse_requests(text: str) -> list[Request]:
|
||
out = []
|
||
for line in text.splitlines():
|
||
parts = line.split(",")
|
||
if len(parts) != 9 or parts[0] != "fo":
|
||
continue
|
||
out.append(
|
||
Request(
|
||
scheduled=float(parts[1]),
|
||
sent=float(parts[2]) if parts[2] else None,
|
||
done=float(parts[3]) if parts[3] else None,
|
||
kind=parts[4],
|
||
status=int(parts[5]),
|
||
instance=parts[6],
|
||
key=parts[7],
|
||
error=parts[8],
|
||
),
|
||
)
|
||
out.sort(key=lambda r: r.scheduled)
|
||
return out
|
||
|
||
|
||
def percentile(values: list[float], fraction: float) -> float | None:
|
||
if not values:
|
||
return None
|
||
ordered = sorted(values)
|
||
return ordered[max(1, math.ceil(fraction * len(ordered))) - 1]
|
||
|
||
|
||
def error_label(request: Request) -> str:
|
||
return request.error or str(request.status)
|
||
|
||
|
||
def window_stats(requests: list[Request], start: float, end: float) -> dict:
|
||
"""Запросы окна [start, end) по моменту расписания."""
|
||
inside = [r for r in requests if start <= r.scheduled < end]
|
||
seconds = max((end - start) / 1000, 1e-9)
|
||
ok = [r for r in inside if r.ok]
|
||
errors: dict[str, int] = {}
|
||
for request in inside:
|
||
if not request.ok:
|
||
errors[error_label(request)] = errors.get(error_label(request), 0) + 1
|
||
latencies = [r.latency for r in ok if r.latency is not None]
|
||
return {
|
||
"seconds": round(seconds, 2),
|
||
"requests": len(inside),
|
||
"ok": len(ok),
|
||
"ok_rps": round(len(ok) / seconds, 1),
|
||
"ok_write_rps": round(sum(r.kind == "write" for r in ok) / seconds, 1),
|
||
"ok_read_rps": round(sum(r.kind == "read" for r in ok) / seconds, 1),
|
||
"errors": sum(errors.values()),
|
||
"error_kinds": errors,
|
||
"p50_ms": _r(percentile(latencies, 0.5)),
|
||
"p95_ms": _r(percentile(latencies, 0.95)),
|
||
"p99_ms": _r(percentile(latencies, 0.99)),
|
||
"max_ms": _r(max(latencies) if latencies else None),
|
||
}
|
||
|
||
|
||
def _r(value: float | None, digits: int = 1) -> float | None:
|
||
return None if value is None else round(value, digits)
|
||
|
||
|
||
def timeline(
|
||
requests: list[Request],
|
||
origin: float,
|
||
start: float,
|
||
end: float,
|
||
groups: dict[str, set[str]] | None = None,
|
||
) -> list[dict]:
|
||
"""Посекундная шкала от `origin`: успешные по видам, ошибки, реплики.
|
||
|
||
`groups` — именованные наборы X-Instance-Id (цель и остальные, старые
|
||
и новые): у каждого — число успешных ответов в секунду.
|
||
"""
|
||
first = math.floor((start - origin) / 1000)
|
||
last = math.ceil((end - origin) / 1000)
|
||
rows = []
|
||
for second in range(first, last):
|
||
lo, hi = origin + second * 1000, origin + (second + 1) * 1000
|
||
inside = [r for r in requests if lo <= r.scheduled < hi]
|
||
ok = [r for r in inside if r.ok]
|
||
errors: dict[str, int] = {}
|
||
for request in inside:
|
||
if not request.ok:
|
||
errors[error_label(request)] = errors.get(error_label(request), 0) + 1
|
||
latencies = [r.latency for r in ok if r.latency is not None]
|
||
row = {
|
||
"second": second,
|
||
"sent": len(inside),
|
||
"ok_write": sum(r.kind == "write" for r in ok),
|
||
"ok_read": sum(r.kind == "read" for r in ok),
|
||
"errors": sum(errors.values()),
|
||
"error_kinds": ";".join(f"{k}:{v}" for k, v in sorted(errors.items())),
|
||
"p95_ms": _r(percentile(latencies, 0.95)),
|
||
"max_ms": _r(max(latencies) if latencies else None),
|
||
}
|
||
for name, ids in (groups or {}).items():
|
||
row[f"ok_{name}"] = sum(r.instance in ids for r in ok)
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def recovery_seconds(
|
||
requests: list[Request],
|
||
action: float,
|
||
end: float,
|
||
offered_per_s: float,
|
||
) -> float | None:
|
||
"""Время восстановления, с: от действия до начала первой секунды, после
|
||
которой до `end` нет ни одной ошибки и успешных не меньше 90 % от
|
||
поданной нагрузки в каждой секунде. None — не восстановилось."""
|
||
rows = timeline(requests, action, action, end)
|
||
good = [
|
||
row["errors"] == 0 and row["ok_write"] + row["ok_read"] >= 0.9 * offered_per_s
|
||
for row in rows
|
||
]
|
||
for index in range(len(good)):
|
||
if all(good[index:]):
|
||
return float(rows[index]["second"])
|
||
return None
|
||
|
||
|
||
# --- события Docker и журнал HAProxy ------------------------------------------------
|
||
|
||
|
||
def parse_events(text: str) -> list[tuple[float, str, str]]:
|
||
"""`docker events --format '{{json .}}'`: (мс, действие, id контейнера)."""
|
||
events = []
|
||
for line in text.splitlines():
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
data = json.loads(line)
|
||
action = data.get("Action") or data.get("status") or ""
|
||
actor = (data.get("Actor") or {}).get("ID") or data.get("id") or ""
|
||
events.append((int(data["timeNano"]) / 1e6, action.split(":")[0], actor))
|
||
events.sort()
|
||
return events
|
||
|
||
|
||
_LB_STATE = re.compile(
|
||
r"Server api/(?P<slot>api-\d+)\b.*?(?P<state>is DOWN|is UP|is going DOWN"
|
||
r"|enters drain|enters maintenance|is leaving forced drain|is leaving maintenance"
|
||
r"|administratively READY)",
|
||
)
|
||
|
||
|
||
def parse_docker_time(stamp: str) -> float:
|
||
"""RFC 3339 с наносекундами из `docker logs -t` → мс эпохи."""
|
||
stamp = stamp.rstrip("Z")
|
||
main, _, fraction = stamp.partition(".")
|
||
base = datetime.datetime.fromisoformat(main).replace(tzinfo=datetime.UTC)
|
||
return base.timestamp() * 1000 + float(f"0.{fraction or 0}") * 1000
|
||
|
||
|
||
def parse_lb_log(text: str) -> list[tuple[float, str, str]]:
|
||
"""Смены состояния слотов: (мс, слот, состояние). Дубли строк HAProxy
|
||
(мастер и воркер пишут одну и ту же) схлопываются."""
|
||
seen = set()
|
||
out = []
|
||
for line in text.splitlines():
|
||
stamp, _, message = line.partition(" ")
|
||
match = _LB_STATE.search(message)
|
||
if not match:
|
||
continue
|
||
when = parse_docker_time(stamp)
|
||
item = (round(when / 50), match.group("slot"), match.group("state"))
|
||
if item in seen:
|
||
continue
|
||
seen.add(item)
|
||
out.append((when, match.group("slot"), match.group("state")))
|
||
out.sort()
|
||
return out
|
||
|
||
|
||
def lb_http_errors(text: str) -> dict[str, int]:
|
||
"""Ответы с ошибкой в журнале HAProxy (option dontlog-normal пишет
|
||
только их): статус → число."""
|
||
counts: dict[str, int] = {}
|
||
for line in text.splitlines():
|
||
match = re.search(r'\s(\d{3}) \d+ - - (\S{4}) .*"(GET|POST) ', line)
|
||
if match and match.group(1)[0] in "45":
|
||
counts[match.group(1)] = counts.get(match.group(1), 0) + 1
|
||
return counts
|
||
|
||
|
||
# --- сэмплер ---------------------------------------------------------------------------
|
||
|
||
|
||
def window_ticks(ticks: list[dict], start: float, end: float) -> list[dict]:
|
||
"""Тики окна [start, end] (секунды эпохи) плюс по одному снаружи:
|
||
приращения считаются по краям окна."""
|
||
inside = [t for t in ticks if start <= t["ts"] <= end]
|
||
before = [t for t in ticks if t["ts"] < start]
|
||
after = [t for t in ticks if t["ts"] > end]
|
||
return ([before[-1]] if before else []) + inside + ([after[0]] if after else [])
|
||
|
||
|
||
def cgroup_usage(ticks: list[dict], container: str) -> dict | None:
|
||
"""CPU (ядер), доля троттлинга и память контейнера по тикам окна.
|
||
|
||
Каждый соседний интервал считается отдельно: новая cgroup (контейнер
|
||
перезапущен) даёт отрицательное приращение — такой интервал
|
||
пропускается, а не обнуляет окно.
|
||
"""
|
||
points = [(t["ts"], t["cg"].get(container[:12])) for t in ticks]
|
||
points = [(ts, line) for ts, line in points if line]
|
||
if len(points) < 2:
|
||
return None
|
||
cpu_us = seconds = periods = throttled = 0.0
|
||
memory = 0
|
||
for (t0, a), (t1, b) in zip(points, points[1:], strict=False):
|
||
if b[0] < a[0]:
|
||
continue
|
||
cpu_us += b[0] - a[0]
|
||
seconds += t1 - t0
|
||
periods += b[1] - a[1]
|
||
throttled += b[2] - a[2]
|
||
memory = max(memory, a[4], b[4])
|
||
if seconds <= 0:
|
||
return None
|
||
return {
|
||
"cpu_cores": round(cpu_us / seconds / 1e6, 3),
|
||
"throttled_share": round(throttled / periods, 4) if periods else 0.0,
|
||
"memory_mib": round(memory / 2**20, 1),
|
||
}
|
||
|
||
|
||
def vm_busy(ticks: list[dict]) -> float | None:
|
||
if len(ticks) < 2:
|
||
return None
|
||
busy = ticks[-1]["vm"][0] - ticks[0]["vm"][0]
|
||
total = ticks[-1]["vm"][1] - ticks[0]["vm"][1]
|
||
return round(busy / total, 4) if total else None
|
||
|
||
|
||
def lb_delta(ticks: list[dict], field: str) -> int | None:
|
||
"""Приращение счётчика HAProxy по всем слотам за окно."""
|
||
with_lb = [t for t in ticks if t.get("lb")]
|
||
if len(with_lb) < 2:
|
||
return None
|
||
|
||
def total(tick: dict) -> int:
|
||
return sum(
|
||
row.get(field, 0) for name, row in tick["lb"].items() if name != "FRONTEND"
|
||
)
|
||
|
||
return total(with_lb[-1]) - total(with_lb[0])
|
||
|
||
|
||
def pg_stats(ticks: list[dict], replica_addrs: set[str]) -> dict:
|
||
"""Соединения приложения и счётчики базы за окно."""
|
||
with_pg = [t for t in ticks if t.get("pg")]
|
||
if not with_pg:
|
||
return {}
|
||
per_tick_total, per_replica_max, active, lock_waits = [], 0, [], []
|
||
for tick in with_pg:
|
||
by_addr: dict[str, int] = {}
|
||
busy = waiting = 0
|
||
for addr, state, wait, count in tick["pg"]["conn"]:
|
||
by_addr[addr] = by_addr.get(addr, 0) + count
|
||
if state == "active":
|
||
busy += count
|
||
if wait == "Lock":
|
||
waiting += count
|
||
per_tick_total.append(sum(by_addr.values()))
|
||
known = [
|
||
n for a, n in by_addr.items() if not replica_addrs or a in replica_addrs
|
||
]
|
||
per_replica_max = max([per_replica_max, *known])
|
||
active.append(busy)
|
||
lock_waits.append(waiting)
|
||
first, last = with_pg[0]["pg"]["db"], with_pg[-1]["pg"]["db"]
|
||
seconds = max(with_pg[-1]["ts"] - with_pg[0]["ts"], 1e-9)
|
||
return {
|
||
"pg_conn_max": max(per_tick_total),
|
||
"pg_conn_replica_max": per_replica_max,
|
||
"pg_active_mean": round(sum(active) / len(active), 2),
|
||
"pg_lock_wait_mean": round(sum(lock_waits) / len(lock_waits), 2),
|
||
"pg_new_sessions": last["sessions"] - first["sessions"],
|
||
"pg_commits_per_s": round(
|
||
(last["xact_commit"] - first["xact_commit"]) / seconds, 1
|
||
),
|
||
}
|
||
|
||
|
||
# --- признаки узкого места (серии трека 1) ------------------------------------------
|
||
|
||
# Пороги заданы до прогонов (REPORT.md, гипотезы трека 1). Сработавший
|
||
# признак на первой нарушившей ступени называет, во что упёрлась система
|
||
THRESHOLDS = {
|
||
# реплика: доля своего лимита CPU или заметный троттлинг CFS
|
||
"app_cpu_share": 0.85,
|
||
"app_throttled_share": 0.20,
|
||
# PostgreSQL: доля лимита CPU, средние ожидания блокировок
|
||
"db_cpu_share": 0.85,
|
||
"db_lock_wait_mean": 1.0,
|
||
# пул: соединений одной реплики не меньше доли её предела (размер + переполнение)
|
||
"pool_share": 0.90,
|
||
# переоткрытие соединений: новых сессий PostgreSQL за окно ступени
|
||
"churn_sessions": 20,
|
||
# LB: доля его лимита CPU
|
||
"lb_cpu_share": 0.85,
|
||
# ВМ Docker занята почти целиком: реплики ждут CPU хоста
|
||
"host_busy_share": 0.90,
|
||
# генератор: доля лимита CPU или отставание от расписания
|
||
"gen_cpu_share": 0.90,
|
||
"gen_lag_p99_ms": 50.0,
|
||
}
|
||
|
||
|
||
def signals(row: dict, limits: dict, pool_cap: int) -> list[str]:
|
||
"""Признаки узкого места ступени по агрегатам `row` (см. run.py)."""
|
||
found = []
|
||
if (row.get("api_cpu_max") or 0) >= THRESHOLDS["app_cpu_share"] * limits[
|
||
"api_cpus"
|
||
] or (row.get("api_throttled_max") or 0) >= THRESHOLDS["app_throttled_share"]:
|
||
found.append("app")
|
||
if (row.get("db_cpu_cores") or 0) >= THRESHOLDS["db_cpu_share"] * limits["db_cpus"]:
|
||
found.append("db_cpu")
|
||
if (row.get("pg_lock_wait_mean") or 0) >= THRESHOLDS["db_lock_wait_mean"]:
|
||
found.append("db_lock")
|
||
if (
|
||
pool_cap
|
||
and (row.get("pg_conn_replica_max") or 0) >= THRESHOLDS["pool_share"] * pool_cap
|
||
):
|
||
found.append("pool")
|
||
if (row.get("pg_new_sessions") or 0) >= THRESHOLDS["churn_sessions"]:
|
||
found.append("churn")
|
||
if (row.get("lb_cpu_cores") or 0) >= THRESHOLDS["lb_cpu_share"] * limits["lb_cpus"]:
|
||
found.append("lb")
|
||
if (row.get("vm_busy_share") or 0) >= THRESHOLDS["host_busy_share"]:
|
||
found.append("host")
|
||
if (
|
||
(row.get("gen_cpu_cores") or 0)
|
||
>= THRESHOLDS["gen_cpu_share"] * limits["gen_cpus"]
|
||
or (row.get("lag_p99_ms") or 0) >= THRESHOLDS["gen_lag_p99_ms"]
|
||
or (row.get("dropped") or 0) > 0
|
||
):
|
||
found.append("generator")
|
||
return found
|
||
|
||
|
||
# --- поиск R* ---------------------------------------------------------------------------
|
||
|
||
|
||
@dataclasses.dataclass
|
||
class Search:
|
||
"""Автомат поиска R*: наибольшей интенсивности, на которой ступень
|
||
выдерживает SLO.
|
||
|
||
Сначала фиксированные уровни (сравнение конфигураций на одной
|
||
нагрузке), затем лестница `base × factor^i`: начало — первая ступенька
|
||
не ниже `start`, подъём до первой нарушившей (если нарушена первая —
|
||
спуск до первой прошедшей), затем `bisections` делений интервала
|
||
«прошла — нарушила» по среднему геометрическому, с округлением до
|
||
целого. R* — наибольшая прошедшая ступень поиска.
|
||
"""
|
||
|
||
fixed: tuple[float, ...]
|
||
base: float
|
||
factor: float
|
||
start: float
|
||
bisections: int
|
||
cap: float
|
||
floor: float = 25.0
|
||
results: list[tuple[float, bool, str]] = dataclasses.field(default_factory=list)
|
||
|
||
def ladder(self, rate: float) -> float:
|
||
steps = math.ceil(math.log(rate / self.base, self.factor) - 1e-9)
|
||
return round(self.base * self.factor ** max(steps, 0))
|
||
|
||
def record(self, rate: float, passed: bool, phase: str) -> None:
|
||
self.results.append((rate, passed, phase))
|
||
|
||
def next_rate(self) -> tuple[float, str] | None:
|
||
done_fixed = [r for r in self.results if r[2] == "fixed"]
|
||
if len(done_fixed) < len(self.fixed):
|
||
return self.fixed[len(done_fixed)], "fixed"
|
||
search = [r for r in self.results if r[2] != "fixed"]
|
||
passed = [rate for rate, ok, _ in search if ok]
|
||
failed = [rate for rate, ok, _ in search if not ok]
|
||
if not search:
|
||
fixed_pass = [rate for rate, ok, _ in done_fixed if ok]
|
||
start = max([self.start, *fixed_pass])
|
||
return min(self.ladder(start), self.cap), "ladder"
|
||
phase = search[-1][2]
|
||
if phase == "ladder":
|
||
last_rate, last_ok, _ = search[-1]
|
||
if last_ok and not failed:
|
||
if last_rate >= self.cap:
|
||
return None
|
||
return min(round(last_rate * self.factor), self.cap), "ladder"
|
||
if not last_ok and not passed:
|
||
lower = round(last_rate / self.factor)
|
||
return (lower, "ladder") if lower >= self.floor else None
|
||
# интервал найден: деления
|
||
if not passed or not failed:
|
||
return None
|
||
bisections = sum(1 for r in search if r[2] == "bisect")
|
||
if bisections >= self.bisections:
|
||
return None
|
||
lo = max(passed)
|
||
hi = (
|
||
min(rate for rate in failed if rate > lo)
|
||
if any(r > lo for r in failed)
|
||
else None
|
||
)
|
||
if hi is None:
|
||
return None
|
||
middle = round(math.sqrt(lo * hi))
|
||
if middle <= lo or middle >= hi:
|
||
return None
|
||
return middle, "bisect"
|
||
|
||
def r_star(self) -> float | None:
|
||
passed = [rate for rate, ok, phase in self.results if ok and phase != "fixed"]
|
||
return max(passed) if passed else None
|
||
|
||
def first_fail(self) -> float | None:
|
||
r_star = self.r_star() or 0
|
||
failed = [
|
||
rate for rate, ok, phase in self.results if not ok and phase != "fixed"
|
||
]
|
||
above = [rate for rate in failed if rate > r_star]
|
||
return min(above) if above else None
|
||
|
||
|
||
# --- самопроверка ---------------------------------------------------------------------
|
||
|
||
|
||
def _selftest() -> None:
|
||
# поиск: подъём, нарушение, два деления
|
||
search = Search(
|
||
fixed=(250, 500), base=100, factor=1.25, start=600, bisections=2, cap=8000
|
||
)
|
||
capacity = 1000
|
||
plan = []
|
||
while (step := search.next_rate()) is not None:
|
||
rate, phase = step
|
||
plan.append((rate, phase))
|
||
search.record(rate, rate <= capacity, phase)
|
||
assert plan[:2] == [(250, "fixed"), (500, "fixed")], plan
|
||
# лестница 100 × 1,25^i: первая ступенька не ниже 600 — 745
|
||
assert plan[2] == (745, "ladder"), plan
|
||
assert search.r_star() is not None, plan
|
||
assert search.r_star() <= capacity, plan
|
||
assert search.first_fail() > capacity, plan
|
||
assert sum(phase == "bisect" for _, phase in plan) == 2, plan
|
||
# первая ступень поиска нарушена — спуск
|
||
search = Search(fixed=(), base=100, factor=1.25, start=300, bisections=1, cap=8000)
|
||
plan = []
|
||
while (step := search.next_rate()) is not None:
|
||
plan.append(step)
|
||
search.record(step[0], step[0] <= 200, step[1])
|
||
assert plan[0] == (305, "ladder"), plan
|
||
assert plan[1] == (244, "ladder"), plan
|
||
assert search.r_star() <= 200, plan
|
||
|
||
# окно и время восстановления
|
||
reqs = [
|
||
Request(
|
||
1000 + i * 10, 1000 + i * 10, 1005 + i * 10, "write", 201, "a", f"k{i}", ""
|
||
)
|
||
for i in range(500)
|
||
]
|
||
reqs[200] = dataclasses.replace(reqs[200], status=502)
|
||
stats = window_stats(reqs, 1000, 6000)
|
||
assert stats["errors"] == 1, stats
|
||
assert stats["ok"] == 499, stats
|
||
assert recovery_seconds(reqs, 1000, 6000, 100) == 3.0
|
||
rows = timeline(reqs, 1000, 1000, 6000, {"a": {"a"}})
|
||
assert rows[2]["errors"] == 1, rows[:3]
|
||
assert rows[0]["ok_a"] == 100, rows[:3]
|
||
|
||
# сэмплер: приращения, перезапуск контейнера
|
||
ticks = [
|
||
{"ts": 0.0, "cg": {"abc": [0, 0, 0, 0, 100]}, "vm": [0, 100]},
|
||
{"ts": 1.0, "cg": {"abc": [500_000, 10, 1, 0, 200]}, "vm": [50, 200]},
|
||
{"ts": 2.0, "cg": {"abc": [100_000, 2, 0, 0, 50]}, "vm": [60, 300]},
|
||
{"ts": 3.0, "cg": {"abc": [600_000, 12, 0, 0, 80]}, "vm": [160, 400]},
|
||
]
|
||
usage = cgroup_usage(ticks, "abc")
|
||
assert usage["cpu_cores"] == 0.5, usage
|
||
assert vm_busy(ticks) == 0.5333, vm_busy(ticks)
|
||
|
||
# журнал LB: дубли схлопываются
|
||
log = (
|
||
"2026-10-14T10:00:00.100000000Z Server api/api-1 is DOWN, reason: x\n"
|
||
"2026-10-14T10:00:00.101000000Z [WARNING] (8) : Server api/api-1 is DOWN, reason: x\n"
|
||
"2026-10-14T10:00:03.000000000Z Server api/api-1 is UP, reason: y\n"
|
||
)
|
||
parsed = parse_lb_log(log)
|
||
assert [p[2] for p in parsed] == ["is DOWN", "is UP"], parsed
|
||
print("analysis selftest: ok")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if sys.argv[1:] == ["--selftest"]:
|
||
_selftest()
|
||
else:
|
||
print("usage: python bench/analysis.py --selftest", file=sys.stderr)
|
||
sys.exit(2)
|