forked from mtusi2026/mtusi2026lab3
Найдено тестом сценария рецензента (docker network disconnect одной API от сети хранилищ): по дедлайну задача psycopg отменялась, psycopg слал cancel-запрос, а libpq PQcancelStart синхронно резолвит имя хоста в потоке event loop — вся реплика, включая /health/live, висела ~1 минуту и не возвращалась в ротацию сама. Теперь задача по дедлайну не отменяется (запрос сразу получает 503, задача доживает сама), а мёртвые соединения рвёт TCP keepalive + tcp_user_timeout = DB_TIMEOUT_SECONDS. - compose: сеть edge (lb, api) и internal-сеть data (api, migrate, postgres, valkey) — LB не видит хранилища. - тесты: неготовая реплика (pg_hba и network disconnect) выводится из ротации, процесс отвечает на live, после восстановления сам UP за <= 15 с; пробельные Unicode-строки, 1e1000, байт FF в пароле, 101 участник, курсор одного списка в другом. - bench/track1: CPU реплик суммируется по каждому замеру, в raw и CSV — фактические лимиты контейнеров из docker inspect. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
397 lines
20 KiB
Python
397 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 1 orchestrator: cost of scaling. For every configuration (replicas x CPU per replica x
|
|
PostgreSQL CPU x pool size x env) and every repeat a FRESH bench stand is started (separate
|
|
compose project, empty volumes), synthetic users/chats are created, then k6 runs an open-loop
|
|
arrival-rate sweep. During each step the orchestrator samples CPU of every container
|
|
(docker stats), DB pool waits (API stats log), PostgreSQL counters (pg_stat_database,
|
|
pg_stat_statements) and HAProxy counters (show stat).
|
|
|
|
python bench/track1/run.py --series fixed-total --label baseline \
|
|
--config replicas=1,api_cpus=2.0 --config replicas=2,api_cpus=1.0 --config replicas=4,api_cpus=0.5 \
|
|
--rates 200,400,600,800 --repeats 3
|
|
|
|
Output: evidence/track1/<series>/<label>/raw/*.json (one per step, no tokens) and
|
|
results.csv (one row per step). The main stand (project mtusi-lab3) is not touched,
|
|
but stop it during experiments so it does not compete for CPU.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import csv
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PROJECT = "mtusi-lab3-bench"
|
|
PORT = 18080
|
|
FILES = ["-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "bench" / "track1" / "compose.bench.yaml")]
|
|
MARK = "@@SUMMARY@@"
|
|
|
|
DEFAULTS = {"replicas": 2, "api_cpus": 1.0, "api_memory": "384m", "pg_cpus": 2.0, "pool": 10}
|
|
|
|
|
|
# ------------------------------------------------------------------ helpers
|
|
def compose(*args: str, env: dict | None = None, check: bool = True, timeout: float = 600) -> str:
|
|
result = subprocess.run(["docker", "compose", "-p", PROJECT, *FILES, *args], cwd=ROOT, capture_output=True,
|
|
text=True, encoding="utf-8", env={**os.environ, **(env or {})}, timeout=timeout)
|
|
if check and result.returncode != 0:
|
|
raise RuntimeError(f"compose {' '.join(args[:3])} failed: {result.stderr.strip()[-1500:]}")
|
|
return result.stdout
|
|
|
|
|
|
def psql(sql: str) -> str:
|
|
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t",
|
|
"-v", "ON_ERROR_STOP=1", "-c", sql).strip()
|
|
|
|
|
|
def http(method: str, path: str, body=None, headers=None):
|
|
data = None if body is None else json.dumps(body).encode()
|
|
h = {"Content-Type": "application/json", **(headers or {})}
|
|
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", data=data, headers=h, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return r.status, json.loads(r.read() or b"null")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, json.loads(e.read() or b"null")
|
|
|
|
|
|
def parse_config(text: str) -> dict:
|
|
cfg = dict(DEFAULTS)
|
|
cfg["env"] = {}
|
|
for part in filter(None, text.split(",")):
|
|
key, value = part.split("=", 1)
|
|
if key in ("replicas", "pool"):
|
|
cfg[key] = int(value)
|
|
elif key in ("api_cpus", "pg_cpus"):
|
|
cfg[key] = float(value)
|
|
elif key == "api_memory":
|
|
cfg[key] = value
|
|
else:
|
|
cfg["env"][key] = value # any API/compose env, e.g. MESSAGE_WRITE_PATH=function
|
|
return cfg
|
|
|
|
|
|
def config_name(cfg: dict) -> str:
|
|
extra = "".join(f"-{k.lower()}_{v}" for k, v in sorted(cfg["env"].items()))
|
|
return f"n{cfg['replicas']}-api{cfg['api_cpus']:g}-pg{cfg['pg_cpus']:g}-pool{cfg['pool']}{extra}"
|
|
|
|
|
|
def build_image(ref: str) -> tuple[str, str]:
|
|
"""Image of the API built from a COMMIT (git archive), not from the working tree, so a
|
|
series is reproducible and edits made while it runs cannot leak into it."""
|
|
sha = subprocess.run(["git", "rev-parse", "--short", ref], cwd=ROOT, capture_output=True, text=True,
|
|
check=True).stdout.strip()
|
|
tag = f"mtusi-messenger:bench-{sha}"
|
|
archive = subprocess.run(["git", "archive", "--format=tar", sha], cwd=ROOT, capture_output=True, check=True).stdout
|
|
subprocess.run(["docker", "build", "-q", "--build-arg", f"BUILD_REVISION={sha}", "-t", tag, "-"], input=archive,
|
|
capture_output=True, check=True, timeout=900)
|
|
return sha, tag
|
|
|
|
|
|
IMAGE = {"tag": "mtusi-messenger:lab3"}
|
|
|
|
|
|
def stand_env(cfg: dict) -> dict:
|
|
return {"API_IMAGE": IMAGE["tag"], "API_REPLICAS": str(cfg["replicas"]), "API_CPUS": f"{cfg['api_cpus']:g}", "API_MEMORY": cfg["api_memory"],
|
|
"PG_CPUS": f"{cfg['pg_cpus']:g}", "DB_POOL_MIN": str(min(cfg["pool"], 4)), "DB_POOL_MAX": str(cfg["pool"]),
|
|
"APP_PUBLISH_PORT": str(PORT), "LB_STATS_PORT": "18404", "BUILD_REVISION": "bench",
|
|
**cfg["env"]}
|
|
|
|
|
|
def start_stand(cfg: dict) -> None:
|
|
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
|
|
compose("up", "-d", "--no-build", "--wait", env=stand_env(cfg), timeout=900)
|
|
psql("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline: # every replica UP in HAProxy before measuring
|
|
if lb_stat()["up"] == cfg["replicas"]:
|
|
return
|
|
time.sleep(0.5)
|
|
raise RuntimeError("not all replicas UP in HAProxy")
|
|
|
|
|
|
def actual_limits() -> dict:
|
|
"""What Docker really applied (not what the config intended): CPU quota, memory, cpuset."""
|
|
ids = compose("ps", "-q").split()
|
|
result = {}
|
|
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
|
|
check=True).stdout):
|
|
host = item["HostConfig"]
|
|
result[item["Name"].lstrip("/")[len(PROJECT) + 1:]] = {
|
|
"cpus": host["NanoCpus"] / 1e9, "memory_mib": host["Memory"] // (1024 * 1024),
|
|
"cpuset": host["CpusetCpus"], "image": item["Config"]["Image"]}
|
|
return result
|
|
|
|
|
|
def make_chats(users: int) -> list[dict]:
|
|
created = []
|
|
stamp = uuid.uuid4().hex[:8]
|
|
for i in range(users):
|
|
username, password = f"b_{stamp}_{i}", f"Bench-password-{stamp}-{i}"
|
|
status, user = http("POST", "/api/v1/users", {"username": username, "display_name": f"bench {i}", "password": password})
|
|
assert status == 201, (status, user)
|
|
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
status, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
|
assert status == 201, (status, session)
|
|
created.append({"id": user["id"], "token": session["token"]})
|
|
chats = []
|
|
for i, me in enumerate(created):
|
|
other = created[(i + 1) % len(created)]
|
|
status, chat = http("POST", "/api/v1/chats", {"title": f"bench {i}", "member_ids": [me["id"], other["id"]]},
|
|
headers={"Authorization": "Bearer " + me["token"]})
|
|
assert status == 201, (status, chat)
|
|
chats.append({"token": me["token"], "chat": chat["id"]})
|
|
return chats
|
|
|
|
|
|
class Sampler(threading.Thread):
|
|
"""docker stats every ~1-2 s: CPU% (100 = one core) and memory per container."""
|
|
|
|
def __init__(self):
|
|
super().__init__(daemon=True)
|
|
self.samples: list[tuple[float, dict]] = []
|
|
self.stop_event = threading.Event()
|
|
|
|
def run(self) -> None:
|
|
while not self.stop_event.is_set():
|
|
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}"],
|
|
capture_output=True, text=True).stdout
|
|
now = time.monotonic()
|
|
row = {}
|
|
for line in out.splitlines():
|
|
item = json.loads(line)
|
|
if not item["Name"].startswith(PROJECT + "-"):
|
|
continue # only this bench project (k6 run containers included)
|
|
short = item["Name"][len(PROJECT) + 1:]
|
|
if short.startswith("k6-run-"):
|
|
short = "k6"
|
|
row[short] = {"cpu": float(item["CPUPerc"].rstrip("%") or 0), "mem": item["MemUsage"].split(" / ")[0]}
|
|
self.samples.append((now, row))
|
|
|
|
def window(self, start: float, end: float) -> dict:
|
|
rows = [r for t, r in self.samples if start <= t <= end]
|
|
result = {}
|
|
# All API replicas summed PER SAMPLE first, then the median over time (not a sum of medians).
|
|
totals = [sum(v["cpu"] for k, v in r.items() if k.startswith("api-")) for r in rows]
|
|
if totals:
|
|
result["api_total"] = {"cpu_median": round(statistics.median(totals), 1), "cpu_max": round(max(totals), 1),
|
|
"samples": len(totals)}
|
|
for name in sorted({n for r in rows for n in r}):
|
|
values = [r[name]["cpu"] for r in rows if name in r]
|
|
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
|
|
"mem_last": [r[name]["mem"] for r in rows if name in r][-1], "samples": len(values)}
|
|
return result
|
|
|
|
|
|
def lb_stat() -> dict:
|
|
raw = compose("exec", "-T", "lb", "sh", "-c", 'echo "show stat" | socat stdio /var/lib/haproxy/admin.sock')
|
|
lines = [line for line in raw.splitlines() if line]
|
|
header = lines[0].lstrip("# ").split(",")
|
|
servers = [dict(zip(header, line.split(","))) for line in lines[1:]]
|
|
api = [s for s in servers if s["pxname"] == "api" and s["svname"].startswith("api") and s["status"] != "MAINT"]
|
|
backend = next(s for s in servers if s["pxname"] == "api" and s["svname"] == "BACKEND")
|
|
return {"up": sum(1 for s in api if s["status"].startswith("UP")),
|
|
"chkdown": sum(int(s["chkdown"] or 0) for s in api), "qmax": int(backend["qmax"] or 0),
|
|
"smax": int(backend["smax"] or 0), "econ": int(backend["econ"] or 0), "eresp": int(backend["eresp"] or 0),
|
|
"wredis": int(backend["wredis"] or 0), "wretr": int(backend["wretr"] or 0)}
|
|
|
|
|
|
def pg_counters() -> dict:
|
|
row = psql("SELECT xact_commit, xact_rollback, tup_inserted, blks_read, blks_hit FROM pg_stat_database "
|
|
"WHERE datname = 'messenger'").split("\t")
|
|
wal = psql("SELECT wal_records, wal_bytes FROM pg_stat_wal").split("\t")
|
|
return {"xact_commit": int(row[0]), "xact_rollback": int(row[1]), "tup_inserted": int(row[2]),
|
|
"blks_read": int(row[3]), "blks_hit": int(row[4]), "wal_records": int(wal[0]), "wal_bytes": int(float(wal[1]))}
|
|
|
|
|
|
def top_statements(limit: int = 12) -> list[dict]:
|
|
out = psql("SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 4), "
|
|
"left(regexp_replace(query, '\\s+', ' ', 'g'), 110) FROM pg_stat_statements "
|
|
"WHERE dbid = (SELECT oid FROM pg_database WHERE datname = 'messenger') "
|
|
f"ORDER BY total_exec_time DESC LIMIT {limit}")
|
|
rows = []
|
|
for line in out.splitlines():
|
|
calls, total, mean, query = line.split("\t", 3)
|
|
rows.append({"calls": int(calls), "total_ms": float(total), "mean_ms": float(mean), "query": query})
|
|
return rows
|
|
|
|
|
|
def pool_window(since_iso: str, start_wall: float, end_wall: float) -> dict:
|
|
logs = compose("logs", "--no-log-prefix", "--since", since_iso, "api")
|
|
waits, nums, waiting, cpu, usage, intervals = 0, 0, 0, 0.0, 0, 0
|
|
for line in logs.splitlines():
|
|
if '"pool_stats"' not in line:
|
|
continue
|
|
item = json.loads(line)
|
|
ts = dt.datetime.fromisoformat(item["ts"].replace("Z", "+00:00")).timestamp()
|
|
if not start_wall <= ts <= end_wall:
|
|
continue
|
|
intervals += 1
|
|
waits += item["requests_wait_ms"]
|
|
nums += item["requests_num"]
|
|
waiting = max(waiting, item["requests_waiting"])
|
|
cpu += item["cpu_s"]
|
|
usage += item["usage_ms"]
|
|
return {"pool_requests": nums, "pool_wait_ms_avg": round(waits / nums, 3) if nums else None,
|
|
"pool_waiting_max": waiting, "pool_usage_ms_avg": round(usage / nums, 3) if nums else None,
|
|
"api_cpu_s": round(cpu, 2), "intervals": intervals}
|
|
|
|
|
|
def metric(data: dict, name: str, field: str):
|
|
m = data["metrics"].get(name)
|
|
return None if m is None else m["values"].get(field)
|
|
|
|
|
|
def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> dict:
|
|
psql("SELECT pg_stat_statements_reset()")
|
|
pg0, lb0 = pg_counters(), lb_stat()
|
|
since = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
started = time.monotonic()
|
|
wall0 = time.time()
|
|
max_vus = max(50, min(args.max_vus, rate * 2))
|
|
out = compose("run", "--rm", "--no-deps", "-v", f"{chats_dir}:/data:ro", "-e", f"RATE={rate}",
|
|
"-e", f"WARMUP={args.warmup}s", "-e", f"DURATION={args.duration}s", "-e", f"MAX_VUS={max_vus}",
|
|
"k6", "run", "--quiet", "/scripts/messages.js", timeout=args.warmup + args.duration + 180)
|
|
ended = time.monotonic()
|
|
data = json.loads(out[out.index(MARK) + len(MARK):].strip().splitlines()[0])
|
|
# k6 starts ~1-3 s after the command; the measure phase is the last `duration` seconds before exit.
|
|
m_end = ended - 1.0
|
|
m_start = max(started, m_end - args.duration)
|
|
pg1, lb1 = pg_counters(), lb_stat()
|
|
ok = metric(data, "ok_messages{phase:measure}", "count") or 0
|
|
reqs = metric(data, "http_reqs{phase:measure}", "count") or 0
|
|
dropped = metric(data, "dropped_iterations{scenario:measure}", "count") or 0
|
|
result = {
|
|
"config": config_name(cfg), "rate": rate, "duration_s": args.duration,
|
|
"offered": rate * args.duration, "requests": int(reqs), "ok": int(ok), "dropped": int(dropped),
|
|
"ok_rps": round(ok / args.duration, 1),
|
|
"error_ratio": round(metric(data, "http_req_failed{phase:measure}", "rate") or 0, 5),
|
|
"ok_p50_ms": metric(data, "ok_latency{phase:measure}", "p(50)"),
|
|
"ok_p95_ms": metric(data, "ok_latency{phase:measure}", "p(95)"),
|
|
"ok_p99_ms": metric(data, "ok_latency{phase:measure}", "p(99)"),
|
|
"vus_max": metric(data, "vus_max", "value"),
|
|
"cpu": sampler.window(m_start, m_end),
|
|
"pool": pool_window(since, wall0 + (m_start - started), wall0 + (m_end - started)),
|
|
"pg": {k: pg1[k] - pg0[k] for k in pg1},
|
|
"lb": {"chkdown_delta": lb1["chkdown"] - lb0["chkdown"], "qmax": lb1["qmax"], "up_after": lb1["up"],
|
|
"econ_delta": lb1["econ"] - lb0["econ"], "eresp_delta": lb1["eresp"] - lb0["eresp"],
|
|
"redispatch_delta": lb1["wredis"] - lb0["wredis"]},
|
|
"statements": top_statements(),
|
|
}
|
|
commits = result["pg"]["xact_commit"]
|
|
result["pg_commits_per_ok"] = round(commits / ok, 2) if ok else None
|
|
result["statements_per_ok"] = round(sum(s["calls"] for s in result["statements"]) / ok, 2) if ok else None
|
|
return result
|
|
|
|
|
|
def flat(r: dict) -> dict:
|
|
cpu = r["cpu"]
|
|
api_cpu = cpu.get("api_total", {}).get("cpu_median", 0.0)
|
|
limits = r.get("limits", {})
|
|
api_limits = [v for k, v in limits.items() if k.startswith("api-")]
|
|
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
|
|
"replicas_running": len(api_limits), "api_cpus_limit_total": round(sum(v["cpus"] for v in api_limits), 2),
|
|
"api_mem_mib_each": api_limits[0]["memory_mib"] if api_limits else None,
|
|
"pg_cpus_limit": limits.get("postgres-1", {}).get("cpus"),
|
|
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
|
|
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
|
|
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
|
|
"lb_cpu_pct": cpu.get("lb-1", {}).get("cpu_median"), "valkey_cpu_pct": cpu.get("valkey-1", {}).get("cpu_median"),
|
|
"k6_cpu_pct": cpu.get("k6", {}).get("cpu_median"),
|
|
"pool_wait_ms_avg": r["pool"]["pool_wait_ms_avg"], "pool_waiting_max": r["pool"]["pool_waiting_max"],
|
|
"pg_commits_per_ok": r["pg_commits_per_ok"], "statements_per_ok": r["statements_per_ok"],
|
|
"lb_chkdown": r["lb"]["chkdown_delta"], "lb_qmax": r["lb"]["qmax"], "wal_bytes": r["pg"]["wal_bytes"]}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
parser.add_argument("--series", required=True)
|
|
parser.add_argument("--label", default="baseline")
|
|
parser.add_argument("--config", action="append", required=True, help="replicas=2,api_cpus=1.0,pg_cpus=2.0,pool=10,ENV=val")
|
|
parser.add_argument("--rates", required=True, help="comma separated arrival rates, req/s")
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--warmup", type=int, default=10)
|
|
parser.add_argument("--duration", type=int, default=20)
|
|
parser.add_argument("--users", type=int, default=64)
|
|
parser.add_argument("--max-vus", type=int, default=600)
|
|
parser.add_argument("--git-ref", default="HEAD", help="commit whose code is measured (image built via git archive)")
|
|
args = parser.parse_args()
|
|
sha, IMAGE["tag"] = build_image(args.git_ref)
|
|
out = ROOT / "evidence" / "track1" / args.series / args.label
|
|
(out / "raw").mkdir(parents=True, exist_ok=True)
|
|
rates = [int(r) for r in args.rates.split(",")]
|
|
configs = [parse_config(c) for c in args.config]
|
|
meta = {"started": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
|
"measured_commit": sha, "image": IMAGE["tag"],
|
|
"bench_tooling_dirty": bool(subprocess.run(["git", "status", "--porcelain", "--", "bench", "compose.yaml",
|
|
"infra"], cwd=ROOT, capture_output=True, text=True).stdout.strip()),
|
|
"host": {"os": f"{platform.system()} {platform.release()}",
|
|
"docker": subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} CPU "
|
|
"{{.MemTotal}} B {{.KernelVersion}}"], capture_output=True,
|
|
text=True).stdout.strip()},
|
|
"cpusets": "k6 0-1, lb+valkey 2, postgres 3-4, api 5-7 (bench/track1/compose.bench.yaml)",
|
|
"workload": f"k6 constant-arrival-rate POST message, new Idempotency-Key, {args.users} chats, "
|
|
f"warmup {args.warmup}s + measure {args.duration}s per step, fresh stand per config x repeat",
|
|
"rates": rates, "repeats": args.repeats, "configs": configs, "args": sys_argv()}
|
|
(out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
csv_path = out / "results.csv"
|
|
new_file = not csv_path.exists()
|
|
with csv_path.open("a", newline="", encoding="utf-8") as f:
|
|
writer = None
|
|
for repeat in range(1, args.repeats + 1):
|
|
for cfg in configs:
|
|
name = config_name(cfg)
|
|
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
|
|
start_stand(cfg)
|
|
limits = actual_limits()
|
|
chats_dir = tempfile.mkdtemp(prefix="track1-")
|
|
try:
|
|
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
|
|
sampler = Sampler()
|
|
sampler.start()
|
|
for rate in rates:
|
|
result = run_step(cfg, rate, args, chats_dir, sampler)
|
|
result["repeat"] = repeat
|
|
result["limits"] = limits
|
|
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
row = flat(result)
|
|
if writer is None:
|
|
writer = csv.DictWriter(f, fieldnames=list(row))
|
|
if new_file:
|
|
writer.writeheader()
|
|
writer.writerow(row)
|
|
f.flush()
|
|
print(f" rate {rate}: ok_rps {row['ok_rps']} p95 {row['p95_ms'] and round(row['p95_ms'])} ms "
|
|
f"err {row['error_ratio']} dropped {row['dropped']} | api {row['api_cpu_pct_total']}% "
|
|
f"pg {row['pg_cpu_pct']}% lb {row['lb_cpu_pct']}% k6 {row['k6_cpu_pct']}% "
|
|
f"pool_wait {row['pool_wait_ms_avg']} ms chkdown {row['lb_chkdown']}", flush=True)
|
|
sampler.stop_event.set()
|
|
finally:
|
|
shutil.rmtree(chats_dir, ignore_errors=True) # synthetic tokens never stay on disk
|
|
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
|
|
print(f"results: {csv_path}")
|
|
return 0
|
|
|
|
|
|
def sys_argv() -> list[str]:
|
|
import sys
|
|
return sys.argv[1:]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|