Files
mtusi2026lab3/bench/track1/run.py
T
Alexey BodanyukandClaude Opus 5.5 3f45eb1b51 fix: реплика не «замерзает» при потере сети к PostgreSQL; сети edge/data
Найдено тестом сценария рецензента (docker network disconnect одной API
от сети хранилищ): по дедлайну задача psycopg отменялась, psycopg слал
cancel-запрос, а libpq PQcancelStart синхронно резолвит имя хоста в потоке
event loop — вся реплика, включая /health/live, висела ~1 минуту и не
возвращалась в ротацию сама. Теперь задача по дедлайну не отменяется
(запрос сразу получает 503, задача доживает сама), а мёртвые соединения
рвёт TCP keepalive + tcp_user_timeout = DB_TIMEOUT_SECONDS.

- compose: сеть edge (lb, api) и internal-сеть data (api, migrate,
  postgres, valkey) — LB не видит хранилища.
- тесты: неготовая реплика (pg_hba и network disconnect) выводится из
  ротации, процесс отвечает на live, после восстановления сам UP за <= 15 с;
  пробельные Unicode-строки, 1e1000, байт FF в пароле, 101 участник,
  курсор одного списка в другом.
- bench/track1: CPU реплик суммируется по каждому замеру, в raw и CSV —
  фактические лимиты контейнеров из docker inspect.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 14:01:11 +03:00

397 lines
20 KiB
Python

#!/usr/bin/env python3
"""Track 1 orchestrator: cost of scaling. For every configuration (replicas x CPU per replica x
PostgreSQL CPU x pool size x env) and every repeat a FRESH bench stand is started (separate
compose project, empty volumes), synthetic users/chats are created, then k6 runs an open-loop
arrival-rate sweep. During each step the orchestrator samples CPU of every container
(docker stats), DB pool waits (API stats log), PostgreSQL counters (pg_stat_database,
pg_stat_statements) and HAProxy counters (show stat).
python bench/track1/run.py --series fixed-total --label baseline \
--config replicas=1,api_cpus=2.0 --config replicas=2,api_cpus=1.0 --config replicas=4,api_cpus=0.5 \
--rates 200,400,600,800 --repeats 3
Output: evidence/track1/<series>/<label>/raw/*.json (one per step, no tokens) and
results.csv (one row per step). The main stand (project mtusi-lab3) is not touched,
but stop it during experiments so it does not compete for CPU.
"""
from __future__ import annotations
import argparse
import base64
import csv
import datetime as dt
import json
import os
import platform
import shutil
import statistics
import subprocess
import tempfile
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PROJECT = "mtusi-lab3-bench"
PORT = 18080
FILES = ["-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "bench" / "track1" / "compose.bench.yaml")]
MARK = "@@SUMMARY@@"
DEFAULTS = {"replicas": 2, "api_cpus": 1.0, "api_memory": "384m", "pg_cpus": 2.0, "pool": 10}
# ------------------------------------------------------------------ helpers
def compose(*args: str, env: dict | None = None, check: bool = True, timeout: float = 600) -> str:
result = subprocess.run(["docker", "compose", "-p", PROJECT, *FILES, *args], cwd=ROOT, capture_output=True,
text=True, encoding="utf-8", env={**os.environ, **(env or {})}, timeout=timeout)
if check and result.returncode != 0:
raise RuntimeError(f"compose {' '.join(args[:3])} failed: {result.stderr.strip()[-1500:]}")
return result.stdout
def psql(sql: str) -> str:
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t",
"-v", "ON_ERROR_STOP=1", "-c", sql).strip()
def http(method: str, path: str, body=None, headers=None):
data = None if body is None else json.dumps(body).encode()
h = {"Content-Type": "application/json", **(headers or {})}
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, json.loads(r.read() or b"null")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def parse_config(text: str) -> dict:
cfg = dict(DEFAULTS)
cfg["env"] = {}
for part in filter(None, text.split(",")):
key, value = part.split("=", 1)
if key in ("replicas", "pool"):
cfg[key] = int(value)
elif key in ("api_cpus", "pg_cpus"):
cfg[key] = float(value)
elif key == "api_memory":
cfg[key] = value
else:
cfg["env"][key] = value # any API/compose env, e.g. MESSAGE_WRITE_PATH=function
return cfg
def config_name(cfg: dict) -> str:
extra = "".join(f"-{k.lower()}_{v}" for k, v in sorted(cfg["env"].items()))
return f"n{cfg['replicas']}-api{cfg['api_cpus']:g}-pg{cfg['pg_cpus']:g}-pool{cfg['pool']}{extra}"
def build_image(ref: str) -> tuple[str, str]:
"""Image of the API built from a COMMIT (git archive), not from the working tree, so a
series is reproducible and edits made while it runs cannot leak into it."""
sha = subprocess.run(["git", "rev-parse", "--short", ref], cwd=ROOT, capture_output=True, text=True,
check=True).stdout.strip()
tag = f"mtusi-messenger:bench-{sha}"
archive = subprocess.run(["git", "archive", "--format=tar", sha], cwd=ROOT, capture_output=True, check=True).stdout
subprocess.run(["docker", "build", "-q", "--build-arg", f"BUILD_REVISION={sha}", "-t", tag, "-"], input=archive,
capture_output=True, check=True, timeout=900)
return sha, tag
IMAGE = {"tag": "mtusi-messenger:lab3"}
def stand_env(cfg: dict) -> dict:
return {"API_IMAGE": IMAGE["tag"], "API_REPLICAS": str(cfg["replicas"]), "API_CPUS": f"{cfg['api_cpus']:g}", "API_MEMORY": cfg["api_memory"],
"PG_CPUS": f"{cfg['pg_cpus']:g}", "DB_POOL_MIN": str(min(cfg["pool"], 4)), "DB_POOL_MAX": str(cfg["pool"]),
"APP_PUBLISH_PORT": str(PORT), "LB_STATS_PORT": "18404", "BUILD_REVISION": "bench",
**cfg["env"]}
def start_stand(cfg: dict) -> None:
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
compose("up", "-d", "--no-build", "--wait", env=stand_env(cfg), timeout=900)
psql("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
deadline = time.monotonic() + 60
while time.monotonic() < deadline: # every replica UP in HAProxy before measuring
if lb_stat()["up"] == cfg["replicas"]:
return
time.sleep(0.5)
raise RuntimeError("not all replicas UP in HAProxy")
def actual_limits() -> dict:
"""What Docker really applied (not what the config intended): CPU quota, memory, cpuset."""
ids = compose("ps", "-q").split()
result = {}
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
host = item["HostConfig"]
result[item["Name"].lstrip("/")[len(PROJECT) + 1:]] = {
"cpus": host["NanoCpus"] / 1e9, "memory_mib": host["Memory"] // (1024 * 1024),
"cpuset": host["CpusetCpus"], "image": item["Config"]["Image"]}
return result
def make_chats(users: int) -> list[dict]:
created = []
stamp = uuid.uuid4().hex[:8]
for i in range(users):
username, password = f"b_{stamp}_{i}", f"Bench-password-{stamp}-{i}"
status, user = http("POST", "/api/v1/users", {"username": username, "display_name": f"bench {i}", "password": password})
assert status == 201, (status, user)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, (status, session)
created.append({"id": user["id"], "token": session["token"]})
chats = []
for i, me in enumerate(created):
other = created[(i + 1) % len(created)]
status, chat = http("POST", "/api/v1/chats", {"title": f"bench {i}", "member_ids": [me["id"], other["id"]]},
headers={"Authorization": "Bearer " + me["token"]})
assert status == 201, (status, chat)
chats.append({"token": me["token"], "chat": chat["id"]})
return chats
class Sampler(threading.Thread):
"""docker stats every ~1-2 s: CPU% (100 = one core) and memory per container."""
def __init__(self):
super().__init__(daemon=True)
self.samples: list[tuple[float, dict]] = []
self.stop_event = threading.Event()
def run(self) -> None:
while not self.stop_event.is_set():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}"],
capture_output=True, text=True).stdout
now = time.monotonic()
row = {}
for line in out.splitlines():
item = json.loads(line)
if not item["Name"].startswith(PROJECT + "-"):
continue # only this bench project (k6 run containers included)
short = item["Name"][len(PROJECT) + 1:]
if short.startswith("k6-run-"):
short = "k6"
row[short] = {"cpu": float(item["CPUPerc"].rstrip("%") or 0), "mem": item["MemUsage"].split(" / ")[0]}
self.samples.append((now, row))
def window(self, start: float, end: float) -> dict:
rows = [r for t, r in self.samples if start <= t <= end]
result = {}
# All API replicas summed PER SAMPLE first, then the median over time (not a sum of medians).
totals = [sum(v["cpu"] for k, v in r.items() if k.startswith("api-")) for r in rows]
if totals:
result["api_total"] = {"cpu_median": round(statistics.median(totals), 1), "cpu_max": round(max(totals), 1),
"samples": len(totals)}
for name in sorted({n for r in rows for n in r}):
values = [r[name]["cpu"] for r in rows if name in r]
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
"mem_last": [r[name]["mem"] for r in rows if name in r][-1], "samples": len(values)}
return result
def lb_stat() -> dict:
raw = compose("exec", "-T", "lb", "sh", "-c", 'echo "show stat" | socat stdio /var/lib/haproxy/admin.sock')
lines = [line for line in raw.splitlines() if line]
header = lines[0].lstrip("# ").split(",")
servers = [dict(zip(header, line.split(","))) for line in lines[1:]]
api = [s for s in servers if s["pxname"] == "api" and s["svname"].startswith("api") and s["status"] != "MAINT"]
backend = next(s for s in servers if s["pxname"] == "api" and s["svname"] == "BACKEND")
return {"up": sum(1 for s in api if s["status"].startswith("UP")),
"chkdown": sum(int(s["chkdown"] or 0) for s in api), "qmax": int(backend["qmax"] or 0),
"smax": int(backend["smax"] or 0), "econ": int(backend["econ"] or 0), "eresp": int(backend["eresp"] or 0),
"wredis": int(backend["wredis"] or 0), "wretr": int(backend["wretr"] or 0)}
def pg_counters() -> dict:
row = psql("SELECT xact_commit, xact_rollback, tup_inserted, blks_read, blks_hit FROM pg_stat_database "
"WHERE datname = 'messenger'").split("\t")
wal = psql("SELECT wal_records, wal_bytes FROM pg_stat_wal").split("\t")
return {"xact_commit": int(row[0]), "xact_rollback": int(row[1]), "tup_inserted": int(row[2]),
"blks_read": int(row[3]), "blks_hit": int(row[4]), "wal_records": int(wal[0]), "wal_bytes": int(float(wal[1]))}
def top_statements(limit: int = 12) -> list[dict]:
out = psql("SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 4), "
"left(regexp_replace(query, '\\s+', ' ', 'g'), 110) FROM pg_stat_statements "
"WHERE dbid = (SELECT oid FROM pg_database WHERE datname = 'messenger') "
f"ORDER BY total_exec_time DESC LIMIT {limit}")
rows = []
for line in out.splitlines():
calls, total, mean, query = line.split("\t", 3)
rows.append({"calls": int(calls), "total_ms": float(total), "mean_ms": float(mean), "query": query})
return rows
def pool_window(since_iso: str, start_wall: float, end_wall: float) -> dict:
logs = compose("logs", "--no-log-prefix", "--since", since_iso, "api")
waits, nums, waiting, cpu, usage, intervals = 0, 0, 0, 0.0, 0, 0
for line in logs.splitlines():
if '"pool_stats"' not in line:
continue
item = json.loads(line)
ts = dt.datetime.fromisoformat(item["ts"].replace("Z", "+00:00")).timestamp()
if not start_wall <= ts <= end_wall:
continue
intervals += 1
waits += item["requests_wait_ms"]
nums += item["requests_num"]
waiting = max(waiting, item["requests_waiting"])
cpu += item["cpu_s"]
usage += item["usage_ms"]
return {"pool_requests": nums, "pool_wait_ms_avg": round(waits / nums, 3) if nums else None,
"pool_waiting_max": waiting, "pool_usage_ms_avg": round(usage / nums, 3) if nums else None,
"api_cpu_s": round(cpu, 2), "intervals": intervals}
def metric(data: dict, name: str, field: str):
m = data["metrics"].get(name)
return None if m is None else m["values"].get(field)
def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> dict:
psql("SELECT pg_stat_statements_reset()")
pg0, lb0 = pg_counters(), lb_stat()
since = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
started = time.monotonic()
wall0 = time.time()
max_vus = max(50, min(args.max_vus, rate * 2))
out = compose("run", "--rm", "--no-deps", "-v", f"{chats_dir}:/data:ro", "-e", f"RATE={rate}",
"-e", f"WARMUP={args.warmup}s", "-e", f"DURATION={args.duration}s", "-e", f"MAX_VUS={max_vus}",
"k6", "run", "--quiet", "/scripts/messages.js", timeout=args.warmup + args.duration + 180)
ended = time.monotonic()
data = json.loads(out[out.index(MARK) + len(MARK):].strip().splitlines()[0])
# k6 starts ~1-3 s after the command; the measure phase is the last `duration` seconds before exit.
m_end = ended - 1.0
m_start = max(started, m_end - args.duration)
pg1, lb1 = pg_counters(), lb_stat()
ok = metric(data, "ok_messages{phase:measure}", "count") or 0
reqs = metric(data, "http_reqs{phase:measure}", "count") or 0
dropped = metric(data, "dropped_iterations{scenario:measure}", "count") or 0
result = {
"config": config_name(cfg), "rate": rate, "duration_s": args.duration,
"offered": rate * args.duration, "requests": int(reqs), "ok": int(ok), "dropped": int(dropped),
"ok_rps": round(ok / args.duration, 1),
"error_ratio": round(metric(data, "http_req_failed{phase:measure}", "rate") or 0, 5),
"ok_p50_ms": metric(data, "ok_latency{phase:measure}", "p(50)"),
"ok_p95_ms": metric(data, "ok_latency{phase:measure}", "p(95)"),
"ok_p99_ms": metric(data, "ok_latency{phase:measure}", "p(99)"),
"vus_max": metric(data, "vus_max", "value"),
"cpu": sampler.window(m_start, m_end),
"pool": pool_window(since, wall0 + (m_start - started), wall0 + (m_end - started)),
"pg": {k: pg1[k] - pg0[k] for k in pg1},
"lb": {"chkdown_delta": lb1["chkdown"] - lb0["chkdown"], "qmax": lb1["qmax"], "up_after": lb1["up"],
"econ_delta": lb1["econ"] - lb0["econ"], "eresp_delta": lb1["eresp"] - lb0["eresp"],
"redispatch_delta": lb1["wredis"] - lb0["wredis"]},
"statements": top_statements(),
}
commits = result["pg"]["xact_commit"]
result["pg_commits_per_ok"] = round(commits / ok, 2) if ok else None
result["statements_per_ok"] = round(sum(s["calls"] for s in result["statements"]) / ok, 2) if ok else None
return result
def flat(r: dict) -> dict:
cpu = r["cpu"]
api_cpu = cpu.get("api_total", {}).get("cpu_median", 0.0)
limits = r.get("limits", {})
api_limits = [v for k, v in limits.items() if k.startswith("api-")]
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
"replicas_running": len(api_limits), "api_cpus_limit_total": round(sum(v["cpus"] for v in api_limits), 2),
"api_mem_mib_each": api_limits[0]["memory_mib"] if api_limits else None,
"pg_cpus_limit": limits.get("postgres-1", {}).get("cpus"),
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
"lb_cpu_pct": cpu.get("lb-1", {}).get("cpu_median"), "valkey_cpu_pct": cpu.get("valkey-1", {}).get("cpu_median"),
"k6_cpu_pct": cpu.get("k6", {}).get("cpu_median"),
"pool_wait_ms_avg": r["pool"]["pool_wait_ms_avg"], "pool_waiting_max": r["pool"]["pool_waiting_max"],
"pg_commits_per_ok": r["pg_commits_per_ok"], "statements_per_ok": r["statements_per_ok"],
"lb_chkdown": r["lb"]["chkdown_delta"], "lb_qmax": r["lb"]["qmax"], "wal_bytes": r["pg"]["wal_bytes"]}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--series", required=True)
parser.add_argument("--label", default="baseline")
parser.add_argument("--config", action="append", required=True, help="replicas=2,api_cpus=1.0,pg_cpus=2.0,pool=10,ENV=val")
parser.add_argument("--rates", required=True, help="comma separated arrival rates, req/s")
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=20)
parser.add_argument("--users", type=int, default=64)
parser.add_argument("--max-vus", type=int, default=600)
parser.add_argument("--git-ref", default="HEAD", help="commit whose code is measured (image built via git archive)")
args = parser.parse_args()
sha, IMAGE["tag"] = build_image(args.git_ref)
out = ROOT / "evidence" / "track1" / args.series / args.label
(out / "raw").mkdir(parents=True, exist_ok=True)
rates = [int(r) for r in args.rates.split(",")]
configs = [parse_config(c) for c in args.config]
meta = {"started": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"measured_commit": sha, "image": IMAGE["tag"],
"bench_tooling_dirty": bool(subprocess.run(["git", "status", "--porcelain", "--", "bench", "compose.yaml",
"infra"], cwd=ROOT, capture_output=True, text=True).stdout.strip()),
"host": {"os": f"{platform.system()} {platform.release()}",
"docker": subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} CPU "
"{{.MemTotal}} B {{.KernelVersion}}"], capture_output=True,
text=True).stdout.strip()},
"cpusets": "k6 0-1, lb+valkey 2, postgres 3-4, api 5-7 (bench/track1/compose.bench.yaml)",
"workload": f"k6 constant-arrival-rate POST message, new Idempotency-Key, {args.users} chats, "
f"warmup {args.warmup}s + measure {args.duration}s per step, fresh stand per config x repeat",
"rates": rates, "repeats": args.repeats, "configs": configs, "args": sys_argv()}
(out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
csv_path = out / "results.csv"
new_file = not csv_path.exists()
with csv_path.open("a", newline="", encoding="utf-8") as f:
writer = None
for repeat in range(1, args.repeats + 1):
for cfg in configs:
name = config_name(cfg)
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
start_stand(cfg)
limits = actual_limits()
chats_dir = tempfile.mkdtemp(prefix="track1-")
try:
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
sampler = Sampler()
sampler.start()
for rate in rates:
result = run_step(cfg, rate, args, chats_dir, sampler)
result["repeat"] = repeat
result["limits"] = limits
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
row = flat(result)
if writer is None:
writer = csv.DictWriter(f, fieldnames=list(row))
if new_file:
writer.writeheader()
writer.writerow(row)
f.flush()
print(f" rate {rate}: ok_rps {row['ok_rps']} p95 {row['p95_ms'] and round(row['p95_ms'])} ms "
f"err {row['error_ratio']} dropped {row['dropped']} | api {row['api_cpu_pct_total']}% "
f"pg {row['pg_cpu_pct']}% lb {row['lb_cpu_pct']}% k6 {row['k6_cpu_pct']}% "
f"pool_wait {row['pool_wait_ms_avg']} ms chkdown {row['lb_chkdown']}", flush=True)
sampler.stop_event.set()
finally:
shutil.rmtree(chats_dir, ignore_errors=True) # synthetic tokens never stay on disk
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
print(f"results: {csv_path}")
return 0
def sys_argv() -> list[str]:
import sys
return sys.argv[1:]
if __name__ == "__main__":
raise SystemExit(main())