bench: трек 1 — оркестратор серий, k6 open-loop, py-spy sidecar

- bench/track1/run.py: на каждую конфигурацию x повтор — свежий стенд
  (отдельный compose-проект, пустые volumes), образ API из git-коммита
  (git archive), ступени constant-arrival-rate; CPU каждого контейнера,
  ожидание пула (STATS_LOG_SECONDS), pg_stat_database/pg_stat_statements,
  счётчики HAProxy; raw JSON + results.csv, без токенов.
- compose.bench.yaml: cpuset для k6 / LB+Valkey / PostgreSQL / API, чтобы
  генератор и LB не отнимали CPU у реплик.
- bench/track1/profile.py + pyspy/: py-spy с хешем пакета, профиль одной
  реплики под нагрузкой.
- API: STATS_LOG_SECONDS (по умолчанию выключено) — счётчики пула и CPU
  процесса в лог; API_IMAGE в compose для образа из коммита.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-16 13:51:37 +03:00
co-authored by Claude Opus 5.5
parent c050642ec2
commit 2033440cdb
9 changed files with 628 additions and 4 deletions
+30
View File
@@ -0,0 +1,30 @@
# Track 1 overlay: docker compose -p mtusi-lab3-bench -f compose.yaml -f bench/track1/compose.bench.yaml ...
# Used only by bench/track1/run.py. A separate project (own volumes, own ports) so the
# experiments never touch the main stand's data. Every role is pinned to its own CPU set,
# so the load generator, the LB and PostgreSQL do not steal CPU from the API replicas and
# the "total API CPU" of a configuration is exactly what the limits say:
# k6 0-1 | lb + valkey 2 | postgres 3-4 | api 5-7 (all replicas share these 3 CPUs; per-replica quota API_CPUS)
services:
postgres:
cpuset: "${PG_CPUSET:-3-4}"
valkey:
cpuset: "${VALKEY_CPUSET:-2}"
lb:
cpuset: "${LB_CPUSET:-2}"
migrate:
cpuset: "${API_CPUSET:-5-7}"
api:
cpuset: "${API_CPUSET:-5-7}"
environment:
STATS_LOG_SECONDS: "${STATS_LOG_SECONDS:-2}"
k6:
image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34
profiles: ["bench"]
cpuset: "${K6_CPUSET:-0-1}"
environment:
BASE_URL: http://lb:8080
volumes:
- ./bench/track1/k6:/scripts:ro
deploy:
resources:
limits: {cpus: "2.0", memory: 1g}
+67
View File
@@ -0,0 +1,67 @@
// Track 1 workload: OPEN-LOOP arrival rate of "send message" requests through the LB.
// Every iteration = one POST /api/v1/chats/{id}/messages with a NEW Idempotency-Key into one
// of USERS chats (user i writes into chat i, shared with user i+1), so writers are spread over
// many chat rows and no single-row lock dominates. constant-arrival-rate starts iterations at
// RATE/s regardless of response time (no coordinated omission); iterations that cannot start
// because all VUs are busy are counted in dropped_iterations - they are failed demand.
// Phases: `warmup` (not reported) then `measure`. Only status 201 counts as success.
// Env: BASE_URL, RATE, WARMUP, DURATION, MAX_VUS, CHATS_FILE.
import http from 'k6/http';
import { check } from 'k6';
import exec from 'k6/execution';
import { Counter, Trend } from 'k6/metrics';
const BASE = __ENV.BASE_URL || 'http://lb:8080';
const RATE = parseInt(__ENV.RATE || '100', 10);
const WARMUP = __ENV.WARMUP || '10s';
const DURATION = __ENV.DURATION || '20s';
const MAX_VUS = parseInt(__ENV.MAX_VUS || '400', 10);
const ok = new Counter('ok_messages'); // 201 only
const okLatency = new Trend('ok_latency', true); // latency of 201 only
function phase(name, startTime, duration) {
return {
executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', duration, startTime,
preAllocatedVUs: Math.min(MAX_VUS, Math.max(20, Math.ceil(RATE / 10))), maxVUs: MAX_VUS,
gracefulStop: '5s', exec: 'send', tags: { phase: name },
};
}
export const options = {
discardResponseBodies: true,
scenarios: { warmup: phase('warmup', '0s', WARMUP), measure: phase('measure', WARMUP, DURATION) },
summaryTrendStats: ['avg', 'p(50)', 'p(95)', 'p(99)', 'max'],
// Thresholds only create per-phase sub-metrics in the summary; they never fail the run.
thresholds: {
'http_reqs{phase:measure}': ['count>=0'],
'http_req_failed{phase:measure}': ['rate>=0'],
'http_req_duration{phase:measure}': ['max>=0'],
'ok_messages{phase:measure}': ['count>=0'],
'ok_latency{phase:measure}': ['max>=0'],
'dropped_iterations{scenario:measure}': ['count>=0'],
'iterations{scenario:measure}': ['count>=0'],
},
};
// Users, sessions and chats are created by bench/track1/run.py before k6 starts (so the timed
// phases start at once); the file with the synthetic tokens is mounted read-only and deleted after.
const chats = JSON.parse(open(__ENV.CHATS_FILE || '/data/chats.json'));
export function send() {
const n = exec.scenario.iterationInTest;
const c = chats[n % chats.length];
const res = http.post(`${BASE}/api/v1/chats/${c.chat}/messages`,
JSON.stringify({ text: `bench ${exec.vu.idInTest}-${n}` }),
{ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${c.token}`,
'Idempotency-Key': `${exec.scenario.name}-${exec.vu.idInTest}-${n}-${Date.now()}` },
tags: { name: 'post_message' }, timeout: '10s' });
if (check(res, { '201': (r) => r.status === 201 })) {
ok.add(1);
okLatency.add(res.timings.duration);
}
}
export function handleSummary(data) {
return { stdout: '\n@@SUMMARY@@' + JSON.stringify(data) + '\n' };
}
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Where does one API replica spend its CPU under the track 1 workload? (py-spy sidecar)
python bench/track1/profile.py --label baseline --rate 260 --config replicas=2,api_cpus=1.0
Starts a fresh bench stand, runs k6 at a fixed arrival rate and samples ONE replica with
py-spy (sidecar sharing its PID namespace, SYS_PTRACE only for the sidecar, image
bench/track1/pyspy with a hash-pinned py-spy). Output evidence/track1/profile-<label>.txt:
share of samples per library ("where the time goes") and the top functions by self/total time.
"""
from __future__ import annotations
import argparse
import collections
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import run as bench # noqa: E402
ROOT = bench.ROOT
IMAGE = "mtusi-pyspy:lab3"
CATEGORIES = [ # first match wins; path fragments of the frames' files
("psycopg (PostgreSQL driver + pool)", ("psycopg",)),
("redis-py (Valkey client)", ("redis/",)),
("uvicorn + httptools (HTTP server)", ("uvicorn", "httptools")),
("starlette (routing, middleware, responses)", ("starlette",)),
("logging (access log JSON lines)", ("logging/", "logging_setup.py")),
("json (stdlib)", ("json/",)),
("asyncio / uvloop (event loop)", ("asyncio", "uvloop")),
("messenger (our code)", ("messenger/",)),
]
def category(stack: list[str]) -> str:
"""The innermost frame that belongs to a known library decides the category."""
for frame in reversed(stack):
for name, needles in CATEGORIES:
if any(n in frame for n in needles):
return name
return "other"
def short(frame: str) -> str:
name, _, where = frame.partition(" (")
file = where.rstrip(")").split(":")[0].rsplit("/", 2)[-2:]
return f"{name} ({'/'.join(file)})"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--label", required=True)
parser.add_argument("--config", default="replicas=2,api_cpus=1.0")
parser.add_argument("--rate", type=int, default=260)
parser.add_argument("--seconds", type=int, default=20)
parser.add_argument("--top", type=int, default=25)
parser.add_argument("--git-ref", default="HEAD")
args = parser.parse_args()
subprocess.run(["docker", "build", "-q", "-t", IMAGE, str(ROOT / "bench" / "track1" / "pyspy")], check=True,
capture_output=True)
cfg = bench.parse_config(args.config)
sha, bench.IMAGE["tag"] = bench.build_image(args.git_ref)
bench.start_stand(cfg)
work = tempfile.mkdtemp(prefix="track1-profile-")
try:
Path(work, "chats.json").write_text(__import__("json").dumps(bench.make_chats(64)), encoding="utf-8")
warmup, duration = 8, args.seconds + 8
k6 = subprocess.Popen(["docker", "compose", "-p", bench.PROJECT, *bench.FILES, "run", "--rm", "--no-deps",
"-v", f"{work}:/data:ro", "-e", f"RATE={args.rate}", "-e", f"WARMUP={warmup}s",
"-e", f"DURATION={duration}s", "-e", f"MAX_VUS={args.rate * 2}", "k6", "run", "--quiet",
"/scripts/messages.js"], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, encoding="utf-8")
time.sleep(warmup + 4)
target = bench.compose("ps", "-q", "api").split()[0]
subprocess.run(["docker", "run", "--rm", "--pid", f"container:{target}", "--cap-add", "SYS_PTRACE",
"-v", f"{work}:/out", IMAGE, "record", "--pid", "1", "--duration", str(args.seconds),
"--rate", "250", "--format", "raw", "--nonblocking", "--output", "/out/stacks.txt"],
check=True, capture_output=True, timeout=args.seconds + 120)
out, _ = k6.communicate(timeout=300)
data = __import__("json").loads(out[out.index(bench.MARK) + len(bench.MARK):].strip().splitlines()[0])
ok_rps = (bench.metric(data, "ok_messages{phase:measure}", "count") or 0) / duration
stacks = []
for line in Path(work, "stacks.txt").read_text(encoding="utf-8").splitlines():
stack, _, count = line.rpartition(" ")
frames = [f for f in stack.split(";") if f and not f.startswith("process ") and not f.startswith("thread ")]
if frames:
stacks.append((frames, int(count)))
finally:
shutil.rmtree(work, ignore_errors=True)
bench.compose("down", "-v", "--remove-orphans", check=False, timeout=300)
total = sum(c for _, c in stacks)
idle = sum(c for f, c in stacks if any("select" in x and ("selectors" in x or "uvloop" in x) for x in f[-2:]))
cats = collections.Counter()
self_time, total_time = collections.Counter(), collections.Counter()
for frames, count in stacks:
cats[category(frames)] += count
self_time[short(frames[-1])] += count
for name in {short(f) for f in frames}:
total_time[name] += count
lines = [f"# py-spy, {args.seconds} s at 250 Hz, ONE replica of {bench.config_name(cfg)}, k6 {args.rate} req/s "
f"(ok {ok_rps:.1f} req/s for the whole stand)", f"# label {args.label}, commit {sha}; samples {total} "
f"(idle in the event loop poll: {100 * idle / max(total, 1):.1f} %)", "",
"## Share of samples by library (innermost known frame)"]
for name, count in cats.most_common():
lines.append(f"{100 * count / total:6.1f} % {name}")
lines += ["", f"## Top {args.top} functions by SELF time (on top of the stack)"]
for name, count in self_time.most_common(args.top):
lines.append(f"{100 * count / total:6.1f} % {name}")
lines += ["", f"## Top {args.top} functions by TOTAL time (anywhere in the stack)"]
for name, count in total_time.most_common(args.top):
lines.append(f"{100 * count / total:6.1f} % {name}")
dest = ROOT / "evidence" / "track1" / f"profile-{args.label}.txt"
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text("\n".join(lines) + "\n", encoding="utf-8")
print("\n".join(lines[:40]))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+6
View File
@@ -0,0 +1,6 @@
# Profiler sidecar: shares the PID namespace of ONE api container (--pid=container:<id>) and
# gets SYS_PTRACE only for itself; the API image stays unprivileged and unchanged.
FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
COPY requirements.txt /tmp/requirements.txt
RUN pip install --no-cache-dir --root-user-action=ignore --require-hashes --no-deps -r /tmp/requirements.txt
ENTRYPOINT ["py-spy"]
+3
View File
@@ -0,0 +1,3 @@
# py-spy sampling profiler for the track 1 sidecar (bench/track1/profile.py), pinned by hash.
py-spy==0.4.2 \
--hash=sha256:aeb0323409199c785f730645e9f4bb7a7b9ca2c481f2c331a55642b5d13fa52f
+371
View File
@@ -0,0 +1,371 @@
#!/usr/bin/env python3
"""Track 1 orchestrator: cost of scaling. For every configuration (replicas x CPU per replica x
PostgreSQL CPU x pool size x env) and every repeat a FRESH bench stand is started (separate
compose project, empty volumes), synthetic users/chats are created, then k6 runs an open-loop
arrival-rate sweep. During each step the orchestrator samples CPU of every container
(docker stats), DB pool waits (API stats log), PostgreSQL counters (pg_stat_database,
pg_stat_statements) and HAProxy counters (show stat).
python bench/track1/run.py --series fixed-total --label baseline \
--config replicas=1,api_cpus=2.0 --config replicas=2,api_cpus=1.0 --config replicas=4,api_cpus=0.5 \
--rates 200,400,600,800 --repeats 3
Output: evidence/track1/<series>/<label>/raw/*.json (one per step, no tokens) and
results.csv (one row per step). The main stand (project mtusi-lab3) is not touched,
but stop it during experiments so it does not compete for CPU.
"""
from __future__ import annotations
import argparse
import base64
import csv
import datetime as dt
import json
import os
import platform
import shutil
import statistics
import subprocess
import tempfile
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PROJECT = "mtusi-lab3-bench"
PORT = 18080
FILES = ["-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "bench" / "track1" / "compose.bench.yaml")]
MARK = "@@SUMMARY@@"
DEFAULTS = {"replicas": 2, "api_cpus": 1.0, "api_memory": "384m", "pg_cpus": 2.0, "pool": 10}
# ------------------------------------------------------------------ helpers
def compose(*args: str, env: dict | None = None, check: bool = True, timeout: float = 600) -> str:
result = subprocess.run(["docker", "compose", "-p", PROJECT, *FILES, *args], cwd=ROOT, capture_output=True,
text=True, encoding="utf-8", env={**os.environ, **(env or {})}, timeout=timeout)
if check and result.returncode != 0:
raise RuntimeError(f"compose {' '.join(args[:3])} failed: {result.stderr.strip()[-1500:]}")
return result.stdout
def psql(sql: str) -> str:
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t",
"-v", "ON_ERROR_STOP=1", "-c", sql).strip()
def http(method: str, path: str, body=None, headers=None):
data = None if body is None else json.dumps(body).encode()
h = {"Content-Type": "application/json", **(headers or {})}
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, json.loads(r.read() or b"null")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def parse_config(text: str) -> dict:
cfg = dict(DEFAULTS)
cfg["env"] = {}
for part in filter(None, text.split(",")):
key, value = part.split("=", 1)
if key in ("replicas", "pool"):
cfg[key] = int(value)
elif key in ("api_cpus", "pg_cpus"):
cfg[key] = float(value)
elif key == "api_memory":
cfg[key] = value
else:
cfg["env"][key] = value # any API/compose env, e.g. MESSAGE_WRITE_PATH=function
return cfg
def config_name(cfg: dict) -> str:
extra = "".join(f"-{k.lower()}_{v}" for k, v in sorted(cfg["env"].items()))
return f"n{cfg['replicas']}-api{cfg['api_cpus']:g}-pg{cfg['pg_cpus']:g}-pool{cfg['pool']}{extra}"
def build_image(ref: str) -> tuple[str, str]:
"""Image of the API built from a COMMIT (git archive), not from the working tree, so a
series is reproducible and edits made while it runs cannot leak into it."""
sha = subprocess.run(["git", "rev-parse", "--short", ref], cwd=ROOT, capture_output=True, text=True,
check=True).stdout.strip()
tag = f"mtusi-messenger:bench-{sha}"
archive = subprocess.run(["git", "archive", "--format=tar", sha], cwd=ROOT, capture_output=True, check=True).stdout
subprocess.run(["docker", "build", "-q", "--build-arg", f"BUILD_REVISION={sha}", "-t", tag, "-"], input=archive,
capture_output=True, check=True, timeout=900)
return sha, tag
IMAGE = {"tag": "mtusi-messenger:lab3"}
def stand_env(cfg: dict) -> dict:
return {"API_IMAGE": IMAGE["tag"], "API_REPLICAS": str(cfg["replicas"]), "API_CPUS": f"{cfg['api_cpus']:g}", "API_MEMORY": cfg["api_memory"],
"PG_CPUS": f"{cfg['pg_cpus']:g}", "DB_POOL_MIN": str(min(cfg["pool"], 4)), "DB_POOL_MAX": str(cfg["pool"]),
"APP_PUBLISH_PORT": str(PORT), "LB_STATS_PORT": "18404", "BUILD_REVISION": "bench",
**cfg["env"]}
def start_stand(cfg: dict) -> None:
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
compose("up", "-d", "--no-build", "--wait", env=stand_env(cfg), timeout=900)
psql("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
deadline = time.monotonic() + 60
while time.monotonic() < deadline: # every replica UP in HAProxy before measuring
if lb_stat()["up"] == cfg["replicas"]:
return
time.sleep(0.5)
raise RuntimeError("not all replicas UP in HAProxy")
def make_chats(users: int) -> list[dict]:
created = []
stamp = uuid.uuid4().hex[:8]
for i in range(users):
username, password = f"b_{stamp}_{i}", f"Bench-password-{stamp}-{i}"
status, user = http("POST", "/api/v1/users", {"username": username, "display_name": f"bench {i}", "password": password})
assert status == 201, (status, user)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, (status, session)
created.append({"id": user["id"], "token": session["token"]})
chats = []
for i, me in enumerate(created):
other = created[(i + 1) % len(created)]
status, chat = http("POST", "/api/v1/chats", {"title": f"bench {i}", "member_ids": [me["id"], other["id"]]},
headers={"Authorization": "Bearer " + me["token"]})
assert status == 201, (status, chat)
chats.append({"token": me["token"], "chat": chat["id"]})
return chats
class Sampler(threading.Thread):
"""docker stats every ~1-2 s: CPU% (100 = one core) and memory per container."""
def __init__(self):
super().__init__(daemon=True)
self.samples: list[tuple[float, dict]] = []
self.stop_event = threading.Event()
def run(self) -> None:
while not self.stop_event.is_set():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}"],
capture_output=True, text=True).stdout
now = time.monotonic()
row = {}
for line in out.splitlines():
item = json.loads(line)
if not item["Name"].startswith(PROJECT + "-"):
continue # only this bench project (k6 run containers included)
short = item["Name"][len(PROJECT) + 1:]
if short.startswith("k6-run-"):
short = "k6"
row[short] = {"cpu": float(item["CPUPerc"].rstrip("%") or 0), "mem": item["MemUsage"].split(" / ")[0]}
self.samples.append((now, row))
def window(self, start: float, end: float) -> dict:
rows = [r for t, r in self.samples if start <= t <= end]
result = {}
for name in sorted({n for r in rows for n in r}):
values = [r[name]["cpu"] for r in rows if name in r]
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
"mem_last": [r[name]["mem"] for r in rows if name in r][-1], "samples": len(values)}
return result
def lb_stat() -> dict:
raw = compose("exec", "-T", "lb", "sh", "-c", 'echo "show stat" | socat stdio /var/lib/haproxy/admin.sock')
lines = [line for line in raw.splitlines() if line]
header = lines[0].lstrip("# ").split(",")
servers = [dict(zip(header, line.split(","))) for line in lines[1:]]
api = [s for s in servers if s["pxname"] == "api" and s["svname"].startswith("api") and s["status"] != "MAINT"]
backend = next(s for s in servers if s["pxname"] == "api" and s["svname"] == "BACKEND")
return {"up": sum(1 for s in api if s["status"].startswith("UP")),
"chkdown": sum(int(s["chkdown"] or 0) for s in api), "qmax": int(backend["qmax"] or 0),
"smax": int(backend["smax"] or 0), "econ": int(backend["econ"] or 0), "eresp": int(backend["eresp"] or 0),
"wredis": int(backend["wredis"] or 0), "wretr": int(backend["wretr"] or 0)}
def pg_counters() -> dict:
row = psql("SELECT xact_commit, xact_rollback, tup_inserted, blks_read, blks_hit FROM pg_stat_database "
"WHERE datname = 'messenger'").split("\t")
wal = psql("SELECT wal_records, wal_bytes FROM pg_stat_wal").split("\t")
return {"xact_commit": int(row[0]), "xact_rollback": int(row[1]), "tup_inserted": int(row[2]),
"blks_read": int(row[3]), "blks_hit": int(row[4]), "wal_records": int(wal[0]), "wal_bytes": int(float(wal[1]))}
def top_statements(limit: int = 12) -> list[dict]:
out = psql("SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 4), "
"left(regexp_replace(query, '\\s+', ' ', 'g'), 110) FROM pg_stat_statements "
"WHERE dbid = (SELECT oid FROM pg_database WHERE datname = 'messenger') "
f"ORDER BY total_exec_time DESC LIMIT {limit}")
rows = []
for line in out.splitlines():
calls, total, mean, query = line.split("\t", 3)
rows.append({"calls": int(calls), "total_ms": float(total), "mean_ms": float(mean), "query": query})
return rows
def pool_window(since_iso: str, start_wall: float, end_wall: float) -> dict:
logs = compose("logs", "--no-log-prefix", "--since", since_iso, "api")
waits, nums, waiting, cpu, usage, intervals = 0, 0, 0, 0.0, 0, 0
for line in logs.splitlines():
if '"pool_stats"' not in line:
continue
item = json.loads(line)
ts = dt.datetime.fromisoformat(item["ts"].replace("Z", "+00:00")).timestamp()
if not start_wall <= ts <= end_wall:
continue
intervals += 1
waits += item["requests_wait_ms"]
nums += item["requests_num"]
waiting = max(waiting, item["requests_waiting"])
cpu += item["cpu_s"]
usage += item["usage_ms"]
return {"pool_requests": nums, "pool_wait_ms_avg": round(waits / nums, 3) if nums else None,
"pool_waiting_max": waiting, "pool_usage_ms_avg": round(usage / nums, 3) if nums else None,
"api_cpu_s": round(cpu, 2), "intervals": intervals}
def metric(data: dict, name: str, field: str):
m = data["metrics"].get(name)
return None if m is None else m["values"].get(field)
def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> dict:
psql("SELECT pg_stat_statements_reset()")
pg0, lb0 = pg_counters(), lb_stat()
since = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
started = time.monotonic()
wall0 = time.time()
max_vus = max(50, min(args.max_vus, rate * 2))
out = compose("run", "--rm", "--no-deps", "-v", f"{chats_dir}:/data:ro", "-e", f"RATE={rate}",
"-e", f"WARMUP={args.warmup}s", "-e", f"DURATION={args.duration}s", "-e", f"MAX_VUS={max_vus}",
"k6", "run", "--quiet", "/scripts/messages.js", timeout=args.warmup + args.duration + 180)
ended = time.monotonic()
data = json.loads(out[out.index(MARK) + len(MARK):].strip().splitlines()[0])
# k6 starts ~1-3 s after the command; the measure phase is the last `duration` seconds before exit.
m_end = ended - 1.0
m_start = max(started, m_end - args.duration)
pg1, lb1 = pg_counters(), lb_stat()
ok = metric(data, "ok_messages{phase:measure}", "count") or 0
reqs = metric(data, "http_reqs{phase:measure}", "count") or 0
dropped = metric(data, "dropped_iterations{scenario:measure}", "count") or 0
result = {
"config": config_name(cfg), "rate": rate, "duration_s": args.duration,
"offered": rate * args.duration, "requests": int(reqs), "ok": int(ok), "dropped": int(dropped),
"ok_rps": round(ok / args.duration, 1),
"error_ratio": round(metric(data, "http_req_failed{phase:measure}", "rate") or 0, 5),
"ok_p50_ms": metric(data, "ok_latency{phase:measure}", "p(50)"),
"ok_p95_ms": metric(data, "ok_latency{phase:measure}", "p(95)"),
"ok_p99_ms": metric(data, "ok_latency{phase:measure}", "p(99)"),
"vus_max": metric(data, "vus_max", "value"),
"cpu": sampler.window(m_start, m_end),
"pool": pool_window(since, wall0 + (m_start - started), wall0 + (m_end - started)),
"pg": {k: pg1[k] - pg0[k] for k in pg1},
"lb": {"chkdown_delta": lb1["chkdown"] - lb0["chkdown"], "qmax": lb1["qmax"], "up_after": lb1["up"],
"econ_delta": lb1["econ"] - lb0["econ"], "eresp_delta": lb1["eresp"] - lb0["eresp"],
"redispatch_delta": lb1["wredis"] - lb0["wredis"]},
"statements": top_statements(),
}
commits = result["pg"]["xact_commit"]
result["pg_commits_per_ok"] = round(commits / ok, 2) if ok else None
result["statements_per_ok"] = round(sum(s["calls"] for s in result["statements"]) / ok, 2) if ok else None
return result
def flat(r: dict) -> dict:
cpu = r["cpu"]
api_cpu = sum(v["cpu_median"] for k, v in cpu.items() if k.startswith("api-"))
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
"lb_cpu_pct": cpu.get("lb-1", {}).get("cpu_median"), "valkey_cpu_pct": cpu.get("valkey-1", {}).get("cpu_median"),
"k6_cpu_pct": cpu.get("k6", {}).get("cpu_median"),
"pool_wait_ms_avg": r["pool"]["pool_wait_ms_avg"], "pool_waiting_max": r["pool"]["pool_waiting_max"],
"pg_commits_per_ok": r["pg_commits_per_ok"], "statements_per_ok": r["statements_per_ok"],
"lb_chkdown": r["lb"]["chkdown_delta"], "lb_qmax": r["lb"]["qmax"], "wal_bytes": r["pg"]["wal_bytes"]}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--series", required=True)
parser.add_argument("--label", default="baseline")
parser.add_argument("--config", action="append", required=True, help="replicas=2,api_cpus=1.0,pg_cpus=2.0,pool=10,ENV=val")
parser.add_argument("--rates", required=True, help="comma separated arrival rates, req/s")
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=20)
parser.add_argument("--users", type=int, default=64)
parser.add_argument("--max-vus", type=int, default=600)
parser.add_argument("--git-ref", default="HEAD", help="commit whose code is measured (image built via git archive)")
args = parser.parse_args()
sha, IMAGE["tag"] = build_image(args.git_ref)
out = ROOT / "evidence" / "track1" / args.series / args.label
(out / "raw").mkdir(parents=True, exist_ok=True)
rates = [int(r) for r in args.rates.split(",")]
configs = [parse_config(c) for c in args.config]
meta = {"started": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
"measured_commit": sha, "image": IMAGE["tag"],
"bench_tooling_dirty": bool(subprocess.run(["git", "status", "--porcelain", "--", "bench", "compose.yaml",
"infra"], cwd=ROOT, capture_output=True, text=True).stdout.strip()),
"host": {"os": f"{platform.system()} {platform.release()}",
"docker": subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} CPU "
"{{.MemTotal}} B {{.KernelVersion}}"], capture_output=True,
text=True).stdout.strip()},
"cpusets": "k6 0-1, lb+valkey 2, postgres 3-4, api 5-7 (bench/track1/compose.bench.yaml)",
"workload": f"k6 constant-arrival-rate POST message, new Idempotency-Key, {args.users} chats, "
f"warmup {args.warmup}s + measure {args.duration}s per step, fresh stand per config x repeat",
"rates": rates, "repeats": args.repeats, "configs": configs, "args": sys_argv()}
(out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
csv_path = out / "results.csv"
new_file = not csv_path.exists()
with csv_path.open("a", newline="", encoding="utf-8") as f:
writer = None
for repeat in range(1, args.repeats + 1):
for cfg in configs:
name = config_name(cfg)
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
start_stand(cfg)
chats_dir = tempfile.mkdtemp(prefix="track1-")
try:
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
sampler = Sampler()
sampler.start()
for rate in rates:
result = run_step(cfg, rate, args, chats_dir, sampler)
result["repeat"] = repeat
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
row = flat(result)
if writer is None:
writer = csv.DictWriter(f, fieldnames=list(row))
if new_file:
writer.writeheader()
writer.writerow(row)
f.flush()
print(f" rate {rate}: ok_rps {row['ok_rps']} p95 {row['p95_ms'] and round(row['p95_ms'])} ms "
f"err {row['error_ratio']} dropped {row['dropped']} | api {row['api_cpu_pct_total']}% "
f"pg {row['pg_cpu_pct']}% lb {row['lb_cpu_pct']}% k6 {row['k6_cpu_pct']}% "
f"pool_wait {row['pool_wait_ms_avg']} ms chkdown {row['lb_chkdown']}", flush=True)
sampler.stop_event.set()
finally:
shutil.rmtree(chats_dir, ignore_errors=True) # synthetic tokens never stay on disk
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
print(f"results: {csv_path}")
return 0
def sys_argv() -> list[str]:
import sys
return sys.argv[1:]
if __name__ == "__main__":
raise SystemExit(main())
+2 -2
View File
@@ -84,7 +84,7 @@ services:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: mtusi-messenger:lab3
image: ${API_IMAGE:-mtusi-messenger:lab3}
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
@@ -104,7 +104,7 @@ services:
context: .
args:
BUILD_REVISION: ${BUILD_REVISION:-dev}
image: mtusi-messenger:lab3
image: ${API_IMAGE:-mtusi-messenger:lab3}
environment: *api-env
secrets: [database_url, cache_url]
# No `ports` and no container_name: replicas are reachable only through `lb`,
+21 -2
View File
@@ -338,6 +338,22 @@ def create_app(settings: Settings | None = None) -> Starlette:
except ApiError:
pass # database unavailable right now; next round
stats_log = logging.getLogger("messenger.stats")
async def stats_loop(every: float) -> None:
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
cpu = time.process_time()
db.pool.pop_stats()
while True:
await asyncio.sleep(every)
stats = db.pool.pop_stats()
now_cpu = time.process_time()
stats_log.info("pool_stats", extra={"fields": {
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
cpu = now_cpu
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
passwords.configure(settings.hash_concurrency)
@@ -346,11 +362,14 @@ def create_app(settings: Settings | None = None) -> Starlette:
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds}})
purge = asyncio.create_task(purge_loop())
tasks = [asyncio.create_task(purge_loop())]
if settings.stats_log_seconds > 0:
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
try:
yield
finally:
purge.cancel()
for task in tasks:
task.cancel()
await sessions.close()
await db.close()
+3
View File
@@ -106,6 +106,7 @@ class Settings:
instance_id: str = "local"
drain_seconds: float = 0.0
idempotency_ttl_seconds: int = 86400
stats_log_seconds: float = 0.0
@property
def db_statement_timeout_ms(self) -> int:
@@ -151,4 +152,6 @@ class Settings:
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
# Contract: a replay within 24 h returns the original result.
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
)