forked from mtusi2026/mtusi2026lab3
bench: трек 1 — оркестратор серий, k6 open-loop, py-spy sidecar
- bench/track1/run.py: на каждую конфигурацию x повтор — свежий стенд (отдельный compose-проект, пустые volumes), образ API из git-коммита (git archive), ступени constant-arrival-rate; CPU каждого контейнера, ожидание пула (STATS_LOG_SECONDS), pg_stat_database/pg_stat_statements, счётчики HAProxy; raw JSON + results.csv, без токенов. - compose.bench.yaml: cpuset для k6 / LB+Valkey / PostgreSQL / API, чтобы генератор и LB не отнимали CPU у реплик. - bench/track1/profile.py + pyspy/: py-spy с хешем пакета, профиль одной реплики под нагрузкой. - API: STATS_LOG_SECONDS (по умолчанию выключено) — счётчики пула и CPU процесса в лог; API_IMAGE в compose для образа из коммита. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c050642ec2
commit
2033440cdb
@@ -0,0 +1,30 @@
|
||||
# Track 1 overlay: docker compose -p mtusi-lab3-bench -f compose.yaml -f bench/track1/compose.bench.yaml ...
|
||||
# Used only by bench/track1/run.py. A separate project (own volumes, own ports) so the
|
||||
# experiments never touch the main stand's data. Every role is pinned to its own CPU set,
|
||||
# so the load generator, the LB and PostgreSQL do not steal CPU from the API replicas and
|
||||
# the "total API CPU" of a configuration is exactly what the limits say:
|
||||
# k6 0-1 | lb + valkey 2 | postgres 3-4 | api 5-7 (all replicas share these 3 CPUs; per-replica quota API_CPUS)
|
||||
services:
|
||||
postgres:
|
||||
cpuset: "${PG_CPUSET:-3-4}"
|
||||
valkey:
|
||||
cpuset: "${VALKEY_CPUSET:-2}"
|
||||
lb:
|
||||
cpuset: "${LB_CPUSET:-2}"
|
||||
migrate:
|
||||
cpuset: "${API_CPUSET:-5-7}"
|
||||
api:
|
||||
cpuset: "${API_CPUSET:-5-7}"
|
||||
environment:
|
||||
STATS_LOG_SECONDS: "${STATS_LOG_SECONDS:-2}"
|
||||
k6:
|
||||
image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34
|
||||
profiles: ["bench"]
|
||||
cpuset: "${K6_CPUSET:-0-1}"
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
volumes:
|
||||
- ./bench/track1/k6:/scripts:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "2.0", memory: 1g}
|
||||
@@ -0,0 +1,67 @@
|
||||
// Track 1 workload: OPEN-LOOP arrival rate of "send message" requests through the LB.
|
||||
// Every iteration = one POST /api/v1/chats/{id}/messages with a NEW Idempotency-Key into one
|
||||
// of USERS chats (user i writes into chat i, shared with user i+1), so writers are spread over
|
||||
// many chat rows and no single-row lock dominates. constant-arrival-rate starts iterations at
|
||||
// RATE/s regardless of response time (no coordinated omission); iterations that cannot start
|
||||
// because all VUs are busy are counted in dropped_iterations - they are failed demand.
|
||||
// Phases: `warmup` (not reported) then `measure`. Only status 201 counts as success.
|
||||
// Env: BASE_URL, RATE, WARMUP, DURATION, MAX_VUS, CHATS_FILE.
|
||||
import http from 'k6/http';
|
||||
import { check } from 'k6';
|
||||
import exec from 'k6/execution';
|
||||
import { Counter, Trend } from 'k6/metrics';
|
||||
|
||||
const BASE = __ENV.BASE_URL || 'http://lb:8080';
|
||||
const RATE = parseInt(__ENV.RATE || '100', 10);
|
||||
const WARMUP = __ENV.WARMUP || '10s';
|
||||
const DURATION = __ENV.DURATION || '20s';
|
||||
const MAX_VUS = parseInt(__ENV.MAX_VUS || '400', 10);
|
||||
|
||||
const ok = new Counter('ok_messages'); // 201 only
|
||||
const okLatency = new Trend('ok_latency', true); // latency of 201 only
|
||||
|
||||
function phase(name, startTime, duration) {
|
||||
return {
|
||||
executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', duration, startTime,
|
||||
preAllocatedVUs: Math.min(MAX_VUS, Math.max(20, Math.ceil(RATE / 10))), maxVUs: MAX_VUS,
|
||||
gracefulStop: '5s', exec: 'send', tags: { phase: name },
|
||||
};
|
||||
}
|
||||
|
||||
export const options = {
|
||||
discardResponseBodies: true,
|
||||
scenarios: { warmup: phase('warmup', '0s', WARMUP), measure: phase('measure', WARMUP, DURATION) },
|
||||
summaryTrendStats: ['avg', 'p(50)', 'p(95)', 'p(99)', 'max'],
|
||||
// Thresholds only create per-phase sub-metrics in the summary; they never fail the run.
|
||||
thresholds: {
|
||||
'http_reqs{phase:measure}': ['count>=0'],
|
||||
'http_req_failed{phase:measure}': ['rate>=0'],
|
||||
'http_req_duration{phase:measure}': ['max>=0'],
|
||||
'ok_messages{phase:measure}': ['count>=0'],
|
||||
'ok_latency{phase:measure}': ['max>=0'],
|
||||
'dropped_iterations{scenario:measure}': ['count>=0'],
|
||||
'iterations{scenario:measure}': ['count>=0'],
|
||||
},
|
||||
};
|
||||
|
||||
// Users, sessions and chats are created by bench/track1/run.py before k6 starts (so the timed
|
||||
// phases start at once); the file with the synthetic tokens is mounted read-only and deleted after.
|
||||
const chats = JSON.parse(open(__ENV.CHATS_FILE || '/data/chats.json'));
|
||||
|
||||
export function send() {
|
||||
const n = exec.scenario.iterationInTest;
|
||||
const c = chats[n % chats.length];
|
||||
const res = http.post(`${BASE}/api/v1/chats/${c.chat}/messages`,
|
||||
JSON.stringify({ text: `bench ${exec.vu.idInTest}-${n}` }),
|
||||
{ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${c.token}`,
|
||||
'Idempotency-Key': `${exec.scenario.name}-${exec.vu.idInTest}-${n}-${Date.now()}` },
|
||||
tags: { name: 'post_message' }, timeout: '10s' });
|
||||
if (check(res, { '201': (r) => r.status === 201 })) {
|
||||
ok.add(1);
|
||||
okLatency.add(res.timings.duration);
|
||||
}
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
return { stdout: '\n@@SUMMARY@@' + JSON.stringify(data) + '\n' };
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Where does one API replica spend its CPU under the track 1 workload? (py-spy sidecar)
|
||||
|
||||
python bench/track1/profile.py --label baseline --rate 260 --config replicas=2,api_cpus=1.0
|
||||
|
||||
Starts a fresh bench stand, runs k6 at a fixed arrival rate and samples ONE replica with
|
||||
py-spy (sidecar sharing its PID namespace, SYS_PTRACE only for the sidecar, image
|
||||
bench/track1/pyspy with a hash-pinned py-spy). Output evidence/track1/profile-<label>.txt:
|
||||
share of samples per library ("where the time goes") and the top functions by self/total time.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import run as bench # noqa: E402
|
||||
|
||||
ROOT = bench.ROOT
|
||||
IMAGE = "mtusi-pyspy:lab3"
|
||||
|
||||
CATEGORIES = [ # first match wins; path fragments of the frames' files
|
||||
("psycopg (PostgreSQL driver + pool)", ("psycopg",)),
|
||||
("redis-py (Valkey client)", ("redis/",)),
|
||||
("uvicorn + httptools (HTTP server)", ("uvicorn", "httptools")),
|
||||
("starlette (routing, middleware, responses)", ("starlette",)),
|
||||
("logging (access log JSON lines)", ("logging/", "logging_setup.py")),
|
||||
("json (stdlib)", ("json/",)),
|
||||
("asyncio / uvloop (event loop)", ("asyncio", "uvloop")),
|
||||
("messenger (our code)", ("messenger/",)),
|
||||
]
|
||||
|
||||
|
||||
def category(stack: list[str]) -> str:
|
||||
"""The innermost frame that belongs to a known library decides the category."""
|
||||
for frame in reversed(stack):
|
||||
for name, needles in CATEGORIES:
|
||||
if any(n in frame for n in needles):
|
||||
return name
|
||||
return "other"
|
||||
|
||||
|
||||
def short(frame: str) -> str:
|
||||
name, _, where = frame.partition(" (")
|
||||
file = where.rstrip(")").split(":")[0].rsplit("/", 2)[-2:]
|
||||
return f"{name} ({'/'.join(file)})"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--label", required=True)
|
||||
parser.add_argument("--config", default="replicas=2,api_cpus=1.0")
|
||||
parser.add_argument("--rate", type=int, default=260)
|
||||
parser.add_argument("--seconds", type=int, default=20)
|
||||
parser.add_argument("--top", type=int, default=25)
|
||||
parser.add_argument("--git-ref", default="HEAD")
|
||||
args = parser.parse_args()
|
||||
subprocess.run(["docker", "build", "-q", "-t", IMAGE, str(ROOT / "bench" / "track1" / "pyspy")], check=True,
|
||||
capture_output=True)
|
||||
cfg = bench.parse_config(args.config)
|
||||
sha, bench.IMAGE["tag"] = bench.build_image(args.git_ref)
|
||||
bench.start_stand(cfg)
|
||||
work = tempfile.mkdtemp(prefix="track1-profile-")
|
||||
try:
|
||||
Path(work, "chats.json").write_text(__import__("json").dumps(bench.make_chats(64)), encoding="utf-8")
|
||||
warmup, duration = 8, args.seconds + 8
|
||||
k6 = subprocess.Popen(["docker", "compose", "-p", bench.PROJECT, *bench.FILES, "run", "--rm", "--no-deps",
|
||||
"-v", f"{work}:/data:ro", "-e", f"RATE={args.rate}", "-e", f"WARMUP={warmup}s",
|
||||
"-e", f"DURATION={duration}s", "-e", f"MAX_VUS={args.rate * 2}", "k6", "run", "--quiet",
|
||||
"/scripts/messages.js"], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
||||
text=True, encoding="utf-8")
|
||||
time.sleep(warmup + 4)
|
||||
target = bench.compose("ps", "-q", "api").split()[0]
|
||||
subprocess.run(["docker", "run", "--rm", "--pid", f"container:{target}", "--cap-add", "SYS_PTRACE",
|
||||
"-v", f"{work}:/out", IMAGE, "record", "--pid", "1", "--duration", str(args.seconds),
|
||||
"--rate", "250", "--format", "raw", "--nonblocking", "--output", "/out/stacks.txt"],
|
||||
check=True, capture_output=True, timeout=args.seconds + 120)
|
||||
out, _ = k6.communicate(timeout=300)
|
||||
data = __import__("json").loads(out[out.index(bench.MARK) + len(bench.MARK):].strip().splitlines()[0])
|
||||
ok_rps = (bench.metric(data, "ok_messages{phase:measure}", "count") or 0) / duration
|
||||
stacks = []
|
||||
for line in Path(work, "stacks.txt").read_text(encoding="utf-8").splitlines():
|
||||
stack, _, count = line.rpartition(" ")
|
||||
frames = [f for f in stack.split(";") if f and not f.startswith("process ") and not f.startswith("thread ")]
|
||||
if frames:
|
||||
stacks.append((frames, int(count)))
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
bench.compose("down", "-v", "--remove-orphans", check=False, timeout=300)
|
||||
total = sum(c for _, c in stacks)
|
||||
idle = sum(c for f, c in stacks if any("select" in x and ("selectors" in x or "uvloop" in x) for x in f[-2:]))
|
||||
cats = collections.Counter()
|
||||
self_time, total_time = collections.Counter(), collections.Counter()
|
||||
for frames, count in stacks:
|
||||
cats[category(frames)] += count
|
||||
self_time[short(frames[-1])] += count
|
||||
for name in {short(f) for f in frames}:
|
||||
total_time[name] += count
|
||||
lines = [f"# py-spy, {args.seconds} s at 250 Hz, ONE replica of {bench.config_name(cfg)}, k6 {args.rate} req/s "
|
||||
f"(ok {ok_rps:.1f} req/s for the whole stand)", f"# label {args.label}, commit {sha}; samples {total} "
|
||||
f"(idle in the event loop poll: {100 * idle / max(total, 1):.1f} %)", "",
|
||||
"## Share of samples by library (innermost known frame)"]
|
||||
for name, count in cats.most_common():
|
||||
lines.append(f"{100 * count / total:6.1f} % {name}")
|
||||
lines += ["", f"## Top {args.top} functions by SELF time (on top of the stack)"]
|
||||
for name, count in self_time.most_common(args.top):
|
||||
lines.append(f"{100 * count / total:6.1f} % {name}")
|
||||
lines += ["", f"## Top {args.top} functions by TOTAL time (anywhere in the stack)"]
|
||||
for name, count in total_time.most_common(args.top):
|
||||
lines.append(f"{100 * count / total:6.1f} % {name}")
|
||||
dest = ROOT / "evidence" / "track1" / f"profile-{args.label}.txt"
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print("\n".join(lines[:40]))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
# Profiler sidecar: shares the PID namespace of ONE api container (--pid=container:<id>) and
|
||||
# gets SYS_PTRACE only for itself; the API image stays unprivileged and unchanged.
|
||||
FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
|
||||
COPY requirements.txt /tmp/requirements.txt
|
||||
RUN pip install --no-cache-dir --root-user-action=ignore --require-hashes --no-deps -r /tmp/requirements.txt
|
||||
ENTRYPOINT ["py-spy"]
|
||||
@@ -0,0 +1,3 @@
|
||||
# py-spy sampling profiler for the track 1 sidecar (bench/track1/profile.py), pinned by hash.
|
||||
py-spy==0.4.2 \
|
||||
--hash=sha256:aeb0323409199c785f730645e9f4bb7a7b9ca2c481f2c331a55642b5d13fa52f
|
||||
@@ -0,0 +1,371 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1 orchestrator: cost of scaling. For every configuration (replicas x CPU per replica x
|
||||
PostgreSQL CPU x pool size x env) and every repeat a FRESH bench stand is started (separate
|
||||
compose project, empty volumes), synthetic users/chats are created, then k6 runs an open-loop
|
||||
arrival-rate sweep. During each step the orchestrator samples CPU of every container
|
||||
(docker stats), DB pool waits (API stats log), PostgreSQL counters (pg_stat_database,
|
||||
pg_stat_statements) and HAProxy counters (show stat).
|
||||
|
||||
python bench/track1/run.py --series fixed-total --label baseline \
|
||||
--config replicas=1,api_cpus=2.0 --config replicas=2,api_cpus=1.0 --config replicas=4,api_cpus=0.5 \
|
||||
--rates 200,400,600,800 --repeats 3
|
||||
|
||||
Output: evidence/track1/<series>/<label>/raw/*.json (one per step, no tokens) and
|
||||
results.csv (one row per step). The main stand (project mtusi-lab3) is not touched,
|
||||
but stop it during experiments so it does not compete for CPU.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import csv
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
PROJECT = "mtusi-lab3-bench"
|
||||
PORT = 18080
|
||||
FILES = ["-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "bench" / "track1" / "compose.bench.yaml")]
|
||||
MARK = "@@SUMMARY@@"
|
||||
|
||||
DEFAULTS = {"replicas": 2, "api_cpus": 1.0, "api_memory": "384m", "pg_cpus": 2.0, "pool": 10}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ helpers
|
||||
def compose(*args: str, env: dict | None = None, check: bool = True, timeout: float = 600) -> str:
|
||||
result = subprocess.run(["docker", "compose", "-p", PROJECT, *FILES, *args], cwd=ROOT, capture_output=True,
|
||||
text=True, encoding="utf-8", env={**os.environ, **(env or {})}, timeout=timeout)
|
||||
if check and result.returncode != 0:
|
||||
raise RuntimeError(f"compose {' '.join(args[:3])} failed: {result.stderr.strip()[-1500:]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t",
|
||||
"-v", "ON_ERROR_STOP=1", "-c", sql).strip()
|
||||
|
||||
|
||||
def http(method: str, path: str, body=None, headers=None):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
h = {"Content-Type": "application/json", **(headers or {})}
|
||||
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, json.loads(r.read() or b"null")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read() or b"null")
|
||||
|
||||
|
||||
def parse_config(text: str) -> dict:
|
||||
cfg = dict(DEFAULTS)
|
||||
cfg["env"] = {}
|
||||
for part in filter(None, text.split(",")):
|
||||
key, value = part.split("=", 1)
|
||||
if key in ("replicas", "pool"):
|
||||
cfg[key] = int(value)
|
||||
elif key in ("api_cpus", "pg_cpus"):
|
||||
cfg[key] = float(value)
|
||||
elif key == "api_memory":
|
||||
cfg[key] = value
|
||||
else:
|
||||
cfg["env"][key] = value # any API/compose env, e.g. MESSAGE_WRITE_PATH=function
|
||||
return cfg
|
||||
|
||||
|
||||
def config_name(cfg: dict) -> str:
|
||||
extra = "".join(f"-{k.lower()}_{v}" for k, v in sorted(cfg["env"].items()))
|
||||
return f"n{cfg['replicas']}-api{cfg['api_cpus']:g}-pg{cfg['pg_cpus']:g}-pool{cfg['pool']}{extra}"
|
||||
|
||||
|
||||
def build_image(ref: str) -> tuple[str, str]:
|
||||
"""Image of the API built from a COMMIT (git archive), not from the working tree, so a
|
||||
series is reproducible and edits made while it runs cannot leak into it."""
|
||||
sha = subprocess.run(["git", "rev-parse", "--short", ref], cwd=ROOT, capture_output=True, text=True,
|
||||
check=True).stdout.strip()
|
||||
tag = f"mtusi-messenger:bench-{sha}"
|
||||
archive = subprocess.run(["git", "archive", "--format=tar", sha], cwd=ROOT, capture_output=True, check=True).stdout
|
||||
subprocess.run(["docker", "build", "-q", "--build-arg", f"BUILD_REVISION={sha}", "-t", tag, "-"], input=archive,
|
||||
capture_output=True, check=True, timeout=900)
|
||||
return sha, tag
|
||||
|
||||
|
||||
IMAGE = {"tag": "mtusi-messenger:lab3"}
|
||||
|
||||
|
||||
def stand_env(cfg: dict) -> dict:
|
||||
return {"API_IMAGE": IMAGE["tag"], "API_REPLICAS": str(cfg["replicas"]), "API_CPUS": f"{cfg['api_cpus']:g}", "API_MEMORY": cfg["api_memory"],
|
||||
"PG_CPUS": f"{cfg['pg_cpus']:g}", "DB_POOL_MIN": str(min(cfg["pool"], 4)), "DB_POOL_MAX": str(cfg["pool"]),
|
||||
"APP_PUBLISH_PORT": str(PORT), "LB_STATS_PORT": "18404", "BUILD_REVISION": "bench",
|
||||
**cfg["env"]}
|
||||
|
||||
|
||||
def start_stand(cfg: dict) -> None:
|
||||
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
|
||||
compose("up", "-d", "--no-build", "--wait", env=stand_env(cfg), timeout=900)
|
||||
psql("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline: # every replica UP in HAProxy before measuring
|
||||
if lb_stat()["up"] == cfg["replicas"]:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError("not all replicas UP in HAProxy")
|
||||
|
||||
|
||||
def make_chats(users: int) -> list[dict]:
|
||||
created = []
|
||||
stamp = uuid.uuid4().hex[:8]
|
||||
for i in range(users):
|
||||
username, password = f"b_{stamp}_{i}", f"Bench-password-{stamp}-{i}"
|
||||
status, user = http("POST", "/api/v1/users", {"username": username, "display_name": f"bench {i}", "password": password})
|
||||
assert status == 201, (status, user)
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201, (status, session)
|
||||
created.append({"id": user["id"], "token": session["token"]})
|
||||
chats = []
|
||||
for i, me in enumerate(created):
|
||||
other = created[(i + 1) % len(created)]
|
||||
status, chat = http("POST", "/api/v1/chats", {"title": f"bench {i}", "member_ids": [me["id"], other["id"]]},
|
||||
headers={"Authorization": "Bearer " + me["token"]})
|
||||
assert status == 201, (status, chat)
|
||||
chats.append({"token": me["token"], "chat": chat["id"]})
|
||||
return chats
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
"""docker stats every ~1-2 s: CPU% (100 = one core) and memory per container."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(daemon=True)
|
||||
self.samples: list[tuple[float, dict]] = []
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
def run(self) -> None:
|
||||
while not self.stop_event.is_set():
|
||||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{json .}}"],
|
||||
capture_output=True, text=True).stdout
|
||||
now = time.monotonic()
|
||||
row = {}
|
||||
for line in out.splitlines():
|
||||
item = json.loads(line)
|
||||
if not item["Name"].startswith(PROJECT + "-"):
|
||||
continue # only this bench project (k6 run containers included)
|
||||
short = item["Name"][len(PROJECT) + 1:]
|
||||
if short.startswith("k6-run-"):
|
||||
short = "k6"
|
||||
row[short] = {"cpu": float(item["CPUPerc"].rstrip("%") or 0), "mem": item["MemUsage"].split(" / ")[0]}
|
||||
self.samples.append((now, row))
|
||||
|
||||
def window(self, start: float, end: float) -> dict:
|
||||
rows = [r for t, r in self.samples if start <= t <= end]
|
||||
result = {}
|
||||
for name in sorted({n for r in rows for n in r}):
|
||||
values = [r[name]["cpu"] for r in rows if name in r]
|
||||
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
|
||||
"mem_last": [r[name]["mem"] for r in rows if name in r][-1], "samples": len(values)}
|
||||
return result
|
||||
|
||||
|
||||
def lb_stat() -> dict:
|
||||
raw = compose("exec", "-T", "lb", "sh", "-c", 'echo "show stat" | socat stdio /var/lib/haproxy/admin.sock')
|
||||
lines = [line for line in raw.splitlines() if line]
|
||||
header = lines[0].lstrip("# ").split(",")
|
||||
servers = [dict(zip(header, line.split(","))) for line in lines[1:]]
|
||||
api = [s for s in servers if s["pxname"] == "api" and s["svname"].startswith("api") and s["status"] != "MAINT"]
|
||||
backend = next(s for s in servers if s["pxname"] == "api" and s["svname"] == "BACKEND")
|
||||
return {"up": sum(1 for s in api if s["status"].startswith("UP")),
|
||||
"chkdown": sum(int(s["chkdown"] or 0) for s in api), "qmax": int(backend["qmax"] or 0),
|
||||
"smax": int(backend["smax"] or 0), "econ": int(backend["econ"] or 0), "eresp": int(backend["eresp"] or 0),
|
||||
"wredis": int(backend["wredis"] or 0), "wretr": int(backend["wretr"] or 0)}
|
||||
|
||||
|
||||
def pg_counters() -> dict:
|
||||
row = psql("SELECT xact_commit, xact_rollback, tup_inserted, blks_read, blks_hit FROM pg_stat_database "
|
||||
"WHERE datname = 'messenger'").split("\t")
|
||||
wal = psql("SELECT wal_records, wal_bytes FROM pg_stat_wal").split("\t")
|
||||
return {"xact_commit": int(row[0]), "xact_rollback": int(row[1]), "tup_inserted": int(row[2]),
|
||||
"blks_read": int(row[3]), "blks_hit": int(row[4]), "wal_records": int(wal[0]), "wal_bytes": int(float(wal[1]))}
|
||||
|
||||
|
||||
def top_statements(limit: int = 12) -> list[dict]:
|
||||
out = psql("SELECT calls, round(total_exec_time::numeric, 1), round(mean_exec_time::numeric, 4), "
|
||||
"left(regexp_replace(query, '\\s+', ' ', 'g'), 110) FROM pg_stat_statements "
|
||||
"WHERE dbid = (SELECT oid FROM pg_database WHERE datname = 'messenger') "
|
||||
f"ORDER BY total_exec_time DESC LIMIT {limit}")
|
||||
rows = []
|
||||
for line in out.splitlines():
|
||||
calls, total, mean, query = line.split("\t", 3)
|
||||
rows.append({"calls": int(calls), "total_ms": float(total), "mean_ms": float(mean), "query": query})
|
||||
return rows
|
||||
|
||||
|
||||
def pool_window(since_iso: str, start_wall: float, end_wall: float) -> dict:
|
||||
logs = compose("logs", "--no-log-prefix", "--since", since_iso, "api")
|
||||
waits, nums, waiting, cpu, usage, intervals = 0, 0, 0, 0.0, 0, 0
|
||||
for line in logs.splitlines():
|
||||
if '"pool_stats"' not in line:
|
||||
continue
|
||||
item = json.loads(line)
|
||||
ts = dt.datetime.fromisoformat(item["ts"].replace("Z", "+00:00")).timestamp()
|
||||
if not start_wall <= ts <= end_wall:
|
||||
continue
|
||||
intervals += 1
|
||||
waits += item["requests_wait_ms"]
|
||||
nums += item["requests_num"]
|
||||
waiting = max(waiting, item["requests_waiting"])
|
||||
cpu += item["cpu_s"]
|
||||
usage += item["usage_ms"]
|
||||
return {"pool_requests": nums, "pool_wait_ms_avg": round(waits / nums, 3) if nums else None,
|
||||
"pool_waiting_max": waiting, "pool_usage_ms_avg": round(usage / nums, 3) if nums else None,
|
||||
"api_cpu_s": round(cpu, 2), "intervals": intervals}
|
||||
|
||||
|
||||
def metric(data: dict, name: str, field: str):
|
||||
m = data["metrics"].get(name)
|
||||
return None if m is None else m["values"].get(field)
|
||||
|
||||
|
||||
def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> dict:
|
||||
psql("SELECT pg_stat_statements_reset()")
|
||||
pg0, lb0 = pg_counters(), lb_stat()
|
||||
since = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
started = time.monotonic()
|
||||
wall0 = time.time()
|
||||
max_vus = max(50, min(args.max_vus, rate * 2))
|
||||
out = compose("run", "--rm", "--no-deps", "-v", f"{chats_dir}:/data:ro", "-e", f"RATE={rate}",
|
||||
"-e", f"WARMUP={args.warmup}s", "-e", f"DURATION={args.duration}s", "-e", f"MAX_VUS={max_vus}",
|
||||
"k6", "run", "--quiet", "/scripts/messages.js", timeout=args.warmup + args.duration + 180)
|
||||
ended = time.monotonic()
|
||||
data = json.loads(out[out.index(MARK) + len(MARK):].strip().splitlines()[0])
|
||||
# k6 starts ~1-3 s after the command; the measure phase is the last `duration` seconds before exit.
|
||||
m_end = ended - 1.0
|
||||
m_start = max(started, m_end - args.duration)
|
||||
pg1, lb1 = pg_counters(), lb_stat()
|
||||
ok = metric(data, "ok_messages{phase:measure}", "count") or 0
|
||||
reqs = metric(data, "http_reqs{phase:measure}", "count") or 0
|
||||
dropped = metric(data, "dropped_iterations{scenario:measure}", "count") or 0
|
||||
result = {
|
||||
"config": config_name(cfg), "rate": rate, "duration_s": args.duration,
|
||||
"offered": rate * args.duration, "requests": int(reqs), "ok": int(ok), "dropped": int(dropped),
|
||||
"ok_rps": round(ok / args.duration, 1),
|
||||
"error_ratio": round(metric(data, "http_req_failed{phase:measure}", "rate") or 0, 5),
|
||||
"ok_p50_ms": metric(data, "ok_latency{phase:measure}", "p(50)"),
|
||||
"ok_p95_ms": metric(data, "ok_latency{phase:measure}", "p(95)"),
|
||||
"ok_p99_ms": metric(data, "ok_latency{phase:measure}", "p(99)"),
|
||||
"vus_max": metric(data, "vus_max", "value"),
|
||||
"cpu": sampler.window(m_start, m_end),
|
||||
"pool": pool_window(since, wall0 + (m_start - started), wall0 + (m_end - started)),
|
||||
"pg": {k: pg1[k] - pg0[k] for k in pg1},
|
||||
"lb": {"chkdown_delta": lb1["chkdown"] - lb0["chkdown"], "qmax": lb1["qmax"], "up_after": lb1["up"],
|
||||
"econ_delta": lb1["econ"] - lb0["econ"], "eresp_delta": lb1["eresp"] - lb0["eresp"],
|
||||
"redispatch_delta": lb1["wredis"] - lb0["wredis"]},
|
||||
"statements": top_statements(),
|
||||
}
|
||||
commits = result["pg"]["xact_commit"]
|
||||
result["pg_commits_per_ok"] = round(commits / ok, 2) if ok else None
|
||||
result["statements_per_ok"] = round(sum(s["calls"] for s in result["statements"]) / ok, 2) if ok else None
|
||||
return result
|
||||
|
||||
|
||||
def flat(r: dict) -> dict:
|
||||
cpu = r["cpu"]
|
||||
api_cpu = sum(v["cpu_median"] for k, v in cpu.items() if k.startswith("api-"))
|
||||
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
|
||||
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
|
||||
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
|
||||
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
|
||||
"lb_cpu_pct": cpu.get("lb-1", {}).get("cpu_median"), "valkey_cpu_pct": cpu.get("valkey-1", {}).get("cpu_median"),
|
||||
"k6_cpu_pct": cpu.get("k6", {}).get("cpu_median"),
|
||||
"pool_wait_ms_avg": r["pool"]["pool_wait_ms_avg"], "pool_waiting_max": r["pool"]["pool_waiting_max"],
|
||||
"pg_commits_per_ok": r["pg_commits_per_ok"], "statements_per_ok": r["statements_per_ok"],
|
||||
"lb_chkdown": r["lb"]["chkdown_delta"], "lb_qmax": r["lb"]["qmax"], "wal_bytes": r["pg"]["wal_bytes"]}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--series", required=True)
|
||||
parser.add_argument("--label", default="baseline")
|
||||
parser.add_argument("--config", action="append", required=True, help="replicas=2,api_cpus=1.0,pg_cpus=2.0,pool=10,ENV=val")
|
||||
parser.add_argument("--rates", required=True, help="comma separated arrival rates, req/s")
|
||||
parser.add_argument("--repeats", type=int, default=3)
|
||||
parser.add_argument("--warmup", type=int, default=10)
|
||||
parser.add_argument("--duration", type=int, default=20)
|
||||
parser.add_argument("--users", type=int, default=64)
|
||||
parser.add_argument("--max-vus", type=int, default=600)
|
||||
parser.add_argument("--git-ref", default="HEAD", help="commit whose code is measured (image built via git archive)")
|
||||
args = parser.parse_args()
|
||||
sha, IMAGE["tag"] = build_image(args.git_ref)
|
||||
out = ROOT / "evidence" / "track1" / args.series / args.label
|
||||
(out / "raw").mkdir(parents=True, exist_ok=True)
|
||||
rates = [int(r) for r in args.rates.split(",")]
|
||||
configs = [parse_config(c) for c in args.config]
|
||||
meta = {"started": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
||||
"measured_commit": sha, "image": IMAGE["tag"],
|
||||
"bench_tooling_dirty": bool(subprocess.run(["git", "status", "--porcelain", "--", "bench", "compose.yaml",
|
||||
"infra"], cwd=ROOT, capture_output=True, text=True).stdout.strip()),
|
||||
"host": {"os": f"{platform.system()} {platform.release()}",
|
||||
"docker": subprocess.run(["docker", "info", "--format", "{{.ServerVersion}} {{.NCPU}} CPU "
|
||||
"{{.MemTotal}} B {{.KernelVersion}}"], capture_output=True,
|
||||
text=True).stdout.strip()},
|
||||
"cpusets": "k6 0-1, lb+valkey 2, postgres 3-4, api 5-7 (bench/track1/compose.bench.yaml)",
|
||||
"workload": f"k6 constant-arrival-rate POST message, new Idempotency-Key, {args.users} chats, "
|
||||
f"warmup {args.warmup}s + measure {args.duration}s per step, fresh stand per config x repeat",
|
||||
"rates": rates, "repeats": args.repeats, "configs": configs, "args": sys_argv()}
|
||||
(out / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
csv_path = out / "results.csv"
|
||||
new_file = not csv_path.exists()
|
||||
with csv_path.open("a", newline="", encoding="utf-8") as f:
|
||||
writer = None
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
for cfg in configs:
|
||||
name = config_name(cfg)
|
||||
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
|
||||
start_stand(cfg)
|
||||
chats_dir = tempfile.mkdtemp(prefix="track1-")
|
||||
try:
|
||||
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
|
||||
sampler = Sampler()
|
||||
sampler.start()
|
||||
for rate in rates:
|
||||
result = run_step(cfg, rate, args, chats_dir, sampler)
|
||||
result["repeat"] = repeat
|
||||
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
row = flat(result)
|
||||
if writer is None:
|
||||
writer = csv.DictWriter(f, fieldnames=list(row))
|
||||
if new_file:
|
||||
writer.writeheader()
|
||||
writer.writerow(row)
|
||||
f.flush()
|
||||
print(f" rate {rate}: ok_rps {row['ok_rps']} p95 {row['p95_ms'] and round(row['p95_ms'])} ms "
|
||||
f"err {row['error_ratio']} dropped {row['dropped']} | api {row['api_cpu_pct_total']}% "
|
||||
f"pg {row['pg_cpu_pct']}% lb {row['lb_cpu_pct']}% k6 {row['k6_cpu_pct']}% "
|
||||
f"pool_wait {row['pool_wait_ms_avg']} ms chkdown {row['lb_chkdown']}", flush=True)
|
||||
sampler.stop_event.set()
|
||||
finally:
|
||||
shutil.rmtree(chats_dir, ignore_errors=True) # synthetic tokens never stay on disk
|
||||
compose("down", "-v", "--remove-orphans", check=False, timeout=300)
|
||||
print(f"results: {csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
def sys_argv() -> list[str]:
|
||||
import sys
|
||||
return sys.argv[1:]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+2
-2
@@ -84,7 +84,7 @@ services:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: mtusi-messenger:lab3
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab3}
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
@@ -104,7 +104,7 @@ services:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: mtusi-messenger:lab3
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab3}
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url]
|
||||
# No `ports` and no container_name: replicas are reachable only through `lb`,
|
||||
|
||||
+21
-2
@@ -338,6 +338,22 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
except ApiError:
|
||||
pass # database unavailable right now; next round
|
||||
|
||||
stats_log = logging.getLogger("messenger.stats")
|
||||
|
||||
async def stats_loop(every: float) -> None:
|
||||
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
|
||||
cpu = time.process_time()
|
||||
db.pool.pop_stats()
|
||||
while True:
|
||||
await asyncio.sleep(every)
|
||||
stats = db.pool.pop_stats()
|
||||
now_cpu = time.process_time()
|
||||
stats_log.info("pool_stats", extra={"fields": {
|
||||
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
|
||||
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
|
||||
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
|
||||
cpu = now_cpu
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def lifespan(app: Starlette):
|
||||
passwords.configure(settings.hash_concurrency)
|
||||
@@ -346,11 +362,14 @@ def create_app(settings: Settings | None = None) -> Starlette:
|
||||
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
|
||||
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
|
||||
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds}})
|
||||
purge = asyncio.create_task(purge_loop())
|
||||
tasks = [asyncio.create_task(purge_loop())]
|
||||
if settings.stats_log_seconds > 0:
|
||||
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
purge.cancel()
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await sessions.close()
|
||||
await db.close()
|
||||
|
||||
|
||||
@@ -106,6 +106,7 @@ class Settings:
|
||||
instance_id: str = "local"
|
||||
drain_seconds: float = 0.0
|
||||
idempotency_ttl_seconds: int = 86400
|
||||
stats_log_seconds: float = 0.0
|
||||
|
||||
@property
|
||||
def db_statement_timeout_ms(self) -> int:
|
||||
@@ -151,4 +152,6 @@ class Settings:
|
||||
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
|
||||
# Contract: a replay within 24 h returns the original result.
|
||||
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
|
||||
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
|
||||
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user