Files

320 lines
17 KiB
Python

#!/usr/bin/env python3
"""Трек 1 «Цена масштабирования» (stdlib, Python 3.9+). Оркестратор серий нагрузки.
python3 research/track1.py run <series>... # fixed-cpu | growth | machine | admission | all
python3 research/track1.py report # сводка + графики из evidence/track1/*.json
Для каждой конфигурации: пересоздать реплики с заданными CPU/RAM/числом (compose.yaml +
research/compose.track1.yaml), дождаться, пока LB видит все реплики UP, запустить генератор
(research/loadgen, open-loop, ступени rate) внутри сети стенда и параллельно собирать:
CPU%/RAM контейнеров (docker stats), очередь/сессии HAProxy (stats CSV), состояния соединений
PostgreSQL (pg_stat_activity), счётчики пула psycopg в каждой реплике (до/после серии).
Результат — evidence/track1/<config>.json. Токенов и паролей в выводе нет.
"""
import argparse
import csv
import io
import json
import os
import re
import shlex
import statistics
import subprocess
import sys
import threading
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "track1"
BASE_FILES = os.environ.get("COMPOSE_FILE", "compose.yaml:research/compose.track1.yaml")
DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env"))
STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
SLO_P99_MS, SLO_ERR = 250.0, 0.01
# Ступени offered rate (req/s). Генератор останавливает серию после 2 ступеней с >30 % ошибок.
RATES = "40,80,120,160,200,240,280,320,360,400,460,520,600,700,800"
CONFIGS = {
# Серия A: фиксированный суммарный бюджет API = 1 ядро и 512 MiB (KDF-параллелизм 4) делим на 1/2/4.
"fixed-1x1.00": dict(series="fixed-cpu", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4),
"fixed-2x0.50": dict(series="fixed-cpu", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M", KDF_CONCURRENCY=2),
"fixed-4x0.25": dict(series="fixed-cpu", API_REPLICAS=4, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
# Серия B: рост ресурсов — добавляем реплики по 0.25 ядра/128 MiB (4x0.25 = fixed-4x0.25).
"growth-1x0.25": dict(series="growth", API_REPLICAS=1, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
"growth-2x0.25": dict(series="growth", API_REPLICAS=2, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
"growth-3x0.25": dict(series="growth", API_REPLICAS=3, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
# Серия C: снять изоляцию ядер — API делит всю машину (2 vCPU) с LB/БД/генератором.
"machine-2x1.00": dict(series="machine", API_REPLICAS=2, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2,
API_CPUSET="0-1", AUX_CPUSET="0-1"),
"machine-4x1.00": dict(series="machine", API_REPLICAS=4, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2,
API_CPUSET="0-1", AUX_CPUSET="0-1"),
# Серия P: размер пула соединений при 1x1.00 (p10 = fixed-1x1.00).
"pool-1x1.00-p5": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=5),
"pool-1x1.00-p20": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=20),
"pool-1x1.00-p40": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=40),
# Серия D: улучшение — ограничение конкуренции на LB (maxconn на реплику; mc0 = fixed-1x1.00).
"admission-1x1.00-mc10": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10),
"admission-1x1.00-mc20": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=20),
"admission-1x1.00-mc10-q250": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10, LB_QUEUE_TIMEOUT="250ms"),
"admission-2x0.50-mc10": dict(series="admission", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M",
KDF_CONCURRENCY=2, LB_SERVER_MAXCONN=10),
}
DEFAULTS = dict(API_CPUSET="0", AUX_CPUSET="1", DB_POOL_MAX="10", SESSION_CACHE="redis",
LB_SERVER_MAXCONN="0", LB_QUEUE_TIMEOUT="2s")
def run(args, env=None, check=True, capture=True, timeout=900):
return subprocess.run(args, cwd=ROOT, env=env, check=check, timeout=timeout,
stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE, text=True)
def cfg_env(cfg):
env = dict(os.environ)
env.setdefault("COMPOSE_FILE", BASE_FILES)
for k, v in {**DEFAULTS, **cfg}.items():
if k.isupper():
env[k] = str(v)
return env
def lb_rows():
with urllib.request.urlopen(STATS_URL, timeout=2) as r:
text = r.read().decode().lstrip("# ")
return list(csv.DictReader(io.StringIO(text)))
def wait_lb(n, timeout=90):
t = time.monotonic()
while time.monotonic() - t < timeout:
try:
up = [r for r in lb_rows() if r["pxname"] == "api" and r["svname"].startswith("api")
and not r["svname"].startswith("apibk") and r["status"] == "UP"]
if len(up) == n:
return
except OSError:
pass
time.sleep(1)
raise SystemExit("LB does not see %d UP replicas" % n)
def pool_stats(env):
res = {}
for cid in run(DC + ["ps", "-q", "api"], env).stdout.split():
out = run(["docker", "exec", cid, "python", "-m", "messenger", "stats"], check=False).stdout
try:
d = json.loads(out)
res[d["instance_id"]] = {"pool": d["pool"], "cache": d["cache"], "requests": d["app"]["requests"]}
except (ValueError, KeyError):
pass
return res
class Sampler:
"""docker stats (CPU%, RAM), HAProxy CSV и pg_stat_activity раз в ~1 с с отметками времени."""
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
def __init__(self, env):
self.env, self.stop = env, threading.Event()
self.docker, self.lb, self.pg, self.throttle = [], [], [], []
def _docker(self):
p = subprocess.Popen(["docker", "stats", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, cwd=ROOT)
try:
for line in p.stdout:
if self.stop.is_set():
break
line = self.ANSI.sub("", line).strip()
parts = line.split("\t")
if len(parts) == 3 and parts[1].endswith("%") and parts[0].startswith("mtusi-lab3-"):
mem = parts[2].split("/")[0].strip()
mib = float(re.sub(r"[^0-9.]", "", mem) or 0) * (1024 if "GiB" in mem else 1 / 1024 if "KiB" in mem else 1)
self.docker.append((time.time(), parts[0].replace("mtusi-lab3-", ""), float(parts[1][:-1]), mib))
finally:
p.kill()
@staticmethod
def _throttle(cid):
for path in ("/sys/fs/cgroup/cpu/docker/%s/cpu.stat" % cid, "/sys/fs/cgroup/system.slice/docker-%s.scope/cpu.stat" % cid):
try:
d = dict(line.split() for line in open(path))
return int(d.get("nr_periods", 0)), int(d.get("nr_throttled", 0))
except OSError:
continue
return None
def _poll(self):
ids = [c for c in run(DC + ["ps", "-q", "--no-trunc", "api"], self.env, check=False).stdout.split()]
while not self.stop.is_set():
t = time.time()
thr = [self._throttle(c) for c in ids]
if all(thr) and thr:
self.throttle.append((t, sum(x[0] for x in thr), sum(x[1] for x in thr)))
try:
rows = lb_rows()
api = [r for r in rows if r["pxname"] == "api" and r["svname"] not in ("FRONTEND",)]
be = next(r for r in api if r["svname"] == "BACKEND")
self.lb.append((t, int(be["qcur"] or 0), int(be["scur"] or 0),
sum(int(r["qcur"] or 0) for r in api if r["svname"].startswith("api"))))
except (OSError, StopIteration, ValueError):
pass
self.stop.wait(1.0)
PG_SQL = ("SELECT 'S', extract(epoch FROM now()), coalesce(state, '?'), count(*) FROM pg_stat_activity"
" WHERE datname = 'messenger' AND backend_type = 'client backend' AND pid <> pg_backend_pid() GROUP BY 3"
" UNION ALL SELECT 'W', extract(epoch FROM now()), coalesce(wait_event_type || ':' || wait_event, 'CPU'), count(*)"
" FROM pg_stat_activity WHERE datname = 'messenger' AND state = 'active' AND pid <> pg_backend_pid() GROUP BY 3")
def _pg(self):
"""ОДНА долгоживущая сессия psql с \\watch 1 (не docker exec каждую секунду — он сам грузит хост)."""
p = subprocess.Popen(DC + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
cwd=ROOT, env=self.env)
p.stdin.write(self.PG_SQL + " \\watch 1\n"); p.stdin.flush()
snap, cur_ts = {}, None
try:
for line in p.stdout:
if self.stop.is_set():
break
parts = line.strip().split("|")
if len(parts) != 4:
continue
kind, ts, key, n = parts
ts = float(ts)
if cur_ts is not None and ts != cur_ts:
self.pg.append((cur_ts, snap)); snap = {}
cur_ts = ts
snap[("state:" if kind == "S" else "wait:") + key] = int(n)
finally:
p.kill()
def __enter__(self):
self.threads = [threading.Thread(target=self._docker, daemon=True), threading.Thread(target=self._poll, daemon=True),
threading.Thread(target=self._pg, daemon=True)]
[t.start() for t in self.threads]
return self
def __exit__(self, *exc):
self.stop.set()
def window(self, a, b):
cpu, mem = {}, {}
for t, name, c, m in self.docker:
if a <= t <= b:
key = "api" if re.fullmatch(r"api-\d+", name) else re.sub(r"-\d+$", "", name)
cpu.setdefault((key, name), []).append(c)
mem.setdefault(key, []).append(m)
agg = {}
for (key, name), vals in cpu.items():
agg.setdefault(key, 0.0)
agg[key] += statistics.mean(vals)
lb = [x for x in self.lb if a <= x[0] <= b]
th = [x for x in self.throttle if a <= x[0] <= b]
throttled = (round((th[-1][2] - th[0][2]) / max(1, th[-1][1] - th[0][1]), 3) if len(th) >= 2 else None)
pg = [x[1] for x in self.pg if a <= x[0] <= b]
return {
"cpu_percent_mean": {k: round(v, 1) for k, v in sorted(agg.items())},
"mem_mib_max": {k: round(max(v), 1) for k, v in sorted(mem.items())},
"lb_queue_max": max((x[1] + x[3] for x in lb), default=None),
"lb_sessions_mean": round(statistics.mean(x[2] for x in lb), 1) if lb else None,
"pg_active_mean": round(statistics.mean(p.get("state:active", 0) for p in pg), 2) if pg else None,
"pg_wait_events_mean": ({k[5:]: round(sum(p.get(k, 0) for p in pg) / len(pg), 2)
for k in sorted({k for p in pg for k in p if k.startswith("wait:")})} if pg else None),
"api_cfs_throttled_period_ratio": throttled,
"pg_connections_mean": round(statistics.mean(sum(v for k, v in p.items() if k.startswith("state:")) for p in pg), 1) if pg else None,
}
def run_config(name, cfg, args, rep=1):
if rep > 1:
name = "%s-r%d" % (name, rep)
env = cfg_env(cfg)
print("== %s %s" % (name, {k: v for k, v in cfg.items() if k.isupper()}), flush=True)
run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], env, timeout=300)
wait_lb(int(cfg["API_REPLICAS"]))
time.sleep(3)
pools_before = pool_stats(env)
rates = cfg.get("RATES", args.rates)
cmd = DC + ["--profile", "bench", "run", "--rm", "-T", "loadgen", "-rates", rates,
"-duration", "%ds" % args.duration, "-warmup", "%ds" % args.warmup,
"-users", str(args.users), "-seed", str(args.seed), "-label", name]
steps, starts = [], {}
with Sampler(env) as sampler:
p = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
for line in p.stdout:
try:
ev = json.loads(line)
except ValueError:
continue
if ev["event"] == "step_start":
starts[ev["rate"]] = ev["ts"] + ev["warmup_s"]
elif ev["event"] == "step":
a = starts[ev["rate"]]
ev["resources"] = sampler.window(a, a + ev["duration_s"])
ev["slo_ok"] = ev["latency_ms"]["p99"] <= SLO_P99_MS and ev["error_ratio"] <= SLO_ERR
steps.append(ev)
print(" rate %5.0f ok %6.1f rps err %5.1f%% p50 %7.1f p99 %8.1f ms cpu %s pgActive %s lbQ %s" % (
ev["rate"], ev["throughput_ok_rps"], 100 * ev["error_ratio"], ev["latency_ms"]["p50"],
ev["latency_ms"]["p99"], ev["resources"]["cpu_percent_mean"], ev["resources"]["pg_active_mean"],
ev["resources"]["lb_queue_max"]), flush=True)
elif ev["event"] == "setup":
print(" setup %.1fs" % ev["seconds"], flush=True)
p.wait()
if p.returncode:
print(p.stderr.read()[-1500:])
pools_after = pool_stats(env)
ok = [s for s in steps if s["slo_ok"]]
result = {
"config": name, "repeat": rep, "series": cfg["series"], "params": {**DEFAULTS, **{k: v for k, v in cfg.items() if k.isupper()}},
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"slo": {"p99_ms": SLO_P99_MS, "error_ratio": SLO_ERR},
"capacity_at_slo_rps": max((s["throughput_ok_rps"] for s in ok), default=0),
"peak_ok_rps": max((s["throughput_ok_rps"] for s in steps), default=0),
"steps": steps, "pool_before": pools_before, "pool_after": pools_after,
"loadgen": {"users": args.users, "seed": args.seed, "duration_s": args.duration, "warmup_s": args.warmup,
"mix": "history=70,post=20,me=10", "model": "open-loop, latency from scheduled time"},
}
OUT.mkdir(parents=True, exist_ok=True)
(OUT / (name + ".json")).write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n")
print(" => capacity@SLO %.1f rps, peak %.1f rps" % (result["capacity_at_slo_rps"], result["peak_ok_rps"]), flush=True)
def report(args):
from track1_report import build # noqa: E402 (рядом в research/)
build(OUT)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("action", choices=["run", "report", "list"])
p.add_argument("targets", nargs="*", default=["all"])
p.add_argument("--rates", default=RATES)
p.add_argument("--duration", type=int, default=10)
p.add_argument("--warmup", type=int, default=3)
p.add_argument("--users", type=int, default=48)
p.add_argument("--seed", type=int, default=10)
p.add_argument("--rep", type=int, default=1, help="номер повтора (сохраняется как <config>-r<rep>.json)")
args = p.parse_args()
if args.action == "list":
for k, v in CONFIGS.items():
print(k, v)
return
if args.action == "report":
sys.path.insert(0, str(Path(__file__).parent))
return report(args)
names = [n for n, c in CONFIGS.items() if "all" in args.targets or c["series"] in args.targets or n in args.targets]
for n in names:
run_config(n, CONFIGS[n], args, args.rep)
# вернуть штатную конфигурацию стенда
run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], cfg_env({"API_REPLICAS": 2}), timeout=300)
if __name__ == "__main__":
main()