forked from mtusi2026/mtusi2026lab3
320 lines
17 KiB
Python
320 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Трек 1 «Цена масштабирования» (stdlib, Python 3.9+). Оркестратор серий нагрузки.
|
|
|
|
python3 research/track1.py run <series>... # fixed-cpu | growth | machine | admission | all
|
|
python3 research/track1.py report # сводка + графики из evidence/track1/*.json
|
|
|
|
Для каждой конфигурации: пересоздать реплики с заданными CPU/RAM/числом (compose.yaml +
|
|
research/compose.track1.yaml), дождаться, пока LB видит все реплики UP, запустить генератор
|
|
(research/loadgen, open-loop, ступени rate) внутри сети стенда и параллельно собирать:
|
|
CPU%/RAM контейнеров (docker stats), очередь/сессии HAProxy (stats CSV), состояния соединений
|
|
PostgreSQL (pg_stat_activity), счётчики пула psycopg в каждой реплике (до/после серии).
|
|
Результат — evidence/track1/<config>.json. Токенов и паролей в выводе нет.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / "evidence" / "track1"
|
|
BASE_FILES = os.environ.get("COMPOSE_FILE", "compose.yaml:research/compose.track1.yaml")
|
|
DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env"))
|
|
STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
|
|
SLO_P99_MS, SLO_ERR = 250.0, 0.01
|
|
|
|
# Ступени offered rate (req/s). Генератор останавливает серию после 2 ступеней с >30 % ошибок.
|
|
RATES = "40,80,120,160,200,240,280,320,360,400,460,520,600,700,800"
|
|
CONFIGS = {
|
|
# Серия A: фиксированный суммарный бюджет API = 1 ядро и 512 MiB (KDF-параллелизм 4) делим на 1/2/4.
|
|
"fixed-1x1.00": dict(series="fixed-cpu", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4),
|
|
"fixed-2x0.50": dict(series="fixed-cpu", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M", KDF_CONCURRENCY=2),
|
|
"fixed-4x0.25": dict(series="fixed-cpu", API_REPLICAS=4, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
|
|
# Серия B: рост ресурсов — добавляем реплики по 0.25 ядра/128 MiB (4x0.25 = fixed-4x0.25).
|
|
"growth-1x0.25": dict(series="growth", API_REPLICAS=1, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
|
|
"growth-2x0.25": dict(series="growth", API_REPLICAS=2, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
|
|
"growth-3x0.25": dict(series="growth", API_REPLICAS=3, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
|
|
# Серия C: снять изоляцию ядер — API делит всю машину (2 vCPU) с LB/БД/генератором.
|
|
"machine-2x1.00": dict(series="machine", API_REPLICAS=2, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2,
|
|
API_CPUSET="0-1", AUX_CPUSET="0-1"),
|
|
"machine-4x1.00": dict(series="machine", API_REPLICAS=4, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2,
|
|
API_CPUSET="0-1", AUX_CPUSET="0-1"),
|
|
# Серия P: размер пула соединений при 1x1.00 (p10 = fixed-1x1.00).
|
|
"pool-1x1.00-p5": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=5),
|
|
"pool-1x1.00-p20": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=20),
|
|
"pool-1x1.00-p40": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=40),
|
|
# Серия D: улучшение — ограничение конкуренции на LB (maxconn на реплику; mc0 = fixed-1x1.00).
|
|
"admission-1x1.00-mc10": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
|
|
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10),
|
|
"admission-1x1.00-mc20": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
|
|
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=20),
|
|
"admission-1x1.00-mc10-q250": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
|
|
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10, LB_QUEUE_TIMEOUT="250ms"),
|
|
"admission-2x0.50-mc10": dict(series="admission", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M",
|
|
KDF_CONCURRENCY=2, LB_SERVER_MAXCONN=10),
|
|
}
|
|
DEFAULTS = dict(API_CPUSET="0", AUX_CPUSET="1", DB_POOL_MAX="10", SESSION_CACHE="redis",
|
|
LB_SERVER_MAXCONN="0", LB_QUEUE_TIMEOUT="2s")
|
|
|
|
|
|
def run(args, env=None, check=True, capture=True, timeout=900):
|
|
return subprocess.run(args, cwd=ROOT, env=env, check=check, timeout=timeout,
|
|
stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE, text=True)
|
|
|
|
|
|
def cfg_env(cfg):
|
|
env = dict(os.environ)
|
|
env.setdefault("COMPOSE_FILE", BASE_FILES)
|
|
for k, v in {**DEFAULTS, **cfg}.items():
|
|
if k.isupper():
|
|
env[k] = str(v)
|
|
return env
|
|
|
|
|
|
def lb_rows():
|
|
with urllib.request.urlopen(STATS_URL, timeout=2) as r:
|
|
text = r.read().decode().lstrip("# ")
|
|
return list(csv.DictReader(io.StringIO(text)))
|
|
|
|
|
|
def wait_lb(n, timeout=90):
|
|
t = time.monotonic()
|
|
while time.monotonic() - t < timeout:
|
|
try:
|
|
up = [r for r in lb_rows() if r["pxname"] == "api" and r["svname"].startswith("api")
|
|
and not r["svname"].startswith("apibk") and r["status"] == "UP"]
|
|
if len(up) == n:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(1)
|
|
raise SystemExit("LB does not see %d UP replicas" % n)
|
|
|
|
|
|
def pool_stats(env):
|
|
res = {}
|
|
for cid in run(DC + ["ps", "-q", "api"], env).stdout.split():
|
|
out = run(["docker", "exec", cid, "python", "-m", "messenger", "stats"], check=False).stdout
|
|
try:
|
|
d = json.loads(out)
|
|
res[d["instance_id"]] = {"pool": d["pool"], "cache": d["cache"], "requests": d["app"]["requests"]}
|
|
except (ValueError, KeyError):
|
|
pass
|
|
return res
|
|
|
|
|
|
class Sampler:
|
|
"""docker stats (CPU%, RAM), HAProxy CSV и pg_stat_activity раз в ~1 с с отметками времени."""
|
|
|
|
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
|
|
|
|
def __init__(self, env):
|
|
self.env, self.stop = env, threading.Event()
|
|
self.docker, self.lb, self.pg, self.throttle = [], [], [], []
|
|
|
|
def _docker(self):
|
|
p = subprocess.Popen(["docker", "stats", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, cwd=ROOT)
|
|
try:
|
|
for line in p.stdout:
|
|
if self.stop.is_set():
|
|
break
|
|
line = self.ANSI.sub("", line).strip()
|
|
parts = line.split("\t")
|
|
if len(parts) == 3 and parts[1].endswith("%") and parts[0].startswith("mtusi-lab3-"):
|
|
mem = parts[2].split("/")[0].strip()
|
|
mib = float(re.sub(r"[^0-9.]", "", mem) or 0) * (1024 if "GiB" in mem else 1 / 1024 if "KiB" in mem else 1)
|
|
self.docker.append((time.time(), parts[0].replace("mtusi-lab3-", ""), float(parts[1][:-1]), mib))
|
|
finally:
|
|
p.kill()
|
|
|
|
@staticmethod
|
|
def _throttle(cid):
|
|
for path in ("/sys/fs/cgroup/cpu/docker/%s/cpu.stat" % cid, "/sys/fs/cgroup/system.slice/docker-%s.scope/cpu.stat" % cid):
|
|
try:
|
|
d = dict(line.split() for line in open(path))
|
|
return int(d.get("nr_periods", 0)), int(d.get("nr_throttled", 0))
|
|
except OSError:
|
|
continue
|
|
return None
|
|
|
|
def _poll(self):
|
|
ids = [c for c in run(DC + ["ps", "-q", "--no-trunc", "api"], self.env, check=False).stdout.split()]
|
|
while not self.stop.is_set():
|
|
t = time.time()
|
|
thr = [self._throttle(c) for c in ids]
|
|
if all(thr) and thr:
|
|
self.throttle.append((t, sum(x[0] for x in thr), sum(x[1] for x in thr)))
|
|
try:
|
|
rows = lb_rows()
|
|
api = [r for r in rows if r["pxname"] == "api" and r["svname"] not in ("FRONTEND",)]
|
|
be = next(r for r in api if r["svname"] == "BACKEND")
|
|
self.lb.append((t, int(be["qcur"] or 0), int(be["scur"] or 0),
|
|
sum(int(r["qcur"] or 0) for r in api if r["svname"].startswith("api"))))
|
|
except (OSError, StopIteration, ValueError):
|
|
pass
|
|
self.stop.wait(1.0)
|
|
|
|
PG_SQL = ("SELECT 'S', extract(epoch FROM now()), coalesce(state, '?'), count(*) FROM pg_stat_activity"
|
|
" WHERE datname = 'messenger' AND backend_type = 'client backend' AND pid <> pg_backend_pid() GROUP BY 3"
|
|
" UNION ALL SELECT 'W', extract(epoch FROM now()), coalesce(wait_event_type || ':' || wait_event, 'CPU'), count(*)"
|
|
" FROM pg_stat_activity WHERE datname = 'messenger' AND state = 'active' AND pid <> pg_backend_pid() GROUP BY 3")
|
|
|
|
def _pg(self):
|
|
"""ОДНА долгоживущая сессия psql с \\watch 1 (не docker exec каждую секунду — он сам грузит хост)."""
|
|
p = subprocess.Popen(DC + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|"],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
|
|
cwd=ROOT, env=self.env)
|
|
p.stdin.write(self.PG_SQL + " \\watch 1\n"); p.stdin.flush()
|
|
snap, cur_ts = {}, None
|
|
try:
|
|
for line in p.stdout:
|
|
if self.stop.is_set():
|
|
break
|
|
parts = line.strip().split("|")
|
|
if len(parts) != 4:
|
|
continue
|
|
kind, ts, key, n = parts
|
|
ts = float(ts)
|
|
if cur_ts is not None and ts != cur_ts:
|
|
self.pg.append((cur_ts, snap)); snap = {}
|
|
cur_ts = ts
|
|
snap[("state:" if kind == "S" else "wait:") + key] = int(n)
|
|
finally:
|
|
p.kill()
|
|
|
|
def __enter__(self):
|
|
self.threads = [threading.Thread(target=self._docker, daemon=True), threading.Thread(target=self._poll, daemon=True),
|
|
threading.Thread(target=self._pg, daemon=True)]
|
|
[t.start() for t in self.threads]
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
self.stop.set()
|
|
|
|
def window(self, a, b):
|
|
cpu, mem = {}, {}
|
|
for t, name, c, m in self.docker:
|
|
if a <= t <= b:
|
|
key = "api" if re.fullmatch(r"api-\d+", name) else re.sub(r"-\d+$", "", name)
|
|
cpu.setdefault((key, name), []).append(c)
|
|
mem.setdefault(key, []).append(m)
|
|
agg = {}
|
|
for (key, name), vals in cpu.items():
|
|
agg.setdefault(key, 0.0)
|
|
agg[key] += statistics.mean(vals)
|
|
lb = [x for x in self.lb if a <= x[0] <= b]
|
|
th = [x for x in self.throttle if a <= x[0] <= b]
|
|
throttled = (round((th[-1][2] - th[0][2]) / max(1, th[-1][1] - th[0][1]), 3) if len(th) >= 2 else None)
|
|
pg = [x[1] for x in self.pg if a <= x[0] <= b]
|
|
return {
|
|
"cpu_percent_mean": {k: round(v, 1) for k, v in sorted(agg.items())},
|
|
"mem_mib_max": {k: round(max(v), 1) for k, v in sorted(mem.items())},
|
|
"lb_queue_max": max((x[1] + x[3] for x in lb), default=None),
|
|
"lb_sessions_mean": round(statistics.mean(x[2] for x in lb), 1) if lb else None,
|
|
"pg_active_mean": round(statistics.mean(p.get("state:active", 0) for p in pg), 2) if pg else None,
|
|
"pg_wait_events_mean": ({k[5:]: round(sum(p.get(k, 0) for p in pg) / len(pg), 2)
|
|
for k in sorted({k for p in pg for k in p if k.startswith("wait:")})} if pg else None),
|
|
"api_cfs_throttled_period_ratio": throttled,
|
|
"pg_connections_mean": round(statistics.mean(sum(v for k, v in p.items() if k.startswith("state:")) for p in pg), 1) if pg else None,
|
|
}
|
|
|
|
|
|
def run_config(name, cfg, args, rep=1):
|
|
if rep > 1:
|
|
name = "%s-r%d" % (name, rep)
|
|
env = cfg_env(cfg)
|
|
print("== %s %s" % (name, {k: v for k, v in cfg.items() if k.isupper()}), flush=True)
|
|
run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], env, timeout=300)
|
|
wait_lb(int(cfg["API_REPLICAS"]))
|
|
time.sleep(3)
|
|
pools_before = pool_stats(env)
|
|
rates = cfg.get("RATES", args.rates)
|
|
cmd = DC + ["--profile", "bench", "run", "--rm", "-T", "loadgen", "-rates", rates,
|
|
"-duration", "%ds" % args.duration, "-warmup", "%ds" % args.warmup,
|
|
"-users", str(args.users), "-seed", str(args.seed), "-label", name]
|
|
steps, starts = [], {}
|
|
with Sampler(env) as sampler:
|
|
p = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
for line in p.stdout:
|
|
try:
|
|
ev = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if ev["event"] == "step_start":
|
|
starts[ev["rate"]] = ev["ts"] + ev["warmup_s"]
|
|
elif ev["event"] == "step":
|
|
a = starts[ev["rate"]]
|
|
ev["resources"] = sampler.window(a, a + ev["duration_s"])
|
|
ev["slo_ok"] = ev["latency_ms"]["p99"] <= SLO_P99_MS and ev["error_ratio"] <= SLO_ERR
|
|
steps.append(ev)
|
|
print(" rate %5.0f ok %6.1f rps err %5.1f%% p50 %7.1f p99 %8.1f ms cpu %s pgActive %s lbQ %s" % (
|
|
ev["rate"], ev["throughput_ok_rps"], 100 * ev["error_ratio"], ev["latency_ms"]["p50"],
|
|
ev["latency_ms"]["p99"], ev["resources"]["cpu_percent_mean"], ev["resources"]["pg_active_mean"],
|
|
ev["resources"]["lb_queue_max"]), flush=True)
|
|
elif ev["event"] == "setup":
|
|
print(" setup %.1fs" % ev["seconds"], flush=True)
|
|
p.wait()
|
|
if p.returncode:
|
|
print(p.stderr.read()[-1500:])
|
|
pools_after = pool_stats(env)
|
|
ok = [s for s in steps if s["slo_ok"]]
|
|
result = {
|
|
"config": name, "repeat": rep, "series": cfg["series"], "params": {**DEFAULTS, **{k: v for k, v in cfg.items() if k.isupper()}},
|
|
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
"slo": {"p99_ms": SLO_P99_MS, "error_ratio": SLO_ERR},
|
|
"capacity_at_slo_rps": max((s["throughput_ok_rps"] for s in ok), default=0),
|
|
"peak_ok_rps": max((s["throughput_ok_rps"] for s in steps), default=0),
|
|
"steps": steps, "pool_before": pools_before, "pool_after": pools_after,
|
|
"loadgen": {"users": args.users, "seed": args.seed, "duration_s": args.duration, "warmup_s": args.warmup,
|
|
"mix": "history=70,post=20,me=10", "model": "open-loop, latency from scheduled time"},
|
|
}
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
(OUT / (name + ".json")).write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n")
|
|
print(" => capacity@SLO %.1f rps, peak %.1f rps" % (result["capacity_at_slo_rps"], result["peak_ok_rps"]), flush=True)
|
|
|
|
|
|
def report(args):
|
|
from track1_report import build # noqa: E402 (рядом в research/)
|
|
build(OUT)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("action", choices=["run", "report", "list"])
|
|
p.add_argument("targets", nargs="*", default=["all"])
|
|
p.add_argument("--rates", default=RATES)
|
|
p.add_argument("--duration", type=int, default=10)
|
|
p.add_argument("--warmup", type=int, default=3)
|
|
p.add_argument("--users", type=int, default=48)
|
|
p.add_argument("--seed", type=int, default=10)
|
|
p.add_argument("--rep", type=int, default=1, help="номер повтора (сохраняется как <config>-r<rep>.json)")
|
|
args = p.parse_args()
|
|
if args.action == "list":
|
|
for k, v in CONFIGS.items():
|
|
print(k, v)
|
|
return
|
|
if args.action == "report":
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
return report(args)
|
|
names = [n for n, c in CONFIGS.items() if "all" in args.targets or c["series"] in args.targets or n in args.targets]
|
|
for n in names:
|
|
run_config(n, CONFIGS[n], args, args.rep)
|
|
# вернуть штатную конфигурацию стенда
|
|
run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], cfg_env({"API_REPLICAS": 2}), timeout=300)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|