forked from mtusi2026/mtusi2026lab3
192 lines
8.8 KiB
Python
192 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
||
"""Трек 1 (цена масштабирования), lab3.
|
||
|
||
Серия A (фиксированный суммарный CPU/RAM API): 1×2.0 CPU, 2×1.0, 4×0.5 — один и тот же профиль нагрузки.
|
||
Серия B (рост ресурсов): 1, 2, 4, 8 реплик по 1.0 CPU; показывает точку насыщения (БД / LB / пул / клиент).
|
||
Для каждой конфигурации: прогрев 5 с, затем N с нагрузки несколькими процессами-клиентами (closed-loop, 90 % POST / 10 % GET),
|
||
параллельно семплируются `docker stats` (CPU каждого сервиса). Ресурсы меняются на лету `docker update`, реплики — `compose --scale`.
|
||
|
||
python research/scaling.py --series A --out evidence --tag base
|
||
python research/scaling.py --series B --out evidence --tag base
|
||
Переменные окружения стенда (SEND_STRATEGY=...) задаются при запуске compose и описываются в --tag.
|
||
"""
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import multiprocessing as mp
|
||
import os
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
import loadgen # noqa: E402
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
BASE = "http://127.0.0.1:8080"
|
||
FILTER = ["--filter", "label=com.docker.compose.project=mtusi-lab3"]
|
||
|
||
|
||
def sh(*args, check=False):
|
||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, env=dict(os.environ, **{"PYTHONUTF8": "1"})).stdout.strip()
|
||
|
||
|
||
def containers(service):
|
||
return sh("docker", "ps", "-q", *FILTER, "--filter", "label=com.docker.compose.service=" + service).split()
|
||
|
||
|
||
def configure(replicas, api_cpus, pg_cpus, lb_cpus):
|
||
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % replicas, "api")
|
||
# лишние реплики (если было больше) удаляем
|
||
ids = containers("api")
|
||
for extra in ids[replicas:]:
|
||
sh("docker", "stop", "--time", "5", extra)
|
||
sh("docker", "rm", extra)
|
||
for cid in containers("api"):
|
||
sh("docker", "update", "--cpus", str(api_cpus), "--memory", "256m", "--memory-swap", "256m", cid)
|
||
for cid in containers("postgres"):
|
||
sh("docker", "update", "--cpus", str(pg_cpus), "--memory", "512m", "--memory-swap", "512m", cid)
|
||
for cid in containers("lb"):
|
||
sh("docker", "update", "--cpus", str(lb_cpus), "--memory", "128m", "--memory-swap", "128m", cid)
|
||
deadline = time.time() + 60
|
||
while time.time() < deadline:
|
||
states = [sh("docker", "inspect", "-f", "{{.State.Health.Status}}", c) for c in containers("api")]
|
||
if len(states) == replicas and all(s == "healthy" for s in states):
|
||
break
|
||
time.sleep(1)
|
||
wait_lb(replicas)
|
||
|
||
|
||
def wait_lb(replicas, timeout=90):
|
||
"""Ждём, пока LB реально раздаёт запросы всем репликам (каждая видна по X-Instance-Id)."""
|
||
import urllib.request
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
seen = set()
|
||
try:
|
||
for _ in range(replicas * 12):
|
||
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as r:
|
||
seen.add(r.headers.get("X-Instance-Id"))
|
||
except Exception:
|
||
time.sleep(1)
|
||
continue
|
||
if len(seen) >= replicas:
|
||
return
|
||
time.sleep(1)
|
||
raise RuntimeError("LB не раздаёт запросы всем %d репликам" % replicas)
|
||
|
||
|
||
def child(args):
|
||
stand_data, seconds, concurrency, seed, t0 = args
|
||
import random
|
||
random.seed(seed)
|
||
stand = loadgen.Stand(BASE)
|
||
stand.users, stand.chats = stand_data["users"], stand_data["chats"]
|
||
rec = loadgen.Recorder()
|
||
rec.t0 = t0
|
||
|
||
async def go():
|
||
await loadgen.run_load(stand, rec, seconds, concurrency)
|
||
asyncio.run(go())
|
||
return rec.events, len(rec.confirmed), rec.final_failures
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
def __init__(self):
|
||
super().__init__(daemon=True)
|
||
self.rows, self.stop = [], threading.Event()
|
||
|
||
def run(self):
|
||
while not self.stop.is_set():
|
||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.CPUPerc}} {{.MemUsage}}"],
|
||
capture_output=True, text=True).stdout.splitlines()
|
||
self.rows.append((time.time(), [l.split() for l in out if "mtusi-lab3" in l]))
|
||
|
||
def summary(self, t_from, t_to):
|
||
agg = {}
|
||
for ts, rows in self.rows:
|
||
if not t_from <= ts <= t_to:
|
||
continue
|
||
for r in rows:
|
||
service = r[0].replace("mtusi-lab3-", "").rsplit("-", 1)[0]
|
||
agg.setdefault(service, []).append(float(r[1].rstrip("%")))
|
||
return {k: {"avg_cpu_percent_per_container": round(statistics.mean(v), 1), "samples": len(v)} for k, v in agg.items()}
|
||
|
||
|
||
def run_config(name, replicas, api_cpus, pg_cpus, lb_cpus, args, stand_data):
|
||
configure(replicas, api_cpus, pg_cpus, lb_cpus)
|
||
# прогрев
|
||
t0 = time.time()
|
||
with mp.get_context("spawn").Pool(args.procs) as pool:
|
||
pool.map(child, [(stand_data, 5, max(1, args.concurrency // args.procs), i, t0) for i in range(args.procs)])
|
||
sampler = Sampler()
|
||
sampler.start()
|
||
t_start = time.time()
|
||
with mp.get_context("spawn").Pool(args.procs) as pool:
|
||
results = pool.map(child, [(stand_data, args.seconds, args.concurrency // args.procs, 100 + i, t_start) for i in range(args.procs)])
|
||
t_end = time.time()
|
||
sampler.stop.set()
|
||
rec = loadgen.Recorder()
|
||
rec.t0 = t_start
|
||
for events, confirmed, failures in results:
|
||
rec.events.extend(events)
|
||
post = loadgen.summarize(rec, 0, args.seconds, "POST")
|
||
get = loadgen.summarize(rec, 0, args.seconds, "GET")
|
||
instances = {}
|
||
for e in rec.events:
|
||
if e[1] == "POST" and e[4]:
|
||
instances[e[4]] = instances.get(e[4], 0) + 1
|
||
out = {"config": name, "replicas": replicas, "api_cpus_each": api_cpus, "api_cpus_total": round(replicas * api_cpus, 2),
|
||
"postgres_cpus": pg_cpus, "lb_cpus": lb_cpus, "concurrency": args.concurrency, "client_processes": args.procs,
|
||
"post": post, "get": get, "total_rps": round((post.get("ok", 0) + get.get("ok", 0)) / args.seconds, 1),
|
||
"replica_share": sorted(instances.values()), "docker_stats": sampler.summary(t_start + 2, t_end)}
|
||
print(json.dumps({k: out[k] for k in ("config", "total_rps")} | {"p95": post.get("p95_ms"), "err": post.get("error_ratio"),
|
||
"stats": out["docker_stats"]}, ensure_ascii=False))
|
||
return out
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--series", choices=["A", "B", "C"], required=True)
|
||
ap.add_argument("--out", default="evidence")
|
||
ap.add_argument("--tag", default="base")
|
||
ap.add_argument("--seconds", type=int, default=30)
|
||
ap.add_argument("--concurrency", type=int, default=256)
|
||
ap.add_argument("--procs", type=int, default=8)
|
||
ap.add_argument("--pg-cpus", type=float, default=2.0)
|
||
ap.add_argument("--lb-cpus", type=float, default=2.0)
|
||
ap.add_argument("--repeat", type=int, default=2)
|
||
args = ap.parse_args()
|
||
configure(2, 1.0, args.pg_cpus, args.lb_cpus)
|
||
stand = loadgen.Stand(BASE, users=40, chats=40)
|
||
asyncio.run(stand.setup())
|
||
stand_data = {"users": stand.users, "chats": stand.chats}
|
||
if args.series == "A":
|
||
configs = [("1x2.0", 1, 2.0), ("2x1.0", 2, 1.0), ("4x0.5", 4, 0.5)]
|
||
elif args.series == "B":
|
||
configs = [("1x1.0", 1, 1.0), ("2x1.0", 2, 1.0), ("4x1.0", 4, 1.0), ("8x1.0", 8, 1.0)]
|
||
else: # C: только 2 и 4 реплики при увеличенном CPU БД — проверка гипотезы «узкое место — БД»
|
||
configs = [("2x1.0", 2, 1.0), ("4x1.0", 4, 1.0)]
|
||
results = []
|
||
for name, n, cpus in configs:
|
||
runs = [run_config(name, n, cpus, args.pg_cpus, args.lb_cpus, args, stand_data) for _ in range(args.repeat)]
|
||
best = dict(runs[0])
|
||
best["total_rps_runs"] = [r["total_rps"] for r in runs]
|
||
best["total_rps"] = round(statistics.mean(r["total_rps"] for r in runs), 1)
|
||
best["post_p95_runs"] = [r["post"].get("p95_ms") for r in runs]
|
||
best["runs"] = runs
|
||
results.append(best)
|
||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "series": args.series, "tag": args.tag, "args": vars(args), "results": results}
|
||
path = os.path.join(args.out, "scaling_%s_%s.json" % (args.series, args.tag))
|
||
with open(path, "w", encoding="utf-8") as fh:
|
||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||
configure(2, 1.0, 1.0, 1.0)
|
||
print("saved", path)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
mp.freeze_support()
|
||
main()
|