Files
mtusi2026lab3/research/scaling.py
T

192 lines
8.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Трек 1 (цена масштабирования), lab3.
Серия A (фиксированный суммарный CPU/RAM API): 1×2.0 CPU, 2×1.0, 4×0.5 — один и тот же профиль нагрузки.
Серия B (рост ресурсов): 1, 2, 4, 8 реплик по 1.0 CPU; показывает точку насыщения (БД / LB / пул / клиент).
Для каждой конфигурации: прогрев 5 с, затем N с нагрузки несколькими процессами-клиентами (closed-loop, 90 % POST / 10 % GET),
параллельно семплируются `docker stats` (CPU каждого сервиса). Ресурсы меняются на лету `docker update`, реплики — `compose --scale`.
python research/scaling.py --series A --out evidence --tag base
python research/scaling.py --series B --out evidence --tag base
Переменные окружения стенда (SEND_STRATEGY=...) задаются при запуске compose и описываются в --tag.
"""
import argparse
import asyncio
import json
import multiprocessing as mp
import os
import statistics
import subprocess
import sys
import threading
import time
sys.path.insert(0, os.path.dirname(__file__))
import loadgen # noqa: E402
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
BASE = "http://127.0.0.1:8080"
FILTER = ["--filter", "label=com.docker.compose.project=mtusi-lab3"]
def sh(*args, check=False):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, env=dict(os.environ, **{"PYTHONUTF8": "1"})).stdout.strip()
def containers(service):
return sh("docker", "ps", "-q", *FILTER, "--filter", "label=com.docker.compose.service=" + service).split()
def configure(replicas, api_cpus, pg_cpus, lb_cpus):
sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % replicas, "api")
# лишние реплики (если было больше) удаляем
ids = containers("api")
for extra in ids[replicas:]:
sh("docker", "stop", "--time", "5", extra)
sh("docker", "rm", extra)
for cid in containers("api"):
sh("docker", "update", "--cpus", str(api_cpus), "--memory", "256m", "--memory-swap", "256m", cid)
for cid in containers("postgres"):
sh("docker", "update", "--cpus", str(pg_cpus), "--memory", "512m", "--memory-swap", "512m", cid)
for cid in containers("lb"):
sh("docker", "update", "--cpus", str(lb_cpus), "--memory", "128m", "--memory-swap", "128m", cid)
deadline = time.time() + 60
while time.time() < deadline:
states = [sh("docker", "inspect", "-f", "{{.State.Health.Status}}", c) for c in containers("api")]
if len(states) == replicas and all(s == "healthy" for s in states):
break
time.sleep(1)
wait_lb(replicas)
def wait_lb(replicas, timeout=90):
"""Ждём, пока LB реально раздаёт запросы всем репликам (каждая видна по X-Instance-Id)."""
import urllib.request
deadline = time.time() + timeout
while time.time() < deadline:
seen = set()
try:
for _ in range(replicas * 12):
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as r:
seen.add(r.headers.get("X-Instance-Id"))
except Exception:
time.sleep(1)
continue
if len(seen) >= replicas:
return
time.sleep(1)
raise RuntimeError("LB не раздаёт запросы всем %d репликам" % replicas)
def child(args):
stand_data, seconds, concurrency, seed, t0 = args
import random
random.seed(seed)
stand = loadgen.Stand(BASE)
stand.users, stand.chats = stand_data["users"], stand_data["chats"]
rec = loadgen.Recorder()
rec.t0 = t0
async def go():
await loadgen.run_load(stand, rec, seconds, concurrency)
asyncio.run(go())
return rec.events, len(rec.confirmed), rec.final_failures
class Sampler(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.rows, self.stop = [], threading.Event()
def run(self):
while not self.stop.is_set():
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.CPUPerc}} {{.MemUsage}}"],
capture_output=True, text=True).stdout.splitlines()
self.rows.append((time.time(), [l.split() for l in out if "mtusi-lab3" in l]))
def summary(self, t_from, t_to):
agg = {}
for ts, rows in self.rows:
if not t_from <= ts <= t_to:
continue
for r in rows:
service = r[0].replace("mtusi-lab3-", "").rsplit("-", 1)[0]
agg.setdefault(service, []).append(float(r[1].rstrip("%")))
return {k: {"avg_cpu_percent_per_container": round(statistics.mean(v), 1), "samples": len(v)} for k, v in agg.items()}
def run_config(name, replicas, api_cpus, pg_cpus, lb_cpus, args, stand_data):
configure(replicas, api_cpus, pg_cpus, lb_cpus)
# прогрев
t0 = time.time()
with mp.get_context("spawn").Pool(args.procs) as pool:
pool.map(child, [(stand_data, 5, max(1, args.concurrency // args.procs), i, t0) for i in range(args.procs)])
sampler = Sampler()
sampler.start()
t_start = time.time()
with mp.get_context("spawn").Pool(args.procs) as pool:
results = pool.map(child, [(stand_data, args.seconds, args.concurrency // args.procs, 100 + i, t_start) for i in range(args.procs)])
t_end = time.time()
sampler.stop.set()
rec = loadgen.Recorder()
rec.t0 = t_start
for events, confirmed, failures in results:
rec.events.extend(events)
post = loadgen.summarize(rec, 0, args.seconds, "POST")
get = loadgen.summarize(rec, 0, args.seconds, "GET")
instances = {}
for e in rec.events:
if e[1] == "POST" and e[4]:
instances[e[4]] = instances.get(e[4], 0) + 1
out = {"config": name, "replicas": replicas, "api_cpus_each": api_cpus, "api_cpus_total": round(replicas * api_cpus, 2),
"postgres_cpus": pg_cpus, "lb_cpus": lb_cpus, "concurrency": args.concurrency, "client_processes": args.procs,
"post": post, "get": get, "total_rps": round((post.get("ok", 0) + get.get("ok", 0)) / args.seconds, 1),
"replica_share": sorted(instances.values()), "docker_stats": sampler.summary(t_start + 2, t_end)}
print(json.dumps({k: out[k] for k in ("config", "total_rps")} | {"p95": post.get("p95_ms"), "err": post.get("error_ratio"),
"stats": out["docker_stats"]}, ensure_ascii=False))
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--series", choices=["A", "B", "C"], required=True)
ap.add_argument("--out", default="evidence")
ap.add_argument("--tag", default="base")
ap.add_argument("--seconds", type=int, default=30)
ap.add_argument("--concurrency", type=int, default=256)
ap.add_argument("--procs", type=int, default=8)
ap.add_argument("--pg-cpus", type=float, default=2.0)
ap.add_argument("--lb-cpus", type=float, default=2.0)
ap.add_argument("--repeat", type=int, default=2)
args = ap.parse_args()
configure(2, 1.0, args.pg_cpus, args.lb_cpus)
stand = loadgen.Stand(BASE, users=40, chats=40)
asyncio.run(stand.setup())
stand_data = {"users": stand.users, "chats": stand.chats}
if args.series == "A":
configs = [("1x2.0", 1, 2.0), ("2x1.0", 2, 1.0), ("4x0.5", 4, 0.5)]
elif args.series == "B":
configs = [("1x1.0", 1, 1.0), ("2x1.0", 2, 1.0), ("4x1.0", 4, 1.0), ("8x1.0", 8, 1.0)]
else: # C: только 2 и 4 реплики при увеличенном CPU БД — проверка гипотезы «узкое место — БД»
configs = [("2x1.0", 2, 1.0), ("4x1.0", 4, 1.0)]
results = []
for name, n, cpus in configs:
runs = [run_config(name, n, cpus, args.pg_cpus, args.lb_cpus, args, stand_data) for _ in range(args.repeat)]
best = dict(runs[0])
best["total_rps_runs"] = [r["total_rps"] for r in runs]
best["total_rps"] = round(statistics.mean(r["total_rps"] for r in runs), 1)
best["post_p95_runs"] = [r["post"].get("p95_ms") for r in runs]
best["runs"] = runs
results.append(best)
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "series": args.series, "tag": args.tag, "args": vars(args), "results": results}
path = os.path.join(args.out, "scaling_%s_%s.json" % (args.series, args.tag))
with open(path, "w", encoding="utf-8") as fh:
json.dump(report, fh, ensure_ascii=False, indent=2)
configure(2, 1.0, 1.0, 1.0)
print("saved", path)
if __name__ == "__main__":
mp.freeze_support()
main()