forked from mtusi2026/mtusi2026lab2
scripts/scaling.py applies a replicas x CPU layout through compose variables: workers equal to the replica CPUs, a pool of 8 per worker, a balancer queue of 32 per worker so every layout of a series offers the same concurrency to the API, and the replicas pinned to CPUs 7 and up, next to PostgreSQL on 0-1, the balancer on 2, Valkey on 3 and k6 on 4-6 with 3 CPU and 1 GiB. For every offered rate it runs k6 three times and stops a config after two steps saturated in all repeats. Each step keeps its raw data: k6 summary, docker stats, HAProxy statistics and pg_stat_activity sampled every second, and the top of pg_stat_statements. The summary row carries CPU per component, PostgreSQL active, idle, idle in transaction and lock waiting backends, the balancer queue and the database size. Several variants alternate per repeat for before and after runs. host-snapshot.sh records every container on the host before a series. make track1-a and track1-b run the fixed budget and the growth series from an empty database. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
328 lines
12 KiB
Python
Executable File
328 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import signal
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from probe import ROOT, k6_metrics, lb_up, run_k6
|
|
|
|
SCRIPTS = ROOT / "scripts"
|
|
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
|
|
PINNED = {
|
|
"PG_CPUSET": "0-1",
|
|
"LB_CPUSET": "2",
|
|
"VALKEY_CPUSET": "3",
|
|
"K6_CPUSET": "4-6",
|
|
"K6_CPUS": "3",
|
|
"K6_MEMORY": "1g",
|
|
}
|
|
API_FIRST_CPU = 7
|
|
FIELDS = (
|
|
"variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
|
|
"started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio "
|
|
"dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres "
|
|
"cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max "
|
|
"pg_active_avg pg_active_max pg_idle_avg pg_idle_tx_avg pg_lock_avg pg_lock_max pg_lwlock_avg "
|
|
"pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms"
|
|
).split()
|
|
|
|
|
|
def compose(*args, env=None):
|
|
command = ["docker", "compose", *args]
|
|
return subprocess.run(
|
|
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
|
|
).stdout
|
|
|
|
|
|
def size(text):
|
|
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
|
|
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
|
|
|
|
|
|
def rows(path, start, end):
|
|
with path.open() as handle:
|
|
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
|
|
|
|
|
|
def mean(values):
|
|
return round(statistics.mean(values), 2) if values else 0.0
|
|
|
|
|
|
def docker_usage(path, project, start, end):
|
|
ticks = collections.defaultdict(collections.Counter)
|
|
memory = collections.Counter()
|
|
for r in rows(path, start, end):
|
|
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
|
|
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
|
|
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
|
|
cpu = {
|
|
name: mean([t[name] for t in ticks.values() if name in t])
|
|
for name in ("api", "postgres", "lb", "valkey", "k6")
|
|
}
|
|
usage = {f"cpu_{name}": value for name, value in cpu.items()}
|
|
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
|
|
return usage
|
|
|
|
|
|
def pg_usage(path, start, end):
|
|
ticks = collections.defaultdict(collections.Counter)
|
|
for r in rows(path, start, end):
|
|
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
|
|
tick = ticks[r["ts"]]
|
|
tick["backends"] += count
|
|
tick["active"] += count * (state == "active")
|
|
tick["idle"] += count * (state == "idle")
|
|
tick["idle_tx"] += count * state.startswith("idle in transaction")
|
|
tick["lock"] += count * (wait == "Lock")
|
|
tick["lwlock"] += count * (wait == "LWLock")
|
|
tick["io"] += count * (wait == "IO")
|
|
series = list(ticks.values())
|
|
return {
|
|
"pg_backends_max": max((t["backends"] for t in series), default=0),
|
|
"pg_active_avg": mean([t["active"] for t in series]),
|
|
"pg_active_max": max((t["active"] for t in series), default=0),
|
|
"pg_idle_avg": mean([t["idle"] for t in series]),
|
|
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
|
|
"pg_lock_avg": mean([t["lock"] for t in series]),
|
|
"pg_lock_max": max((t["lock"] for t in series), default=0),
|
|
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
|
|
"pg_io_avg": mean([t["io"] for t in series]),
|
|
}
|
|
|
|
|
|
def lb_usage(path, start, end):
|
|
backend = [
|
|
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
|
|
]
|
|
|
|
def numbers(field):
|
|
return [int(r[field] or 0) for r in backend]
|
|
|
|
return {
|
|
"lb_scur_max": max(numbers("scur"), default=0),
|
|
"lb_qcur_max": max(numbers("qcur"), default=0),
|
|
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
|
|
"lb_rtime_avg_ms": mean(numbers("rtime")),
|
|
}
|
|
|
|
|
|
def trend(path, name):
|
|
metrics = json.loads(Path(path).read_text())["metrics"]
|
|
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
|
|
return round(found.get("values", {}).get("p(95)", 0.0), 2)
|
|
|
|
|
|
def sampler(*command):
|
|
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
|
|
|
|
|
|
def stop(process):
|
|
try:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
process.wait()
|
|
|
|
|
|
def wait_lb(expected, timeout=120):
|
|
deadline, stable = time.monotonic() + timeout, 0
|
|
while stable < 4:
|
|
if time.monotonic() > deadline:
|
|
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
|
|
stable = stable + 1 if lb_up() == expected else 0
|
|
time.sleep(0.5)
|
|
|
|
|
|
def layout(config, args, variant):
|
|
name, _, extra = variant.partition(":")
|
|
replicas, cpus = (float(v) for v in config.split("x"))
|
|
replicas = int(replicas)
|
|
workers = args.workers or max(1, int(cpus))
|
|
width = max(1, math.ceil(replicas * cpus))
|
|
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
|
|
memory = args.api_memory or args.api_memory_total // replicas
|
|
settings = {
|
|
"API_REPLICAS": str(replicas),
|
|
"API_CPUS": f"{cpus:g}",
|
|
"API_MEMORY": f"{memory}m",
|
|
"WEB_CONCURRENCY": str(workers),
|
|
"DB_POOL_MAX_SIZE": str(args.pool),
|
|
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
|
|
"API_CPUSET": cpuset if args.pin else "",
|
|
}
|
|
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
|
|
base = {
|
|
"variant": name,
|
|
"config": config,
|
|
"replicas": replicas,
|
|
"cpus": cpus,
|
|
"memory_mb": memory,
|
|
"workers": int(settings["WEB_CONCURRENCY"]),
|
|
"pool": int(settings["DB_POOL_MAX_SIZE"]),
|
|
"maxconn": int(settings["API_SERVER_MAXCONN"]),
|
|
"api_cpuset": settings["API_CPUSET"],
|
|
}
|
|
return base, settings
|
|
|
|
|
|
def apply(base, settings, env):
|
|
env = dict(env, **settings)
|
|
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
|
|
wait_lb(base["replicas"])
|
|
return env
|
|
|
|
|
|
def measure(folder, args, rate, env):
|
|
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
|
|
database = subprocess.run(
|
|
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
|
|
).stdout.split(",")
|
|
samplers = [
|
|
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
|
|
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
|
|
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
|
|
]
|
|
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
|
|
try:
|
|
code, started = run_k6(folder / "k6", load)
|
|
finally:
|
|
for process in samplers:
|
|
stop(process)
|
|
subprocess.run(
|
|
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
|
|
)
|
|
if started is None:
|
|
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
|
|
return code, started + args.warmup, database
|
|
|
|
|
|
def step(base, rate, repeat, args, env):
|
|
folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
code, start, database = measure(folder, args, rate, env)
|
|
end = start + args.duration
|
|
summary = folder / "k6/summary.json"
|
|
metrics = k6_metrics(summary, args.duration)
|
|
saturated = (
|
|
metrics["failed_ratio"] > args.max_error_ratio
|
|
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
|
|
or metrics["p95_ms"] > args.slo_p95_ms
|
|
)
|
|
row = dict(base, rate=rate, repeat=repeat, k6_exit=code, saturated=saturated, **metrics)
|
|
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(start))
|
|
row["db_mb"] = round(int(database[0]) / 2**20, 1)
|
|
row["messages"] = int(database[1])
|
|
row["p95_post_ms"] = trend(summary, "post_message")
|
|
row["p95_get_ms"] = trend(summary, "get_history")
|
|
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
|
|
row.update(pg_usage(folder / "pg_activity.csv", start, end))
|
|
row.update(lb_usage(folder / "lb_stats.csv", start, end))
|
|
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
|
return row
|
|
|
|
|
|
def rates(args, base):
|
|
if args.rates:
|
|
return [int(r) for r in args.rates.split(",")]
|
|
factor, top = (float(v) for v in args.search.split(":"))
|
|
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
|
|
while value <= top:
|
|
values.append(int(round(value, -1)))
|
|
value *= factor
|
|
return values
|
|
|
|
|
|
def arguments():
|
|
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
|
|
parser.add_argument(
|
|
"--variant",
|
|
dest="variants",
|
|
action="append",
|
|
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
|
|
)
|
|
parser.add_argument("--fresh-db", action="store_true", help="drop volumes before a config")
|
|
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
|
|
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
|
|
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
|
|
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
|
|
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
|
|
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
|
|
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
|
|
parser.add_argument("--rates", help="comma separated offered rates, rps")
|
|
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
|
|
parser.add_argument("--start-per-cpu", type=float, default=100.0)
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--warmup", type=int, default=10)
|
|
parser.add_argument("--duration", type=int, default=30)
|
|
parser.add_argument("--pause", type=float, default=5.0)
|
|
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
|
|
parser.add_argument("--max-error-ratio", type=float, default=0.01)
|
|
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
|
|
args = parser.parse_args()
|
|
args.variants = args.variants or ["base"]
|
|
if len(args.variants) > 1 and not args.rates:
|
|
parser.error("several variants need explicit --rates")
|
|
return args
|
|
|
|
|
|
def block(args, output, config, variant, repeats):
|
|
base, settings = layout(config, args, variant)
|
|
current, streak = apply(base, settings, args.base_env), 0
|
|
for rate in rates(args, base):
|
|
results = []
|
|
for repeat in repeats:
|
|
results.append(step(base, rate, repeat, args, current))
|
|
time.sleep(args.pause)
|
|
output.writerows(results)
|
|
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
|
if streak == 2:
|
|
break
|
|
|
|
|
|
def series(args, output):
|
|
every = range(1, args.repeats + 1)
|
|
for config in args.configs.split(","):
|
|
if args.fresh_db:
|
|
compose("down", "-v", env=args.base_env)
|
|
if len(args.variants) == 1:
|
|
block(args, output, config, args.variants[0], every)
|
|
continue
|
|
for repeat in every:
|
|
for variant in args.variants:
|
|
block(args, output, config, variant, [repeat])
|
|
|
|
|
|
def main():
|
|
args = arguments()
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
|
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
|
summary = out / "summary.csv"
|
|
fresh = not summary.exists()
|
|
try:
|
|
with summary.open("a", newline="", buffering=1) as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
|
if fresh:
|
|
writer.writeheader()
|
|
series(args, writer)
|
|
finally:
|
|
if not args.keep:
|
|
compose("up", "-d", "--wait", "--wait-timeout", "180")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|