Files
mtusi2026labs/scripts/scaling.py
T

479 lines
18 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import collections
import contextlib
import csv
import json
import math
import os
import re
import signal
import statistics
import subprocess
import sys
import time
from pathlib import Path
from probe import ROOT, k6_metrics, lb_up, run_k6
SCRIPTS = ROOT / "scripts"
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
PINNED = {
"PG_CPUSET": "0-1",
"LB_CPUSET": "2",
"VALKEY_CPUSET": "3",
"K6_CPUSET": "4-6",
"K6_CPUS": "3",
"K6_MEMORY": "1g",
}
API_FIRST_CPU = 7
FIELDS = (
"raw_dir variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
"started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio "
"dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres "
"cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max "
"pg_active_avg pg_active_max pg_idle_avg pg_idle_tx_avg pg_lock_avg pg_lock_max pg_lwlock_avg "
"pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms cpu_vm cpu_other "
"gap_s wall_s disturbed"
).split()
def compose(*args, env=None):
command = ["docker", "compose", *args]
return subprocess.run(
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
).stdout
def size(text):
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
def rows(path, start, end):
with path.open() as handle:
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
def mean(values):
return round(statistics.mean(values), 2) if values else 0.0
def docker_usage(path, project, start, end):
ticks = collections.defaultdict(collections.Counter)
memory = collections.Counter()
for r in rows(path, start, end):
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
cpu = {
name: mean([t[name] for t in ticks.values() if name in t])
for name in ("api", "postgres", "lb", "valkey", "k6")
}
usage = {f"cpu_{name}": value for name, value in cpu.items()}
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
return usage
def vm_usage(path, start, end):
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
return 0.0
first, last = samples[0], samples[-1]
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
def gap(path, start, end):
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
edges = [start, *[t for t in ticks if start <= t <= end], end]
return max(b - a for a, b in zip(edges, edges[1:]))
def vm_counters():
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
values = [int(v) for v in lines[0].split()[1:]]
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
return sum(values) - values[3] - values[4], sum(values), cpus
def wait_quiet(args):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = vm_counters()
time.sleep(5)
last = vm_counters()
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
calm = calm + 1 if busy < args.max_other else 0
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
def pg_usage(path, start, end):
ticks = collections.defaultdict(collections.Counter)
for r in rows(path, start, end):
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
tick = ticks[r["ts"]]
tick["backends"] += count
tick["active"] += count * (state == "active")
tick["idle"] += count * (state == "idle")
tick["idle_tx"] += count * state.startswith("idle in transaction")
tick["lock"] += count * (wait == "Lock")
tick["lwlock"] += count * (wait == "LWLock")
tick["io"] += count * (wait == "IO")
series = list(ticks.values())
return {
"pg_backends_max": max((t["backends"] for t in series), default=0),
"pg_active_avg": mean([t["active"] for t in series]),
"pg_active_max": max((t["active"] for t in series), default=0),
"pg_idle_avg": mean([t["idle"] for t in series]),
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
"pg_lock_avg": mean([t["lock"] for t in series]),
"pg_lock_max": max((t["lock"] for t in series), default=0),
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
"pg_io_avg": mean([t["io"] for t in series]),
}
def lb_usage(path, start, end):
backend = [
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
]
def numbers(field):
return [int(r[field] or 0) for r in backend]
return {
"lb_scur_max": max(numbers("scur"), default=0),
"lb_qcur_max": max(numbers("qcur"), default=0),
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
"lb_rtime_avg_ms": mean(numbers("rtime")),
}
def trend(path, name):
metrics = json.loads(Path(path).read_text())["metrics"]
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
return round(found.get("values", {}).get("p(95)", 0.0), 2)
def sampler(*command):
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
def stop(process):
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
process.wait()
def wait_lb(expected, timeout=120):
deadline, stable = time.monotonic() + timeout, 0
while stable < 4:
if time.monotonic() > deadline:
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
stable = stable + 1 if lb_up() == expected else 0
time.sleep(0.5)
def layout(config, args, variant):
name, _, extra = variant.partition(":")
replicas, cpus = (float(v) for v in config.split("x"))
replicas = int(replicas)
workers = args.workers or max(1, int(cpus))
width = max(1, math.ceil(replicas * cpus))
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
memory = args.api_memory or args.api_memory_total // replicas
settings = {
"API_REPLICAS": str(replicas),
"API_CPUS": f"{cpus:g}",
"API_MEMORY": f"{memory}m",
"WEB_CONCURRENCY": str(workers),
"DB_POOL_MAX_SIZE": str(args.pool),
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
"API_CPUSET": cpuset if args.pin else "",
}
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
base = {
"variant": name,
"config": config,
"replicas": replicas,
"cpus": cpus,
"memory_mb": memory,
"workers": int(settings["WEB_CONCURRENCY"]),
"pool": int(settings["DB_POOL_MAX_SIZE"]),
"maxconn": int(settings["API_SERVER_MAXCONN"]),
"api_cpuset": settings["API_CPUSET"],
}
return base, settings
def apply(base, settings, env, out):
env = dict(env, **settings)
for attempt in range(3):
try:
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
break
except subprocess.CalledProcessError as error:
log = Path(out) / "compose-up-failed.txt"
with log.open("a") as handle:
stamp = time.strftime("%FT%TZ", time.gmtime())
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"date: {stamp} commit: {commit}\n")
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
handle.write(error.stdout + error.stderr + "\n")
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
if attempt == 2:
raise
time.sleep(30)
wait_lb(base["replicas"])
return env
def deployment(env):
ids = compose("ps", "-q", env=env).split()
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
result = subprocess.run(
["docker", "inspect", "--format", template, *ids],
check=True,
capture_output=True,
text=True,
)
containers = []
for line in result.stdout.splitlines():
name, image, config = line.split(" ", 2)
limits = json.loads(config)
containers.append(
dict(
name=json.loads(name),
image=json.loads(image),
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
)
)
return containers
def provenance(folder, args, env):
stamp = time.strftime("%FT%TZ", time.gmtime())
metadata = {
"date": stamp,
"commit": args.commit,
"project": args.project,
"command": sys.argv,
"api_image": env.get("API_IMAGE", "compose default"),
"warmup_seconds": args.warmup,
"duration_seconds": args.duration,
"containers": args.deployment,
}
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
for path in folder.rglob("*"):
if path.suffix in {".csv", ".txt"}:
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
elif path.name == "summary.json":
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
path.write_text(json.dumps(data) + "\n")
def attempt_folder(out, base, rate, repeat):
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
attempt = 1
while (Path(out) / f"{name}_attempt{attempt}").exists():
attempt += 1
folder = Path(out) / f"{name}_attempt{attempt}"
folder.mkdir(parents=True)
return folder
def measure(folder, args, rate, env):
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
database = subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
).stdout.split(",")
samplers = [
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
]
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
began = time.time()
try:
code, started = run_k6(folder / "k6", load)
finally:
wall = time.time() - began
for process in samplers:
stop(process)
subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
)
if started is None:
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
return code, started + args.warmup, database, wall
def step(base, rate, repeat, args, env):
folder = attempt_folder(args.out, base, rate, repeat)
code, start, database, wall = measure(folder, args, rate, env)
end = start + args.duration
summary = folder / "k6/summary.json"
metrics = k6_metrics(summary, args.duration)
saturated = (
metrics["failed_ratio"] > args.max_error_ratio
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
or metrics["p95_ms"] > args.slo_p95_ms
)
row = dict(
base,
raw_dir=folder.name,
rate=rate,
repeat=repeat,
k6_exit=code,
saturated=saturated,
**metrics,
)
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
row["db_mb"] = round(int(database[0]) / 2**20, 1)
row["messages"] = int(database[1])
row["p95_post_ms"] = trend(summary, "post_message")
row["p95_get_ms"] = trend(summary, "get_history")
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
row.update(pg_usage(folder / "pg_activity.csv", start, end))
row.update(lb_usage(folder / "lb_stats.csv", start, end))
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
row["wall_s"] = round(wall, 1)
row["disturbed"] = (
row["cpu_other"] > args.max_other
or row["gap_s"] > args.max_gap
or wall > args.warmup + args.duration + args.max_overhead
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
or not rows(folder / "pg_activity.csv", start, end)
or not rows(folder / "lb_stats.csv", start, end)
)
provenance(folder, args, env)
if code or not metrics["requests"]:
raise SystemExit(f"invalid k6 run: {folder}")
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
return row
def rates(args, base):
if args.rates:
return [int(r) for r in args.rates.split(",")]
factor, top = (float(v) for v in args.search.split(":"))
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
while value <= top:
values.append(int(round(value, -1)))
value *= factor
return values
def arguments():
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
parser.add_argument("--out", required=True)
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
parser.add_argument(
"--variant",
dest="variants",
action="append",
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
)
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
parser.add_argument("--rates", help="comma separated offered rates, rps")
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
parser.add_argument("--start-per-cpu", type=float, default=100.0)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--pause", type=float, default=5.0)
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
parser.add_argument("--max-error-ratio", type=float, default=0.01)
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
args = parser.parse_args()
args.variants = args.variants or ["base"]
if len(args.variants) > 1 and not args.rates:
parser.error("several variants need explicit --rates")
return args
def block(args, output, config, variant, repeats):
base, settings = layout(config, args, variant)
current, streak = apply(base, settings, args.base_env, args.out), 0
args.deployment = deployment(current)
for rate in rates(args, base):
results = []
for repeat in repeats:
for _ in range(args.retries + 1):
row = step(base, rate, repeat, args, current)
time.sleep(args.pause)
if not row["disturbed"]:
break
args.rejected.writerow(row)
wait_quiet(args)
else:
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
results.append(row)
output.writerows(results)
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
if streak == 2:
break
def series(args, output):
every = range(args.first_repeat, args.first_repeat + args.repeats)
configs = args.configs.split(",")
if not args.rates:
for config in configs:
block(args, output, config, args.variants[0], every)
return
for repeat in every:
for config in configs:
for variant in args.variants:
block(args, output, config, variant, [repeat])
@contextlib.contextmanager
def table(path):
fresh = not path.exists()
with path.open("a", newline="", buffering=1) as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
if fresh:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
writer.writeheader()
yield writer
def main():
args = arguments()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
args.project = json.loads(compose("config", "--format", "json"))["name"]
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
try:
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
series(args, writer)
finally:
if not args.keep:
compose("up", "-d", "--wait", "--wait-timeout", "180")
return 0
if __name__ == "__main__":
sys.exit(main())