Files
mtusi2026lab2/scripts/scaling.py
T
Artyom 12db77133f Forward the lab 5 foundation while preserving lab 6 telemetry
Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.

Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
2026-11-06 12:29:35 +03:00

525 lines
19 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import collections
import contextlib
import csv
import json
import math
import os
import re
import signal
import statistics
import subprocess
import sys
import time
from pathlib import Path
from probe import ROOT, k6_metrics, lb_up, run_k6
SCRIPTS = ROOT / "scripts"
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
PINNED = {
"PG_CPUSET": "0-1",
"LB_CPUSET": "2",
"VALKEY_CPUSET": "3",
"K6_CPUSET": "4-6",
"K6_CPUS": "3",
"K6_MEMORY": "1g",
}
API_FIRST_CPU = 7
FIELDS = [
"raw_dir",
"variant",
"config",
"replicas",
"cpus",
"memory_mb",
"workers",
"pool",
"maxconn",
"api_cpuset",
"rate",
"repeat",
"started",
"db_mb",
"messages",
"k6_exit",
"saturated",
"requests",
"rps",
"ok_rps",
"failed_ratio",
"dropped_iterations",
"p50_ms",
"p95_ms",
"p99_ms",
"p95_post_ms",
"p95_get_ms",
"cpu_api",
"cpu_postgres",
"cpu_lb",
"cpu_valkey",
"cpu_k6",
"mem_api_mb",
"mem_postgres_mb",
"mem_k6_mb",
"pg_backends_max",
"pg_active_avg",
"pg_active_max",
"pg_idle_avg",
"pg_idle_tx_avg",
"pg_lock_avg",
"pg_lock_max",
"pg_lwlock_avg",
"pg_io_avg",
"lb_scur_max",
"lb_qcur_max",
"lb_qtime_max_ms",
"lb_rtime_avg_ms",
"cpu_vm",
"cpu_other",
"gap_s",
"wall_s",
"disturbed",
]
def compose(*args, env=None):
command = ["docker", "compose", *args]
return subprocess.run(
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
).stdout
def size(text):
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
def rows(path, start, end):
with path.open() as handle:
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
def mean(values):
return round(statistics.mean(values), 2) if values else 0.0
def docker_usage(path, project, start, end):
ticks = collections.defaultdict(collections.Counter)
memory = collections.Counter()
for r in rows(path, start, end):
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
cpu = {
name: mean([t[name] for t in ticks.values() if name in t])
for name in ("api", "postgres", "lb", "valkey", "k6")
}
usage = {f"cpu_{name}": value for name, value in cpu.items()}
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
return usage
def vm_usage(path, start, end):
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
return 0.0
first, last = samples[0], samples[-1]
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
def gap(path, start, end):
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
edges = [start, *[t for t in ticks if start <= t <= end], end]
return max(b - a for a, b in zip(edges, edges[1:]))
def vm_counters():
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
values = [int(v) for v in lines[0].split()[1:]]
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
return sum(values) - values[3] - values[4], sum(values), cpus
def wait_quiet(args):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = vm_counters()
time.sleep(5)
last = vm_counters()
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
calm = calm + 1 if busy < args.max_other else 0
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
def pg_usage(path, start, end):
ticks = collections.defaultdict(collections.Counter)
for r in rows(path, start, end):
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
tick = ticks[r["ts"]]
tick["backends"] += count
tick["active"] += count * (state == "active")
tick["idle"] += count * (state == "idle")
tick["idle_tx"] += count * state.startswith("idle in transaction")
tick["lock"] += count * (wait == "Lock")
tick["lwlock"] += count * (wait == "LWLock")
tick["io"] += count * (wait == "IO")
series = list(ticks.values())
return {
"pg_backends_max": max((t["backends"] for t in series), default=0),
"pg_active_avg": mean([t["active"] for t in series]),
"pg_active_max": max((t["active"] for t in series), default=0),
"pg_idle_avg": mean([t["idle"] for t in series]),
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
"pg_lock_avg": mean([t["lock"] for t in series]),
"pg_lock_max": max((t["lock"] for t in series), default=0),
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
"pg_io_avg": mean([t["io"] for t in series]),
}
def lb_usage(path, start, end):
backend = [
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
]
def numbers(field):
return [int(r[field] or 0) for r in backend]
return {
"lb_scur_max": max(numbers("scur"), default=0),
"lb_qcur_max": max(numbers("qcur"), default=0),
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
"lb_rtime_avg_ms": mean(numbers("rtime")),
}
def trend(path, name):
metrics = json.loads(Path(path).read_text())["metrics"]
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
return round(found.get("values", {}).get("p(95)", 0.0), 2)
def sampler(*command):
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
def stop(process):
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
process.wait()
def wait_lb(expected, timeout=120):
deadline, stable = time.monotonic() + timeout, 0
while stable < 4:
if time.monotonic() > deadline:
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
stable = stable + 1 if lb_up() == expected else 0
time.sleep(0.5)
def layout(config, args, variant):
name, _, extra = variant.partition(":")
replicas, cpus = (float(v) for v in config.split("x"))
replicas = int(replicas)
workers = args.workers or max(1, int(cpus))
width = max(1, math.ceil(replicas * cpus))
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
memory = args.api_memory or args.api_memory_total // replicas
settings = {
"API_REPLICAS": str(replicas),
"API_CPUS": f"{cpus:g}",
"API_MEMORY": f"{memory}m",
"WEB_CONCURRENCY": str(workers),
"DB_POOL_MAX_SIZE": str(args.pool),
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
"API_CPUSET": cpuset if args.pin else "",
}
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
base = {
"variant": name,
"config": config,
"replicas": replicas,
"cpus": cpus,
"memory_mb": memory,
"workers": int(settings["WEB_CONCURRENCY"]),
"pool": int(settings["DB_POOL_MAX_SIZE"]),
"maxconn": int(settings["API_SERVER_MAXCONN"]),
"api_cpuset": settings["API_CPUSET"],
}
return base, settings
def apply(base, settings, env, out):
env = dict(env, **settings)
for attempt in range(3):
try:
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
break
except subprocess.CalledProcessError as error:
log = Path(out) / "compose-up-failed.txt"
with log.open("a") as handle:
stamp = time.strftime("%FT%TZ", time.gmtime())
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"date: {stamp} commit: {commit}\n")
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
handle.write(error.stdout + error.stderr + "\n")
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
if attempt == 2:
raise
time.sleep(30)
wait_lb(base["replicas"])
return env
def deployment(env):
ids = compose("ps", "-q", env=env).split()
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
result = subprocess.run(
["docker", "inspect", "--format", template, *ids],
check=True,
capture_output=True,
text=True,
)
containers = []
for line in result.stdout.splitlines():
name, image, config = line.split(" ", 2)
limits = json.loads(config)
containers.append(
dict(
name=json.loads(name),
image=json.loads(image),
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
)
)
return containers
def provenance(folder, args, env):
stamp = time.strftime("%FT%TZ", time.gmtime())
metadata = {
"date": stamp,
"commit": args.commit,
"project": args.project,
"command": sys.argv,
"api_image": env.get("API_IMAGE", "compose default"),
"warmup_seconds": args.warmup,
"duration_seconds": args.duration,
"containers": args.deployment,
}
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
for path in folder.rglob("*"):
if path.suffix in {".csv", ".txt"}:
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
elif path.name == "summary.json":
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
path.write_text(json.dumps(data) + "\n")
def attempt_folder(out, base, rate, repeat):
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
attempt = 1
while (Path(out) / f"{name}_attempt{attempt}").exists():
attempt += 1
folder = Path(out) / f"{name}_attempt{attempt}"
folder.mkdir(parents=True)
return folder
def measure(folder, args, rate, env):
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
database = subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
).stdout.split(",")
samplers = [
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
]
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
began = time.time()
try:
code, started = run_k6(folder / "k6", load)
finally:
wall = time.time() - began
for process in samplers:
stop(process)
subprocess.run(
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
)
if started is None:
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
return code, started + args.warmup, database, wall
def step(base, rate, repeat, args, env):
folder = attempt_folder(args.out, base, rate, repeat)
code, start, database, wall = measure(folder, args, rate, env)
end = start + args.duration
summary = folder / "k6/summary.json"
metrics = k6_metrics(summary, args.duration)
saturated = (
metrics["failed_ratio"] > args.max_error_ratio
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
or metrics["p95_ms"] > args.slo_p95_ms
)
row = dict(
base,
raw_dir=folder.name,
rate=rate,
repeat=repeat,
k6_exit=code,
saturated=saturated,
**metrics,
)
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
row["db_mb"] = round(int(database[0]) / 2**20, 1)
row["messages"] = int(database[1])
row["p95_post_ms"] = trend(summary, "post_message")
row["p95_get_ms"] = trend(summary, "get_history")
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
row.update(pg_usage(folder / "pg_activity.csv", start, end))
row.update(lb_usage(folder / "lb_stats.csv", start, end))
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
row["wall_s"] = round(wall, 1)
row["disturbed"] = (
row["cpu_other"] > args.max_other
or row["gap_s"] > args.max_gap
or wall > args.warmup + args.duration + args.max_overhead
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
or not rows(folder / "pg_activity.csv", start, end)
or not rows(folder / "lb_stats.csv", start, end)
)
provenance(folder, args, env)
if code or not metrics["requests"]:
raise SystemExit(f"invalid k6 run: {folder}")
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
return row
def rates(args, base):
if args.rates:
return [int(r) for r in args.rates.split(",")]
factor, top = (float(v) for v in args.search.split(":"))
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
while value <= top:
values.append(int(round(value, -1)))
value *= factor
return values
def arguments():
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
parser.add_argument("--out", required=True)
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
parser.add_argument(
"--variant",
dest="variants",
action="append",
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
)
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
parser.add_argument("--rates", help="comma separated offered rates, rps")
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
parser.add_argument("--start-per-cpu", type=float, default=100.0)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
parser.add_argument("--warmup", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--pause", type=float, default=5.0)
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
parser.add_argument("--max-error-ratio", type=float, default=0.01)
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
args = parser.parse_args()
args.variants = args.variants or ["base"]
if len(args.variants) > 1 and not args.rates:
parser.error("several variants need explicit --rates")
return args
def block(args, output, config, variant, repeats):
base, settings = layout(config, args, variant)
current, streak = apply(base, settings, args.base_env, args.out), 0
args.deployment = deployment(current)
for rate in rates(args, base):
results = []
for repeat in repeats:
for _ in range(args.retries + 1):
row = step(base, rate, repeat, args, current)
time.sleep(args.pause)
if not row["disturbed"]:
break
args.rejected.writerow(row)
wait_quiet(args)
else:
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
results.append(row)
output.writerows(results)
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
if streak == 2:
break
def series(args, output):
every = range(args.first_repeat, args.first_repeat + args.repeats)
configs = args.configs.split(",")
if not args.rates:
for config in configs:
block(args, output, config, args.variants[0], every)
return
for repeat in every:
for config in configs:
for variant in args.variants:
block(args, output, config, variant, [repeat])
@contextlib.contextmanager
def table(path):
fresh = not path.exists()
with path.open("a", newline="", buffering=1) as handle:
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
if fresh:
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
writer.writeheader()
yield writer
def main():
args = arguments()
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
args.project = json.loads(compose("config", "--format", "json"))["name"]
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
try:
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
series(args, writer)
finally:
if not args.keep:
compose("up", "-d", "--wait", "--wait-timeout", "180")
return 0
if __name__ == "__main__":
sys.exit(main())