forked from mtusi2026/mtusi2026lab3
475 lines
18 KiB
Python
Executable File
475 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import collections
|
|
import contextlib
|
|
import csv
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import signal
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from probe import ROOT, k6_metrics, lb_up, run_k6
|
|
|
|
SCRIPTS = ROOT / "scripts"
|
|
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
|
|
PINNED = {
|
|
"PG_CPUSET": "0-1",
|
|
"LB_CPUSET": "2",
|
|
"VALKEY_CPUSET": "3",
|
|
"K6_CPUSET": "4-6",
|
|
"K6_CPUS": "3",
|
|
"K6_MEMORY": "1g",
|
|
}
|
|
API_FIRST_CPU = 7
|
|
FIELDS = (
|
|
"raw_dir variant config replicas cpus memory_mb workers pool maxconn api_cpuset rate repeat "
|
|
"started db_mb messages k6_exit saturated requests rps ok_rps failed_ratio "
|
|
"dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres "
|
|
"cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max "
|
|
"pg_active_avg pg_active_max pg_idle_avg pg_idle_tx_avg pg_lock_avg pg_lock_max pg_lwlock_avg "
|
|
"pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms cpu_vm cpu_other "
|
|
"gap_s wall_s disturbed"
|
|
).split()
|
|
|
|
|
|
def compose(*args, env=None):
|
|
command = ["docker", "compose", *args]
|
|
return subprocess.run(
|
|
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
|
|
).stdout
|
|
|
|
|
|
def size(text):
|
|
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
|
|
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
|
|
|
|
|
|
def rows(path, start, end):
|
|
with path.open() as handle:
|
|
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
|
|
|
|
|
|
def mean(values):
|
|
return round(statistics.mean(values), 2) if values else 0.0
|
|
|
|
|
|
def docker_usage(path, project, start, end):
|
|
ticks = collections.defaultdict(collections.Counter)
|
|
memory = collections.Counter()
|
|
for r in rows(path, start, end):
|
|
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
|
|
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
|
|
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
|
|
cpu = {
|
|
name: mean([t[name] for t in ticks.values() if name in t])
|
|
for name in ("api", "postgres", "lb", "valkey", "k6")
|
|
}
|
|
usage = {f"cpu_{name}": value for name, value in cpu.items()}
|
|
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
|
|
return usage
|
|
|
|
|
|
def vm_usage(path, start, end):
|
|
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
|
|
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
|
|
return 0.0
|
|
first, last = samples[0], samples[-1]
|
|
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
|
|
|
|
|
|
def gap(path, start, end):
|
|
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
|
|
edges = [start, *[t for t in ticks if start <= t <= end], end]
|
|
return max(b - a for a, b in zip(edges, edges[1:]))
|
|
|
|
|
|
def vm_counters():
|
|
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
|
|
values = [int(v) for v in lines[0].split()[1:]]
|
|
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
|
|
return sum(values) - values[3] - values[4], sum(values), cpus
|
|
|
|
|
|
def wait_quiet(args):
|
|
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
|
while calm < 3 and time.monotonic() < deadline:
|
|
first = vm_counters()
|
|
time.sleep(5)
|
|
last = vm_counters()
|
|
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
|
|
calm = calm + 1 if busy < args.max_other else 0
|
|
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
|
|
|
|
|
|
def pg_usage(path, start, end):
|
|
ticks = collections.defaultdict(collections.Counter)
|
|
for r in rows(path, start, end):
|
|
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
|
|
tick = ticks[r["ts"]]
|
|
tick["backends"] += count
|
|
tick["active"] += count * (state == "active")
|
|
tick["idle"] += count * (state == "idle")
|
|
tick["idle_tx"] += count * state.startswith("idle in transaction")
|
|
tick["lock"] += count * (wait == "Lock")
|
|
tick["lwlock"] += count * (wait == "LWLock")
|
|
tick["io"] += count * (wait == "IO")
|
|
series = list(ticks.values())
|
|
return {
|
|
"pg_backends_max": max((t["backends"] for t in series), default=0),
|
|
"pg_active_avg": mean([t["active"] for t in series]),
|
|
"pg_active_max": max((t["active"] for t in series), default=0),
|
|
"pg_idle_avg": mean([t["idle"] for t in series]),
|
|
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
|
|
"pg_lock_avg": mean([t["lock"] for t in series]),
|
|
"pg_lock_max": max((t["lock"] for t in series), default=0),
|
|
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
|
|
"pg_io_avg": mean([t["io"] for t in series]),
|
|
}
|
|
|
|
|
|
def lb_usage(path, start, end):
|
|
backend = [
|
|
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
|
|
]
|
|
|
|
def numbers(field):
|
|
return [int(r[field] or 0) for r in backend]
|
|
|
|
return {
|
|
"lb_scur_max": max(numbers("scur"), default=0),
|
|
"lb_qcur_max": max(numbers("qcur"), default=0),
|
|
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
|
|
"lb_rtime_avg_ms": mean(numbers("rtime")),
|
|
}
|
|
|
|
|
|
def trend(path, name):
|
|
metrics = json.loads(Path(path).read_text())["metrics"]
|
|
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
|
|
return round(found.get("values", {}).get("p(95)", 0.0), 2)
|
|
|
|
|
|
def sampler(*command):
|
|
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
|
|
|
|
|
|
def stop(process):
|
|
try:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
process.wait()
|
|
|
|
|
|
def wait_lb(expected, timeout=120):
|
|
deadline, stable = time.monotonic() + timeout, 0
|
|
while stable < 4:
|
|
if time.monotonic() > deadline:
|
|
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
|
|
stable = stable + 1 if lb_up() == expected else 0
|
|
time.sleep(0.5)
|
|
|
|
|
|
def layout(config, args, variant):
|
|
name, _, extra = variant.partition(":")
|
|
replicas, cpus = (float(v) for v in config.split("x"))
|
|
replicas = int(replicas)
|
|
workers = args.workers or max(1, int(cpus))
|
|
width = max(1, math.ceil(replicas * cpus))
|
|
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
|
|
memory = args.api_memory or args.api_memory_total // replicas
|
|
settings = {
|
|
"API_REPLICAS": str(replicas),
|
|
"API_CPUS": f"{cpus:g}",
|
|
"API_MEMORY": f"{memory}m",
|
|
"WEB_CONCURRENCY": str(workers),
|
|
"DB_POOL_MAX_SIZE": str(args.pool),
|
|
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
|
|
"API_CPUSET": cpuset if args.pin else "",
|
|
}
|
|
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
|
|
base = {
|
|
"variant": name,
|
|
"config": config,
|
|
"replicas": replicas,
|
|
"cpus": cpus,
|
|
"memory_mb": memory,
|
|
"workers": int(settings["WEB_CONCURRENCY"]),
|
|
"pool": int(settings["DB_POOL_MAX_SIZE"]),
|
|
"maxconn": int(settings["API_SERVER_MAXCONN"]),
|
|
"api_cpuset": settings["API_CPUSET"],
|
|
}
|
|
return base, settings
|
|
|
|
|
|
def apply(base, settings, env, out):
|
|
env = dict(env, **settings)
|
|
for attempt in range(3):
|
|
try:
|
|
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
|
|
break
|
|
except subprocess.CalledProcessError as error:
|
|
log = Path(out) / "compose-up-failed.log"
|
|
with log.open("a") as handle:
|
|
stamp = time.strftime("%FT%TZ", time.gmtime())
|
|
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
|
|
handle.write(error.stdout + error.stderr + "\n")
|
|
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
|
|
if attempt == 2:
|
|
raise
|
|
time.sleep(30)
|
|
wait_lb(base["replicas"])
|
|
return env
|
|
|
|
|
|
def deployment(env):
|
|
ids = compose("ps", "-q", env=env).split()
|
|
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
|
|
result = subprocess.run(
|
|
["docker", "inspect", "--format", template, *ids],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
containers = []
|
|
for line in result.stdout.splitlines():
|
|
name, image, config = line.split(" ", 2)
|
|
limits = json.loads(config)
|
|
containers.append(
|
|
dict(
|
|
name=json.loads(name),
|
|
image=json.loads(image),
|
|
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
|
|
)
|
|
)
|
|
return containers
|
|
|
|
|
|
def provenance(folder, args, env):
|
|
stamp = time.strftime("%FT%TZ", time.gmtime())
|
|
metadata = {
|
|
"date": stamp,
|
|
"commit": args.commit,
|
|
"project": args.project,
|
|
"command": sys.argv,
|
|
"api_image": env.get("API_IMAGE", "compose default"),
|
|
"warmup_seconds": args.warmup,
|
|
"duration_seconds": args.duration,
|
|
"containers": args.deployment,
|
|
}
|
|
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
|
|
for path in folder.rglob("*"):
|
|
if path.suffix in {".csv", ".txt"}:
|
|
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
|
|
elif path.name == "summary.json":
|
|
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
|
|
path.write_text(json.dumps(data) + "\n")
|
|
|
|
|
|
def attempt_folder(out, base, rate, repeat):
|
|
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
|
attempt = 1
|
|
while (Path(out) / f"{name}_attempt{attempt}").exists():
|
|
attempt += 1
|
|
folder = Path(out) / f"{name}_attempt{attempt}"
|
|
folder.mkdir(parents=True)
|
|
return folder
|
|
|
|
|
|
def measure(folder, args, rate, env):
|
|
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
|
|
database = subprocess.run(
|
|
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
|
|
).stdout.split(",")
|
|
samplers = [
|
|
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
|
|
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
|
|
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
|
|
]
|
|
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
|
|
began = time.time()
|
|
try:
|
|
code, started = run_k6(folder / "k6", load)
|
|
finally:
|
|
wall = time.time() - began
|
|
for process in samplers:
|
|
stop(process)
|
|
subprocess.run(
|
|
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
|
|
)
|
|
if started is None:
|
|
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
|
|
return code, started + args.warmup, database, wall
|
|
|
|
|
|
def step(base, rate, repeat, args, env):
|
|
folder = attempt_folder(args.out, base, rate, repeat)
|
|
code, start, database, wall = measure(folder, args, rate, env)
|
|
end = start + args.duration
|
|
summary = folder / "k6/summary.json"
|
|
metrics = k6_metrics(summary, args.duration)
|
|
saturated = (
|
|
metrics["failed_ratio"] > args.max_error_ratio
|
|
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
|
|
or metrics["p95_ms"] > args.slo_p95_ms
|
|
)
|
|
row = dict(
|
|
base,
|
|
raw_dir=folder.name,
|
|
rate=rate,
|
|
repeat=repeat,
|
|
k6_exit=code,
|
|
saturated=saturated,
|
|
**metrics,
|
|
)
|
|
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
|
|
row["db_mb"] = round(int(database[0]) / 2**20, 1)
|
|
row["messages"] = int(database[1])
|
|
row["p95_post_ms"] = trend(summary, "post_message")
|
|
row["p95_get_ms"] = trend(summary, "get_history")
|
|
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
|
|
row.update(pg_usage(folder / "pg_activity.csv", start, end))
|
|
row.update(lb_usage(folder / "lb_stats.csv", start, end))
|
|
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
|
|
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
|
|
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
|
|
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
|
|
row["wall_s"] = round(wall, 1)
|
|
row["disturbed"] = (
|
|
row["cpu_other"] > args.max_other
|
|
or row["gap_s"] > args.max_gap
|
|
or wall > args.warmup + args.duration + args.max_overhead
|
|
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
|
|
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
|
|
or not rows(folder / "pg_activity.csv", start, end)
|
|
or not rows(folder / "lb_stats.csv", start, end)
|
|
)
|
|
provenance(folder, args, env)
|
|
if code or not metrics["requests"]:
|
|
raise SystemExit(f"invalid k6 run: {folder}")
|
|
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
|
return row
|
|
|
|
|
|
def rates(args, base):
|
|
if args.rates:
|
|
return [int(r) for r in args.rates.split(",")]
|
|
factor, top = (float(v) for v in args.search.split(":"))
|
|
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
|
|
while value <= top:
|
|
values.append(int(round(value, -1)))
|
|
value *= factor
|
|
return values
|
|
|
|
|
|
def arguments():
|
|
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
|
|
parser.add_argument(
|
|
"--variant",
|
|
dest="variants",
|
|
action="append",
|
|
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
|
|
)
|
|
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
|
|
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
|
|
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
|
|
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
|
|
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
|
|
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
|
|
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
|
|
parser.add_argument("--rates", help="comma separated offered rates, rps")
|
|
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
|
|
parser.add_argument("--start-per-cpu", type=float, default=100.0)
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
|
|
parser.add_argument("--warmup", type=int, default=10)
|
|
parser.add_argument("--duration", type=int, default=30)
|
|
parser.add_argument("--pause", type=float, default=5.0)
|
|
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
|
|
parser.add_argument("--max-error-ratio", type=float, default=0.01)
|
|
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
|
|
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
|
|
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
|
|
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
|
|
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
|
|
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
|
|
args = parser.parse_args()
|
|
args.variants = args.variants or ["base"]
|
|
if len(args.variants) > 1 and not args.rates:
|
|
parser.error("several variants need explicit --rates")
|
|
return args
|
|
|
|
|
|
def block(args, output, config, variant, repeats):
|
|
base, settings = layout(config, args, variant)
|
|
current, streak = apply(base, settings, args.base_env, args.out), 0
|
|
args.deployment = deployment(current)
|
|
for rate in rates(args, base):
|
|
results = []
|
|
for repeat in repeats:
|
|
for _ in range(args.retries + 1):
|
|
row = step(base, rate, repeat, args, current)
|
|
time.sleep(args.pause)
|
|
if not row["disturbed"]:
|
|
break
|
|
args.rejected.writerow(row)
|
|
wait_quiet(args)
|
|
else:
|
|
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
|
|
results.append(row)
|
|
output.writerows(results)
|
|
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
|
if streak == 2:
|
|
break
|
|
|
|
|
|
def series(args, output):
|
|
every = range(args.first_repeat, args.first_repeat + args.repeats)
|
|
for config in args.configs.split(","):
|
|
if len(args.variants) == 1:
|
|
block(args, output, config, args.variants[0], every)
|
|
continue
|
|
for repeat in every:
|
|
for variant in args.variants:
|
|
block(args, output, config, variant, [repeat])
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def table(path):
|
|
fresh = not path.exists()
|
|
with path.open("a", newline="", buffering=1) as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
|
if fresh:
|
|
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
|
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
|
|
writer.writeheader()
|
|
yield writer
|
|
|
|
|
|
def main():
|
|
args = arguments()
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
|
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
|
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
|
try:
|
|
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
|
|
series(args, writer)
|
|
finally:
|
|
if not args.keep:
|
|
compose("up", "-d", "--wait", "--wait-timeout", "180")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|