forked from mtusi2026/mtusi2026lab4
Carry source f591fa64e525679a31c9db7f942d36c789e1a12f forward from 72a1e57. Preserve lab4 attachment transactions, reconciliation locks, quarantine, and existing schema data through verified legacy adoption. Keep report and evidence work in progress untouched.
525 lines
19 KiB
Python
Executable File
525 lines
19 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import collections
|
|
import contextlib
|
|
import csv
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import signal
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from probe import ROOT, k6_metrics, lb_up, run_k6
|
|
|
|
SCRIPTS = ROOT / "scripts"
|
|
UNITS = {"B": 1, "KIB": 2**10, "MIB": 2**20, "GIB": 2**30, "KB": 10**3, "MB": 10**6, "GB": 10**9}
|
|
PINNED = {
|
|
"PG_CPUSET": "0-1",
|
|
"LB_CPUSET": "2",
|
|
"VALKEY_CPUSET": "3",
|
|
"K6_CPUSET": "4-6",
|
|
"K6_CPUS": "3",
|
|
"K6_MEMORY": "1g",
|
|
}
|
|
API_FIRST_CPU = 7
|
|
FIELDS = [
|
|
"raw_dir",
|
|
"variant",
|
|
"config",
|
|
"replicas",
|
|
"cpus",
|
|
"memory_mb",
|
|
"workers",
|
|
"pool",
|
|
"maxconn",
|
|
"api_cpuset",
|
|
"rate",
|
|
"repeat",
|
|
"started",
|
|
"db_mb",
|
|
"messages",
|
|
"k6_exit",
|
|
"saturated",
|
|
"requests",
|
|
"rps",
|
|
"ok_rps",
|
|
"failed_ratio",
|
|
"dropped_iterations",
|
|
"p50_ms",
|
|
"p95_ms",
|
|
"p99_ms",
|
|
"p95_post_ms",
|
|
"p95_get_ms",
|
|
"cpu_api",
|
|
"cpu_postgres",
|
|
"cpu_lb",
|
|
"cpu_valkey",
|
|
"cpu_k6",
|
|
"mem_api_mb",
|
|
"mem_postgres_mb",
|
|
"mem_k6_mb",
|
|
"pg_backends_max",
|
|
"pg_active_avg",
|
|
"pg_active_max",
|
|
"pg_idle_avg",
|
|
"pg_idle_tx_avg",
|
|
"pg_lock_avg",
|
|
"pg_lock_max",
|
|
"pg_lwlock_avg",
|
|
"pg_io_avg",
|
|
"lb_scur_max",
|
|
"lb_qcur_max",
|
|
"lb_qtime_max_ms",
|
|
"lb_rtime_avg_ms",
|
|
"cpu_vm",
|
|
"cpu_other",
|
|
"gap_s",
|
|
"wall_s",
|
|
"disturbed",
|
|
]
|
|
|
|
|
|
def compose(*args, env=None):
|
|
command = ["docker", "compose", *args]
|
|
return subprocess.run(
|
|
command, cwd=str(ROOT), env=env, check=True, capture_output=True, text=True
|
|
).stdout
|
|
|
|
|
|
def size(text):
|
|
match = re.match(r"([\d.]+)\s*([A-Za-z]+)", text.strip())
|
|
return float(match.group(1)) * UNITS.get(match.group(2).upper(), 1) if match else 0.0
|
|
|
|
|
|
def rows(path, start, end):
|
|
with path.open() as handle:
|
|
return [r for r in csv.DictReader(handle) if r["ts"] and start <= float(r["ts"]) <= end]
|
|
|
|
|
|
def mean(values):
|
|
return round(statistics.mean(values), 2) if values else 0.0
|
|
|
|
|
|
def docker_usage(path, project, start, end):
|
|
ticks = collections.defaultdict(collections.Counter)
|
|
memory = collections.Counter()
|
|
for r in rows(path, start, end):
|
|
name = re.sub(r"(-run-[0-9a-f]+|-\d+)$", "", r["name"][len(project) + 1 :])
|
|
ticks[r["ts"]][name] += float(r["cpu_pct"].rstrip("%") or 0)
|
|
memory[name] = max(memory[name], size(r["mem_usage"].split("/")[0]) / 2**20)
|
|
cpu = {
|
|
name: mean([t[name] for t in ticks.values() if name in t])
|
|
for name in ("api", "postgres", "lb", "valkey", "k6")
|
|
}
|
|
usage = {f"cpu_{name}": value for name, value in cpu.items()}
|
|
usage.update({f"mem_{name}_mb": round(memory[name], 1) for name in ("api", "postgres", "k6")})
|
|
return usage
|
|
|
|
|
|
def vm_usage(path, start, end):
|
|
samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)]
|
|
if len(samples) < 2 or samples[-1][1] == samples[0][1]:
|
|
return 0.0
|
|
first, last = samples[0], samples[-1]
|
|
return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1)
|
|
|
|
|
|
def gap(path, start, end):
|
|
ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)})
|
|
edges = [start, *[t for t in ticks if start <= t <= end], end]
|
|
return max(b - a for a, b in zip(edges, edges[1:]))
|
|
|
|
|
|
def vm_counters():
|
|
lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines()
|
|
values = [int(v) for v in lines[0].split()[1:]]
|
|
cpus = sum(1 for line in lines if re.match(r"cpu\d", line))
|
|
return sum(values) - values[3] - values[4], sum(values), cpus
|
|
|
|
|
|
def wait_quiet(args):
|
|
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
|
while calm < 3 and time.monotonic() < deadline:
|
|
first = vm_counters()
|
|
time.sleep(5)
|
|
last = vm_counters()
|
|
busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1])
|
|
calm = calm + 1 if busy < args.max_other else 0
|
|
print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True)
|
|
|
|
|
|
def pg_usage(path, start, end):
|
|
ticks = collections.defaultdict(collections.Counter)
|
|
for r in rows(path, start, end):
|
|
count, state, wait = int(r["backends"]), r["state"], r["wait_event_type"]
|
|
tick = ticks[r["ts"]]
|
|
tick["backends"] += count
|
|
tick["active"] += count * (state == "active")
|
|
tick["idle"] += count * (state == "idle")
|
|
tick["idle_tx"] += count * state.startswith("idle in transaction")
|
|
tick["lock"] += count * (wait == "Lock")
|
|
tick["lwlock"] += count * (wait == "LWLock")
|
|
tick["io"] += count * (wait == "IO")
|
|
series = list(ticks.values())
|
|
return {
|
|
"pg_backends_max": max((t["backends"] for t in series), default=0),
|
|
"pg_active_avg": mean([t["active"] for t in series]),
|
|
"pg_active_max": max((t["active"] for t in series), default=0),
|
|
"pg_idle_avg": mean([t["idle"] for t in series]),
|
|
"pg_idle_tx_avg": mean([t["idle_tx"] for t in series]),
|
|
"pg_lock_avg": mean([t["lock"] for t in series]),
|
|
"pg_lock_max": max((t["lock"] for t in series), default=0),
|
|
"pg_lwlock_avg": mean([t["lwlock"] for t in series]),
|
|
"pg_io_avg": mean([t["io"] for t in series]),
|
|
}
|
|
|
|
|
|
def lb_usage(path, start, end):
|
|
backend = [
|
|
r for r in rows(path, start, end) if r["pxname"] == "api" and r["svname"] == "BACKEND"
|
|
]
|
|
|
|
def numbers(field):
|
|
return [int(r[field] or 0) for r in backend]
|
|
|
|
return {
|
|
"lb_scur_max": max(numbers("scur"), default=0),
|
|
"lb_qcur_max": max(numbers("qcur"), default=0),
|
|
"lb_qtime_max_ms": max(numbers("qtime"), default=0),
|
|
"lb_rtime_avg_ms": mean(numbers("rtime")),
|
|
}
|
|
|
|
|
|
def trend(path, name):
|
|
metrics = json.loads(Path(path).read_text())["metrics"]
|
|
found = metrics.get(f"http_req_duration{{scenario:measure,name:{name}}}", {})
|
|
return round(found.get("values", {}).get("p(95)", 0.0), 2)
|
|
|
|
|
|
def sampler(*command):
|
|
return subprocess.Popen([str(SCRIPTS / command[0]), *command[1:]], start_new_session=True)
|
|
|
|
|
|
def stop(process):
|
|
try:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
process.wait()
|
|
|
|
|
|
def wait_lb(expected, timeout=120):
|
|
deadline, stable = time.monotonic() + timeout, 0
|
|
while stable < 4:
|
|
if time.monotonic() > deadline:
|
|
raise SystemExit(f"LB does not settle on {expected} UP servers (now {lb_up()})")
|
|
stable = stable + 1 if lb_up() == expected else 0
|
|
time.sleep(0.5)
|
|
|
|
|
|
def layout(config, args, variant):
|
|
name, _, extra = variant.partition(":")
|
|
replicas, cpus = (float(v) for v in config.split("x"))
|
|
replicas = int(replicas)
|
|
workers = args.workers or max(1, int(cpus))
|
|
width = max(1, math.ceil(replicas * cpus))
|
|
cpuset = str(API_FIRST_CPU) + (f"-{API_FIRST_CPU + width - 1}" if width > 1 else "")
|
|
memory = args.api_memory or args.api_memory_total // replicas
|
|
settings = {
|
|
"API_REPLICAS": str(replicas),
|
|
"API_CPUS": f"{cpus:g}",
|
|
"API_MEMORY": f"{memory}m",
|
|
"WEB_CONCURRENCY": str(workers),
|
|
"DB_POOL_MAX_SIZE": str(args.pool),
|
|
"API_SERVER_MAXCONN": str(args.conn_per_worker * workers),
|
|
"API_CPUSET": cpuset if args.pin else "",
|
|
}
|
|
settings.update(dict(kv.split("=", 1) for kv in [*args.env, *filter(None, extra.split(","))]))
|
|
base = {
|
|
"variant": name,
|
|
"config": config,
|
|
"replicas": replicas,
|
|
"cpus": cpus,
|
|
"memory_mb": memory,
|
|
"workers": int(settings["WEB_CONCURRENCY"]),
|
|
"pool": int(settings["DB_POOL_MAX_SIZE"]),
|
|
"maxconn": int(settings["API_SERVER_MAXCONN"]),
|
|
"api_cpuset": settings["API_CPUSET"],
|
|
}
|
|
return base, settings
|
|
|
|
|
|
def apply(base, settings, env, out):
|
|
env = dict(env, **settings)
|
|
for attempt in range(3):
|
|
try:
|
|
compose("up", "-d", "--wait", "--wait-timeout", "180", env=env)
|
|
break
|
|
except subprocess.CalledProcessError as error:
|
|
log = Path(out) / "compose-up-failed.txt"
|
|
with log.open("a") as handle:
|
|
stamp = time.strftime("%FT%TZ", time.gmtime())
|
|
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
|
handle.write(f"date: {stamp} commit: {commit}\n")
|
|
handle.write(f"== {stamp} {base} attempt {attempt + 1}\n")
|
|
handle.write(error.stdout + error.stderr + "\n")
|
|
print(json.dumps({"compose_up_failed": attempt + 1, "log": str(log)}))
|
|
if attempt == 2:
|
|
raise
|
|
time.sleep(30)
|
|
wait_lb(base["replicas"])
|
|
return env
|
|
|
|
|
|
def deployment(env):
|
|
ids = compose("ps", "-q", env=env).split()
|
|
template = "{{json .Name}} {{json .Image}} {{json .HostConfig}}"
|
|
result = subprocess.run(
|
|
["docker", "inspect", "--format", template, *ids],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
containers = []
|
|
for line in result.stdout.splitlines():
|
|
name, image, config = line.split(" ", 2)
|
|
limits = json.loads(config)
|
|
containers.append(
|
|
dict(
|
|
name=json.loads(name),
|
|
image=json.loads(image),
|
|
**{k: limits[k] for k in ("NanoCpus", "Memory", "CpusetCpus", "PortBindings")},
|
|
)
|
|
)
|
|
return containers
|
|
|
|
|
|
def provenance(folder, args, env):
|
|
stamp = time.strftime("%FT%TZ", time.gmtime())
|
|
metadata = {
|
|
"date": stamp,
|
|
"commit": args.commit,
|
|
"project": args.project,
|
|
"command": sys.argv,
|
|
"api_image": env.get("API_IMAGE", "compose default"),
|
|
"warmup_seconds": args.warmup,
|
|
"duration_seconds": args.duration,
|
|
"containers": args.deployment,
|
|
}
|
|
(folder / "metadata.json").write_text(json.dumps(metadata, ensure_ascii=False) + "\n")
|
|
for path in folder.rglob("*"):
|
|
if path.suffix in {".csv", ".txt"}:
|
|
path.write_text(f"# date: {stamp} commit: {args.commit}\n" + path.read_text())
|
|
elif path.name == "summary.json":
|
|
data = dict(date=stamp, commit=args.commit, **json.loads(path.read_text()))
|
|
path.write_text(json.dumps(data) + "\n")
|
|
|
|
|
|
def attempt_folder(out, base, rate, repeat):
|
|
name = f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}"
|
|
attempt = 1
|
|
while (Path(out) / f"{name}_attempt{attempt}").exists():
|
|
attempt += 1
|
|
folder = Path(out) / f"{name}_attempt{attempt}"
|
|
folder.mkdir(parents=True)
|
|
return folder
|
|
|
|
|
|
def measure(folder, args, rate, env):
|
|
subprocess.run([str(SCRIPTS / "pg-stats.sh"), "reset"], check=True)
|
|
database = subprocess.run(
|
|
[str(SCRIPTS / "pg-stats.sh"), "size"], check=True, capture_output=True, text=True
|
|
).stdout.split(",")
|
|
samplers = [
|
|
sampler("docker-stats.sh", str(folder / "docker_stats.csv")),
|
|
sampler("lb-stats.sh", str(folder / "lb_stats.csv")),
|
|
sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")),
|
|
]
|
|
load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration))
|
|
began = time.time()
|
|
try:
|
|
code, started = run_k6(folder / "k6", load)
|
|
finally:
|
|
wall = time.time() - began
|
|
for process in samplers:
|
|
stop(process)
|
|
subprocess.run(
|
|
[str(SCRIPTS / "pg-stats.sh"), "statements", str(folder / "pg_statements.csv")], check=True
|
|
)
|
|
if started is None:
|
|
raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}")
|
|
return code, started + args.warmup, database, wall
|
|
|
|
|
|
def step(base, rate, repeat, args, env):
|
|
folder = attempt_folder(args.out, base, rate, repeat)
|
|
code, start, database, wall = measure(folder, args, rate, env)
|
|
end = start + args.duration
|
|
summary = folder / "k6/summary.json"
|
|
metrics = k6_metrics(summary, args.duration)
|
|
saturated = (
|
|
metrics["failed_ratio"] > args.max_error_ratio
|
|
or metrics["dropped_iterations"] > args.max_error_ratio * rate * args.duration
|
|
or metrics["p95_ms"] > args.slo_p95_ms
|
|
)
|
|
row = dict(
|
|
base,
|
|
raw_dir=folder.name,
|
|
rate=rate,
|
|
repeat=repeat,
|
|
k6_exit=code,
|
|
saturated=saturated,
|
|
**metrics,
|
|
)
|
|
row["started"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(start))
|
|
row["db_mb"] = round(int(database[0]) / 2**20, 1)
|
|
row["messages"] = int(database[1])
|
|
row["p95_post_ms"] = trend(summary, "post_message")
|
|
row["p95_get_ms"] = trend(summary, "get_history")
|
|
row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end))
|
|
row.update(pg_usage(folder / "pg_activity.csv", start, end))
|
|
row.update(lb_usage(folder / "lb_stats.csv", start, end))
|
|
row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end)
|
|
own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6"))
|
|
row["cpu_other"] = round(row["cpu_vm"] - own, 1)
|
|
row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1)
|
|
row["wall_s"] = round(wall, 1)
|
|
row["disturbed"] = (
|
|
row["cpu_other"] > args.max_other
|
|
or row["gap_s"] > args.max_gap
|
|
or wall > args.warmup + args.duration + args.max_overhead
|
|
or row["cpu_k6"] >= 95 * float(env.get("K6_CPUS", "2"))
|
|
or any(row[f"cpu_{name}"] <= 0 for name in ("vm", "api", "postgres", "k6"))
|
|
or not rows(folder / "pg_activity.csv", start, end)
|
|
or not rows(folder / "lb_stats.csv", start, end)
|
|
)
|
|
provenance(folder, args, env)
|
|
if code or not metrics["requests"]:
|
|
raise SystemExit(f"invalid k6 run: {folder}")
|
|
print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True)
|
|
return row
|
|
|
|
|
|
def rates(args, base):
|
|
if args.rates:
|
|
return [int(r) for r in args.rates.split(",")]
|
|
factor, top = (float(v) for v in args.search.split(":"))
|
|
value, values = args.start_per_cpu * base["replicas"] * base["cpus"], []
|
|
while value <= top:
|
|
values.append(int(round(value, -1)))
|
|
value *= factor
|
|
return values
|
|
|
|
|
|
def arguments():
|
|
parser = argparse.ArgumentParser(description="Track 1: replicas against throughput")
|
|
parser.add_argument("--out", required=True)
|
|
parser.add_argument("--configs", default="1x4,2x2,4x1", help="replicas x cpus per replica")
|
|
parser.add_argument(
|
|
"--variant",
|
|
dest="variants",
|
|
action="append",
|
|
help="NAME[:KEY=VALUE,...]; several variants alternate per repeat",
|
|
)
|
|
parser.add_argument("--api-memory-total", type=int, default=2048, help="MiB for all replicas")
|
|
parser.add_argument("--api-memory", type=int, default=0, help="MiB per replica")
|
|
parser.add_argument("--workers", type=int, default=0, help="gunicorn workers, default = cpus")
|
|
parser.add_argument("--pool", type=int, default=8, help="DB pool per worker")
|
|
parser.add_argument("--conn-per-worker", type=int, default=32, help="LB maxconn per worker")
|
|
parser.add_argument("--env", action="append", default=[], help="extra compose KEY=VALUE")
|
|
parser.add_argument("--no-pin", dest="pin", action="store_false", help="do not pin CPUs")
|
|
parser.add_argument("--rates", help="comma separated offered rates, rps")
|
|
parser.add_argument("--search", default="1.5:8000", help="factor:max, from start-per-cpu")
|
|
parser.add_argument("--start-per-cpu", type=float, default=100.0)
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--first-repeat", type=int, default=1, help="number of the first repeat")
|
|
parser.add_argument("--warmup", type=int, default=10)
|
|
parser.add_argument("--duration", type=int, default=30)
|
|
parser.add_argument("--pause", type=float, default=5.0)
|
|
parser.add_argument("--slo-p95-ms", type=float, default=500.0)
|
|
parser.add_argument("--max-error-ratio", type=float, default=0.01)
|
|
parser.add_argument("--keep", action="store_true", help="leave the last layout running")
|
|
parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core")
|
|
parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s")
|
|
parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s")
|
|
parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step")
|
|
parser.add_argument("--quiet-timeout", type=float, default=1200.0)
|
|
args = parser.parse_args()
|
|
args.variants = args.variants or ["base"]
|
|
if len(args.variants) > 1 and not args.rates:
|
|
parser.error("several variants need explicit --rates")
|
|
return args
|
|
|
|
|
|
def block(args, output, config, variant, repeats):
|
|
base, settings = layout(config, args, variant)
|
|
current, streak = apply(base, settings, args.base_env, args.out), 0
|
|
args.deployment = deployment(current)
|
|
for rate in rates(args, base):
|
|
results = []
|
|
for repeat in repeats:
|
|
for _ in range(args.retries + 1):
|
|
row = step(base, rate, repeat, args, current)
|
|
time.sleep(args.pause)
|
|
if not row["disturbed"]:
|
|
break
|
|
args.rejected.writerow(row)
|
|
wait_quiet(args)
|
|
else:
|
|
raise SystemExit(f"disturbed step exhausted retries: {row['raw_dir']}")
|
|
results.append(row)
|
|
output.writerows(results)
|
|
streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0
|
|
if streak == 2:
|
|
break
|
|
|
|
|
|
def series(args, output):
|
|
every = range(args.first_repeat, args.first_repeat + args.repeats)
|
|
configs = args.configs.split(",")
|
|
if not args.rates:
|
|
for config in configs:
|
|
block(args, output, config, args.variants[0], every)
|
|
return
|
|
for repeat in every:
|
|
for config in configs:
|
|
for variant in args.variants:
|
|
block(args, output, config, variant, [repeat])
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def table(path):
|
|
fresh = not path.exists()
|
|
with path.open("a", newline="", buffering=1) as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
|
|
if fresh:
|
|
commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
|
handle.write(f"# date: {time.strftime('%FT%TZ', time.gmtime())} commit: {commit}\n")
|
|
writer.writeheader()
|
|
yield writer
|
|
|
|
|
|
def main():
|
|
args = arguments()
|
|
out = Path(args.out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
args.project = json.loads(compose("config", "--format", "json"))["name"]
|
|
args.commit = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip()
|
|
args.base_env = dict(os.environ, **(PINNED if args.pin else {}))
|
|
try:
|
|
with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected:
|
|
series(args, writer)
|
|
finally:
|
|
if not args.keep:
|
|
compose("up", "-d", "--wait", "--wait-timeout", "180")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|