Files
mtusi2026lab3/experiments/lab3-scale/analyze.py
T
2026-10-10 21:32:33 +03:00

392 lines
15 KiB
Python

import argparse
import csv
import gzip
import json
import statistics
from collections import defaultdict
from datetime import datetime
from common import EVIDENCE, write_json
def seconds(value):
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
def diagnostics(directory, manifest):
samples = []
with gzip.open(directory / "telemetry.jsonl.gz", "rt") as file:
samples = [json.loads(line) for line in file]
limits = {row["instance"]: row["cpu"] / 1e9 for row in manifest["containers"]}
previous, cpu, throttled, memory = (
{},
defaultdict(list),
defaultdict(list),
defaultdict(list),
)
queue, active, waits = [], [], defaultdict(int)
for sample in samples:
if "error" in sample:
continue
queue.append(
sum(
int(row["qcur"] or 0)
for row in sample["haproxy"]
if row["svname"].startswith("api")
)
)
active.append(sample["postgres"]["active"])
for row in sample["postgres"]["waits"]:
waits[str(row["event_type"]) + ":" + str(row["event"])] += row["count"]
for resource in sample["resources"]:
name = resource["instance"]
stats = resource["cpu_stats"] or {}
old = previous.get(name)
if old and "system_cpu_usage" in stats and "system_cpu_usage" in old:
system_delta = stats["system_cpu_usage"] - old["system_cpu_usage"]
if system_delta > 0:
delta = (
stats["cpu_usage"]["total_usage"]
- old["cpu_usage"]["total_usage"]
)
cores = delta / system_delta * stats.get("online_cpus", 16)
if delta >= 0:
cpu[resource["service"]].append(cores / limits[name])
current_throttle, old_throttle = (
stats.get("throttling_data", {}),
old.get("throttling_data", {}),
)
periods = current_throttle.get("periods", 0) - old_throttle.get(
"periods", 0
)
if periods > 0:
throttled[resource["service"]].append(
(
current_throttle.get("throttled_periods", 0)
- old_throttle.get("throttled_periods", 0)
)
/ periods
)
previous[name] = stats
memory[resource["service"]].append(
(resource["memory_stats"] or {}).get("usage", 0) / 1048576
)
pools = defaultdict(list)
for row in json.loads(directory.joinpath("pool.json").read_text()):
pools[row["instance"]].append(row)
acquires = duration_ns = empties = 0
for values in pools.values():
if len(values) >= 2:
acquires += values[-1]["acquire_count"] - values[0]["acquire_count"]
duration_ns += (
values[-1]["acquire_duration_ns"] - values[0]["acquire_duration_ns"]
)
empties += (
values[-1]["empty_acquire_count"] - values[0]["empty_acquire_count"]
)
return {
"cpu_fraction_of_limit_mean": {
name: round(statistics.mean(values), 4) for name, values in cpu.items()
},
"throttled_period_fraction_mean": {
name: round(statistics.mean(values), 4)
for name, values in throttled.items()
},
"memory_max_mib": {
name: round(max(values), 3) for name, values in memory.items()
},
"haproxy_max_queue": max(queue, default=0),
"postgres_max_active": max(active, default=0),
"postgres_wait_sample_counts": dict(waits),
"pool_acquire_mean_ms": round(duration_ns / max(acquires, 1) / 1e6, 4),
"pool_empty_acquire_fraction": round(empties / max(acquires, 1), 4),
}
def analyze(series):
directory = EVIDENCE / series
reports, groups = [], defaultdict(list)
for path in sorted(directory.glob("*/summary.json")):
manifest = json.loads(path.with_name("manifest.json").read_text())
summary = json.loads(path.read_text())
diagnostic = diagnostics(path.parent, manifest)
report = {
"run": manifest["run_id"],
"config": manifest["config_name"],
"repeat": manifest["repeat"],
"rate": manifest["rate"],
**summary,
"diagnostics": diagnostic,
}
reports.append(report)
groups[(report["config"], report["repeat"])].append(report)
capacities = []
for (config, repeat), values in sorted(groups.items()):
values.sort(key=lambda row: row["rate"])
passed, failures, invalid = [], [], []
stopped = False
for row in values:
if not row["valid"]:
invalid.append(row["rate"])
if row["slo_pass"] and not stopped:
passed.append(row["rate"])
else:
stopped = True
failures.append(row["rate"])
capacities.append(
{
"config": config,
"repeat": repeat,
"capacity_lower": max(passed, default=0),
"capacity_upper": min(failures) if failures else None,
"invalid_rates": invalid,
"nonmonotonic": any(
row["slo_pass"] and row["rate"] > min(failures) for row in values
)
if failures
else False,
}
)
write_json(directory / "analysis.json", {"runs": reports, "capacities": capacities})
if all(
name in {row["config"] for row in capacities}
for name in ("A1", "A4", "B1", "B4")
):
write_json(directory / "hypotheses.json", hypothesis_decisions(capacities))
with directory.joinpath("summary.csv").open("w", newline="") as file:
fields = [
"config",
"repeat",
"rate",
"goodput_rps",
"success_ratio",
"p95_ms",
"p99_ms",
"dispatch_p95_ms",
"generator_cpu_cores",
"valid",
"slo_pass",
]
writer = csv.DictWriter(file, fieldnames=fields)
writer.writeheader()
for row in reports:
writer.writerow({key: row.get(key) for key in fields})
print(json.dumps(capacities, indent=2))
if reports:
plots(directory, reports, capacities)
return reports, capacities
def plots(directory, reports, capacities):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
colors = {
"A1": "#0072B2",
"A2": "#009E73",
"A4": "#D55E00",
"B1": "#CC79A7",
"B2": "#56B4E9",
"B4": "#E69F00",
}
if not any(row["config"].startswith(("A", "B")) for row in reports):
figure, axes = plt.subplots(1, 3, figsize=(14, 4), layout="constrained")
for ax, metric, label in zip(
axes,
("window_goodput_rps", "p95_ms", "success_ratio"),
(
"Successful responses / s",
"p95 full latency (ms)",
"Successful / scheduled",
),
):
for config in sorted({row["config"] for row in reports}):
grouped = defaultdict(list)
for row in reports:
if row["config"] == config and row.get(metric) is not None:
grouped[row["rate"]].append(row[metric])
rates = sorted(grouped)
ax.plot(
rates,
[statistics.median(grouped[rate]) for rate in rates],
"o-",
label=config,
)
ax.fill_between(
rates,
[min(grouped[rate]) for rate in rates],
[max(grouped[rate]) for rate in rates],
alpha=0.15,
)
ax.set_xlabel("Offered requests / s")
ax.set_ylabel(label)
ax.grid(alpha=0.25)
ax.legend()
figure.savefig(directory / "improvement.png", dpi=160)
plt.close(figure)
return
for metric, label, name in (
(
"window_goodput_rps",
"Successful responses / s (measurement window)",
"throughput",
),
("p95_ms", "p95 full response latency (ms)", "latency"),
("success_ratio", "Successful / scheduled", "success"),
):
figure, axes = plt.subplots(1, 2, figsize=(11, 4), layout="constrained")
for ax, prefix in zip(axes, ("A", "B")):
for config in sorted(
{row["config"] for row in reports if row["config"].startswith(prefix)}
):
grouped = defaultdict(list)
for row in reports:
if row["config"] == config and row.get(metric) is not None:
grouped[row["rate"]].append(row[metric])
rates = sorted(grouped)
median = [statistics.median(grouped[rate]) for rate in rates]
ax.plot(rates, median, "o-", color=colors.get(config), label=config)
ax.fill_between(
rates,
[min(grouped[rate]) for rate in rates],
[max(grouped[rate]) for rate in rates],
alpha=0.15,
color=colors.get(config),
)
ax.set_title(
"Fixed total API resources"
if prefix == "A"
else "Resources increase with replicas"
)
ax.set_xlabel("Offered requests / s")
ax.set_ylabel(label)
ax.grid(alpha=0.25)
ax.legend()
if metric == "p95_ms":
ax.axhline(500, color="black", linestyle="--", linewidth=0.8)
if metric == "success_ratio":
ax.axhline(0.99, color="black", linestyle="--", linewidth=0.8)
figure.savefig(directory / (name + ".png"), dpi=160)
plt.close(figure)
figure, ax = plt.subplots(figsize=(9, 4), layout="constrained")
configs = sorted({row["config"] for row in capacities})
for index, config in enumerate(configs):
values = [
row["capacity_lower"] for row in capacities if row["config"] == config
]
ax.bar(index, statistics.median(values), color=colors.get(config), alpha=0.7)
ax.scatter([index] * len(values), values, color="black", s=22, zorder=3)
ax.set_xticks(range(len(configs)), configs)
ax.set_ylabel("Last contiguous SLO-passing offered rate (req/s)")
ax.set_title("Capacity: medians and all repetitions; finite rate grid")
ax.grid(axis="y", alpha=0.25)
figure.savefig(directory / "capacity.png", dpi=160)
plt.close(figure)
figure, axes = plt.subplots(1, 2, figsize=(11, 4), layout="constrained")
for ax, prefix in zip(axes, ("A", "B")):
for config in sorted(
{row["config"] for row in reports if row["config"].startswith(prefix)}
):
grouped = defaultdict(list)
for row in reports:
if row["config"] == config:
value = row["diagnostics"]["cpu_fraction_of_limit_mean"].get("api")
if value is not None:
grouped[row["rate"]].append(value)
rates = sorted(grouped)
ax.plot(
rates,
[statistics.median(grouped[rate]) for rate in rates],
"o-",
color=colors.get(config),
label=config,
)
ax.axhline(1, color="black", linestyle="--", linewidth=0.8)
ax.set_title("API CPU relative to per-container limit: " + prefix)
ax.set_xlabel("Offered requests / s")
ax.set_ylabel("Mean CPU / configured CPU limit")
ax.grid(alpha=0.25)
ax.legend()
figure.savefig(directory / "api-cpu.png", dpi=160)
plt.close(figure)
def hypothesis_decisions(capacities):
decisions = {}
lookup = {(row["config"], row["repeat"]): row for row in capacities}
for name, prefix, threshold, less in (
("H1", "A", 1.2, True),
("H2", "B", 1.5, False),
):
blocks = []
for repeat in sorted({row["repeat"] for row in capacities}):
one, four = (
lookup.get((prefix + "1", repeat)),
lookup.get((prefix + "4", repeat)),
)
if one is None or four is None:
continue
lower_ratio = (
four["capacity_lower"] / one["capacity_lower"]
if one["capacity_lower"]
else None
)
lower_bound = (
four["capacity_lower"] / one["capacity_upper"]
if one["capacity_upper"]
else None
)
upper_bound = (
four["capacity_upper"] / one["capacity_lower"]
if four["capacity_upper"] and one["capacity_lower"]
else None
)
valid = not (
one["invalid_rates"]
or four["invalid_rates"]
or one["nonmonotonic"]
or four["nonmonotonic"]
)
decision = "inconclusive"
if valid and lower_bound is not None and upper_bound is not None:
if less:
if upper_bound < threshold:
decision = "supported"
elif lower_bound >= threshold:
decision = "refuted"
else:
if lower_bound >= threshold:
decision = "supported"
elif upper_bound < threshold:
decision = "refuted"
blocks.append(
{
"repeat": repeat,
"ratio_of_grid_capacities": lower_ratio,
"ratio_lower_bound": lower_bound,
"ratio_upper_bound": upper_bound,
"decision": decision,
}
)
outcomes = [row["decision"] for row in blocks]
result = (
outcomes[0]
if len(outcomes) == 3 and len(set(outcomes)) == 1
else "inconclusive"
)
decisions[name] = {
"threshold": threshold,
"result": result,
"blocks": blocks,
"note": "Conservative bounds from finite offered-rate grid; not a confidence interval.",
}
return decisions
if __name__ == "__main__":
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("series")
analyze(parser.parse_args().series)