forked from mtusi2026/mtusi2026lab3
392 lines
15 KiB
Python
392 lines
15 KiB
Python
import argparse
|
|
import csv
|
|
import gzip
|
|
import json
|
|
import statistics
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
|
|
from common import EVIDENCE, write_json
|
|
|
|
|
|
def seconds(value):
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
|
|
|
|
|
def diagnostics(directory, manifest):
|
|
samples = []
|
|
with gzip.open(directory / "telemetry.jsonl.gz", "rt") as file:
|
|
samples = [json.loads(line) for line in file]
|
|
limits = {row["instance"]: row["cpu"] / 1e9 for row in manifest["containers"]}
|
|
previous, cpu, throttled, memory = (
|
|
{},
|
|
defaultdict(list),
|
|
defaultdict(list),
|
|
defaultdict(list),
|
|
)
|
|
queue, active, waits = [], [], defaultdict(int)
|
|
for sample in samples:
|
|
if "error" in sample:
|
|
continue
|
|
queue.append(
|
|
sum(
|
|
int(row["qcur"] or 0)
|
|
for row in sample["haproxy"]
|
|
if row["svname"].startswith("api")
|
|
)
|
|
)
|
|
active.append(sample["postgres"]["active"])
|
|
for row in sample["postgres"]["waits"]:
|
|
waits[str(row["event_type"]) + ":" + str(row["event"])] += row["count"]
|
|
for resource in sample["resources"]:
|
|
name = resource["instance"]
|
|
stats = resource["cpu_stats"] or {}
|
|
old = previous.get(name)
|
|
if old and "system_cpu_usage" in stats and "system_cpu_usage" in old:
|
|
system_delta = stats["system_cpu_usage"] - old["system_cpu_usage"]
|
|
if system_delta > 0:
|
|
delta = (
|
|
stats["cpu_usage"]["total_usage"]
|
|
- old["cpu_usage"]["total_usage"]
|
|
)
|
|
cores = delta / system_delta * stats.get("online_cpus", 16)
|
|
if delta >= 0:
|
|
cpu[resource["service"]].append(cores / limits[name])
|
|
current_throttle, old_throttle = (
|
|
stats.get("throttling_data", {}),
|
|
old.get("throttling_data", {}),
|
|
)
|
|
periods = current_throttle.get("periods", 0) - old_throttle.get(
|
|
"periods", 0
|
|
)
|
|
if periods > 0:
|
|
throttled[resource["service"]].append(
|
|
(
|
|
current_throttle.get("throttled_periods", 0)
|
|
- old_throttle.get("throttled_periods", 0)
|
|
)
|
|
/ periods
|
|
)
|
|
previous[name] = stats
|
|
memory[resource["service"]].append(
|
|
(resource["memory_stats"] or {}).get("usage", 0) / 1048576
|
|
)
|
|
pools = defaultdict(list)
|
|
for row in json.loads(directory.joinpath("pool.json").read_text()):
|
|
pools[row["instance"]].append(row)
|
|
acquires = duration_ns = empties = 0
|
|
for values in pools.values():
|
|
if len(values) >= 2:
|
|
acquires += values[-1]["acquire_count"] - values[0]["acquire_count"]
|
|
duration_ns += (
|
|
values[-1]["acquire_duration_ns"] - values[0]["acquire_duration_ns"]
|
|
)
|
|
empties += (
|
|
values[-1]["empty_acquire_count"] - values[0]["empty_acquire_count"]
|
|
)
|
|
return {
|
|
"cpu_fraction_of_limit_mean": {
|
|
name: round(statistics.mean(values), 4) for name, values in cpu.items()
|
|
},
|
|
"throttled_period_fraction_mean": {
|
|
name: round(statistics.mean(values), 4)
|
|
for name, values in throttled.items()
|
|
},
|
|
"memory_max_mib": {
|
|
name: round(max(values), 3) for name, values in memory.items()
|
|
},
|
|
"haproxy_max_queue": max(queue, default=0),
|
|
"postgres_max_active": max(active, default=0),
|
|
"postgres_wait_sample_counts": dict(waits),
|
|
"pool_acquire_mean_ms": round(duration_ns / max(acquires, 1) / 1e6, 4),
|
|
"pool_empty_acquire_fraction": round(empties / max(acquires, 1), 4),
|
|
}
|
|
|
|
|
|
def analyze(series):
|
|
directory = EVIDENCE / series
|
|
reports, groups = [], defaultdict(list)
|
|
for path in sorted(directory.glob("*/summary.json")):
|
|
manifest = json.loads(path.with_name("manifest.json").read_text())
|
|
summary = json.loads(path.read_text())
|
|
diagnostic = diagnostics(path.parent, manifest)
|
|
report = {
|
|
"run": manifest["run_id"],
|
|
"config": manifest["config_name"],
|
|
"repeat": manifest["repeat"],
|
|
"rate": manifest["rate"],
|
|
**summary,
|
|
"diagnostics": diagnostic,
|
|
}
|
|
reports.append(report)
|
|
groups[(report["config"], report["repeat"])].append(report)
|
|
capacities = []
|
|
for (config, repeat), values in sorted(groups.items()):
|
|
values.sort(key=lambda row: row["rate"])
|
|
passed, failures, invalid = [], [], []
|
|
stopped = False
|
|
for row in values:
|
|
if not row["valid"]:
|
|
invalid.append(row["rate"])
|
|
if row["slo_pass"] and not stopped:
|
|
passed.append(row["rate"])
|
|
else:
|
|
stopped = True
|
|
failures.append(row["rate"])
|
|
capacities.append(
|
|
{
|
|
"config": config,
|
|
"repeat": repeat,
|
|
"capacity_lower": max(passed, default=0),
|
|
"capacity_upper": min(failures) if failures else None,
|
|
"invalid_rates": invalid,
|
|
"nonmonotonic": any(
|
|
row["slo_pass"] and row["rate"] > min(failures) for row in values
|
|
)
|
|
if failures
|
|
else False,
|
|
}
|
|
)
|
|
write_json(directory / "analysis.json", {"runs": reports, "capacities": capacities})
|
|
if all(
|
|
name in {row["config"] for row in capacities}
|
|
for name in ("A1", "A4", "B1", "B4")
|
|
):
|
|
write_json(directory / "hypotheses.json", hypothesis_decisions(capacities))
|
|
with directory.joinpath("summary.csv").open("w", newline="") as file:
|
|
fields = [
|
|
"config",
|
|
"repeat",
|
|
"rate",
|
|
"goodput_rps",
|
|
"success_ratio",
|
|
"p95_ms",
|
|
"p99_ms",
|
|
"dispatch_p95_ms",
|
|
"generator_cpu_cores",
|
|
"valid",
|
|
"slo_pass",
|
|
]
|
|
writer = csv.DictWriter(file, fieldnames=fields)
|
|
writer.writeheader()
|
|
for row in reports:
|
|
writer.writerow({key: row.get(key) for key in fields})
|
|
print(json.dumps(capacities, indent=2))
|
|
if reports:
|
|
plots(directory, reports, capacities)
|
|
return reports, capacities
|
|
|
|
|
|
def plots(directory, reports, capacities):
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
|
|
colors = {
|
|
"A1": "#0072B2",
|
|
"A2": "#009E73",
|
|
"A4": "#D55E00",
|
|
"B1": "#CC79A7",
|
|
"B2": "#56B4E9",
|
|
"B4": "#E69F00",
|
|
}
|
|
if not any(row["config"].startswith(("A", "B")) for row in reports):
|
|
figure, axes = plt.subplots(1, 3, figsize=(14, 4), layout="constrained")
|
|
for ax, metric, label in zip(
|
|
axes,
|
|
("window_goodput_rps", "p95_ms", "success_ratio"),
|
|
(
|
|
"Successful responses / s",
|
|
"p95 full latency (ms)",
|
|
"Successful / scheduled",
|
|
),
|
|
):
|
|
for config in sorted({row["config"] for row in reports}):
|
|
grouped = defaultdict(list)
|
|
for row in reports:
|
|
if row["config"] == config and row.get(metric) is not None:
|
|
grouped[row["rate"]].append(row[metric])
|
|
rates = sorted(grouped)
|
|
ax.plot(
|
|
rates,
|
|
[statistics.median(grouped[rate]) for rate in rates],
|
|
"o-",
|
|
label=config,
|
|
)
|
|
ax.fill_between(
|
|
rates,
|
|
[min(grouped[rate]) for rate in rates],
|
|
[max(grouped[rate]) for rate in rates],
|
|
alpha=0.15,
|
|
)
|
|
ax.set_xlabel("Offered requests / s")
|
|
ax.set_ylabel(label)
|
|
ax.grid(alpha=0.25)
|
|
ax.legend()
|
|
figure.savefig(directory / "improvement.png", dpi=160)
|
|
plt.close(figure)
|
|
return
|
|
for metric, label, name in (
|
|
(
|
|
"window_goodput_rps",
|
|
"Successful responses / s (measurement window)",
|
|
"throughput",
|
|
),
|
|
("p95_ms", "p95 full response latency (ms)", "latency"),
|
|
("success_ratio", "Successful / scheduled", "success"),
|
|
):
|
|
figure, axes = plt.subplots(1, 2, figsize=(11, 4), layout="constrained")
|
|
for ax, prefix in zip(axes, ("A", "B")):
|
|
for config in sorted(
|
|
{row["config"] for row in reports if row["config"].startswith(prefix)}
|
|
):
|
|
grouped = defaultdict(list)
|
|
for row in reports:
|
|
if row["config"] == config and row.get(metric) is not None:
|
|
grouped[row["rate"]].append(row[metric])
|
|
rates = sorted(grouped)
|
|
median = [statistics.median(grouped[rate]) for rate in rates]
|
|
ax.plot(rates, median, "o-", color=colors.get(config), label=config)
|
|
ax.fill_between(
|
|
rates,
|
|
[min(grouped[rate]) for rate in rates],
|
|
[max(grouped[rate]) for rate in rates],
|
|
alpha=0.15,
|
|
color=colors.get(config),
|
|
)
|
|
ax.set_title(
|
|
"Fixed total API resources"
|
|
if prefix == "A"
|
|
else "Resources increase with replicas"
|
|
)
|
|
ax.set_xlabel("Offered requests / s")
|
|
ax.set_ylabel(label)
|
|
ax.grid(alpha=0.25)
|
|
ax.legend()
|
|
if metric == "p95_ms":
|
|
ax.axhline(500, color="black", linestyle="--", linewidth=0.8)
|
|
if metric == "success_ratio":
|
|
ax.axhline(0.99, color="black", linestyle="--", linewidth=0.8)
|
|
figure.savefig(directory / (name + ".png"), dpi=160)
|
|
plt.close(figure)
|
|
figure, ax = plt.subplots(figsize=(9, 4), layout="constrained")
|
|
configs = sorted({row["config"] for row in capacities})
|
|
for index, config in enumerate(configs):
|
|
values = [
|
|
row["capacity_lower"] for row in capacities if row["config"] == config
|
|
]
|
|
ax.bar(index, statistics.median(values), color=colors.get(config), alpha=0.7)
|
|
ax.scatter([index] * len(values), values, color="black", s=22, zorder=3)
|
|
ax.set_xticks(range(len(configs)), configs)
|
|
ax.set_ylabel("Last contiguous SLO-passing offered rate (req/s)")
|
|
ax.set_title("Capacity: medians and all repetitions; finite rate grid")
|
|
ax.grid(axis="y", alpha=0.25)
|
|
figure.savefig(directory / "capacity.png", dpi=160)
|
|
plt.close(figure)
|
|
figure, axes = plt.subplots(1, 2, figsize=(11, 4), layout="constrained")
|
|
for ax, prefix in zip(axes, ("A", "B")):
|
|
for config in sorted(
|
|
{row["config"] for row in reports if row["config"].startswith(prefix)}
|
|
):
|
|
grouped = defaultdict(list)
|
|
for row in reports:
|
|
if row["config"] == config:
|
|
value = row["diagnostics"]["cpu_fraction_of_limit_mean"].get("api")
|
|
if value is not None:
|
|
grouped[row["rate"]].append(value)
|
|
rates = sorted(grouped)
|
|
ax.plot(
|
|
rates,
|
|
[statistics.median(grouped[rate]) for rate in rates],
|
|
"o-",
|
|
color=colors.get(config),
|
|
label=config,
|
|
)
|
|
ax.axhline(1, color="black", linestyle="--", linewidth=0.8)
|
|
ax.set_title("API CPU relative to per-container limit: " + prefix)
|
|
ax.set_xlabel("Offered requests / s")
|
|
ax.set_ylabel("Mean CPU / configured CPU limit")
|
|
ax.grid(alpha=0.25)
|
|
ax.legend()
|
|
figure.savefig(directory / "api-cpu.png", dpi=160)
|
|
plt.close(figure)
|
|
|
|
|
|
def hypothesis_decisions(capacities):
|
|
decisions = {}
|
|
lookup = {(row["config"], row["repeat"]): row for row in capacities}
|
|
for name, prefix, threshold, less in (
|
|
("H1", "A", 1.2, True),
|
|
("H2", "B", 1.5, False),
|
|
):
|
|
blocks = []
|
|
for repeat in sorted({row["repeat"] for row in capacities}):
|
|
one, four = (
|
|
lookup.get((prefix + "1", repeat)),
|
|
lookup.get((prefix + "4", repeat)),
|
|
)
|
|
if one is None or four is None:
|
|
continue
|
|
lower_ratio = (
|
|
four["capacity_lower"] / one["capacity_lower"]
|
|
if one["capacity_lower"]
|
|
else None
|
|
)
|
|
lower_bound = (
|
|
four["capacity_lower"] / one["capacity_upper"]
|
|
if one["capacity_upper"]
|
|
else None
|
|
)
|
|
upper_bound = (
|
|
four["capacity_upper"] / one["capacity_lower"]
|
|
if four["capacity_upper"] and one["capacity_lower"]
|
|
else None
|
|
)
|
|
valid = not (
|
|
one["invalid_rates"]
|
|
or four["invalid_rates"]
|
|
or one["nonmonotonic"]
|
|
or four["nonmonotonic"]
|
|
)
|
|
decision = "inconclusive"
|
|
if valid and lower_bound is not None and upper_bound is not None:
|
|
if less:
|
|
if upper_bound < threshold:
|
|
decision = "supported"
|
|
elif lower_bound >= threshold:
|
|
decision = "refuted"
|
|
else:
|
|
if lower_bound >= threshold:
|
|
decision = "supported"
|
|
elif upper_bound < threshold:
|
|
decision = "refuted"
|
|
blocks.append(
|
|
{
|
|
"repeat": repeat,
|
|
"ratio_of_grid_capacities": lower_ratio,
|
|
"ratio_lower_bound": lower_bound,
|
|
"ratio_upper_bound": upper_bound,
|
|
"decision": decision,
|
|
}
|
|
)
|
|
outcomes = [row["decision"] for row in blocks]
|
|
result = (
|
|
outcomes[0]
|
|
if len(outcomes) == 3 and len(set(outcomes)) == 1
|
|
else "inconclusive"
|
|
)
|
|
decisions[name] = {
|
|
"threshold": threshold,
|
|
"result": result,
|
|
"blocks": blocks,
|
|
"note": "Conservative bounds from finite offered-rate grid; not a confidence interval.",
|
|
}
|
|
return decisions
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(__doc__)
|
|
parser.add_argument("series")
|
|
analyze(parser.parse_args().series)
|