forked from mtusi2026/mtusi2026lab6
130 lines
4.5 KiB
Python
130 lines
4.5 KiB
Python
import argparse
|
|
import json
|
|
import statistics
|
|
from pathlib import Path
|
|
|
|
|
|
def percentile(values, fraction):
|
|
ordered = sorted(values)
|
|
return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))]
|
|
|
|
|
|
def total(counters, prefix):
|
|
return sum(v for k, v in counters.items() if k.startswith(prefix + "{"))
|
|
|
|
|
|
def summarize(path):
|
|
run = json.loads(path.read_text())
|
|
before, after = run["before"], run["after"]
|
|
resources = run["resources"]
|
|
elapsed = resources[-1]["at"] - resources[0]["at"]
|
|
first = {r["id"]: r for r in resources[0]["containers"]}
|
|
cpu = {}
|
|
memory = {}
|
|
membership = {}
|
|
for row in resources[-1]["containers"]:
|
|
name = row["name"]
|
|
group = (
|
|
"application"
|
|
if any(part in name for part in ("-api-", "-relay-", "-worker-"))
|
|
else "observability"
|
|
if any(
|
|
part in name
|
|
for part in ("-otel-collector-", "-prometheus-", "-loki-", "-tempo-", "-grafana-")
|
|
)
|
|
else "business_stores_lb"
|
|
)
|
|
membership[row["id"]] = group
|
|
cpu[group] = (
|
|
cpu.get(group, 0) + (row["cpu_ns"] - first[row["id"]]["cpu_ns"]) / 1e9 / elapsed
|
|
)
|
|
memory[name] = max(
|
|
r["working_set_bytes"]
|
|
for sample in resources
|
|
for r in sample["containers"]
|
|
if r["id"] == row["id"]
|
|
)
|
|
latency = [r["latency_ms"] for r in run["requests"]]
|
|
normal = [r["latency_ms"] for r in run["requests"] if r["category"] == "normal"]
|
|
collector = {}
|
|
for name in ("accepted", "sent", "failed"):
|
|
prefix = {
|
|
"accepted": "otelcol_receiver_accepted_spans_total",
|
|
"sent": "otelcol_exporter_sent_spans_total",
|
|
"failed": "otelcol_exporter_send_failed_spans_total",
|
|
}[name]
|
|
collector[name] = total(after["collector"], prefix) - total(before["collector"], prefix)
|
|
ingest = {
|
|
backend: {key: value - before[backend].get(key, 0) for key, value in after[backend].items()}
|
|
for backend in ("tempo", "loki", "prometheus")
|
|
}
|
|
return {
|
|
"file": path.name,
|
|
"mode": run["mode"],
|
|
"repeat": run["repeat"],
|
|
"sha": run["sha"],
|
|
"result": run["result"],
|
|
"requests": len(latency),
|
|
"unexpected_responses": run["unexpected_responses"],
|
|
"p50_ms": percentile(latency, 0.5),
|
|
"p95_ms": percentile(latency, 0.95),
|
|
"p99_ms": percentile(latency, 0.99),
|
|
"normal_p95_ms": percentile(normal, 0.95),
|
|
"max_schedule_lag_ms": max(r["schedule_lag_ms"] for r in run["requests"]),
|
|
"cpu_cores": cpu,
|
|
"memory_peak_bytes": memory,
|
|
"memory_peak_group_bytes": {
|
|
group: max(
|
|
sum(
|
|
r["working_set_bytes"]
|
|
for r in sample["containers"]
|
|
if membership[r["id"]] == group
|
|
)
|
|
for sample in resources
|
|
)
|
|
for group in set(membership.values())
|
|
},
|
|
"achieved_rps": len(latency) / (run["finished"] - run["started"]),
|
|
"resource_samples": len(resources),
|
|
"ingest": ingest,
|
|
"sdk_ended_spans": after["sdk_ended_spans"] - before["sdk_ended_spans"],
|
|
"collector_spans": collector,
|
|
"traces": {
|
|
kind: {
|
|
"expected": sum(r["category"] == kind for r in run["requests"]),
|
|
"retained": len(ids),
|
|
}
|
|
for kind, ids in run["retained_trace_ids"].items()
|
|
},
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("directory", type=Path)
|
|
args = parser.parse_args()
|
|
rows = [
|
|
summarize(p)
|
|
for p in sorted(args.directory.glob("*-*.json"))
|
|
if p.name.split("-")[0] in {"off", "on", "head", "tail", "balanced"}
|
|
]
|
|
groups = {
|
|
mode: {
|
|
key: {
|
|
"median": statistics.median(r[key] for r in rows if r["mode"] == mode),
|
|
"min": min(r[key] for r in rows if r["mode"] == mode),
|
|
"max": max(r[key] for r in rows if r["mode"] == mode),
|
|
}
|
|
for key in ("p50_ms", "p95_ms", "p99_ms", "normal_p95_ms")
|
|
}
|
|
for mode in sorted({r["mode"] for r in rows})
|
|
}
|
|
(args.directory / "comparison.json").write_text(
|
|
json.dumps({"runs": rows, "latency_ranges": groups}, indent=2) + "\n"
|
|
)
|
|
print(json.dumps({"runs": len(rows), "modes": groups}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|