import argparse import json import statistics from pathlib import Path def percentile(values, fraction): ordered = sorted(values) return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))] def total(counters, prefix): return sum(v for k, v in counters.items() if k.startswith(prefix + "{")) def summarize(path): run = json.loads(path.read_text()) before, after = run["before"], run["after"] resources = run["resources"] elapsed = resources[-1]["at"] - resources[0]["at"] first = {r["id"]: r for r in resources[0]["containers"]} cpu = {} memory = {} membership = {} for row in resources[-1]["containers"]: name = row["name"] group = ( "application" if any(part in name for part in ("-api-", "-relay-", "-worker-")) else "observability" if any( part in name for part in ("-otel-collector-", "-prometheus-", "-loki-", "-tempo-", "-grafana-") ) else "business_stores_lb" ) membership[row["id"]] = group cpu[group] = ( cpu.get(group, 0) + (row["cpu_ns"] - first[row["id"]]["cpu_ns"]) / 1e9 / elapsed ) memory[name] = max( r["working_set_bytes"] for sample in resources for r in sample["containers"] if r["id"] == row["id"] ) latency = [r["latency_ms"] for r in run["requests"]] normal = [r["latency_ms"] for r in run["requests"] if r["category"] == "normal"] collector = {} for name in ("accepted", "sent", "failed"): prefix = { "accepted": "otelcol_receiver_accepted_spans_total", "sent": "otelcol_exporter_sent_spans_total", "failed": "otelcol_exporter_send_failed_spans_total", }[name] collector[name] = total(after["collector"], prefix) - total(before["collector"], prefix) ingest = { backend: {key: value - before[backend].get(key, 0) for key, value in after[backend].items()} for backend in ("tempo", "loki", "prometheus") } return { "file": path.name, "mode": run["mode"], "repeat": run["repeat"], "sha": run["sha"], "result": run["result"], "requests": len(latency), "unexpected_responses": run["unexpected_responses"], "p50_ms": percentile(latency, 0.5), "p95_ms": percentile(latency, 0.95), "p99_ms": percentile(latency, 0.99), "normal_p95_ms": percentile(normal, 0.95), "max_schedule_lag_ms": max(r["schedule_lag_ms"] for r in run["requests"]), "cpu_cores": cpu, "memory_peak_bytes": memory, "memory_peak_group_bytes": { group: max( sum( r["working_set_bytes"] for r in sample["containers"] if membership[r["id"]] == group ) for sample in resources ) for group in set(membership.values()) }, "achieved_rps": len(latency) / (run["finished"] - run["started"]), "resource_samples": len(resources), "ingest": ingest, "sdk_ended_spans": after["sdk_ended_spans"] - before["sdk_ended_spans"], "collector_spans": collector, "traces": { kind: { "expected": sum(r["category"] == kind for r in run["requests"]), "retained": len(ids), } for kind, ids in run["retained_trace_ids"].items() }, } def main(): parser = argparse.ArgumentParser() parser.add_argument("directory", type=Path) args = parser.parse_args() rows = [ summarize(p) for p in sorted(args.directory.glob("*-*.json")) if p.name.split("-")[0] in {"off", "on", "head", "tail", "balanced"} ] groups = { mode: { key: { "median": statistics.median(r[key] for r in rows if r["mode"] == mode), "min": min(r[key] for r in rows if r["mode"] == mode), "max": max(r[key] for r in rows if r["mode"] == mode), } for key in ("p50_ms", "p95_ms", "p99_ms", "normal_p95_ms") } for mode in sorted({r["mode"] for r in rows}) } (args.directory / "comparison.json").write_text( json.dumps({"runs": rows, "latency_ranges": groups}, indent=2) + "\n" ) print(json.dumps({"runs": len(rows), "modes": groups}, indent=2)) if __name__ == "__main__": main()