From bc0b1620825b53fd3a3019623326ebf68cbbbeec Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 21:26:45 +0300 Subject: [PATCH] Handle Docker stats without embedded container names --- Makefile | 2 +- scripts/obs_cost.py | 10 +++- scripts/obs_cost_summary.py | 115 ++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 scripts/obs_cost_summary.py diff --git a/Makefile b/Makefile index 0a9963b3..eb278006 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMPOSE ?= docker compose RATE ?= 100 FROM_REV ?= SHIFT ?= 0 -SCRIPTS_PY = scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ +SCRIPTS_PY = scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \ scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py SCENARIO ?= failover-upload diff --git a/scripts/obs_cost.py b/scripts/obs_cost.py index c620e3ad..4329b5f2 100644 --- a/scripts/obs_cost.py +++ b/scripts/obs_cost.py @@ -76,6 +76,14 @@ class Resources: raise RuntimeError("cost measurement requires a local Docker socket") self.path = host[7:] self.ids = compose("ps", "-q").splitlines() + self.names = dict( + zip( + self.ids, + subprocess.check_output( + ["docker", "inspect", "-f", "{{.Name}}", *self.ids], text=True + ).splitlines(), + ) + ) self.samples = [] self.stop = threading.Event() self.thread = threading.Thread(target=self.poll) @@ -90,7 +98,7 @@ class Resources: rows.append( { "id": identity[:12], - "name": data["name"], + "name": self.names[identity], "cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"], "working_set_bytes": memory["usage"] - memory.get("stats", {}).get("inactive_file", 0), diff --git a/scripts/obs_cost_summary.py b/scripts/obs_cost_summary.py new file mode 100644 index 00000000..ca809485 --- /dev/null +++ b/scripts/obs_cost_summary.py @@ -0,0 +1,115 @@ +import argparse +import json +import statistics +from pathlib import Path + + +def percentile(values, fraction): + ordered = sorted(values) + return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))] + + +def total(counters, prefix): + return sum(v for k, v in counters.items() if k.startswith(prefix + "{")) + + +def summarize(path): + run = json.loads(path.read_text()) + before, after = run["before"], run["after"] + resources = run["resources"] + elapsed = resources[-1]["at"] - resources[0]["at"] + first = {r["id"]: r for r in resources[0]["containers"]} + cpu = {} + memory = {} + for row in resources[-1]["containers"]: + name = row["name"] + group = ( + "application" + if any(part in name for part in ("-api-", "-relay-", "-worker-")) + else "observability" + if any( + part in name + for part in ("-otel-collector-", "-prometheus-", "-loki-", "-tempo-", "-grafana-") + ) + else "business_stores_lb" + ) + cpu[group] = ( + cpu.get(group, 0) + (row["cpu_ns"] - first[row["id"]]["cpu_ns"]) / 1e9 / elapsed + ) + memory[name] = max( + r["working_set_bytes"] + for sample in resources + for r in sample["containers"] + if r["id"] == row["id"] + ) + latency = [r["latency_ms"] for r in run["requests"]] + normal = [r["latency_ms"] for r in run["requests"] if r["category"] == "normal"] + collector = {} + for name in ("accepted", "sent", "failed"): + prefix = { + "accepted": "otelcol_receiver_accepted_spans_total", + "sent": "otelcol_exporter_sent_spans_total", + "failed": "otelcol_exporter_send_failed_spans_total", + }[name] + collector[name] = total(after["collector"], prefix) - total(before["collector"], prefix) + ingest = { + backend: {key: value - before[backend].get(key, 0) for key, value in after[backend].items()} + for backend in ("tempo", "loki", "prometheus") + } + return { + "file": path.name, + "mode": run["mode"], + "repeat": run["repeat"], + "sha": run["sha"], + "result": run["result"], + "requests": len(latency), + "unexpected_responses": run["unexpected_responses"], + "p50_ms": percentile(latency, 0.5), + "p95_ms": percentile(latency, 0.95), + "p99_ms": percentile(latency, 0.99), + "normal_p95_ms": percentile(normal, 0.95), + "max_schedule_lag_ms": max(r["schedule_lag_ms"] for r in run["requests"]), + "cpu_cores": cpu, + "memory_peak_bytes": memory, + "resource_samples": len(resources), + "ingest": ingest, + "sdk_ended_spans": after["sdk_ended_spans"] - before["sdk_ended_spans"], + "collector_spans": collector, + "traces": { + kind: { + "expected": sum(r["category"] == kind for r in run["requests"]), + "retained": len(ids), + } + for kind, ids in run["retained_trace_ids"].items() + }, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("directory", type=Path) + args = parser.parse_args() + rows = [ + summarize(p) + for p in sorted(args.directory.glob("*-*.json")) + if p.name.split("-")[0] in {"off", "on", "head", "tail", "balanced"} + ] + groups = { + mode: { + key: { + "median": statistics.median(r[key] for r in rows if r["mode"] == mode), + "min": min(r[key] for r in rows if r["mode"] == mode), + "max": max(r[key] for r in rows if r["mode"] == mode), + } + for key in ("p50_ms", "p95_ms", "p99_ms", "normal_p95_ms") + } + for mode in sorted({r["mode"] for r in rows}) + } + (args.directory / "comparison.json").write_text( + json.dumps({"runs": rows, "latency_ranges": groups}, indent=2) + "\n" + ) + print(json.dumps({"runs": len(rows), "modes": groups}, indent=2)) + + +if __name__ == "__main__": + main()