Handle Docker stats without embedded container names
This commit is contained in:
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
|
||||
RATE ?= 100
|
||||
FROM_REV ?=
|
||||
SHIFT ?= 0
|
||||
SCRIPTS_PY = scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
SCRIPTS_PY = scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
|
||||
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
SCENARIO ?= failover-upload
|
||||
|
||||
+9
-1
@@ -76,6 +76,14 @@ class Resources:
|
||||
raise RuntimeError("cost measurement requires a local Docker socket")
|
||||
self.path = host[7:]
|
||||
self.ids = compose("ps", "-q").splitlines()
|
||||
self.names = dict(
|
||||
zip(
|
||||
self.ids,
|
||||
subprocess.check_output(
|
||||
["docker", "inspect", "-f", "{{.Name}}", *self.ids], text=True
|
||||
).splitlines(),
|
||||
)
|
||||
)
|
||||
self.samples = []
|
||||
self.stop = threading.Event()
|
||||
self.thread = threading.Thread(target=self.poll)
|
||||
@@ -90,7 +98,7 @@ class Resources:
|
||||
rows.append(
|
||||
{
|
||||
"id": identity[:12],
|
||||
"name": data["name"],
|
||||
"name": self.names[identity],
|
||||
"cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"],
|
||||
"working_set_bytes": memory["usage"]
|
||||
- memory.get("stats", {}).get("inactive_file", 0),
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
ordered = sorted(values)
|
||||
return ordered[min(len(ordered) - 1, int((len(ordered) - 1) * fraction))]
|
||||
|
||||
|
||||
def total(counters, prefix):
|
||||
return sum(v for k, v in counters.items() if k.startswith(prefix + "{"))
|
||||
|
||||
|
||||
def summarize(path):
|
||||
run = json.loads(path.read_text())
|
||||
before, after = run["before"], run["after"]
|
||||
resources = run["resources"]
|
||||
elapsed = resources[-1]["at"] - resources[0]["at"]
|
||||
first = {r["id"]: r for r in resources[0]["containers"]}
|
||||
cpu = {}
|
||||
memory = {}
|
||||
for row in resources[-1]["containers"]:
|
||||
name = row["name"]
|
||||
group = (
|
||||
"application"
|
||||
if any(part in name for part in ("-api-", "-relay-", "-worker-"))
|
||||
else "observability"
|
||||
if any(
|
||||
part in name
|
||||
for part in ("-otel-collector-", "-prometheus-", "-loki-", "-tempo-", "-grafana-")
|
||||
)
|
||||
else "business_stores_lb"
|
||||
)
|
||||
cpu[group] = (
|
||||
cpu.get(group, 0) + (row["cpu_ns"] - first[row["id"]]["cpu_ns"]) / 1e9 / elapsed
|
||||
)
|
||||
memory[name] = max(
|
||||
r["working_set_bytes"]
|
||||
for sample in resources
|
||||
for r in sample["containers"]
|
||||
if r["id"] == row["id"]
|
||||
)
|
||||
latency = [r["latency_ms"] for r in run["requests"]]
|
||||
normal = [r["latency_ms"] for r in run["requests"] if r["category"] == "normal"]
|
||||
collector = {}
|
||||
for name in ("accepted", "sent", "failed"):
|
||||
prefix = {
|
||||
"accepted": "otelcol_receiver_accepted_spans_total",
|
||||
"sent": "otelcol_exporter_sent_spans_total",
|
||||
"failed": "otelcol_exporter_send_failed_spans_total",
|
||||
}[name]
|
||||
collector[name] = total(after["collector"], prefix) - total(before["collector"], prefix)
|
||||
ingest = {
|
||||
backend: {key: value - before[backend].get(key, 0) for key, value in after[backend].items()}
|
||||
for backend in ("tempo", "loki", "prometheus")
|
||||
}
|
||||
return {
|
||||
"file": path.name,
|
||||
"mode": run["mode"],
|
||||
"repeat": run["repeat"],
|
||||
"sha": run["sha"],
|
||||
"result": run["result"],
|
||||
"requests": len(latency),
|
||||
"unexpected_responses": run["unexpected_responses"],
|
||||
"p50_ms": percentile(latency, 0.5),
|
||||
"p95_ms": percentile(latency, 0.95),
|
||||
"p99_ms": percentile(latency, 0.99),
|
||||
"normal_p95_ms": percentile(normal, 0.95),
|
||||
"max_schedule_lag_ms": max(r["schedule_lag_ms"] for r in run["requests"]),
|
||||
"cpu_cores": cpu,
|
||||
"memory_peak_bytes": memory,
|
||||
"resource_samples": len(resources),
|
||||
"ingest": ingest,
|
||||
"sdk_ended_spans": after["sdk_ended_spans"] - before["sdk_ended_spans"],
|
||||
"collector_spans": collector,
|
||||
"traces": {
|
||||
kind: {
|
||||
"expected": sum(r["category"] == kind for r in run["requests"]),
|
||||
"retained": len(ids),
|
||||
}
|
||||
for kind, ids in run["retained_trace_ids"].items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("directory", type=Path)
|
||||
args = parser.parse_args()
|
||||
rows = [
|
||||
summarize(p)
|
||||
for p in sorted(args.directory.glob("*-*.json"))
|
||||
if p.name.split("-")[0] in {"off", "on", "head", "tail", "balanced"}
|
||||
]
|
||||
groups = {
|
||||
mode: {
|
||||
key: {
|
||||
"median": statistics.median(r[key] for r in rows if r["mode"] == mode),
|
||||
"min": min(r[key] for r in rows if r["mode"] == mode),
|
||||
"max": max(r[key] for r in rows if r["mode"] == mode),
|
||||
}
|
||||
for key in ("p50_ms", "p95_ms", "p99_ms", "normal_p95_ms")
|
||||
}
|
||||
for mode in sorted({r["mode"] for r in rows})
|
||||
}
|
||||
(args.directory / "comparison.json").write_text(
|
||||
json.dumps({"runs": rows, "latency_ranges": groups}, indent=2) + "\n"
|
||||
)
|
||||
print(json.dumps({"runs": len(rows), "modes": groups}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user