119 lines
5.0 KiB
Python
119 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove business API survives a bounded outage of all telemetry stores."""
|
|
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import urlencode
|
|
from urllib.request import urlopen
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FILES = ["-f", "compose.yaml"]
|
|
if (ROOT / "compose.local.yaml").exists():
|
|
FILES += ["-f", "compose.local.yaml"]
|
|
FILES += ["-f", "compose.telemetry.yaml"]
|
|
COMPOSE = ["docker", "compose", *FILES, "--env-file", ".env"]
|
|
COUNTERS = (
|
|
"otelcol_exporter_enqueue_failed_spans",
|
|
"otelcol_exporter_send_failed_spans",
|
|
"otelcol_exporter_send_failed_log_records",
|
|
"otelcol_exporter_send_failed_metric_points",
|
|
"otelcol_receiver_refused_spans",
|
|
"otelcol_receiver_refused_log_records",
|
|
"otelcol_receiver_refused_metric_points",
|
|
)
|
|
|
|
|
|
def run(*args):
|
|
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
|
|
|
|
|
|
def stats():
|
|
result = run("docker", "stats", "--no-stream", "--format", "{{json .}}",
|
|
"mtusi2026lab6-otel-collector-1", "mtusi2026lab6-api-1", "mtusi2026lab6-worker-1")
|
|
return [json.loads(line) for line in result.stdout.splitlines()] if result.returncode == 0 else []
|
|
|
|
|
|
def metric(name):
|
|
url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": f"sum({name})"})
|
|
try:
|
|
with urlopen(url, timeout=5) as response:
|
|
rows = json.load(response)["data"]["result"]
|
|
except (OSError, ValueError):
|
|
return None
|
|
return float(rows[0]["value"][1]) if rows else None
|
|
|
|
|
|
def stores_up():
|
|
url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": "up{job=~\"prometheus|loki|tempo\"}"})
|
|
try:
|
|
with urlopen(url, timeout=5) as response:
|
|
rows = json.load(response)["data"]["result"]
|
|
except (OSError, ValueError):
|
|
return False
|
|
return len(rows) == 3 and all(row["value"][1] == "1" for row in rows)
|
|
|
|
|
|
def main():
|
|
output = ROOT / "evidence/telemetry-backend-outage-2026-10-02.json"
|
|
data = {"started": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
"stores": ["prometheus", "loki", "tempo"], "stats_before": stats(),
|
|
"collector_counters_before": {name: metric(name) for name in COUNTERS}}
|
|
stopped = False
|
|
try:
|
|
result = run(*COMPOSE, "stop", "prometheus", "loki", "tempo")
|
|
if result.returncode:
|
|
raise RuntimeError(result.stderr[-500:])
|
|
stopped = True
|
|
data["stores_stopped"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
smoke = run("make", "test", "BASE_URL=http://127.0.0.1:8080")
|
|
data["smoke_passed"] = smoke.returncode == 0
|
|
data["smoke_tail"] = (smoke.stdout + smoke.stderr).splitlines()[-5:]
|
|
load = run("python3", "tests/load.py", "--base-url", "http://127.0.0.1:8080",
|
|
"--duration", "30", "--concurrency", "4",
|
|
"--output", "evidence/telemetry-outage-http-load-2026-10-02.json")
|
|
data["load_passed"] = load.returncode == 0
|
|
data["stats_during"] = stats()
|
|
# Longer than Collector's 30-second exporter retry horizon.
|
|
time.sleep(35)
|
|
data["stats_after_35s"] = stats()
|
|
finally:
|
|
if stopped:
|
|
restored = run(*COMPOSE, "up", "-d", "prometheus", "loki", "tempo")
|
|
data["stores_restored"] = restored.returncode == 0
|
|
data["stores_restored_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
healthy_since = None
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
if stores_up():
|
|
healthy_since = healthy_since or time.monotonic()
|
|
if time.monotonic() - healthy_since >= 30:
|
|
break
|
|
else:
|
|
healthy_since = None
|
|
time.sleep(5)
|
|
data["stores_healthy"] = (healthy_since is not None and
|
|
time.monotonic() - healthy_since >= 30 and stores_up())
|
|
data["collector_counters_after"] = {name: metric(name) for name in COUNTERS}
|
|
data["collector_counter_delta"] = {
|
|
name: (data["collector_counters_after"][name] - data["collector_counters_before"][name]
|
|
if data["collector_counters_before"][name] is not None and
|
|
data["collector_counters_after"][name] is not None else None)
|
|
for name in COUNTERS
|
|
}
|
|
output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps({"smoke_passed": data.get("smoke_passed"),
|
|
"load_passed": data.get("load_passed"),
|
|
"stores_restored": data.get("stores_restored"),
|
|
"collector_counter_delta": data["collector_counter_delta"],
|
|
"output": str(output)}, sort_keys=True))
|
|
if not all(data.get(key) for key in ("smoke_passed", "load_passed", "stores_restored", "stores_healthy")):
|
|
raise SystemExit("business path or telemetry recovery failed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|