Files
mtusi2026lab6/student-tests/test_observability_failure.py
T

119 lines
5.0 KiB
Python

#!/usr/bin/env python3
"""Prove business API survives a bounded outage of all telemetry stores."""
import json
import subprocess
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
FILES = ["-f", "compose.yaml"]
if (ROOT / "compose.local.yaml").exists():
FILES += ["-f", "compose.local.yaml"]
FILES += ["-f", "compose.telemetry.yaml"]
COMPOSE = ["docker", "compose", *FILES, "--env-file", ".env"]
COUNTERS = (
"otelcol_exporter_enqueue_failed_spans",
"otelcol_exporter_send_failed_spans",
"otelcol_exporter_send_failed_log_records",
"otelcol_exporter_send_failed_metric_points",
"otelcol_receiver_refused_spans",
"otelcol_receiver_refused_log_records",
"otelcol_receiver_refused_metric_points",
)
def run(*args):
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
def stats():
result = run("docker", "stats", "--no-stream", "--format", "{{json .}}",
"mtusi2026lab6-otel-collector-1", "mtusi2026lab6-api-1", "mtusi2026lab6-worker-1")
return [json.loads(line) for line in result.stdout.splitlines()] if result.returncode == 0 else []
def metric(name):
url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": f"sum({name})"})
try:
with urlopen(url, timeout=5) as response:
rows = json.load(response)["data"]["result"]
except (OSError, ValueError):
return None
return float(rows[0]["value"][1]) if rows else None
def stores_up():
url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": "up{job=~\"prometheus|loki|tempo\"}"})
try:
with urlopen(url, timeout=5) as response:
rows = json.load(response)["data"]["result"]
except (OSError, ValueError):
return False
return len(rows) == 3 and all(row["value"][1] == "1" for row in rows)
def main():
output = ROOT / "evidence/telemetry-backend-outage-2026-10-02.json"
data = {"started": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"stores": ["prometheus", "loki", "tempo"], "stats_before": stats(),
"collector_counters_before": {name: metric(name) for name in COUNTERS}}
stopped = False
try:
result = run(*COMPOSE, "stop", "prometheus", "loki", "tempo")
if result.returncode:
raise RuntimeError(result.stderr[-500:])
stopped = True
data["stores_stopped"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
smoke = run("make", "test", "BASE_URL=http://127.0.0.1:8080")
data["smoke_passed"] = smoke.returncode == 0
data["smoke_tail"] = (smoke.stdout + smoke.stderr).splitlines()[-5:]
load = run("python3", "tests/load.py", "--base-url", "http://127.0.0.1:8080",
"--duration", "30", "--concurrency", "4",
"--output", "evidence/telemetry-outage-http-load-2026-10-02.json")
data["load_passed"] = load.returncode == 0
data["stats_during"] = stats()
# Longer than Collector's 30-second exporter retry horizon.
time.sleep(35)
data["stats_after_35s"] = stats()
finally:
if stopped:
restored = run(*COMPOSE, "up", "-d", "prometheus", "loki", "tempo")
data["stores_restored"] = restored.returncode == 0
data["stores_restored_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
healthy_since = None
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
if stores_up():
healthy_since = healthy_since or time.monotonic()
if time.monotonic() - healthy_since >= 30:
break
else:
healthy_since = None
time.sleep(5)
data["stores_healthy"] = (healthy_since is not None and
time.monotonic() - healthy_since >= 30 and stores_up())
data["collector_counters_after"] = {name: metric(name) for name in COUNTERS}
data["collector_counter_delta"] = {
name: (data["collector_counters_after"][name] - data["collector_counters_before"][name]
if data["collector_counters_before"][name] is not None and
data["collector_counters_after"][name] is not None else None)
for name in COUNTERS
}
output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"smoke_passed": data.get("smoke_passed"),
"load_passed": data.get("load_passed"),
"stores_restored": data.get("stores_restored"),
"collector_counter_delta": data["collector_counter_delta"],
"output": str(output)}, sort_keys=True))
if not all(data.get(key) for key in ("smoke_passed", "load_passed", "stores_restored", "stores_healthy")):
raise SystemExit("business path or telemetry recovery failed")
if __name__ == "__main__":
main()