Files
mtusi2026lab6/scripts/verify_observability_persistence.py
T

146 lines
5.1 KiB
Python

"""Restart actual stores and compare data accepted before the restart."""
import argparse
import json
import os
import time
from datetime import UTC, datetime
import httpx
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, query, spans, wait
from scripts.stand import ROOT, command, compose
def instances():
result = {}
for role in ["prometheus", "loki", "tempo", "grafana"]:
cid = compose("ps", "-q", role)
data = json.loads(command(["docker", "inspect", cid]))[0]
result[role] = {
"started_at": data["State"]["StartedAt"],
"volumes": sorted(
m["Name"] for m in data["Mounts"] if m["Type"] == "volume"
),
"read_only": data["HostConfig"]["ReadonlyRootfs"],
"cap_drop": data["HostConfig"]["CapDrop"],
"user": data["Config"]["User"],
}
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--causal-evidence", default="evidence/telemetry-first.json"
)
parser.add_argument(
"--output", default="evidence/observability-persistence-first.json"
)
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists()
causal = json.loads((ROOT / args.causal_evidence).read_text())
assert causal["complete"]
identity = causal["causal_uploads"][0]["trace_id"]
timestamp = time.time() - 20
historical = query("sum(messenger_http_requests_total)", at=timestamp)
assert historical and spans(identity) and logs(identity)["data"]["result"]
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"trace_id": identity,
"historical_query_at": timestamp,
"historical_before": historical,
"before": instances(),
"complete": False,
}
password = (ROOT / "secrets/grafana_password").read_text().strip()
base = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
with httpx.Client(
base_url=base, auth=("admin", password), timeout=10, trust_env=False
) as grafana:
preferences = grafana.get("/api/user/preferences")
preferences.raise_for_status()
previous = preferences.json()
temporary = {
**previous,
"theme": "light" if previous.get("theme") != "light" else "dark",
}
try:
changed = grafana.put("/api/user/preferences", json=temporary)
changed.raise_for_status()
compose("restart", "prometheus", "loki", "tempo", "grafana")
for role, path in [
("prometheus", "/-/ready"),
("loki", "/ready"),
("tempo", "/ready"),
]:
wait(
lambda role=role, path=path: fetch(
role, path, json_result=False
),
timeout=120,
)
def grafana_ready():
try:
response = grafana.get("/api/health")
return (
response.status_code == 200
and response.json()["database"] == "ok"
)
except httpx.HTTPError:
return False
wait(grafana_ready, timeout=120)
wait(lambda: spans(identity), timeout=120)
wait(lambda: logs(identity)["data"]["result"], timeout=120)
report["historical_after"] = query(
"sum(messenger_http_requests_total)", at=timestamp
)
assert report["historical_after"] == historical
assert (
grafana.get("/api/user/preferences").json()["theme"]
== temporary["theme"]
)
report["after"] = instances()
for role, before in report["before"].items():
after = report["after"][role]
assert before["started_at"] != after["started_at"]
assert (
before["volumes"] == after["volumes"] and before["volumes"]
)
assert after["read_only"] and after["user"] not in {
"",
"0",
"0:0",
"root",
}
assert "ALL" in after["cap_drop"]
report.update(
stored_trace_survives=True,
stored_logs_survive=True,
historical_metric_survives=True,
grafana_sqlite_preference_survives=True,
complete=True,
)
finally:
try:
grafana.put(
"/api/user/preferences", json=previous
).raise_for_status()
finally:
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: metrics, traces, logs and Grafana state survive real store restarts"
)
if __name__ == "__main__":
main()