"""Restart actual stores and compare data accepted before the restart.""" import argparse import json import os import time from datetime import UTC, datetime import httpx from scripts.tls_client import FreshTrustTransport os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") from scripts.observe import fetch, logs, query, spans, wait from scripts.stand import ROOT, command, compose def instances(): result = {} for role in ["prometheus", "loki", "tempo", "grafana"]: cid = compose("ps", "-q", role) data = json.loads(command(["docker", "inspect", cid]))[0] result[role] = { "started_at": data["State"]["StartedAt"], "volumes": sorted( m["Name"] for m in data["Mounts"] if m["Type"] == "volume" ), "read_only": data["HostConfig"]["ReadonlyRootfs"], "cap_drop": data["HostConfig"]["CapDrop"], "user": data["Config"]["User"], } return result def main(): parser = argparse.ArgumentParser() parser.add_argument( "--causal-evidence", default="evidence/telemetry-first.json" ) parser.add_argument( "--output", default="evidence/observability-persistence-first.json" ) args = parser.parse_args() output = ROOT / args.output assert not output.exists() causal = json.loads((ROOT / args.causal_evidence).read_text()) assert causal["complete"] identity = causal["causal_uploads"][0]["trace_id"] timestamp = time.time() - 20 historical = query("sum(messenger_http_requests_total)", at=timestamp) assert historical and spans(identity) and logs(identity)["data"]["result"] report = { "date": datetime.now(UTC).isoformat(), "source_commit": command(["git", "rev-parse", "HEAD"]), "trace_id": identity, "historical_query_at": timestamp, "historical_before": historical, "before": instances(), "complete": False, } password = (ROOT / "secrets/grafana_password").read_text().strip() base = "https://localhost:" + os.getenv("GRAFANA_PORT", "3447") with httpx.Client( transport=FreshTrustTransport(), base_url=base, auth=("admin", password), timeout=10, trust_env=False, ) as grafana: preferences = grafana.get("/api/user/preferences") preferences.raise_for_status() previous = preferences.json() temporary = { **previous, "theme": "light" if previous.get("theme") != "light" else "dark", } try: changed = grafana.put("/api/user/preferences", json=temporary) changed.raise_for_status() compose("restart", "prometheus", "loki", "tempo", "grafana") for role, path in [ ("prometheus", "/-/ready"), ("loki", "/ready"), ("tempo", "/ready"), ]: wait( lambda role=role, path=path: fetch( role, path, json_result=False ), timeout=120, ) def grafana_ready(): try: response = grafana.get("/api/health") return ( response.status_code == 200 and response.json()["database"] == "ok" ) except httpx.HTTPError: return False wait(grafana_ready, timeout=120) wait(lambda: spans(identity), timeout=120) wait(lambda: logs(identity)["data"]["result"], timeout=120) report["historical_after"] = query( "sum(messenger_http_requests_total)", at=timestamp ) assert report["historical_after"] == historical assert ( grafana.get("/api/user/preferences").json()["theme"] == temporary["theme"] ) report["after"] = instances() for role, before in report["before"].items(): after = report["after"][role] assert before["started_at"] != after["started_at"] assert ( before["volumes"] == after["volumes"] and before["volumes"] ) assert after["read_only"] and after["user"] not in { "", "0", "0:0", "root", } assert "ALL" in after["cap_drop"] report.update( stored_trace_survives=True, stored_logs_survive=True, historical_metric_survives=True, grafana_sqlite_preference_survives=True, complete=True, ) finally: try: grafana.put( "/api/user/preferences", json=previous ).raise_for_status() finally: output.parent.mkdir(parents=True, exist_ok=True) output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) print( "PASS: metrics, traces, logs and Grafana state survive real store restarts" ) if __name__ == "__main__": main()