146 lines
5.1 KiB
Python
146 lines
5.1 KiB
Python
"""Restart actual stores and compare data accepted before the restart."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import time
|
|
from datetime import UTC, datetime
|
|
|
|
import httpx
|
|
|
|
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
|
from scripts.observe import fetch, logs, query, spans, wait
|
|
from scripts.stand import ROOT, command, compose
|
|
|
|
|
|
def instances():
|
|
result = {}
|
|
for role in ["prometheus", "loki", "tempo", "grafana"]:
|
|
cid = compose("ps", "-q", role)
|
|
data = json.loads(command(["docker", "inspect", cid]))[0]
|
|
result[role] = {
|
|
"started_at": data["State"]["StartedAt"],
|
|
"volumes": sorted(
|
|
m["Name"] for m in data["Mounts"] if m["Type"] == "volume"
|
|
),
|
|
"read_only": data["HostConfig"]["ReadonlyRootfs"],
|
|
"cap_drop": data["HostConfig"]["CapDrop"],
|
|
"user": data["Config"]["User"],
|
|
}
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--causal-evidence", default="evidence/telemetry-first.json"
|
|
)
|
|
parser.add_argument(
|
|
"--output", default="evidence/observability-persistence-first.json"
|
|
)
|
|
args = parser.parse_args()
|
|
output = ROOT / args.output
|
|
assert not output.exists()
|
|
causal = json.loads((ROOT / args.causal_evidence).read_text())
|
|
assert causal["complete"]
|
|
identity = causal["causal_uploads"][0]["trace_id"]
|
|
timestamp = time.time() - 20
|
|
historical = query("sum(messenger_http_requests_total)", at=timestamp)
|
|
assert historical and spans(identity) and logs(identity)["data"]["result"]
|
|
report = {
|
|
"date": datetime.now(UTC).isoformat(),
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"trace_id": identity,
|
|
"historical_query_at": timestamp,
|
|
"historical_before": historical,
|
|
"before": instances(),
|
|
"complete": False,
|
|
}
|
|
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
|
base = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
|
|
with httpx.Client(
|
|
base_url=base, auth=("admin", password), timeout=10, trust_env=False
|
|
) as grafana:
|
|
preferences = grafana.get("/api/user/preferences")
|
|
preferences.raise_for_status()
|
|
previous = preferences.json()
|
|
temporary = {
|
|
**previous,
|
|
"theme": "light" if previous.get("theme") != "light" else "dark",
|
|
}
|
|
try:
|
|
changed = grafana.put("/api/user/preferences", json=temporary)
|
|
changed.raise_for_status()
|
|
compose("restart", "prometheus", "loki", "tempo", "grafana")
|
|
for role, path in [
|
|
("prometheus", "/-/ready"),
|
|
("loki", "/ready"),
|
|
("tempo", "/ready"),
|
|
]:
|
|
wait(
|
|
lambda role=role, path=path: fetch(
|
|
role, path, json_result=False
|
|
),
|
|
timeout=120,
|
|
)
|
|
|
|
def grafana_ready():
|
|
try:
|
|
response = grafana.get("/api/health")
|
|
return (
|
|
response.status_code == 200
|
|
and response.json()["database"] == "ok"
|
|
)
|
|
except httpx.HTTPError:
|
|
return False
|
|
|
|
wait(grafana_ready, timeout=120)
|
|
wait(lambda: spans(identity), timeout=120)
|
|
wait(lambda: logs(identity)["data"]["result"], timeout=120)
|
|
report["historical_after"] = query(
|
|
"sum(messenger_http_requests_total)", at=timestamp
|
|
)
|
|
assert report["historical_after"] == historical
|
|
assert (
|
|
grafana.get("/api/user/preferences").json()["theme"]
|
|
== temporary["theme"]
|
|
)
|
|
report["after"] = instances()
|
|
for role, before in report["before"].items():
|
|
after = report["after"][role]
|
|
assert before["started_at"] != after["started_at"]
|
|
assert (
|
|
before["volumes"] == after["volumes"] and before["volumes"]
|
|
)
|
|
assert after["read_only"] and after["user"] not in {
|
|
"",
|
|
"0",
|
|
"0:0",
|
|
"root",
|
|
}
|
|
assert "ALL" in after["cap_drop"]
|
|
report.update(
|
|
stored_trace_survives=True,
|
|
stored_logs_survive=True,
|
|
historical_metric_survives=True,
|
|
grafana_sqlite_preference_survives=True,
|
|
complete=True,
|
|
)
|
|
finally:
|
|
try:
|
|
grafana.put(
|
|
"/api/user/preferences", json=previous
|
|
).raise_for_status()
|
|
finally:
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
"PASS: metrics, traces, logs and Grafana state survive real store restarts"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|