import argparse import json import os import subprocess import time from obs_check import Api, Backends, get_json, http, now, tempo_spans, wait_for def backends(): return Backends( argparse.Namespace( prometheus=os.environ["PROMETHEUS_URL"], loki=os.environ["LOKI_URL"], tempo=os.environ["TEMPO_URL"], grafana=os.environ["GRAFANA_URL"], grafana_password_file=os.environ.get( "GRAFANA_PASSWORD_FILE", "/".join(("secrets", "grafana_admin_password")) ), ) ) def compose(*args): return subprocess.run( ["docker", "compose", *args], check=True, capture_output=True, text=True ).stdout.strip() def history(b): started = time.time() _, _, _, rid = Api(os.environ["BASE_URL"]).register() query = '{service_name="messenger-api"} | request_id="' + rid + '"' logs = wait_for("log", lambda: b.logs(query, started - 1, time.time() + 1), 60) trace_id = logs[0][0]["trace_id"] spans = wait_for("trace", lambda: tempo_spans(b.trace(trace_id)[0]), 60) expr = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})' before = wait_for("metrics", lambda: b.promql(expr), 60) at = before[0]["value"][0] compose("restart", "prometheus", "loki", "tempo", "grafana") wait_for("Grafana", lambda: get_json(b.grafana + "/api/health"), 90) after_logs = wait_for("retained log", lambda: b.logs(query, started - 1, time.time() + 1), 90) after_spans = wait_for("retained trace", lambda: tempo_spans(b.trace(trace_id)[0]), 90) after = wait_for("retained metric", lambda: b.promql(expr, at=at), 90) if before[0]["value"][1] != after[0]["value"][1] or len(after_spans) < len(spans): raise RuntimeError("telemetry history changed after restart") return { "request_id": rid, "trace_id": trace_id, "spans_before": len(spans), "spans_after": len(after_spans), "logs_after": len(after_logs), "metric_before": before, "metric_after": after, } def collector(): compose("stop", "otel-collector") try: status = http("GET", os.environ["BASE_URL"] + "/health/ready")[0] result = subprocess.run(["make", "test"], capture_output=True, text=True, check=False) if status != 200 or result.returncode: raise RuntimeError("application failed without collector") replica = compose("ps", "-q", "api").splitlines()[0] started = time.monotonic() subprocess.run(["docker", "stop", "-t", "8", replica], check=True, capture_output=True) elapsed = time.monotonic() - started code = subprocess.check_output( ["docker", "inspect", "-f", "{{.State.ExitCode}}", replica], text=True ).strip() if code != "0" or elapsed > 8: raise RuntimeError("unbounded API shutdown") return { "ready": status, "smoke": result.stdout.strip(), "api_exit_code": code, "api_shutdown_seconds": round(elapsed, 3), } finally: compose("up", "-d", "--no-deps", "--wait", "api", "otel-collector") subprocess.run(["scripts/lb-wait.sh"], check=True) def main(): b = backends() report = { "utc": now(), "sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), } report["history"] = history(b) report["collector_down"] = collector() report["result"] = "PASS" print(json.dumps(report, indent=2)) if __name__ == "__main__": main()