#!/usr/bin/env python3 import argparse import json from pathlib import Path import re import time import urllib.error import urllib.parse import urllib.request TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b') SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher") def get_json(url): with urllib.request.urlopen(url, timeout=10) as response: return json.loads(response.read(8 << 20).decode("utf-8")) def tempo_query(base_url, query): url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100}) payload = get_json(url) return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")} def extract_trace_ids(payload): result = set() for stream in payload.get("data", {}).get("result", []): for value in stream.get("values", []): candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""] if len(value) > 2: candidates.append(value[2]) for candidate in candidates: if isinstance(candidate, dict): trace_id = candidate.get("trace_id") or candidate.get("traceid") if trace_id: result.add(str(trace_id)) candidate = json.dumps(candidate) result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate))) return result def loki_logs(base_url, service, start_ns): query = '{service_name="' + service + '"}' url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({ "query": query, "start": str(start_ns), "limit": 5000, "direction": "backward", }) payload = get_json(url) count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", [])) return count, extract_trace_ids(payload) def capture(tempo_url, loki_url, lookback_seconds): start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES} logs = {} log_trace_ids = {} for service in SERVICES: count, ids = loki_logs(loki_url, service, start_ns) logs[service] = count log_trace_ids[service] = ids api_worker = traces["messenger-api"] & traces["messenger-worker"] all_trace_ids = set().union(*traces.values()) all_log_trace_ids = set().union(*log_trace_ids.values()) return { "traces_by_service": {service: len(ids) for service, ids in traces.items()}, "logs_by_service": logs, "api_worker_shared_traces": len(api_worker), "traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids), } def capture_database(tempo_url, loki_url, lookback_seconds): start_ns = int((time.time() - lookback_seconds) * 1_000_000_000) traces = tempo_query( tempo_url, '{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }', ) log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns) return { "database_error_traces": len(traces), "api_logs": log_count, "database_error_traces_with_logs": len(traces & log_trace_ids), } def complete(report): return ( report["traces_by_service"]["messenger-api"] > 0 and report["traces_by_service"]["messenger-worker"] > 0 and report["logs_by_service"]["messenger-api"] > 0 and report["logs_by_service"]["messenger-worker"] > 0 and report["api_worker_shared_traces"] > 0 and report["traces_with_correlated_logs"] > 0 ) def database_complete(report): return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0 def main(): parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki") parser.add_argument("--tempo-url", default="http://127.0.0.1:3200") parser.add_argument("--loki-url", default="http://127.0.0.1:3100") parser.add_argument("--lookback", type=float, default=900) parser.add_argument("--wait", type=float, default=45) parser.add_argument("--output", required=True) parser.add_argument("--require", action="store_true") parser.add_argument("--mode", choices=("baseline", "database"), default="baseline") args = parser.parse_args() deadline = time.monotonic() + args.wait error = None report = None while time.monotonic() < deadline: try: report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback) error = None if database_complete(report) if args.mode == "database" else complete(report): break except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error: error = str(current_error) time.sleep(2) result = { "schema_version": 1, "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "lookback_seconds": args.lookback, "signals": report, "mode": args.mode, "complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))), "error": error, } destination = Path(args.output) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(json.dumps(result, ensure_ascii=False, indent=2)) if args.require and not result["complete"]: raise SystemExit(1) if __name__ == "__main__": main()