Files
mtusi2026lab6/scripts/lab6-correlation.py
T

149 lines
5.7 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
import re
import time
import urllib.error
import urllib.parse
import urllib.request
TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b')
SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher")
def get_json(url):
with urllib.request.urlopen(url, timeout=10) as response:
return json.loads(response.read(8 << 20).decode("utf-8"))
def tempo_query(base_url, query):
url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100})
payload = get_json(url)
return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")}
def extract_trace_ids(payload):
result = set()
for stream in payload.get("data", {}).get("result", []):
for value in stream.get("values", []):
candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""]
if len(value) > 2:
candidates.append(value[2])
for candidate in candidates:
if isinstance(candidate, dict):
trace_id = candidate.get("trace_id") or candidate.get("traceid")
if trace_id:
result.add(str(trace_id))
candidate = json.dumps(candidate)
result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate)))
return result
def loki_logs(base_url, service, start_ns):
query = '{service_name="' + service + '"}'
url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({
"query": query,
"start": str(start_ns),
"limit": 5000,
"direction": "backward",
})
payload = get_json(url)
count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", []))
return count, extract_trace_ids(payload)
def capture(tempo_url, loki_url, lookback_seconds):
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES}
logs = {}
log_trace_ids = {}
for service in SERVICES:
count, ids = loki_logs(loki_url, service, start_ns)
logs[service] = count
log_trace_ids[service] = ids
api_worker = traces["messenger-api"] & traces["messenger-worker"]
all_trace_ids = set().union(*traces.values())
all_log_trace_ids = set().union(*log_trace_ids.values())
return {
"traces_by_service": {service: len(ids) for service, ids in traces.items()},
"logs_by_service": logs,
"api_worker_shared_traces": len(api_worker),
"traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids),
}
def capture_database(tempo_url, loki_url, lookback_seconds):
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
traces = tempo_query(
tempo_url,
'{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }',
)
log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns)
return {
"database_error_traces": len(traces),
"api_logs": log_count,
"database_error_traces_with_logs": len(traces & log_trace_ids),
}
def complete(report):
return (
report["traces_by_service"]["messenger-api"] > 0
and report["traces_by_service"]["messenger-worker"] > 0
and report["logs_by_service"]["messenger-api"] > 0
and report["logs_by_service"]["messenger-worker"] > 0
and report["api_worker_shared_traces"] > 0
and report["traces_with_correlated_logs"] > 0
)
def database_complete(report):
return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0
def main():
parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki")
parser.add_argument("--tempo-url", default="http://127.0.0.1:3200")
parser.add_argument("--loki-url", default="http://127.0.0.1:3100")
parser.add_argument("--lookback", type=float, default=900)
parser.add_argument("--wait", type=float, default=45)
parser.add_argument("--output", required=True)
parser.add_argument("--require", action="store_true")
parser.add_argument("--mode", choices=("baseline", "database"), default="baseline")
args = parser.parse_args()
deadline = time.monotonic() + args.wait
error = None
report = None
while time.monotonic() < deadline:
try:
report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback)
error = None
if database_complete(report) if args.mode == "database" else complete(report):
break
except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error:
error = str(current_error)
time.sleep(2)
result = {
"schema_version": 1,
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"lookback_seconds": args.lookback,
"signals": report,
"mode": args.mode,
"complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))),
"error": error,
}
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(result, ensure_ascii=False, indent=2))
if args.require and not result["complete"]:
raise SystemExit(1)
if __name__ == "__main__":
main()