forked from mtusi2026/mtusi2026lab7
149 lines
5.7 KiB
Python
Executable File
149 lines
5.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
TRACE_ID = re.compile(r'(?i)\b[0-9a-f]{32}\b')
|
|
SERVICES = ("messenger-api", "messenger-worker", "messenger-dispatcher")
|
|
|
|
|
|
def get_json(url):
|
|
with urllib.request.urlopen(url, timeout=10) as response:
|
|
return json.loads(response.read(8 << 20).decode("utf-8"))
|
|
|
|
|
|
def tempo_query(base_url, query):
|
|
url = base_url.rstrip("/") + "/api/search?" + urllib.parse.urlencode({"q": query, "limit": 100})
|
|
payload = get_json(url)
|
|
return {item.get("traceID") for item in payload.get("traces", []) if item.get("traceID")}
|
|
|
|
|
|
def extract_trace_ids(payload):
|
|
result = set()
|
|
for stream in payload.get("data", {}).get("result", []):
|
|
for value in stream.get("values", []):
|
|
candidates = [stream.get("stream", {}), value[1] if len(value) > 1 else ""]
|
|
if len(value) > 2:
|
|
candidates.append(value[2])
|
|
for candidate in candidates:
|
|
if isinstance(candidate, dict):
|
|
trace_id = candidate.get("trace_id") or candidate.get("traceid")
|
|
if trace_id:
|
|
result.add(str(trace_id))
|
|
candidate = json.dumps(candidate)
|
|
result.update(match.group(0).lower() for match in TRACE_ID.finditer(str(candidate)))
|
|
return result
|
|
|
|
|
|
def loki_logs(base_url, service, start_ns):
|
|
query = '{service_name="' + service + '"}'
|
|
url = base_url.rstrip("/") + "/loki/api/v1/query_range?" + urllib.parse.urlencode({
|
|
"query": query,
|
|
"start": str(start_ns),
|
|
"limit": 5000,
|
|
"direction": "backward",
|
|
})
|
|
payload = get_json(url)
|
|
count = sum(len(stream.get("values", [])) for stream in payload.get("data", {}).get("result", []))
|
|
return count, extract_trace_ids(payload)
|
|
|
|
|
|
def capture(tempo_url, loki_url, lookback_seconds):
|
|
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
|
|
traces = {service: tempo_query(tempo_url, '{ resource.service.name = "' + service + '" }') for service in SERVICES}
|
|
logs = {}
|
|
log_trace_ids = {}
|
|
for service in SERVICES:
|
|
count, ids = loki_logs(loki_url, service, start_ns)
|
|
logs[service] = count
|
|
log_trace_ids[service] = ids
|
|
api_worker = traces["messenger-api"] & traces["messenger-worker"]
|
|
all_trace_ids = set().union(*traces.values())
|
|
all_log_trace_ids = set().union(*log_trace_ids.values())
|
|
return {
|
|
"traces_by_service": {service: len(ids) for service, ids in traces.items()},
|
|
"logs_by_service": logs,
|
|
"api_worker_shared_traces": len(api_worker),
|
|
"traces_with_correlated_logs": len(all_trace_ids & all_log_trace_ids),
|
|
}
|
|
|
|
|
|
def capture_database(tempo_url, loki_url, lookback_seconds):
|
|
start_ns = int((time.time() - lookback_seconds) * 1_000_000_000)
|
|
traces = tempo_query(
|
|
tempo_url,
|
|
'{ resource.service.name = "messenger-api" && span.db.system = "postgresql" && status = error }',
|
|
)
|
|
log_count, log_trace_ids = loki_logs(loki_url, "messenger-api", start_ns)
|
|
return {
|
|
"database_error_traces": len(traces),
|
|
"api_logs": log_count,
|
|
"database_error_traces_with_logs": len(traces & log_trace_ids),
|
|
}
|
|
|
|
|
|
def complete(report):
|
|
return (
|
|
report["traces_by_service"]["messenger-api"] > 0
|
|
and report["traces_by_service"]["messenger-worker"] > 0
|
|
and report["logs_by_service"]["messenger-api"] > 0
|
|
and report["logs_by_service"]["messenger-worker"] > 0
|
|
and report["api_worker_shared_traces"] > 0
|
|
and report["traces_with_correlated_logs"] > 0
|
|
)
|
|
|
|
|
|
def database_complete(report):
|
|
return report["database_error_traces"] > 0 and report["database_error_traces_with_logs"] > 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Verify trace and log correlation in Tempo and Loki")
|
|
parser.add_argument("--tempo-url", default="http://127.0.0.1:3200")
|
|
parser.add_argument("--loki-url", default="http://127.0.0.1:3100")
|
|
parser.add_argument("--lookback", type=float, default=900)
|
|
parser.add_argument("--wait", type=float, default=45)
|
|
parser.add_argument("--output", required=True)
|
|
parser.add_argument("--require", action="store_true")
|
|
parser.add_argument("--mode", choices=("baseline", "database"), default="baseline")
|
|
args = parser.parse_args()
|
|
|
|
deadline = time.monotonic() + args.wait
|
|
error = None
|
|
report = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
report = capture_database(args.tempo_url, args.loki_url, args.lookback) if args.mode == "database" else capture(args.tempo_url, args.loki_url, args.lookback)
|
|
error = None
|
|
if database_complete(report) if args.mode == "database" else complete(report):
|
|
break
|
|
except (urllib.error.URLError, json.JSONDecodeError, KeyError) as current_error:
|
|
error = str(current_error)
|
|
time.sleep(2)
|
|
|
|
result = {
|
|
"schema_version": 1,
|
|
"captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"lookback_seconds": args.lookback,
|
|
"signals": report,
|
|
"mode": args.mode,
|
|
"complete": bool(report and (database_complete(report) if args.mode == "database" else complete(report))),
|
|
"error": error,
|
|
}
|
|
destination = Path(args.output)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
if args.require and not result["complete"]:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|