import json import re import time import urllib.parse import urllib.request LOKI = "http://loki:3100" TEMPO = "http://tempo:3200" TRACE_PATTERN = re.compile(r"^[0-9a-f]{32}$") SPAN_PATTERN = re.compile(r"^[0-9a-f]{16}$") def get_json(url): with urllib.request.urlopen(url, timeout=15) as response: return json.load(response) def find_log(service, event): params = urllib.parse.urlencode({ "query": f'{{service_name="{service}"}}', "start": str(time.time_ns() - 6 * 3600 * 10**9), "end": str(time.time_ns()), "limit": 500, "direction": "BACKWARD", }) payload = get_json( f"{LOKI}/loki/api/v1/query_range?{params}" ) streams = payload.get("data", {}).get("result", []) for stream in streams: labels = stream.get("stream", {}) for entry in stream.get("values", []): if len(entry) < 2: continue message = entry[1] metadata = entry[2] if len(entry) >= 3 else {} if event not in message and ( metadata.get("event") != event ): continue trace_id = ( metadata.get("trace_id") or labels.get("trace_id") ) span_id = ( metadata.get("span_id") or labels.get("span_id") ) if not trace_id or not TRACE_PATTERN.fullmatch( trace_id ): continue if not span_id or not SPAN_PATTERN.fullmatch( span_id ): continue return { "service": service, "event": event, "message": message, "trace_id": trace_id, "span_id": span_id, } return None def verify_trace(log, expected_kind): trace_id = log["trace_id"] payload = get_json( f"{TEMPO}/api/traces/{trace_id}" ) batches = payload.get("batches", []) spans = [] for batch in batches: for scope in batch.get("scopeSpans", []): spans.extend(scope.get("spans", [])) if not spans: raise AssertionError( f"No spans found for {trace_id}" ) kinds = [str(span.get("kind", "")) for span in spans] if not any(expected_kind in kind for kind in kinds): raise AssertionError( f"Expected span kind {expected_kind}; got {kinds}" ) print(f"Service: {log['service']}") print(f"Event: {log['event']}") print(f"Trace ID: {trace_id}") print(f"Span ID: {log['span_id']}") print(f"Tempo spans: {len(spans)}") for span in spans: print( " Span:", span.get("name"), "kind:", span.get("kind"), ) print("PASS: Loki log linked to Tempo trace") print() def main(): cases = [ ( "messenger-api", "HTTP request completed", "SERVER", ), ( "messenger-worker", "Image job handled", "CONSUMER", ), ] for service, event, kind in cases: log = find_log(service, event) if log is None: raise AssertionError( f"No correlated log found for {service}" ) verify_trace(log, kind) print("PASS: API and worker trace correlation") if __name__ == "__main__": main()