"""Reconcile what the application wrote to stdout with what reached Loki and Tempo (stdlib only). A record's stdout line is the ground truth for whether a trace was sampled (trace_sampled) and which SERVER span ID it carries.""" import json import subprocess import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import measurement # noqa: E402 import obs # noqa: E402 from obs import Failure, ROOT # noqa: E402 DEADLINE_S = 60 def stdout_http_logs(since_utc, services=('api',), until_utc=None): """{request_id: parsed stdout JSON} for event=http.request lines.""" args = ['docker', 'compose', '--env-file', obs.stand.ENV_FILE, 'logs', '--no-color', '--no-log-prefix', '--since', since_utc] if until_utc: args += ['--until', until_utc] result = subprocess.run(args + list(services), cwd=ROOT, text=True, capture_output=True, timeout=180) if result.returncode != 0: raise Failure('docker compose logs failed') found = {} for line in result.stdout.splitlines(): if not line.startswith('{'): continue try: row = json.loads(line) except ValueError: continue if row.get('event') == 'http.request' and row.get('request_id'): found[row['request_id']] = row return found def loki_by_request_id(client, request_ids, start_ns, end_ns, deadline_s=DEADLINE_S): """Poll until every id is seen or the deadline passes. Returns ({request_id: labels}, complete) where complete means every query returned a full, un-truncated answer (a Failure would propagate otherwise).""" ids = sorted(request_ids) found = {} deadline = time.monotonic() + deadline_s while True: for i in range(0, len(ids), 100): chunk = [r for r in ids[i:i + 100] if r not in found] if not chunk: continue query = ('{service_name="messenger-api"} | event_name="http.request" ' '| request_id=~"' + '|'.join(chunk) + '"') for row in client.loki_rows(query, start_ns, end_ns): found[row['labels']['request_id']] = row['labels'] if len(found) >= len(ids) or time.monotonic() >= deadline: return found time.sleep(3) def tempo_server_spans(client, targets, deadline_s=DEADLINE_S): """targets: {request_id: (trace_id, span_id)} -> {request_id: bool}.""" found = {} deadline = time.monotonic() + deadline_s while True: for rid, (tid, sid) in targets.items(): if found.get(rid): continue spans = client.get_trace(tid) found[rid] = bool(spans) and any( s['span_id'] == sid and s['kind'] == 'SERVER' for s in spans) if all(found.values()) or time.monotonic() >= deadline: return found time.sleep(3) def reconcile(client, mode, requests, since_utc, start_ns, end_ns): """requests: [{'request_id','status',...}] -> (records, summary). off: stdout only (Loki/Tempo not applicable). on: stdout -> Loki and stdout span_id -> Tempo SERVER span.""" stdout = stdout_http_logs(since_utc) records = [] for req in requests: line = stdout.get(req.get('request_id')) rec = {'request_id': req.get('request_id'), 'status': req.get('status'), 'stdout': None, 'loki': None, 'span': None} if line is not None: rec['stdout'] = {k: line[k] for k in ('trace_id', 'span_id', 'trace_sampled') if k in line} records.append(rec) if mode != 'off': with_stdout = [r for r in records if r['stdout'] is not None] loki = loki_by_request_id( client, [r['request_id'] for r in with_stdout], start_ns, end_ns) for rec in with_stdout: rec['loki'] = rec['request_id'] in loki targets = {r['request_id']: (r['stdout']['trace_id'], r['stdout']['span_id']) for r in with_stdout if r['stdout'].get('trace_sampled') is True} spans = tempo_server_spans(client, targets) for rec in with_stdout: if rec['request_id'] in spans: rec['span'] = spans[rec['request_id']] return records, measurement.summarize_cohort(mode, records)