Files
mtusi2026lab6/scripts/lab6/reconcile.py
T

107 lines
4.3 KiB
Python

"""Reconcile what the application wrote to stdout with what reached Loki and
Tempo (stdlib only). A record's stdout line is the ground truth for whether a
trace was sampled (trace_sampled) and which SERVER span ID it carries."""
import json
import subprocess
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
from obs import Failure, ROOT # noqa: E402
DEADLINE_S = 60
def stdout_http_logs(since_utc, services=('api',), until_utc=None):
"""{request_id: parsed stdout JSON} for event=http.request lines."""
args = ['docker', 'compose', '--env-file', obs.stand.ENV_FILE, 'logs',
'--no-color', '--no-log-prefix', '--since', since_utc]
if until_utc:
args += ['--until', until_utc]
result = subprocess.run(args + list(services), cwd=ROOT, text=True,
capture_output=True, timeout=180)
if result.returncode != 0:
raise Failure('docker compose logs failed')
found = {}
for line in result.stdout.splitlines():
if not line.startswith('{'):
continue
try:
row = json.loads(line)
except ValueError:
continue
if row.get('event') == 'http.request' and row.get('request_id'):
found[row['request_id']] = row
return found
def loki_by_request_id(client, request_ids, start_ns, end_ns, deadline_s=DEADLINE_S):
"""Poll until every id is seen or the deadline passes. Returns
({request_id: labels}, complete) where complete means every query returned
a full, un-truncated answer (a Failure would propagate otherwise)."""
ids = sorted(request_ids)
found = {}
deadline = time.monotonic() + deadline_s
while True:
for i in range(0, len(ids), 100):
chunk = [r for r in ids[i:i + 100] if r not in found]
if not chunk:
continue
query = ('{service_name="messenger-api"} | event_name="http.request" '
'| request_id=~"' + '|'.join(chunk) + '"')
for row in client.loki_rows(query, start_ns, end_ns):
found[row['labels']['request_id']] = row['labels']
if len(found) >= len(ids) or time.monotonic() >= deadline:
return found
time.sleep(3)
def tempo_server_spans(client, targets, deadline_s=DEADLINE_S):
"""targets: {request_id: (trace_id, span_id)} -> {request_id: bool}."""
found = {}
deadline = time.monotonic() + deadline_s
while True:
for rid, (tid, sid) in targets.items():
if found.get(rid):
continue
spans = client.get_trace(tid)
found[rid] = bool(spans) and any(
s['span_id'] == sid and s['kind'] == 'SERVER' for s in spans)
if all(found.values()) or time.monotonic() >= deadline:
return found
time.sleep(3)
def reconcile(client, mode, requests, since_utc, start_ns, end_ns):
"""requests: [{'request_id','status',...}] -> (records, summary).
off: stdout only (Loki/Tempo not applicable). on: stdout -> Loki and
stdout span_id -> Tempo SERVER span."""
stdout = stdout_http_logs(since_utc)
records = []
for req in requests:
line = stdout.get(req.get('request_id'))
rec = {'request_id': req.get('request_id'), 'status': req.get('status'),
'stdout': None, 'loki': None, 'span': None}
if line is not None:
rec['stdout'] = {k: line[k] for k in
('trace_id', 'span_id', 'trace_sampled') if k in line}
records.append(rec)
if mode != 'off':
with_stdout = [r for r in records if r['stdout'] is not None]
loki = loki_by_request_id(
client, [r['request_id'] for r in with_stdout], start_ns, end_ns)
for rec in with_stdout:
rec['loki'] = rec['request_id'] in loki
targets = {r['request_id']: (r['stdout']['trace_id'], r['stdout']['span_id'])
for r in with_stdout
if r['stdout'].get('trace_sampled') is True}
spans = tempo_server_spans(client, targets)
for rec in with_stdout:
if rec['request_id'] in spans:
rec['span'] = spans[rec['request_id']]
return records, measurement.summarize_cohort(mode, records)