forked from mtusi2026/mtusi2026lab6
107 lines
4.3 KiB
Python
107 lines
4.3 KiB
Python
"""Reconcile what the application wrote to stdout with what reached Loki and
|
|
Tempo (stdlib only). A record's stdout line is the ground truth for whether a
|
|
trace was sampled (trace_sampled) and which SERVER span ID it carries."""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import measurement # noqa: E402
|
|
import obs # noqa: E402
|
|
from obs import Failure, ROOT # noqa: E402
|
|
|
|
DEADLINE_S = 60
|
|
|
|
|
|
def stdout_http_logs(since_utc, services=('api',), until_utc=None):
|
|
"""{request_id: parsed stdout JSON} for event=http.request lines."""
|
|
args = ['docker', 'compose', '--env-file', obs.stand.ENV_FILE, 'logs',
|
|
'--no-color', '--no-log-prefix', '--since', since_utc]
|
|
if until_utc:
|
|
args += ['--until', until_utc]
|
|
result = subprocess.run(args + list(services), cwd=ROOT, text=True,
|
|
capture_output=True, timeout=180)
|
|
if result.returncode != 0:
|
|
raise Failure('docker compose logs failed')
|
|
found = {}
|
|
for line in result.stdout.splitlines():
|
|
if not line.startswith('{'):
|
|
continue
|
|
try:
|
|
row = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if row.get('event') == 'http.request' and row.get('request_id'):
|
|
found[row['request_id']] = row
|
|
return found
|
|
|
|
|
|
def loki_by_request_id(client, request_ids, start_ns, end_ns, deadline_s=DEADLINE_S):
|
|
"""Poll until every id is seen or the deadline passes. Returns
|
|
({request_id: labels}, complete) where complete means every query returned
|
|
a full, un-truncated answer (a Failure would propagate otherwise)."""
|
|
ids = sorted(request_ids)
|
|
found = {}
|
|
deadline = time.monotonic() + deadline_s
|
|
while True:
|
|
for i in range(0, len(ids), 100):
|
|
chunk = [r for r in ids[i:i + 100] if r not in found]
|
|
if not chunk:
|
|
continue
|
|
query = ('{service_name="messenger-api"} | event_name="http.request" '
|
|
'| request_id=~"' + '|'.join(chunk) + '"')
|
|
for row in client.loki_rows(query, start_ns, end_ns):
|
|
found[row['labels']['request_id']] = row['labels']
|
|
if len(found) >= len(ids) or time.monotonic() >= deadline:
|
|
return found
|
|
time.sleep(3)
|
|
|
|
|
|
def tempo_server_spans(client, targets, deadline_s=DEADLINE_S):
|
|
"""targets: {request_id: (trace_id, span_id)} -> {request_id: bool}."""
|
|
found = {}
|
|
deadline = time.monotonic() + deadline_s
|
|
while True:
|
|
for rid, (tid, sid) in targets.items():
|
|
if found.get(rid):
|
|
continue
|
|
spans = client.get_trace(tid)
|
|
found[rid] = bool(spans) and any(
|
|
s['span_id'] == sid and s['kind'] == 'SERVER' for s in spans)
|
|
if all(found.values()) or time.monotonic() >= deadline:
|
|
return found
|
|
time.sleep(3)
|
|
|
|
|
|
def reconcile(client, mode, requests, since_utc, start_ns, end_ns):
|
|
"""requests: [{'request_id','status',...}] -> (records, summary).
|
|
|
|
off: stdout only (Loki/Tempo not applicable). on: stdout -> Loki and
|
|
stdout span_id -> Tempo SERVER span."""
|
|
stdout = stdout_http_logs(since_utc)
|
|
records = []
|
|
for req in requests:
|
|
line = stdout.get(req.get('request_id'))
|
|
rec = {'request_id': req.get('request_id'), 'status': req.get('status'),
|
|
'stdout': None, 'loki': None, 'span': None}
|
|
if line is not None:
|
|
rec['stdout'] = {k: line[k] for k in
|
|
('trace_id', 'span_id', 'trace_sampled') if k in line}
|
|
records.append(rec)
|
|
if mode != 'off':
|
|
with_stdout = [r for r in records if r['stdout'] is not None]
|
|
loki = loki_by_request_id(
|
|
client, [r['request_id'] for r in with_stdout], start_ns, end_ns)
|
|
for rec in with_stdout:
|
|
rec['loki'] = rec['request_id'] in loki
|
|
targets = {r['request_id']: (r['stdout']['trace_id'], r['stdout']['span_id'])
|
|
for r in with_stdout
|
|
if r['stdout'].get('trace_sampled') is True}
|
|
spans = tempo_server_spans(client, targets)
|
|
for rec in with_stdout:
|
|
if rec['request_id'] in spans:
|
|
rec['span'] = spans[rec['request_id']]
|
|
return records, measurement.summarize_cohort(mode, records)
|