"""Read stored observations through Grafana's provisioned datasource proxy.""" import json import os from pathlib import Path import subprocess import time from urllib.parse import urlencode from urllib.request import urlopen ROOT = Path(__file__).resolve().parents[1] GRAFANA = os.environ.get('GRAFANA_URL') if not GRAFANA: # Also supports an ephemeral host port (GRAFANA_PORT=0) in isolated checks. address = subprocess.check_output(['docker','compose','port','grafana','3000'], cwd=ROOT, text=True).strip() GRAFANA = 'http://' + address GRAFANA = GRAFANA.rstrip('/') def get(path, **params): url = GRAFANA + path + ('?' + urlencode(params) if params else '') with urlopen(url, timeout=15) as response: return json.load(response) def proxy(source, path, **params): return get('/api/datasources/proxy/uid/' + source + path, **params) def query(expr): result = proxy('prometheus', '/api/v1/query', query=expr) assert result['status'] == 'success', result return result['data']['result'] def scalar(expr, default=0): result = query(expr) return float(result[0]['value'][1]) if result else default def logs(expr, start=None, end=None, limit=1000): return proxy('loki', '/loki/api/v1/query_range', query=expr, start=str(int((start or time.time()-600)*1e9)), end=str(int((end or time.time())*1e9)), limit=limit, direction='forward') def trace(trace_id): return proxy('tempo', '/api/traces/' + trace_id) def compose(*args, check=True, timeout=180): return subprocess.run(['docker', 'compose', *args], cwd=ROOT, text=True, capture_output=True, check=check, timeout=timeout) def sql(statement): return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', statement).stdout.strip() def wait(predicate, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: result = predicate() if result: return result except (OSError, ValueError): pass time.sleep(1) raise AssertionError('Timed out waiting for observed condition') def save(name, value): path = ROOT / 'evidence' / name path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n') def spans(value): result = [] def walk(item, service=''): if isinstance(item, dict): if 'resource' in item: for attr in item['resource'].get('attributes', []): if attr['key'] == 'service.name': service = attr['value']['stringValue'] if 'spanId' in item and 'name' in item: result.append({'service': service, **item}) for child in item.values(): walk(child, service) elif isinstance(item, list): for child in item: walk(child, service) walk(value) return result def check_dashboard(): dashboard = get('/api/dashboards/uid/messenger-lab6')['dashboard'] checked = [] for panel in dashboard['panels']: source = panel.get('datasource', {}).get('uid') for target in panel.get('targets', []): if source == 'prometheus': query(target['expr']) elif source == 'loki': assert logs(target['expr'])['status'] == 'success' elif source == 'tempo': # Grafana v12 routes traceql frontend searches to this proxy endpoint. assert proxy('tempo', '/api/search', q=target['query'], limit=20).get('traces') checked.append({'panel': panel['title'], 'source': source, 'passed': True}) return {'passed': True, 'panels': checked}