Files
mtusi2026lab6/research/observations.py

112 lines
3.8 KiB
Python

"""Read stored observations through Grafana's provisioned datasource proxy."""
import json
import os
from pathlib import Path
import subprocess
import time
from urllib.parse import urlencode
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
GRAFANA = os.environ.get('GRAFANA_URL')
if not GRAFANA:
# Also supports an ephemeral host port (GRAFANA_PORT=0) in isolated checks.
address = subprocess.check_output(['docker','compose','port','grafana','3000'], cwd=ROOT, text=True).strip()
GRAFANA = 'http://' + address
GRAFANA = GRAFANA.rstrip('/')
def get(path, **params):
url = GRAFANA + path + ('?' + urlencode(params) if params else '')
with urlopen(url, timeout=15) as response:
return json.load(response)
def proxy(source, path, **params):
return get('/api/datasources/proxy/uid/' + source + path, **params)
def query(expr):
result = proxy('prometheus', '/api/v1/query', query=expr)
assert result['status'] == 'success', result
return result['data']['result']
def scalar(expr, default=0):
result = query(expr)
return float(result[0]['value'][1]) if result else default
def logs(expr, start=None, end=None, limit=1000):
return proxy('loki', '/loki/api/v1/query_range', query=expr,
start=str(int((start or time.time()-600)*1e9)),
end=str(int((end or time.time())*1e9)), limit=limit, direction='forward')
def trace(trace_id):
return proxy('tempo', '/api/traces/' + trace_id)
def compose(*args, check=True, timeout=180):
return subprocess.run(['docker', 'compose', *args], cwd=ROOT, text=True,
capture_output=True, check=check, timeout=timeout)
def sql(statement):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', statement).stdout.strip()
def wait(predicate, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
result = predicate()
if result:
return result
except (OSError, ValueError):
pass
time.sleep(1)
raise AssertionError('Timed out waiting for observed condition')
def save(name, value):
path = ROOT / 'evidence' / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')
def spans(value):
result = []
def walk(item, service=''):
if isinstance(item, dict):
if 'resource' in item:
for attr in item['resource'].get('attributes', []):
if attr['key'] == 'service.name':
service = attr['value']['stringValue']
if 'spanId' in item and 'name' in item:
result.append({'service': service, **item})
for child in item.values():
walk(child, service)
elif isinstance(item, list):
for child in item:
walk(child, service)
walk(value)
return result
def check_dashboard():
dashboard = get('/api/dashboards/uid/messenger-lab6')['dashboard']
checked = []
for panel in dashboard['panels']:
source = panel.get('datasource', {}).get('uid')
for target in panel.get('targets', []):
if source == 'prometheus':
query(target['expr'])
elif source == 'loki':
assert logs(target['expr'])['status'] == 'success'
elif source == 'tempo':
# Grafana v12 routes traceql frontend searches to this proxy endpoint.
assert proxy('tempo', '/api/search', q=target['query'], limit=20).get('traces')
checked.append({'panel': panel['title'], 'source': source, 'passed': True})
return {'passed': True, 'panels': checked}