forked from mtusi2026/mtusi2026lab6
112 lines
3.8 KiB
Python
112 lines
3.8 KiB
Python
"""Read stored observations through Grafana's provisioned datasource proxy."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
from urllib.parse import urlencode
|
|
from urllib.request import urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
GRAFANA = os.environ.get('GRAFANA_URL')
|
|
if not GRAFANA:
|
|
# Also supports an ephemeral host port (GRAFANA_PORT=0) in isolated checks.
|
|
address = subprocess.check_output(['docker','compose','port','grafana','3000'], cwd=ROOT, text=True).strip()
|
|
GRAFANA = 'http://' + address
|
|
GRAFANA = GRAFANA.rstrip('/')
|
|
|
|
|
|
def get(path, **params):
|
|
url = GRAFANA + path + ('?' + urlencode(params) if params else '')
|
|
with urlopen(url, timeout=15) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def proxy(source, path, **params):
|
|
return get('/api/datasources/proxy/uid/' + source + path, **params)
|
|
|
|
|
|
def query(expr):
|
|
result = proxy('prometheus', '/api/v1/query', query=expr)
|
|
assert result['status'] == 'success', result
|
|
return result['data']['result']
|
|
|
|
|
|
def scalar(expr, default=0):
|
|
result = query(expr)
|
|
return float(result[0]['value'][1]) if result else default
|
|
|
|
|
|
def logs(expr, start=None, end=None, limit=1000):
|
|
return proxy('loki', '/loki/api/v1/query_range', query=expr,
|
|
start=str(int((start or time.time()-600)*1e9)),
|
|
end=str(int((end or time.time())*1e9)), limit=limit, direction='forward')
|
|
|
|
|
|
def trace(trace_id):
|
|
return proxy('tempo', '/api/traces/' + trace_id)
|
|
|
|
|
|
def compose(*args, check=True, timeout=180):
|
|
return subprocess.run(['docker', 'compose', *args], cwd=ROOT, text=True,
|
|
capture_output=True, check=check, timeout=timeout)
|
|
|
|
|
|
def sql(statement):
|
|
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-c', statement).stdout.strip()
|
|
|
|
|
|
def wait(predicate, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
result = predicate()
|
|
if result:
|
|
return result
|
|
except (OSError, ValueError):
|
|
pass
|
|
time.sleep(1)
|
|
raise AssertionError('Timed out waiting for observed condition')
|
|
|
|
|
|
def save(name, value):
|
|
path = ROOT / 'evidence' / name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')
|
|
|
|
|
|
def spans(value):
|
|
result = []
|
|
def walk(item, service=''):
|
|
if isinstance(item, dict):
|
|
if 'resource' in item:
|
|
for attr in item['resource'].get('attributes', []):
|
|
if attr['key'] == 'service.name':
|
|
service = attr['value']['stringValue']
|
|
if 'spanId' in item and 'name' in item:
|
|
result.append({'service': service, **item})
|
|
for child in item.values():
|
|
walk(child, service)
|
|
elif isinstance(item, list):
|
|
for child in item:
|
|
walk(child, service)
|
|
walk(value)
|
|
return result
|
|
|
|
|
|
def check_dashboard():
|
|
dashboard = get('/api/dashboards/uid/messenger-lab6')['dashboard']
|
|
checked = []
|
|
for panel in dashboard['panels']:
|
|
source = panel.get('datasource', {}).get('uid')
|
|
for target in panel.get('targets', []):
|
|
if source == 'prometheus':
|
|
query(target['expr'])
|
|
elif source == 'loki':
|
|
assert logs(target['expr'])['status'] == 'success'
|
|
elif source == 'tempo':
|
|
# Grafana v12 routes traceql frontend searches to this proxy endpoint.
|
|
assert proxy('tempo', '/api/search', q=target['query'], limit=20).get('traces')
|
|
checked.append({'panel': panel['title'], 'source': source, 'passed': True})
|
|
return {'passed': True, 'panels': checked}
|