58 lines
2.4 KiB
Python
Executable File
58 lines
2.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Runs every query of the provisioned dashboard through Grafana datasource proxies and reports empty panels.
|
|
|
|
Run inside the tools container: docker compose run --rm tools python3 scripts/dashboard-check.py
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
|
DASHBOARD = os.path.join(os.path.dirname(__file__), '..', 'infra', 'grafana', 'dashboards', 'messenger.json')
|
|
|
|
|
|
def get(path, **params):
|
|
url = GRAFANA + path + ('?' + urllib.parse.urlencode(params) if params else '')
|
|
with urllib.request.urlopen(url, timeout=15) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def run(target):
|
|
uid = target['datasource']['uid']
|
|
now = int(time.time())
|
|
if uid == 'prometheus':
|
|
expr = target['expr'].replace('$__rate_interval', '1m')
|
|
body = get('/api/datasources/proxy/uid/prometheus/api/v1/query_range', query=expr, start=now - 1800, end=now, step=30)
|
|
series = body['data']['result']
|
|
last = [s['values'][-1][1] for s in series if s['values']]
|
|
return len(series), ', '.join('%s=%s' % (s['metric'].get('outcome') or s['metric'].get('http_response_status_class') or '', v)
|
|
for s, v in zip(series[:4], last[:4]))
|
|
if uid == 'loki':
|
|
body = get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=target['expr'], limit=20, since='30m')
|
|
return sum(len(s['values']) for s in body['data']['result']), ''
|
|
if uid == 'tempo':
|
|
body = get('/api/datasources/proxy/uid/tempo/api/search', q=target['query'], limit=target.get('limit', 20), start=now - 1800, end=now)
|
|
return len(body.get('traces', [])), ''
|
|
return 0, 'unknown datasource'
|
|
|
|
|
|
def main():
|
|
grafana = get('/api/dashboards/uid/messenger-overview')
|
|
print('Grafana dashboard:', grafana['dashboard']['title'], 'folder:', grafana['meta'].get('folderTitle'))
|
|
dashboard = json.load(open(DASHBOARD, encoding='utf-8'))
|
|
empty = 0
|
|
for panel in dashboard['panels']:
|
|
for target in panel.get('targets', []):
|
|
count, sample = run(target)
|
|
empty += count == 0
|
|
print('%-4s %-62s %-3s %4d %s' % ('OK' if count else 'EMPTY', panel['title'][:62], target['refId'], count, sample[:70]))
|
|
print('\n%s: %d empty queries' % ('PASS' if empty == 0 else 'WARN', empty))
|
|
sys.exit(0 if empty == 0 else 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|