#!/usr/bin/env python3 """Verify live provisioned sources, dashboard queries and metric label policy.""" import datetime import json import re import urllib.parse from observability_common import get_json, prom, save def main(): targets = get_json('http://127.0.0.1:19090/api/v1/targets')['data']['activeTargets'] assert targets and all(t['health'] == 'up' for t in targets), 'Prometheus target is down' dashboard = get_json('http://127.0.0.1:13000/api/dashboards/uid/messenger')['dashboard'] assert dashboard['uid'] == 'messenger' and len(dashboard['panels']) >= 10 results = [] for panel in dashboard['panels']: if panel.get('datasource', {}).get('uid') != 'prometheus': continue for target in panel.get('targets', []): query = target['expr'].replace('$__rate_interval', '1m') rows = prom(query) results.append({'panel': panel['title'], 'query': query, 'series': len(rows)}) query = '{__name__=~"messenger_.*"}' series = get_json('http://127.0.0.1:19090/api/v1/series?' + urllib.parse.urlencode({'match[]': query}))['data'] prohibited = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url', 'http_target'} for row in series: assert not prohibited.intersection(row), 'Forbidden metric label' assert not re.search(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', row.get('route', '')), 'Raw UUID in route' required = {'messenger_http_requests_total', 'messenger_http_duration_seconds_bucket', 'messenger_jobs_total', 'messenger_job_duration_seconds_bucket', 'messenger_attachment_ready_seconds_bucket', 'messenger_queue_depth', 'messenger_queue_oldest_age_seconds', 'messenger_db_pool_used', 'messenger_db_pool_waiting', 'messenger_worker_capacity', 'messenger_worker_busy'} names = {row['__name__'] for row in series} assert required <= names, 'Required metrics absent: ' + str(sorted(required-names)) report = {'at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'status': 'PASS', 'targets': {t['labels']['job']: t['health'] for t in targets}, 'dashboard_uid': dashboard['uid'], 'panel_count': len(dashboard['panels']), 'queries': results, 'metric_series_checked': len(series), 'required_metrics': sorted(required), 'label_policy': 'PASS'} save('evidence/lab6-provisioning.json', report) print(json.dumps({'status': 'PASS', 'panels': report['panel_count'], 'queries': len(results), 'label_policy': 'PASS'})) if __name__ == '__main__': main()