38 lines
3.8 KiB
Python
38 lines
3.8 KiB
Python
"""Generate the committed, provisioned dashboard from explicit PromQL."""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
panels=[]
|
|
def panel(title,queries,unit='short'):
|
|
i=len(panels)
|
|
panels.append({'id':i+1,'type':'timeseries','title':title,'datasource':{'type':'prometheus','uid':'prometheus'},
|
|
'gridPos':{'x':(i%2)*12,'y':(i//2)*8,'w':12,'h':8},
|
|
'targets':[{'refId':chr(65+j),'expr':expr,'legendFormat':label} for j,(label,expr) in enumerate(queries)],
|
|
'fieldConfig':{'defaults':{'unit':unit,'min':0},'overrides':[]},'options':{'legend':{'displayMode':'list','placement':'bottom'}}})
|
|
|
|
panel('API requests / second',[('{{method}} {{route}}','sum by(method,route)(rate(http_server_requests_total[1m]))')],'reqps')
|
|
panel('API 5xx / all business requests',[('5xx fraction','(sum(rate(http_server_requests_total{status_class="5xx"}[1m])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[1m])),0.001)')],'percentunit')
|
|
panel('API latency: aggregate buckets across replicas',[(f'p{p}',f'histogram_quantile({p/100},sum by(le)(rate(http_server_duration_seconds_bucket[1m])))') for p in (50,95,99)],'s')
|
|
panel('Jobs / second by outcome',[('{{outcome}}','sum by(outcome)(rate(jobs_processed_total[1m]))')],'ops')
|
|
panel('Worker processing latency',[(f'p{p}',f'histogram_quantile({p/100},sum by(le)(rate(job_processing_duration_seconds_bucket[1m])))') for p in (50,95,99)],'s')
|
|
panel('Acceptance to ready latency',[(f'p{p}',f'histogram_quantile({p/100},sum by(le)(rate(attachment_time_to_ready_seconds_bucket[5m])))') for p in (50,95,99)],'s')
|
|
panel('Unfinished durable jobs',[('queue depth','sum(queue_depth)')])
|
|
panel('Oldest unfinished job age',[('oldest','max(queue_oldest_age_seconds)')],'s')
|
|
panel('DB pool',[('used {{process_instance}}','db_pool_used'),('waiting {{process_instance}}','db_pool_waiting')])
|
|
panel('Worker saturation',[('busy / capacity','sum(worker_busy) / clamp_min(sum(worker_capacity),1)')],'percentunit')
|
|
panel('Current process RSS',[('{{service}} {{process_instance}}','process_rss_bytes')],'bytes')
|
|
panel('Process CPU seconds / wall second',[('{{service}} {{process_instance}}','clamp_min(deriv(process_cpu_time_seconds[1m]),0)')],'percentunit')
|
|
panel('API availability burn rate; budget 0.5%',[(f'{w}s',f'(sum(rate(http_server_requests_total{{status_class="5xx"}}[{w}s])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[{w}s])),0.001) / 0.005') for w in (20,60)])
|
|
panel('Ready <=30s burn rate; budget 1%',[(f'{w}s eligible cohort',f'sum(attachment_slo_bad{{window="{w}"}}) / clamp_min(sum(attachment_slo_eligible{{window="{w}"}}),1) / 0.01') for w in (60,300)])
|
|
panel('Collector exporter queue / capacity',[('{{exporter}} size','otelcol_exporter_queue_size'),('{{exporter}} capacity','otelcol_exporter_queue_capacity')])
|
|
panel('Collector failed exports',[('{{exporter}} spans','rate(otelcol_exporter_send_failed_spans_total[1m])'),('{{exporter}} logs','rate(otelcol_exporter_send_failed_log_records_total[1m])')],'ops')
|
|
panels.append({'id':99,'type':'logs','title':'Structured application logs; TraceID links to Tempo','datasource':{'type':'loki','uid':'loki'},
|
|
'gridPos':{'x':0,'y':64,'w':24,'h':10},'targets':[{'refId':'A','expr':'{service_name=~"messenger-(api|worker|dispatcher)"}'}],
|
|
'options':{'showTime':True,'showLabels':False,'wrapLogMessage':True,'sortOrder':'Descending'}})
|
|
doc={'uid':'mtusi-messenger','title':'MTUSI Messenger - signals and SLO','schemaVersion':39,'version':1,
|
|
'editable':False,'timezone':'browser','time':{'from':'now-15m','to':'now'},'refresh':'15s','tags':['mtusi','lab6'],
|
|
'panels':panels,'annotations':{'list':[]},'templating':{'list':[]}}
|
|
path=Path(__file__).resolve().parents[1]/'infra/grafana/dashboards/messenger.json'
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
path.write_text(json.dumps(doc,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|