forked from mtusi2026/mtusi2026lab6
66 lines
2.7 KiB
Python
66 lines
2.7 KiB
Python
"""Read stored signals through internal backend APIs; no public admin ports."""
|
|
import json
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import urllib.error
|
|
import base64
|
|
from pipeline_helpers import *
|
|
|
|
|
|
def backend(url, text=False):
|
|
# Query via Grafana proxy: observation must work even while worker is stopped.
|
|
target=urllib.parse.urlsplit(url)
|
|
sources={'prometheus:9090':'prometheus','loki:3100':'loki','tempo:3200':'tempo'}
|
|
check(target.netloc in sources,'unknown telemetry backend')
|
|
password=(ROOT/'secrets/grafana_admin_password').read_text().strip()
|
|
base=os.environ.get('GRAFANA_URL','http://localhost:13000').rstrip('/')
|
|
address=base+'/api/datasources/proxy/uid/'+sources[target.netloc]+target.path
|
|
if target.query:address+='?'+target.query
|
|
auth=base64.b64encode(('admin:'+password).encode()).decode()
|
|
request=urllib.request.Request(address,headers={'Accept':'application/json','Authorization':'Basic '+auth})
|
|
try:
|
|
with urllib.request.urlopen(request,timeout=8) as response:output=response.read().decode()
|
|
return output if text else json.loads(output)
|
|
except (urllib.error.URLError,OSError,ValueError):
|
|
raise Failure('telemetry backend not ready: '+sources[target.netloc]) from None
|
|
|
|
|
|
def prom(expr):
|
|
result=backend('http://prometheus:9090/api/v1/query?'+urllib.parse.urlencode({'query':expr}))
|
|
check(result['status']=='success','Prometheus query failed')
|
|
return result['data']['result']
|
|
|
|
|
|
def number(expr, default=0):
|
|
values=prom(expr)
|
|
return float(values[0]['value'][1]) if values else default
|
|
|
|
|
|
def trace_data(trace_id):
|
|
return backend('http://tempo:3200/api/traces/'+trace_id)
|
|
|
|
|
|
def spans(value):
|
|
found=[]
|
|
if isinstance(value,dict):
|
|
if 'spanId' in value and 'name' in value:found.append(value)
|
|
for child in value.values():found.extend(spans(child))
|
|
elif isinstance(value,list):
|
|
for child in value:found.extend(spans(child))
|
|
return found
|
|
|
|
|
|
def logs(trace_id=None, service=None):
|
|
selector='{service_name="'+service+'"}' if service else '{service_name=~"messenger-api|messenger-worker|messenger-dispatcher"}'
|
|
query=selector+' | json'
|
|
if trace_id:query+=' | trace_id="'+trace_id+'"'
|
|
data=backend('http://loki:3100/loki/api/v1/query_range?'+urllib.parse.urlencode({'query':query,'start':str(time.time_ns()-900_000_000_000),'limit':'1000'}))
|
|
return [json.loads(row[1]) for stream in data['data']['result'] for row in stream['values']]
|
|
|
|
|
|
def alert(name):
|
|
result=backend('http://prometheus:9090/api/v1/alerts')
|
|
matches=[a for a in result['data']['alerts'] if a['labels'].get('alertname')==name]
|
|
return matches[0]['state'] if matches else 'resolved'
|