Files
mtusi2026lab6/scripts/observe.py
T

135 lines
4.4 KiB
Python

"""Read-only queries through an internal probe, without exposing admin ports."""
import base64
import json
import time
from urllib.parse import urlencode
from scripts.stand import compose
ENDPOINTS = {
"prometheus": "http://prometheus:9090",
"loki": "http://loki:3100",
"tempo": "http://tempo:3200",
"otel": "http://otel:13133",
}
def fetch(service, path, *, params=None, json_result=True):
url = ENDPOINTS[service] + path
if params:
url += "?" + urlencode(params, doseq=True)
code = (
"import urllib.request; "
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); "
"print(r.read().decode())"
)
text = compose("exec", "-T", "observer", "python", "-c", code)
return json.loads(text) if json_result else text
def query(expression, *, at=None):
params = {"query": expression}
if at is not None:
params["time"] = str(at)
result = fetch("prometheus", "/api/v1/query", params=params)
assert result["status"] == "success"
return result["data"]["result"]
def scalar(expression):
result = query(expression)
return float(result[0]["value"][1]) if result else 0.0
def queries(expressions):
urls = {
name: ENDPOINTS["prometheus"]
+ "/api/v1/query?"
+ urlencode({"query": expression})
for name, expression in expressions.items()
}
code = (
"import json,urllib.request\n"
f"urls={urls!r}\n"
"result={}\n"
"for name,url in urls.items():\n"
" with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n"
"print(json.dumps(result))"
)
return json.loads(compose("exec", "-T", "observer", "python", "-c", code))
def logs(trace_id, *, start=None):
return fetch(
"loki",
"/loki/api/v1/query_range",
params={
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
+ trace_id
+ '"',
"start": str(int((start or time.time() - 3600) * 1e9)),
"end": str(time.time_ns()),
"limit": 500,
},
)
def normalize_id(value):
if not value:
return ""
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
return value
return base64.b64decode(value).hex()
def spans(trace_id):
result = fetch("tempo", "/api/traces/" + trace_id)
flattened = []
for batch in result.get("batches", result.get("resourceSpans", [])):
resource = {
a["key"]: next(iter(a["value"].values()))
for a in batch.get("resource", {}).get("attributes", [])
}
for scope in batch.get("scopeSpans", []):
for item in scope.get("spans", []):
flattened.append(
{
"service": resource.get("service.name"),
"version": resource.get("service.version"),
"instance": resource.get("service.instance.id"),
"name": item["name"],
"kind": item.get("kind"),
"trace_id": normalize_id(item.get("traceId")),
"span_id": normalize_id(item.get("spanId")),
"parent_span_id": normalize_id(
item.get("parentSpanId")
),
"duration_seconds": (
int(item["endTimeUnixNano"])
- int(item["startTimeUnixNano"])
)
/ 1e9,
"status": item.get("status", {}),
"attributes": {
a["key"]: next(iter(a["value"].values()))
for a in item.get("attributes", [])
},
}
)
return flattened
def wait(condition, *, timeout=90, interval=1):
end = time.monotonic() + timeout
last_error = None
while time.monotonic() < end:
try:
result = condition()
if result:
return result
except (RuntimeError, ValueError, KeyError) as exc:
last_error = type(exc).__name__
time.sleep(interval)
raise TimeoutError("Telemetry observation not reached: " + str(last_error))