"""Read-only queries through an internal probe, without exposing admin ports.""" import base64 import json import time from urllib.parse import urlencode from scripts.stand import compose ENDPOINTS = { "prometheus": "http://prometheus:9090", "loki": "http://loki:3100", "tempo": "http://tempo:3200", "otel": "http://otel:13133", } def fetch(service, path, *, params=None, json_result=True): url = ENDPOINTS[service] + path if params: url += "?" + urlencode(params, doseq=True) code = ( "import urllib.request; " f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); " "print(r.read().decode())" ) text = compose("exec", "-T", "observer", "python", "-c", code) return json.loads(text) if json_result else text def query(expression, *, at=None): params = {"query": expression} if at is not None: params["time"] = str(at) result = fetch("prometheus", "/api/v1/query", params=params) assert result["status"] == "success" return result["data"]["result"] def scalar(expression): result = query(expression) return float(result[0]["value"][1]) if result else 0.0 def queries(expressions): urls = { name: ENDPOINTS["prometheus"] + "/api/v1/query?" + urlencode({"query": expression}) for name, expression in expressions.items() } code = ( "import json,urllib.request\n" f"urls={urls!r}\n" "result={}\n" "for name,url in urls.items():\n" " with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n" "print(json.dumps(result))" ) return json.loads(compose("exec", "-T", "observer", "python", "-c", code)) def logs(trace_id, *, start=None): return fetch( "loki", "/loki/api/v1/query_range", params={ "query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="' + trace_id + '"', "start": str(int((start or time.time() - 3600) * 1e9)), "end": str(time.time_ns()), "limit": 500, }, ) def normalize_id(value): if not value: return "" if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value): return value return base64.b64decode(value).hex() def spans(trace_id): result = fetch("tempo", "/api/traces/" + trace_id) flattened = [] for batch in result.get("batches", result.get("resourceSpans", [])): resource = { a["key"]: next(iter(a["value"].values())) for a in batch.get("resource", {}).get("attributes", []) } for scope in batch.get("scopeSpans", []): for item in scope.get("spans", []): flattened.append( { "service": resource.get("service.name"), "version": resource.get("service.version"), "instance": resource.get("service.instance.id"), "name": item["name"], "kind": item.get("kind"), "trace_id": normalize_id(item.get("traceId")), "span_id": normalize_id(item.get("spanId")), "parent_span_id": normalize_id( item.get("parentSpanId") ), "duration_seconds": ( int(item["endTimeUnixNano"]) - int(item["startTimeUnixNano"]) ) / 1e9, "status": item.get("status", {}), "attributes": { a["key"]: next(iter(a["value"].values())) for a in item.get("attributes", []) }, } ) return flattened def wait(condition, *, timeout=90, interval=1): end = time.monotonic() + timeout last_error = None while time.monotonic() < end: try: result = condition() if result: return result except (RuntimeError, ValueError, KeyError) as exc: last_error = type(exc).__name__ time.sleep(interval) raise TimeoutError("Telemetry observation not reached: " + str(last_error))