"""Read-only queries through an internal probe, without exposing admin ports.""" import base64 import json import time from urllib.parse import urlencode from scripts.stand import compose ENDPOINTS = { "prometheus": "https://prometheus:9900", "loki": "https://loki:3310", "tempo": "https://tempo:3320", "otel": "https://otel:13333", } def fetch(service, path, *, params=None, json_result=True): url = ENDPOINTS[service] + path if params: url += "?" + urlencode(params, doseq=True) code = ( f"import urllib.request; from src.security_probe import context; service={service!r}; " f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8,context=context('operator' if 'otel' == service else 'grafana')); " "print(r.read().decode())" ) text = compose( "--profile", "operator", "run", "--rm", "--no-deps", "operator-probe", "-c", code, ) return json.loads(text) if json_result else text def query(expression, *, at=None): params = {"query": expression} if at is not None: params["time"] = str(at) result = fetch("prometheus", "/api/v1/query", params=params) assert result["status"] == "success" return result["data"]["result"] def scalar(expression): result = query(expression) return float(result[0]["value"][1]) if result else 0.0 def queries(expressions): urls = { name: ENDPOINTS["prometheus"] + "/api/v1/query?" + urlencode({"query": expression}) for name, expression in expressions.items() } code = ( "import json,urllib.request\nfrom src.security_probe import context\n" f"urls={urls!r}\n" "result={}\n" "for name,url in urls.items():\n" " with urllib.request.urlopen(url,timeout=8,context=context('grafana')) as r: result[name]=json.load(r)['data']['result']\n" "print(json.dumps(result))" ) return json.loads( compose( "--profile", "operator", "run", "--rm", "--no-deps", "operator-probe", "-c", code, ) ) def logs(trace_id, *, start=None): return fetch( "loki", "/loki/api/v1/query_range", params={ "query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="' + trace_id + '"', "start": str(int((start or time.time() - 3600) * 1e9)), "end": str(time.time_ns()), "limit": 500, }, ) def normalize_id(value): if not value: return "" if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value): return value return base64.b64decode(value).hex() def spans(trace_id): result = fetch("tempo", "/api/traces/" + trace_id) flattened = [] for batch in result.get("batches", result.get("resourceSpans", [])): resource = { a["key"]: next(iter(a["value"].values())) for a in batch.get("resource", {}).get("attributes", []) } for scope in batch.get("scopeSpans", []): for item in scope.get("spans", []): flattened.append( { "service": resource.get("service.name"), "version": resource.get("service.version"), "instance": resource.get("service.instance.id"), "name": item["name"], "kind": item.get("kind"), "trace_id": normalize_id(item.get("traceId")), "span_id": normalize_id(item.get("spanId")), "parent_span_id": normalize_id( item.get("parentSpanId") ), "duration_seconds": ( int(item["endTimeUnixNano"]) - int(item["startTimeUnixNano"]) ) / 1e9, "status": item.get("status", {}), "attributes": { a["key"]: next(iter(a["value"].values())) for a in item.get("attributes", []) }, } ) return flattened def wait(condition, *, timeout=90, interval=1): end = time.monotonic() + timeout last_error = None while time.monotonic() < end: try: result = condition() if result: return result except (RuntimeError, ValueError, KeyError) as exc: last_error = type(exc).__name__ time.sleep(interval) raise TimeoutError("Telemetry observation not reached: " + str(last_error))