155 lines
4.8 KiB
Python
155 lines
4.8 KiB
Python
"""Read-only queries through an internal probe, without exposing admin ports."""
|
|
|
|
import base64
|
|
import json
|
|
import time
|
|
from urllib.parse import urlencode
|
|
|
|
from scripts.stand import compose
|
|
|
|
ENDPOINTS = {
|
|
"prometheus": "https://prometheus:9900",
|
|
"loki": "https://loki:3310",
|
|
"tempo": "https://tempo:3320",
|
|
"otel": "https://otel:13333",
|
|
}
|
|
|
|
|
|
def fetch(service, path, *, params=None, json_result=True):
|
|
url = ENDPOINTS[service] + path
|
|
if params:
|
|
url += "?" + urlencode(params, doseq=True)
|
|
code = (
|
|
f"import urllib.request; from src.security_probe import context; service={service!r}; "
|
|
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8,context=context('operator' if 'otel' == service else 'grafana')); "
|
|
"print(r.read().decode())"
|
|
)
|
|
text = compose(
|
|
"--profile",
|
|
"operator",
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"operator-probe",
|
|
"-c",
|
|
code,
|
|
)
|
|
return json.loads(text) if json_result else text
|
|
|
|
|
|
def query(expression, *, at=None):
|
|
params = {"query": expression}
|
|
if at is not None:
|
|
params["time"] = str(at)
|
|
result = fetch("prometheus", "/api/v1/query", params=params)
|
|
assert result["status"] == "success"
|
|
return result["data"]["result"]
|
|
|
|
|
|
def scalar(expression):
|
|
result = query(expression)
|
|
return float(result[0]["value"][1]) if result else 0.0
|
|
|
|
|
|
def queries(expressions):
|
|
urls = {
|
|
name: ENDPOINTS["prometheus"]
|
|
+ "/api/v1/query?"
|
|
+ urlencode({"query": expression})
|
|
for name, expression in expressions.items()
|
|
}
|
|
code = (
|
|
"import json,urllib.request\nfrom src.security_probe import context\n"
|
|
f"urls={urls!r}\n"
|
|
"result={}\n"
|
|
"for name,url in urls.items():\n"
|
|
" with urllib.request.urlopen(url,timeout=8,context=context('grafana')) as r: result[name]=json.load(r)['data']['result']\n"
|
|
"print(json.dumps(result))"
|
|
)
|
|
return json.loads(
|
|
compose(
|
|
"--profile",
|
|
"operator",
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"operator-probe",
|
|
"-c",
|
|
code,
|
|
)
|
|
)
|
|
|
|
|
|
def logs(trace_id, *, start=None):
|
|
return fetch(
|
|
"loki",
|
|
"/loki/api/v1/query_range",
|
|
params={
|
|
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
|
|
+ trace_id
|
|
+ '"',
|
|
"start": str(int((start or time.time() - 3600) * 1e9)),
|
|
"end": str(time.time_ns()),
|
|
"limit": 500,
|
|
},
|
|
)
|
|
|
|
|
|
def normalize_id(value):
|
|
if not value:
|
|
return ""
|
|
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
|
|
return value
|
|
return base64.b64decode(value).hex()
|
|
|
|
|
|
def spans(trace_id):
|
|
result = fetch("tempo", "/api/traces/" + trace_id)
|
|
flattened = []
|
|
for batch in result.get("batches", result.get("resourceSpans", [])):
|
|
resource = {
|
|
a["key"]: next(iter(a["value"].values()))
|
|
for a in batch.get("resource", {}).get("attributes", [])
|
|
}
|
|
for scope in batch.get("scopeSpans", []):
|
|
for item in scope.get("spans", []):
|
|
flattened.append(
|
|
{
|
|
"service": resource.get("service.name"),
|
|
"version": resource.get("service.version"),
|
|
"instance": resource.get("service.instance.id"),
|
|
"name": item["name"],
|
|
"kind": item.get("kind"),
|
|
"trace_id": normalize_id(item.get("traceId")),
|
|
"span_id": normalize_id(item.get("spanId")),
|
|
"parent_span_id": normalize_id(
|
|
item.get("parentSpanId")
|
|
),
|
|
"duration_seconds": (
|
|
int(item["endTimeUnixNano"])
|
|
- int(item["startTimeUnixNano"])
|
|
)
|
|
/ 1e9,
|
|
"status": item.get("status", {}),
|
|
"attributes": {
|
|
a["key"]: next(iter(a["value"].values()))
|
|
for a in item.get("attributes", [])
|
|
},
|
|
}
|
|
)
|
|
return flattened
|
|
|
|
|
|
def wait(condition, *, timeout=90, interval=1):
|
|
end = time.monotonic() + timeout
|
|
last_error = None
|
|
while time.monotonic() < end:
|
|
try:
|
|
result = condition()
|
|
if result:
|
|
return result
|
|
except (RuntimeError, ValueError, KeyError) as exc:
|
|
last_error = type(exc).__name__
|
|
time.sleep(interval)
|
|
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
|