Files
mtusi2026lab7/scripts/observe.py
T
wh1tet3a f2ea9d65a8
Skeleton contracts / contracts (pull_request) Canceled after 0s
feat(security): add operator rotation and recovery commands
2026-11-20 19:02:00 +03:00

155 lines
4.8 KiB
Python

"""Read-only queries through an internal probe, without exposing admin ports."""
import base64
import json
import time
from urllib.parse import urlencode
from scripts.stand import compose
ENDPOINTS = {
"prometheus": "https://prometheus:9900",
"loki": "https://loki:3310",
"tempo": "https://tempo:3320",
"otel": "https://otel:13333",
}
def fetch(service, path, *, params=None, json_result=True):
url = ENDPOINTS[service] + path
if params:
url += "?" + urlencode(params, doseq=True)
code = (
f"import urllib.request; from src.security_probe import context; service={service!r}; "
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8,context=context('operator' if 'otel' == service else 'grafana')); "
"print(r.read().decode())"
)
text = compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-probe",
"-c",
code,
)
return json.loads(text) if json_result else text
def query(expression, *, at=None):
params = {"query": expression}
if at is not None:
params["time"] = str(at)
result = fetch("prometheus", "/api/v1/query", params=params)
assert result["status"] == "success"
return result["data"]["result"]
def scalar(expression):
result = query(expression)
return float(result[0]["value"][1]) if result else 0.0
def queries(expressions):
urls = {
name: ENDPOINTS["prometheus"]
+ "/api/v1/query?"
+ urlencode({"query": expression})
for name, expression in expressions.items()
}
code = (
"import json,urllib.request\nfrom src.security_probe import context\n"
f"urls={urls!r}\n"
"result={}\n"
"for name,url in urls.items():\n"
" with urllib.request.urlopen(url,timeout=8,context=context('grafana')) as r: result[name]=json.load(r)['data']['result']\n"
"print(json.dumps(result))"
)
return json.loads(
compose(
"--profile",
"operator",
"run",
"--rm",
"--no-deps",
"operator-probe",
"-c",
code,
)
)
def logs(trace_id, *, start=None):
return fetch(
"loki",
"/loki/api/v1/query_range",
params={
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
+ trace_id
+ '"',
"start": str(int((start or time.time() - 3600) * 1e9)),
"end": str(time.time_ns()),
"limit": 500,
},
)
def normalize_id(value):
if not value:
return ""
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
return value
return base64.b64decode(value).hex()
def spans(trace_id):
result = fetch("tempo", "/api/traces/" + trace_id)
flattened = []
for batch in result.get("batches", result.get("resourceSpans", [])):
resource = {
a["key"]: next(iter(a["value"].values()))
for a in batch.get("resource", {}).get("attributes", [])
}
for scope in batch.get("scopeSpans", []):
for item in scope.get("spans", []):
flattened.append(
{
"service": resource.get("service.name"),
"version": resource.get("service.version"),
"instance": resource.get("service.instance.id"),
"name": item["name"],
"kind": item.get("kind"),
"trace_id": normalize_id(item.get("traceId")),
"span_id": normalize_id(item.get("spanId")),
"parent_span_id": normalize_id(
item.get("parentSpanId")
),
"duration_seconds": (
int(item["endTimeUnixNano"])
- int(item["startTimeUnixNano"])
)
/ 1e9,
"status": item.get("status", {}),
"attributes": {
a["key"]: next(iter(a["value"].values()))
for a in item.get("attributes", [])
},
}
)
return flattened
def wait(condition, *, timeout=90, interval=1):
end = time.monotonic() + timeout
last_error = None
while time.monotonic() < end:
try:
result = condition()
if result:
return result
except (RuntimeError, ValueError, KeyError) as exc:
last_error = type(exc).__name__
time.sleep(interval)
raise TimeoutError("Telemetry observation not reached: " + str(last_error))