Files
mtusi2026lab7/student-tests/telemetry_probe.py
T
2026-11-11 13:53:51 +03:00

90 lines
3.7 KiB
Python

"""Query all three TLS backends from the operator probe namespace."""
import json
import sys
import time
from urllib.parse import urlencode
from trust import check, https
def query(host, port, path):
status, raw, _ = https(host, port, path, "operator")
return status, raw
def promql(expression):
status, raw = query("prometheus", 9090, "/api/v1/query?" + urlencode({"query": expression}))
if status != 200:
raise AssertionError("Prometheus TLS query failed")
return json.loads(raw)["data"]["result"]
def event_trace(trace):
return query("tempo", 3200, "/api/traces/" + trace)
def verify_signals(payload):
trace_id = payload["trace_id"]
loki_path = "/loki/api/v1/query_range?" + urlencode({
"query": '{service_name=~"messenger-api|messenger-worker"} |= "' + trace_id + '"',
"start": str(payload["start"]),
"end": str(time.time_ns() + 60_000_000_000),
"limit": "100",
})
spans = (b"POST /api/v1/chats/{chat_id}/attachments", b"attachments publish",
b"attachment.process", b"s3.", b"db.execute")
end = time.monotonic() + 40
while time.monotonic() < end:
loki_code, logs = query("loki", 3100, loki_path)
tempo_code, trace = event_trace(trace_id)
if loki_code == tempo_code == 200 and b"attachment_ready" in logs and all(fragment in trace for fragment in spans):
break
time.sleep(1.5)
check("upload HTTP request ID and ready worker event share one Loki trace ID",
payload["request_id"].encode() in logs and b"attachment_ready" in logs)
check("Tempo trace joins HTTP, producer, consumer, S3 and DB",
tempo_code == 200 and all(fragment in trace for fragment in spans))
window_path = "/loki/api/v1/query_range?" + urlencode({
"query": '{service_name=~"messenger-api|messenger-worker"}',
"start": str(payload["start"]), "end": str(time.time_ns()), "limit": "2000",
})
status, window_logs = query("loki", 3100, window_path)
message_code, message_trace = event_trace(payload["message_trace_id"])
check("Loki and Tempo contain none of the synthetic sensitive values",
status == message_code == 200 and not any(
item.encode() in window_logs or item.encode() in trace or item.encode() in message_trace
for item in payload["canaries"]))
for expression in (
"app_http_requests_total", "app_http_duration_seconds_bucket",
"jobs_processed_total", "attachment_ready_seconds_bucket", "queue_depth",
"queue_oldest_age_seconds", "worker_capacity", "db_pool_used",
"process_memory_bytes", "otelcol_receiver_accepted_spans_total",
):
for _ in range(18):
if promql(expression):
check("metric " + expression, True)
break
time.sleep(1)
else:
raise AssertionError("metric missing: " + expression)
if __name__ == "__main__":
if sys.argv[1:2] == ["--trace-error"]:
trace_id = sys.argv[2]
for _ in range(15):
status, raw = event_trace(trace_id)
if status == 200 and all(x in raw for x in (b"LockNotAvailable", b"db.execute", b"POST /api/v1/chats/{chat_id}/messages")):
print("PASS Tempo HTTP 503 trace includes DB span with LockNotAvailable")
break
time.sleep(1)
else:
raise SystemExit("expected DB error trace missing")
elif sys.argv[1:]:
for expression in sys.argv[1:]:
result = promql(expression)
print(json.dumps({"expression": expression, "values": [row["value"][1] for row in result]}))
else:
verify_signals(json.load(sys.stdin))