Files
mtusi2026lab6/tests/observability_grade3.py

180 lines
5.5 KiB
Python

#!/usr/bin/env python3
"""Verify the grade-3 telemetry path after the public smoke test."""
import argparse
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
TRACE_ID = re.compile(r"^[0-9a-f]{32}$")
FORBIDDEN_LABELS = {
"user_id",
"chat_id",
"message_id",
"attachment_id",
"request_id",
"url",
"filename",
"message_text",
}
REQUIRED_RESOURCE_LABELS = {
"service_name",
"service_version",
"service_instance_id",
"deployment_environment_name",
}
def json_get(base_url, path, params=None, timeout=5):
query = ""
if params:
query = "?" + urllib.parse.urlencode(params)
request = urllib.request.Request(base_url.rstrip("/") + path + query)
with urllib.request.urlopen(request, timeout=timeout) as response:
return json.load(response)
def wait_for(description, action, timeout):
deadline = time.monotonic() + timeout
last_error = None
while time.monotonic() < deadline:
try:
value = action()
if value:
return value
except (AssertionError, KeyError, OSError, urllib.error.URLError, ValueError) as error:
last_error = error
time.sleep(2)
raise AssertionError(f"timeout waiting for {description}: {last_error}")
def prometheus_scalar(base_url, expression):
payload = json_get(base_url, "/api/v1/query", {"query": expression})
assert payload["status"] == "success", payload
results = payload["data"]["result"]
if not results:
return 0.0
return float(results[0]["value"][1])
def trace_id_from_loki(base_url, service_name):
now = time.time_ns()
payload = json_get(
base_url,
"/loki/api/v1/query_range",
{
"query": f'{{service_name="{service_name}"}}',
"start": str(now - 30 * 60 * 1_000_000_000),
"end": str(now),
"direction": "backward",
"limit": "100",
},
)
assert payload["status"] == "success", payload
for stream in payload["data"]["result"]:
trace_id = stream.get("stream", {}).get("trace_id", "")
if TRACE_ID.fullmatch(trace_id):
return trace_id
for value in stream.get("values", []):
for candidate in value[1:]:
if isinstance(candidate, str) and TRACE_ID.fullmatch(candidate.strip()):
return candidate.strip()
if isinstance(candidate, dict):
trace_id = candidate.get("trace_id", "")
if TRACE_ID.fullmatch(trace_id):
return trace_id
return None
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--prometheus-url", default="http://localhost:9090")
parser.add_argument("--loki-url", default="http://localhost:3100")
parser.add_argument("--tempo-url", default="http://localhost:3200")
parser.add_argument("--grafana-url", default="http://localhost:3000")
parser.add_argument("--timeout", type=int, default=90)
args = parser.parse_args()
wait_for(
"Grafana health",
lambda: json_get(args.grafana_url, "/api/health").get("database") == "ok",
args.timeout,
)
dashboard = wait_for(
"provisioned dashboard",
lambda: json_get(
args.grafana_url,
"/api/dashboards/uid/mtusi-lab6-grade3",
),
args.timeout,
)
assert dashboard["dashboard"]["title"] == "MTUSI Lab 6 — Grade 3"
api_series = wait_for(
"API metrics",
lambda: prometheus_scalar(
args.prometheus_url,
"count(messenger_http_server_requests_total)",
),
args.timeout,
)
job_series = wait_for(
"worker metrics",
lambda: prometheus_scalar(
args.prometheus_url,
"count(messenger_worker_jobs_total)",
),
args.timeout,
)
series_payload = json_get(
args.prometheus_url,
"/api/v1/series",
{"match[]": "messenger_http_server_requests_total"},
)
assert series_payload["status"] == "success"
assert series_payload["data"]
for labels in series_payload["data"]:
leaked = FORBIDDEN_LABELS.intersection(labels)
assert not leaked, f"forbidden metric labels: {sorted(leaked)}"
missing = REQUIRED_RESOURCE_LABELS.difference(labels)
assert not missing, f"missing resource labels: {sorted(missing)}"
assert labels["deployment_environment_name"] == "lab"
api_trace_id = wait_for(
"correlated API log",
lambda: trace_id_from_loki(args.loki_url, "messenger-api"),
args.timeout,
)
worker_trace_id = wait_for(
"correlated worker log",
lambda: trace_id_from_loki(args.loki_url, "messenger-worker"),
args.timeout,
)
api_trace = json_get(args.tempo_url, f"/api/traces/{api_trace_id}")
worker_trace = json_get(args.tempo_url, f"/api/traces/{worker_trace_id}")
assert api_trace.get("batches"), "API trace not found in Tempo"
assert worker_trace.get("batches"), "worker trace not found in Tempo"
print(
json.dumps(
{
"status": "PASS",
"api_metric_series": int(api_series),
"worker_metric_series": int(job_series),
"api_trace_id": api_trace_id,
"worker_trace_id": worker_trace_id,
"dashboard_uid": "mtusi-lab6-grade3",
},
ensure_ascii=False,
indent=2,
)
)
if __name__ == "__main__":
main()