forked from mtusi2026/mtusi2026lab6
180 lines
5.5 KiB
Python
180 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify the grade-3 telemetry path after the public smoke test."""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
TRACE_ID = re.compile(r"^[0-9a-f]{32}$")
|
|
FORBIDDEN_LABELS = {
|
|
"user_id",
|
|
"chat_id",
|
|
"message_id",
|
|
"attachment_id",
|
|
"request_id",
|
|
"url",
|
|
"filename",
|
|
"message_text",
|
|
}
|
|
REQUIRED_RESOURCE_LABELS = {
|
|
"service_name",
|
|
"service_version",
|
|
"service_instance_id",
|
|
"deployment_environment_name",
|
|
}
|
|
|
|
|
|
def json_get(base_url, path, params=None, timeout=5):
|
|
query = ""
|
|
if params:
|
|
query = "?" + urllib.parse.urlencode(params)
|
|
request = urllib.request.Request(base_url.rstrip("/") + path + query)
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def wait_for(description, action, timeout):
|
|
deadline = time.monotonic() + timeout
|
|
last_error = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = action()
|
|
if value:
|
|
return value
|
|
except (AssertionError, KeyError, OSError, urllib.error.URLError, ValueError) as error:
|
|
last_error = error
|
|
time.sleep(2)
|
|
raise AssertionError(f"timeout waiting for {description}: {last_error}")
|
|
|
|
|
|
def prometheus_scalar(base_url, expression):
|
|
payload = json_get(base_url, "/api/v1/query", {"query": expression})
|
|
assert payload["status"] == "success", payload
|
|
results = payload["data"]["result"]
|
|
if not results:
|
|
return 0.0
|
|
return float(results[0]["value"][1])
|
|
|
|
|
|
def trace_id_from_loki(base_url, service_name):
|
|
now = time.time_ns()
|
|
payload = json_get(
|
|
base_url,
|
|
"/loki/api/v1/query_range",
|
|
{
|
|
"query": f'{{service_name="{service_name}"}}',
|
|
"start": str(now - 30 * 60 * 1_000_000_000),
|
|
"end": str(now),
|
|
"direction": "backward",
|
|
"limit": "100",
|
|
},
|
|
)
|
|
assert payload["status"] == "success", payload
|
|
for stream in payload["data"]["result"]:
|
|
trace_id = stream.get("stream", {}).get("trace_id", "")
|
|
if TRACE_ID.fullmatch(trace_id):
|
|
return trace_id
|
|
for value in stream.get("values", []):
|
|
for candidate in value[1:]:
|
|
if isinstance(candidate, str) and TRACE_ID.fullmatch(candidate.strip()):
|
|
return candidate.strip()
|
|
if isinstance(candidate, dict):
|
|
trace_id = candidate.get("trace_id", "")
|
|
if TRACE_ID.fullmatch(trace_id):
|
|
return trace_id
|
|
return None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--prometheus-url", default="http://localhost:9090")
|
|
parser.add_argument("--loki-url", default="http://localhost:3100")
|
|
parser.add_argument("--tempo-url", default="http://localhost:3200")
|
|
parser.add_argument("--grafana-url", default="http://localhost:3000")
|
|
parser.add_argument("--timeout", type=int, default=90)
|
|
args = parser.parse_args()
|
|
|
|
wait_for(
|
|
"Grafana health",
|
|
lambda: json_get(args.grafana_url, "/api/health").get("database") == "ok",
|
|
args.timeout,
|
|
)
|
|
dashboard = wait_for(
|
|
"provisioned dashboard",
|
|
lambda: json_get(
|
|
args.grafana_url,
|
|
"/api/dashboards/uid/mtusi-lab6-grade3",
|
|
),
|
|
args.timeout,
|
|
)
|
|
assert dashboard["dashboard"]["title"] == "MTUSI Lab 6 — Grade 3"
|
|
|
|
api_series = wait_for(
|
|
"API metrics",
|
|
lambda: prometheus_scalar(
|
|
args.prometheus_url,
|
|
"count(messenger_http_server_requests_total)",
|
|
),
|
|
args.timeout,
|
|
)
|
|
job_series = wait_for(
|
|
"worker metrics",
|
|
lambda: prometheus_scalar(
|
|
args.prometheus_url,
|
|
"count(messenger_worker_jobs_total)",
|
|
),
|
|
args.timeout,
|
|
)
|
|
series_payload = json_get(
|
|
args.prometheus_url,
|
|
"/api/v1/series",
|
|
{"match[]": "messenger_http_server_requests_total"},
|
|
)
|
|
assert series_payload["status"] == "success"
|
|
assert series_payload["data"]
|
|
for labels in series_payload["data"]:
|
|
leaked = FORBIDDEN_LABELS.intersection(labels)
|
|
assert not leaked, f"forbidden metric labels: {sorted(leaked)}"
|
|
missing = REQUIRED_RESOURCE_LABELS.difference(labels)
|
|
assert not missing, f"missing resource labels: {sorted(missing)}"
|
|
assert labels["deployment_environment_name"] == "lab"
|
|
|
|
api_trace_id = wait_for(
|
|
"correlated API log",
|
|
lambda: trace_id_from_loki(args.loki_url, "messenger-api"),
|
|
args.timeout,
|
|
)
|
|
worker_trace_id = wait_for(
|
|
"correlated worker log",
|
|
lambda: trace_id_from_loki(args.loki_url, "messenger-worker"),
|
|
args.timeout,
|
|
)
|
|
api_trace = json_get(args.tempo_url, f"/api/traces/{api_trace_id}")
|
|
worker_trace = json_get(args.tempo_url, f"/api/traces/{worker_trace_id}")
|
|
assert api_trace.get("batches"), "API trace not found in Tempo"
|
|
assert worker_trace.get("batches"), "worker trace not found in Tempo"
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "PASS",
|
|
"api_metric_series": int(api_series),
|
|
"worker_metric_series": int(job_series),
|
|
"api_trace_id": api_trace_id,
|
|
"worker_trace_id": worker_trace_id,
|
|
"dashboard_uid": "mtusi-lab6-grade3",
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|