Files
mtusi2026lab6/tests/observability_grade4.py
T

181 lines
6.3 KiB
Python

#!/usr/bin/env python3
"""Verify grade-4 metrics and one end-to-end upload trace."""
import argparse
import json
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from attachment_memory import cleanup
from client import Client
def json_get(base_url, path, params=None, timeout=10):
query = "?" + urllib.parse.urlencode(params) if params else ""
with urllib.request.urlopen(base_url.rstrip("/") + path + query, timeout=timeout) as response:
return json.load(response)
def wait_for(description, action, timeout, interval=2):
deadline = time.monotonic() + timeout
last_error = None
while time.monotonic() < deadline:
try:
value = action()
if value:
return value
except (AssertionError, KeyError, OSError, urllib.error.URLError, ValueError) as error:
last_error = error
time.sleep(interval)
raise AssertionError(f"timeout waiting for {description}: {last_error}")
def prometheus_scalar(base_url, expression):
payload = json_get(base_url, "/api/v1/query", {"query": expression})
assert payload["status"] == "success", payload
results = payload["data"]["result"]
return float(results[0]["value"][1]) if results else 0.0
def trace_id_for_request(loki_url, request_id):
payload = json_get(
loki_url,
"/loki/api/v1/query_range",
{
"query": f'{{service_name="messenger-api"}} | request_id="{request_id}"',
"start": str(time.time_ns() - 10 * 60 * 1_000_000_000),
"end": str(time.time_ns()),
"direction": "backward",
"limit": "20",
},
)
assert payload["status"] == "success", payload
for result in payload["data"]["result"]:
trace_id = result.get("stream", {}).get("trace_id")
if trace_id and len(trace_id) == 32:
return trace_id
return None
def flatten_trace(payload):
services = set()
spans = []
for batch in payload.get("batches", []):
attributes = batch.get("resource", {}).get("attributes", [])
service = next(
(
item.get("value", {}).get("stringValue")
for item in attributes
if item.get("key") == "service.name"
),
None,
)
if service:
services.add(service)
for scope in batch.get("scopeSpans", []):
for span in scope.get("spans", []):
spans.append((service, span))
return services, spans
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--prometheus-url", default="http://localhost:9090")
parser.add_argument("--loki-url", default="http://localhost:3100")
parser.add_argument("--tempo-url", default="http://localhost:3200")
parser.add_argument("--grafana-url", default="http://localhost:3000")
parser.add_argument("--timeout", type=int, default=120)
args = parser.parse_args()
client = Client(args.base_url, timeout=20)
alice, bob, eve, chat = client.bootstrap()
users = [alice["id"], bob["id"], eve["id"]]
request_id = str(uuid.uuid4())
content = open("tests/fixtures/sample.png", "rb").read()
try:
response = client.call(
"POST",
f'/api/v1/chats/{chat["id"]}/attachments?filename=grade4.png',
202,
actor=alice,
raw=content,
headers={"Content-Type": "image/png", "X-Request-Id": request_id},
schema="Attachment",
)
attachment = client.wait_ready(alice, response["json"], args.timeout)
trace_id = wait_for(
"upload log correlation",
lambda: trace_id_for_request(args.loki_url, request_id),
args.timeout,
)
trace = wait_for(
"complete upload trace",
lambda: json_get(args.tempo_url, f"/api/traces/{trace_id}"),
args.timeout,
)
services, spans = flatten_trace(trace)
required_services = {
"messenger-api",
"messenger-dispatcher",
"messenger-worker",
}
assert required_services.issubset(services), (required_services, services)
kinds = {span.get("kind") for _, span in spans}
assert "SPAN_KIND_PRODUCER" in kinds, kinds
assert "SPAN_KIND_CONSUMER" in kinds, kinds
assert any(span.get("name", "").startswith("s3.") for _, span in spans)
assert any(span.get("name", "").startswith("postgresql.") for _, span in spans)
metric_checks = {
"queue_depth": "count(messenger_queue_depth)",
"queue_oldest_age": "count(messenger_queue_oldest_job_age_seconds)",
"db_pool": "count(messenger_db_pool_connections)",
"worker_busy": "count(messenger_worker_busy)",
"worker_capacity": "count(messenger_worker_capacity)",
"process_cpu": "count(messenger_process_cpu_time_seconds_total)",
"process_memory": "count(messenger_process_memory_bytes)",
"ready_histogram": "sum(messenger_attachment_ready_duration_seconds_count)",
}
metrics = {
name: wait_for(
name,
lambda expression=expression: prometheus_scalar(
args.prometheus_url, expression
),
args.timeout,
)
for name, expression in metric_checks.items()
}
dashboard = json_get(
args.grafana_url,
"/api/dashboards/uid/mtusi-lab6-grade4",
)
assert dashboard["dashboard"]["title"] == "MTUSI Lab 6 - Grade 4"
print(
json.dumps(
{
"status": "PASS",
"attachment_id": attachment["id"],
"trace_id": trace_id,
"services": sorted(services),
"span_kinds": sorted(kinds),
"span_count": len(spans),
"metrics": metrics,
"dashboard_uid": "mtusi-lab6-grade4",
},
ensure_ascii=False,
indent=2,
)
)
finally:
cleanup(chat["id"], users)
if __name__ == "__main__":
main()