forked from mtusi2026/mtusi2026lab6
181 lines
6.3 KiB
Python
181 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify grade-4 metrics and one end-to-end upload trace."""
|
|
|
|
import argparse
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
from attachment_memory import cleanup
|
|
from client import Client
|
|
|
|
|
|
def json_get(base_url, path, params=None, timeout=10):
|
|
query = "?" + urllib.parse.urlencode(params) if params else ""
|
|
with urllib.request.urlopen(base_url.rstrip("/") + path + query, timeout=timeout) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def wait_for(description, action, timeout, interval=2):
|
|
deadline = time.monotonic() + timeout
|
|
last_error = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = action()
|
|
if value:
|
|
return value
|
|
except (AssertionError, KeyError, OSError, urllib.error.URLError, ValueError) as error:
|
|
last_error = error
|
|
time.sleep(interval)
|
|
raise AssertionError(f"timeout waiting for {description}: {last_error}")
|
|
|
|
|
|
def prometheus_scalar(base_url, expression):
|
|
payload = json_get(base_url, "/api/v1/query", {"query": expression})
|
|
assert payload["status"] == "success", payload
|
|
results = payload["data"]["result"]
|
|
return float(results[0]["value"][1]) if results else 0.0
|
|
|
|
|
|
def trace_id_for_request(loki_url, request_id):
|
|
payload = json_get(
|
|
loki_url,
|
|
"/loki/api/v1/query_range",
|
|
{
|
|
"query": f'{{service_name="messenger-api"}} | request_id="{request_id}"',
|
|
"start": str(time.time_ns() - 10 * 60 * 1_000_000_000),
|
|
"end": str(time.time_ns()),
|
|
"direction": "backward",
|
|
"limit": "20",
|
|
},
|
|
)
|
|
assert payload["status"] == "success", payload
|
|
for result in payload["data"]["result"]:
|
|
trace_id = result.get("stream", {}).get("trace_id")
|
|
if trace_id and len(trace_id) == 32:
|
|
return trace_id
|
|
return None
|
|
|
|
|
|
def flatten_trace(payload):
|
|
services = set()
|
|
spans = []
|
|
for batch in payload.get("batches", []):
|
|
attributes = batch.get("resource", {}).get("attributes", [])
|
|
service = next(
|
|
(
|
|
item.get("value", {}).get("stringValue")
|
|
for item in attributes
|
|
if item.get("key") == "service.name"
|
|
),
|
|
None,
|
|
)
|
|
if service:
|
|
services.add(service)
|
|
for scope in batch.get("scopeSpans", []):
|
|
for span in scope.get("spans", []):
|
|
spans.append((service, span))
|
|
return services, spans
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--prometheus-url", default="http://localhost:9090")
|
|
parser.add_argument("--loki-url", default="http://localhost:3100")
|
|
parser.add_argument("--tempo-url", default="http://localhost:3200")
|
|
parser.add_argument("--grafana-url", default="http://localhost:3000")
|
|
parser.add_argument("--timeout", type=int, default=120)
|
|
args = parser.parse_args()
|
|
|
|
client = Client(args.base_url, timeout=20)
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
users = [alice["id"], bob["id"], eve["id"]]
|
|
request_id = str(uuid.uuid4())
|
|
content = open("tests/fixtures/sample.png", "rb").read()
|
|
try:
|
|
response = client.call(
|
|
"POST",
|
|
f'/api/v1/chats/{chat["id"]}/attachments?filename=grade4.png',
|
|
202,
|
|
actor=alice,
|
|
raw=content,
|
|
headers={"Content-Type": "image/png", "X-Request-Id": request_id},
|
|
schema="Attachment",
|
|
)
|
|
attachment = client.wait_ready(alice, response["json"], args.timeout)
|
|
trace_id = wait_for(
|
|
"upload log correlation",
|
|
lambda: trace_id_for_request(args.loki_url, request_id),
|
|
args.timeout,
|
|
)
|
|
trace = wait_for(
|
|
"complete upload trace",
|
|
lambda: json_get(args.tempo_url, f"/api/traces/{trace_id}"),
|
|
args.timeout,
|
|
)
|
|
services, spans = flatten_trace(trace)
|
|
required_services = {
|
|
"messenger-api",
|
|
"messenger-dispatcher",
|
|
"messenger-worker",
|
|
}
|
|
assert required_services.issubset(services), (required_services, services)
|
|
kinds = {span.get("kind") for _, span in spans}
|
|
assert "SPAN_KIND_PRODUCER" in kinds, kinds
|
|
assert "SPAN_KIND_CONSUMER" in kinds, kinds
|
|
assert any(span.get("name", "").startswith("s3.") for _, span in spans)
|
|
assert any(span.get("name", "").startswith("postgresql.") for _, span in spans)
|
|
|
|
metric_checks = {
|
|
"queue_depth": "count(messenger_queue_depth)",
|
|
"queue_oldest_age": "count(messenger_queue_oldest_job_age_seconds)",
|
|
"db_pool": "count(messenger_db_pool_connections)",
|
|
"worker_busy": "count(messenger_worker_busy)",
|
|
"worker_capacity": "count(messenger_worker_capacity)",
|
|
"process_cpu": "count(messenger_process_cpu_time_seconds_total)",
|
|
"process_memory": "count(messenger_process_memory_bytes)",
|
|
"ready_histogram": "sum(messenger_attachment_ready_duration_seconds_count)",
|
|
}
|
|
metrics = {
|
|
name: wait_for(
|
|
name,
|
|
lambda expression=expression: prometheus_scalar(
|
|
args.prometheus_url, expression
|
|
),
|
|
args.timeout,
|
|
)
|
|
for name, expression in metric_checks.items()
|
|
}
|
|
dashboard = json_get(
|
|
args.grafana_url,
|
|
"/api/dashboards/uid/mtusi-lab6-grade4",
|
|
)
|
|
assert dashboard["dashboard"]["title"] == "MTUSI Lab 6 - Grade 4"
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "PASS",
|
|
"attachment_id": attachment["id"],
|
|
"trace_id": trace_id,
|
|
"services": sorted(services),
|
|
"span_kinds": sorted(kinds),
|
|
"span_count": len(spans),
|
|
"metrics": metrics,
|
|
"dashboard_uid": "mtusi-lab6-grade4",
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
finally:
|
|
cleanup(chat["id"], users)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|