Files
mtusi2026lab6/scripts/verify_telemetry.py
T

341 lines
13 KiB
Python

"""Actual API-to-worker causality, log correlation, provisioning and privacy."""
import argparse
import hashlib
import io
import json
import os
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, query, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def verify_chain(items, trace_id, *, retry):
assert items and {s["trace_id"] for s in items} == {trace_id}
by_id = {s["span_id"]: s for s in items}
producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"]
consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"]
processing = [s for s in items if s["name"] == "attachment.process"]
assert producers and consumers and processing
assert {s["service"] for s in items} >= {
"messenger-api",
"messenger-relay",
"messenger-worker",
}
for consumer in consumers:
assert (
by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER"
)
for process in processing:
assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER"
assert any(
s["name"] == "s3.put" and s["service"] == "messenger-worker"
for s in items
)
assert any(
s["name"] == "db.postgresql" and s["service"] == "messenger-worker"
for s in items
)
if retry:
assert len(processing) >= 2
assert any(
s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in processing
)
return items
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/telemetry-first.json")
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready(timeout=90)
for service, path in [
("prometheus", "/-/ready"),
("loki", "/ready"),
("tempo", "/ready"),
("otel", "/"),
]:
wait(lambda s=service, p=path: fetch(s, p, json_result=False))
report["backends_ready"] = True
buffer = io.BytesIO()
Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG")
raw = buffer.getvalue()
sentinel = "PrivacyProbe-" + uuid4().hex
with httpx.Client(
base_url=BASE, timeout=15, trust_env=False
) as client:
user, _, session = actor(client)
headers = auth(session)
chat = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic telemetry",
"member_ids": [user["id"]],
},
).json()["id"]
message = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**headers, "Idempotency-Key": uuid4().hex},
json={"text": sentinel},
)
assert message.status_code == 201
cases = []
for retry in [False, True]:
if retry:
compose("stop", "--timeout", "1", "worker")
trace_id = uuid4().hex
trace_header = (
"00-" + trace_id + "-" + uuid4().hex[:16] + "-01"
)
accepted = client.post(
f"/api/v1/chats/{chat}/attachments",
params={"filename": sentinel + ".png"},
headers={
**headers,
"Content-Type": "image/png",
"traceparent": trace_header,
},
content=raw,
)
assert (
accepted.status_code == 202
and accepted.json()["status"] == "queued"
)
attachment = accepted.json()["id"]
job = sql(
f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid"
)
if retry:
sql(
f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)"
)
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
def finished(identity=attachment):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(finished, timeout=90)
original = client.get(
f"/api/v1/attachments/{attachment}/content",
headers=headers,
)
assert original.content == raw
def traced(identity=trace_id, retry_expected=retry):
items = spans(identity)
if len(
[s for s in items if s["name"] == "attachment.process"]
) < (2 if retry_expected else 1):
return None
if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items):
# Independent export batches may arrive in a different
# order. Require the whole chain before accepting it.
try:
return verify_chain(
items, identity, retry=retry_expected
)
except (AssertionError, KeyError):
return None
items = wait(traced, timeout=90)
def correlated(identity=trace_id):
result = logs(identity)
streams = result.get("data", {}).get("result", [])
services = {
x["stream"].get("service_name") for x in streams
}
return (
result
if {"messenger-api", "messenger-worker"} <= services
else None
)
correlated_logs = wait(correlated, timeout=90)
span_ids = {s["span_id"] for s in items}
streams = correlated_logs["data"]["result"]
assert all(
row["stream"].get("span_id") in span_ids for row in streams
)
assert any(
row["stream"].get("request_id")
== accepted.headers["X-Request-Id"]
for row in streams
)
exported = json.dumps(
{"spans": items, "logs": correlated_logs}
)
assert (
sentinel not in exported
and session["token"] not in exported
)
cases.append(
{
"retry": retry,
"attachment_id": attachment,
"job_id": job,
"trace_id": trace_id,
"request_id": accepted.headers["X-Request-Id"],
"original_sha256": hashlib.sha256(raw).hexdigest(),
"attempt": int(
sql(
f"SELECT attempt FROM jobs WHERE id='{job}'::uuid"
)
),
"spans": items,
"correlated_logs": correlated_logs,
}
)
report["causal_uploads"] = cases
assert cases[1]["attempt"] >= 2
report["privacy_sentinels_absent_from_traces_and_logs"] = True
names = wait(
lambda: fetch("prometheus", "/api/v1/label/__name__/values")[
"data"
]
)
required = {
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
"messenger_job_attempts_total",
"messenger_job_duration_seconds_bucket",
"messenger_time_to_ready_seconds_bucket",
"messenger_queue_depth",
"messenger_oldest_job_age_seconds",
"messenger_db_pool_used",
"messenger_db_pool_waiting",
"messenger_worker_busy",
"messenger_worker_capacity",
"messenger_process_cpu_seconds_total",
"messenger_process_memory_bytes",
"messenger_slo_decisions_total",
}
wait(
lambda: (
required
<= set(
fetch("prometheus", "/api/v1/label/__name__/values")[
"data"
]
)
)
)
names = fetch("prometheus", "/api/v1/label/__name__/values")["data"]
report["metric_names"] = [
n for n in names if n.startswith("messenger_")
]
series = fetch(
"prometheus",
"/api/v1/series",
params={"match[]": '{__name__=~"messenger_.*"}'},
)["data"]
forbidden = {
"user_id",
"chat_id",
"message_id",
"attachment_id",
"job_id",
"request_id",
"trace_id",
"filename",
}
assert all(not forbidden.intersection(s) for s in series)
assert all(sentinel not in json.dumps(s) for s in series)
report["metric_series_count"] = len(series)
report["metric_labels_low_cardinality"] = True
url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
password = (ROOT / "secrets/grafana_password").read_text().strip()
with httpx.Client(
base_url=url, auth=("admin", password), trust_env=False, timeout=15
) as grafana:
dashboard = grafana.get("/api/dashboards/uid/messenger-lab6")
dashboard.raise_for_status()
imported = dashboard.json()["dashboard"]
assert len(imported["panels"]) == 18
sources = grafana.get("/api/datasources").json()
assert {s["uid"] for s in sources} == {
"prometheus",
"loki",
"tempo",
}
checks = {}
for uid in ["prometheus", "loki", "tempo"]:
r = grafana.get(f"/api/datasources/uid/{uid}/health")
r.raise_for_status()
assert r.json()["status"] == "OK", uid
checks[uid] = r.json()["status"]
result = grafana.post(
"/api/ds/query",
json={
"queries": [
{
"refId": "A",
"datasource": {
"uid": "prometheus",
"type": "prometheus",
},
"expr": "sum(messenger_http_requests_total)",
"instant": True,
"format": "table",
"intervalMs": 5000,
"maxDataPoints": 10,
}
],
"from": "now-5m",
"to": "now",
},
)
result.raise_for_status()
assert result.json()["results"]["A"]["frames"]
report["grafana"] = {
"dashboard_uid": imported["uid"],
"panels": len(imported["panels"]),
"datasources_health": checks,
"actual_query_frames": len(
result.json()["results"]["A"]["frames"]
),
}
report["quantiles"] = {
str(q): query(
f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))"
)
for q in [0.5, 0.95, 0.99]
}
assert all(report["quantiles"].values())
report["complete"] = True
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana"
)
if __name__ == "__main__":
main()