"""Actual API-to-worker causality, log correlation, provisioning and privacy.""" import argparse import hashlib import io import json import os from datetime import UTC, datetime from uuid import uuid4 import httpx from PIL import Image os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") from scripts.observe import fetch, logs, query, spans, wait from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql def verify_chain(items, trace_id, *, retry): assert items and {s["trace_id"] for s in items} == {trace_id} by_id = {s["span_id"]: s for s in items} producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"] consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"] processing = [s for s in items if s["name"] == "attachment.process"] assert producers and consumers and processing assert {s["service"] for s in items} >= { "messenger-api", "messenger-relay", "messenger-worker", } for consumer in consumers: assert ( by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER" ) for process in processing: assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER" assert any( s["name"] == "s3.put" and s["service"] == "messenger-worker" for s in items ) assert any( s["name"] == "db.postgresql" and s["service"] == "messenger-worker" for s in items ) if retry: assert len(processing) >= 2 assert any( s["status"].get("code") in {2, "STATUS_CODE_ERROR"} for s in processing ) return items def main(): parser = argparse.ArgumentParser() parser.add_argument("--output", default="evidence/telemetry-first.json") args = parser.parse_args() output = ROOT / args.output assert not output.exists(), "Choose a new output path" report = { "date": datetime.now(UTC).isoformat(), "source_commit": command(["git", "rev-parse", "HEAD"]), "complete": False, } output.parent.mkdir(parents=True, exist_ok=True) try: ready(timeout=90) for service, path in [ ("prometheus", "/-/ready"), ("loki", "/ready"), ("tempo", "/ready"), ("otel", "/"), ]: wait(lambda s=service, p=path: fetch(s, p, json_result=False)) report["backends_ready"] = True buffer = io.BytesIO() Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG") raw = buffer.getvalue() sentinel = "PrivacyProbe-" + uuid4().hex with httpx.Client( base_url=BASE, timeout=15, trust_env=False ) as client: user, _, session = actor(client) headers = auth(session) chat = client.post( "/api/v1/chats", headers=headers, json={ "title": "Synthetic telemetry", "member_ids": [user["id"]], }, ).json()["id"] message = client.post( f"/api/v1/chats/{chat}/messages", headers={**headers, "Idempotency-Key": uuid4().hex}, json={"text": sentinel}, ) assert message.status_code == 201 cases = [] for retry in [False, True]: if retry: compose("stop", "--timeout", "1", "worker") trace_id = uuid4().hex trace_header = ( "00-" + trace_id + "-" + uuid4().hex[:16] + "-01" ) accepted = client.post( f"/api/v1/chats/{chat}/attachments", params={"filename": sentinel + ".png"}, headers={ **headers, "Content-Type": "image/png", "traceparent": trace_header, }, content=raw, ) assert ( accepted.status_code == 202 and accepted.json()["status"] == "queued" ) attachment = accepted.json()["id"] job = sql( f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid" ) if retry: sql( f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)" ) compose("up", "-d", "--no-deps", "--no-recreate", "worker") def finished(identity=attachment): response = client.get( f"/api/v1/attachments/{identity}", headers=headers ) return ( response.status_code == 200 and response.json()["status"] == "ready" ) wait(finished, timeout=90) original = client.get( f"/api/v1/attachments/{attachment}/content", headers=headers, ) assert original.content == raw def traced(identity=trace_id, retry_expected=retry): items = spans(identity) if len( [s for s in items if s["name"] == "attachment.process"] ) < (2 if retry_expected else 1): return None if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items): # Independent export batches may arrive in a different # order. Require the whole chain before accepting it. try: return verify_chain( items, identity, retry=retry_expected ) except (AssertionError, KeyError): return None items = wait(traced, timeout=90) def correlated(identity=trace_id): result = logs(identity) streams = result.get("data", {}).get("result", []) services = { x["stream"].get("service_name") for x in streams } return ( result if {"messenger-api", "messenger-worker"} <= services else None ) correlated_logs = wait(correlated, timeout=90) span_ids = {s["span_id"] for s in items} streams = correlated_logs["data"]["result"] assert all( row["stream"].get("span_id") in span_ids for row in streams ) assert any( row["stream"].get("request_id") == accepted.headers["X-Request-Id"] for row in streams ) exported = json.dumps( {"spans": items, "logs": correlated_logs} ) assert ( sentinel not in exported and session["token"] not in exported ) cases.append( { "retry": retry, "attachment_id": attachment, "job_id": job, "trace_id": trace_id, "request_id": accepted.headers["X-Request-Id"], "original_sha256": hashlib.sha256(raw).hexdigest(), "attempt": int( sql( f"SELECT attempt FROM jobs WHERE id='{job}'::uuid" ) ), "spans": items, "correlated_logs": correlated_logs, } ) report["causal_uploads"] = cases assert cases[1]["attempt"] >= 2 report["privacy_sentinels_absent_from_traces_and_logs"] = True names = wait( lambda: fetch("prometheus", "/api/v1/label/__name__/values")[ "data" ] ) required = { "messenger_http_requests_total", "messenger_http_duration_seconds_bucket", "messenger_job_attempts_total", "messenger_job_duration_seconds_bucket", "messenger_time_to_ready_seconds_bucket", "messenger_queue_depth", "messenger_oldest_job_age_seconds", "messenger_db_pool_used", "messenger_db_pool_waiting", "messenger_worker_busy", "messenger_worker_capacity", "messenger_process_cpu_seconds_total", "messenger_process_memory_bytes", "messenger_slo_decisions_total", } wait( lambda: ( required <= set( fetch("prometheus", "/api/v1/label/__name__/values")[ "data" ] ) ) ) names = fetch("prometheus", "/api/v1/label/__name__/values")["data"] report["metric_names"] = [ n for n in names if n.startswith("messenger_") ] series = fetch( "prometheus", "/api/v1/series", params={"match[]": '{__name__=~"messenger_.*"}'}, )["data"] forbidden = { "user_id", "chat_id", "message_id", "attachment_id", "job_id", "request_id", "trace_id", "filename", } assert all(not forbidden.intersection(s) for s in series) assert all(sentinel not in json.dumps(s) for s in series) report["metric_series_count"] = len(series) report["metric_labels_low_cardinality"] = True url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006") password = (ROOT / "secrets/grafana_password").read_text().strip() with httpx.Client( base_url=url, auth=("admin", password), trust_env=False, timeout=15 ) as grafana: dashboard = grafana.get("/api/dashboards/uid/messenger-lab6") dashboard.raise_for_status() imported = dashboard.json()["dashboard"] assert len(imported["panels"]) == 18 sources = grafana.get("/api/datasources").json() assert {s["uid"] for s in sources} == { "prometheus", "loki", "tempo", } checks = {} for uid in ["prometheus", "loki", "tempo"]: r = grafana.get(f"/api/datasources/uid/{uid}/health") r.raise_for_status() assert r.json()["status"] == "OK", uid checks[uid] = r.json()["status"] result = grafana.post( "/api/ds/query", json={ "queries": [ { "refId": "A", "datasource": { "uid": "prometheus", "type": "prometheus", }, "expr": "sum(messenger_http_requests_total)", "instant": True, "format": "table", "intervalMs": 5000, "maxDataPoints": 10, } ], "from": "now-5m", "to": "now", }, ) result.raise_for_status() assert result.json()["results"]["A"]["frames"] report["grafana"] = { "dashboard_uid": imported["uid"], "panels": len(imported["panels"]), "datasources_health": checks, "actual_query_frames": len( result.json()["results"]["A"]["frames"] ), } report["quantiles"] = { str(q): query( f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))" ) for q in [0.5, 0.95, 0.99] } assert all(report["quantiles"].values()) report["complete"] = True finally: compose("up", "-d", "--no-deps", "--no-recreate", "worker") output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) print( "PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana" ) if __name__ == "__main__": main()