341 lines
13 KiB
Python
341 lines
13 KiB
Python
"""Actual API-to-worker causality, log correlation, provisioning and privacy."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
from datetime import UTC, datetime
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
from PIL import Image
|
|
|
|
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
|
from scripts.observe import fetch, logs, query, spans, wait
|
|
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
|
|
|
|
|
def verify_chain(items, trace_id, *, retry):
|
|
assert items and {s["trace_id"] for s in items} == {trace_id}
|
|
by_id = {s["span_id"]: s for s in items}
|
|
producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"]
|
|
consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"]
|
|
processing = [s for s in items if s["name"] == "attachment.process"]
|
|
assert producers and consumers and processing
|
|
assert {s["service"] for s in items} >= {
|
|
"messenger-api",
|
|
"messenger-relay",
|
|
"messenger-worker",
|
|
}
|
|
for consumer in consumers:
|
|
assert (
|
|
by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER"
|
|
)
|
|
for process in processing:
|
|
assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER"
|
|
assert any(
|
|
s["name"] == "s3.put" and s["service"] == "messenger-worker"
|
|
for s in items
|
|
)
|
|
assert any(
|
|
s["name"] == "db.postgresql" and s["service"] == "messenger-worker"
|
|
for s in items
|
|
)
|
|
if retry:
|
|
assert len(processing) >= 2
|
|
assert any(
|
|
s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
|
for s in processing
|
|
)
|
|
return items
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", default="evidence/telemetry-first.json")
|
|
args = parser.parse_args()
|
|
output = ROOT / args.output
|
|
assert not output.exists(), "Choose a new output path"
|
|
report = {
|
|
"date": datetime.now(UTC).isoformat(),
|
|
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
|
"complete": False,
|
|
}
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
ready(timeout=90)
|
|
for service, path in [
|
|
("prometheus", "/-/ready"),
|
|
("loki", "/ready"),
|
|
("tempo", "/ready"),
|
|
("otel", "/"),
|
|
]:
|
|
wait(lambda s=service, p=path: fetch(s, p, json_result=False))
|
|
report["backends_ready"] = True
|
|
buffer = io.BytesIO()
|
|
Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG")
|
|
raw = buffer.getvalue()
|
|
sentinel = "PrivacyProbe-" + uuid4().hex
|
|
with httpx.Client(
|
|
base_url=BASE, timeout=15, trust_env=False
|
|
) as client:
|
|
user, _, session = actor(client)
|
|
headers = auth(session)
|
|
chat = client.post(
|
|
"/api/v1/chats",
|
|
headers=headers,
|
|
json={
|
|
"title": "Synthetic telemetry",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
).json()["id"]
|
|
message = client.post(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers={**headers, "Idempotency-Key": uuid4().hex},
|
|
json={"text": sentinel},
|
|
)
|
|
assert message.status_code == 201
|
|
cases = []
|
|
for retry in [False, True]:
|
|
if retry:
|
|
compose("stop", "--timeout", "1", "worker")
|
|
trace_id = uuid4().hex
|
|
trace_header = (
|
|
"00-" + trace_id + "-" + uuid4().hex[:16] + "-01"
|
|
)
|
|
accepted = client.post(
|
|
f"/api/v1/chats/{chat}/attachments",
|
|
params={"filename": sentinel + ".png"},
|
|
headers={
|
|
**headers,
|
|
"Content-Type": "image/png",
|
|
"traceparent": trace_header,
|
|
},
|
|
content=raw,
|
|
)
|
|
assert (
|
|
accepted.status_code == 202
|
|
and accepted.json()["status"] == "queued"
|
|
)
|
|
attachment = accepted.json()["id"]
|
|
job = sql(
|
|
f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid"
|
|
)
|
|
if retry:
|
|
sql(
|
|
f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)"
|
|
)
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
|
|
|
def finished(identity=attachment):
|
|
response = client.get(
|
|
f"/api/v1/attachments/{identity}", headers=headers
|
|
)
|
|
return (
|
|
response.status_code == 200
|
|
and response.json()["status"] == "ready"
|
|
)
|
|
|
|
wait(finished, timeout=90)
|
|
original = client.get(
|
|
f"/api/v1/attachments/{attachment}/content",
|
|
headers=headers,
|
|
)
|
|
assert original.content == raw
|
|
|
|
def traced(identity=trace_id, retry_expected=retry):
|
|
items = spans(identity)
|
|
if len(
|
|
[s for s in items if s["name"] == "attachment.process"]
|
|
) < (2 if retry_expected else 1):
|
|
return None
|
|
if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items):
|
|
# Independent export batches may arrive in a different
|
|
# order. Require the whole chain before accepting it.
|
|
try:
|
|
return verify_chain(
|
|
items, identity, retry=retry_expected
|
|
)
|
|
except (AssertionError, KeyError):
|
|
return None
|
|
|
|
items = wait(traced, timeout=90)
|
|
|
|
def correlated(identity=trace_id):
|
|
result = logs(identity)
|
|
streams = result.get("data", {}).get("result", [])
|
|
services = {
|
|
x["stream"].get("service_name") for x in streams
|
|
}
|
|
return (
|
|
result
|
|
if {"messenger-api", "messenger-worker"} <= services
|
|
else None
|
|
)
|
|
|
|
correlated_logs = wait(correlated, timeout=90)
|
|
span_ids = {s["span_id"] for s in items}
|
|
streams = correlated_logs["data"]["result"]
|
|
assert all(
|
|
row["stream"].get("span_id") in span_ids for row in streams
|
|
)
|
|
assert any(
|
|
row["stream"].get("request_id")
|
|
== accepted.headers["X-Request-Id"]
|
|
for row in streams
|
|
)
|
|
exported = json.dumps(
|
|
{"spans": items, "logs": correlated_logs}
|
|
)
|
|
assert (
|
|
sentinel not in exported
|
|
and session["token"] not in exported
|
|
)
|
|
cases.append(
|
|
{
|
|
"retry": retry,
|
|
"attachment_id": attachment,
|
|
"job_id": job,
|
|
"trace_id": trace_id,
|
|
"request_id": accepted.headers["X-Request-Id"],
|
|
"original_sha256": hashlib.sha256(raw).hexdigest(),
|
|
"attempt": int(
|
|
sql(
|
|
f"SELECT attempt FROM jobs WHERE id='{job}'::uuid"
|
|
)
|
|
),
|
|
"spans": items,
|
|
"correlated_logs": correlated_logs,
|
|
}
|
|
)
|
|
report["causal_uploads"] = cases
|
|
assert cases[1]["attempt"] >= 2
|
|
report["privacy_sentinels_absent_from_traces_and_logs"] = True
|
|
names = wait(
|
|
lambda: fetch("prometheus", "/api/v1/label/__name__/values")[
|
|
"data"
|
|
]
|
|
)
|
|
required = {
|
|
"messenger_http_requests_total",
|
|
"messenger_http_duration_seconds_bucket",
|
|
"messenger_job_attempts_total",
|
|
"messenger_job_duration_seconds_bucket",
|
|
"messenger_time_to_ready_seconds_bucket",
|
|
"messenger_queue_depth",
|
|
"messenger_oldest_job_age_seconds",
|
|
"messenger_db_pool_used",
|
|
"messenger_db_pool_waiting",
|
|
"messenger_worker_busy",
|
|
"messenger_worker_capacity",
|
|
"messenger_process_cpu_seconds_total",
|
|
"messenger_process_memory_bytes",
|
|
"messenger_slo_decisions_total",
|
|
}
|
|
wait(
|
|
lambda: (
|
|
required
|
|
<= set(
|
|
fetch("prometheus", "/api/v1/label/__name__/values")[
|
|
"data"
|
|
]
|
|
)
|
|
)
|
|
)
|
|
names = fetch("prometheus", "/api/v1/label/__name__/values")["data"]
|
|
report["metric_names"] = [
|
|
n for n in names if n.startswith("messenger_")
|
|
]
|
|
series = fetch(
|
|
"prometheus",
|
|
"/api/v1/series",
|
|
params={"match[]": '{__name__=~"messenger_.*"}'},
|
|
)["data"]
|
|
forbidden = {
|
|
"user_id",
|
|
"chat_id",
|
|
"message_id",
|
|
"attachment_id",
|
|
"job_id",
|
|
"request_id",
|
|
"trace_id",
|
|
"filename",
|
|
}
|
|
assert all(not forbidden.intersection(s) for s in series)
|
|
assert all(sentinel not in json.dumps(s) for s in series)
|
|
report["metric_series_count"] = len(series)
|
|
report["metric_labels_low_cardinality"] = True
|
|
url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
|
|
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
|
with httpx.Client(
|
|
base_url=url, auth=("admin", password), trust_env=False, timeout=15
|
|
) as grafana:
|
|
dashboard = grafana.get("/api/dashboards/uid/messenger-lab6")
|
|
dashboard.raise_for_status()
|
|
imported = dashboard.json()["dashboard"]
|
|
assert len(imported["panels"]) == 18
|
|
sources = grafana.get("/api/datasources").json()
|
|
assert {s["uid"] for s in sources} == {
|
|
"prometheus",
|
|
"loki",
|
|
"tempo",
|
|
}
|
|
checks = {}
|
|
for uid in ["prometheus", "loki", "tempo"]:
|
|
r = grafana.get(f"/api/datasources/uid/{uid}/health")
|
|
r.raise_for_status()
|
|
assert r.json()["status"] == "OK", uid
|
|
checks[uid] = r.json()["status"]
|
|
result = grafana.post(
|
|
"/api/ds/query",
|
|
json={
|
|
"queries": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": {
|
|
"uid": "prometheus",
|
|
"type": "prometheus",
|
|
},
|
|
"expr": "sum(messenger_http_requests_total)",
|
|
"instant": True,
|
|
"format": "table",
|
|
"intervalMs": 5000,
|
|
"maxDataPoints": 10,
|
|
}
|
|
],
|
|
"from": "now-5m",
|
|
"to": "now",
|
|
},
|
|
)
|
|
result.raise_for_status()
|
|
assert result.json()["results"]["A"]["frames"]
|
|
report["grafana"] = {
|
|
"dashboard_uid": imported["uid"],
|
|
"panels": len(imported["panels"]),
|
|
"datasources_health": checks,
|
|
"actual_query_frames": len(
|
|
result.json()["results"]["A"]["frames"]
|
|
),
|
|
}
|
|
report["quantiles"] = {
|
|
str(q): query(
|
|
f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))"
|
|
)
|
|
for q in [0.5, 0.95, 0.99]
|
|
}
|
|
assert all(report["quantiles"].values())
|
|
report["complete"] = True
|
|
finally:
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
|
output.write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
"PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|