test(observability): verify causal signals and controlled SLO incidents

This commit is contained in:
2026-11-20 16:03:47 +03:00
parent 8f3ff33b2d
commit 752a5e6c0d
8 changed files with 1099 additions and 21 deletions
+2 -2
View File
@@ -39,8 +39,8 @@ up: bootstrap
down:
docker compose down
lint:
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
test-student:
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
verify:
+8
View File
@@ -0,0 +1,8 @@
{
"source_commit": "8f3ff33b2dcb1cf3082535acf99ee4045a80400d",
"command": "BASE_URL=http://127.0.0.1:8086 uv run --locked pytest student-tests -q",
"elapsed_seconds": 417.533,
"exit_code": 0,
"summary": "112 passed in 416.52s (0:06:56)",
"result": "PASS"
}
+2 -2
View File
@@ -679,7 +679,7 @@
},
"targets": [
{
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
"expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))",
"refId": "A",
"legendFormat": "spans/s",
"datasource": {
@@ -688,7 +688,7 @@
}
},
{
"expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))",
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))",
"refId": "B",
"legendFormat": "logs/s",
"datasource": {
+34 -17
View File
@@ -15,9 +15,13 @@ processors:
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
metric_expiration: 5m
resource_constant_labels:
included:
- service.name
- service.instance.id
- service.version
- deployment.environment.name
otlp_http/tempo:
endpoint: http://tempo:4318
timeout: 2s
@@ -44,28 +48,41 @@ extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
extensions:
- health_check
telemetry:
logs:
level: warn
metrics:
level: detailed
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/tempo]
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp_http/tempo
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp_http/loki]
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- otlp_http/loki
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
receivers:
- otlp
processors:
- memory_limiter
- batch
exporters:
- prometheus
+134
View File
@@ -0,0 +1,134 @@
"""Read-only queries through an internal probe, without exposing admin ports."""
import base64
import json
import time
from urllib.parse import urlencode
from scripts.stand import compose
ENDPOINTS = {
"prometheus": "http://prometheus:9090",
"loki": "http://loki:3100",
"tempo": "http://tempo:3200",
"otel": "http://otel:13133",
}
def fetch(service, path, *, params=None, json_result=True):
url = ENDPOINTS[service] + path
if params:
url += "?" + urlencode(params, doseq=True)
code = (
"import urllib.request; "
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); "
"print(r.read().decode())"
)
text = compose("exec", "-T", "observer", "python", "-c", code)
return json.loads(text) if json_result else text
def query(expression, *, at=None):
params = {"query": expression}
if at is not None:
params["time"] = str(at)
result = fetch("prometheus", "/api/v1/query", params=params)
assert result["status"] == "success"
return result["data"]["result"]
def scalar(expression):
result = query(expression)
return float(result[0]["value"][1]) if result else 0.0
def queries(expressions):
urls = {
name: ENDPOINTS["prometheus"]
+ "/api/v1/query?"
+ urlencode({"query": expression})
for name, expression in expressions.items()
}
code = (
"import json,urllib.request\n"
f"urls={urls!r}\n"
"result={}\n"
"for name,url in urls.items():\n"
" with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n"
"print(json.dumps(result))"
)
return json.loads(compose("exec", "-T", "observer", "python", "-c", code))
def logs(trace_id, *, start=None):
return fetch(
"loki",
"/loki/api/v1/query_range",
params={
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
+ trace_id
+ '"',
"start": str(int((start or time.time() - 3600) * 1e9)),
"end": str(time.time_ns()),
"limit": 500,
},
)
def normalize_id(value):
if not value:
return ""
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
return value
return base64.b64decode(value).hex()
def spans(trace_id):
result = fetch("tempo", "/api/traces/" + trace_id)
flattened = []
for batch in result.get("batches", result.get("resourceSpans", [])):
resource = {
a["key"]: next(iter(a["value"].values()))
for a in batch.get("resource", {}).get("attributes", [])
}
for scope in batch.get("scopeSpans", []):
for item in scope.get("spans", []):
flattened.append(
{
"service": resource.get("service.name"),
"version": resource.get("service.version"),
"instance": resource.get("service.instance.id"),
"name": item["name"],
"kind": item.get("kind"),
"trace_id": normalize_id(item.get("traceId")),
"span_id": normalize_id(item.get("spanId")),
"parent_span_id": normalize_id(
item.get("parentSpanId")
),
"duration_seconds": (
int(item["endTimeUnixNano"])
- int(item["startTimeUnixNano"])
)
/ 1e9,
"status": item.get("status", {}),
"attributes": {
a["key"]: next(iter(a["value"].values()))
for a in item.get("attributes", [])
},
}
)
return flattened
def wait(condition, *, timeout=90, interval=1):
end = time.monotonic() + timeout
last_error = None
while time.monotonic() < end:
try:
result = condition()
if result:
return result
except (RuntimeError, ValueError, KeyError) as exc:
last_error = type(exc).__name__
time.sleep(interval)
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
+577
View File
@@ -0,0 +1,577 @@
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
import argparse
import csv
import io
import json
import math
import os
import threading
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, queries, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
SIGNALS = {
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
"queue_depth": "max(messenger_queue_depth)",
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
"worker_busy": "sum(messenger_worker_busy)",
"worker_attempts": "sum(messenger_job_attempts_total)",
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
"collector_queue": "sum(otelcol_exporter_queue_size)",
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
"collector_memory": 'process_resident_memory_bytes{job="collector"}',
"app_memory": "sum(messenger_process_memory_bytes)",
}
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
def snapshot(phase):
observed = queries(SIGNALS)
values = {}
for key, rows in observed.items():
value = float(rows[0]["value"][1]) if rows else 0.0
values[key] = value if math.isfinite(value) else None
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
return {
"at": time.time(),
"phase": phase,
"signals": values,
"alerts": [
{
"name": a["labels"]["alertname"],
"state": a["state"],
"active_at": a.get("activeAt"),
}
for a in alerts
],
}
def firing(samples):
return {
a["name"]
for sample in samples
for a in sample["alerts"]
if a["state"] == "firing"
}
class Workload:
def __init__(self, client, headers, chat):
self.client, self.headers, self.chat = client, headers, chat
self.stop = threading.Event()
self.phase = "warmup"
self.requests = []
self.uploads = []
self.lock = threading.Lock()
self.slots = threading.BoundedSemaphore(16)
self.pool = ThreadPoolExecutor(max_workers=16)
image = io.BytesIO()
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
self.raw = image.getvalue()
self.threads = [
threading.Thread(target=self.schedule_messages),
threading.Thread(target=self.schedule_uploads),
]
def write(self):
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/messages",
headers={
**self.headers,
"Idempotency-Key": uuid4().hex,
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
json={"text": "Synthetic SLO message " + uuid4().hex},
)
status = response.status_code
if status == 201:
identity = response.json()["id"]
except httpx.HTTPError:
pass
finally:
with self.lock:
self.requests.append(
[
started,
phase,
status,
time.time() - started,
trace_id,
identity,
]
)
self.slots.release()
def schedule_messages(self):
next_at = time.monotonic()
while not self.stop.is_set():
if self.slots.acquire(blocking=False):
self.pool.submit(self.write)
else:
with self.lock:
self.requests.append(
[time.time(), self.phase, -1, 0, "", ""]
)
next_at += 0.25
self.stop.wait(max(0, next_at - time.monotonic()))
def schedule_uploads(self):
while not self.stop.is_set():
started, phase, trace_id = time.time(), self.phase, uuid4().hex
status, identity = 0, ""
try:
response = self.client.post(
f"/api/v1/chats/{self.chat}/attachments",
params={"filename": "synthetic-slo.png"},
content=self.raw,
headers={
**self.headers,
"Content-Type": "image/png",
"traceparent": "00-"
+ trace_id
+ "-"
+ uuid4().hex[:16]
+ "-01",
},
)
status = response.status_code
if status == 202:
identity = response.json()["id"]
except httpx.HTTPError:
pass
with self.lock:
self.uploads.append(
[started, phase, status, identity, trace_id]
)
self.stop.wait(max(0, 5 - (time.time() - started)))
def start(self):
for thread in self.threads:
thread.start()
def close(self):
self.stop.set()
for thread in self.threads:
thread.join(20)
self.pool.shutdown(wait=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/slo-research-first.json")
parser.add_argument("--minimum-seconds", type=int, default=1800)
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"scope": "full30m"
if args.minimum_seconds >= 1800
else "short-development",
"profile": {
"messages_per_second": 4,
"max_inflight": 16,
"upload_every_seconds": 5,
"image_pixels": [320, 200],
},
"complete": False,
"samples": [],
"phases": [],
}
study_started = time.monotonic()
output.parent.mkdir(parents=True, exist_ok=True)
load = None
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
with httpx.Client(
base_url=BASE,
timeout=8,
trust_env=False,
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
) as client:
ready(timeout=90)
user, _, session = actor(client)
headers = auth(session)
response = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic SLO experiment",
"member_ids": [user["id"]],
},
)
assert response.status_code == 201
chat = response.json()["id"]
load = Workload(client, headers, chat)
workload = load
def save():
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
def hold(name, seconds):
load.phase = name
phase = {
"name": name,
"start": time.time(),
"planned_seconds": seconds,
}
report["phases"].append(phase)
deadline = time.monotonic() + seconds
start_index = len(report["samples"])
while time.monotonic() < deadline:
report["samples"].append(snapshot(name))
save()
time.sleep(min(5, max(0, deadline - time.monotonic())))
phase["end"] = time.time()
phase["firing"] = sorted(firing(report["samples"][start_index:]))
return phase
def db_delay(on):
if on:
sql(
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
)
else:
sql(
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
)
try:
load.start()
warm = hold("warmup", 300)
assert not BURN_ALERTS.intersection(warm["firing"]), (
"Baseline is not clean"
)
report["baseline"] = snapshot("baseline")
for signal in ["api", "ready"]:
if signal == "api":
activate, restore = (
lambda: db_delay(True),
lambda: db_delay(False),
)
else:
def activate():
return compose("stop", "--timeout", "1", "worker")
def restore():
return compose(
"up", "-d", "--no-deps", "--no-recreate", "worker"
)
brief_started = time.time()
activate()
brief = hold(signal + "_brief", 5)
restore()
settle = hold(
signal + "_brief_recovery", 120 if signal == "api" else 60
)
assert not BURN_ALERTS.intersection(
brief["firing"] + settle["firing"]
), "Brief incident triggered paging"
report.setdefault("short_spikes", []).append(
{
"signal": signal,
"action_started": brief_started,
"paging_events": 0,
}
)
action_started = time.time()
activate()
action_finished = time.time()
degraded = hold(
signal + "_sustained", 180 if signal == "api" else 150
)
restore_started = time.time()
restore()
restore_finished = time.time()
recovered = hold(
signal + "_recovery", 150 if signal == "api" else 240
)
expected = (
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
if signal == "api"
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
)
assert expected <= set(degraded["firing"]), (
signal,
degraded["firing"],
)
last = report["samples"][-1]
assert not expected.intersection(firing([last])), (
"Incident alerts did not resolve"
)
points = [
s
for s in report["samples"]
if action_finished <= s["at"] <= recovered["end"]
]
timings = {}
for name in sorted(expected):
first = next(
s["at"] for s in points if name in firing([s])
)
resolved = next(
s["at"]
for s in points
if s["at"] >= restore_finished
and name not in firing([s])
)
timings[name] = {
"detect_seconds": round(first - action_finished, 3),
"resolve_seconds": round(
resolved - restore_finished, 3
),
}
report.setdefault("incidents", []).append(
{
"signal": signal,
"action_started": action_started,
"action_finished": action_finished,
"restore_started": restore_started,
"restore_finished": restore_finished,
"timings": timings,
}
)
if signal == "api":
candidates = [
r
for r in load.requests
if action_finished <= r[0] <= restore_started
and r[2] >= 500
]
assert candidates
items = wait(
lambda identity=candidates[0][4]: spans(identity)
)
assert any(
s["name"] == "db.postgresql"
and s["duration_seconds"] >= 2
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in items
)
report["database_diagnostic_trace"] = items
else:
assert (
max(
s["signals"]["oldest_job_age"] or 0 for s in points
)
> 20
)
assert (
max(s["signals"]["queue_depth"] or 0 for s in points)
> 0
)
accepted = [
r
for r in load.uploads
if action_finished <= r[0] <= restore_started
and r[2] == 202
]
assert accepted
report["worker_recovered_trace"] = wait(
lambda identity=accepted[0][4]: spans(identity)
)
report["worker_recovered_attachment"] = accepted[0][3]
outage_before = snapshot("backend_before")
outage_started = time.time()
compose("stop", "--timeout", "1", "loki", "tempo")
outage = hold("backend_unavailable", 90)
points = [
s
for s in report["samples"]
if s["phase"] == "backend_unavailable"
]
assert max(
s["signals"]["collector_failed_spans"] or 0 for s in points
) > (outage_before["signals"]["collector_failed_spans"] or 0)
assert all(
(s["signals"]["collector_queue"] or 0)
<= (s["signals"]["collector_queue_capacity"] or 0)
for s in points
)
assert (
max(s["signals"]["collector_memory"] or 0 for s in points)
< 256 * 1024**2
)
assert not BURN_ALERTS.intersection(outage["firing"])
during = [
r for r in load.requests if r[1] == "backend_unavailable"
]
assert during and all(r[2] == 201 for r in during)
before_ready = sql(
"SELECT count(*) FROM attachments WHERE status='ready'"
)
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
wait(lambda: fetch("loki", "/ready", json_result=False))
wait(lambda: fetch("tempo", "/ready", json_result=False))
hold("backend_recovery", 60)
restored = next(
r
for r in reversed(load.uploads)
if r[1] == "backend_recovery" and r[2] == 202
)
wait(
lambda: any(
s["name"] == "attachment.process"
for s in spans(restored[4])
)
)
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
report["backend_outage"] = {
"started": outage_started,
"finished": outage["end"],
"before": outage_before,
"after": snapshot("backend_after"),
"ready_count_before_backend_recovery": int(before_ready),
"successful_writes_during_outage": len(during),
"restored_trace_id": restored[4],
"queue_bounded": True,
"collector_rss_below_limit": True,
"send_failure_counter_counts_attempts_not_unique_loss": True,
}
remaining = args.minimum_seconds - (
time.monotonic() - study_started
)
if remaining > 0:
hold("steady_after_recovery", remaining)
load.close()
load = None
# Read through the public API: a successful SQL insert alone is not
# proof that every acknowledged message remains accessible.
seen, cursor = set(), None
while True:
page = client.get(
f"/api/v1/chats/{chat}/messages",
headers=headers,
params={
"limit": 100,
**({"cursor": cursor} if cursor else {}),
},
)
page.raise_for_status()
data = page.json()
ids = [row["id"] for row in data["items"]]
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
seen.update(ids)
cursor = data.get("next_cursor")
if not cursor:
break
expected_messages = {
r[5] for r in workload.requests if r[2] == 201
}
assert seen == expected_messages
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
for row in accepted_uploads:
def completed(identity=row[3]):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(completed, timeout=120)
for row in accepted_uploads[
:: max(1, len(accepted_uploads) // 10)
]:
response = client.get(
f"/api/v1/attachments/{row[3]}/content", headers=headers
)
assert (
response.status_code == 200
and response.content == workload.raw
)
report["preservation"] = {
"acknowledged_messages": len(expected_messages),
"accessible_messages": len(seen),
"accepted_uploads": len(accepted_uploads),
"all_accepted_uploads_ready": True,
"sampled_originals_match": True,
}
report["rules"] = fetch("prometheus", "/api/v1/rules")
report["final_30m_sli"] = queries(
{
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
}
)
report["complete"] = True
finally:
try:
db_delay(False)
finally:
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"worker",
"loki",
"tempo",
)
if load is not None:
load.close()
report["elapsed_seconds"] = round(
time.monotonic() - study_started, 3
)
request_path = output.with_name(output.stem + "-requests.csv")
with request_path.open("w") as stream:
writer = csv.writer(stream)
writer.writerow(
[
"timestamp",
"phase",
"status",
"seconds",
"trace_id",
"message_id",
]
)
writer.writerows(workload.requests)
report["requests_csv"] = request_path.name
report["request_statuses"] = dict(
Counter(str(r[2]) for r in workload.requests)
)
report["uploads"] = workload.uploads
save()
print(
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
)
if __name__ == "__main__":
main()
+340
View File
@@ -0,0 +1,340 @@
"""Actual API-to-worker causality, log correlation, provisioning and privacy."""
import argparse
import hashlib
import io
import json
import os
from datetime import UTC, datetime
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
from scripts.observe import fetch, logs, query, spans, wait
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
def verify_chain(items, trace_id, *, retry):
assert items and {s["trace_id"] for s in items} == {trace_id}
by_id = {s["span_id"]: s for s in items}
producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"]
consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"]
processing = [s for s in items if s["name"] == "attachment.process"]
assert producers and consumers and processing
assert {s["service"] for s in items} >= {
"messenger-api",
"messenger-relay",
"messenger-worker",
}
for consumer in consumers:
assert (
by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER"
)
for process in processing:
assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER"
assert any(
s["name"] == "s3.put" and s["service"] == "messenger-worker"
for s in items
)
assert any(
s["name"] == "db.postgresql" and s["service"] == "messenger-worker"
for s in items
)
if retry:
assert len(processing) >= 2
assert any(
s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
for s in processing
)
return items
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/telemetry-first.json")
args = parser.parse_args()
output = ROOT / args.output
assert not output.exists(), "Choose a new output path"
report = {
"date": datetime.now(UTC).isoformat(),
"source_commit": command(["git", "rev-parse", "HEAD"]),
"complete": False,
}
output.parent.mkdir(parents=True, exist_ok=True)
try:
ready(timeout=90)
for service, path in [
("prometheus", "/-/ready"),
("loki", "/ready"),
("tempo", "/ready"),
("otel", "/"),
]:
wait(lambda s=service, p=path: fetch(s, p, json_result=False))
report["backends_ready"] = True
buffer = io.BytesIO()
Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG")
raw = buffer.getvalue()
sentinel = "PrivacyProbe-" + uuid4().hex
with httpx.Client(
base_url=BASE, timeout=15, trust_env=False
) as client:
user, _, session = actor(client)
headers = auth(session)
chat = client.post(
"/api/v1/chats",
headers=headers,
json={
"title": "Synthetic telemetry",
"member_ids": [user["id"]],
},
).json()["id"]
message = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**headers, "Idempotency-Key": uuid4().hex},
json={"text": sentinel},
)
assert message.status_code == 201
cases = []
for retry in [False, True]:
if retry:
compose("stop", "--timeout", "1", "worker")
trace_id = uuid4().hex
trace_header = (
"00-" + trace_id + "-" + uuid4().hex[:16] + "-01"
)
accepted = client.post(
f"/api/v1/chats/{chat}/attachments",
params={"filename": sentinel + ".png"},
headers={
**headers,
"Content-Type": "image/png",
"traceparent": trace_header,
},
content=raw,
)
assert (
accepted.status_code == 202
and accepted.json()["status"] == "queued"
)
attachment = accepted.json()["id"]
job = sql(
f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid"
)
if retry:
sql(
f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)"
)
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
def finished(identity=attachment):
response = client.get(
f"/api/v1/attachments/{identity}", headers=headers
)
return (
response.status_code == 200
and response.json()["status"] == "ready"
)
wait(finished, timeout=90)
original = client.get(
f"/api/v1/attachments/{attachment}/content",
headers=headers,
)
assert original.content == raw
def traced(identity=trace_id, retry_expected=retry):
items = spans(identity)
if len(
[s for s in items if s["name"] == "attachment.process"]
) < (2 if retry_expected else 1):
return None
if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items):
# Independent export batches may arrive in a different
# order. Require the whole chain before accepting it.
try:
return verify_chain(
items, identity, retry=retry_expected
)
except (AssertionError, KeyError):
return None
items = wait(traced, timeout=90)
def correlated(identity=trace_id):
result = logs(identity)
streams = result.get("data", {}).get("result", [])
services = {
x["stream"].get("service_name") for x in streams
}
return (
result
if {"messenger-api", "messenger-worker"} <= services
else None
)
correlated_logs = wait(correlated, timeout=90)
span_ids = {s["span_id"] for s in items}
streams = correlated_logs["data"]["result"]
assert all(
row["stream"].get("span_id") in span_ids for row in streams
)
assert any(
row["stream"].get("request_id")
== accepted.headers["X-Request-Id"]
for row in streams
)
exported = json.dumps(
{"spans": items, "logs": correlated_logs}
)
assert (
sentinel not in exported
and session["token"] not in exported
)
cases.append(
{
"retry": retry,
"attachment_id": attachment,
"job_id": job,
"trace_id": trace_id,
"request_id": accepted.headers["X-Request-Id"],
"original_sha256": hashlib.sha256(raw).hexdigest(),
"attempt": int(
sql(
f"SELECT attempt FROM jobs WHERE id='{job}'::uuid"
)
),
"spans": items,
"correlated_logs": correlated_logs,
}
)
report["causal_uploads"] = cases
assert cases[1]["attempt"] >= 2
report["privacy_sentinels_absent_from_traces_and_logs"] = True
names = wait(
lambda: fetch("prometheus", "/api/v1/label/__name__/values")[
"data"
]
)
required = {
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
"messenger_job_attempts_total",
"messenger_job_duration_seconds_bucket",
"messenger_time_to_ready_seconds_bucket",
"messenger_queue_depth",
"messenger_oldest_job_age_seconds",
"messenger_db_pool_used",
"messenger_db_pool_waiting",
"messenger_worker_busy",
"messenger_worker_capacity",
"messenger_process_cpu_seconds_total",
"messenger_process_memory_bytes",
"messenger_slo_decisions_total",
}
wait(
lambda: (
required
<= set(
fetch("prometheus", "/api/v1/label/__name__/values")[
"data"
]
)
)
)
names = fetch("prometheus", "/api/v1/label/__name__/values")["data"]
report["metric_names"] = [
n for n in names if n.startswith("messenger_")
]
series = fetch(
"prometheus",
"/api/v1/series",
params={"match[]": '{__name__=~"messenger_.*"}'},
)["data"]
forbidden = {
"user_id",
"chat_id",
"message_id",
"attachment_id",
"job_id",
"request_id",
"trace_id",
"filename",
}
assert all(not forbidden.intersection(s) for s in series)
assert all(sentinel not in json.dumps(s) for s in series)
report["metric_series_count"] = len(series)
report["metric_labels_low_cardinality"] = True
url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
password = (ROOT / "secrets/grafana_password").read_text().strip()
with httpx.Client(
base_url=url, auth=("admin", password), trust_env=False, timeout=15
) as grafana:
dashboard = grafana.get("/api/dashboards/uid/messenger-lab6")
dashboard.raise_for_status()
imported = dashboard.json()["dashboard"]
assert len(imported["panels"]) == 18
sources = grafana.get("/api/datasources").json()
assert {s["uid"] for s in sources} == {
"prometheus",
"loki",
"tempo",
}
checks = {}
for uid in ["prometheus", "loki", "tempo"]:
r = grafana.get(f"/api/datasources/uid/{uid}/health")
r.raise_for_status()
assert r.json()["status"] == "OK", uid
checks[uid] = r.json()["status"]
result = grafana.post(
"/api/ds/query",
json={
"queries": [
{
"refId": "A",
"datasource": {
"uid": "prometheus",
"type": "prometheus",
},
"expr": "sum(messenger_http_requests_total)",
"instant": True,
"format": "table",
"intervalMs": 5000,
"maxDataPoints": 10,
}
],
"from": "now-5m",
"to": "now",
},
)
result.raise_for_status()
assert result.json()["results"]["A"]["frames"]
report["grafana"] = {
"dashboard_uid": imported["uid"],
"panels": len(imported["panels"]),
"datasources_health": checks,
"actual_query_frames": len(
result.json()["results"]["A"]["frames"]
),
}
report["quantiles"] = {
str(q): query(
f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))"
)
for q in [0.5, 0.95, 0.99]
}
assert all(report["quantiles"].values())
report["complete"] = True
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print(
"PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana"
)
if __name__ == "__main__":
main()
+2
View File
@@ -306,6 +306,8 @@ def duration(name, seconds, attributes=None):
def carrier():
result = {}
_propagator.inject(result)
if len(result.get("tracestate", "")) > 512:
result.pop("tracestate", None)
return result