test(observability): verify causal signals and controlled SLO incidents
This commit is contained in:
@@ -39,8 +39,8 @@ up: bootstrap
|
||||
down:
|
||||
docker compose down
|
||||
lint:
|
||||
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
|
||||
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py)
|
||||
$(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
|
||||
$(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/observe.py scripts/research_slo.py $(wildcard scripts/verify_*.py)
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests
|
||||
verify:
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"source_commit": "8f3ff33b2dcb1cf3082535acf99ee4045a80400d",
|
||||
"command": "BASE_URL=http://127.0.0.1:8086 uv run --locked pytest student-tests -q",
|
||||
"elapsed_seconds": 417.533,
|
||||
"exit_code": 0,
|
||||
"summary": "112 passed in 416.52s (0:06:56)",
|
||||
"result": "PASS"
|
||||
}
|
||||
@@ -679,7 +679,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))",
|
||||
"refId": "A",
|
||||
"legendFormat": "spans/s",
|
||||
"datasource": {
|
||||
@@ -688,7 +688,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))",
|
||||
"refId": "B",
|
||||
"legendFormat": "logs/s",
|
||||
"datasource": {
|
||||
|
||||
+34
-17
@@ -15,9 +15,13 @@ processors:
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
metric_expiration: 5m
|
||||
resource_constant_labels:
|
||||
included:
|
||||
- service.name
|
||||
- service.instance.id
|
||||
- service.version
|
||||
- deployment.environment.name
|
||||
otlp_http/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
@@ -44,28 +48,41 @@ extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
service:
|
||||
extensions: [health_check]
|
||||
extensions:
|
||||
- health_check
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/tempo]
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp_http/tempo
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- otlp_http/loki
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
receivers:
|
||||
- otlp
|
||||
processors:
|
||||
- memory_limiter
|
||||
- batch
|
||||
exporters:
|
||||
- prometheus
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Read-only queries through an internal probe, without exposing admin ports."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from scripts.stand import compose
|
||||
|
||||
ENDPOINTS = {
|
||||
"prometheus": "http://prometheus:9090",
|
||||
"loki": "http://loki:3100",
|
||||
"tempo": "http://tempo:3200",
|
||||
"otel": "http://otel:13133",
|
||||
}
|
||||
|
||||
|
||||
def fetch(service, path, *, params=None, json_result=True):
|
||||
url = ENDPOINTS[service] + path
|
||||
if params:
|
||||
url += "?" + urlencode(params, doseq=True)
|
||||
code = (
|
||||
"import urllib.request; "
|
||||
f"r=urllib.request.urlopen(urllib.request.Request({url!r},headers={{'Accept':'application/json'}}),timeout=8); "
|
||||
"print(r.read().decode())"
|
||||
)
|
||||
text = compose("exec", "-T", "observer", "python", "-c", code)
|
||||
return json.loads(text) if json_result else text
|
||||
|
||||
|
||||
def query(expression, *, at=None):
|
||||
params = {"query": expression}
|
||||
if at is not None:
|
||||
params["time"] = str(at)
|
||||
result = fetch("prometheus", "/api/v1/query", params=params)
|
||||
assert result["status"] == "success"
|
||||
return result["data"]["result"]
|
||||
|
||||
|
||||
def scalar(expression):
|
||||
result = query(expression)
|
||||
return float(result[0]["value"][1]) if result else 0.0
|
||||
|
||||
|
||||
def queries(expressions):
|
||||
urls = {
|
||||
name: ENDPOINTS["prometheus"]
|
||||
+ "/api/v1/query?"
|
||||
+ urlencode({"query": expression})
|
||||
for name, expression in expressions.items()
|
||||
}
|
||||
code = (
|
||||
"import json,urllib.request\n"
|
||||
f"urls={urls!r}\n"
|
||||
"result={}\n"
|
||||
"for name,url in urls.items():\n"
|
||||
" with urllib.request.urlopen(url,timeout=8) as r: result[name]=json.load(r)['data']['result']\n"
|
||||
"print(json.dumps(result))"
|
||||
)
|
||||
return json.loads(compose("exec", "-T", "observer", "python", "-c", code))
|
||||
|
||||
|
||||
def logs(trace_id, *, start=None):
|
||||
return fetch(
|
||||
"loki",
|
||||
"/loki/api/v1/query_range",
|
||||
params={
|
||||
"query": '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | trace_id="'
|
||||
+ trace_id
|
||||
+ '"',
|
||||
"start": str(int((start or time.time() - 3600) * 1e9)),
|
||||
"end": str(time.time_ns()),
|
||||
"limit": 500,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def normalize_id(value):
|
||||
if not value:
|
||||
return ""
|
||||
if len(value) in {16, 32} and all(c in "0123456789abcdef" for c in value):
|
||||
return value
|
||||
return base64.b64decode(value).hex()
|
||||
|
||||
|
||||
def spans(trace_id):
|
||||
result = fetch("tempo", "/api/traces/" + trace_id)
|
||||
flattened = []
|
||||
for batch in result.get("batches", result.get("resourceSpans", [])):
|
||||
resource = {
|
||||
a["key"]: next(iter(a["value"].values()))
|
||||
for a in batch.get("resource", {}).get("attributes", [])
|
||||
}
|
||||
for scope in batch.get("scopeSpans", []):
|
||||
for item in scope.get("spans", []):
|
||||
flattened.append(
|
||||
{
|
||||
"service": resource.get("service.name"),
|
||||
"version": resource.get("service.version"),
|
||||
"instance": resource.get("service.instance.id"),
|
||||
"name": item["name"],
|
||||
"kind": item.get("kind"),
|
||||
"trace_id": normalize_id(item.get("traceId")),
|
||||
"span_id": normalize_id(item.get("spanId")),
|
||||
"parent_span_id": normalize_id(
|
||||
item.get("parentSpanId")
|
||||
),
|
||||
"duration_seconds": (
|
||||
int(item["endTimeUnixNano"])
|
||||
- int(item["startTimeUnixNano"])
|
||||
)
|
||||
/ 1e9,
|
||||
"status": item.get("status", {}),
|
||||
"attributes": {
|
||||
a["key"]: next(iter(a["value"].values()))
|
||||
for a in item.get("attributes", [])
|
||||
},
|
||||
}
|
||||
)
|
||||
return flattened
|
||||
|
||||
|
||||
def wait(condition, *, timeout=90, interval=1):
|
||||
end = time.monotonic() + timeout
|
||||
last_error = None
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
result = condition()
|
||||
if result:
|
||||
return result
|
||||
except (RuntimeError, ValueError, KeyError) as exc:
|
||||
last_error = type(exc).__name__
|
||||
time.sleep(interval)
|
||||
raise TimeoutError("Telemetry observation not reached: " + str(last_error))
|
||||
@@ -0,0 +1,577 @@
|
||||
"""Thirty-minute controlled SLO study with real DB/worker/backend failures."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
from scripts.observe import fetch, logs, queries, spans, wait
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
ERRORS = 'haproxy_backend_http_responses_total{proxy="messenger",code="5xx"}'
|
||||
RESPONSES = 'haproxy_backend_http_responses_total{proxy="messenger"}'
|
||||
SIGNALS = {
|
||||
"api_error_ratio_1m": f"sum(rate({ERRORS}[1m])) / clamp_min(sum(rate({RESPONSES}[1m])),0.001)",
|
||||
"queue_depth": "max(messenger_queue_depth)",
|
||||
"oldest_job_age": "max(messenger_oldest_job_age_seconds)",
|
||||
"worker_busy": "sum(messenger_worker_busy)",
|
||||
"worker_attempts": "sum(messenger_job_attempts_total)",
|
||||
"pool_used": 'sum(messenger_db_pool_used{service_name="messenger-api"})',
|
||||
"pool_waiting": 'sum(messenger_db_pool_waiting{service_name="messenger-api"})',
|
||||
"http_p95": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"slo_good": 'sum(messenger_slo_decisions_total{outcome="good"})',
|
||||
"slo_bad": 'sum(messenger_slo_decisions_total{outcome="bad"})',
|
||||
"collector_received_spans": "sum(otelcol_receiver_accepted_spans)",
|
||||
"collector_sent_spans": "sum(otelcol_exporter_sent_spans)",
|
||||
"collector_received_logs": "sum(otelcol_receiver_accepted_log_records)",
|
||||
"collector_sent_logs": "sum(otelcol_exporter_sent_log_records)",
|
||||
"collector_failed_spans": "sum(otelcol_exporter_send_failed_spans)",
|
||||
"collector_queue": "sum(otelcol_exporter_queue_size)",
|
||||
"collector_queue_capacity": "sum(otelcol_exporter_queue_capacity)",
|
||||
"collector_memory": 'process_resident_memory_bytes{job="collector"}',
|
||||
"app_memory": "sum(messenger_process_memory_bytes)",
|
||||
}
|
||||
BURN_ALERTS = {"ApiFastBurn", "ApiSlowBurn", "ReadyFastBurn", "ReadySlowBurn"}
|
||||
|
||||
|
||||
def snapshot(phase):
|
||||
observed = queries(SIGNALS)
|
||||
values = {}
|
||||
for key, rows in observed.items():
|
||||
value = float(rows[0]["value"][1]) if rows else 0.0
|
||||
values[key] = value if math.isfinite(value) else None
|
||||
alerts = fetch("prometheus", "/api/v1/alerts")["data"]["alerts"]
|
||||
return {
|
||||
"at": time.time(),
|
||||
"phase": phase,
|
||||
"signals": values,
|
||||
"alerts": [
|
||||
{
|
||||
"name": a["labels"]["alertname"],
|
||||
"state": a["state"],
|
||||
"active_at": a.get("activeAt"),
|
||||
}
|
||||
for a in alerts
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def firing(samples):
|
||||
return {
|
||||
a["name"]
|
||||
for sample in samples
|
||||
for a in sample["alerts"]
|
||||
if a["state"] == "firing"
|
||||
}
|
||||
|
||||
|
||||
class Workload:
|
||||
def __init__(self, client, headers, chat):
|
||||
self.client, self.headers, self.chat = client, headers, chat
|
||||
self.stop = threading.Event()
|
||||
self.phase = "warmup"
|
||||
self.requests = []
|
||||
self.uploads = []
|
||||
self.lock = threading.Lock()
|
||||
self.slots = threading.BoundedSemaphore(16)
|
||||
self.pool = ThreadPoolExecutor(max_workers=16)
|
||||
image = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (39, 102, 157)).save(image, "PNG")
|
||||
self.raw = image.getvalue()
|
||||
self.threads = [
|
||||
threading.Thread(target=self.schedule_messages),
|
||||
threading.Thread(target=self.schedule_uploads),
|
||||
]
|
||||
|
||||
def write(self):
|
||||
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
||||
status, identity = 0, ""
|
||||
try:
|
||||
response = self.client.post(
|
||||
f"/api/v1/chats/{self.chat}/messages",
|
||||
headers={
|
||||
**self.headers,
|
||||
"Idempotency-Key": uuid4().hex,
|
||||
"traceparent": "00-"
|
||||
+ trace_id
|
||||
+ "-"
|
||||
+ uuid4().hex[:16]
|
||||
+ "-01",
|
||||
},
|
||||
json={"text": "Synthetic SLO message " + uuid4().hex},
|
||||
)
|
||||
status = response.status_code
|
||||
if status == 201:
|
||||
identity = response.json()["id"]
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
finally:
|
||||
with self.lock:
|
||||
self.requests.append(
|
||||
[
|
||||
started,
|
||||
phase,
|
||||
status,
|
||||
time.time() - started,
|
||||
trace_id,
|
||||
identity,
|
||||
]
|
||||
)
|
||||
self.slots.release()
|
||||
|
||||
def schedule_messages(self):
|
||||
next_at = time.monotonic()
|
||||
while not self.stop.is_set():
|
||||
if self.slots.acquire(blocking=False):
|
||||
self.pool.submit(self.write)
|
||||
else:
|
||||
with self.lock:
|
||||
self.requests.append(
|
||||
[time.time(), self.phase, -1, 0, "", ""]
|
||||
)
|
||||
next_at += 0.25
|
||||
self.stop.wait(max(0, next_at - time.monotonic()))
|
||||
|
||||
def schedule_uploads(self):
|
||||
while not self.stop.is_set():
|
||||
started, phase, trace_id = time.time(), self.phase, uuid4().hex
|
||||
status, identity = 0, ""
|
||||
try:
|
||||
response = self.client.post(
|
||||
f"/api/v1/chats/{self.chat}/attachments",
|
||||
params={"filename": "synthetic-slo.png"},
|
||||
content=self.raw,
|
||||
headers={
|
||||
**self.headers,
|
||||
"Content-Type": "image/png",
|
||||
"traceparent": "00-"
|
||||
+ trace_id
|
||||
+ "-"
|
||||
+ uuid4().hex[:16]
|
||||
+ "-01",
|
||||
},
|
||||
)
|
||||
status = response.status_code
|
||||
if status == 202:
|
||||
identity = response.json()["id"]
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
with self.lock:
|
||||
self.uploads.append(
|
||||
[started, phase, status, identity, trace_id]
|
||||
)
|
||||
self.stop.wait(max(0, 5 - (time.time() - started)))
|
||||
|
||||
def start(self):
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
for thread in self.threads:
|
||||
thread.join(20)
|
||||
self.pool.shutdown(wait=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/slo-research-first.json")
|
||||
parser.add_argument("--minimum-seconds", type=int, default=1800)
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
assert not output.exists(), "Choose a new output path"
|
||||
report = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"scope": "full30m"
|
||||
if args.minimum_seconds >= 1800
|
||||
else "short-development",
|
||||
"profile": {
|
||||
"messages_per_second": 4,
|
||||
"max_inflight": 16,
|
||||
"upload_every_seconds": 5,
|
||||
"image_pixels": [320, 200],
|
||||
},
|
||||
"complete": False,
|
||||
"samples": [],
|
||||
"phases": [],
|
||||
}
|
||||
study_started = time.monotonic()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
load = None
|
||||
trigger = "lab6_slo_delay_" + uuid4().hex[:8]
|
||||
with httpx.Client(
|
||||
base_url=BASE,
|
||||
timeout=8,
|
||||
trust_env=False,
|
||||
limits=httpx.Limits(max_connections=24, max_keepalive_connections=24),
|
||||
) as client:
|
||||
ready(timeout=90)
|
||||
user, _, session = actor(client)
|
||||
headers = auth(session)
|
||||
response = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Synthetic SLO experiment",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 201
|
||||
chat = response.json()["id"]
|
||||
load = Workload(client, headers, chat)
|
||||
workload = load
|
||||
|
||||
def save():
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
|
||||
def hold(name, seconds):
|
||||
load.phase = name
|
||||
phase = {
|
||||
"name": name,
|
||||
"start": time.time(),
|
||||
"planned_seconds": seconds,
|
||||
}
|
||||
report["phases"].append(phase)
|
||||
deadline = time.monotonic() + seconds
|
||||
start_index = len(report["samples"])
|
||||
while time.monotonic() < deadline:
|
||||
report["samples"].append(snapshot(name))
|
||||
save()
|
||||
time.sleep(min(5, max(0, deadline - time.monotonic())))
|
||||
phase["end"] = time.time()
|
||||
phase["firing"] = sorted(firing(report["samples"][start_index:]))
|
||||
return phase
|
||||
|
||||
def db_delay(on):
|
||||
if on:
|
||||
sql(
|
||||
f"CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.chat_id='{chat}'::uuid THEN PERFORM pg_sleep(3); END IF; RETURN NEW; END $$; CREATE TRIGGER {trigger} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {trigger}()"
|
||||
)
|
||||
else:
|
||||
sql(
|
||||
f"DROP TRIGGER IF EXISTS {trigger} ON messages; DROP FUNCTION IF EXISTS {trigger}()"
|
||||
)
|
||||
|
||||
try:
|
||||
load.start()
|
||||
warm = hold("warmup", 300)
|
||||
assert not BURN_ALERTS.intersection(warm["firing"]), (
|
||||
"Baseline is not clean"
|
||||
)
|
||||
report["baseline"] = snapshot("baseline")
|
||||
for signal in ["api", "ready"]:
|
||||
if signal == "api":
|
||||
activate, restore = (
|
||||
lambda: db_delay(True),
|
||||
lambda: db_delay(False),
|
||||
)
|
||||
else:
|
||||
|
||||
def activate():
|
||||
return compose("stop", "--timeout", "1", "worker")
|
||||
|
||||
def restore():
|
||||
return compose(
|
||||
"up", "-d", "--no-deps", "--no-recreate", "worker"
|
||||
)
|
||||
|
||||
brief_started = time.time()
|
||||
activate()
|
||||
brief = hold(signal + "_brief", 5)
|
||||
restore()
|
||||
settle = hold(
|
||||
signal + "_brief_recovery", 120 if signal == "api" else 60
|
||||
)
|
||||
assert not BURN_ALERTS.intersection(
|
||||
brief["firing"] + settle["firing"]
|
||||
), "Brief incident triggered paging"
|
||||
report.setdefault("short_spikes", []).append(
|
||||
{
|
||||
"signal": signal,
|
||||
"action_started": brief_started,
|
||||
"paging_events": 0,
|
||||
}
|
||||
)
|
||||
action_started = time.time()
|
||||
activate()
|
||||
action_finished = time.time()
|
||||
degraded = hold(
|
||||
signal + "_sustained", 180 if signal == "api" else 150
|
||||
)
|
||||
restore_started = time.time()
|
||||
restore()
|
||||
restore_finished = time.time()
|
||||
recovered = hold(
|
||||
signal + "_recovery", 150 if signal == "api" else 240
|
||||
)
|
||||
expected = (
|
||||
{"ApiFastBurn", "ApiSlowBurn", "ApiHighErrorRate"}
|
||||
if signal == "api"
|
||||
else {"ReadyFastBurn", "ReadySlowBurn", "QueueLagGrowing"}
|
||||
)
|
||||
assert expected <= set(degraded["firing"]), (
|
||||
signal,
|
||||
degraded["firing"],
|
||||
)
|
||||
last = report["samples"][-1]
|
||||
assert not expected.intersection(firing([last])), (
|
||||
"Incident alerts did not resolve"
|
||||
)
|
||||
points = [
|
||||
s
|
||||
for s in report["samples"]
|
||||
if action_finished <= s["at"] <= recovered["end"]
|
||||
]
|
||||
timings = {}
|
||||
for name in sorted(expected):
|
||||
first = next(
|
||||
s["at"] for s in points if name in firing([s])
|
||||
)
|
||||
resolved = next(
|
||||
s["at"]
|
||||
for s in points
|
||||
if s["at"] >= restore_finished
|
||||
and name not in firing([s])
|
||||
)
|
||||
timings[name] = {
|
||||
"detect_seconds": round(first - action_finished, 3),
|
||||
"resolve_seconds": round(
|
||||
resolved - restore_finished, 3
|
||||
),
|
||||
}
|
||||
report.setdefault("incidents", []).append(
|
||||
{
|
||||
"signal": signal,
|
||||
"action_started": action_started,
|
||||
"action_finished": action_finished,
|
||||
"restore_started": restore_started,
|
||||
"restore_finished": restore_finished,
|
||||
"timings": timings,
|
||||
}
|
||||
)
|
||||
if signal == "api":
|
||||
candidates = [
|
||||
r
|
||||
for r in load.requests
|
||||
if action_finished <= r[0] <= restore_started
|
||||
and r[2] >= 500
|
||||
]
|
||||
assert candidates
|
||||
items = wait(
|
||||
lambda identity=candidates[0][4]: spans(identity)
|
||||
)
|
||||
assert any(
|
||||
s["name"] == "db.postgresql"
|
||||
and s["duration_seconds"] >= 2
|
||||
and s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
||||
for s in items
|
||||
)
|
||||
report["database_diagnostic_trace"] = items
|
||||
else:
|
||||
assert (
|
||||
max(
|
||||
s["signals"]["oldest_job_age"] or 0 for s in points
|
||||
)
|
||||
> 20
|
||||
)
|
||||
assert (
|
||||
max(s["signals"]["queue_depth"] or 0 for s in points)
|
||||
> 0
|
||||
)
|
||||
accepted = [
|
||||
r
|
||||
for r in load.uploads
|
||||
if action_finished <= r[0] <= restore_started
|
||||
and r[2] == 202
|
||||
]
|
||||
assert accepted
|
||||
report["worker_recovered_trace"] = wait(
|
||||
lambda identity=accepted[0][4]: spans(identity)
|
||||
)
|
||||
report["worker_recovered_attachment"] = accepted[0][3]
|
||||
outage_before = snapshot("backend_before")
|
||||
outage_started = time.time()
|
||||
compose("stop", "--timeout", "1", "loki", "tempo")
|
||||
outage = hold("backend_unavailable", 90)
|
||||
points = [
|
||||
s
|
||||
for s in report["samples"]
|
||||
if s["phase"] == "backend_unavailable"
|
||||
]
|
||||
assert max(
|
||||
s["signals"]["collector_failed_spans"] or 0 for s in points
|
||||
) > (outage_before["signals"]["collector_failed_spans"] or 0)
|
||||
assert all(
|
||||
(s["signals"]["collector_queue"] or 0)
|
||||
<= (s["signals"]["collector_queue_capacity"] or 0)
|
||||
for s in points
|
||||
)
|
||||
assert (
|
||||
max(s["signals"]["collector_memory"] or 0 for s in points)
|
||||
< 256 * 1024**2
|
||||
)
|
||||
assert not BURN_ALERTS.intersection(outage["firing"])
|
||||
during = [
|
||||
r for r in load.requests if r[1] == "backend_unavailable"
|
||||
]
|
||||
assert during and all(r[2] == 201 for r in during)
|
||||
before_ready = sql(
|
||||
"SELECT count(*) FROM attachments WHERE status='ready'"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "loki", "tempo")
|
||||
wait(lambda: fetch("loki", "/ready", json_result=False))
|
||||
wait(lambda: fetch("tempo", "/ready", json_result=False))
|
||||
hold("backend_recovery", 60)
|
||||
restored = next(
|
||||
r
|
||||
for r in reversed(load.uploads)
|
||||
if r[1] == "backend_recovery" and r[2] == 202
|
||||
)
|
||||
wait(
|
||||
lambda: any(
|
||||
s["name"] == "attachment.process"
|
||||
for s in spans(restored[4])
|
||||
)
|
||||
)
|
||||
wait(lambda: logs(restored[4]).get("data", {}).get("result"))
|
||||
report["backend_outage"] = {
|
||||
"started": outage_started,
|
||||
"finished": outage["end"],
|
||||
"before": outage_before,
|
||||
"after": snapshot("backend_after"),
|
||||
"ready_count_before_backend_recovery": int(before_ready),
|
||||
"successful_writes_during_outage": len(during),
|
||||
"restored_trace_id": restored[4],
|
||||
"queue_bounded": True,
|
||||
"collector_rss_below_limit": True,
|
||||
"send_failure_counter_counts_attempts_not_unique_loss": True,
|
||||
}
|
||||
remaining = args.minimum_seconds - (
|
||||
time.monotonic() - study_started
|
||||
)
|
||||
if remaining > 0:
|
||||
hold("steady_after_recovery", remaining)
|
||||
load.close()
|
||||
load = None
|
||||
# Read through the public API: a successful SQL insert alone is not
|
||||
# proof that every acknowledged message remains accessible.
|
||||
seen, cursor = set(), None
|
||||
while True:
|
||||
page = client.get(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers=headers,
|
||||
params={
|
||||
"limit": 100,
|
||||
**({"cursor": cursor} if cursor else {}),
|
||||
},
|
||||
)
|
||||
page.raise_for_status()
|
||||
data = page.json()
|
||||
ids = [row["id"] for row in data["items"]]
|
||||
assert len(ids) == len(set(ids)) and not seen.intersection(ids)
|
||||
seen.update(ids)
|
||||
cursor = data.get("next_cursor")
|
||||
if not cursor:
|
||||
break
|
||||
expected_messages = {
|
||||
r[5] for r in workload.requests if r[2] == 201
|
||||
}
|
||||
assert seen == expected_messages
|
||||
accepted_uploads = [r for r in workload.uploads if r[2] == 202]
|
||||
for row in accepted_uploads:
|
||||
|
||||
def completed(identity=row[3]):
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{identity}", headers=headers
|
||||
)
|
||||
return (
|
||||
response.status_code == 200
|
||||
and response.json()["status"] == "ready"
|
||||
)
|
||||
|
||||
wait(completed, timeout=120)
|
||||
for row in accepted_uploads[
|
||||
:: max(1, len(accepted_uploads) // 10)
|
||||
]:
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{row[3]}/content", headers=headers
|
||||
)
|
||||
assert (
|
||||
response.status_code == 200
|
||||
and response.content == workload.raw
|
||||
)
|
||||
report["preservation"] = {
|
||||
"acknowledged_messages": len(expected_messages),
|
||||
"accessible_messages": len(seen),
|
||||
"accepted_uploads": len(accepted_uploads),
|
||||
"all_accepted_uploads_ready": True,
|
||||
"sampled_originals_match": True,
|
||||
}
|
||||
report["rules"] = fetch("prometheus", "/api/v1/rules")
|
||||
report["final_30m_sli"] = queries(
|
||||
{
|
||||
"api_good_ratio": f"1 - sum(increase({ERRORS}[30m])) / clamp_min(sum(increase({RESPONSES}[30m])),1)",
|
||||
"ready_good_ratio": 'sum(increase(messenger_slo_decisions_total{outcome="good"}[30m])) / clamp_min(sum(increase(messenger_slo_decisions_total[30m])),1)',
|
||||
}
|
||||
)
|
||||
report["complete"] = True
|
||||
finally:
|
||||
try:
|
||||
db_delay(False)
|
||||
finally:
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"worker",
|
||||
"loki",
|
||||
"tempo",
|
||||
)
|
||||
if load is not None:
|
||||
load.close()
|
||||
report["elapsed_seconds"] = round(
|
||||
time.monotonic() - study_started, 3
|
||||
)
|
||||
request_path = output.with_name(output.stem + "-requests.csv")
|
||||
with request_path.open("w") as stream:
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(
|
||||
[
|
||||
"timestamp",
|
||||
"phase",
|
||||
"status",
|
||||
"seconds",
|
||||
"trace_id",
|
||||
"message_id",
|
||||
]
|
||||
)
|
||||
writer.writerows(workload.requests)
|
||||
report["requests_csv"] = request_path.name
|
||||
report["request_statuses"] = dict(
|
||||
Counter(str(r[2]) for r in workload.requests)
|
||||
)
|
||||
report["uploads"] = workload.uploads
|
||||
save()
|
||||
print(
|
||||
"PASS: short/sustained SLO incidents, detection/resolution and controlled false paging"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Actual API-to-worker causality, log correlation, provisioning and privacy."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
|
||||
from scripts.observe import fetch, logs, query, spans, wait
|
||||
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql
|
||||
|
||||
|
||||
def verify_chain(items, trace_id, *, retry):
|
||||
assert items and {s["trace_id"] for s in items} == {trace_id}
|
||||
by_id = {s["span_id"]: s for s in items}
|
||||
producers = [s for s in items if s["kind"] == "SPAN_KIND_PRODUCER"]
|
||||
consumers = [s for s in items if s["kind"] == "SPAN_KIND_CONSUMER"]
|
||||
processing = [s for s in items if s["name"] == "attachment.process"]
|
||||
assert producers and consumers and processing
|
||||
assert {s["service"] for s in items} >= {
|
||||
"messenger-api",
|
||||
"messenger-relay",
|
||||
"messenger-worker",
|
||||
}
|
||||
for consumer in consumers:
|
||||
assert (
|
||||
by_id[consumer["parent_span_id"]]["kind"] == "SPAN_KIND_PRODUCER"
|
||||
)
|
||||
for process in processing:
|
||||
assert by_id[process["parent_span_id"]]["kind"] == "SPAN_KIND_CONSUMER"
|
||||
assert any(
|
||||
s["name"] == "s3.put" and s["service"] == "messenger-worker"
|
||||
for s in items
|
||||
)
|
||||
assert any(
|
||||
s["name"] == "db.postgresql" and s["service"] == "messenger-worker"
|
||||
for s in items
|
||||
)
|
||||
if retry:
|
||||
assert len(processing) >= 2
|
||||
assert any(
|
||||
s["status"].get("code") in {2, "STATUS_CODE_ERROR"}
|
||||
for s in processing
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/telemetry-first.json")
|
||||
args = parser.parse_args()
|
||||
output = ROOT / args.output
|
||||
assert not output.exists(), "Choose a new output path"
|
||||
report = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
}
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ready(timeout=90)
|
||||
for service, path in [
|
||||
("prometheus", "/-/ready"),
|
||||
("loki", "/ready"),
|
||||
("tempo", "/ready"),
|
||||
("otel", "/"),
|
||||
]:
|
||||
wait(lambda s=service, p=path: fetch(s, p, json_result=False))
|
||||
report["backends_ready"] = True
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (320, 200), (24, 110, 153)).save(buffer, "PNG")
|
||||
raw = buffer.getvalue()
|
||||
sentinel = "PrivacyProbe-" + uuid4().hex
|
||||
with httpx.Client(
|
||||
base_url=BASE, timeout=15, trust_env=False
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
headers = auth(session)
|
||||
chat = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=headers,
|
||||
json={
|
||||
"title": "Synthetic telemetry",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
).json()["id"]
|
||||
message = client.post(
|
||||
f"/api/v1/chats/{chat}/messages",
|
||||
headers={**headers, "Idempotency-Key": uuid4().hex},
|
||||
json={"text": sentinel},
|
||||
)
|
||||
assert message.status_code == 201
|
||||
cases = []
|
||||
for retry in [False, True]:
|
||||
if retry:
|
||||
compose("stop", "--timeout", "1", "worker")
|
||||
trace_id = uuid4().hex
|
||||
trace_header = (
|
||||
"00-" + trace_id + "-" + uuid4().hex[:16] + "-01"
|
||||
)
|
||||
accepted = client.post(
|
||||
f"/api/v1/chats/{chat}/attachments",
|
||||
params={"filename": sentinel + ".png"},
|
||||
headers={
|
||||
**headers,
|
||||
"Content-Type": "image/png",
|
||||
"traceparent": trace_header,
|
||||
},
|
||||
content=raw,
|
||||
)
|
||||
assert (
|
||||
accepted.status_code == 202
|
||||
and accepted.json()["status"] == "queued"
|
||||
)
|
||||
attachment = accepted.json()["id"]
|
||||
job = sql(
|
||||
f"SELECT id FROM jobs WHERE attachment_id='{attachment}'::uuid"
|
||||
)
|
||||
if retry:
|
||||
sql(
|
||||
f"INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES('{job}'::uuid,'before_effect','transient',1)"
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
||||
|
||||
def finished(identity=attachment):
|
||||
response = client.get(
|
||||
f"/api/v1/attachments/{identity}", headers=headers
|
||||
)
|
||||
return (
|
||||
response.status_code == 200
|
||||
and response.json()["status"] == "ready"
|
||||
)
|
||||
|
||||
wait(finished, timeout=90)
|
||||
original = client.get(
|
||||
f"/api/v1/attachments/{attachment}/content",
|
||||
headers=headers,
|
||||
)
|
||||
assert original.content == raw
|
||||
|
||||
def traced(identity=trace_id, retry_expected=retry):
|
||||
items = spans(identity)
|
||||
if len(
|
||||
[s for s in items if s["name"] == "attachment.process"]
|
||||
) < (2 if retry_expected else 1):
|
||||
return None
|
||||
if any(s["kind"] == "SPAN_KIND_CONSUMER" for s in items):
|
||||
# Independent export batches may arrive in a different
|
||||
# order. Require the whole chain before accepting it.
|
||||
try:
|
||||
return verify_chain(
|
||||
items, identity, retry=retry_expected
|
||||
)
|
||||
except (AssertionError, KeyError):
|
||||
return None
|
||||
|
||||
items = wait(traced, timeout=90)
|
||||
|
||||
def correlated(identity=trace_id):
|
||||
result = logs(identity)
|
||||
streams = result.get("data", {}).get("result", [])
|
||||
services = {
|
||||
x["stream"].get("service_name") for x in streams
|
||||
}
|
||||
return (
|
||||
result
|
||||
if {"messenger-api", "messenger-worker"} <= services
|
||||
else None
|
||||
)
|
||||
|
||||
correlated_logs = wait(correlated, timeout=90)
|
||||
span_ids = {s["span_id"] for s in items}
|
||||
streams = correlated_logs["data"]["result"]
|
||||
assert all(
|
||||
row["stream"].get("span_id") in span_ids for row in streams
|
||||
)
|
||||
assert any(
|
||||
row["stream"].get("request_id")
|
||||
== accepted.headers["X-Request-Id"]
|
||||
for row in streams
|
||||
)
|
||||
exported = json.dumps(
|
||||
{"spans": items, "logs": correlated_logs}
|
||||
)
|
||||
assert (
|
||||
sentinel not in exported
|
||||
and session["token"] not in exported
|
||||
)
|
||||
cases.append(
|
||||
{
|
||||
"retry": retry,
|
||||
"attachment_id": attachment,
|
||||
"job_id": job,
|
||||
"trace_id": trace_id,
|
||||
"request_id": accepted.headers["X-Request-Id"],
|
||||
"original_sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"attempt": int(
|
||||
sql(
|
||||
f"SELECT attempt FROM jobs WHERE id='{job}'::uuid"
|
||||
)
|
||||
),
|
||||
"spans": items,
|
||||
"correlated_logs": correlated_logs,
|
||||
}
|
||||
)
|
||||
report["causal_uploads"] = cases
|
||||
assert cases[1]["attempt"] >= 2
|
||||
report["privacy_sentinels_absent_from_traces_and_logs"] = True
|
||||
names = wait(
|
||||
lambda: fetch("prometheus", "/api/v1/label/__name__/values")[
|
||||
"data"
|
||||
]
|
||||
)
|
||||
required = {
|
||||
"messenger_http_requests_total",
|
||||
"messenger_http_duration_seconds_bucket",
|
||||
"messenger_job_attempts_total",
|
||||
"messenger_job_duration_seconds_bucket",
|
||||
"messenger_time_to_ready_seconds_bucket",
|
||||
"messenger_queue_depth",
|
||||
"messenger_oldest_job_age_seconds",
|
||||
"messenger_db_pool_used",
|
||||
"messenger_db_pool_waiting",
|
||||
"messenger_worker_busy",
|
||||
"messenger_worker_capacity",
|
||||
"messenger_process_cpu_seconds_total",
|
||||
"messenger_process_memory_bytes",
|
||||
"messenger_slo_decisions_total",
|
||||
}
|
||||
wait(
|
||||
lambda: (
|
||||
required
|
||||
<= set(
|
||||
fetch("prometheus", "/api/v1/label/__name__/values")[
|
||||
"data"
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
names = fetch("prometheus", "/api/v1/label/__name__/values")["data"]
|
||||
report["metric_names"] = [
|
||||
n for n in names if n.startswith("messenger_")
|
||||
]
|
||||
series = fetch(
|
||||
"prometheus",
|
||||
"/api/v1/series",
|
||||
params={"match[]": '{__name__=~"messenger_.*"}'},
|
||||
)["data"]
|
||||
forbidden = {
|
||||
"user_id",
|
||||
"chat_id",
|
||||
"message_id",
|
||||
"attachment_id",
|
||||
"job_id",
|
||||
"request_id",
|
||||
"trace_id",
|
||||
"filename",
|
||||
}
|
||||
assert all(not forbidden.intersection(s) for s in series)
|
||||
assert all(sentinel not in json.dumps(s) for s in series)
|
||||
report["metric_series_count"] = len(series)
|
||||
report["metric_labels_low_cardinality"] = True
|
||||
url = "http://127.0.0.1:" + os.getenv("GRAFANA_PORT", "3006")
|
||||
password = (ROOT / "secrets/grafana_password").read_text().strip()
|
||||
with httpx.Client(
|
||||
base_url=url, auth=("admin", password), trust_env=False, timeout=15
|
||||
) as grafana:
|
||||
dashboard = grafana.get("/api/dashboards/uid/messenger-lab6")
|
||||
dashboard.raise_for_status()
|
||||
imported = dashboard.json()["dashboard"]
|
||||
assert len(imported["panels"]) == 18
|
||||
sources = grafana.get("/api/datasources").json()
|
||||
assert {s["uid"] for s in sources} == {
|
||||
"prometheus",
|
||||
"loki",
|
||||
"tempo",
|
||||
}
|
||||
checks = {}
|
||||
for uid in ["prometheus", "loki", "tempo"]:
|
||||
r = grafana.get(f"/api/datasources/uid/{uid}/health")
|
||||
r.raise_for_status()
|
||||
assert r.json()["status"] == "OK", uid
|
||||
checks[uid] = r.json()["status"]
|
||||
result = grafana.post(
|
||||
"/api/ds/query",
|
||||
json={
|
||||
"queries": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"uid": "prometheus",
|
||||
"type": "prometheus",
|
||||
},
|
||||
"expr": "sum(messenger_http_requests_total)",
|
||||
"instant": True,
|
||||
"format": "table",
|
||||
"intervalMs": 5000,
|
||||
"maxDataPoints": 10,
|
||||
}
|
||||
],
|
||||
"from": "now-5m",
|
||||
"to": "now",
|
||||
},
|
||||
)
|
||||
result.raise_for_status()
|
||||
assert result.json()["results"]["A"]["frames"]
|
||||
report["grafana"] = {
|
||||
"dashboard_uid": imported["uid"],
|
||||
"panels": len(imported["panels"]),
|
||||
"datasources_health": checks,
|
||||
"actual_query_frames": len(
|
||||
result.json()["results"]["A"]["frames"]
|
||||
),
|
||||
}
|
||||
report["quantiles"] = {
|
||||
str(q): query(
|
||||
f"histogram_quantile({q},sum by(le)(rate(messenger_http_duration_seconds_bucket[2m])))"
|
||||
)
|
||||
for q in [0.5, 0.95, 0.99]
|
||||
}
|
||||
assert all(report["quantiles"].values())
|
||||
report["complete"] = True
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "worker")
|
||||
output.write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: causal API/publish/consumer/child/DB/S3, retry, correlated logs, privacy and Grafana"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -306,6 +306,8 @@ def duration(name, seconds, attributes=None):
|
||||
def carrier():
|
||||
result = {}
|
||||
_propagator.inject(result)
|
||||
if len(result.get("tracestate", "")) > 512:
|
||||
result.pop("tracestate", None)
|
||||
return result
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user