681 lines
34 KiB
Python
681 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""Capture real lab6 observations. Standard library; local synthetic stand only.
|
|
|
|
Start Compose first. Commands: --baseline, --chain-with-retry, --incidents.
|
|
Evidence contains actual backend responses, never credentials or message bodies.
|
|
Every stopped service is restarted in finally, including failed assertions.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
PROMETHEUS = os.environ.get("PROMETHEUS_URL", "http://localhost:9090").rstrip("/")
|
|
LOKI = os.environ.get("LOKI_URL", "http://localhost:3100").rstrip("/")
|
|
TEMPO = os.environ.get("TEMPO_URL", "http://localhost:3200").rstrip("/")
|
|
GRAFANA = os.environ.get("GRAFANA_URL", "http://localhost:3000").rstrip("/")
|
|
BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
|
START_NS = int((time.time() - 5) * 1e9)
|
|
SECRETS: list[str] = []
|
|
|
|
|
|
def now() -> str:
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def check(condition, message: str) -> None:
|
|
if not condition:
|
|
raise AssertionError(message)
|
|
|
|
|
|
def http(method: str, url: str, body=None, headers=None, raw=None, timeout=15):
|
|
request_headers = {"Accept": "application/json", **(headers or {})}
|
|
if body is not None:
|
|
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
|
request_headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(url, data=raw, headers=request_headers, method=method)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=timeout)
|
|
except urllib.error.HTTPError as exc:
|
|
response = exc
|
|
with response:
|
|
data = response.read(16 * 1024 * 1024 + 1)
|
|
check(len(data) <= 16 * 1024 * 1024, "Backend response exceeds safety limit")
|
|
try:
|
|
payload = json.loads(data)
|
|
except (ValueError, UnicodeError):
|
|
payload = None
|
|
return response.status, payload, dict(response.headers), data
|
|
|
|
|
|
def header(headers, name):
|
|
return next((value for key, value in headers.items() if key.lower() == name.lower()), None)
|
|
|
|
|
|
def get(url):
|
|
status, data, _, _ = http("GET", url)
|
|
check(status == 200 and data is not None, "Telemetry query failed: HTTP " + str(status))
|
|
return data
|
|
|
|
|
|
def query(expression):
|
|
result = get(PROMETHEUS + "/api/v1/query?" + urllib.parse.urlencode({"query": expression}))
|
|
check(result.get("status") == "success", "Prometheus query is unsuccessful")
|
|
return result["data"]["result"]
|
|
|
|
|
|
def number(expression):
|
|
rows = query(expression)
|
|
return float(rows[0]["value"][1]) if rows else None
|
|
|
|
|
|
def wait(label, predicate, timeout=150, interval=2):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
result = predicate()
|
|
except (urllib.error.URLError, TimeoutError, ConnectionError):
|
|
# Socket errors are expected during startup/recovery. Assertions
|
|
# still propagate immediately: bad telemetry must never be masked.
|
|
result = None
|
|
if result:
|
|
return result
|
|
time.sleep(interval)
|
|
raise AssertionError("Timed out: " + label)
|
|
|
|
|
|
def wait_backends():
|
|
def ready():
|
|
endpoints = ((PROMETHEUS, "/-/ready"), (LOKI, "/ready"), (TEMPO, "/ready"))
|
|
for base, path in endpoints:
|
|
if http("GET", base + path, timeout=3)[0] != 200:
|
|
return False
|
|
status, payload, _, _ = http("GET", GRAFANA + "/api/health", timeout=3)
|
|
return status == 200 and isinstance(payload, dict) and payload.get("database") == "ok"
|
|
wait("Prometheus, Loki, Tempo and Grafana readiness", ready, timeout=150)
|
|
|
|
|
|
def safe_json(value):
|
|
encoded = json.dumps(value, ensure_ascii=False)
|
|
for secret in SECRETS:
|
|
check(secret not in encoded, "Sensitive synthetic marker found in exported telemetry")
|
|
# These keys are unnecessary for this stand and can expose request data.
|
|
forbidden = {"db.statement", "db.query.text", "exception.message", "exception.stacktrace",
|
|
"url.full", "http.url", "authorization", "http.request.body", "password"}
|
|
def walk(item):
|
|
if isinstance(item, dict):
|
|
check(not forbidden.intersection(key.lower() for key in item), "Forbidden telemetry key")
|
|
if item.get("key", "").lower() in forbidden:
|
|
raise AssertionError("Forbidden telemetry attribute")
|
|
for child in item.values():
|
|
walk(child)
|
|
elif isinstance(item, list):
|
|
for child in item:
|
|
walk(child)
|
|
walk(value)
|
|
return encoded
|
|
|
|
|
|
def save(name, value):
|
|
safe_json(value)
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
(EVIDENCE / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def compose(*arguments, timeout=90):
|
|
result = subprocess.run(["docker", "compose", *arguments], cwd=ROOT, capture_output=True,
|
|
text=True, encoding="utf-8", timeout=timeout)
|
|
check(result.returncode == 0, "docker compose " + " ".join(arguments[:2]) + " failed")
|
|
return result.stdout.strip()
|
|
|
|
|
|
class Stand:
|
|
def __init__(self):
|
|
self.stopped = set()
|
|
|
|
def stop(self, *services):
|
|
# Register before the command: even a partial stop must be undone.
|
|
self.stopped.update(services)
|
|
compose("stop", "-t", "3", *services)
|
|
|
|
def start(self, *services):
|
|
# Compose start --wait may also start dependencies (including S3 while
|
|
# testing its outage). Restart only these exact existing containers.
|
|
ids = compose("ps", "-a", "-q", *services).splitlines()
|
|
check(bool(ids), "No existing containers for requested services")
|
|
result = subprocess.run(["docker", "start", *ids], cwd=ROOT, capture_output=True, timeout=30)
|
|
check(result.returncode == 0, "docker start failed")
|
|
def healthy():
|
|
status = subprocess.run(["docker", "inspect", "--format",
|
|
"{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}healthy{{end}}", *ids],
|
|
cwd=ROOT, capture_output=True, text=True, timeout=10)
|
|
check(status.returncode == 0, "Container state lookup failed")
|
|
return all(line.strip() == "true healthy" for line in status.stdout.splitlines())
|
|
wait("requested containers healthy", healthy, timeout=120)
|
|
self.stopped.difference_update(services)
|
|
|
|
def restore(self):
|
|
if self.stopped:
|
|
self.start(*sorted(self.stopped))
|
|
|
|
|
|
class Business:
|
|
def __init__(self):
|
|
self.token = None
|
|
self.chat = None
|
|
|
|
def call(self, method, path, expected=200, body=None, headers=None, raw=None):
|
|
request_headers = dict(headers or {})
|
|
if self.token:
|
|
request_headers["Authorization"] = "Bearer " + self.token
|
|
status, data, response_headers, content = http(method, BASE + path, body, request_headers, raw)
|
|
check(status == expected, "Business request returned " + str(status) + ", expected " + str(expected))
|
|
check(bool(header(response_headers, "X-Request-Id")), "Missing request ID")
|
|
check(bool(header(response_headers, "X-Instance-Id")), "Missing API instance ID")
|
|
return data, response_headers, content
|
|
|
|
def bootstrap(self):
|
|
username = "obs_" + uuid.uuid4().hex[:20]
|
|
password = "Never-export-" + uuid.uuid4().hex
|
|
SECRETS.append(password)
|
|
user, _, _ = self.call("POST", "/api/v1/users", 201,
|
|
{"username": username, "display_name": "Observability", "password": password})
|
|
credential = base64.b64encode((username + ":" + password).encode()).decode()
|
|
SECRETS.append(credential)
|
|
session, _, _ = self.call("POST", "/api/v1/auth/sessions", 201,
|
|
headers={"Authorization": "Basic " + credential})
|
|
self.token = session["token"]
|
|
SECRETS.append(self.token)
|
|
self.chat, _, _ = self.call("POST", "/api/v1/chats", 201,
|
|
{"title": "Synthetic observations", "member_ids": [user["id"]]})
|
|
|
|
def upload(self):
|
|
rid = "observe-" + uuid.uuid4().hex
|
|
filename = "never-export-name-" + uuid.uuid4().hex + ".png"
|
|
SECRETS.append(filename)
|
|
data, headers, _ = self.call("POST", "/api/v1/chats/" + self.chat["id"] +
|
|
"/attachments?" + urllib.parse.urlencode({"filename": filename}), 202,
|
|
headers={"Content-Type": "image/png", "X-Request-Id": rid},
|
|
raw=(ROOT / "tests/fixtures/sample.png").read_bytes())
|
|
check(data["status"] == "queued", "202 must contain queued snapshot")
|
|
check(header(headers, "X-Request-Id") == rid, "Request ID was not preserved")
|
|
return {"attachment_id": data["id"], "request_id": rid,
|
|
"api_trace_id": header(headers, "X-Trace-Id"), "accepted_at": now()}
|
|
|
|
def ready(self, upload):
|
|
data, _, _ = self.call("GET", "/api/v1/attachments/" + upload["attachment_id"])
|
|
check(data["status"] != "failed", "Synthetic attachment reached failed state")
|
|
return data["status"] == "ready"
|
|
|
|
def verify_content(self, upload):
|
|
import hashlib
|
|
attachment_id = upload["attachment_id"]
|
|
data, _, _ = self.call("GET", "/api/v1/attachments/" + attachment_id)
|
|
_, _, original = self.call("GET", "/api/v1/attachments/" + attachment_id + "/content")
|
|
fixture = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
check(original == fixture, "Recovered original differs from accepted bytes")
|
|
check(data["sha256"] == hashlib.sha256(original).hexdigest(), "Original checksum mismatch")
|
|
thumbnail = next((v for v in data["variants"] if v["name"] == "thumbnail"), None)
|
|
check(thumbnail and (thumbnail["width"], thumbnail["height"]) == (256, 160),
|
|
"Recovered thumbnail dimensions are incorrect")
|
|
return {"attachment_id": attachment_id, "original_sha256": data["sha256"],
|
|
"original_bytes": len(original), "thumbnail_size": [256, 160]}
|
|
|
|
def message(self, prefix="observe-message"):
|
|
text = "Never-export-message-" + uuid.uuid4().hex
|
|
SECRETS.append(text)
|
|
rid = prefix + "-" + uuid.uuid4().hex
|
|
_, headers, _ = self.call("POST", "/api/v1/chats/" + self.chat["id"] + "/messages", 201,
|
|
{"text": text}, {"Idempotency-Key": uuid.uuid4().hex, "X-Request-Id": rid})
|
|
return {"request_id": rid, "trace_id": header(headers, "X-Trace-Id")}
|
|
|
|
|
|
def logs(service, marker):
|
|
field = "request_id" if service == "messenger-api" else "attachment_id"
|
|
# OTel attributes become Loki structured metadata. Body is the source-defined
|
|
# event name; no request body or arbitrary message text is exported.
|
|
expression = '{service_name="' + service + '"} | ' + field + "=" + json.dumps(marker)
|
|
result = get(LOKI + "/loki/api/v1/query_range?" + urllib.parse.urlencode(
|
|
{"query": expression, "start": START_NS, "end": time.time_ns(), "limit": 1000, "direction": "forward"}))
|
|
check(result.get("status") == "success", "Loki query failed")
|
|
records = []
|
|
for stream in result["data"]["result"]:
|
|
for sample in stream.get("values", []):
|
|
try:
|
|
record = json.loads(sample[1])
|
|
except (ValueError, TypeError):
|
|
record = {"event": sample[1]}
|
|
if not isinstance(record, dict):
|
|
record = {"event": sample[1]}
|
|
if len(sample) > 2 and isinstance(sample[2], dict):
|
|
record.update(sample[2])
|
|
# Some Loki versions put structured metadata in stream labels.
|
|
record = {**stream.get("stream", {}), **record}
|
|
records.append(record)
|
|
safe_json(result)
|
|
return result, records
|
|
|
|
|
|
def hex_id(value):
|
|
if not value:
|
|
return ""
|
|
if len(value) in (16, 32) and all(char in "0123456789abcdefABCDEF" for char in value):
|
|
return value.lower()
|
|
try:
|
|
return base64.b64decode(value, validate=True).hex()
|
|
except (ValueError, TypeError):
|
|
return value
|
|
|
|
|
|
def wait_logs(service, marker, predicate, label):
|
|
def matching():
|
|
result = logs(service, marker)
|
|
return result if any(predicate(record) for record in result[1]) else None
|
|
return wait(label, matching)
|
|
|
|
|
|
def trace(trace_id):
|
|
status, data, _, _ = http("GET", TEMPO + "/api/traces/" + trace_id)
|
|
if status == 404:
|
|
return None
|
|
check(status == 200 and data is not None, "Tempo trace lookup failed")
|
|
safe_json(data)
|
|
return data
|
|
|
|
|
|
def attribute_value(value):
|
|
if isinstance(value, dict):
|
|
return next(iter(value.values()), None)
|
|
return value
|
|
|
|
|
|
def spans(data):
|
|
found = []
|
|
def walk(value):
|
|
if isinstance(value, dict):
|
|
if "resource" in value and ("scopeSpans" in value or "instrumentationLibrarySpans" in value):
|
|
attributes = {item["key"]: attribute_value(item["value"])
|
|
for item in value.get("resource", {}).get("attributes", [])}
|
|
for scope in value.get("scopeSpans", value.get("instrumentationLibrarySpans", [])):
|
|
for span in scope.get("spans", []):
|
|
found.append({"service": attributes.get("service.name"), "span": span})
|
|
else:
|
|
for child in value.values():
|
|
walk(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
walk(child)
|
|
walk(data)
|
|
return found
|
|
|
|
|
|
def correlated(upload, prefix="baseline", require_chain=False, require_retry=False):
|
|
api_result = wait_logs("messenger-api", upload["request_id"], lambda record:
|
|
record.get("event") == "http.request.complete" and
|
|
record.get("request_id") == upload["request_id"], "API request log")
|
|
api_log = next(record for record in api_result[1]
|
|
if record.get("event") == "http.request.complete" and record.get("request_id") == upload["request_id"])
|
|
api_tid = hex_id(api_log.get("trace_id"))
|
|
check(len(api_tid) == 32 and int(api_tid, 16) != 0, "API log has no valid trace ID")
|
|
if upload["api_trace_id"]:
|
|
check(api_tid == upload["api_trace_id"], "API response and log disagree on trace ID")
|
|
api_trace = wait("API trace", lambda: trace(api_tid))
|
|
matching_api = [entry for entry in spans(api_trace) if entry["service"] == "messenger-api"
|
|
and hex_id(entry["span"].get("spanId")) == hex_id(api_log.get("span_id"))]
|
|
check(bool(matching_api), "API log span is absent from Tempo trace")
|
|
check(str(matching_api[0]["span"].get("kind")) in ("2", "SPAN_KIND_SERVER"), "API span is not SERVER")
|
|
worker_result = wait_logs("messenger-worker", upload["attachment_id"], lambda record:
|
|
record.get("event") == "job.complete" and record.get("outcome") == "success",
|
|
"worker completion log")
|
|
job_logs = [record for record in worker_result[1] if record.get("event") == "job.complete"]
|
|
worker_log = next(record for record in job_logs if record.get("outcome") == "success")
|
|
check(bool(worker_log.get("job_id")), "Worker log does not identify logical job")
|
|
worker_tid = hex_id(worker_log.get("trace_id"))
|
|
worker_trace = wait("worker trace", lambda: trace(worker_tid))
|
|
matching_worker = [entry for entry in spans(worker_trace) if entry["service"] == "messenger-worker"
|
|
and hex_id(entry["span"].get("spanId")) == hex_id(worker_log.get("span_id"))]
|
|
check(bool(matching_worker), "Worker log span is absent from Tempo trace")
|
|
check(str(matching_worker[0]["span"].get("kind")) in ("5", "SPAN_KIND_CONSUMER"), "Worker span is not CONSUMER")
|
|
check(all(int(entry["span"].get("endTimeUnixNano", 0)) > int(entry["span"].get("startTimeUnixNano", 0))
|
|
for entry in matching_api + matching_worker), "Trace contains zero-duration synthetic span")
|
|
if require_chain:
|
|
check(api_tid == worker_tid, "Upload and worker are not in the same propagated trace")
|
|
def complete_chain():
|
|
value = trace(api_tid)
|
|
return value if value and any(str(entry["span"].get("kind")) in ("4", "SPAN_KIND_PRODUCER")
|
|
for entry in spans(value)) else None
|
|
chain = wait("complete producer/consumer trace", complete_chain)
|
|
names = [entry["span"].get("name", "") for entry in spans(chain)]
|
|
check(any(name.startswith("s3.") for name in names), "S3 operation is absent from upload trace")
|
|
check(any(name.startswith("db.") for name in names), "DB operation is absent from upload trace")
|
|
def has_parent_chain(value):
|
|
nodes = {hex_id(entry["span"].get("spanId")): entry["span"] for entry in spans(value)}
|
|
server_id = hex_id(api_log.get("span_id"))
|
|
for record in job_logs:
|
|
consumer_id = hex_id(record.get("span_id"))
|
|
consumer = nodes.get(consumer_id)
|
|
if consumer is None:
|
|
return False
|
|
producer = nodes.get(hex_id(consumer.get("parentSpanId")))
|
|
if producer is None or str(producer.get("kind")) not in ("4", "SPAN_KIND_PRODUCER"):
|
|
return False
|
|
current, visited = consumer_id, set()
|
|
while current != server_id:
|
|
if not current or current in visited or current not in nodes:
|
|
return False
|
|
visited.add(current)
|
|
current = hex_id(nodes[current].get("parentSpanId"))
|
|
return True
|
|
chain = wait("actual producer/consumer parents lead to upload SERVER span", lambda:
|
|
value if (value := trace(api_tid)) and has_parent_chain(value) else None)
|
|
if require_retry:
|
|
check(any(record.get("outcome") == "error" for record in job_logs), "No failed worker attempt was recorded")
|
|
check(len({record.get("attempt") for record in job_logs}) >= 2, "Retry did not create a later attempt")
|
|
check(all(hex_id(record.get("trace_id")) == api_tid for record in job_logs), "Retry broke propagated trace")
|
|
api_trace = worker_trace = chain
|
|
save(prefix + "-api-logs.json", api_result[0])
|
|
save(prefix + "-worker-logs.json", worker_result[0])
|
|
save(prefix + "-api-trace.json", api_trace)
|
|
save(prefix + "-worker-trace.json", worker_trace)
|
|
return {**upload, "api_trace_id": api_tid, "worker_trace_id": worker_tid,
|
|
"job_id": worker_log["job_id"], "worker_attempts": [record.get("attempt") for record in job_logs],
|
|
"same_trace": api_tid == worker_tid, "correlated_at": now()}
|
|
|
|
|
|
def metric_snapshot():
|
|
expressions = {
|
|
"requests": "messenger_http_requests_total",
|
|
"http_histogram": "messenger_http_duration_seconds_bucket",
|
|
"jobs": "messenger_jobs_processed_total",
|
|
"job_histogram": "messenger_jobs_duration_seconds_bucket",
|
|
}
|
|
result = {name: query(expression) for name, expression in expressions.items()}
|
|
forbidden_labels = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "filename", "url"}
|
|
for rows in result.values():
|
|
for row in rows:
|
|
check(not forbidden_labels.intersection(row["metric"]), "High-cardinality metric labels")
|
|
route = row["metric"].get("route", "")
|
|
check("?" not in route and not re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", route),
|
|
"Raw URL label")
|
|
# SDK export and Prometheus scrape run independently. An empty vector is
|
|
# temporarily missing evidence, while invalid labels above are a failure.
|
|
if not all(result.values()):
|
|
return None
|
|
if not any(float(row["value"][1]) > 0 for row in result["requests"]):
|
|
return None
|
|
if not any(row["metric"].get("outcome") == "success" and float(row["value"][1]) > 0
|
|
for row in result["jobs"]):
|
|
return None
|
|
return {"captured_at": now(), "queries": expressions, "responses": result}
|
|
|
|
|
|
def baseline(client):
|
|
check(get(GRAFANA + "/api/health").get("database") == "ok", "Grafana database is unhealthy")
|
|
upload = client.upload()
|
|
wait("baseline attachment ready", lambda: client.ready(upload), timeout=90)
|
|
correlation = correlated(upload)
|
|
metrics = wait("all application signals", lambda: metric_snapshot())
|
|
result = {"started_at": datetime.fromtimestamp(START_NS / 1e9, timezone.utc).isoformat(),
|
|
"finished_at": now(), "scenario": "real control upload", "correlation": correlation, "metrics": metrics}
|
|
save("observability-baseline.json", result)
|
|
print("PASS: real API/worker logs, traces and metrics; evidence/observability-baseline.json", flush=True)
|
|
|
|
|
|
def chain_with_retry(client, stand):
|
|
stand.stop("worker")
|
|
upload = client.upload()
|
|
try:
|
|
stand.stop("s3")
|
|
stand.start("worker")
|
|
wait("failed attempt during S3 outage", lambda: any(
|
|
record.get("event") == "job.complete" and record.get("outcome") == "error"
|
|
for record in logs("messenger-worker", upload["attachment_id"])[1]), timeout=120)
|
|
finally:
|
|
if "s3" in stand.stopped:
|
|
stand.start("s3")
|
|
if "worker" in stand.stopped:
|
|
stand.start("worker")
|
|
wait("retry attachment ready", lambda: client.ready(upload), timeout=180)
|
|
result = correlated(upload, "retry", require_chain=True, require_retry=True)
|
|
save("observability-retry.json", {"scenario": "S3 unavailable during first attempt, then restored", **result,
|
|
"content": client.verify_content(upload)})
|
|
print("PASS: propagated upload/publish/consume/S3/DB trace including retry", flush=True)
|
|
|
|
|
|
def alert(name):
|
|
payload = get(PROMETHEUS + "/api/v1/alerts")
|
|
return [item for item in payload["data"]["alerts"] if item["labels"].get("alertname") == name]
|
|
|
|
|
|
def alert_state(name, state):
|
|
current = alert(name)
|
|
return current if state == "firing" and any(item["state"] == "firing" for item in current) else (
|
|
{"alertname": name, "state": "resolved", "observed_at": now()} if state == "resolved" and not current else None)
|
|
|
|
|
|
def lag_incident(client, stand, name):
|
|
wait("clean queue and inactive lag alert before injection", lambda:
|
|
number('max(messenger_queue_depth{service_name="messenger-publisher"})') == 0 and
|
|
alert_state(name, "resolved"), timeout=180)
|
|
started = now()
|
|
stand.stop("worker")
|
|
try:
|
|
uploads = [client.upload() for _ in range(4)]
|
|
firing = wait("queue lag alert firing", lambda: alert_state(name, "firing"), timeout=180)
|
|
backlog = query('{__name__=~"messenger_queue_(depth|oldest_age.*|sample_age.*)"}')
|
|
check(bool(backlog), "Queue gauges are absent during worker outage")
|
|
finally:
|
|
stand.start("worker")
|
|
wait("queue drain", lambda: all(client.ready(upload) for upload in uploads), timeout=180)
|
|
resolved = wait("queue lag alert resolved", lambda: alert_state(name, "resolved"), timeout=180)
|
|
return {"incident": "worker stopped", "started_at": started, "finished_at": now(),
|
|
"accepted_attachments": [item["attachment_id"] for item in uploads], "firing": firing,
|
|
"backlog": backlog, "resolved": resolved,
|
|
"content": [client.verify_content(upload) for upload in uploads]}
|
|
|
|
|
|
def direct_db_failure(container_id):
|
|
# Runs inside the existing API container so HAProxy readiness cannot mask
|
|
# the API's real dependency error. It prints only status and correlation IDs.
|
|
code = '''import json,urllib.request,urllib.error,uuid
|
|
body=json.dumps({"username":"incident_"+uuid.uuid4().hex[:16],"display_name":"Synthetic DB incident","password":"SyntheticPasswordForLocalIncidentOnly"}).encode()
|
|
req=urllib.request.Request("http://127.0.0.1:8080/api/v1/users",data=body,headers={"Content-Type":"application/json"},method="POST")
|
|
try:r=urllib.request.urlopen(req,timeout=10)
|
|
except urllib.error.HTTPError as e:r=e
|
|
with r:
|
|
p=json.loads(r.read());print(json.dumps({"status":r.status,"request_id":r.headers.get("X-Request-Id"),"trace_id":r.headers.get("X-Trace-Id"),"error_code":p.get("error",{}).get("code")}))
|
|
'''
|
|
result = subprocess.run(["docker", "exec", container_id, "python", "-c", code], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=20)
|
|
check(result.returncode == 0, "Direct API request during DB outage failed before receiving HTTP")
|
|
value = json.loads(result.stdout)
|
|
check(value["status"] == 503 and value["error_code"] == "unavailable", "DB outage must produce API 503")
|
|
check(bool(value["request_id"]), "DB error has no request ID")
|
|
return value
|
|
|
|
|
|
def db_incident(client, stand, name):
|
|
wait("inactive HTTP error alert before injection", lambda: alert_state(name, "resolved"), timeout=120)
|
|
started = now()
|
|
containers = compose("ps", "-q", "api").splitlines()
|
|
check(len(containers) >= 2, "DB incident requires two actual API containers")
|
|
samples = []
|
|
stopped = threading.Event()
|
|
def flood(container):
|
|
while not stopped.is_set():
|
|
samples.append(direct_db_failure(container))
|
|
stopped.wait(0.5)
|
|
stand.stop("db")
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=len(containers)) as pool:
|
|
futures = [pool.submit(flood, container) for container in containers]
|
|
try:
|
|
def firing_or_error():
|
|
for future in futures:
|
|
if future.done():
|
|
future.result()
|
|
return alert_state(name, "firing")
|
|
firing = wait("HTTP error alert firing", firing_or_error, timeout=180)
|
|
database_metrics = query('{__name__=~"messenger_db_pool_.*"}')
|
|
check(bool(database_metrics), "DB pool metrics are absent")
|
|
finally:
|
|
stopped.set()
|
|
finally:
|
|
stand.start("db")
|
|
wait("API ready after DB recovery", lambda: http("GET", BASE + "/health/ready")[0] == 200, timeout=120)
|
|
client.message()
|
|
resolved = wait("HTTP error alert resolved", lambda: alert_state(name, "resolved"), timeout=180)
|
|
check(bool(samples), "No API dependency errors captured")
|
|
first = samples[0]
|
|
raw_logs, records = wait_logs("messenger-api", first["request_id"], lambda _record: True,
|
|
"DB error correlation log")
|
|
trace_id = first.get("trace_id") or next((row.get("trace_id") for row in records if row.get("trace_id")), None)
|
|
check(bool(trace_id), "DB error log has no trace ID")
|
|
data = wait("DB error trace", lambda: trace(hex_id(trace_id)))
|
|
check(any(entry["span"].get("name", "").startswith("db.") and
|
|
str(entry["span"].get("status", {}).get("code")) in ("2", "STATUS_CODE_ERROR")
|
|
for entry in spans(data)), "No failing DB operation in dependency-error trace")
|
|
save("incident-db-logs.json", raw_logs)
|
|
save("incident-db-trace.json", data)
|
|
return {"incident": "PostgreSQL unavailable", "started_at": started, "finished_at": now(),
|
|
"api_error_responses": samples, "firing": firing, "db_pool": database_metrics, "resolved": resolved}
|
|
|
|
|
|
def collector_snapshot():
|
|
return query('{__name__=~"otelcol_(exporter_(send_failed|enqueue_failed|queue).*|process_memory_rss.*)"}')
|
|
|
|
|
|
def telemetry_incident(client, stand):
|
|
before = collector_snapshot()
|
|
check(bool(before), "Collector self metrics are absent")
|
|
started = now()
|
|
samples = []
|
|
controls = []
|
|
successes = 0
|
|
stand.stop("tempo", "loki")
|
|
try:
|
|
deadline = time.monotonic() + 65
|
|
while time.monotonic() < deadline:
|
|
controls.append(client.message("telemetry-loss"))
|
|
successes += 1
|
|
if not samples or time.time() - samples[-1]["time"] >= 5:
|
|
samples.append({"time": time.time(), "observed_at": now(), "metrics": collector_snapshot()})
|
|
time.sleep(0.15)
|
|
after = collector_snapshot()
|
|
def failure_count(rows):
|
|
return sum(float(row["value"][1]) for row in rows
|
|
if "send_failed" in row["metric"].get("__name__", ""))
|
|
check(failure_count(after) > failure_count(before), "No backend export failures observed")
|
|
rss = [float(row["value"][1]) for sample in samples for row in sample["metrics"]
|
|
if "process_memory_rss" in row["metric"].get("__name__", "")]
|
|
check(bool(rss) and max(rss) < 256 * 1024 * 1024, "Collector RSS missing or exceeds 256 MiB limit")
|
|
for sample in samples:
|
|
capacity = {row["metric"].get("exporter"): float(row["value"][1]) for row in sample["metrics"]
|
|
if "queue_capacity" in row["metric"].get("__name__", "")}
|
|
queue_sizes = [row for row in sample["metrics"] if "queue_size" in row["metric"].get("__name__", "")]
|
|
check(bool(capacity) and bool(queue_sizes), "Bounded exporter queue metrics are absent")
|
|
for row in queue_sizes:
|
|
check(float(row["value"][1]) <= capacity[row["metric"].get("exporter")], "Exporter queue exceeds configured capacity")
|
|
finally:
|
|
stand.start("tempo", "loki")
|
|
wait_backends()
|
|
client.message()
|
|
upload = client.upload()
|
|
wait("job remains functional after telemetry recovery", lambda: client.ready(upload), timeout=90)
|
|
correlation = correlated(upload, "telemetry-recovery", require_chain=True)
|
|
def queues_drained():
|
|
value = number('sum(otelcol_exporter_queue_size{exporter=~"otlphttp/.*"})')
|
|
return value is not None and value == 0
|
|
wait("Collector export queues drained after recovery", queues_drained)
|
|
# Queued metrics can be zero while the last two batches are in flight.
|
|
time.sleep(5)
|
|
availability = []
|
|
for control in controls:
|
|
data = wait("control trace lookup", lambda: (trace(control["trace_id"]),), timeout=60)[0]
|
|
server_present = bool(data) and any(
|
|
entry["service"] == "messenger-api" and
|
|
str(entry["span"].get("kind")) in ("2", "SPAN_KIND_SERVER")
|
|
for entry in spans(data))
|
|
availability.append({**control, "trace_available": data is not None,
|
|
"server_span_available": server_present})
|
|
log_query = '{service_name="messenger-api"} | request_id=~"telemetry-loss-.*"'
|
|
raw_logs = get(LOKI + "/loki/api/v1/query_range?" + urllib.parse.urlencode({
|
|
"query": log_query, "start": START_NS, "end": time.time_ns(), "limit": 5000}))
|
|
present_ids = set()
|
|
for stream in raw_logs["data"]["result"]:
|
|
for sample in stream.get("values", []):
|
|
metadata = {**stream.get("stream", {}), **(sample[2] if len(sample) > 2 else {})}
|
|
if sample[1] == "http.request.complete":
|
|
present_ids.add(metadata.get("request_id"))
|
|
for control in availability:
|
|
control["log_available"] = control["request_id"] in present_ids
|
|
save("telemetry-loss-control-logs.json", raw_logs)
|
|
losses = {"expected_control_requests": len(availability),
|
|
"missing_control_traces": sum(not x["trace_available"] for x in availability),
|
|
"missing_control_server_spans": sum(not x["server_span_available"] for x in availability),
|
|
"missing_control_logs": sum(not x["log_available"] for x in availability),
|
|
"checked_at": now(), "controls": availability}
|
|
return {"incident": "Tempo and Loki unavailable", "started_at": started, "finished_at": now(),
|
|
"successful_message_writes_during_outage": successes, "before": before, "samples": samples,
|
|
"after": after, "max_collector_rss_bytes": max(rss), "recovery": correlation,
|
|
"loss_measurement": losses,
|
|
"interpretation": "send_failed counts failed export attempts, not unique lost spans/log records; "
|
|
"loss_measurement counts absent control request traces/logs after recovery and queue drain, "
|
|
"not all internal spans. Business writes stayed successful."}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--baseline", action="store_true")
|
|
parser.add_argument("--chain-with-retry", action="store_true")
|
|
parser.add_argument("--incidents", action="store_true")
|
|
parser.add_argument("--queue-alert", default="MessengerQueueLag")
|
|
parser.add_argument("--http-alert", default="MessengerHighErrorRate")
|
|
args = parser.parse_args()
|
|
check(args.baseline or args.chain_with_retry or args.incidents, "Choose at least one scenario")
|
|
client, stand = Business(), Stand()
|
|
try:
|
|
wait_backends()
|
|
wait("business API readiness", lambda: http("GET", BASE + "/health/ready", timeout=3)[0] == 200,
|
|
timeout=120)
|
|
client.bootstrap()
|
|
if args.baseline:
|
|
baseline(client)
|
|
if args.chain_with_retry:
|
|
chain_with_retry(client, stand)
|
|
if args.incidents:
|
|
incidents = []
|
|
for scenario in (lambda: lag_incident(client, stand, args.queue_alert),
|
|
lambda: db_incident(client, stand, args.http_alert),
|
|
lambda: telemetry_incident(client, stand)):
|
|
incidents.append(scenario())
|
|
save("observability-incidents.json", {"captured_at": now(), "incidents": incidents})
|
|
print("PASS: " + incidents[-1]["incident"], flush=True)
|
|
finally:
|
|
stand.restore()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (AssertionError, OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError) as exc:
|
|
# Never print backend bodies, exception URLs or subprocess output.
|
|
print("FAIL: " + (str(exc) if isinstance(exc, AssertionError) else type(exc).__name__), flush=True)
|
|
raise SystemExit(1)
|