Files
mtusi2026lab6/scripts/observe_lab6.py
T

681 lines
34 KiB
Python

#!/usr/bin/env python3
"""Capture real lab6 observations. Standard library; local synthetic stand only.
Start Compose first. Commands: --baseline, --chain-with-retry, --incidents.
Evidence contains actual backend responses, never credentials or message bodies.
Every stopped service is restarted in finally, including failed assertions.
"""
from __future__ import annotations
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import re
import subprocess
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
PROMETHEUS = os.environ.get("PROMETHEUS_URL", "http://localhost:9090").rstrip("/")
LOKI = os.environ.get("LOKI_URL", "http://localhost:3100").rstrip("/")
TEMPO = os.environ.get("TEMPO_URL", "http://localhost:3200").rstrip("/")
GRAFANA = os.environ.get("GRAFANA_URL", "http://localhost:3000").rstrip("/")
BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
START_NS = int((time.time() - 5) * 1e9)
SECRETS: list[str] = []
def now() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def check(condition, message: str) -> None:
if not condition:
raise AssertionError(message)
def http(method: str, url: str, body=None, headers=None, raw=None, timeout=15):
request_headers = {"Accept": "application/json", **(headers or {})}
if body is not None:
raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
request_headers["Content-Type"] = "application/json"
request = urllib.request.Request(url, data=raw, headers=request_headers, method=method)
try:
response = urllib.request.urlopen(request, timeout=timeout)
except urllib.error.HTTPError as exc:
response = exc
with response:
data = response.read(16 * 1024 * 1024 + 1)
check(len(data) <= 16 * 1024 * 1024, "Backend response exceeds safety limit")
try:
payload = json.loads(data)
except (ValueError, UnicodeError):
payload = None
return response.status, payload, dict(response.headers), data
def header(headers, name):
return next((value for key, value in headers.items() if key.lower() == name.lower()), None)
def get(url):
status, data, _, _ = http("GET", url)
check(status == 200 and data is not None, "Telemetry query failed: HTTP " + str(status))
return data
def query(expression):
result = get(PROMETHEUS + "/api/v1/query?" + urllib.parse.urlencode({"query": expression}))
check(result.get("status") == "success", "Prometheus query is unsuccessful")
return result["data"]["result"]
def number(expression):
rows = query(expression)
return float(rows[0]["value"][1]) if rows else None
def wait(label, predicate, timeout=150, interval=2):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
result = predicate()
except (urllib.error.URLError, TimeoutError, ConnectionError):
# Socket errors are expected during startup/recovery. Assertions
# still propagate immediately: bad telemetry must never be masked.
result = None
if result:
return result
time.sleep(interval)
raise AssertionError("Timed out: " + label)
def wait_backends():
def ready():
endpoints = ((PROMETHEUS, "/-/ready"), (LOKI, "/ready"), (TEMPO, "/ready"))
for base, path in endpoints:
if http("GET", base + path, timeout=3)[0] != 200:
return False
status, payload, _, _ = http("GET", GRAFANA + "/api/health", timeout=3)
return status == 200 and isinstance(payload, dict) and payload.get("database") == "ok"
wait("Prometheus, Loki, Tempo and Grafana readiness", ready, timeout=150)
def safe_json(value):
encoded = json.dumps(value, ensure_ascii=False)
for secret in SECRETS:
check(secret not in encoded, "Sensitive synthetic marker found in exported telemetry")
# These keys are unnecessary for this stand and can expose request data.
forbidden = {"db.statement", "db.query.text", "exception.message", "exception.stacktrace",
"url.full", "http.url", "authorization", "http.request.body", "password"}
def walk(item):
if isinstance(item, dict):
check(not forbidden.intersection(key.lower() for key in item), "Forbidden telemetry key")
if item.get("key", "").lower() in forbidden:
raise AssertionError("Forbidden telemetry attribute")
for child in item.values():
walk(child)
elif isinstance(item, list):
for child in item:
walk(child)
walk(value)
return encoded
def save(name, value):
safe_json(value)
EVIDENCE.mkdir(exist_ok=True)
(EVIDENCE / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def compose(*arguments, timeout=90):
result = subprocess.run(["docker", "compose", *arguments], cwd=ROOT, capture_output=True,
text=True, encoding="utf-8", timeout=timeout)
check(result.returncode == 0, "docker compose " + " ".join(arguments[:2]) + " failed")
return result.stdout.strip()
class Stand:
def __init__(self):
self.stopped = set()
def stop(self, *services):
# Register before the command: even a partial stop must be undone.
self.stopped.update(services)
compose("stop", "-t", "3", *services)
def start(self, *services):
# Compose start --wait may also start dependencies (including S3 while
# testing its outage). Restart only these exact existing containers.
ids = compose("ps", "-a", "-q", *services).splitlines()
check(bool(ids), "No existing containers for requested services")
result = subprocess.run(["docker", "start", *ids], cwd=ROOT, capture_output=True, timeout=30)
check(result.returncode == 0, "docker start failed")
def healthy():
status = subprocess.run(["docker", "inspect", "--format",
"{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}healthy{{end}}", *ids],
cwd=ROOT, capture_output=True, text=True, timeout=10)
check(status.returncode == 0, "Container state lookup failed")
return all(line.strip() == "true healthy" for line in status.stdout.splitlines())
wait("requested containers healthy", healthy, timeout=120)
self.stopped.difference_update(services)
def restore(self):
if self.stopped:
self.start(*sorted(self.stopped))
class Business:
def __init__(self):
self.token = None
self.chat = None
def call(self, method, path, expected=200, body=None, headers=None, raw=None):
request_headers = dict(headers or {})
if self.token:
request_headers["Authorization"] = "Bearer " + self.token
status, data, response_headers, content = http(method, BASE + path, body, request_headers, raw)
check(status == expected, "Business request returned " + str(status) + ", expected " + str(expected))
check(bool(header(response_headers, "X-Request-Id")), "Missing request ID")
check(bool(header(response_headers, "X-Instance-Id")), "Missing API instance ID")
return data, response_headers, content
def bootstrap(self):
username = "obs_" + uuid.uuid4().hex[:20]
password = "Never-export-" + uuid.uuid4().hex
SECRETS.append(password)
user, _, _ = self.call("POST", "/api/v1/users", 201,
{"username": username, "display_name": "Observability", "password": password})
credential = base64.b64encode((username + ":" + password).encode()).decode()
SECRETS.append(credential)
session, _, _ = self.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": "Basic " + credential})
self.token = session["token"]
SECRETS.append(self.token)
self.chat, _, _ = self.call("POST", "/api/v1/chats", 201,
{"title": "Synthetic observations", "member_ids": [user["id"]]})
def upload(self):
rid = "observe-" + uuid.uuid4().hex
filename = "never-export-name-" + uuid.uuid4().hex + ".png"
SECRETS.append(filename)
data, headers, _ = self.call("POST", "/api/v1/chats/" + self.chat["id"] +
"/attachments?" + urllib.parse.urlencode({"filename": filename}), 202,
headers={"Content-Type": "image/png", "X-Request-Id": rid},
raw=(ROOT / "tests/fixtures/sample.png").read_bytes())
check(data["status"] == "queued", "202 must contain queued snapshot")
check(header(headers, "X-Request-Id") == rid, "Request ID was not preserved")
return {"attachment_id": data["id"], "request_id": rid,
"api_trace_id": header(headers, "X-Trace-Id"), "accepted_at": now()}
def ready(self, upload):
data, _, _ = self.call("GET", "/api/v1/attachments/" + upload["attachment_id"])
check(data["status"] != "failed", "Synthetic attachment reached failed state")
return data["status"] == "ready"
def verify_content(self, upload):
import hashlib
attachment_id = upload["attachment_id"]
data, _, _ = self.call("GET", "/api/v1/attachments/" + attachment_id)
_, _, original = self.call("GET", "/api/v1/attachments/" + attachment_id + "/content")
fixture = (ROOT / "tests/fixtures/sample.png").read_bytes()
check(original == fixture, "Recovered original differs from accepted bytes")
check(data["sha256"] == hashlib.sha256(original).hexdigest(), "Original checksum mismatch")
thumbnail = next((v for v in data["variants"] if v["name"] == "thumbnail"), None)
check(thumbnail and (thumbnail["width"], thumbnail["height"]) == (256, 160),
"Recovered thumbnail dimensions are incorrect")
return {"attachment_id": attachment_id, "original_sha256": data["sha256"],
"original_bytes": len(original), "thumbnail_size": [256, 160]}
def message(self, prefix="observe-message"):
text = "Never-export-message-" + uuid.uuid4().hex
SECRETS.append(text)
rid = prefix + "-" + uuid.uuid4().hex
_, headers, _ = self.call("POST", "/api/v1/chats/" + self.chat["id"] + "/messages", 201,
{"text": text}, {"Idempotency-Key": uuid.uuid4().hex, "X-Request-Id": rid})
return {"request_id": rid, "trace_id": header(headers, "X-Trace-Id")}
def logs(service, marker):
field = "request_id" if service == "messenger-api" else "attachment_id"
# OTel attributes become Loki structured metadata. Body is the source-defined
# event name; no request body or arbitrary message text is exported.
expression = '{service_name="' + service + '"} | ' + field + "=" + json.dumps(marker)
result = get(LOKI + "/loki/api/v1/query_range?" + urllib.parse.urlencode(
{"query": expression, "start": START_NS, "end": time.time_ns(), "limit": 1000, "direction": "forward"}))
check(result.get("status") == "success", "Loki query failed")
records = []
for stream in result["data"]["result"]:
for sample in stream.get("values", []):
try:
record = json.loads(sample[1])
except (ValueError, TypeError):
record = {"event": sample[1]}
if not isinstance(record, dict):
record = {"event": sample[1]}
if len(sample) > 2 and isinstance(sample[2], dict):
record.update(sample[2])
# Some Loki versions put structured metadata in stream labels.
record = {**stream.get("stream", {}), **record}
records.append(record)
safe_json(result)
return result, records
def hex_id(value):
if not value:
return ""
if len(value) in (16, 32) and all(char in "0123456789abcdefABCDEF" for char in value):
return value.lower()
try:
return base64.b64decode(value, validate=True).hex()
except (ValueError, TypeError):
return value
def wait_logs(service, marker, predicate, label):
def matching():
result = logs(service, marker)
return result if any(predicate(record) for record in result[1]) else None
return wait(label, matching)
def trace(trace_id):
status, data, _, _ = http("GET", TEMPO + "/api/traces/" + trace_id)
if status == 404:
return None
check(status == 200 and data is not None, "Tempo trace lookup failed")
safe_json(data)
return data
def attribute_value(value):
if isinstance(value, dict):
return next(iter(value.values()), None)
return value
def spans(data):
found = []
def walk(value):
if isinstance(value, dict):
if "resource" in value and ("scopeSpans" in value or "instrumentationLibrarySpans" in value):
attributes = {item["key"]: attribute_value(item["value"])
for item in value.get("resource", {}).get("attributes", [])}
for scope in value.get("scopeSpans", value.get("instrumentationLibrarySpans", [])):
for span in scope.get("spans", []):
found.append({"service": attributes.get("service.name"), "span": span})
else:
for child in value.values():
walk(child)
elif isinstance(value, list):
for child in value:
walk(child)
walk(data)
return found
def correlated(upload, prefix="baseline", require_chain=False, require_retry=False):
api_result = wait_logs("messenger-api", upload["request_id"], lambda record:
record.get("event") == "http.request.complete" and
record.get("request_id") == upload["request_id"], "API request log")
api_log = next(record for record in api_result[1]
if record.get("event") == "http.request.complete" and record.get("request_id") == upload["request_id"])
api_tid = hex_id(api_log.get("trace_id"))
check(len(api_tid) == 32 and int(api_tid, 16) != 0, "API log has no valid trace ID")
if upload["api_trace_id"]:
check(api_tid == upload["api_trace_id"], "API response and log disagree on trace ID")
api_trace = wait("API trace", lambda: trace(api_tid))
matching_api = [entry for entry in spans(api_trace) if entry["service"] == "messenger-api"
and hex_id(entry["span"].get("spanId")) == hex_id(api_log.get("span_id"))]
check(bool(matching_api), "API log span is absent from Tempo trace")
check(str(matching_api[0]["span"].get("kind")) in ("2", "SPAN_KIND_SERVER"), "API span is not SERVER")
worker_result = wait_logs("messenger-worker", upload["attachment_id"], lambda record:
record.get("event") == "job.complete" and record.get("outcome") == "success",
"worker completion log")
job_logs = [record for record in worker_result[1] if record.get("event") == "job.complete"]
worker_log = next(record for record in job_logs if record.get("outcome") == "success")
check(bool(worker_log.get("job_id")), "Worker log does not identify logical job")
worker_tid = hex_id(worker_log.get("trace_id"))
worker_trace = wait("worker trace", lambda: trace(worker_tid))
matching_worker = [entry for entry in spans(worker_trace) if entry["service"] == "messenger-worker"
and hex_id(entry["span"].get("spanId")) == hex_id(worker_log.get("span_id"))]
check(bool(matching_worker), "Worker log span is absent from Tempo trace")
check(str(matching_worker[0]["span"].get("kind")) in ("5", "SPAN_KIND_CONSUMER"), "Worker span is not CONSUMER")
check(all(int(entry["span"].get("endTimeUnixNano", 0)) > int(entry["span"].get("startTimeUnixNano", 0))
for entry in matching_api + matching_worker), "Trace contains zero-duration synthetic span")
if require_chain:
check(api_tid == worker_tid, "Upload and worker are not in the same propagated trace")
def complete_chain():
value = trace(api_tid)
return value if value and any(str(entry["span"].get("kind")) in ("4", "SPAN_KIND_PRODUCER")
for entry in spans(value)) else None
chain = wait("complete producer/consumer trace", complete_chain)
names = [entry["span"].get("name", "") for entry in spans(chain)]
check(any(name.startswith("s3.") for name in names), "S3 operation is absent from upload trace")
check(any(name.startswith("db.") for name in names), "DB operation is absent from upload trace")
def has_parent_chain(value):
nodes = {hex_id(entry["span"].get("spanId")): entry["span"] for entry in spans(value)}
server_id = hex_id(api_log.get("span_id"))
for record in job_logs:
consumer_id = hex_id(record.get("span_id"))
consumer = nodes.get(consumer_id)
if consumer is None:
return False
producer = nodes.get(hex_id(consumer.get("parentSpanId")))
if producer is None or str(producer.get("kind")) not in ("4", "SPAN_KIND_PRODUCER"):
return False
current, visited = consumer_id, set()
while current != server_id:
if not current or current in visited or current not in nodes:
return False
visited.add(current)
current = hex_id(nodes[current].get("parentSpanId"))
return True
chain = wait("actual producer/consumer parents lead to upload SERVER span", lambda:
value if (value := trace(api_tid)) and has_parent_chain(value) else None)
if require_retry:
check(any(record.get("outcome") == "error" for record in job_logs), "No failed worker attempt was recorded")
check(len({record.get("attempt") for record in job_logs}) >= 2, "Retry did not create a later attempt")
check(all(hex_id(record.get("trace_id")) == api_tid for record in job_logs), "Retry broke propagated trace")
api_trace = worker_trace = chain
save(prefix + "-api-logs.json", api_result[0])
save(prefix + "-worker-logs.json", worker_result[0])
save(prefix + "-api-trace.json", api_trace)
save(prefix + "-worker-trace.json", worker_trace)
return {**upload, "api_trace_id": api_tid, "worker_trace_id": worker_tid,
"job_id": worker_log["job_id"], "worker_attempts": [record.get("attempt") for record in job_logs],
"same_trace": api_tid == worker_tid, "correlated_at": now()}
def metric_snapshot():
expressions = {
"requests": "messenger_http_requests_total",
"http_histogram": "messenger_http_duration_seconds_bucket",
"jobs": "messenger_jobs_processed_total",
"job_histogram": "messenger_jobs_duration_seconds_bucket",
}
result = {name: query(expression) for name, expression in expressions.items()}
forbidden_labels = {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "filename", "url"}
for rows in result.values():
for row in rows:
check(not forbidden_labels.intersection(row["metric"]), "High-cardinality metric labels")
route = row["metric"].get("route", "")
check("?" not in route and not re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", route),
"Raw URL label")
# SDK export and Prometheus scrape run independently. An empty vector is
# temporarily missing evidence, while invalid labels above are a failure.
if not all(result.values()):
return None
if not any(float(row["value"][1]) > 0 for row in result["requests"]):
return None
if not any(row["metric"].get("outcome") == "success" and float(row["value"][1]) > 0
for row in result["jobs"]):
return None
return {"captured_at": now(), "queries": expressions, "responses": result}
def baseline(client):
check(get(GRAFANA + "/api/health").get("database") == "ok", "Grafana database is unhealthy")
upload = client.upload()
wait("baseline attachment ready", lambda: client.ready(upload), timeout=90)
correlation = correlated(upload)
metrics = wait("all application signals", lambda: metric_snapshot())
result = {"started_at": datetime.fromtimestamp(START_NS / 1e9, timezone.utc).isoformat(),
"finished_at": now(), "scenario": "real control upload", "correlation": correlation, "metrics": metrics}
save("observability-baseline.json", result)
print("PASS: real API/worker logs, traces and metrics; evidence/observability-baseline.json", flush=True)
def chain_with_retry(client, stand):
stand.stop("worker")
upload = client.upload()
try:
stand.stop("s3")
stand.start("worker")
wait("failed attempt during S3 outage", lambda: any(
record.get("event") == "job.complete" and record.get("outcome") == "error"
for record in logs("messenger-worker", upload["attachment_id"])[1]), timeout=120)
finally:
if "s3" in stand.stopped:
stand.start("s3")
if "worker" in stand.stopped:
stand.start("worker")
wait("retry attachment ready", lambda: client.ready(upload), timeout=180)
result = correlated(upload, "retry", require_chain=True, require_retry=True)
save("observability-retry.json", {"scenario": "S3 unavailable during first attempt, then restored", **result,
"content": client.verify_content(upload)})
print("PASS: propagated upload/publish/consume/S3/DB trace including retry", flush=True)
def alert(name):
payload = get(PROMETHEUS + "/api/v1/alerts")
return [item for item in payload["data"]["alerts"] if item["labels"].get("alertname") == name]
def alert_state(name, state):
current = alert(name)
return current if state == "firing" and any(item["state"] == "firing" for item in current) else (
{"alertname": name, "state": "resolved", "observed_at": now()} if state == "resolved" and not current else None)
def lag_incident(client, stand, name):
wait("clean queue and inactive lag alert before injection", lambda:
number('max(messenger_queue_depth{service_name="messenger-publisher"})') == 0 and
alert_state(name, "resolved"), timeout=180)
started = now()
stand.stop("worker")
try:
uploads = [client.upload() for _ in range(4)]
firing = wait("queue lag alert firing", lambda: alert_state(name, "firing"), timeout=180)
backlog = query('{__name__=~"messenger_queue_(depth|oldest_age.*|sample_age.*)"}')
check(bool(backlog), "Queue gauges are absent during worker outage")
finally:
stand.start("worker")
wait("queue drain", lambda: all(client.ready(upload) for upload in uploads), timeout=180)
resolved = wait("queue lag alert resolved", lambda: alert_state(name, "resolved"), timeout=180)
return {"incident": "worker stopped", "started_at": started, "finished_at": now(),
"accepted_attachments": [item["attachment_id"] for item in uploads], "firing": firing,
"backlog": backlog, "resolved": resolved,
"content": [client.verify_content(upload) for upload in uploads]}
def direct_db_failure(container_id):
# Runs inside the existing API container so HAProxy readiness cannot mask
# the API's real dependency error. It prints only status and correlation IDs.
code = '''import json,urllib.request,urllib.error,uuid
body=json.dumps({"username":"incident_"+uuid.uuid4().hex[:16],"display_name":"Synthetic DB incident","password":"SyntheticPasswordForLocalIncidentOnly"}).encode()
req=urllib.request.Request("http://127.0.0.1:8080/api/v1/users",data=body,headers={"Content-Type":"application/json"},method="POST")
try:r=urllib.request.urlopen(req,timeout=10)
except urllib.error.HTTPError as e:r=e
with r:
p=json.loads(r.read());print(json.dumps({"status":r.status,"request_id":r.headers.get("X-Request-Id"),"trace_id":r.headers.get("X-Trace-Id"),"error_code":p.get("error",{}).get("code")}))
'''
result = subprocess.run(["docker", "exec", container_id, "python", "-c", code], cwd=ROOT,
capture_output=True, text=True, timeout=20)
check(result.returncode == 0, "Direct API request during DB outage failed before receiving HTTP")
value = json.loads(result.stdout)
check(value["status"] == 503 and value["error_code"] == "unavailable", "DB outage must produce API 503")
check(bool(value["request_id"]), "DB error has no request ID")
return value
def db_incident(client, stand, name):
wait("inactive HTTP error alert before injection", lambda: alert_state(name, "resolved"), timeout=120)
started = now()
containers = compose("ps", "-q", "api").splitlines()
check(len(containers) >= 2, "DB incident requires two actual API containers")
samples = []
stopped = threading.Event()
def flood(container):
while not stopped.is_set():
samples.append(direct_db_failure(container))
stopped.wait(0.5)
stand.stop("db")
try:
with ThreadPoolExecutor(max_workers=len(containers)) as pool:
futures = [pool.submit(flood, container) for container in containers]
try:
def firing_or_error():
for future in futures:
if future.done():
future.result()
return alert_state(name, "firing")
firing = wait("HTTP error alert firing", firing_or_error, timeout=180)
database_metrics = query('{__name__=~"messenger_db_pool_.*"}')
check(bool(database_metrics), "DB pool metrics are absent")
finally:
stopped.set()
finally:
stand.start("db")
wait("API ready after DB recovery", lambda: http("GET", BASE + "/health/ready")[0] == 200, timeout=120)
client.message()
resolved = wait("HTTP error alert resolved", lambda: alert_state(name, "resolved"), timeout=180)
check(bool(samples), "No API dependency errors captured")
first = samples[0]
raw_logs, records = wait_logs("messenger-api", first["request_id"], lambda _record: True,
"DB error correlation log")
trace_id = first.get("trace_id") or next((row.get("trace_id") for row in records if row.get("trace_id")), None)
check(bool(trace_id), "DB error log has no trace ID")
data = wait("DB error trace", lambda: trace(hex_id(trace_id)))
check(any(entry["span"].get("name", "").startswith("db.") and
str(entry["span"].get("status", {}).get("code")) in ("2", "STATUS_CODE_ERROR")
for entry in spans(data)), "No failing DB operation in dependency-error trace")
save("incident-db-logs.json", raw_logs)
save("incident-db-trace.json", data)
return {"incident": "PostgreSQL unavailable", "started_at": started, "finished_at": now(),
"api_error_responses": samples, "firing": firing, "db_pool": database_metrics, "resolved": resolved}
def collector_snapshot():
return query('{__name__=~"otelcol_(exporter_(send_failed|enqueue_failed|queue).*|process_memory_rss.*)"}')
def telemetry_incident(client, stand):
before = collector_snapshot()
check(bool(before), "Collector self metrics are absent")
started = now()
samples = []
controls = []
successes = 0
stand.stop("tempo", "loki")
try:
deadline = time.monotonic() + 65
while time.monotonic() < deadline:
controls.append(client.message("telemetry-loss"))
successes += 1
if not samples or time.time() - samples[-1]["time"] >= 5:
samples.append({"time": time.time(), "observed_at": now(), "metrics": collector_snapshot()})
time.sleep(0.15)
after = collector_snapshot()
def failure_count(rows):
return sum(float(row["value"][1]) for row in rows
if "send_failed" in row["metric"].get("__name__", ""))
check(failure_count(after) > failure_count(before), "No backend export failures observed")
rss = [float(row["value"][1]) for sample in samples for row in sample["metrics"]
if "process_memory_rss" in row["metric"].get("__name__", "")]
check(bool(rss) and max(rss) < 256 * 1024 * 1024, "Collector RSS missing or exceeds 256 MiB limit")
for sample in samples:
capacity = {row["metric"].get("exporter"): float(row["value"][1]) for row in sample["metrics"]
if "queue_capacity" in row["metric"].get("__name__", "")}
queue_sizes = [row for row in sample["metrics"] if "queue_size" in row["metric"].get("__name__", "")]
check(bool(capacity) and bool(queue_sizes), "Bounded exporter queue metrics are absent")
for row in queue_sizes:
check(float(row["value"][1]) <= capacity[row["metric"].get("exporter")], "Exporter queue exceeds configured capacity")
finally:
stand.start("tempo", "loki")
wait_backends()
client.message()
upload = client.upload()
wait("job remains functional after telemetry recovery", lambda: client.ready(upload), timeout=90)
correlation = correlated(upload, "telemetry-recovery", require_chain=True)
def queues_drained():
value = number('sum(otelcol_exporter_queue_size{exporter=~"otlphttp/.*"})')
return value is not None and value == 0
wait("Collector export queues drained after recovery", queues_drained)
# Queued metrics can be zero while the last two batches are in flight.
time.sleep(5)
availability = []
for control in controls:
data = wait("control trace lookup", lambda: (trace(control["trace_id"]),), timeout=60)[0]
server_present = bool(data) and any(
entry["service"] == "messenger-api" and
str(entry["span"].get("kind")) in ("2", "SPAN_KIND_SERVER")
for entry in spans(data))
availability.append({**control, "trace_available": data is not None,
"server_span_available": server_present})
log_query = '{service_name="messenger-api"} | request_id=~"telemetry-loss-.*"'
raw_logs = get(LOKI + "/loki/api/v1/query_range?" + urllib.parse.urlencode({
"query": log_query, "start": START_NS, "end": time.time_ns(), "limit": 5000}))
present_ids = set()
for stream in raw_logs["data"]["result"]:
for sample in stream.get("values", []):
metadata = {**stream.get("stream", {}), **(sample[2] if len(sample) > 2 else {})}
if sample[1] == "http.request.complete":
present_ids.add(metadata.get("request_id"))
for control in availability:
control["log_available"] = control["request_id"] in present_ids
save("telemetry-loss-control-logs.json", raw_logs)
losses = {"expected_control_requests": len(availability),
"missing_control_traces": sum(not x["trace_available"] for x in availability),
"missing_control_server_spans": sum(not x["server_span_available"] for x in availability),
"missing_control_logs": sum(not x["log_available"] for x in availability),
"checked_at": now(), "controls": availability}
return {"incident": "Tempo and Loki unavailable", "started_at": started, "finished_at": now(),
"successful_message_writes_during_outage": successes, "before": before, "samples": samples,
"after": after, "max_collector_rss_bytes": max(rss), "recovery": correlation,
"loss_measurement": losses,
"interpretation": "send_failed counts failed export attempts, not unique lost spans/log records; "
"loss_measurement counts absent control request traces/logs after recovery and queue drain, "
"not all internal spans. Business writes stayed successful."}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--baseline", action="store_true")
parser.add_argument("--chain-with-retry", action="store_true")
parser.add_argument("--incidents", action="store_true")
parser.add_argument("--queue-alert", default="MessengerQueueLag")
parser.add_argument("--http-alert", default="MessengerHighErrorRate")
args = parser.parse_args()
check(args.baseline or args.chain_with_retry or args.incidents, "Choose at least one scenario")
client, stand = Business(), Stand()
try:
wait_backends()
wait("business API readiness", lambda: http("GET", BASE + "/health/ready", timeout=3)[0] == 200,
timeout=120)
client.bootstrap()
if args.baseline:
baseline(client)
if args.chain_with_retry:
chain_with_retry(client, stand)
if args.incidents:
incidents = []
for scenario in (lambda: lag_incident(client, stand, args.queue_alert),
lambda: db_incident(client, stand, args.http_alert),
lambda: telemetry_incident(client, stand)):
incidents.append(scenario())
save("observability-incidents.json", {"captured_at": now(), "incidents": incidents})
print("PASS: " + incidents[-1]["incident"], flush=True)
finally:
stand.restore()
if __name__ == "__main__":
try:
main()
except (AssertionError, OSError, ValueError, subprocess.SubprocessError, urllib.error.URLError) as exc:
# Never print backend bodies, exception URLs or subprocess output.
print("FAIL: " + (str(exc) if isinstance(exc, AssertionError) else type(exc).__name__), flush=True)
raise SystemExit(1)