#!/usr/bin/env python3 """Collect sanitized evidence from the local lab6 stand.""" import argparse import json import subprocess import sys import time from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import urlopen ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 def compose(*args, **kwargs): return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs) def api_json(port, path, **params): url = f"http://127.0.0.1:{port}{path}" if params: url += "?" + urlencode(params) with urlopen(url, timeout=5) as response: return json.load(response) def prom(query): data = api_json(9096, "/api/v1/query", query=query)["data"]["result"] return float(data[0]["value"][1]) if data else 0 def until(check, timeout=45): end = time.monotonic() + timeout while time.monotonic() < end: try: result = check() if result: return result except (HTTPError, URLError): pass time.sleep(0.5) raise RuntimeError(f"Timed out waiting for {check.__name__}") def log_for(request_id): data = api_json( 3106, "/loki/api/v1/query_range", query='{service_name="messenger-api"} | request_id="' + request_id + '"', start=str(time.time_ns() - 180_000_000_000), limit=5, )["data"]["result"] for stream in data: for value in stream["values"]: return value[2] if len(value) > 2 else stream["stream"] return None def span_names(trace_id): data = api_json(3206, "/api/traces/" + trace_id) return [ (span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId")) for batch in data.get("batches", data.get("resourceSpans", [])) for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])) for span in group["spans"] ] def alerts(): return { a["labels"]["alertname"]: a["state"] for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"] } def correlation(client, actor, chat, retry=False): if retry: compose("stop", "worker", stdout=subprocess.DEVNULL) try: attachment = client.call( "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png", 202, actor=actor, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(), headers={"Content-Type": "image/png"}, schema="Attachment", ) if retry: job_attachment = attachment["json"]["id"] compose( "exec", "-T", "postgres", "psql", "-U", "messenger", "-Xq", "-v", "ON_ERROR_STOP=1", "-v", "attachment=" + job_attachment, input=( "INSERT INTO pipeline_faults(job_id,phase,action,remaining) " "SELECT job_id,'before_effect','transient',1 FROM attachment_jobs " "WHERE attachment_id=:'attachment';\n" ), text=True, stdout=subprocess.DEVNULL, ) finally: if retry: compose("start", "worker", stdout=subprocess.DEVNULL) client.wait_ready(actor, attachment["json"]) request_id = attachment["headers"]["X-Request-Id"] log = until(lambda: log_for(request_id)) trace_id = log["trace_id"] expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"} def connected(): spans = span_names(trace_id) names = {row[0] for row in spans} consumers = sum(row[0] == "jobs.consume" for row in spans) return spans if expected <= names and consumers >= (2 if retry else 1) else None spans = until(connected) by_id = {row[2]: row for row in spans} consumers = [row for row in spans if row[0] == "jobs.consume"] assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers) logs = until( lambda: api_json( 3106, "/loki/api/v1/query_range", query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"', start=str(time.time_ns() - 180_000_000_000), limit=5, )["data"]["result"] ) assert any(value[1] == "job_processed" for row in logs for value in row["values"]) return { "request_id": request_id, "trace_id": trace_id, "span_names": sorted({r[0] for r in spans}), "worker_parent": by_id[consumers[0][3]][0], "worker_attempts": len(consumers), "api_and_worker_loki": True, } def worker_incident(client, actor, chat): compose("stop", "worker", stdout=subprocess.DEVNULL) try: uploads = [client.attachment(actor, chat)[0] for _ in range(3)] until(lambda: alerts().get("MessengerQueueLag") == "firing") record = { "depth_at_firing": prom("max(messenger_queue_depth)"), "oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"), } assert record["depth_at_firing"] >= 3 finally: compose("start", "worker", stdout=subprocess.DEVNULL) for attachment in uploads: client.wait_ready(actor, attachment) until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts()) return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)} def database_incident(client, actor): # Independent, bounded transaction. Lock sessions only; readiness can still use the schema. command = [ "docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-XqAt", "-v", "ON_ERROR_STOP=1", ] start = time.monotonic() lock = subprocess.Popen( command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True ) lock.stdin.write( "BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n" "\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n" ) lock.stdin.close() try: assert lock.stdout.readline().strip() == "locked" attempts = [] while time.monotonic() - start < 34: attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User") attempts.append(attempt) states = alerts() if all( states.get(a) == "firing" for a in ("MessengerHttpErrors", "MessengerDatabaseErrors") ): break assert any(item["status"] == 503 for item in attempts) active = alerts() assert active.get("MessengerDatabaseErrors") == "firing", active assert active.get("MessengerHttpErrors") == "firing", active finally: lock.wait(timeout=46) lock.stdout.close() client.call("GET", "/api/v1/users/me", actor=actor, schema="User") # Keep successful traffic to publish fresh data while the alert window slides. def resolved(): client.call("GET", "/api/v1/users/me", actor=actor, schema="User") return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys()) until(resolved) failure = next(item for item in attempts if item["status"] == 503) failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"])) names = span_names(failure_log["trace_id"]) assert "db.read" in {row[0] for row in names} return { "statuses": [item["status"] for item in attempts], "max_latency_ms": max(item["elapsed"] for item in attempts) * 1000, "trace_id": failure_log["trace_id"], "state": ["firing", "resolved"], "spans": sorted({row[0] for row in names}), } def backend_incident(client, actor, chat): lost = "sum(otelcol_exporter_send_failed_spans_total)" baseline = prom(lost) compose("stop", "tempo", stdout=subprocess.DEVNULL) try: for _ in range(15): client.message(actor, chat, "synthetic telemetry outage probe") time.sleep(0.5) delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None) memory = prom("max(otelcol_process_memory_rss_bytes)") assert memory > 0 finally: compose("start", "tempo", stdout=subprocess.DEVNULL) client.call("GET", "/health/ready", schema="Health") return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:8086") parser.add_argument("--output", type=Path, required=True) parser.add_argument("--incidents", action="store_true") parser.add_argument("--retry", action="store_true") args = parser.parse_args() if args.output.exists(): parser.error("choose a new evidence file") client = Client(args.base_url) alice, _, _, chat = client.bootstrap() result = {"correlation": correlation(client, alice, chat, retry=args.retry)} print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True) if args.incidents: result["worker_stop"] = worker_incident(client, alice, chat) print("PASS: queue alert firing and resolved", flush=True) result["database_lock"] = database_incident(client, alice) print("PASS: database and HTTP alerts firing and resolved", flush=True) result["tempo_stop"] = backend_incident(client, alice, chat) print("PASS: messages persisted during Tempo outage", flush=True) args.output.write_text(json.dumps(result, indent=2) + "\n") if __name__ == "__main__": main()