#!/usr/bin/env python3 """Grade-6 observability evidence. Prints statuses, ids, and metric values only.""" import base64 import json import subprocess import sys import time import urllib.parse import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure # noqa: E402 OUT = ROOT / "evidence" / "grade6.txt" lines = [] def log(message): print(message, flush=True) lines.append(message) def run(args, check=True): result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True) if check and result.returncode != 0: raise SystemExit(args[0] + " failed: " + (result.stderr or result.stdout)[-400:]) return result def compose(*args): run(["docker", "compose", "--env-file", ".env", *args]) def env_value(name, default): for line in (ROOT / ".env").read_text().splitlines(): if line.startswith(name + "="): return line.split("=", 1)[1].strip() return default def prom(query): url = "http://127.0.0.1:9090/api/v1/query?" + urllib.parse.urlencode({"query": query}) with urllib.request.urlopen(url, timeout=10) as response: data = json.loads(response.read().decode()) if data.get("status") != "success": return "error" result = data["data"]["result"] if not result: return "empty" if len(result) == 1 and not result[0]["metric"]: return result[0]["value"][1] return "; ".join( ",".join(f"{k}={v}" for k, v in item["metric"].items() if k != "__name__") + "=" + item["value"][1] for item in result[:6] ) def grafana(path): user = env_value("GRAFANA_ADMIN_USER", "admin") password = env_value("GRAFANA_ADMIN_PASSWORD", "lab-grafana") token = base64.b64encode((user + ":" + password).encode()).decode() request = urllib.request.Request( "http://127.0.0.1:3000" + path, headers={"Authorization": "Basic " + token}, ) with urllib.request.urlopen(request, timeout=10) as response: return json.loads(response.read().decode()) def alert_states(): payload = grafana("/api/prometheus/grafana/api/v1/alerts") states = {} for alert in payload.get("data", {}).get("alerts", []): states[alert.get("labels", {}).get("alertname", "?")] = alert.get("state", "?") return states def wait_state(name, wanted, timeout): deadline = time.monotonic() + timeout last = {} while time.monotonic() < deadline: last = alert_states() state = last.get(name, "missing") if state in wanted: return state time.sleep(5) return "timeout:" + last.get(name, "missing") def container_logs(name, since): result = run(["docker", "logs", "--since", since, name], check=False) return result.stdout + result.stderr def trace_id_for(attachment_id=None, route=None, since="5m"): deadline = time.monotonic() + 25 while time.monotonic() < deadline: blob = "" for name in ("messenger-lab6-api-1", "messenger-lab6-api-2", "messenger-lab6-worker-1"): blob += container_logs(name, since) for line in blob.splitlines(): if "trace_id=" not in line: continue if attachment_id and attachment_id not in line: continue if route and route not in line: continue if "password" in line.lower() or "Bearer " in line: continue marker = line.split("trace_id=", 1)[1] return marker.split()[0] time.sleep(1) return None def fetch_trace(trace_id): last = "" for _ in range(8): result = run([ "docker", "exec", "messenger-lab6-api-1", "curl", "-fsS", "http://tempo:3200/api/traces/" + trace_id, ], check=False) if result.returncode == 0 and result.stdout.strip().startswith("{"): return json.loads(result.stdout) last = (result.stderr or result.stdout)[-200:] time.sleep(2) raise SystemExit("tempo trace " + trace_id + " unavailable: " + last) def hex_id(value): if not value: return "" return base64.b64decode(value).hex() def span_rows(trace): rows = {} for batch in trace.get("batches", []): service = "?" for attribute in batch.get("resource", {}).get("attributes", []): if attribute.get("key") == "service.name": service = attribute.get("value", {}).get("stringValue", "?") for scope in batch.get("scopeSpans", []): for span in scope.get("spans", []): start = int(span.get("startTimeUnixNano", "0")) end = int(span.get("endTimeUnixNano", "0")) status = (span.get("status") or {}).get("code", "") rows[hex_id(span.get("spanId"))] = { "service": service, "name": span.get("name", ""), "parent": hex_id(span.get("parentSpanId")), "ms": (end - start) / 1_000_000, "status": status, } return rows def describe(trace_id, interesting): rows = span_rows(fetch_trace(trace_id)) log("trace " + trace_id + " spans=" + str(len(rows))) for sid, row in rows.items(): if not any(token in row["name"] for token in interesting): continue parent = rows.get(row["parent"], {}).get("name", "-") log( " " + row["service"] + " " + row["name"] + " parent=" + parent + " dur_ms=" + format(row["ms"], ".1f") + (" " + row["status"] if row["status"] else "") ) query_parents = {} for row in rows.values(): if row["name"] != "query": continue parent = rows.get(row["parent"], {}).get("name", "?") query_parents[parent] = query_parents.get(parent, 0) + 1 if query_parents: log(" jdbc query parents " + json.dumps(query_parents, ensure_ascii=False)) return rows def stats(): result = run([ "docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}", "messenger-lab6-otel-collector-1", "messenger-lab6-api-1", "messenger-lab6-api-2", "messenger-lab6-worker-1", ]) return " | ".join(line.strip() for line in result.stdout.splitlines()) def main(): OUT.parent.mkdir(exist_ok=True) log("make test") test = subprocess.run(["make", "test"], cwd=ROOT, text=True, capture_output=True) (ROOT / "evidence" / "results.txt").write_text(test.stdout + test.stderr) tail = [line for line in (test.stdout + test.stderr).splitlines() if line.strip()] log("make test exit=" + str(test.returncode) + " last=" + (tail[-1] if tail else "")) client = Client("http://127.0.0.1:8080") alice, _bob, _eve, chat = client.bootstrap() token = env_value("DEMO_TOKEN", "lab-demo") log("upload chain") attachment, _content = client.attachment(alice, chat) ready = client.wait_ready(alice, attachment) log("attachment " + ready["id"] + " status=" + ready["status"]) trace = trace_id_for(attachment_id=attachment["id"]) log("accepted trace_id=" + str(trace)) if trace: describe(trace, ("http post", "attachments.process", "s3.")) log("transient retry") run(["docker", "exec", "messenger-lab6-worker-1", "touch", "/tmp/messenger-fail-transient"]) retried, _content = client.attachment(alice, chat) retried_ready = client.wait_ready(alice, retried, wait_seconds=60) log("retry attachment " + retried_ready["id"] + " status=" + retried_ready["status"]) retry_trace = trace_id_for(attachment_id=retried["id"]) log("retry trace_id=" + str(retry_trace)) if retry_trace: rows = describe(retry_trace, ("attachments.process", "s3.")) consumers = [row for row in rows.values() if row["name"] == "attachments.process process"] publishers = [row for row in rows.values() if row["name"] == "attachments.process publish"] log("retry process spans=" + str(len(consumers)) + " publish spans=" + str(len(publishers))) log("latency before faults p50_s=" + str(prom( 'histogram_quantile(0.50, sum by (le) (rate(http_server_requests_milliseconds_bucket{uri!~"/health.*"}[5m]))) / 1000' ))) log("latency before faults p95_s=" + str(prom( 'histogram_quantile(0.95, sum by (le) (rate(http_server_requests_milliseconds_bucket{uri!~"/health.*"}[5m]))) / 1000' ))) log("latency before faults p99_s=" + str(prom( 'histogram_quantile(0.99, sum by (le) (rate(http_server_requests_milliseconds_bucket{uri!~"/health.*"}[5m]))) / 1000' ))) log("slow database vs slow handler") client.call( "POST", "/internal/demo/slow-db?seconds=3", expected=200, actor=alice, headers={"X-Demo-Token": token}, schema=None, ) slow_db = trace_id_for(route="route=/internal/demo/slow-db") log("slow-db trace_id=" + str(slow_db)) if slow_db: rows = describe(slow_db, ("http post",)) slowest = max((row["ms"] for row in rows.values() if row["name"] == "query"), default=0) log("slow-db longest jdbc query ms=" + format(slowest, ".1f")) client.call( "POST", "/internal/demo/slow-handler?seconds=3", expected=200, actor=alice, headers={"X-Demo-Token": token}, schema=None, ) slow_handler = trace_id_for(route="route=/internal/demo/slow-handler") log("slow-handler trace_id=" + str(slow_handler)) if slow_handler: rows = describe(slow_handler, ("http post",)) slowest = max((row["ms"] for row in rows.values() if row["name"] == "query"), default=0) server = max((row["ms"] for row in rows.values() if row["name"].startswith("http post")), default=0) log("slow-handler server ms=" + format(server, ".1f") + " longest jdbc query ms=" + format(slowest, ".1f")) log("queue lag incident") compose("stop", "worker") queued, _content = client.attachment(alice, chat) log("queued while worker stopped " + queued["id"]) deadline = time.monotonic() + 40 age = "empty" while time.monotonic() < deadline: age = prom('max(messenger_queue_oldest_age_seconds{queue="attachments.process"})') try: if float(age) > 15: break except ValueError: pass time.sleep(5) log("queue age after wait s=" + str(age)) state = wait_state("Attachment queue lag", {"Alerting"}, 150) log("queue alert while stopped=" + state + " age_s=" + str(prom( 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})' ))) compose("start", "worker") recovered = client.wait_ready(alice, queued, wait_seconds=90) log("queue recovered " + recovered["id"] + " status=" + recovered["status"]) resolved = wait_state("Attachment queue lag", {"Normal", "Normal (NoData)"}, 90) log("queue alert after recovery=" + resolved + " age_s=" + str(prom( 'max(messenger_queue_oldest_age_seconds{queue="attachments.process"})' ))) log("error rate incident") started = time.monotonic() sent = 0 while time.monotonic() - started < 100: client.call("POST", "/internal/demo/error", expected=500, actor=alice, headers={"X-Demo-Token": token}) sent += 1 time.sleep(0.15) log("demo 500 responses=" + str(sent)) error_state = wait_state("High business HTTP error rate", {"Alerting", "Pending"}, 40) ratio = prom( 'sum(rate(http_server_requests_milliseconds_count{status=~"5..",uri!~"/health.*"}[2m]))' ' / clamp_min(sum(rate(http_server_requests_milliseconds_count{uri!~"/health.*"}[2m])), 1e-9)' ) log("error alert during burst=" + error_state + " ratio=" + str(ratio)) resolved = wait_state("High business HTTP error rate", {"Normal", "Normal (NoData)"}, 200) log("error alert after quiet=" + resolved + " ratio=" + str(prom( 'sum(rate(http_server_requests_milliseconds_count{status=~"5..",uri!~"/health.*"}[2m]))' ' / clamp_min(sum(rate(http_server_requests_milliseconds_count{uri!~"/health.*"}[2m])), 1e-9)' ))) log("telemetry backends down") before = stats() log("mem before " + before) compose("stop", "prometheus", "loki", "tempo") message = client.message(alice, chat, "telemetry-down") log("message while stores down status=201 id=" + message["json"]["id"]) held, _content = client.attachment(alice, chat) held_ready = client.wait_ready(alice, held, wait_seconds=60) log("attachment while stores down " + held_ready["id"] + " status=" + held_ready["status"]) log("mem during " + stats()) time.sleep(20) log("mem 20s later " + stats()) collector = container_logs("messenger-lab6-otel-collector-1", "2m") drops = 0 for line in collector.splitlines(): lowered = line.lower() if "export" in lowered and ("fail" in lowered or "error" in lowered or "drop" in lowered): drops += 1 log("collector export failure lines in last 2m=" + str(drops)) if __name__ == "__main__": try: main() except (Failure, OSError, subprocess.SubprocessError, json.JSONDecodeError) as exc: log("stopped: " + type(exc).__name__ + " " + str(exc).splitlines()[0][:300]) raise finally: subprocess.run( ["docker", "compose", "--env-file", ".env", "start", "worker", "prometheus", "loki", "tempo"], cwd=ROOT, text=True, capture_output=True, ) OUT.write_text("\n".join(lines) + "\n")