forked from mtusi2026/mtusi2026lab6
85 lines
5.3 KiB
Python
85 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab6 level-4 observability scenarios against a RUNNING compose stack (stdlib only, run from the repo root).
|
|
|
|
python3 student-tests/level4_observability.py trace_chain | queue_lag | db_slow | telemetry_down | all
|
|
Redirect to evidence: ... all | tee evidence/level4-observability.txt (no tokens are printed)
|
|
"""
|
|
import json, subprocess, sys, threading, time, urllib.parse
|
|
sys.path.insert(0, __import__("pathlib").Path(__file__).parent.as_posix())
|
|
from level4_failures import DC, http, setup, upload, wait, sh, sql
|
|
|
|
def prom(query):
|
|
out = sh(*DC, "exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?query=" + urllib.parse.quote(query))
|
|
return json.loads(out)["data"]["result"]
|
|
|
|
def firing(name):
|
|
return any(r["metric"].get("alertname") == name and r["metric"].get("alertstate") == "firing" for r in prom("ALERTS"))
|
|
|
|
def wait_alert(name, want, timeout):
|
|
end = time.time() + timeout
|
|
while time.time() < end:
|
|
if firing(name) == want:
|
|
return round(timeout - (end - time.time()))
|
|
time.sleep(5)
|
|
raise AssertionError(f"alert {name} did not become {'firing' if want else 'resolved'} in {timeout}s")
|
|
|
|
def grafana_get(url):
|
|
return sh(*DC, "exec", "-T", "grafana", "wget", "-qO-", url)
|
|
|
|
def trace_chain():
|
|
token, chat = setup(); aid = upload(token, chat); wait(token, aid, "ready")
|
|
time.sleep(10) # batch export + ingestion
|
|
q = urllib.parse.quote(f'{{service_name="messenger-api"}} | event="attachment_accepted" | attachment_id="{aid}"')
|
|
res = json.loads(grafana_get(f"http://loki:3100/loki/api/v1/query_range?query={q}"))["data"]["result"]
|
|
tid = res[0]["stream"]["trace_id"]
|
|
tr = json.loads(grafana_get(f"http://tempo:3200/api/traces/{tid}"))
|
|
spans = {}
|
|
for b in tr["batches"]:
|
|
svc = [a["value"]["stringValue"] for a in b["resource"]["attributes"] if a["key"] == "service.name"][0]
|
|
for ss in b["scopeSpans"]:
|
|
for s in ss["spans"]:
|
|
spans[s["spanId"]] = (svc, s["name"], s.get("kind"), s.get("parentSpanId", ""))
|
|
kinds = {(v[0], v[2]) for v in spans.values()}
|
|
assert ("messenger-api", "SPAN_KIND_SERVER") in kinds and ("messenger-dispatcher", "SPAN_KIND_PRODUCER") in kinds and ("messenger-worker", "SPAN_KIND_CONSUMER") in kinds, kinds
|
|
consumer = next(v for v in spans.values() if v[2] == "SPAN_KIND_CONSUMER")
|
|
assert spans[consumer[3]][2] == "SPAN_KIND_PRODUCER", "consumer must be a child of the producer span"
|
|
print("PASS trace_chain: trace", tid, "spans:", sorted({f"{v[0]}:{v[1]}" for v in spans.values()}))
|
|
|
|
def queue_lag(): # incident 1: worker stopped -> oldest job age grows -> QueueLagGrowing firing -> worker back -> resolved
|
|
token, chat = setup(); sh(*DC, "stop", "worker"); aid = upload(token, chat)
|
|
t = wait_alert("QueueLagGrowing", True, 240); age = prom("max(messenger_queue_oldest_job_age_seconds)")[0]["value"][1]
|
|
print(f"queue_lag: FIRING after ~{t}s, oldest_job_age={float(age):.0f}s, depth={prom('max(messenger_queue_depth)')[0]['value'][1]}")
|
|
sh(*DC, "up", "-d", "worker"); wait(token, aid, "ready", 120)
|
|
print(f"PASS queue_lag: ready after worker restart; RESOLVED after ~{wait_alert('QueueLagGrowing', False, 240)}s")
|
|
|
|
def db_slow(): # incident 2: ACCESS EXCLUSIVE lock on messages -> INSERT waits until statement_timeout (5s) -> 503
|
|
token, chat = setup(); stop = threading.Event(); codes = []
|
|
def load():
|
|
i = 0
|
|
while not stop.is_set():
|
|
i += 1
|
|
codes.append(http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "x"}, {"Idempotency-Key": f"k{threading.get_ident()}-{i}-{time.time()}"})[0])
|
|
lock = subprocess.Popen([*DC, "exec", "-T", "postgres", "sh", "-c", 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(150); COMMIT;"'], stdout=subprocess.DEVNULL)
|
|
ts = [threading.Thread(target=load) for _ in range(4)]; [t.start() for t in ts]
|
|
t = wait_alert("HighApiErrorRate", True, 240)
|
|
print(f"db_slow: FIRING after ~{t}s; 5xx so far: {codes.count(503)} of {len(codes)}")
|
|
sql("select pg_terminate_backend(pid) from pg_stat_activity where query like '%pg_sleep(150)%' and pid<>pg_backend_pid()")
|
|
lock.wait(); print(f"PASS db_slow: lock released; RESOLVED after ~{wait_alert('HighApiErrorRate', False, 300)}s")
|
|
stop.set(); [t.join() for t in ts]
|
|
|
|
def telemetry_down(): # scenario 3: stop all telemetry stores, API keeps working, Collector RAM stays bounded
|
|
token, chat = setup(); sh(*DC, "stop", "prometheus", "loki", "tempo")
|
|
bad = 0; end = time.time() + 90
|
|
while time.time() < end:
|
|
bad += http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "x"}, {"Idempotency-Key": f"t{time.time()}"})[0] != 201
|
|
aid = upload(token, chat); wait(token, aid, "ready", 90)
|
|
mem = sh("docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", sh(*DC, "ps", "-q", "otel-collector"))
|
|
assert bad == 0, bad
|
|
sh(*DC, "up", "-d", "prometheus", "loki", "tempo"); time.sleep(60)
|
|
print(f"PASS telemetry_down: 0 failed writes with all telemetry stores down; collector memory {mem} (limit 256MiB); stores restarted")
|
|
|
|
if __name__ == "__main__":
|
|
names = {"trace_chain": trace_chain, "queue_lag": queue_lag, "db_slow": db_slow, "telemetry_down": telemetry_down}
|
|
for n in (names if sys.argv[1] == "all" else [sys.argv[1]]):
|
|
names[n]()
|