Files
mtusi2026lab6-fork/student-tests/level4_observability.py
T
2026-11-21 00:02:11 +03:00

85 lines
5.3 KiB
Python

#!/usr/bin/env python3
"""Lab6 level-4 observability scenarios against a RUNNING compose stack (stdlib only, run from the repo root).
python3 student-tests/level4_observability.py trace_chain | queue_lag | db_slow | telemetry_down | all
Redirect to evidence: ... all | tee evidence/level4-observability.txt (no tokens are printed)
"""
import json, subprocess, sys, threading, time, urllib.parse
sys.path.insert(0, __import__("pathlib").Path(__file__).parent.as_posix())
from level4_failures import DC, http, setup, upload, wait, sh, sql
def prom(query):
out = sh(*DC, "exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?query=" + urllib.parse.quote(query))
return json.loads(out)["data"]["result"]
def firing(name):
return any(r["metric"].get("alertname") == name and r["metric"].get("alertstate") == "firing" for r in prom("ALERTS"))
def wait_alert(name, want, timeout):
end = time.time() + timeout
while time.time() < end:
if firing(name) == want:
return round(timeout - (end - time.time()))
time.sleep(5)
raise AssertionError(f"alert {name} did not become {'firing' if want else 'resolved'} in {timeout}s")
def grafana_get(url):
return sh(*DC, "exec", "-T", "grafana", "wget", "-qO-", url)
def trace_chain():
token, chat = setup(); aid = upload(token, chat); wait(token, aid, "ready")
time.sleep(10) # batch export + ingestion
q = urllib.parse.quote(f'{{service_name="messenger-api"}} | event="attachment_accepted" | attachment_id="{aid}"')
res = json.loads(grafana_get(f"http://loki:3100/loki/api/v1/query_range?query={q}"))["data"]["result"]
tid = res[0]["stream"]["trace_id"]
tr = json.loads(grafana_get(f"http://tempo:3200/api/traces/{tid}"))
spans = {}
for b in tr["batches"]:
svc = [a["value"]["stringValue"] for a in b["resource"]["attributes"] if a["key"] == "service.name"][0]
for ss in b["scopeSpans"]:
for s in ss["spans"]:
spans[s["spanId"]] = (svc, s["name"], s.get("kind"), s.get("parentSpanId", ""))
kinds = {(v[0], v[2]) for v in spans.values()}
assert ("messenger-api", "SPAN_KIND_SERVER") in kinds and ("messenger-dispatcher", "SPAN_KIND_PRODUCER") in kinds and ("messenger-worker", "SPAN_KIND_CONSUMER") in kinds, kinds
consumer = next(v for v in spans.values() if v[2] == "SPAN_KIND_CONSUMER")
assert spans[consumer[3]][2] == "SPAN_KIND_PRODUCER", "consumer must be a child of the producer span"
print("PASS trace_chain: trace", tid, "spans:", sorted({f"{v[0]}:{v[1]}" for v in spans.values()}))
def queue_lag(): # incident 1: worker stopped -> oldest job age grows -> QueueLagGrowing firing -> worker back -> resolved
token, chat = setup(); sh(*DC, "stop", "worker"); aid = upload(token, chat)
t = wait_alert("QueueLagGrowing", True, 240); age = prom("max(messenger_queue_oldest_job_age_seconds)")[0]["value"][1]
print(f"queue_lag: FIRING after ~{t}s, oldest_job_age={float(age):.0f}s, depth={prom('max(messenger_queue_depth)')[0]['value'][1]}")
sh(*DC, "up", "-d", "worker"); wait(token, aid, "ready", 120)
print(f"PASS queue_lag: ready after worker restart; RESOLVED after ~{wait_alert('QueueLagGrowing', False, 240)}s")
def db_slow(): # incident 2: ACCESS EXCLUSIVE lock on messages -> INSERT waits until statement_timeout (5s) -> 503
token, chat = setup(); stop = threading.Event(); codes = []
def load():
i = 0
while not stop.is_set():
i += 1
codes.append(http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "x"}, {"Idempotency-Key": f"k{threading.get_ident()}-{i}-{time.time()}"})[0])
lock = subprocess.Popen([*DC, "exec", "-T", "postgres", "sh", "-c", 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(150); COMMIT;"'], stdout=subprocess.DEVNULL)
ts = [threading.Thread(target=load) for _ in range(4)]; [t.start() for t in ts]
t = wait_alert("HighApiErrorRate", True, 240)
print(f"db_slow: FIRING after ~{t}s; 5xx so far: {codes.count(503)} of {len(codes)}")
sql("select pg_terminate_backend(pid) from pg_stat_activity where query like '%pg_sleep(150)%' and pid<>pg_backend_pid()")
lock.wait(); print(f"PASS db_slow: lock released; RESOLVED after ~{wait_alert('HighApiErrorRate', False, 300)}s")
stop.set(); [t.join() for t in ts]
def telemetry_down(): # scenario 3: stop all telemetry stores, API keeps working, Collector RAM stays bounded
token, chat = setup(); sh(*DC, "stop", "prometheus", "loki", "tempo")
bad = 0; end = time.time() + 90
while time.time() < end:
bad += http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "x"}, {"Idempotency-Key": f"t{time.time()}"})[0] != 201
aid = upload(token, chat); wait(token, aid, "ready", 90)
mem = sh("docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", sh(*DC, "ps", "-q", "otel-collector"))
assert bad == 0, bad
sh(*DC, "up", "-d", "prometheus", "loki", "tempo"); time.sleep(60)
print(f"PASS telemetry_down: 0 failed writes with all telemetry stores down; collector memory {mem} (limit 256MiB); stores restarted")
if __name__ == "__main__":
names = {"trace_chain": trace_chain, "queue_lag": queue_lag, "db_slow": db_slow, "telemetry_down": telemetry_down}
for n in (names if sys.argv[1] == "all" else [sys.argv[1]]):
names[n]()