Files
mtusi2026lab6/student-tests/test_lab6.py
T

148 lines
5.8 KiB
Python

#!/usr/bin/env python3
"""Lab6: controlled incidents + observability correlation."""
import base64, hashlib, json, os, subprocess, sys, time, urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
def prom(expr):
"""Запрос к Prometheus через docker exec."""
url = f"http://localhost:9090/api/v1/query?query={urllib.parse.quote(expr)}"
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
"wget", "-qO-", url],
capture_output=True, text=True, cwd=ROOT, timeout=15)
if r.returncode != 0:
return None
try:
return json.loads(r.stdout)["data"]["result"]
except Exception:
return None
def compose(*args, timeout=120):
r = subprocess.run(["docker", "compose", "--env-file", ".env", *args],
capture_output=True, text=True, cwd=ROOT, timeout=timeout)
return r.returncode, r.stdout + r.stderr
def loki(service_name):
url = ("http://localhost:3100/loki/api/v1/query_range?"
"query=%7Bservice_name%3D%22" + service_name + "%22%7D&limit=5")
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "loki",
"wget", "-qO-", url], capture_output=True, text=True, cwd=ROOT, timeout=15)
return r.stdout
def tempo():
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "tempo",
"wget", "-qO-", "http://localhost:3200/api/search?limit=10"],
capture_output=True, text=True, cwd=ROOT, timeout=15)
return r.stdout
def main():
args = arguments("Lab6 controlled incidents")
c = Client(args.base_url, args.ca_file)
# 1. Smoke: данные текут.
alice, bob, eve, chat = c.bootstrap()
print("1. bootstrap OK, chat:", chat["id"])
att, _ = c.attachment(alice, chat)
ready = c.wait_ready(bob, att, wait_seconds=60)
check(ready["status"] == "ready", "worker pipeline not ready")
# 2. Метрики существуют.
jobs = prom('messenger_jobs_processed_total{outcome="ready"}')
check(jobs and len(jobs) > 0, "messenger_jobs_processed_total missing")
print("2. jobs metric OK:", jobs[0]["value"][1])
http_count = prom('sum(http_server_duration_milliseconds_count{http_target!~"/health/.*"})')
check(http_count and len(http_count) > 0, "http_server_duration_milliseconds_count missing")
print(" http metric OK:", http_count[0]["value"][1])
# 3. Корреляция trace → log.
l = loki("messenger-api")
check("trace_id" in l, "no trace_id in Loki logs")
print("3. Loki correlation OK (trace_id present in logs)")
t = tempo()
check("traces" in t and "messenger-api" in t, "no traces in Tempo")
print(" Tempo traces OK")
# 4. Инцидент 1: остановить worker → очередь растёт.
print("4. stopping worker...")
compose("stop", "worker", timeout=60)
time.sleep(2)
for _ in range(5):
c.attachment(alice, chat)
print(" 5 uploads queued")
# Ждём, пока alert станет firing (for: 2m).
firing = False
deadline = time.monotonic() + 200
while time.monotonic() < deadline:
time.sleep(15)
rules = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
"wget", "-qO-", "http://localhost:9090/api/v1/rules"],
capture_output=True, text=True, cwd=ROOT, timeout=15).stdout
if '"name":"QueueLagHigh"' in rules and '"state":"firing"' in rules:
firing = True
break
check(firing, "QueueLagHigh did not reach firing")
print(" QueueLagHigh firing ✓")
# 5. Инцидент 2: восстановить worker → alert resolved.
print("5. starting worker...")
compose("start", "worker", timeout=60)
resolved = False
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
time.sleep(15)
rules = subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
"wget", "-qO-", "http://localhost:9090/api/v1/rules"],
capture_output=True, text=True, cwd=ROOT, timeout=15).stdout
if ('"name":"QueueLagHigh"' in rules
and '"state":"inactive"' in rules.split('"name":"QueueLagHigh"')[1][:2000]):
resolved = True
break
check(resolved, "QueueLagHigh did not resolve after worker restart")
print(" QueueLagHigh resolved ✓")
# 6. Backlog рассосался.
time.sleep(5)
depth = prom("sum(messenger_queue_depth)")
if depth and depth[0]["value"][1]:
check(float(depth[0]["value"][1]) == 0.0, "queue not drained: " + depth[0]["value"][1])
print("6. queue drained")
# 7. Телеметрия-бэкенд недоступен → API продолжает работать.
print("7. stopping telemetry backends (loki, tempo)...")
compose("stop", "loki", "tempo", timeout=60)
time.sleep(3)
# smoke через API — должен пройти, несмотря на потерю traces/logs.
# В этом состоянии в api пойдёт WARN от exporter — но оно не падает.
a2, b2, e2, chat2 = c.bootstrap()
att2, _ = c.attachment(a2, chat2)
ready2 = c.wait_ready(b2, att2, wait_seconds=60)
check(ready2["status"] == "ready", "API broken when telemetry down")
print(" API still works with telemetry down ✓")
compose("start", "loki", "tempo", timeout=60)
time.sleep(10)
print("\nPASS: lab6 observability (metrics, traces, logs, alerts firing→resolved, telemetry-down resilience)")
if __name__ == "__main__":
run(main)