148 lines
5.8 KiB
Python
148 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab6: controlled incidents + observability correlation."""
|
|
import base64, hashlib, json, os, subprocess, sys, time, urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run # noqa: E402
|
|
|
|
|
|
def prom(expr):
|
|
"""Запрос к Prometheus через docker exec."""
|
|
url = f"http://localhost:9090/api/v1/query?query={urllib.parse.quote(expr)}"
|
|
r = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
|
|
"wget", "-qO-", url],
|
|
capture_output=True, text=True, cwd=ROOT, timeout=15)
|
|
if r.returncode != 0:
|
|
return None
|
|
try:
|
|
return json.loads(r.stdout)["data"]["result"]
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def compose(*args, timeout=120):
|
|
r = subprocess.run(["docker", "compose", "--env-file", ".env", *args],
|
|
capture_output=True, text=True, cwd=ROOT, timeout=timeout)
|
|
return r.returncode, r.stdout + r.stderr
|
|
|
|
|
|
def loki(service_name):
|
|
url = ("http://localhost:3100/loki/api/v1/query_range?"
|
|
"query=%7Bservice_name%3D%22" + service_name + "%22%7D&limit=5")
|
|
r = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "exec", "-T", "loki",
|
|
"wget", "-qO-", url], capture_output=True, text=True, cwd=ROOT, timeout=15)
|
|
return r.stdout
|
|
|
|
|
|
def tempo():
|
|
r = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "exec", "-T", "tempo",
|
|
"wget", "-qO-", "http://localhost:3200/api/search?limit=10"],
|
|
capture_output=True, text=True, cwd=ROOT, timeout=15)
|
|
return r.stdout
|
|
|
|
|
|
def main():
|
|
args = arguments("Lab6 controlled incidents")
|
|
c = Client(args.base_url, args.ca_file)
|
|
|
|
# 1. Smoke: данные текут.
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
print("1. bootstrap OK, chat:", chat["id"])
|
|
|
|
att, _ = c.attachment(alice, chat)
|
|
ready = c.wait_ready(bob, att, wait_seconds=60)
|
|
check(ready["status"] == "ready", "worker pipeline not ready")
|
|
|
|
# 2. Метрики существуют.
|
|
jobs = prom('messenger_jobs_processed_total{outcome="ready"}')
|
|
check(jobs and len(jobs) > 0, "messenger_jobs_processed_total missing")
|
|
print("2. jobs metric OK:", jobs[0]["value"][1])
|
|
|
|
http_count = prom('sum(http_server_duration_milliseconds_count{http_target!~"/health/.*"})')
|
|
check(http_count and len(http_count) > 0, "http_server_duration_milliseconds_count missing")
|
|
print(" http metric OK:", http_count[0]["value"][1])
|
|
|
|
# 3. Корреляция trace → log.
|
|
l = loki("messenger-api")
|
|
check("trace_id" in l, "no trace_id in Loki logs")
|
|
print("3. Loki correlation OK (trace_id present in logs)")
|
|
|
|
t = tempo()
|
|
check("traces" in t and "messenger-api" in t, "no traces in Tempo")
|
|
print(" Tempo traces OK")
|
|
|
|
# 4. Инцидент 1: остановить worker → очередь растёт.
|
|
print("4. stopping worker...")
|
|
compose("stop", "worker", timeout=60)
|
|
time.sleep(2)
|
|
|
|
for _ in range(5):
|
|
c.attachment(alice, chat)
|
|
print(" 5 uploads queued")
|
|
|
|
# Ждём, пока alert станет firing (for: 2m).
|
|
firing = False
|
|
deadline = time.monotonic() + 200
|
|
while time.monotonic() < deadline:
|
|
time.sleep(15)
|
|
rules = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
|
|
"wget", "-qO-", "http://localhost:9090/api/v1/rules"],
|
|
capture_output=True, text=True, cwd=ROOT, timeout=15).stdout
|
|
if '"name":"QueueLagHigh"' in rules and '"state":"firing"' in rules:
|
|
firing = True
|
|
break
|
|
check(firing, "QueueLagHigh did not reach firing")
|
|
print(" QueueLagHigh firing ✓")
|
|
|
|
# 5. Инцидент 2: восстановить worker → alert resolved.
|
|
print("5. starting worker...")
|
|
compose("start", "worker", timeout=60)
|
|
|
|
resolved = False
|
|
deadline = time.monotonic() + 180
|
|
while time.monotonic() < deadline:
|
|
time.sleep(15)
|
|
rules = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus",
|
|
"wget", "-qO-", "http://localhost:9090/api/v1/rules"],
|
|
capture_output=True, text=True, cwd=ROOT, timeout=15).stdout
|
|
if ('"name":"QueueLagHigh"' in rules
|
|
and '"state":"inactive"' in rules.split('"name":"QueueLagHigh"')[1][:2000]):
|
|
resolved = True
|
|
break
|
|
check(resolved, "QueueLagHigh did not resolve after worker restart")
|
|
print(" QueueLagHigh resolved ✓")
|
|
|
|
# 6. Backlog рассосался.
|
|
time.sleep(5)
|
|
depth = prom("sum(messenger_queue_depth)")
|
|
if depth and depth[0]["value"][1]:
|
|
check(float(depth[0]["value"][1]) == 0.0, "queue not drained: " + depth[0]["value"][1])
|
|
print("6. queue drained")
|
|
|
|
# 7. Телеметрия-бэкенд недоступен → API продолжает работать.
|
|
print("7. stopping telemetry backends (loki, tempo)...")
|
|
compose("stop", "loki", "tempo", timeout=60)
|
|
time.sleep(3)
|
|
# smoke через API — должен пройти, несмотря на потерю traces/logs.
|
|
# В этом состоянии в api пойдёт WARN от exporter — но оно не падает.
|
|
a2, b2, e2, chat2 = c.bootstrap()
|
|
att2, _ = c.attachment(a2, chat2)
|
|
ready2 = c.wait_ready(b2, att2, wait_seconds=60)
|
|
check(ready2["status"] == "ready", "API broken when telemetry down")
|
|
print(" API still works with telemetry down ✓")
|
|
|
|
compose("start", "loki", "tempo", timeout=60)
|
|
time.sleep(10)
|
|
|
|
print("\nPASS: lab6 observability (metrics, traces, logs, alerts firing→resolved, telemetry-down resilience)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |