537 lines
19 KiB
Python
537 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
"""Level 4 для lab6: сквозной trace API→broker→worker, метрики, логи, dashboard,
|
||
alert rules, поведение при отказе telemetry-backend.
|
||
|
||
Запуск:
|
||
$env:PYTHONUTF8="1"
|
||
python student-tests/lab6_test_level4.py --base-url http://localhost:8080
|
||
"""
|
||
import base64
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
|
||
|
||
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost:3000")
|
||
GRAFANA_ADMIN_USER = os.environ.get("GRAFANA_ADMIN_USER", "admin")
|
||
GRAFANA_ADMIN_PASSWORD = os.environ.get("GRAFANA_ADMIN_PASSWORD", "admin")
|
||
|
||
READY_TIMEOUT = int(os.environ.get("READY_TIMEOUT", "180"))
|
||
|
||
|
||
def _compose_exec(service: str, *args: str, timeout: int = 30):
|
||
return subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", "exec", "-T", service, *args],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
|
||
)
|
||
|
||
|
||
def _compose(action: str, service: str, timeout: int = 90):
|
||
return subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", action, service],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
|
||
)
|
||
|
||
|
||
def _compose_logs(service: str, tail: int = 20) -> str:
|
||
r = subprocess.run(
|
||
["docker", "compose", "--env-file", ".env", "logs", "--no-color",
|
||
"--tail=%d" % tail, service],
|
||
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
|
||
)
|
||
return (r.stdout or "") + (r.stderr or "")
|
||
|
||
|
||
def _http_get(url: str, timeout: float = 10.0, basic_auth: tuple | None = None):
|
||
headers = {"Accept": "application/json"}
|
||
if basic_auth is not None:
|
||
token = base64.b64encode(("%s:%s" % basic_auth).encode()).decode()
|
||
headers["Authorization"] = "Basic " + token
|
||
req = urllib.request.Request(url, headers=headers, method="GET")
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
return resp.status, resp.read().decode("utf-8", "replace")
|
||
except urllib.error.HTTPError as e:
|
||
return e.code, e.read().decode("utf-8", "replace")
|
||
except (urllib.error.URLError, OSError) as e:
|
||
return 0, "%s: %s" % (type(e).__name__, e)
|
||
|
||
|
||
def _prom_query(query: str) -> dict:
|
||
qs = "query=" + urllib.parse.quote(query, safe="")
|
||
r = _compose_exec(
|
||
"prometheus", "sh", "-c",
|
||
"wget -qO- 'http://localhost:9090/api/v1/query?%s'" % qs,
|
||
)
|
||
if r.returncode != 0 or not r.stdout.strip():
|
||
raise Failure("prometheus query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||
try:
|
||
return json.loads(r.stdout)
|
||
except Exception:
|
||
raise Failure("prometheus non-JSON: %s" % r.stdout[:300])
|
||
|
||
|
||
def _tempo_search(query: str, since_seconds: int = 300, limit: int = 20) -> list[dict]:
|
||
now = int(time.time())
|
||
start = now - since_seconds
|
||
qs = "q=%s&start=%d&end=%d&limit=%d" % (
|
||
urllib.parse.quote(query, safe=""), start, now, limit,
|
||
)
|
||
r = _compose_exec(
|
||
"tempo", "sh", "-c",
|
||
"wget -qO- 'http://localhost:3200/api/search?%s'" % qs,
|
||
)
|
||
if r.returncode != 0 or not r.stdout.strip():
|
||
return []
|
||
try:
|
||
data = json.loads(r.stdout)
|
||
except Exception:
|
||
return []
|
||
return data.get("traces", []) or []
|
||
|
||
|
||
def _tempo_get_trace(trace_id: str) -> dict:
|
||
r = _compose_exec(
|
||
"tempo", "sh", "-c",
|
||
"wget -qO- 'http://localhost:3200/api/traces/%s'" % trace_id,
|
||
)
|
||
if r.returncode != 0 or not r.stdout.strip():
|
||
raise Failure("tempo trace fetch failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||
return json.loads(r.stdout)
|
||
|
||
|
||
def _tempo_span_names(trace: dict) -> set[str]:
|
||
names: set[str] = set()
|
||
for batch in trace.get("batches", []):
|
||
scopes = batch.get("scopeSpans") or batch.get("instrumentationLibrarySpans") or []
|
||
for scope in scopes:
|
||
for span in scope.get("spans", []) or []:
|
||
names.add(span.get("name", ""))
|
||
return names
|
||
|
||
|
||
def _loki_query(query: str, since_seconds: int = 300, limit: int = 200) -> dict:
|
||
end = int(time.time() * 1e9)
|
||
start = end - since_seconds * 1_000_000_000
|
||
qs = "query=%s&start=%d&end=%d&limit=%d" % (
|
||
urllib.parse.quote(query, safe=""), start, end, limit,
|
||
)
|
||
r = _compose_exec(
|
||
"loki", "sh", "-c",
|
||
"wget -qO- 'http://localhost:3100/loki/api/v1/query_range?%s'" % qs,
|
||
)
|
||
if r.returncode != 0 or not r.stdout.strip():
|
||
raise Failure("loki query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
|
||
try:
|
||
return json.loads(r.stdout)
|
||
except Exception:
|
||
raise Failure("loki returned non-JSON: %s" % r.stdout[:300])
|
||
|
||
|
||
def _raw_upload(c: Client, actor, chat_id: str, filename: str,
|
||
data: bytes, ctype: str):
|
||
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
|
||
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
|
||
req = urllib.request.Request(
|
||
url, data=data, method="POST",
|
||
headers={
|
||
"Authorization": "Bearer " + actor["token"],
|
||
"Content-Type": ctype,
|
||
},
|
||
)
|
||
try:
|
||
with c.opener.open(req, timeout=30) as resp:
|
||
return resp.code, json.loads(resp.read()), resp.headers
|
||
except urllib.error.HTTPError as e:
|
||
body = e.read()
|
||
try:
|
||
return e.code, json.loads(body), e.headers
|
||
except Exception:
|
||
return e.code, None, e.headers
|
||
|
||
|
||
def _wait_ready(c, actor, att, timeout: int) -> dict:
|
||
"""Обёртка с диагностикой: если долго не ready — показываем статус и логи."""
|
||
deadline = time.monotonic() + timeout
|
||
last_status = None
|
||
while time.monotonic() < deadline:
|
||
a = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor,
|
||
schema="Attachment")["json"]
|
||
last_status = a.get("status")
|
||
if last_status == "ready":
|
||
return a
|
||
if last_status == "failed":
|
||
raise Failure("attachment failed: error_code=%r" % a.get("error_code"))
|
||
time.sleep(0.5)
|
||
diag = (
|
||
"\nlast_status=%r (waited %ss)\n--- outbox logs ---\n%s\n--- worker logs ---\n%s"
|
||
% (last_status, timeout, _compose_logs("outbox", 20),
|
||
_compose_logs("worker", 20))
|
||
)
|
||
raise Failure("attachment did not become ready" + diag)
|
||
|
||
|
||
def _wait_trace_with_spans(since_seconds: int, need_substrings: list[str],
|
||
timeout: float = 180.0) -> dict:
|
||
deadline = time.monotonic() + timeout
|
||
last_seen: list[dict] = []
|
||
while time.monotonic() < deadline:
|
||
for svc in ("messenger-api", "messenger-worker", "messenger-outbox"):
|
||
candidates = _tempo_search(
|
||
'{resource.service.name = "%s"}' % svc,
|
||
since_seconds=since_seconds,
|
||
)
|
||
for t in candidates:
|
||
trace_id = t.get("traceID")
|
||
if not trace_id:
|
||
continue
|
||
try:
|
||
full = _tempo_get_trace(trace_id)
|
||
except Failure:
|
||
continue
|
||
names = _tempo_span_names(full)
|
||
last_seen.append({"trace_id": trace_id, "names": sorted(names)})
|
||
if all(any(s in n for n in names) for s in need_substrings):
|
||
return {"trace_id": trace_id, "names": sorted(names)}
|
||
time.sleep(3)
|
||
tail = "; ".join(
|
||
"%s -> %s" % (x["trace_id"][:8], x["names"][:6]) for x in last_seen[:5]
|
||
)
|
||
raise Failure(
|
||
"не дождались trace со span-names, содержащими %s за %ss; последние trace: %s"
|
||
% (need_substrings, timeout, tail or "нет")
|
||
)
|
||
|
||
|
||
def test_trace_api_to_worker(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = SAMPLE.read_bytes()
|
||
started = int(time.time())
|
||
|
||
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
|
||
check(status == 202, "upload must be 202, got %s" % status)
|
||
check(att["status"] == "queued", "upload status must be queued")
|
||
|
||
ready = _wait_ready(c, alice, att, timeout=READY_TIMEOUT)
|
||
check(ready["status"] == "ready", "attachment must become ready")
|
||
|
||
since = max(60, int(time.time()) - started + 180)
|
||
trace = _wait_trace_with_spans(
|
||
since_seconds=since,
|
||
need_substrings=[
|
||
"/api/v1/chats/{chat_id}/attachments",
|
||
"job.publish",
|
||
"job.consume",
|
||
],
|
||
timeout=180,
|
||
)
|
||
print(" trace_id:", trace["trace_id"])
|
||
print(" spans:", ", ".join(trace["names"]))
|
||
|
||
|
||
def test_metrics_in_prometheus(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
|
||
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
|
||
data = SAMPLE.read_bytes()
|
||
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
|
||
check(status == 202, "upload must be 202")
|
||
_wait_ready(c, alice, att, timeout=READY_TIMEOUT)
|
||
|
||
deadline = time.monotonic() + 120
|
||
required = {
|
||
"http_requests_total",
|
||
"http_request_duration_seconds_bucket",
|
||
"jobs_processed_total",
|
||
"job_processing_duration_seconds_bucket",
|
||
}
|
||
got: set[str] = set()
|
||
while time.monotonic() < deadline:
|
||
got = set()
|
||
for name in required:
|
||
r = _prom_query("count(%s)" % name)
|
||
series = r.get("data", {}).get("result", [])
|
||
if series:
|
||
got.add(name)
|
||
if got == required:
|
||
break
|
||
time.sleep(3)
|
||
missing = required - got
|
||
check(not missing, "Prometheus missing metrics: %s" % sorted(missing))
|
||
print(" metrics in Prometheus:", sorted(got))
|
||
|
||
r = _prom_query("count by (http_route, http_status_class) (http_requests_total)")
|
||
series = r.get("data", {}).get("result", [])
|
||
check(series, "http_requests_total has no series")
|
||
import re as _re
|
||
for s in series:
|
||
metric = s.get("metric", {})
|
||
route = metric.get("http_route", "")
|
||
check(route.startswith("/api/") or route.startswith("/health"),
|
||
"unexpected http_route: %r" % route)
|
||
check(_re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
|
||
route) is None,
|
||
"http_route contains raw UUID: %r" % route)
|
||
|
||
r = _prom_query("sum(jobs_processed_total{outcome=\"success\"})")
|
||
series = r.get("data", {}).get("result", [])
|
||
check(series, "jobs_processed_total{outcome=success} absent")
|
||
value = float(series[0]["value"][1])
|
||
check(value >= 1, "jobs success metric is %r" % value)
|
||
print(" jobs_processed_total{outcome=success} =", value)
|
||
|
||
|
||
def test_logs_in_loki(c):
|
||
time.sleep(15)
|
||
|
||
deadline = time.monotonic() + 90
|
||
streams = []
|
||
while time.monotonic() < deadline:
|
||
r = _loki_query('{service_name="messenger-api"} |= "event=http"',
|
||
since_seconds=300)
|
||
streams = r.get("data", {}).get("result", [])
|
||
if streams:
|
||
break
|
||
time.sleep(3)
|
||
check(streams, "Loki: no messenger-api logs with event=http")
|
||
lines = [v[1] for s in streams for v in s.get("values", [])]
|
||
check(any("request_id=" in line for line in lines),
|
||
"API logs missing request_id")
|
||
|
||
r = _loki_query('{service_name="messenger-worker"} |= "event="', since_seconds=300)
|
||
check(r.get("data", {}).get("result", []),
|
||
"Loki: no messenger-worker logs")
|
||
|
||
# Ищем запись event=http с ненулевым trace_id (корреляция лог↔trace).
|
||
deadline = time.monotonic() + 90
|
||
ok = False
|
||
while time.monotonic() < deadline:
|
||
r = _loki_query(
|
||
'{service_name=~"messenger-api|messenger-worker"} '
|
||
'|~ "event=(http|job_ready)" |~ "trace_id=" ',
|
||
since_seconds=300,
|
||
)
|
||
streams = r.get("data", {}).get("result", [])
|
||
for s in streams:
|
||
for v in s.get("values", []):
|
||
line = v[1]
|
||
idx = line.find("trace_id=")
|
||
if idx < 0:
|
||
continue
|
||
tail = line[idx + len("trace_id="):]
|
||
tid = tail.split()[0].strip()
|
||
if len(tid) == 32 and set(tid) != {"0"}:
|
||
ok = True
|
||
break
|
||
if ok:
|
||
break
|
||
if ok:
|
||
break
|
||
time.sleep(3)
|
||
check(ok, "Loki: no logs with non-zero trace_id (log↔trace correlation)")
|
||
print(" Loki OK: api+worker logs with non-zero trace_id found")
|
||
|
||
|
||
def test_grafana_provisioning(c):
|
||
deadline = time.monotonic() + 60
|
||
auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD)
|
||
while time.monotonic() < deadline:
|
||
code, _ = _http_get(GRAFANA_URL + "/api/health")
|
||
if code == 200:
|
||
break
|
||
time.sleep(2)
|
||
else:
|
||
raise Failure("Grafana API не поднялась за 60 с")
|
||
|
||
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
|
||
check(code == 200, "grafana datasources API %s" % code)
|
||
datasources = {d["type"]: d for d in json.loads(body)}
|
||
for t in ("prometheus", "loki", "tempo"):
|
||
check(t in datasources, "Grafana missing datasource: %s (have %s)"
|
||
% (t, list(datasources)))
|
||
print(" datasources:", sorted(datasources))
|
||
|
||
code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6",
|
||
basic_auth=auth)
|
||
check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code)
|
||
dash = json.loads(body)
|
||
panels = dash.get("dashboard", {}).get("panels", [])
|
||
titles = {p.get("title") for p in panels}
|
||
check(any("p95" in (t or "") for t in titles),
|
||
"dashboard missing p95 panel")
|
||
check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles),
|
||
"dashboard missing jobs panel")
|
||
print(" dashboard panels:", sorted(t for t in titles if t))
|
||
|
||
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
|
||
basic_auth=auth)
|
||
check(code == 200, "grafana alert-rules API %s" % code)
|
||
rules = json.loads(body)
|
||
check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules))
|
||
print(" alert rules:", [r.get("title") for r in rules])
|
||
|
||
|
||
def _prom_alerts() -> list[dict]:
|
||
r = _compose_exec("prometheus", "sh", "-c",
|
||
"wget -qO- 'http://localhost:9090/api/v1/alerts'")
|
||
if r.returncode != 0 or not r.stdout.strip():
|
||
return []
|
||
try:
|
||
data = json.loads(r.stdout)
|
||
except Exception:
|
||
return []
|
||
return data.get("data", {}).get("alerts", []) or []
|
||
|
||
|
||
def _wait_alert(name: str, state: str, timeout: float = 240.0):
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
alerts = _prom_alerts()
|
||
for a in alerts:
|
||
labels = a.get("labels", {})
|
||
if labels.get("alertname") == name and a.get("state") == state:
|
||
return
|
||
time.sleep(3)
|
||
raise Failure("alert %s never reached %s" % (name, state))
|
||
|
||
|
||
def test_alert_firing_and_resolved(c):
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
|
||
stopped = _compose("stop", "postgres")
|
||
check(stopped.returncode == 0, "stop postgres failed")
|
||
|
||
try:
|
||
deadline = time.monotonic() + 60
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
c.call("GET", "/api/v1/users/me", (401, 503), actor=alice)
|
||
except Failure:
|
||
pass
|
||
time.sleep(0.3)
|
||
_wait_alert("MessengerHighErrorRate", "firing", timeout=240)
|
||
print(" MessengerHighErrorRate firing")
|
||
finally:
|
||
started = _compose("start", "postgres")
|
||
check(started.returncode == 0, "start postgres failed")
|
||
|
||
deadline = time.monotonic() + 90
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
c.call("GET", "/health/ready", 200)
|
||
break
|
||
except Failure:
|
||
time.sleep(1)
|
||
else:
|
||
raise Failure("API didn't become ready after postgres restart")
|
||
|
||
deadline = time.monotonic() + 60
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||
except Failure:
|
||
pass
|
||
time.sleep(0.3)
|
||
_wait_alert("MessengerHighErrorRate", "inactive", timeout=300)
|
||
print(" MessengerHighErrorRate resolved")
|
||
|
||
|
||
def test_telemetry_backend_down(c):
|
||
for svc in ("tempo", "loki", "prometheus"):
|
||
_compose("start", svc)
|
||
time.sleep(3)
|
||
|
||
stopped = {}
|
||
for svc in ("prometheus", "loki", "tempo"):
|
||
r = _compose("stop", svc)
|
||
check(r.returncode == 0, "stop %s failed: %s" % (svc, r.stderr[:300]))
|
||
stopped[svc] = True
|
||
|
||
try:
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
deadline = time.monotonic() + 30
|
||
ok = 0
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
|
||
c.message(alice, chat, "telem-down-" + uuid.uuid4().hex[:8])
|
||
ok += 1
|
||
except Failure:
|
||
pass
|
||
time.sleep(0.2)
|
||
check(ok >= 5, "API unavailable while telemetry backend is down (ok=%d)" % ok)
|
||
|
||
ps = _compose_exec("otel-collector", "true")
|
||
check(ps.returncode == 0,
|
||
"otel-collector died while backends were unavailable")
|
||
|
||
stats = subprocess.run(
|
||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}",
|
||
"mtuci-otel-collector"],
|
||
capture_output=True, text=True, timeout=15,
|
||
)
|
||
if stats.returncode == 0:
|
||
used = stats.stdout.strip().split("/")[0].strip()
|
||
mb = None
|
||
if used.endswith("MiB"):
|
||
mb = float(used[:-3])
|
||
elif used.endswith("GiB"):
|
||
mb = float(used[:-3]) * 1024
|
||
if mb is not None:
|
||
check(mb < 260, "otel-collector RSS=%s MiB exceeds limit" % mb)
|
||
print(" otel-collector RSS while backends down:", mb, "MiB")
|
||
finally:
|
||
for svc in stopped:
|
||
_compose("start", svc)
|
||
time.sleep(3)
|
||
|
||
print(" API and Collector remained healthy while backends were down")
|
||
|
||
|
||
def main():
|
||
args = arguments("Lab6 level 4 checks")
|
||
c = Client(args.base_url, args.ca_file)
|
||
c.call("GET", "/health/ready", schema="Health")
|
||
|
||
print("=== trace API -> broker -> worker (Tempo) ===")
|
||
test_trace_api_to_worker(c)
|
||
print("PASS: single trace with upload + job.publish + job.consume")
|
||
|
||
print("=== metrics in Prometheus ===")
|
||
test_metrics_in_prometheus(c)
|
||
print("PASS: http_requests_total + duration + jobs_processed_total present, "
|
||
"low-cardinality route labels")
|
||
|
||
print("=== logs in Loki ===")
|
||
test_logs_in_loki(c)
|
||
print("PASS: structured logs of api+worker with trace_id")
|
||
|
||
print("=== Grafana provisioning ===")
|
||
test_grafana_provisioning(c)
|
||
print("PASS: dashboard messenger-lab6 + >=2 alert rules provisioned")
|
||
|
||
print("=== alert firing and resolved ===")
|
||
test_alert_firing_and_resolved(c)
|
||
print("PASS: alert fires on artificial outage and resolves on recovery")
|
||
|
||
print("=== telemetry backend down ===")
|
||
test_telemetry_backend_down(c)
|
||
print("PASS: API works, Collector stays within memory limit")
|
||
|
||
print("PASS: all lab6 level-4 checks passed")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main) |