Files
mtusi2026lab6/student-tests/lab6_test_level4.py
T

537 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Level 4 для lab6: сквозной trace API→broker→worker, метрики, логи, dashboard,
alert rules, поведение при отказе telemetry-backend.
Запуск:
$env:PYTHONUTF8="1"
python student-tests/lab6_test_level4.py --base-url http://localhost:8080
"""
import base64
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run # noqa: E402
ROOT = Path(__file__).resolve().parents[1]
SAMPLE = ROOT / "tests" / "fixtures" / "sample.png"
GRAFANA_URL = os.environ.get("GRAFANA_URL", "http://localhost:3000")
GRAFANA_ADMIN_USER = os.environ.get("GRAFANA_ADMIN_USER", "admin")
GRAFANA_ADMIN_PASSWORD = os.environ.get("GRAFANA_ADMIN_PASSWORD", "admin")
READY_TIMEOUT = int(os.environ.get("READY_TIMEOUT", "180"))
def _compose_exec(service: str, *args: str, timeout: int = 30):
return subprocess.run(
["docker", "compose", "--env-file", ".env", "exec", "-T", service, *args],
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
)
def _compose(action: str, service: str, timeout: int = 90):
return subprocess.run(
["docker", "compose", "--env-file", ".env", action, service],
cwd=str(ROOT), capture_output=True, text=True, timeout=timeout,
)
def _compose_logs(service: str, tail: int = 20) -> str:
r = subprocess.run(
["docker", "compose", "--env-file", ".env", "logs", "--no-color",
"--tail=%d" % tail, service],
cwd=str(ROOT), capture_output=True, text=True, timeout=30,
)
return (r.stdout or "") + (r.stderr or "")
def _http_get(url: str, timeout: float = 10.0, basic_auth: tuple | None = None):
headers = {"Accept": "application/json"}
if basic_auth is not None:
token = base64.b64encode(("%s:%s" % basic_auth).encode()).decode()
headers["Authorization"] = "Basic " + token
req = urllib.request.Request(url, headers=headers, method="GET")
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.status, resp.read().decode("utf-8", "replace")
except urllib.error.HTTPError as e:
return e.code, e.read().decode("utf-8", "replace")
except (urllib.error.URLError, OSError) as e:
return 0, "%s: %s" % (type(e).__name__, e)
def _prom_query(query: str) -> dict:
qs = "query=" + urllib.parse.quote(query, safe="")
r = _compose_exec(
"prometheus", "sh", "-c",
"wget -qO- 'http://localhost:9090/api/v1/query?%s'" % qs,
)
if r.returncode != 0 or not r.stdout.strip():
raise Failure("prometheus query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
try:
return json.loads(r.stdout)
except Exception:
raise Failure("prometheus non-JSON: %s" % r.stdout[:300])
def _tempo_search(query: str, since_seconds: int = 300, limit: int = 20) -> list[dict]:
now = int(time.time())
start = now - since_seconds
qs = "q=%s&start=%d&end=%d&limit=%d" % (
urllib.parse.quote(query, safe=""), start, now, limit,
)
r = _compose_exec(
"tempo", "sh", "-c",
"wget -qO- 'http://localhost:3200/api/search?%s'" % qs,
)
if r.returncode != 0 or not r.stdout.strip():
return []
try:
data = json.loads(r.stdout)
except Exception:
return []
return data.get("traces", []) or []
def _tempo_get_trace(trace_id: str) -> dict:
r = _compose_exec(
"tempo", "sh", "-c",
"wget -qO- 'http://localhost:3200/api/traces/%s'" % trace_id,
)
if r.returncode != 0 or not r.stdout.strip():
raise Failure("tempo trace fetch failed: %s" % (r.stderr[:300] or r.stdout[:300]))
return json.loads(r.stdout)
def _tempo_span_names(trace: dict) -> set[str]:
names: set[str] = set()
for batch in trace.get("batches", []):
scopes = batch.get("scopeSpans") or batch.get("instrumentationLibrarySpans") or []
for scope in scopes:
for span in scope.get("spans", []) or []:
names.add(span.get("name", ""))
return names
def _loki_query(query: str, since_seconds: int = 300, limit: int = 200) -> dict:
end = int(time.time() * 1e9)
start = end - since_seconds * 1_000_000_000
qs = "query=%s&start=%d&end=%d&limit=%d" % (
urllib.parse.quote(query, safe=""), start, end, limit,
)
r = _compose_exec(
"loki", "sh", "-c",
"wget -qO- 'http://localhost:3100/loki/api/v1/query_range?%s'" % qs,
)
if r.returncode != 0 or not r.stdout.strip():
raise Failure("loki query failed: %s" % (r.stderr[:300] or r.stdout[:300]))
try:
return json.loads(r.stdout)
except Exception:
raise Failure("loki returned non-JSON: %s" % r.stdout[:300])
def _raw_upload(c: Client, actor, chat_id: str, filename: str,
data: bytes, ctype: str):
qs = urllib.parse.urlencode({"filename": filename}, quote_via=urllib.parse.quote)
url = c.base + "/api/v1/chats/" + chat_id + "/attachments?" + qs
req = urllib.request.Request(
url, data=data, method="POST",
headers={
"Authorization": "Bearer " + actor["token"],
"Content-Type": ctype,
},
)
try:
with c.opener.open(req, timeout=30) as resp:
return resp.code, json.loads(resp.read()), resp.headers
except urllib.error.HTTPError as e:
body = e.read()
try:
return e.code, json.loads(body), e.headers
except Exception:
return e.code, None, e.headers
def _wait_ready(c, actor, att, timeout: int) -> dict:
"""Обёртка с диагностикой: если долго не ready — показываем статус и логи."""
deadline = time.monotonic() + timeout
last_status = None
while time.monotonic() < deadline:
a = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor,
schema="Attachment")["json"]
last_status = a.get("status")
if last_status == "ready":
return a
if last_status == "failed":
raise Failure("attachment failed: error_code=%r" % a.get("error_code"))
time.sleep(0.5)
diag = (
"\nlast_status=%r (waited %ss)\n--- outbox logs ---\n%s\n--- worker logs ---\n%s"
% (last_status, timeout, _compose_logs("outbox", 20),
_compose_logs("worker", 20))
)
raise Failure("attachment did not become ready" + diag)
def _wait_trace_with_spans(since_seconds: int, need_substrings: list[str],
timeout: float = 180.0) -> dict:
deadline = time.monotonic() + timeout
last_seen: list[dict] = []
while time.monotonic() < deadline:
for svc in ("messenger-api", "messenger-worker", "messenger-outbox"):
candidates = _tempo_search(
'{resource.service.name = "%s"}' % svc,
since_seconds=since_seconds,
)
for t in candidates:
trace_id = t.get("traceID")
if not trace_id:
continue
try:
full = _tempo_get_trace(trace_id)
except Failure:
continue
names = _tempo_span_names(full)
last_seen.append({"trace_id": trace_id, "names": sorted(names)})
if all(any(s in n for n in names) for s in need_substrings):
return {"trace_id": trace_id, "names": sorted(names)}
time.sleep(3)
tail = "; ".join(
"%s -> %s" % (x["trace_id"][:8], x["names"][:6]) for x in last_seen[:5]
)
raise Failure(
"не дождались trace со span-names, содержащими %s за %ss; последние trace: %s"
% (need_substrings, timeout, tail or "нет")
)
def test_trace_api_to_worker(c):
alice, bob, eve, chat = c.bootstrap()
data = SAMPLE.read_bytes()
started = int(time.time())
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
check(status == 202, "upload must be 202, got %s" % status)
check(att["status"] == "queued", "upload status must be queued")
ready = _wait_ready(c, alice, att, timeout=READY_TIMEOUT)
check(ready["status"] == "ready", "attachment must become ready")
since = max(60, int(time.time()) - started + 180)
trace = _wait_trace_with_spans(
since_seconds=since,
need_substrings=[
"/api/v1/chats/{chat_id}/attachments",
"job.publish",
"job.consume",
],
timeout=180,
)
print(" trace_id:", trace["trace_id"])
print(" spans:", ", ".join(trace["names"]))
def test_metrics_in_prometheus(c):
alice, bob, eve, chat = c.bootstrap()
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
c.message(alice, chat, "metrics-" + uuid.uuid4().hex)
data = SAMPLE.read_bytes()
status, att, _ = _raw_upload(c, alice, chat["id"], "sample.png", data, "image/png")
check(status == 202, "upload must be 202")
_wait_ready(c, alice, att, timeout=READY_TIMEOUT)
deadline = time.monotonic() + 120
required = {
"http_requests_total",
"http_request_duration_seconds_bucket",
"jobs_processed_total",
"job_processing_duration_seconds_bucket",
}
got: set[str] = set()
while time.monotonic() < deadline:
got = set()
for name in required:
r = _prom_query("count(%s)" % name)
series = r.get("data", {}).get("result", [])
if series:
got.add(name)
if got == required:
break
time.sleep(3)
missing = required - got
check(not missing, "Prometheus missing metrics: %s" % sorted(missing))
print(" metrics in Prometheus:", sorted(got))
r = _prom_query("count by (http_route, http_status_class) (http_requests_total)")
series = r.get("data", {}).get("result", [])
check(series, "http_requests_total has no series")
import re as _re
for s in series:
metric = s.get("metric", {})
route = metric.get("http_route", "")
check(route.startswith("/api/") or route.startswith("/health"),
"unexpected http_route: %r" % route)
check(_re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}",
route) is None,
"http_route contains raw UUID: %r" % route)
r = _prom_query("sum(jobs_processed_total{outcome=\"success\"})")
series = r.get("data", {}).get("result", [])
check(series, "jobs_processed_total{outcome=success} absent")
value = float(series[0]["value"][1])
check(value >= 1, "jobs success metric is %r" % value)
print(" jobs_processed_total{outcome=success} =", value)
def test_logs_in_loki(c):
time.sleep(15)
deadline = time.monotonic() + 90
streams = []
while time.monotonic() < deadline:
r = _loki_query('{service_name="messenger-api"} |= "event=http"',
since_seconds=300)
streams = r.get("data", {}).get("result", [])
if streams:
break
time.sleep(3)
check(streams, "Loki: no messenger-api logs with event=http")
lines = [v[1] for s in streams for v in s.get("values", [])]
check(any("request_id=" in line for line in lines),
"API logs missing request_id")
r = _loki_query('{service_name="messenger-worker"} |= "event="', since_seconds=300)
check(r.get("data", {}).get("result", []),
"Loki: no messenger-worker logs")
# Ищем запись event=http с ненулевым trace_id (корреляция лог↔trace).
deadline = time.monotonic() + 90
ok = False
while time.monotonic() < deadline:
r = _loki_query(
'{service_name=~"messenger-api|messenger-worker"} '
'|~ "event=(http|job_ready)" |~ "trace_id=" ',
since_seconds=300,
)
streams = r.get("data", {}).get("result", [])
for s in streams:
for v in s.get("values", []):
line = v[1]
idx = line.find("trace_id=")
if idx < 0:
continue
tail = line[idx + len("trace_id="):]
tid = tail.split()[0].strip()
if len(tid) == 32 and set(tid) != {"0"}:
ok = True
break
if ok:
break
if ok:
break
time.sleep(3)
check(ok, "Loki: no logs with non-zero trace_id (log↔trace correlation)")
print(" Loki OK: api+worker logs with non-zero trace_id found")
def test_grafana_provisioning(c):
deadline = time.monotonic() + 60
auth = (GRAFANA_ADMIN_USER, GRAFANA_ADMIN_PASSWORD)
while time.monotonic() < deadline:
code, _ = _http_get(GRAFANA_URL + "/api/health")
if code == 200:
break
time.sleep(2)
else:
raise Failure("Grafana API не поднялась за 60 с")
code, body = _http_get(GRAFANA_URL + "/api/datasources", basic_auth=auth)
check(code == 200, "grafana datasources API %s" % code)
datasources = {d["type"]: d for d in json.loads(body)}
for t in ("prometheus", "loki", "tempo"):
check(t in datasources, "Grafana missing datasource: %s (have %s)"
% (t, list(datasources)))
print(" datasources:", sorted(datasources))
code, body = _http_get(GRAFANA_URL + "/api/dashboards/uid/messenger-lab6",
basic_auth=auth)
check(code == 200, "Grafana dashboard messenger-lab6 not found (%s)" % code)
dash = json.loads(body)
panels = dash.get("dashboard", {}).get("panels", [])
titles = {p.get("title") for p in panels}
check(any("p95" in (t or "") for t in titles),
"dashboard missing p95 panel")
check(any("Jobs" in (t or "") or "job" in (t or "").lower() for t in titles),
"dashboard missing jobs panel")
print(" dashboard panels:", sorted(t for t in titles if t))
code, body = _http_get(GRAFANA_URL + "/api/v1/provisioning/alert-rules",
basic_auth=auth)
check(code == 200, "grafana alert-rules API %s" % code)
rules = json.loads(body)
check(len(rules) >= 2, "expected >=2 alert rules, got %d" % len(rules))
print(" alert rules:", [r.get("title") for r in rules])
def _prom_alerts() -> list[dict]:
r = _compose_exec("prometheus", "sh", "-c",
"wget -qO- 'http://localhost:9090/api/v1/alerts'")
if r.returncode != 0 or not r.stdout.strip():
return []
try:
data = json.loads(r.stdout)
except Exception:
return []
return data.get("data", {}).get("alerts", []) or []
def _wait_alert(name: str, state: str, timeout: float = 240.0):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
alerts = _prom_alerts()
for a in alerts:
labels = a.get("labels", {})
if labels.get("alertname") == name and a.get("state") == state:
return
time.sleep(3)
raise Failure("alert %s never reached %s" % (name, state))
def test_alert_firing_and_resolved(c):
alice, bob, eve, chat = c.bootstrap()
stopped = _compose("stop", "postgres")
check(stopped.returncode == 0, "stop postgres failed")
try:
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call("GET", "/api/v1/users/me", (401, 503), actor=alice)
except Failure:
pass
time.sleep(0.3)
_wait_alert("MessengerHighErrorRate", "firing", timeout=240)
print(" MessengerHighErrorRate firing")
finally:
started = _compose("start", "postgres")
check(started.returncode == 0, "start postgres failed")
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", 200)
break
except Failure:
time.sleep(1)
else:
raise Failure("API didn't become ready after postgres restart")
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
except Failure:
pass
time.sleep(0.3)
_wait_alert("MessengerHighErrorRate", "inactive", timeout=300)
print(" MessengerHighErrorRate resolved")
def test_telemetry_backend_down(c):
for svc in ("tempo", "loki", "prometheus"):
_compose("start", svc)
time.sleep(3)
stopped = {}
for svc in ("prometheus", "loki", "tempo"):
r = _compose("stop", svc)
check(r.returncode == 0, "stop %s failed: %s" % (svc, r.stderr[:300]))
stopped[svc] = True
try:
alice, bob, eve, chat = c.bootstrap()
deadline = time.monotonic() + 30
ok = 0
while time.monotonic() < deadline:
try:
c.call("GET", "/api/v1/users/me", 200, actor=alice, schema="User")
c.message(alice, chat, "telem-down-" + uuid.uuid4().hex[:8])
ok += 1
except Failure:
pass
time.sleep(0.2)
check(ok >= 5, "API unavailable while telemetry backend is down (ok=%d)" % ok)
ps = _compose_exec("otel-collector", "true")
check(ps.returncode == 0,
"otel-collector died while backends were unavailable")
stats = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}",
"mtuci-otel-collector"],
capture_output=True, text=True, timeout=15,
)
if stats.returncode == 0:
used = stats.stdout.strip().split("/")[0].strip()
mb = None
if used.endswith("MiB"):
mb = float(used[:-3])
elif used.endswith("GiB"):
mb = float(used[:-3]) * 1024
if mb is not None:
check(mb < 260, "otel-collector RSS=%s MiB exceeds limit" % mb)
print(" otel-collector RSS while backends down:", mb, "MiB")
finally:
for svc in stopped:
_compose("start", svc)
time.sleep(3)
print(" API and Collector remained healthy while backends were down")
def main():
args = arguments("Lab6 level 4 checks")
c = Client(args.base_url, args.ca_file)
c.call("GET", "/health/ready", schema="Health")
print("=== trace API -> broker -> worker (Tempo) ===")
test_trace_api_to_worker(c)
print("PASS: single trace with upload + job.publish + job.consume")
print("=== metrics in Prometheus ===")
test_metrics_in_prometheus(c)
print("PASS: http_requests_total + duration + jobs_processed_total present, "
"low-cardinality route labels")
print("=== logs in Loki ===")
test_logs_in_loki(c)
print("PASS: structured logs of api+worker with trace_id")
print("=== Grafana provisioning ===")
test_grafana_provisioning(c)
print("PASS: dashboard messenger-lab6 + >=2 alert rules provisioned")
print("=== alert firing and resolved ===")
test_alert_firing_and_resolved(c)
print("PASS: alert fires on artificial outage and resolves on recovery")
print("=== telemetry backend down ===")
test_telemetry_backend_down(c)
print("PASS: API works, Collector stays within memory limit")
print("PASS: all lab6 level-4 checks passed")
if __name__ == "__main__":
run(main)