Files
mtusi2026lab6/student-tests/test_observability.py
T

143 lines
8.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Проверка наблюдаемости lab6: метрики, логи, трассы, связь API -> брокер -> worker, отсутствие секретов в телеметрии.
python -m unittest discover -s student-tests -p test_observability.py -v (стенд запущен; нужен ~1 минута)
Запросы к Prometheus/Loki/Tempo выполняются изнутри сети стенда (их порты наружу не опубликованы).
"""
import json
import os
import subprocess
import sys
import time
import unittest
import urllib.parse
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import BASE, ROOT, call, register # noqa: E402
from test_attachments import PNG, upload # noqa: E402
from test_pipeline import wait_status # noqa: E402
def net_get(url, accept=None):
cmd = ["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus", "wget", "-q", "-O", "-"]
if accept:
cmd += ["--header", "Accept: " + accept]
out = subprocess.run(cmd + [url], cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
return out.stdout
def prom(query):
data = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query)))
return data["data"]["result"]
def loki(query, limit=200):
now = int(time.time() * 1e9)
url = "http://loki:3100/loki/api/v1/query_range?limit=%d&start=%d&end=%d&query=%s" % (limit, now - 600 * 10 ** 9, now, urllib.parse.quote(query))
data = json.loads(net_get(url))
return [(s["stream"], v) for s in data["data"]["result"] for v in s["values"]]
def tempo_trace(trace_id):
raw = net_get("http://tempo:3200/api/traces/" + trace_id, "application/json")
return json.loads(raw) if raw.strip() else None
def spans_of(trace):
out = []
for batch in trace.get("batches", trace.get("resourceSpans", [])):
service = next((a["value"].get("stringValue") for a in batch["resource"]["attributes"] if a["key"] == "service.name"), None)
for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])):
for sp in scope["spans"]:
attrs = {a["key"]: next(iter(a["value"].values())) for a in sp.get("attributes", [])}
out.append({"service": service, "name": sp["name"], "id": sp["spanId"], "parent": sp.get("parentSpanId", ""), "kind": sp.get("kind"), "attrs": attrs})
return out
class ObservabilityTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.a = register("obs")
cls.chat = call("POST", "/api/v1/chats", {"title": "наблюдаемость", "member_ids": [cls.a["user"]["id"]]}, cls.a["token"])[2]
cls.trace_id = uuid.uuid4().hex
cls.secret_text = "СЕКРЕТНЫЙ-ТЕКСТ-" + uuid.uuid4().hex
# загрузка с входящим traceparent: так мы знаем trace_id заранее
import urllib.request
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=obs.png" % (BASE, cls.chat["id"]), data=PNG, method="POST",
headers={"Authorization": "Bearer " + cls.a["token"], "Content-Type": "image/png",
"traceparent": "00-%s-%s-01" % (cls.trace_id, uuid.uuid4().hex[:16])})
with urllib.request.urlopen(req, timeout=30) as r:
cls.att = json.loads(r.read())
wait_status(cls.a["token"], cls.att["id"], ("ready",), 60)
call("POST", "/api/v1/chats/%s/messages" % cls.chat["id"], {"text": cls.secret_text, "attachment_ids": [cls.att["id"]]}, cls.a["token"])
deadline = time.time() + 40 # экспорт SDK (2 с) + пакетная обработка Collector + индексация
cls.trace = None
while time.time() < deadline:
cls.trace = tempo_trace(cls.trace_id)
if cls.trace and {s["service"] for s in spans_of(cls.trace)} >= {"messenger-api", "messenger-worker"}:
break
time.sleep(2)
def test_one_trace_spans_api_outbox_broker_and_worker(self):
self.assertIsNotNone(self.trace, "трасса не найдена в Tempo")
spans = spans_of(self.trace)
by_name = {s["name"]: s for s in spans}
self.assertEqual({s["service"] for s in spans} & {"messenger-api", "messenger-worker"}, {"messenger-api", "messenger-worker"})
server = by_name["HTTP POST /api/v1/chats/{chat_id}/attachments"]
producer, consumer = by_name["attachment.process publish"], by_name["attachment.process process"]
self.assertEqual((server["service"], producer["service"], consumer["service"]), ("messenger-api", "messenger-api", "messenger-worker"))
# цепочка parent-child: server (API) -> producer (publish из outbox) -> consumer (worker)
self.assertEqual(producer["parent"], server["id"])
self.assertEqual(consumer["parent"], producer["id"])
names = {s["name"] for s in spans}
self.assertIn("s3.put_object", names) # запись staging (API) и результатов (worker)
self.assertIn("s3.get_object", names) # чтение staging worker-ом
self.assertIn("db.connection", names)
self.assertEqual(server["attrs"].get("http.route"), "/api/v1/chats/{chat_id}/attachments")
self.assertEqual(int(server["attrs"].get("http.response.status_code")), 202)
def test_logs_are_structured_and_linked_by_trace_id(self):
rows = loki('{service_name=~"messenger-.*"} | trace_id="%s"' % self.trace_id)
services = {stream["service_name"] for stream, _ in rows}
messages = {v[1] for _, v in rows}
self.assertIn("messenger-api", services)
self.assertIn("messenger-worker", services)
self.assertIn("job_ready", messages)
self.assertTrue(all(stream.get("span_id") for stream, _ in rows))
self.assertTrue(all(stream.get("service_version") is not None and stream.get("service_instance_id") for stream, _ in rows))
def test_metrics_for_http_jobs_queue_pool_worker(self):
for query in ('http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}',
'messenger_jobs_processed_total{outcome="ready"}',
'messenger_job_duration_seconds_bucket', 'messenger_job_time_to_ready_seconds_bucket',
'messenger_queue_depth', 'messenger_queue_oldest_job_age_seconds', 'messenger_worker_capacity', 'messenger_worker_busy',
'db_client_connections_usage{state="used"}', 'db_client_connections_waiting', 'process_memory_rss_bytes',
'process_cpu_time_seconds_total'):
with self.subTest(query):
deadline = time.time() + 40 # экспорт SDK (5 с) + scrape Prometheus (5 с)
while not prom(query) and time.time() < deadline:
time.sleep(2)
self.assertTrue(prom(query), "нет данных: " + query)
def test_no_high_cardinality_or_secret_labels(self):
forbidden = ("user_id", "chat_id", "message_id", "attachment_id", "request_id", "filename")
series = json.loads(net_get("http://127.0.0.1:9090/api/v1/series?match[]=%s" % urllib.parse.quote('{__name__=~"http_server.*|messenger_.*|db_client.*|process_.*"}')))["data"]
labels = {k for s in series for k in s}
self.assertFalse(labels & set(forbidden), labels & set(forbidden))
text = json.dumps(series)
self.assertNotIn(self.a["token"], text)
route_values = {s.get("http_route") for s in series if "http_route" in s}
import re
self.assertFalse([r for r in route_values if re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-", r)], route_values) # шаблоны, не реальные UUID
def test_secrets_and_message_bodies_are_not_exported(self):
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.secret_text), [])
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["token"]), [])
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["password"]), [])
raw = json.dumps(self.trace)
for secret in (self.secret_text, self.a["token"], self.a["password"]):
self.assertNotIn(secret, raw)
if __name__ == "__main__":
unittest.main()