forked from mtusi2026/mtusi2026lab6
143 lines
8.3 KiB
Python
143 lines
8.3 KiB
Python
"""Проверка наблюдаемости lab6: метрики, логи, трассы, связь API -> брокер -> worker, отсутствие секретов в телеметрии.
|
||
python -m unittest discover -s student-tests -p test_observability.py -v (стенд запущен; нужен ~1 минута)
|
||
Запросы к Prometheus/Loki/Tempo выполняются изнутри сети стенда (их порты наружу не опубликованы).
|
||
"""
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import unittest
|
||
import urllib.parse
|
||
import uuid
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
from test_api import BASE, ROOT, call, register # noqa: E402
|
||
from test_attachments import PNG, upload # noqa: E402
|
||
from test_pipeline import wait_status # noqa: E402
|
||
|
||
|
||
def net_get(url, accept=None):
|
||
cmd = ["docker", "compose", "--env-file", ".env", "exec", "-T", "prometheus", "wget", "-q", "-O", "-"]
|
||
if accept:
|
||
cmd += ["--header", "Accept: " + accept]
|
||
out = subprocess.run(cmd + [url], cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
|
||
return out.stdout
|
||
|
||
|
||
def prom(query):
|
||
data = json.loads(net_get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(query)))
|
||
return data["data"]["result"]
|
||
|
||
|
||
def loki(query, limit=200):
|
||
now = int(time.time() * 1e9)
|
||
url = "http://loki:3100/loki/api/v1/query_range?limit=%d&start=%d&end=%d&query=%s" % (limit, now - 600 * 10 ** 9, now, urllib.parse.quote(query))
|
||
data = json.loads(net_get(url))
|
||
return [(s["stream"], v) for s in data["data"]["result"] for v in s["values"]]
|
||
|
||
|
||
def tempo_trace(trace_id):
|
||
raw = net_get("http://tempo:3200/api/traces/" + trace_id, "application/json")
|
||
return json.loads(raw) if raw.strip() else None
|
||
|
||
|
||
def spans_of(trace):
|
||
out = []
|
||
for batch in trace.get("batches", trace.get("resourceSpans", [])):
|
||
service = next((a["value"].get("stringValue") for a in batch["resource"]["attributes"] if a["key"] == "service.name"), None)
|
||
for scope in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", [])):
|
||
for sp in scope["spans"]:
|
||
attrs = {a["key"]: next(iter(a["value"].values())) for a in sp.get("attributes", [])}
|
||
out.append({"service": service, "name": sp["name"], "id": sp["spanId"], "parent": sp.get("parentSpanId", ""), "kind": sp.get("kind"), "attrs": attrs})
|
||
return out
|
||
|
||
|
||
class ObservabilityTest(unittest.TestCase):
|
||
@classmethod
|
||
def setUpClass(cls):
|
||
cls.a = register("obs")
|
||
cls.chat = call("POST", "/api/v1/chats", {"title": "наблюдаемость", "member_ids": [cls.a["user"]["id"]]}, cls.a["token"])[2]
|
||
cls.trace_id = uuid.uuid4().hex
|
||
cls.secret_text = "СЕКРЕТНЫЙ-ТЕКСТ-" + uuid.uuid4().hex
|
||
# загрузка с входящим traceparent: так мы знаем trace_id заранее
|
||
import urllib.request
|
||
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=obs.png" % (BASE, cls.chat["id"]), data=PNG, method="POST",
|
||
headers={"Authorization": "Bearer " + cls.a["token"], "Content-Type": "image/png",
|
||
"traceparent": "00-%s-%s-01" % (cls.trace_id, uuid.uuid4().hex[:16])})
|
||
with urllib.request.urlopen(req, timeout=30) as r:
|
||
cls.att = json.loads(r.read())
|
||
wait_status(cls.a["token"], cls.att["id"], ("ready",), 60)
|
||
call("POST", "/api/v1/chats/%s/messages" % cls.chat["id"], {"text": cls.secret_text, "attachment_ids": [cls.att["id"]]}, cls.a["token"])
|
||
deadline = time.time() + 40 # экспорт SDK (2 с) + пакетная обработка Collector + индексация
|
||
cls.trace = None
|
||
while time.time() < deadline:
|
||
cls.trace = tempo_trace(cls.trace_id)
|
||
if cls.trace and {s["service"] for s in spans_of(cls.trace)} >= {"messenger-api", "messenger-worker"}:
|
||
break
|
||
time.sleep(2)
|
||
|
||
def test_one_trace_spans_api_outbox_broker_and_worker(self):
|
||
self.assertIsNotNone(self.trace, "трасса не найдена в Tempo")
|
||
spans = spans_of(self.trace)
|
||
by_name = {s["name"]: s for s in spans}
|
||
self.assertEqual({s["service"] for s in spans} & {"messenger-api", "messenger-worker"}, {"messenger-api", "messenger-worker"})
|
||
server = by_name["HTTP POST /api/v1/chats/{chat_id}/attachments"]
|
||
producer, consumer = by_name["attachment.process publish"], by_name["attachment.process process"]
|
||
self.assertEqual((server["service"], producer["service"], consumer["service"]), ("messenger-api", "messenger-api", "messenger-worker"))
|
||
# цепочка parent-child: server (API) -> producer (publish из outbox) -> consumer (worker)
|
||
self.assertEqual(producer["parent"], server["id"])
|
||
self.assertEqual(consumer["parent"], producer["id"])
|
||
names = {s["name"] for s in spans}
|
||
self.assertIn("s3.put_object", names) # запись staging (API) и результатов (worker)
|
||
self.assertIn("s3.get_object", names) # чтение staging worker-ом
|
||
self.assertIn("db.connection", names)
|
||
self.assertEqual(server["attrs"].get("http.route"), "/api/v1/chats/{chat_id}/attachments")
|
||
self.assertEqual(int(server["attrs"].get("http.response.status_code")), 202)
|
||
|
||
def test_logs_are_structured_and_linked_by_trace_id(self):
|
||
rows = loki('{service_name=~"messenger-.*"} | trace_id="%s"' % self.trace_id)
|
||
services = {stream["service_name"] for stream, _ in rows}
|
||
messages = {v[1] for _, v in rows}
|
||
self.assertIn("messenger-api", services)
|
||
self.assertIn("messenger-worker", services)
|
||
self.assertIn("job_ready", messages)
|
||
self.assertTrue(all(stream.get("span_id") for stream, _ in rows))
|
||
self.assertTrue(all(stream.get("service_version") is not None and stream.get("service_instance_id") for stream, _ in rows))
|
||
|
||
def test_metrics_for_http_jobs_queue_pool_worker(self):
|
||
for query in ('http_server_request_duration_seconds_count{http_route="/api/v1/chats/{chat_id}/attachments"}',
|
||
'messenger_jobs_processed_total{outcome="ready"}',
|
||
'messenger_job_duration_seconds_bucket', 'messenger_job_time_to_ready_seconds_bucket',
|
||
'messenger_queue_depth', 'messenger_queue_oldest_job_age_seconds', 'messenger_worker_capacity', 'messenger_worker_busy',
|
||
'db_client_connections_usage{state="used"}', 'db_client_connections_waiting', 'process_memory_rss_bytes',
|
||
'process_cpu_time_seconds_total'):
|
||
with self.subTest(query):
|
||
deadline = time.time() + 40 # экспорт SDK (5 с) + scrape Prometheus (5 с)
|
||
while not prom(query) and time.time() < deadline:
|
||
time.sleep(2)
|
||
self.assertTrue(prom(query), "нет данных: " + query)
|
||
|
||
def test_no_high_cardinality_or_secret_labels(self):
|
||
forbidden = ("user_id", "chat_id", "message_id", "attachment_id", "request_id", "filename")
|
||
series = json.loads(net_get("http://127.0.0.1:9090/api/v1/series?match[]=%s" % urllib.parse.quote('{__name__=~"http_server.*|messenger_.*|db_client.*|process_.*"}')))["data"]
|
||
labels = {k for s in series for k in s}
|
||
self.assertFalse(labels & set(forbidden), labels & set(forbidden))
|
||
text = json.dumps(series)
|
||
self.assertNotIn(self.a["token"], text)
|
||
route_values = {s.get("http_route") for s in series if "http_route" in s}
|
||
import re
|
||
self.assertFalse([r for r in route_values if re.search(r"[0-9a-f]{8}-[0-9a-f]{4}-", r)], route_values) # шаблоны, не реальные UUID
|
||
|
||
def test_secrets_and_message_bodies_are_not_exported(self):
|
||
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.secret_text), [])
|
||
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["token"]), [])
|
||
self.assertEqual(loki('{service_name=~"messenger-.*"} |= "%s"' % self.a["password"]), [])
|
||
raw = json.dumps(self.trace)
|
||
for secret in (self.secret_text, self.a["token"], self.a["password"]):
|
||
self.assertNotIn(secret, raw)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|