#!/usr/bin/env python3 """Проверка сохранённых сигналов через источники Grafana.""" import argparse import base64 import json from pathlib import Path import re import struct import sys import time import urllib.parse import urllib.request import uuid import zlib ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check def oversized_image(): # 9 million pixels fit the API limit but exceed the default worker limit. def chunk(kind, data): return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0) return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b"")) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--grafana-url", default="http://127.0.0.1:3000") parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline", "dashboard"]) args = parser.parse_args() def query(path, **params): suffix = "?" + urllib.parse.urlencode(params) if params else "" with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response: return json.load(response) def source(uid, path, **params): return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params) if args.criterion == "provisioning": for uid in ("prometheus", "loki", "tempo"): check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch") print("Источник Grafana:", uid) dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"] check(len(dashboard["panels"]) >= 6, "dashboard panels missing") print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"])) return if args.criterion == "dashboard": dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"] check(len(dashboard["panels"]) >= 17, "grade 4 dashboard panels missing") for panel in dashboard["panels"]: if panel["datasource"]["type"] != "prometheus": continue for target in panel["targets"]: result = source("prometheus", "/api/v1/query", query=target["expr"]) check(result["status"] == "success", "PromQL query failed") print(panel["title"], "—", target["expr"], "— рядов:", len(result["data"]["result"])) expression = ('1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) ' 'or 0 * sum(increase(messenger_http_requests_total[30m]))) ' '/ sum(increase(messenger_http_requests_total[30m]))') result = source("prometheus", "/api/v1/query", query=expression)["data"]["result"] check(bool(result), "SLI series missing") print("SLI за 30 минут:", result[0]["value"][1], "цель: >= 0.99") return client = Client(args.base_url) start = time.time_ns() alice, bob, _, chat = client.bootstrap() if args.criterion == "pipeline": upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", 202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(), headers={"Content-Type": "image/png"}, schema="Attachment") client.wait_ready(bob, upload["json"]) request_id = upload["headers"]["X-Request-Id"] deadline = time.monotonic() + 45 while time.monotonic() < deadline: logs = source("loki", "/loki/api/v1/query_range", query='{service_name="messenger-api"} | request_id="' + request_id + '"', start=start, end=time.time_ns())["data"]["result"] if logs: trace_id = logs[0]["stream"]["trace_id"] try: trace = source("tempo", "/api/traces/" + trace_id) except urllib.error.HTTPError as error: if error.code != 404: raise time.sleep(2) continue spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]] by_id = {span["spanId"]: span for span in spans} for consumer in (span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"): producer = by_id.get(consumer.get("parentSpanId"), {}) api = by_id.get(producer.get("parentSpanId"), {}) children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]} if (producer.get("kind") == "SPAN_KIND_PRODUCER" and api.get("kind") == "SPAN_KIND_SERVER" and {"postgresql finalize attachment", "s3 read staging and write variants"} <= children): print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id) return time.sleep(2) raise Failure("upload trace did not include the worker pipeline") marker = "telemetry-secret-" + uuid.uuid4().hex message = client.message(alice, chat, marker) request_id = message["headers"]["X-Request-Id"] attachment, original = client.attachment(alice, chat) client.wait_ready(bob, attachment) oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png", 202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"}, schema="Attachment")["json"] deadline = time.monotonic() + 45 while time.monotonic() < deadline: oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"] if oversized["status"] in ("failed", "ready"): break time.sleep(0.5) check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit") print("Worker: изображение 3000×3000 завершилось в состоянии failed.") bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode() client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth}) forbidden = [ marker, alice["token"], alice["registration"]["password"], chat["id"], message["json"]["id"], "sample.png", base64.b64encode(original).decode(), ] def metrics(expr): return source("prometheus", "/api/v1/query", query=expr)["data"]["result"] def correlate(service, event, predicate, span_kind): deadline = time.monotonic() + 45 while time.monotonic() < deadline: logs = source("loki", "/loki/api/v1/query_range", query='{service_name="' + service + '"} |= "' + event + '"', start=start, end=time.time_ns(), limit=100)["data"]["result"] for stream in logs: for entry in stream["values"]: metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})} if not predicate(metadata): continue trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "") check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID") check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID") try: trace = source("tempo", "/api/traces/" + trace_id) except urllib.error.HTTPError as exc: if exc.code == 404: continue raise spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]] span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode() matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)] if not matched: continue check(len(matched) == 1, "duplicate span ID in trace") span = matched[0] check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind") check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing") print(f"{service}: {event}, trace={trace_id}, span={span_id}") return [logs, trace] time.sleep(2) raise Failure(service + ": correlated log and span not found") # Loki may include each structured-metadata value in its own result stream. api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2) worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5) error_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5) expressions = [ "messenger_http_requests_total", "messenger_http_duration_seconds_bucket", 'messenger_worker_jobs_total{job_outcome="success"}', 'messenger_worker_jobs_total{job_outcome="error"}', "messenger_worker_duration_seconds_bucket", ] series = [] for expression in expressions: deadline = time.monotonic() + 30 samples = metrics(expression) while not samples and time.monotonic() < deadline: time.sleep(2) samples = metrics(expression) check(bool(samples), "missing metric: " + expression) for sample in samples: labels = sample["metric"] check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels") series.extend(samples) print(f"Prometheus: {expression}, рядов={len(samples)}") captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False) check(all(value not in captured for value in forbidden), "sensitive test value in telemetry") print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.") if __name__ == "__main__": try: main() except (Failure, OSError, ValueError, KeyError) as error: print("FAIL:", error) sys.exit(1)