Files
mtusi2026lab6/scripts/check_observability.py
T

152 lines
7.7 KiB
Python

#!/usr/bin/env python3
"""Проверка сохранённых сигналов через источники Grafana."""
import argparse
import base64
import json
from pathlib import Path
import re
import struct
import sys
import time
import urllib.parse
import urllib.request
import uuid
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
def oversized_image():
# 9 million pixels fit the API limit but exceed the default worker limit.
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals"])
args = parser.parse_args()
def query(path, **params):
suffix = "?" + urllib.parse.urlencode(params) if params else ""
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
return json.load(response)
def source(uid, path, **params):
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
if args.criterion == "provisioning":
for uid in ("prometheus", "loki", "tempo"):
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]
attachment, original = client.attachment(alice, chat)
client.wait_ready(bob, attachment)
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
schema="Attachment")["json"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
if oversized["status"] in ("failed", "ready"):
break
time.sleep(0.5)
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
forbidden = [
marker, alice["token"], alice["registration"]["password"], chat["id"],
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
]
def metrics(expr):
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
def correlate(service, event, predicate, span_kind):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="' + service + '"} |= "' + event + '"',
start=start, end=time.time_ns(), limit=100)["data"]["result"]
for stream in logs:
for entry in stream["values"]:
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
if not predicate(metadata):
continue
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
try:
trace = source("tempo", "/api/traces/" + trace_id)
except urllib.error.HTTPError as exc:
if exc.code == 404:
continue
raise
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
check(len(matched) == 1, "span ID from log missing in trace")
span = matched[0]
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
return [logs, trace]
time.sleep(2)
raise Failure(service + ": correlated log and span not found")
# Loki may include each structured-metadata value in its own result stream.
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
error_data = correlate("messenger-worker", "job.processing.completed",
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
expressions = [
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
'messenger_worker_jobs_total{job_outcome="success"}',
'messenger_worker_jobs_total{job_outcome="error"}',
"messenger_worker_duration_seconds_bucket",
]
series = []
for expression in expressions:
deadline = time.monotonic() + 30
samples = metrics(expression)
while not samples and time.monotonic() < deadline:
time.sleep(2)
samples = metrics(expression)
check(bool(samples), "missing metric: " + expression)
for sample in samples:
labels = sample["metric"]
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
series.extend(samples)
print(f"Prometheus: {expression}, рядов={len(samples)}")
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
if __name__ == "__main__":
try:
main()
except (Failure, OSError, ValueError, KeyError) as error:
print("FAIL:", error)
sys.exit(1)