152 lines
7.7 KiB
Python
152 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Проверка сохранённых сигналов через источники Grafana."""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check
|
|
|
|
|
|
def oversized_image():
|
|
# 9 million pixels fit the API limit but exceed the default worker limit.
|
|
def chunk(kind, data):
|
|
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
|
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
|
|
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
|
parser.add_argument("criterion", choices=["provisioning", "signals"])
|
|
args = parser.parse_args()
|
|
|
|
def query(path, **params):
|
|
suffix = "?" + urllib.parse.urlencode(params) if params else ""
|
|
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
|
|
return json.load(response)
|
|
|
|
def source(uid, path, **params):
|
|
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
|
|
|
|
if args.criterion == "provisioning":
|
|
for uid in ("prometheus", "loki", "tempo"):
|
|
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
|
|
print("Источник Grafana:", uid)
|
|
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
|
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
|
|
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
|
return
|
|
|
|
client = Client(args.base_url)
|
|
start = time.time_ns()
|
|
alice, bob, _, chat = client.bootstrap()
|
|
marker = "telemetry-secret-" + uuid.uuid4().hex
|
|
message = client.message(alice, chat, marker)
|
|
request_id = message["headers"]["X-Request-Id"]
|
|
attachment, original = client.attachment(alice, chat)
|
|
client.wait_ready(bob, attachment)
|
|
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
|
|
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
|
|
schema="Attachment")["json"]
|
|
deadline = time.monotonic() + 45
|
|
while time.monotonic() < deadline:
|
|
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
|
|
if oversized["status"] in ("failed", "ready"):
|
|
break
|
|
time.sleep(0.5)
|
|
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
|
|
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
|
|
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
|
|
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
|
|
forbidden = [
|
|
marker, alice["token"], alice["registration"]["password"], chat["id"],
|
|
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
|
|
]
|
|
|
|
def metrics(expr):
|
|
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
|
|
|
|
def correlate(service, event, predicate, span_kind):
|
|
deadline = time.monotonic() + 45
|
|
while time.monotonic() < deadline:
|
|
logs = source("loki", "/loki/api/v1/query_range",
|
|
query='{service_name="' + service + '"} |= "' + event + '"',
|
|
start=start, end=time.time_ns(), limit=100)["data"]["result"]
|
|
for stream in logs:
|
|
for entry in stream["values"]:
|
|
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
|
|
if not predicate(metadata):
|
|
continue
|
|
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
|
|
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
|
|
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
|
|
try:
|
|
trace = source("tempo", "/api/traces/" + trace_id)
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
continue
|
|
raise
|
|
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
|
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
|
|
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
|
|
check(len(matched) == 1, "span ID from log missing in trace")
|
|
span = matched[0]
|
|
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
|
|
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
|
|
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
|
|
return [logs, trace]
|
|
time.sleep(2)
|
|
raise Failure(service + ": correlated log and span not found")
|
|
|
|
# Loki may include each structured-metadata value in its own result stream.
|
|
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
|
|
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
|
|
error_data = correlate("messenger-worker", "job.processing.completed",
|
|
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
|
|
expressions = [
|
|
"messenger_http_requests_total",
|
|
"messenger_http_duration_seconds_bucket",
|
|
'messenger_worker_jobs_total{job_outcome="success"}',
|
|
'messenger_worker_jobs_total{job_outcome="error"}',
|
|
"messenger_worker_duration_seconds_bucket",
|
|
]
|
|
series = []
|
|
for expression in expressions:
|
|
deadline = time.monotonic() + 30
|
|
samples = metrics(expression)
|
|
while not samples and time.monotonic() < deadline:
|
|
time.sleep(2)
|
|
samples = metrics(expression)
|
|
check(bool(samples), "missing metric: " + expression)
|
|
for sample in samples:
|
|
labels = sample["metric"]
|
|
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
|
|
series.extend(samples)
|
|
print(f"Prometheus: {expression}, рядов={len(samples)}")
|
|
|
|
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
|
|
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
|
|
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (Failure, OSError, ValueError, KeyError) as error:
|
|
print("FAIL:", error)
|
|
sys.exit(1)
|