feat: add baseline observability for lab 6

This commit is contained in:
Dmitrii Grishko
2026-11-13 18:24:14 +03:00
parent d993866881
commit 549edc8d7a
17 changed files with 967 additions and 33 deletions
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Проверка сохранённых сигналов через источники Grafana."""
import argparse
import base64
import json
from pathlib import Path
import re
import struct
import sys
import time
import urllib.parse
import urllib.request
import uuid
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
def oversized_image():
# 9 million pixels fit the API limit but exceed the default worker limit.
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals"])
args = parser.parse_args()
def query(path, **params):
suffix = "?" + urllib.parse.urlencode(params) if params else ""
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
return json.load(response)
def source(uid, path, **params):
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
if args.criterion == "provisioning":
for uid in ("prometheus", "loki", "tempo"):
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]
attachment, original = client.attachment(alice, chat)
client.wait_ready(bob, attachment)
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
schema="Attachment")["json"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
if oversized["status"] in ("failed", "ready"):
break
time.sleep(0.5)
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
forbidden = [
marker, alice["token"], alice["registration"]["password"], chat["id"],
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
]
def metrics(expr):
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
def correlate(service, event, predicate, span_kind):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="' + service + '"} |= "' + event + '"',
start=start, end=time.time_ns(), limit=100)["data"]["result"]
for stream in logs:
for entry in stream["values"]:
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
if not predicate(metadata):
continue
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
try:
trace = source("tempo", "/api/traces/" + trace_id)
except urllib.error.HTTPError as exc:
if exc.code == 404:
continue
raise
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
check(len(matched) == 1, "span ID from log missing in trace")
span = matched[0]
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
return [logs, trace]
time.sleep(2)
raise Failure(service + ": correlated log and span not found")
# Loki may include each structured-metadata value in its own result stream.
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
error_data = correlate("messenger-worker", "job.processing.completed",
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
expressions = [
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
'messenger_worker_jobs_total{job_outcome="success"}',
'messenger_worker_jobs_total{job_outcome="error"}',
"messenger_worker_duration_seconds_bucket",
]
series = []
for expression in expressions:
deadline = time.monotonic() + 30
samples = metrics(expression)
while not samples and time.monotonic() < deadline:
time.sleep(2)
samples = metrics(expression)
check(bool(samples), "missing metric: " + expression)
for sample in samples:
labels = sample["metric"]
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
series.extend(samples)
print(f"Prometheus: {expression}, рядов={len(samples)}")
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
if __name__ == "__main__":
try:
main()
except (Failure, OSError, ValueError, KeyError) as error:
print("FAIL:", error)
sys.exit(1)
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Сбор воспроизводимых доказательств уровня 3."""
import argparse
from pathlib import Path
import re
import shlex
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
"--grafana-url", args.grafana_url]
checks = [
("grade3-stand", "Стенд, тома и provisioning",
[["docker", "compose", "config", "--quiet"],
["docker", "compose", "ps", "--all"],
["docker", "compose", "config", "--volumes"],
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
probe + ["provisioning"]],
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
("grade3-smoke", "Совместимость API при включённой телеметрии",
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
[probe + ["signals"]],
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
]
failed = False
for name, title, commands, explanation in checks:
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
"", explanation, ""]
for command in commands:
try:
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, timeout=180)
code, output = result.returncode, result.stdout
except subprocess.TimeoutExpired:
code, output = 124, "Истёк лимит времени."
except OSError:
code, output = 127, "Команда недоступна."
failed |= code != 0
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**.", "", "```text", output or "(пусто)", "```", ""])
path = ROOT / "evidence" / (name + ".md")
path.write_text("\n".join(lines), encoding="utf-8")
print(path.relative_to(ROOT))
return int(failed)
if __name__ == "__main__":
sys.exit(main())