feat: add baseline observability for lab 6
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка сохранённых сигналов через источники Grafana."""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check
|
||||
|
||||
|
||||
def oversized_image():
|
||||
# 9 million pixels fit the API limit but exceed the default worker limit.
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
|
||||
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals"])
|
||||
args = parser.parse_args()
|
||||
|
||||
def query(path, **params):
|
||||
suffix = "?" + urllib.parse.urlencode(params) if params else ""
|
||||
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
def source(uid, path, **params):
|
||||
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
|
||||
|
||||
if args.criterion == "provisioning":
|
||||
for uid in ("prometheus", "loki", "tempo"):
|
||||
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
|
||||
print("Источник Grafana:", uid)
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
|
||||
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
||||
return
|
||||
|
||||
client = Client(args.base_url)
|
||||
start = time.time_ns()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
marker = "telemetry-secret-" + uuid.uuid4().hex
|
||||
message = client.message(alice, chat, marker)
|
||||
request_id = message["headers"]["X-Request-Id"]
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
client.wait_ready(bob, attachment)
|
||||
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
|
||||
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
|
||||
schema="Attachment")["json"]
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
|
||||
if oversized["status"] in ("failed", "ready"):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
|
||||
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
|
||||
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
|
||||
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
|
||||
forbidden = [
|
||||
marker, alice["token"], alice["registration"]["password"], chat["id"],
|
||||
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
|
||||
]
|
||||
|
||||
def metrics(expr):
|
||||
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
|
||||
|
||||
def correlate(service, event, predicate, span_kind):
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
logs = source("loki", "/loki/api/v1/query_range",
|
||||
query='{service_name="' + service + '"} |= "' + event + '"',
|
||||
start=start, end=time.time_ns(), limit=100)["data"]["result"]
|
||||
for stream in logs:
|
||||
for entry in stream["values"]:
|
||||
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
|
||||
if not predicate(metadata):
|
||||
continue
|
||||
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
|
||||
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
|
||||
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
|
||||
try:
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
continue
|
||||
raise
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
|
||||
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
|
||||
check(len(matched) == 1, "span ID from log missing in trace")
|
||||
span = matched[0]
|
||||
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
|
||||
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
|
||||
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
|
||||
return [logs, trace]
|
||||
time.sleep(2)
|
||||
raise Failure(service + ": correlated log and span not found")
|
||||
|
||||
# Loki may include each structured-metadata value in its own result stream.
|
||||
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
|
||||
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
|
||||
error_data = correlate("messenger-worker", "job.processing.completed",
|
||||
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
|
||||
expressions = [
|
||||
"messenger_http_requests_total",
|
||||
"messenger_http_duration_seconds_bucket",
|
||||
'messenger_worker_jobs_total{job_outcome="success"}',
|
||||
'messenger_worker_jobs_total{job_outcome="error"}',
|
||||
"messenger_worker_duration_seconds_bucket",
|
||||
]
|
||||
series = []
|
||||
for expression in expressions:
|
||||
deadline = time.monotonic() + 30
|
||||
samples = metrics(expression)
|
||||
while not samples and time.monotonic() < deadline:
|
||||
time.sleep(2)
|
||||
samples = metrics(expression)
|
||||
check(bool(samples), "missing metric: " + expression)
|
||||
for sample in samples:
|
||||
labels = sample["metric"]
|
||||
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
|
||||
series.extend(samples)
|
||||
print(f"Prometheus: {expression}, рядов={len(samples)}")
|
||||
|
||||
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
|
||||
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
|
||||
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, OSError, ValueError, KeyError) as error:
|
||||
print("FAIL:", error)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сбор воспроизводимых доказательств уровня 3."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
|
||||
"--grafana-url", args.grafana_url]
|
||||
checks = [
|
||||
("grade3-stand", "Стенд, тома и provisioning",
|
||||
[["docker", "compose", "config", "--quiet"],
|
||||
["docker", "compose", "ps", "--all"],
|
||||
["docker", "compose", "config", "--volumes"],
|
||||
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
|
||||
probe + ["provisioning"]],
|
||||
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
|
||||
("grade3-smoke", "Совместимость API при включённой телеметрии",
|
||||
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
|
||||
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
|
||||
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
|
||||
[probe + ["signals"]],
|
||||
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
|
||||
]
|
||||
failed = False
|
||||
for name, title, commands, explanation in checks:
|
||||
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
|
||||
"", explanation, ""]
|
||||
for command in commands:
|
||||
try:
|
||||
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, timeout=180)
|
||||
code, output = result.returncode, result.stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
code, output = 124, "Истёк лимит времени."
|
||||
except OSError:
|
||||
code, output = 127, "Команда недоступна."
|
||||
failed |= code != 0
|
||||
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
|
||||
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**.", "", "```text", output or "(пусто)", "```", ""])
|
||||
path = ROOT / "evidence" / (name + ".md")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(path.relative_to(ROOT))
|
||||
return int(failed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user