Files
mtusi2026lab6/scripts/lab6.py
T

1867 lines
72 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Операторские команды стенда лабы 6 (свои, не skeleton). Python 3.9+, stdlib.
python3 scripts/lab6.py env
.env из .env.example со случайными паролями и ключами S3 (make env);
несекретные переменные, заданные в окружении (`make env
HOST_PORT=8180`), берутся оттуда — так стенд встаёт рядом с чужим
на других портах. Существующий .env не перезаписывается,
дописываются только переменные шаблона, которых в нём нет.
Печатает только имена;
python3 scripts/lab5.py instances [--base-url URL] [--requests 12]
вход одним синтетическим пользователем и серия GET /users/me с одним
токеном через LB: X-Instance-Id каждого ответа и итог по репликам
(шаг 1 сценария защиты лабы 3);
python3 scripts/lab5.py lb-stats
слоты HAProxy рабочего стенда: адрес, состояние проверки, запросы в
полёте, всего, ошибки соединения, повторы, перенаправления;
python3 scripts/lab5.py failover-each [--base-url URL] [--replicas 2]
[--mode stop|kill]
make test-failover для каждой реплики по очереди: перед прогоном все
реплики возвращены (make replicas), тест останавливает реплику,
ответившую на его первую запись; если это реплика, которую уже
проверили, очередь LB сдвигается одним запросом и прогон
повторяется. Печатает docker compose ps до и после, строку
STOP_ONE_LOG и итог теста — это evidence «какой контейнер
действительно остановлен»;
python3 scripts/lab5.py attachment-failover [--base-url URL]
[--replicas 2] [--mode stop|kill]
загрузка tests/fixtures/sample.png через LB (202, затем ready),
сообщение со ссылкой, остановка именно реплики, принявшей загрузку
(scripts/stop-one.sh по X-Instance-Id ответа), скачивание вторым
участником через другую реплику — тот же SHA-256, 404 постороннему;
затем реплики возвращаются;
python3 scripts/lab5.py defense [--base-url URL]
сценарий защиты лабы 5, шаги 2–3: worker остановлен → загрузка 202 →
сообщение со ссылкой 201 → content 409, текст 201 → scripts/restart.sh
(LB, реплики, PostgreSQL, Valkey, S3, брокер) → worker запущен → ready,
оригинал и thumbnail 256×160. Печатает id и StartedAt без токенов;
python3 scripts/lab5.py pipeline
задания по состояниям, возраст старейшего, сообщения в очереди;
python3 scripts/lab5.py s3-check
приватность bucket вне API (make s3-check): bootstrap в режиме
check (права ключей API, worker и уборщика, чужой bucket,
административные запросы), анонимные запросы с хоста и из сети
стенда, порты контейнера S3 из сети, затем cleanup. Таблица «где →
проверка → ожидаемо → получено», код 1 при нарушении;
python3 scripts/lab6.py telemetry-check [--base-url URL]
лаба 6 (make telemetry-check): provisioning Grafana; запрос со своим
X-Request-Id → трасса в Tempo, запись в Loki и строка stdout с тем
же trace_id; загрузка → span PostgreSQL и S3, трасса worker с
исходом ready и запись «attachment processed» с её trace_id;
метрики API, worker и relay и ресурс в Prometheus, метки без id;
пароля, токена, текста сообщения, имени файла и Authorization нет
ни в логах, ни в трассах. Хранилища — через прокси источников
Grafana (127.0.0.1:GRAFANA_HOST_PORT, пароль admin из .env). Итог
PASS/FAIL по каждой проверке, код 1 при FAIL.
python3 scripts/lab6.py trace-chain [--base-url URL]
уровень 4 (make trace-chain, ~1 мин): worker остановлен → загрузка
202 → S3 остановлен → worker берёт задание, попытка 1 — retry →
S3 и worker обратно → ready. В Tempo по id вложения ровно одна
трасса: server span загрузки → producer span relay на каждую
публикацию → consumer span worker той же попытки → S3 и
PostgreSQL готовой попытки. Печатает дерево span'ов;
python3 scripts/lab6.py alerts-check [--base-url URL]
уровень 4 (make alerts-check, ~8–10 мин): (а) worker остановлен,
3 загрузки → MessengerQueueLagGrowing firing → worker запущен →
backlog разобран, resolved; (б) таблица sessions под ACCESS
EXCLUSIVE блокировкой при фоновой нагрузке GET /users/me → 503 по
statement_timeout → MessengerHighErrorRate firing → блокировка
снята → resolved. Состояния — Prometheus /api/v1/alerts;
python3 scripts/lab6.py telemetry-outage [--base-url URL] [--seconds 120]
уровень 4 (make telemetry-outage, ~3–4 мин): Tempo остановлен под
нагрузкой (GET /users/me и загрузка раз в 10 с): ответы и ready
без отказов, пик памяти Collector меньше лимита контейнера,
потерянные span'ы посчитаны счётчиками Collector, после возврата
Tempo экспорт идёт и очередь пуста.
Пароли синтетических пользователей и токены живут только в памяти и не
печатаются. Работает с проектом compose репозитория (compose.yaml и .env)
или с тем, что задают COMPOSE_PROJECT_NAME / COMPOSE_FILE /
COMPOSE_ENV_FILES (стенды student-tests/stand.py).
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import os
import re
import secrets
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from collections import Counter
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
LB_STATS = (
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
)
STAT_FIELDS = ("addr", "status", "scur", "stot", "econ", "eresp", "wretr", "wredis")
# пустые значения этих переменных шаблона — секреты: make env их генерирует
SECRET_SUFFIXES = ("_PASSWORD", "_KEY")
# порты API S3 и gRPC шлюза: единственные, что открыты в сеть стенда
S3_NETWORK_PORTS = ("8333", "18333")
def call(base: str, method: str, path: str, headers=None, body=None, raw=None):
data = json.dumps(body).encode() if body is not None else raw
request = urllib.request.Request(base + path, data=data, method=method)
for key, value in (headers or {}).items():
request.add_header(key, value)
if body is not None:
request.add_header("Content-Type", "application/json")
try:
with urllib.request.urlopen(request, timeout=10) as response:
return response.status, dict(response.headers), response.read()
except urllib.error.HTTPError as error:
return error.code, dict(error.headers), error.read()
def register(base: str, label: str = "ops") -> dict:
"""Синтетический пользователь с паролем и входом: id и токен."""
username = f"{label}_" + secrets.token_hex(6)
password = secrets.token_urlsafe(18)
status, _, data = call(
base,
"POST",
"/api/v1/users",
body={"username": username, "display_name": "Оператор", "password": password},
)
if status != 201:
raise SystemExit(f"register -> {status}")
user_id = json.loads(data)["id"]
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data = call(
base,
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + basic},
)
if status != 201:
raise SystemExit(f"login -> {status}")
return {"id": user_id, "token": json.loads(data)["token"]}
def login(base: str) -> str:
return register(base)["token"]
def bearer(actor: dict) -> dict:
return {"Authorization": "Bearer " + actor["token"]}
def instance_of(headers: dict) -> str:
return headers.get("X-Instance-Id") or headers.get("x-instance-id") or "-"
# --- env ------------------------------------------------------------------------
def _assignments(text: str) -> list[tuple[str, str]]:
pairs = []
for line in text.splitlines():
match = re.match(r"([A-Z][A-Z0-9_]*)=(.*)$", line)
if match:
pairs.append((match.group(1), match.group(2)))
return pairs
def _value(name: str, value: str) -> str:
"""Значение для нового .env: секрет — случайный, несекретная переменная
из окружения (`make env HOST_PORT=8180`) — как задано, иначе шаблон."""
if name.endswith(SECRET_SUFFIXES):
# ключи S3 и пароли — случайный hex; длина ключа S3 — 64 символа
return value or secrets.token_hex(32 if name.startswith("S3_") else 16)
return os.environ.get(name, value)
def env(_args: argparse.Namespace) -> int:
template = (ROOT / ".env.example").read_text(encoding="utf-8")
target = ROOT / ".env"
if not target.exists():
lines = []
generated = []
for line in template.splitlines():
match = re.match(r"([A-Z][A-Z0-9_]*)=(.*)$", line)
if match:
value = _value(match.group(1), match.group(2))
if value != match.group(2) and match.group(1).endswith(SECRET_SUFFIXES):
generated.append(match.group(1))
line = f"{match.group(1)}={value}"
lines.append(line)
target.write_text("\n".join(lines) + "\n", encoding="utf-8")
target.chmod(0o600)
print(f".env created; generated: {' '.join(generated) or '-'}")
return 0
existing = {name for name, _ in _assignments(target.read_text(encoding="utf-8"))}
missing = [(n, v) for n, v in _assignments(template) if n not in existing]
if missing:
with target.open("a", encoding="utf-8") as handle:
handle.write("\n# added by make env from .env.example\n")
for name, value in missing:
handle.write(f"{name}={_value(name, value)}\n")
print("added to existing .env: " + " ".join(name for name, _ in missing))
else:
print(".env ready")
return 0
def instances(args: argparse.Namespace) -> int:
token = login(args.base_url)
seen: Counter = Counter()
print(f"one session, {args.requests} x GET /api/v1/users/me via {args.base_url}")
for number in range(1, args.requests + 1):
status, headers, _ = call(
args.base_url,
"GET",
"/api/v1/users/me",
headers={"Authorization": "Bearer " + token},
)
instance = headers.get("X-Instance-Id") or headers.get("x-instance-id") or "-"
seen[instance] += 1
print(f" {number:2d}: {status} X-Instance-Id={instance}")
print(f"replicas seen: {len(seen)} {dict(seen)}")
return 0 if len(seen) >= 2 else 1
def compose(*args: str, check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(
["docker", "compose", *args],
cwd=ROOT,
capture_output=True,
text=True,
check=check,
)
def lb_rows() -> list[dict]:
done = compose("exec", "-T", "haproxy", "bash", "-c", LB_STATS)
lines = [
line for line in done.stdout.splitlines() if line.startswith(("# ", "api,"))
]
header = lines[0][2:].split(",")
rows = []
for line in lines[1:]:
# zip(strict=) — только с Python 3.10, скрипт работает и на 3.9
row = dict(zip(header, line.split(","))) # noqa: B905
if row["svname"] in ("BACKEND", "FRONTEND") or not row.get("addr"):
continue
rows.append(row)
return rows
def lb_stats(_args: argparse.Namespace) -> int:
print("slot " + " ".join(f"{field:>12}" for field in STAT_FIELDS))
for row in lb_rows():
print(
f"{row['svname']:<9} "
+ " ".join(f"{row.get(f, ''):>12}" for f in STAT_FIELDS)
)
return 0
def wait_lb_up(replicas: int, timeout: float = 30) -> bool:
"""Все реплики UP в LB: `--wait` compose ждёт только healthcheck
контейнера, а слот вернувшейся реплики включается после проверок LB."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
up = [row for row in lb_rows() if row.get("status") == "UP"]
if len(up) >= replicas:
return True
time.sleep(0.5)
return False
def ps() -> str:
return compose(
"ps", "-a", "--format", "table {{.Name}}\t{{.ID}}\t{{.State}}\t{{.Status}}"
).stdout
def failover_each(args: argparse.Namespace) -> int:
done: set[str] = set()
log = Path(tempfile.mkdtemp(prefix="stop-one-")) / "stop-one.log"
env = {**os.environ, "STOP_ONE_LOG": str(log), "STOP_MODE": args.mode}
attempt = 0
while len(done) < args.replicas and attempt < args.replicas * 4:
attempt += 1
replicas = compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"--wait",
"--scale",
f"api={args.replicas}",
"api",
check=False,
)
if replicas.returncode != 0 or not wait_lb_up(args.replicas):
print(replicas.stderr or "replicas are not UP in the LB")
return 2
# сдвиг очереди round-robin: тест останавливает реплику, ответившую
# на его первую запись; с каждой попыткой сдвиг на запрос больше
for _ in range(attempt - 1):
call(args.base_url, "GET", "/health/live")
print(f"=== attempt {attempt}: docker compose ps before\n{ps()}", flush=True)
before = log.read_text().splitlines() if log.exists() else []
test = subprocess.run(
[
sys.executable,
"tests/resilience.py",
"--mode",
"failover",
"--action-script",
"scripts/stop-one.sh",
"--base-url",
args.base_url,
],
cwd=ROOT,
env=env,
capture_output=True,
text=True,
check=False,
)
print(
"$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh "
f"(STOP_MODE={args.mode})"
)
print(test.stdout.strip())
lines = log.read_text().splitlines() if log.exists() else []
new = lines[len(before) :]
print("STOP_ONE_LOG: " + (new[-1] if new else "(no line)"))
print(f"docker compose ps after\n{ps()}", flush=True)
if test.returncode != 0:
return test.returncode
stopped = new[-1].split("container=")[1].split()[0][:12] if new else ""
if stopped:
done.add(stopped)
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"--wait",
"--scale",
f"api={args.replicas}",
"api",
)
print(f"replicas stopped one by one: {sorted(done)}; all restored:\n{ps()}")
return 0 if len(done) >= args.replicas else 1
# --- вложение и отказ принявшей реплики ---------------------------------------
def wait_status(
base: str, actor: dict, attachment_id: str, wanted: tuple, timeout: float = 60
) -> dict:
"""Опрос метаданных, пока статус не войдёт в `wanted`."""
deadline = time.monotonic() + timeout
while True:
status, _, data = call(
base, "GET", f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
)
if status == 200:
meta = json.loads(data)
if meta["status"] in wanted:
return meta
if time.monotonic() > deadline:
raise SystemExit(f"attachment did not reach {wanted} in {timeout:.0f} s")
time.sleep(0.3)
def restore_replicas(replicas: int) -> None:
compose(
"up",
"-d",
"--no-deps",
"--no-recreate",
"--wait",
"--scale",
f"api={replicas}",
"api",
)
wait_lb_up(replicas)
def attachment_failover(args: argparse.Namespace) -> int:
base = args.base_url
restore_replicas(args.replicas)
alice, bob, eve = (register(base, label) for label in ("alice", "bob", "eve"))
status, _, data = call(
base,
"POST",
"/api/v1/chats",
headers=bearer(alice),
body={"title": "Отказ реплики", "member_ids": [alice["id"], bob["id"]]},
)
if status != 201:
raise SystemExit(f"create chat -> {status}")
chat = json.loads(data)["id"]
content = FIXTURE.read_bytes()
expected = hashlib.sha256(content).hexdigest()
status, headers, data = call(
base,
"POST",
f"/api/v1/chats/{chat}/attachments?filename=sample.png",
headers={**bearer(alice), "Content-Type": "image/png"},
raw=content,
)
if status != 202:
raise SystemExit(f"upload -> {status} {data[:200]!r}")
attachment = json.loads(data)
uploader = instance_of(headers)
print(
f"upload: 202 {attachment['status']} X-Instance-Id={uploader} "
f"size={attachment['size_bytes']} sha256={attachment['sha256'][:16]}… "
f"(fixture {expected[:16]}…)"
)
ready = wait_status(base, alice, attachment["id"], ("ready", "failed"))
print(f"processed: {ready['status']} variants={len(ready['variants'])}")
status, headers, data = call(
base,
"POST",
f"/api/v1/chats/{chat}/messages",
headers={**bearer(alice), "Idempotency-Key": uuid.uuid4().hex},
body={"text": "", "attachment_ids": [attachment["id"]]},
)
print(
f"message with attachment: {status} attachment_ids="
f"{json.loads(data).get('attachment_ids') if status == 201 else '-'}"
)
log = Path(tempfile.mkdtemp(prefix="stop-one-")) / "stop-one.log"
print(f"=== docker compose ps before\n{ps()}", flush=True)
stop = subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env={
**os.environ,
"TARGET_INSTANCE_ID": uploader,
"STOP_MODE": args.mode,
"STOP_ONE_LOG": str(log),
},
capture_output=True,
text=True,
check=False,
)
print(f"$ TARGET_INSTANCE_ID={uploader} STOP_MODE={args.mode} scripts/stop-one.sh")
stopped = log.read_text().strip() if log.exists() else "-"
print(f"exit {stop.returncode}; STOP_ONE_LOG: {stopped}")
if stop.returncode != 0:
print(stop.stderr.strip())
return 1
# LB исключает реплику за ≤ 2,5 с; ответы ещё могут прийти от неё —
# ждём ответа другой реплики не дольше 10 с
deadline = time.monotonic() + 10
started = time.monotonic()
result = None
while time.monotonic() < deadline:
try:
status, headers, data = call(
base,
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
headers=bearer(bob),
)
except OSError:
status, headers, data = 0, {}, b""
if status == 200 and instance_of(headers) != uploader:
result = (status, headers, data)
break
time.sleep(0.2)
ok = result is not None
if result is not None:
status, headers, data = result
digest = hashlib.sha256(data).hexdigest()
ok = digest == expected and instance_of(headers) != uploader
kind = headers.get("content-type") or headers.get("Content-Type")
print(
f"download by bob after stop: {status} "
f"X-Instance-Id={instance_of(headers)} "
f"after {time.monotonic() - started:.2f}s, bytes={len(data)}, "
f"sha256 {'== fixture' if digest == expected else '!= fixture'}, "
f"content-type={kind}"
)
else:
print("download by bob after stop: no 200 from another replica in 10 s")
for suffix in ("", "/content"):
status, _, _ = call(
base,
"GET",
f"/api/v1/attachments/{attachment['id']}{suffix}",
headers=bearer(eve),
)
print(f"outsider GET /attachments/{{id}}{suffix}: {status}")
ok = ok and status == 404
print(f"=== docker compose ps after\n{ps()}", flush=True)
restore_replicas(args.replicas)
print(f"replicas restored:\n{ps()}")
print("PASS" if ok else "FAIL")
return 0 if ok else 1
# --- лаба 5: сценарий защиты и состояние pipeline ------------------------------
def started_at(service: str) -> str:
ids = compose("ps", "-q", service, check=False).stdout.split()
if not ids:
return f"{service}: not running"
done = subprocess.run(
["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids],
capture_output=True,
text=True,
check=False,
)
return "; ".join(line.strip().lstrip("/") for line in done.stdout.splitlines())
def jpeg_size(data: bytes) -> tuple:
"""Размеры JPEG по маркеру SOF (как tests/image_probe.py)."""
i = 2
while i + 9 < len(data):
if data[i] != 0xFF:
i += 1
continue
marker = data[i + 1]
length = int.from_bytes(data[i + 2 : i + 4], "big")
if marker in (0xC0, 0xC1, 0xC2):
height = int.from_bytes(data[i + 5 : i + 7], "big")
width = int.from_bytes(data[i + 7 : i + 9], "big")
return width, height
i += 2 + length
return (0, 0)
def defense(args: argparse.Namespace) -> int:
base = args.base_url
services = ("api", "postgres", "valkey", "s3", "rabbitmq", "worker", "relay")
print("$ docker compose stop worker")
compose("stop", "worker")
ok = True
try:
alice, bob = (register(base, label) for label in ("alice", "bob"))
status, _, data = call(
base,
"POST",
"/api/v1/chats",
headers=bearer(alice),
body={"title": "Защита", "member_ids": [alice["id"], bob["id"]]},
)
chat = json.loads(data)["id"]
content = FIXTURE.read_bytes()
status, _, data = call(
base,
"POST",
f"/api/v1/chats/{chat}/attachments?filename=sample.png",
headers={**bearer(alice), "Content-Type": "image/png"},
raw=content,
)
attachment = json.loads(data)
print(f"worker stopped; upload -> {status} {attachment.get('status')}")
print(f"attachment_id={attachment.get('id')}")
ok = ok and status == 202
status, _, data = call(
base,
"POST",
f"/api/v1/chats/{chat}/messages",
headers={**bearer(alice), "Idempotency-Key": uuid.uuid4().hex},
body={"text": "", "attachment_ids": [attachment["id"]]},
)
print(f"message with queued attachment -> {status}")
ok = ok and status == 201
status, _, _ = call(
base,
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
headers=bearer(bob),
)
print(f"content while worker is stopped -> {status}")
ok = ok and status == 409
status, _, _ = call(
base,
"POST",
f"/api/v1/chats/{chat}/messages",
headers={**bearer(bob), "Idempotency-Key": uuid.uuid4().hex},
body={"text": "текст без вложения"},
)
print(f"plain text message meanwhile -> {status}")
ok = ok and status == 201
print("StartedAt before restart:")
for service in services:
print(" " + started_at(service))
restart = subprocess.run(
["scripts/restart.sh"],
cwd=ROOT,
capture_output=True,
text=True,
check=False,
)
print(f"$ scripts/restart.sh -> exit {restart.returncode}")
ok = ok and restart.returncode == 0
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
if call(base, "GET", "/health/ready")[0] == 200:
break
except OSError:
pass
time.sleep(0.5)
meta = wait_status(base, bob, attachment["id"], ("queued", "processing"), 30)
print(f"after restart, worker still stopped: {meta['status']}")
finally:
print("$ docker compose start worker")
compose("up", "-d", "--no-deps", "worker")
meta = wait_status(base, bob, attachment["id"], ("ready", "failed"))
print(f"after worker start: {meta['status']} error_code={meta['error_code']}")
ok = ok and meta["status"] == "ready"
status, _, data = call(
base,
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
headers=bearer(bob),
)
same = hashlib.sha256(data).hexdigest() == hashlib.sha256(content).hexdigest()
print(f"original -> {status}, sha256 {'== fixture' if same else '!= fixture'}")
ok = ok and status == 200 and same
status, _, data = call(
base,
"GET",
f"/api/v1/attachments/{attachment['id']}/content?variant=thumbnail",
headers=bearer(bob),
)
print(f"thumbnail -> {status}, JPEG {jpeg_size(data)}")
ok = ok and status == 200 and jpeg_size(data) == (256, 160)
print("StartedAt after:")
for service in services:
print(" " + started_at(service))
print("PASS" if ok else "FAIL")
return 0 if ok else 1
PIPELINE_SQL = (
"SELECT state, count(*), "
"coalesce(round(extract(epoch FROM now() - min(created_at))::numeric, 1), 0) "
"FROM attachment_jobs GROUP BY state ORDER BY state"
)
def pipeline(_args: argparse.Namespace) -> int:
db = _env_value("POSTGRES_DB", "messenger")
user = _env_value("POSTGRES_USER", "messenger")
done = compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
user,
"-d",
db,
"-At",
"-F",
" ",
"-c",
PIPELINE_SQL,
check=False,
)
print("state count oldest_s")
print(done.stdout.strip() or done.stderr.strip())
queues = compose(
"exec",
"-T",
"rabbitmq",
"rabbitmqctl",
"-q",
"list_queues",
"name",
"messages_ready",
"messages_unacknowledged",
"consumers",
check=False,
)
print("queue ready unacked consumers")
print(queues.stdout.strip() or queues.stderr.strip())
return done.returncode
# --- приватность bucket вне API ------------------------------------------------
def _env_value(name: str, default: str) -> str:
env_file = os.environ.get("COMPOSE_ENV_FILES") or str(ROOT / ".env")
path = Path(env_file.split(",")[0])
if path.exists():
for key, value in _assignments(path.read_text(encoding="utf-8")):
if key == name and value:
return value
return os.environ.get(name) or default
def _http_status(url: str, method: str = "GET", data: bytes | None = None) -> str:
request = urllib.request.Request(url, data=data, method=method)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return str(response.status)
except urllib.error.HTTPError as error:
return str(error.code)
except OSError as error:
return type(error).__name__
def _network_status(path: str, method: str = "GET") -> str:
"""Анонимный запрос к API S3 из сети стенда (контейнер LB, bash /dev/tcp)."""
request = (
f"{method} {path} HTTP/1.0\\r\\nHost: s3\\r\\nContent-Length: 0\\r\\n\\r\\n"
)
script = f"exec 3<>/dev/tcp/s3/8333 && printf '{request}' >&3 && head -n 1 <&3"
done = compose("exec", "-T", "haproxy", "bash", "-c", script, check=False)
match = re.search(r"HTTP/1\.[01] (\d{3})", done.stdout)
return match.group(1) if match else "no answer"
def _port_open(port: str) -> bool:
script = f"timeout 2 bash -c 'exec 3<>/dev/tcp/s3/{port}'"
return (
compose("exec", "-T", "haproxy", "bash", "-c", script, check=False).returncode
== 0
)
def s3_check(_args: argparse.Namespace) -> int:
rows: list[tuple[str, str, str, str]] = []
print("$ docker compose run --rm --no-deps s3-bootstrap check")
check = compose("run", "--rm", "--no-deps", "s3-bootstrap", "check", check=False)
for line in check.stdout.splitlines():
parts = line.split("\t")
if line.startswith("s3-bootstrap: check") and len(parts) == 5:
rows.append(("bootstrap (loopback s3)", parts[1], parts[2], parts[3]))
elif line.strip():
print(line)
if check.returncode != 0:
print(check.stderr.strip())
bucket = _env_value("S3_BUCKET", "messenger-attachments")
host = f"http://127.0.0.1:{_env_value('S3_HOST_PORT', '8334')}"
probe = f"/{bucket}/.s3-check/probe"
try:
for name, url, method in (
("anonymous get object", host + probe, "GET"),
("anonymous head object", host + probe, "HEAD"),
("anonymous list objects", f"{host}/{bucket}?list-type=2", "GET"),
("anonymous list buckets", host + "/", "GET"),
):
rows.append(("host 127.0.0.1", name, "403", _http_status(url, method)))
rows.append(
(
"host 127.0.0.1",
"anonymous put object",
"403",
_http_status(f"{host}/{bucket}/.s3-check/anonymous", "PUT", b"x"),
)
)
for name, path, method in (
("anonymous get object", probe, "GET"),
("anonymous list objects", f"/{bucket}?list-type=2", "GET"),
("anonymous put object", f"/{bucket}/.s3-check/anonymous", "PUT"),
):
rows.append(
("stand network (haproxy)", name, "403", _network_status(path, method))
)
listening = compose(
"exec", "-T", "s3", "sh", "-c", "netstat -tln", check=False
).stdout
ports = sorted(
{
m.group(1)
for m in re.finditer(r"\s\S+:(\d+)\s+\S+:\*?\S*\s+LISTEN", listening)
},
key=int,
)
for port in ports:
expected = "open" if port in S3_NETWORK_PORTS else "closed"
got = "open" if _port_open(port) else "closed"
rows.append(("stand network (haproxy)", f"s3 port {port}", expected, got))
finally:
cleanup = compose(
"run", "--rm", "--no-deps", "s3-bootstrap", "cleanup", check=False
)
print(
"$ docker compose run --rm --no-deps s3-bootstrap cleanup -> exit",
cleanup.returncode,
)
failed = 0
print(f"{'where':<26} {'check':<46} {'expected':>8} {'got':>8} verdict")
for where, name, expected, got in rows:
verdict = "ok" if expected == got else "FAIL"
failed += verdict != "ok"
print(f"{where:<26} {name:<46} {expected:>8} {got:>8} {verdict}")
print(f"checks: {len(rows)}, failed: {failed}, bootstrap exit {check.returncode}")
return 1 if failed or check.returncode != 0 else 0
# --- лаба 6: телеметрия ----------------------------------------------------------
# метки метрик, в имени которых не должно быть ничего из этого
# (contracts/observability.md, «Не labels метрик»)
FORBIDDEN_LABEL_PARTS = (
"user",
"chat",
"message_id",
"attachment",
"request_id",
"url",
"path",
"filename",
"text",
)
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
# сколько ждать сигналы: экспорт метрик 15 с + scrape 15 с + пакеты логов
SIGNAL_TIMEOUT = 120.0
class Grafana:
"""HTTP API хранилищ через прокси источников Grafana (uid из
infra/otel/grafana/datasources.yaml). Пароль admin — из .env, не
печатается."""
def __init__(self, port: str, password: str) -> None:
self.base = f"http://127.0.0.1:{port}"
token = base64.b64encode(f"admin:{password}".encode()).decode()
self.headers = {"Authorization": "Basic " + token}
def get(self, path: str, params: dict | None = None):
query = ("?" + urllib.parse.urlencode(params)) if params else ""
request = urllib.request.Request(self.base + path + query, headers=self.headers)
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status, json.loads(response.read() or b"null")
except urllib.error.HTTPError as error:
return error.code, None
except (OSError, ValueError) as error:
return type(error).__name__, None
def source(self, uid: str, path: str, params: dict | None = None):
return self.get(f"/api/datasources/proxy/uid/{uid}{path}", params)
class Checks:
def __init__(self) -> None:
self.results: list[bool] = []
def add(self, ok: bool, name: str, detail: str = "") -> bool:
self.results.append(ok)
print(f"{'PASS' if ok else 'FAIL'} {name}" + (f": {detail}" if detail else ""))
return ok
def _poll(fetch, timeout: float = SIGNAL_TIMEOUT, pause: float = 3.0):
"""Повторять `fetch()`, пока он не вернёт непустое, или до срока."""
deadline = time.monotonic() + timeout
while True:
value = fetch()
if value or time.monotonic() >= deadline:
return value
time.sleep(pause)
def _tempo_find(grafana: Grafana, traceql: str, since: float) -> list[str]:
status, data = grafana.source(
"traces",
"/api/search",
{"q": traceql, "start": int(since) - 60, "end": int(time.time()) + 60},
)
if status != 200 or not data:
return []
# Tempo отдаёт id без ведущих нулей, а журнал — все 32 hex-знака
return [item["traceID"].zfill(32) for item in data.get("traces") or []]
def _tempo_trace(grafana: Grafana, trace_id: str) -> dict:
status, data = grafana.source("traces", f"/api/v2/traces/{trace_id}")
return data if status == 200 and data else {}
def _spans(trace: dict) -> list[tuple[str, str, dict]]:
"""(service.name, имя span, атрибуты) из ответа /api/v2/traces."""
found = []
body = trace.get("trace", trace)
for resource in body.get("resourceSpans") or body.get("batches") or []:
service = ""
for attr in resource.get("resource", {}).get("attributes", []):
if attr["key"] == "service.name":
service = attr["value"].get("stringValue", "")
for scope in resource.get("scopeSpans", []):
for span in scope.get("spans", []):
attrs = {
a["key"]: next(iter(a["value"].values()), None)
for a in span.get("attributes", [])
}
found.append((service, span.get("name", ""), attrs))
return found
def _loki_lines(grafana: Grafana, query: str, since: float, limit: int = 1000):
"""[(метки и structured metadata потока, строка)] за окно с `since`."""
status, data = grafana.source(
"logs",
"/loki/api/v1/query_range",
{
"query": query,
"start": int((since - 60) * 1e9),
"end": int((time.time() + 60) * 1e9),
"limit": limit,
"direction": "backward",
},
)
if status != 200 or not data:
return []
rows = []
for stream in data.get("data", {}).get("result", []):
for _, line in stream.get("values", []):
rows.append((stream.get("stream", {}), line))
return rows
def _prom(grafana: Grafana, expr: str) -> list:
status, data = grafana.source("metrics", "/api/v1/query", {"query": expr})
if status != 200 or not data:
return []
return data.get("data", {}).get("result", [])
def _prom_value(grafana: Grafana, expr: str) -> float:
result = _prom(grafana, expr)
return float(result[0]["value"][1]) if result else 0.0
def telemetry_check(args: argparse.Namespace) -> int:
base = args.base_url
checks = Checks()
password_env = _env_value("GRAFANA_ADMIN_PASSWORD", "")
if not password_env:
print("FAIL GRAFANA_ADMIN_PASSWORD is empty: run make env")
return 1
grafana = Grafana(_env_value("GRAFANA_HOST_PORT", "3180"), password_env)
started = time.time()
# 1. Grafana: provisioning из git
status, health = grafana.get("/api/health")
checks.add(status == 200, "grafana /api/health", f"status {status}")
status, sources = grafana.get("/api/datasources")
uids = sorted(item["uid"] for item in sources or [])
checks.add(
{"metrics", "logs", "traces"} <= set(uids), "datasources provisioned", str(uids)
)
status, board = grafana.get("/api/dashboards/uid/messenger-lab6")
meta = (board or {}).get("meta", {})
panels = (board or {}).get("dashboard", {}).get("panels", [])
checks.add(
status == 200 and meta.get("provisioned") is True,
"dashboard messenger-lab6 provisioned",
f"status {status}, panels {len(panels)}",
)
# 2. контролируемые запросы: свой X-Request-Id, известные пароль, токен,
# текст сообщения и имя файла — потом их не должно быть в телеметрии
marker = secrets.token_hex(4)
rid_me = f"tcheck-{marker}-me"
rid_upload = f"tcheck-{marker}-upload"
password = "Tc-" + secrets.token_urlsafe(18)
text_marker = f"telemetry body {secrets.token_hex(8)}"
filename = f"tcheck-{secrets.token_hex(6)}.png"
username = f"tcheck_{marker}"
status, _, data = call(
base,
"POST",
"/api/v1/users",
body={"username": username, "display_name": "Проверка", "password": password},
)
if status != 201:
print(f"FAIL register -> {status}")
return 1
user_id = json.loads(data)["id"]
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data = call(
base,
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + basic},
)
token = json.loads(data)["token"]
auth = {"Authorization": "Bearer " + token}
status, headers, _ = call(
base, "GET", "/api/v1/users/me", headers={**auth, "X-Request-Id": rid_me}
)
checks.add(
status == 200
and {k.lower(): v for k, v in headers.items()}.get("x-request-id") == rid_me,
"controlled request GET /api/v1/users/me",
f"status {status}, X-Request-Id {rid_me}",
)
status, _, data = call(
base,
"POST",
"/api/v1/chats",
headers=auth,
body={"title": "Телеметрия", "member_ids": [user_id]},
)
chat = json.loads(data)["id"]
status, _, data = call(
base,
"POST",
f"/api/v1/chats/{chat}/attachments?filename={filename}",
headers={**auth, "Content-Type": "image/png", "X-Request-Id": rid_upload},
raw=FIXTURE.read_bytes(),
)
attachment = json.loads(data)["id"] if status == 202 else ""
checks.add(status == 202, "upload accepted", f"status {status}")
actor = {"id": user_id, "token": token}
meta_doc = (
wait_status(base, actor, attachment, ("ready", "failed")) if attachment else {}
)
checks.add(
meta_doc.get("status") == "ready",
"attachment ready",
str(meta_doc.get("status")),
)
status, _, _ = call(
base,
"POST",
f"/api/v1/chats/{chat}/messages",
headers={**auth, "Idempotency-Key": f"tcheck-{marker}"},
body={"text": text_marker, "attachment_ids": [attachment]},
)
checks.add(status == 201, "message with attachment", f"status {status}")
# 3. трасса запроса в Tempo и связанный лог в Loki
api_traces = _poll(
lambda: _tempo_find(
grafana, f'{{ span.messenger.request_id = "{rid_me}" }}', started
)
)
checks.add(
len(api_traces) == 1, "API trace found by request id", ",".join(api_traces)
)
api_trace = _tempo_trace(grafana, api_traces[0]) if api_traces else {}
spans = _spans(api_trace)
server = [s for s in spans if s[1] == "GET /api/v1/users/me"]
checks.add(
bool(server)
and server[0][0] == "messenger-api"
and server[0][2].get("http.route") == "/api/v1/users/me",
"server span with route template",
f"{len(spans)} spans: " + ", ".join(sorted({s[1] for s in spans})),
)
# встроенная телеметрия FastAPI выключена: span только наши
foreign = sorted({s[1] for s in spans if s[0] != "messenger-api"})
checks.add(not foreign, "no spans from FastAPI native telemetry", str(foreign))
api_logs = _poll(
lambda: _loki_lines(
grafana,
f'{{service_name="messenger-api"}} | request_id="{rid_me}"',
started,
)
)
linked = [
row
for row in api_logs
if api_traces and row[0].get("trace_id") == api_traces[0]
]
checks.add(
bool(linked),
"API log record in Loki with the same trace_id",
linked[0][1][:100] if linked else f"{len(api_logs)} records",
)
stdout = compose(
"logs", "--no-log-prefix", "--since", "15m", "api", check=False
).stdout
stdout_lines = [line for line in stdout.splitlines() if f"[rid={rid_me}]" in line]
checks.add(
any(
api_traces and f"trace_id={api_traces[0]}" in line for line in stdout_lines
),
"API stdout line carries the same trace_id",
stdout_lines[0][:140] if stdout_lines else "no line",
)
upload_traces = _poll(
lambda: _tempo_find(
grafana, f'{{ span.messenger.request_id = "{rid_upload}" }}', started
)
)
upload_spans = (
{s[1] for s in _spans(_tempo_trace(grafana, upload_traces[0]))}
if upload_traces
else set()
)
checks.add(
{"POST /api/v1/chats/{chat_id}/attachments", "S3 put_object"} <= upload_spans
and any(name.startswith("postgresql ") for name in upload_spans),
"upload trace has PostgreSQL and S3 child spans",
", ".join(sorted(upload_spans)),
)
# 4. трасса задания worker и его лог
job_traces = _poll(
lambda: _tempo_find(
grafana,
f'{{ resource.service.name = "messenger-worker" && '
f'span.messenger.attachment.id = "{attachment}" }}',
started,
)
)
job_trace = _tempo_trace(grafana, job_traces[0]) if job_traces else {}
job_spans = _spans(job_trace)
consumer = [s for s in job_spans if s[1].startswith("process ")]
checks.add(
bool(consumer) and consumer[0][2].get("messenger.job.outcome") == "ready",
"worker consumer span with outcome ready",
", ".join(sorted({s[1] for s in job_spans})) or "not found",
)
worker_logs = _poll(
lambda: _loki_lines(
grafana,
f'{{service_name="messenger-worker"}} |= "{attachment}" '
'| event="attachment processed"',
started,
)
)
checks.add(
any(
job_traces and row[0].get("trace_id") == job_traces[0]
for row in worker_logs
),
"worker log 'attachment processed' with the job trace_id",
worker_logs[0][1][:100] if worker_logs else "no record",
)
# 5. метрики в Prometheus
me_count = _poll(
lambda: _prom_value(
grafana,
'sum(http_server_request_duration_seconds_count{job="messenger-api",'
'http_route="/api/v1/users/me",http_response_status_code="200"})',
)
)
checks.add(me_count > 0, "HTTP duration histogram (API)", f"count {me_count:g}")
ready_count = _poll(
lambda: _prom_value(
grafana,
'sum(messenger_job_duration_seconds_count{job="messenger-worker",'
'messenger_job_outcome="ready"})',
)
)
checks.add(
ready_count > 0,
"job duration histogram (worker, ready)",
f"count {ready_count:g}",
)
published = _poll(
lambda: _prom_value(
grafana,
'sum(messenger_relay_jobs_total{messenger_relay_action="published"})',
)
)
checks.add(published > 0, "relay published counter", f"{published:g}")
info = _prom(grafana, 'target_info{job=~"messenger-.+"}')
services = sorted(
{
f"{row['metric'].get('job')}@{row['metric'].get('service_version')}"
f"/{row['metric'].get('deployment_environment_name')}"
for row in info
}
)
checks.add(
all(item.endswith("/lab") and "@None" not in item for item in services)
and len(services) == 3,
"resource attributes (target_info)",
", ".join(services),
)
status, series = grafana.source(
"metrics",
"/api/v1/series",
{
"match[]": '{__name__=~"http_server_request_duration_seconds_count'
'|messenger_.+_count|messenger_.+_total", job=~"messenger-.+"}',
"start": int(started),
},
)
# второй инструмент той же метрики (например, встроенная телеметрия
# FastAPI) за последнюю минуту не прибавлял
duplicate = _prom_value(
grafana,
"sum(rate(http_server_request_duration_seconds_count"
'{otel_scope_name!="messenger"}[1m]))',
)
checks.add(
duplicate == 0, "HTTP histogram has a single source", f"rate {duplicate:g}"
)
bad = set()
for row in (series or {}).get("data", []):
for key, value in row.items():
if any(part in key for part in FORBIDDEN_LABEL_PARTS) or UUID_RE.search(
value
):
bad.add(f"{key}={value}")
checks.add(
status == 200 and not bad,
"metric labels without ids, raw URLs and filenames",
f"{len((series or {}).get('data', []))} series"
+ (", bad: " + ", ".join(sorted(bad)[:5]) if bad else ""),
)
# 6. секретов и тел в телеметрии нет: ищем у себя в выгрузке, а не
# запросом к Loki — иначе пароль попал бы в журнал запросов
secrets_list = {
"password": password,
"token": token,
"message text": text_marker,
"filename": filename,
}
logs = _loki_lines(grafana, '{service_name=~"messenger-.+"}', started, limit=5000)
traces_blob = json.dumps([api_trace, job_trace]) + "".join(
json.dumps(_tempo_trace(grafana, trace_id))
for trace_id in _tempo_find(
grafana, '{ resource.service.name =~ "messenger-.+" }', started
)[:20]
)
logs_blob = json.dumps(logs)
leaked = [
f"{name} in {where}"
for name, value in secrets_list.items()
for where, blob in (("logs", logs_blob), ("traces", traces_blob))
if value in blob
]
authorization = "authorization" in traces_blob.lower()
checks.add(
not leaked and not authorization and bool(logs),
"no password, token, message text, filename or Authorization in telemetry",
f"{len(logs)} log records, {len(traces_blob)} bytes of traces"
+ (", leaked: " + ", ".join(leaked) if leaked else "")
+ (", Authorization in traces" if authorization else ""),
)
passed = sum(checks.results)
verdict = "PASS" if all(checks.results) else "FAIL"
print(f"telemetry-check: {verdict} ({passed}/{len(checks.results)} checks)")
return 0 if verdict == "PASS" else 1
# --- уровень 4: цепочка трассы, alerts, отказ хранилища телеметрии -------------
def _psql(sql: str) -> str:
done = compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
_env_value("POSTGRES_USER", "messenger"),
"-d",
_env_value("POSTGRES_DB", "messenger"),
"-At",
"-F",
" ",
"-c",
sql,
check=False,
)
return done.stdout.strip()
def _grafana_or_none():
password = _env_value("GRAFANA_ADMIN_PASSWORD", "")
if not password:
print("FAIL GRAFANA_ADMIN_PASSWORD is empty: run make env")
return None
return Grafana(_env_value("GRAFANA_HOST_PORT", "3180"), password)
def _actor_with_chat(base: str, label: str):
actor = register(base, label)
status, _, data = call(
base,
"POST",
"/api/v1/chats",
headers=bearer(actor),
body={"title": "Уровень 4", "member_ids": [actor["id"]]},
)
if status != 201:
raise SystemExit(f"create chat -> {status}")
return actor, json.loads(data)["id"]
def _upload(base: str, actor: dict, chat: str):
status, _, data = call(
base,
"POST",
f"/api/v1/chats/{chat}/attachments?filename=sample.png",
headers={**bearer(actor), "Content-Type": "image/png"},
raw=FIXTURE.read_bytes(),
)
return status, (json.loads(data)["id"] if status == 202 else "")
def _job_row(attachment: str) -> str:
"""attempt state last_error задания вложения (id проверен по шаблону)."""
if not UUID_RE.fullmatch(attachment):
return ""
return _psql(
"SELECT attempt, state, coalesce(last_error, '-') FROM attachment_jobs "
f"WHERE attachment_id = '{attachment}'"
)
def _trace_spans(trace: dict) -> list[dict]:
"""Span трассы Tempo с id, родителем, видом и атрибутами."""
found = []
body = trace.get("trace", trace)
for resource in body.get("resourceSpans") or body.get("batches") or []:
service = ""
for attr in resource.get("resource", {}).get("attributes", []):
if attr["key"] == "service.name":
service = attr["value"].get("stringValue", "")
for scope in resource.get("scopeSpans", []):
for span in scope.get("spans", []):
attrs = {
a["key"]: next(iter(a["value"].values()), None)
for a in span.get("attributes", [])
}
found.append(
{
"service": service,
"name": span.get("name", ""),
"id": span.get("spanId", ""),
"parent": span.get("parentSpanId", ""),
"kind": span.get("kind", ""),
"start": int(span.get("startTimeUnixNano", 0)),
"ms": (
int(span.get("endTimeUnixNano", 0))
- int(span.get("startTimeUnixNano", 0))
)
/ 1e6,
"error": span.get("status", {}).get("code")
== "STATUS_CODE_ERROR",
"attrs": attrs,
}
)
return found
def _print_tree(spans: list[dict]) -> None:
children: dict[str, list[dict]] = {}
ids = {span["id"] for span in spans}
for span in sorted(spans, key=lambda s: s["start"]):
key = span["parent"] if span["parent"] in ids else ""
children.setdefault(key, []).append(span)
def walk(parent: str, depth: int) -> None:
for span in children.get(parent, []):
attrs = span["attrs"]
extra = " ".join(
f"{key.split('.')[-1]}={attrs[key]}"
for key in ("messenger.job.attempt", "messenger.job.outcome")
if key in attrs
)
mark = " ERROR" if span["error"] else ""
print(
f" {' ' * depth}{span['service']}: {span['name']} "
f"[{span['kind'].replace('SPAN_KIND_', '').lower()}] "
f"{span['ms']:.1f} ms {extra}{mark}"
)
walk(span["id"], depth + 1)
walk("", 0)
def _descendants(spans: list[dict], root: str) -> list[dict]:
found, frontier = [], {root}
while frontier:
level = [span for span in spans if span["parent"] in frontier]
found.extend(level)
frontier = {span["id"] for span in level}
return found
def trace_chain(args: argparse.Namespace) -> int:
base = args.base_url
grafana = _grafana_or_none()
if grafana is None:
return 1
checks = Checks()
began = time.monotonic()
started = time.time()
actor, chat = _actor_with_chat(base, "trace")
compose("stop", "worker")
try:
status, attachment = _upload(base, actor, chat)
checks.add(status == 202, "upload accepted, worker stopped", f"status {status}")
if not attachment:
return 1
print(f"attachment_id={attachment}")
# первая попытка падает на чтении staging: S3 остановлен. После
# первого повтора worker снова останавливается, чтобы следующая
# попытка не сожгла бюджет, пока S3 поднимается
compose("stop", "s3")
try:
# `start` поднял бы и зависимость s3 (depends_on), --no-deps — нет
compose("up", "-d", "--no-deps", "worker")
def retried() -> str:
# queued с last_error — это retry_job; просто опубликованное
# тоже queued, но без ошибки
row = _job_row(attachment).split(" ", 2)
failed = len(row) == 3 and row[1] == "queued" and row[2] != "-"
return " ".join(row) if failed else ""
row = _poll(retried, timeout=60, pause=1)
checks.add(
bool(row),
"attempt 1 failed (S3 stopped), job back to queued",
row or "no retry",
)
compose("stop", "worker")
finally:
compose("up", "-d", "--wait", "s3")
finally:
compose("start", "worker")
meta = wait_status(base, actor, attachment, ("ready", "failed"), timeout=120)
checks.add(
meta["status"] == "ready", "attachment ready on a later attempt", meta["status"]
)
print(f"job: {_job_row(attachment)}")
query = f'{{span.messenger.attachment.id="{attachment}"}}'
def whole_trace():
ids = sorted(set(_tempo_find(grafana, query, started)))
if len(ids) != 1:
return None
spans = _trace_spans(_tempo_trace(grafana, ids[0]))
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
servers = [s for s in spans if s["kind"] == "SPAN_KIND_SERVER"]
ready = [
s for s in consumers if s["attrs"].get("messenger.job.outcome") == "ready"
]
if servers and ready and len(consumers) >= 2:
return ids[0], spans
return None
found = _poll(whole_trace, timeout=90)
ids = sorted(set(_tempo_find(grafana, query, started)))
checks.add(
bool(found), "one trace for the upload in Tempo", f"trace ids found: {len(ids)}"
)
if not found:
print(f"trace-chain: FAIL ({sum(checks.results)}/{len(checks.results)} checks)")
return 1
trace_id, spans = found
print(f"trace_id={trace_id} (Grafana → Explore → Tempo → {trace_id})")
_print_tree(spans)
by_id = {span["id"]: span for span in spans}
server = next(s for s in spans if s["kind"] == "SPAN_KIND_SERVER")
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
checks.add(
server["service"] == "messenger-api"
and server["name"].startswith("POST /api/v1/chats/{chat_id}/attachments"),
"root: API server span of the upload",
server["name"],
)
checks.add(
len(producers) >= 2
and all(
p["service"] == "messenger-relay" and p["parent"] == server["id"]
for p in producers
),
"relay producer spans are children of the upload span",
f"{len(producers)} publish spans",
)
linked = [
c
for c in consumers
if by_id.get(c["parent"], {}).get("kind") == "SPAN_KIND_PRODUCER"
and by_id[c["parent"]]["attrs"].get("messenger.job.attempt")
== c["attrs"].get("messenger.job.attempt")
]
attempts = sorted(
int(c["attrs"].get("messenger.job.attempt") or 0) for c in consumers
)
checks.add(
len(linked) == len(consumers) >= 2,
"worker consumer spans are children of the producer of the same attempt",
f"attempts {attempts}",
)
first = min(consumers, key=lambda s: s["start"])
checks.add(
first["attrs"].get("messenger.job.outcome") == "retry",
"attempt 1 outcome retry in the same trace",
str(first["attrs"].get("messenger.job.outcome")),
)
last = next(
c for c in consumers if c["attrs"].get("messenger.job.outcome") == "ready"
)
below = {span["name"] for span in _descendants(spans, last["id"])}
checks.add(
"S3 put_object" in below and "postgresql transaction" in below,
"ready attempt reaches S3 and PostgreSQL",
", ".join(sorted(below)),
)
return _finish("trace-chain", checks, began)
def _finish(name: str, checks: Checks, began: float) -> int:
verdict = "PASS" if all(checks.results) else "FAIL"
passed, total = sum(checks.results), len(checks.results)
took = time.monotonic() - began
print(f"{name}: {verdict} ({passed}/{total} checks, {took:.0f} s)")
return 0 if verdict == "PASS" else 1
def _alerts(grafana: Grafana) -> dict[str, str]:
"""alertname → state (pending/firing) из Prometheus; нет в списке — resolved."""
status, data = grafana.source("metrics", "/api/v1/alerts")
if status != 200 or not data:
return {}
return {a["labels"]["alertname"]: a["state"] for a in data["data"]["alerts"]}
def _wait_alert(
grafana: Grafana, name: str, firing: bool, timeout: float
) -> float | None:
"""Секунды до firing (или до ухода из списка); None — не дождались."""
began = time.monotonic()
last = None
while time.monotonic() - began < timeout:
state = _alerts(grafana).get(name, "inactive")
if state != last:
print(f" {time.monotonic() - began:5.0f} s {name}: {state}")
last = state
if (state == "firing") == firing and (firing or state == "inactive"):
return time.monotonic() - began
time.sleep(5)
return None
class _Load:
"""Фоновая нагрузка GET /users/me: статусы ответов по фазам."""
def __init__(self, base: str, actor: dict, threads: int = 4) -> None:
import threading
self.base, self.actor = base, actor
self.statuses: Counter = Counter()
self.stop = threading.Event()
self.threads = [
threading.Thread(target=self._run, daemon=True) for _ in range(threads)
]
def _run(self) -> None:
while not self.stop.is_set():
try:
status, _, _ = call(
self.base, "GET", "/api/v1/users/me", headers=bearer(self.actor)
)
except OSError as error:
status = type(error).__name__
self.statuses[status] += 1
time.sleep(0.2)
def __enter__(self):
for thread in self.threads:
thread.start()
return self
def take(self) -> dict:
seen, self.statuses = dict(self.statuses), Counter()
return seen
def __exit__(self, *_exc) -> None:
self.stop.set()
for thread in self.threads:
thread.join(timeout=15)
LOCK_SQL = (
"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;"
)
def alerts_check(args: argparse.Namespace) -> int:
base = args.base_url
grafana = _grafana_or_none()
if grafana is None:
return 1
checks = Checks()
began = time.monotonic()
rules = grafana.source("metrics", "/api/v1/rules")[1] or {}
names = sorted(
rule["name"]
for group in rules.get("data", {}).get("groups", [])
for rule in group["rules"]
if rule.get("type") == "alerting"
)
checks.add(
{"MessengerHighErrorRate", "MessengerQueueLagGrowing"} <= set(names),
"alert rules loaded in Prometheus",
", ".join(names),
)
actor, chat = _actor_with_chat(base, "alerts")
# (а) worker остановлен → возраст задач растёт → queue lag
print("(a) worker stopped: oldest job age grows")
phase = time.monotonic()
compose("stop", "worker")
try:
uploads = [_upload(base, actor, chat) for _ in range(3)]
checks.add(
all(s == 202 for s, _ in uploads), "3 uploads accepted with worker stopped"
)
fired = _wait_alert(grafana, "MessengerQueueLagGrowing", True, 300)
depth = _prom_value(grafana, "sum(messenger_job_queue_depth)")
age = _prom_value(grafana, "max(messenger_job_queue_oldest_age_seconds)")
checks.add(
fired is not None,
"MessengerQueueLagGrowing firing",
f"after {fired or 0:.0f} s; backlog {depth:.0f} jobs, oldest {age:.0f} s",
)
finally:
compose("start", "worker")
resolved = _wait_alert(grafana, "MessengerQueueLagGrowing", False, 240)
after = _prom_value(grafana, "sum(messenger_job_queue_depth)")
checks.add(
resolved is not None and after < depth,
"worker started: backlog drained, alert resolved",
f"after {resolved or 0:.0f} s; backlog {depth:.0f} -> {after:.0f}",
)
print(f" phase (a): {time.monotonic() - phase:.0f} s")
# (б) таблица сессий под блокировкой: каждый запрос с токеном ждёт
# statement_timeout и получает 503 — «медленная база»
print("(b) sessions table locked: authenticated requests wait and get 503")
phase = time.monotonic()
with _Load(base, actor) as load:
time.sleep(10)
print(f" baseline statuses: {load.take()}")
locker = subprocess.Popen(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
_env_value("POSTGRES_USER", "messenger"),
"-d",
_env_value("POSTGRES_DB", "messenger"),
"-c",
LOCK_SQL,
],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
fired = _wait_alert(grafana, "MessengerHighErrorRate", True, 300)
ratio = _prom_value(grafana, "messenger:http_errors:ratio_rate2m")
waiting = _prom_value(
grafana,
'max(db_client_connection_pending_requests{job="messenger-api"})',
)
used = _prom_value(grafana, USED_EXPR)
checks.add(
fired is not None,
"MessengerHighErrorRate firing",
f"after {fired or 0:.0f} s; 5xx ratio {ratio:.2f}, "
f"pool used {used:.0f}, waiting {waiting:.0f}; "
f"statuses {load.take()}",
)
finally:
_psql(
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE query LIKE '%pg_sleep(600)%' AND pid <> pg_backend_pid()"
)
locker.wait(timeout=30)
# запросы, начатые до снятия блокировки, ещё доходят с 503
time.sleep(8)
print(f" statuses around release: {load.take()}")
resolved = _wait_alert(grafana, "MessengerHighErrorRate", False, 300)
statuses = load.take()
checks.add(
resolved is not None and set(statuses) <= {200},
"lock released: errors stopped, alert resolved",
f"after {resolved or 0:.0f} s; statuses after release {statuses}",
)
print(f" phase (b): {time.monotonic() - phase:.0f} s")
return _finish("alerts-check", checks, began)
USED_EXPR = (
'sum(db_client_connection_count{job="messenger-api", '
'db_client_connection_state="used"})'
)
def _container_mib(service: str) -> float:
cid = compose("ps", "-q", service, check=False).stdout.strip()
if not cid:
return 0.0
done = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", cid],
capture_output=True,
text=True,
check=False,
)
used = (done.stdout.split("/")[0] or "0").strip()
number = float(re.sub(r"[^0-9.]", "", used) or 0)
return number * {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024}.get(
re.sub(r"[0-9.]", "", used), 1
)
# Collector 0.164 отдаёт свои счётчики без суффикса _total; enqueue_failed
# появляется только после первого отказа очереди
LOSS_EXPR = (
'(sum(otelcol_exporter_send_failed_spans{exporter="otlp_http/tempo"}) or vector(0))'
' + (sum(otelcol_exporter_enqueue_failed_spans{exporter="otlp_http/tempo"})'
" or vector(0))"
)
SENT_EXPR = 'sum(otelcol_exporter_sent_spans{exporter="otlp_http/tempo"})'
QUEUE_EXPR = 'sum(otelcol_exporter_queue_size{exporter="otlp_http/tempo"})'
def telemetry_outage(args: argparse.Namespace) -> int:
base = args.base_url
grafana = _grafana_or_none()
if grafana is None:
return 1
checks = Checks()
began = time.monotonic()
actor, chat = _actor_with_chat(base, "outage")
lost_before = _prom_value(grafana, LOSS_EXPR)
memory = _container_mib("otel-collector")
print(f"before: collector {memory:.0f} MiB, spans lost so far {lost_before:.0f}")
compose("stop", "tempo")
peak, queue_peak, uploads = 0.0, 0.0, []
try:
with _Load(base, actor, threads=2) as load:
deadline = time.monotonic() + args.seconds
while time.monotonic() < deadline:
uploads.append(_upload(base, actor, chat))
peak = max(peak, _container_mib("otel-collector"))
queue_peak = max(queue_peak, _prom_value(grafana, QUEUE_EXPR))
time.sleep(10)
statuses = load.take()
ready = [
wait_status(base, actor, a, ("ready", "failed"), timeout=90)["status"]
for s, a in uploads
if s == 202
]
checks.add(
set(statuses) <= {200},
f"API answers during {args.seconds} s without Tempo",
f"GET /users/me statuses {statuses}",
)
checks.add(
len(ready) == len(uploads) and set(ready) == {"ready"},
"uploads accepted and processed without Tempo",
f"{len(uploads)} uploads, ready {ready.count('ready')}",
)
limit = 256.0
checks.add(
0 < peak < limit,
"Collector memory bounded",
f"peak {peak:.0f} MiB of {limit:.0f} MiB limit; "
f"export queue peak {queue_peak:.0f} of 20000 items",
)
finally:
compose("start", "tempo")
# счётчики отказов Collector доходят до Prometheus за scrape
lost = _poll(
lambda: max(_prom_value(grafana, LOSS_EXPR) - lost_before, 0),
timeout=90,
pause=5,
)
checks.add(
bool(lost),
"lost spans are counted by the Collector",
f"{lost or 0:.0f} spans (send_failed + enqueue_failed, otlp_http/tempo)",
)
sent_before = _prom_value(grafana, SENT_EXPR)
recovered = _poll(
lambda: (
_prom_value(grafana, SENT_EXPR) > sent_before
and _prom_value(grafana, QUEUE_EXPR) < 100
),
timeout=120,
pause=5,
)
checks.add(
bool(recovered),
"Tempo back: export resumes, queue drained",
f"queue now {_prom_value(grafana, QUEUE_EXPR):.0f} items",
)
return _finish("telemetry-outage", checks, began)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
commands = parser.add_subparsers(dest="command", required=True)
commands.add_parser("env")
one = commands.add_parser("instances")
one.add_argument("--base-url", default="http://localhost:8080")
one.add_argument("--requests", type=int, default=12)
commands.add_parser("lb-stats")
each = commands.add_parser("failover-each")
each.add_argument("--base-url", default="http://localhost:8080")
each.add_argument("--replicas", type=int, default=2)
each.add_argument("--mode", choices=("stop", "kill"), default="stop")
attachment = commands.add_parser("attachment-failover")
attachment.add_argument("--base-url", default="http://localhost:8080")
attachment.add_argument("--replicas", type=int, default=2)
attachment.add_argument("--mode", choices=("stop", "kill"), default="stop")
commands.add_parser("s3-check")
scenario = commands.add_parser("defense")
scenario.add_argument("--base-url", default="http://localhost:8080")
commands.add_parser("pipeline")
telemetry = commands.add_parser("telemetry-check")
telemetry.add_argument("--base-url", default="http://localhost:8080")
for name in ("trace-chain", "alerts-check", "telemetry-outage"):
level4 = commands.add_parser(name)
level4.add_argument("--base-url", default="http://localhost:8080")
if name == "telemetry-outage":
level4.add_argument("--seconds", type=int, default=120)
args = parser.parse_args()
handler = {
"env": env,
"instances": instances,
"lb-stats": lb_stats,
"failover-each": failover_each,
"attachment-failover": attachment_failover,
"s3-check": s3_check,
"defense": defense,
"pipeline": pipeline,
"telemetry-check": telemetry_check,
"trace-chain": trace_chain,
"alerts-check": alerts_check,
"telemetry-outage": telemetry_outage,
}
return handler[args.command](args)
if __name__ == "__main__":
sys.exit(main())