1867 lines
72 KiB
Python
Executable File
1867 lines
72 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Операторские команды стенда лабы 6 (свои, не skeleton). Python 3.9+, stdlib.
|
||
|
||
python3 scripts/lab6.py env
|
||
.env из .env.example со случайными паролями и ключами S3 (make env);
|
||
несекретные переменные, заданные в окружении (`make env
|
||
HOST_PORT=8180`), берутся оттуда — так стенд встаёт рядом с чужим
|
||
на других портах. Существующий .env не перезаписывается,
|
||
дописываются только переменные шаблона, которых в нём нет.
|
||
Печатает только имена;
|
||
python3 scripts/lab5.py instances [--base-url URL] [--requests 12]
|
||
вход одним синтетическим пользователем и серия GET /users/me с одним
|
||
токеном через LB: X-Instance-Id каждого ответа и итог по репликам
|
||
(шаг 1 сценария защиты лабы 3);
|
||
python3 scripts/lab5.py lb-stats
|
||
слоты HAProxy рабочего стенда: адрес, состояние проверки, запросы в
|
||
полёте, всего, ошибки соединения, повторы, перенаправления;
|
||
python3 scripts/lab5.py failover-each [--base-url URL] [--replicas 2]
|
||
[--mode stop|kill]
|
||
make test-failover для каждой реплики по очереди: перед прогоном все
|
||
реплики возвращены (make replicas), тест останавливает реплику,
|
||
ответившую на его первую запись; если это реплика, которую уже
|
||
проверили, очередь LB сдвигается одним запросом и прогон
|
||
повторяется. Печатает docker compose ps до и после, строку
|
||
STOP_ONE_LOG и итог теста — это evidence «какой контейнер
|
||
действительно остановлен»;
|
||
python3 scripts/lab5.py attachment-failover [--base-url URL]
|
||
[--replicas 2] [--mode stop|kill]
|
||
загрузка tests/fixtures/sample.png через LB (202, затем ready),
|
||
сообщение со ссылкой, остановка именно реплики, принявшей загрузку
|
||
(scripts/stop-one.sh по X-Instance-Id ответа), скачивание вторым
|
||
участником через другую реплику — тот же SHA-256, 404 постороннему;
|
||
затем реплики возвращаются;
|
||
python3 scripts/lab5.py defense [--base-url URL]
|
||
сценарий защиты лабы 5, шаги 2–3: worker остановлен → загрузка 202 →
|
||
сообщение со ссылкой 201 → content 409, текст 201 → scripts/restart.sh
|
||
(LB, реплики, PostgreSQL, Valkey, S3, брокер) → worker запущен → ready,
|
||
оригинал и thumbnail 256×160. Печатает id и StartedAt без токенов;
|
||
python3 scripts/lab5.py pipeline
|
||
задания по состояниям, возраст старейшего, сообщения в очереди;
|
||
python3 scripts/lab5.py s3-check
|
||
приватность bucket вне API (make s3-check): bootstrap в режиме
|
||
check (права ключей API, worker и уборщика, чужой bucket,
|
||
административные запросы), анонимные запросы с хоста и из сети
|
||
стенда, порты контейнера S3 из сети, затем cleanup. Таблица «где →
|
||
проверка → ожидаемо → получено», код 1 при нарушении;
|
||
python3 scripts/lab6.py telemetry-check [--base-url URL]
|
||
лаба 6 (make telemetry-check): provisioning Grafana; запрос со своим
|
||
X-Request-Id → трасса в Tempo, запись в Loki и строка stdout с тем
|
||
же trace_id; загрузка → span PostgreSQL и S3, трасса worker с
|
||
исходом ready и запись «attachment processed» с её trace_id;
|
||
метрики API, worker и relay и ресурс в Prometheus, метки без id;
|
||
пароля, токена, текста сообщения, имени файла и Authorization нет
|
||
ни в логах, ни в трассах. Хранилища — через прокси источников
|
||
Grafana (127.0.0.1:GRAFANA_HOST_PORT, пароль admin из .env). Итог
|
||
PASS/FAIL по каждой проверке, код 1 при FAIL.
|
||
python3 scripts/lab6.py trace-chain [--base-url URL]
|
||
уровень 4 (make trace-chain, ~1 мин): worker остановлен → загрузка
|
||
202 → S3 остановлен → worker берёт задание, попытка 1 — retry →
|
||
S3 и worker обратно → ready. В Tempo по id вложения ровно одна
|
||
трасса: server span загрузки → producer span relay на каждую
|
||
публикацию → consumer span worker той же попытки → S3 и
|
||
PostgreSQL готовой попытки. Печатает дерево span'ов;
|
||
python3 scripts/lab6.py alerts-check [--base-url URL]
|
||
уровень 4 (make alerts-check, ~8–10 мин): (а) worker остановлен,
|
||
3 загрузки → MessengerQueueLagGrowing firing → worker запущен →
|
||
backlog разобран, resolved; (б) таблица sessions под ACCESS
|
||
EXCLUSIVE блокировкой при фоновой нагрузке GET /users/me → 503 по
|
||
statement_timeout → MessengerHighErrorRate firing → блокировка
|
||
снята → resolved. Состояния — Prometheus /api/v1/alerts;
|
||
python3 scripts/lab6.py telemetry-outage [--base-url URL] [--seconds 120]
|
||
уровень 4 (make telemetry-outage, ~3–4 мин): Tempo остановлен под
|
||
нагрузкой (GET /users/me и загрузка раз в 10 с): ответы и ready
|
||
без отказов, пик памяти Collector меньше лимита контейнера,
|
||
потерянные span'ы посчитаны счётчиками Collector, после возврата
|
||
Tempo экспорт идёт и очередь пуста.
|
||
|
||
Пароли синтетических пользователей и токены живут только в памяти и не
|
||
печатаются. Работает с проектом compose репозитория (compose.yaml и .env)
|
||
или с тем, что задают COMPOSE_PROJECT_NAME / COMPOSE_FILE /
|
||
COMPOSE_ENV_FILES (стенды student-tests/stand.py).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import secrets
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||
LB_STATS = (
|
||
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
|
||
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
|
||
)
|
||
STAT_FIELDS = ("addr", "status", "scur", "stot", "econ", "eresp", "wretr", "wredis")
|
||
# пустые значения этих переменных шаблона — секреты: make env их генерирует
|
||
SECRET_SUFFIXES = ("_PASSWORD", "_KEY")
|
||
# порты API S3 и gRPC шлюза: единственные, что открыты в сеть стенда
|
||
S3_NETWORK_PORTS = ("8333", "18333")
|
||
|
||
|
||
def call(base: str, method: str, path: str, headers=None, body=None, raw=None):
|
||
data = json.dumps(body).encode() if body is not None else raw
|
||
request = urllib.request.Request(base + path, data=data, method=method)
|
||
for key, value in (headers or {}).items():
|
||
request.add_header(key, value)
|
||
if body is not None:
|
||
request.add_header("Content-Type", "application/json")
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=10) as response:
|
||
return response.status, dict(response.headers), response.read()
|
||
except urllib.error.HTTPError as error:
|
||
return error.code, dict(error.headers), error.read()
|
||
|
||
|
||
def register(base: str, label: str = "ops") -> dict:
|
||
"""Синтетический пользователь с паролем и входом: id и токен."""
|
||
username = f"{label}_" + secrets.token_hex(6)
|
||
password = secrets.token_urlsafe(18)
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/users",
|
||
body={"username": username, "display_name": "Оператор", "password": password},
|
||
)
|
||
if status != 201:
|
||
raise SystemExit(f"register -> {status}")
|
||
user_id = json.loads(data)["id"]
|
||
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/auth/sessions",
|
||
headers={"Authorization": "Basic " + basic},
|
||
)
|
||
if status != 201:
|
||
raise SystemExit(f"login -> {status}")
|
||
return {"id": user_id, "token": json.loads(data)["token"]}
|
||
|
||
|
||
def login(base: str) -> str:
|
||
return register(base)["token"]
|
||
|
||
|
||
def bearer(actor: dict) -> dict:
|
||
return {"Authorization": "Bearer " + actor["token"]}
|
||
|
||
|
||
def instance_of(headers: dict) -> str:
|
||
return headers.get("X-Instance-Id") or headers.get("x-instance-id") or "-"
|
||
|
||
|
||
# --- env ------------------------------------------------------------------------
|
||
|
||
|
||
def _assignments(text: str) -> list[tuple[str, str]]:
|
||
pairs = []
|
||
for line in text.splitlines():
|
||
match = re.match(r"([A-Z][A-Z0-9_]*)=(.*)$", line)
|
||
if match:
|
||
pairs.append((match.group(1), match.group(2)))
|
||
return pairs
|
||
|
||
|
||
def _value(name: str, value: str) -> str:
|
||
"""Значение для нового .env: секрет — случайный, несекретная переменная
|
||
из окружения (`make env HOST_PORT=8180`) — как задано, иначе шаблон."""
|
||
if name.endswith(SECRET_SUFFIXES):
|
||
# ключи S3 и пароли — случайный hex; длина ключа S3 — 64 символа
|
||
return value or secrets.token_hex(32 if name.startswith("S3_") else 16)
|
||
return os.environ.get(name, value)
|
||
|
||
|
||
def env(_args: argparse.Namespace) -> int:
|
||
template = (ROOT / ".env.example").read_text(encoding="utf-8")
|
||
target = ROOT / ".env"
|
||
if not target.exists():
|
||
lines = []
|
||
generated = []
|
||
for line in template.splitlines():
|
||
match = re.match(r"([A-Z][A-Z0-9_]*)=(.*)$", line)
|
||
if match:
|
||
value = _value(match.group(1), match.group(2))
|
||
if value != match.group(2) and match.group(1).endswith(SECRET_SUFFIXES):
|
||
generated.append(match.group(1))
|
||
line = f"{match.group(1)}={value}"
|
||
lines.append(line)
|
||
target.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
target.chmod(0o600)
|
||
print(f".env created; generated: {' '.join(generated) or '-'}")
|
||
return 0
|
||
existing = {name for name, _ in _assignments(target.read_text(encoding="utf-8"))}
|
||
missing = [(n, v) for n, v in _assignments(template) if n not in existing]
|
||
if missing:
|
||
with target.open("a", encoding="utf-8") as handle:
|
||
handle.write("\n# added by make env from .env.example\n")
|
||
for name, value in missing:
|
||
handle.write(f"{name}={_value(name, value)}\n")
|
||
print("added to existing .env: " + " ".join(name for name, _ in missing))
|
||
else:
|
||
print(".env ready")
|
||
return 0
|
||
|
||
|
||
def instances(args: argparse.Namespace) -> int:
|
||
token = login(args.base_url)
|
||
seen: Counter = Counter()
|
||
print(f"one session, {args.requests} x GET /api/v1/users/me via {args.base_url}")
|
||
for number in range(1, args.requests + 1):
|
||
status, headers, _ = call(
|
||
args.base_url,
|
||
"GET",
|
||
"/api/v1/users/me",
|
||
headers={"Authorization": "Bearer " + token},
|
||
)
|
||
instance = headers.get("X-Instance-Id") or headers.get("x-instance-id") or "-"
|
||
seen[instance] += 1
|
||
print(f" {number:2d}: {status} X-Instance-Id={instance}")
|
||
print(f"replicas seen: {len(seen)} {dict(seen)}")
|
||
return 0 if len(seen) >= 2 else 1
|
||
|
||
|
||
def compose(*args: str, check: bool = True) -> subprocess.CompletedProcess:
|
||
return subprocess.run(
|
||
["docker", "compose", *args],
|
||
cwd=ROOT,
|
||
capture_output=True,
|
||
text=True,
|
||
check=check,
|
||
)
|
||
|
||
|
||
def lb_rows() -> list[dict]:
|
||
done = compose("exec", "-T", "haproxy", "bash", "-c", LB_STATS)
|
||
lines = [
|
||
line for line in done.stdout.splitlines() if line.startswith(("# ", "api,"))
|
||
]
|
||
header = lines[0][2:].split(",")
|
||
rows = []
|
||
for line in lines[1:]:
|
||
# zip(strict=) — только с Python 3.10, скрипт работает и на 3.9
|
||
row = dict(zip(header, line.split(","))) # noqa: B905
|
||
if row["svname"] in ("BACKEND", "FRONTEND") or not row.get("addr"):
|
||
continue
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def lb_stats(_args: argparse.Namespace) -> int:
|
||
print("slot " + " ".join(f"{field:>12}" for field in STAT_FIELDS))
|
||
for row in lb_rows():
|
||
print(
|
||
f"{row['svname']:<9} "
|
||
+ " ".join(f"{row.get(f, ''):>12}" for f in STAT_FIELDS)
|
||
)
|
||
return 0
|
||
|
||
|
||
def wait_lb_up(replicas: int, timeout: float = 30) -> bool:
|
||
"""Все реплики UP в LB: `--wait` compose ждёт только healthcheck
|
||
контейнера, а слот вернувшейся реплики включается после проверок LB."""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
up = [row for row in lb_rows() if row.get("status") == "UP"]
|
||
if len(up) >= replicas:
|
||
return True
|
||
time.sleep(0.5)
|
||
return False
|
||
|
||
|
||
def ps() -> str:
|
||
return compose(
|
||
"ps", "-a", "--format", "table {{.Name}}\t{{.ID}}\t{{.State}}\t{{.Status}}"
|
||
).stdout
|
||
|
||
|
||
def failover_each(args: argparse.Namespace) -> int:
|
||
done: set[str] = set()
|
||
log = Path(tempfile.mkdtemp(prefix="stop-one-")) / "stop-one.log"
|
||
env = {**os.environ, "STOP_ONE_LOG": str(log), "STOP_MODE": args.mode}
|
||
attempt = 0
|
||
while len(done) < args.replicas and attempt < args.replicas * 4:
|
||
attempt += 1
|
||
replicas = compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-recreate",
|
||
"--wait",
|
||
"--scale",
|
||
f"api={args.replicas}",
|
||
"api",
|
||
check=False,
|
||
)
|
||
if replicas.returncode != 0 or not wait_lb_up(args.replicas):
|
||
print(replicas.stderr or "replicas are not UP in the LB")
|
||
return 2
|
||
# сдвиг очереди round-robin: тест останавливает реплику, ответившую
|
||
# на его первую запись; с каждой попыткой сдвиг на запрос больше
|
||
for _ in range(attempt - 1):
|
||
call(args.base_url, "GET", "/health/live")
|
||
print(f"=== attempt {attempt}: docker compose ps before\n{ps()}", flush=True)
|
||
before = log.read_text().splitlines() if log.exists() else []
|
||
test = subprocess.run(
|
||
[
|
||
sys.executable,
|
||
"tests/resilience.py",
|
||
"--mode",
|
||
"failover",
|
||
"--action-script",
|
||
"scripts/stop-one.sh",
|
||
"--base-url",
|
||
args.base_url,
|
||
],
|
||
cwd=ROOT,
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
print(
|
||
"$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh "
|
||
f"(STOP_MODE={args.mode})"
|
||
)
|
||
print(test.stdout.strip())
|
||
lines = log.read_text().splitlines() if log.exists() else []
|
||
new = lines[len(before) :]
|
||
print("STOP_ONE_LOG: " + (new[-1] if new else "(no line)"))
|
||
print(f"docker compose ps after\n{ps()}", flush=True)
|
||
if test.returncode != 0:
|
||
return test.returncode
|
||
stopped = new[-1].split("container=")[1].split()[0][:12] if new else ""
|
||
if stopped:
|
||
done.add(stopped)
|
||
compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-recreate",
|
||
"--wait",
|
||
"--scale",
|
||
f"api={args.replicas}",
|
||
"api",
|
||
)
|
||
print(f"replicas stopped one by one: {sorted(done)}; all restored:\n{ps()}")
|
||
return 0 if len(done) >= args.replicas else 1
|
||
|
||
|
||
# --- вложение и отказ принявшей реплики ---------------------------------------
|
||
|
||
|
||
def wait_status(
|
||
base: str, actor: dict, attachment_id: str, wanted: tuple, timeout: float = 60
|
||
) -> dict:
|
||
"""Опрос метаданных, пока статус не войдёт в `wanted`."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
status, _, data = call(
|
||
base, "GET", f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
|
||
)
|
||
if status == 200:
|
||
meta = json.loads(data)
|
||
if meta["status"] in wanted:
|
||
return meta
|
||
if time.monotonic() > deadline:
|
||
raise SystemExit(f"attachment did not reach {wanted} in {timeout:.0f} s")
|
||
time.sleep(0.3)
|
||
|
||
|
||
def restore_replicas(replicas: int) -> None:
|
||
compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-recreate",
|
||
"--wait",
|
||
"--scale",
|
||
f"api={replicas}",
|
||
"api",
|
||
)
|
||
wait_lb_up(replicas)
|
||
|
||
|
||
def attachment_failover(args: argparse.Namespace) -> int:
|
||
base = args.base_url
|
||
restore_replicas(args.replicas)
|
||
alice, bob, eve = (register(base, label) for label in ("alice", "bob", "eve"))
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/chats",
|
||
headers=bearer(alice),
|
||
body={"title": "Отказ реплики", "member_ids": [alice["id"], bob["id"]]},
|
||
)
|
||
if status != 201:
|
||
raise SystemExit(f"create chat -> {status}")
|
||
chat = json.loads(data)["id"]
|
||
content = FIXTURE.read_bytes()
|
||
expected = hashlib.sha256(content).hexdigest()
|
||
status, headers, data = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/attachments?filename=sample.png",
|
||
headers={**bearer(alice), "Content-Type": "image/png"},
|
||
raw=content,
|
||
)
|
||
if status != 202:
|
||
raise SystemExit(f"upload -> {status} {data[:200]!r}")
|
||
attachment = json.loads(data)
|
||
uploader = instance_of(headers)
|
||
print(
|
||
f"upload: 202 {attachment['status']} X-Instance-Id={uploader} "
|
||
f"size={attachment['size_bytes']} sha256={attachment['sha256'][:16]}… "
|
||
f"(fixture {expected[:16]}…)"
|
||
)
|
||
ready = wait_status(base, alice, attachment["id"], ("ready", "failed"))
|
||
print(f"processed: {ready['status']} variants={len(ready['variants'])}")
|
||
status, headers, data = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/messages",
|
||
headers={**bearer(alice), "Idempotency-Key": uuid.uuid4().hex},
|
||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||
)
|
||
print(
|
||
f"message with attachment: {status} attachment_ids="
|
||
f"{json.loads(data).get('attachment_ids') if status == 201 else '-'}"
|
||
)
|
||
log = Path(tempfile.mkdtemp(prefix="stop-one-")) / "stop-one.log"
|
||
print(f"=== docker compose ps before\n{ps()}", flush=True)
|
||
stop = subprocess.run(
|
||
["scripts/stop-one.sh"],
|
||
cwd=ROOT,
|
||
env={
|
||
**os.environ,
|
||
"TARGET_INSTANCE_ID": uploader,
|
||
"STOP_MODE": args.mode,
|
||
"STOP_ONE_LOG": str(log),
|
||
},
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
print(f"$ TARGET_INSTANCE_ID={uploader} STOP_MODE={args.mode} scripts/stop-one.sh")
|
||
stopped = log.read_text().strip() if log.exists() else "-"
|
||
print(f"exit {stop.returncode}; STOP_ONE_LOG: {stopped}")
|
||
if stop.returncode != 0:
|
||
print(stop.stderr.strip())
|
||
return 1
|
||
# LB исключает реплику за ≤ 2,5 с; ответы ещё могут прийти от неё —
|
||
# ждём ответа другой реплики не дольше 10 с
|
||
deadline = time.monotonic() + 10
|
||
started = time.monotonic()
|
||
result = None
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
status, headers, data = call(
|
||
base,
|
||
"GET",
|
||
f"/api/v1/attachments/{attachment['id']}/content",
|
||
headers=bearer(bob),
|
||
)
|
||
except OSError:
|
||
status, headers, data = 0, {}, b""
|
||
if status == 200 and instance_of(headers) != uploader:
|
||
result = (status, headers, data)
|
||
break
|
||
time.sleep(0.2)
|
||
ok = result is not None
|
||
if result is not None:
|
||
status, headers, data = result
|
||
digest = hashlib.sha256(data).hexdigest()
|
||
ok = digest == expected and instance_of(headers) != uploader
|
||
kind = headers.get("content-type") or headers.get("Content-Type")
|
||
print(
|
||
f"download by bob after stop: {status} "
|
||
f"X-Instance-Id={instance_of(headers)} "
|
||
f"after {time.monotonic() - started:.2f}s, bytes={len(data)}, "
|
||
f"sha256 {'== fixture' if digest == expected else '!= fixture'}, "
|
||
f"content-type={kind}"
|
||
)
|
||
else:
|
||
print("download by bob after stop: no 200 from another replica in 10 s")
|
||
for suffix in ("", "/content"):
|
||
status, _, _ = call(
|
||
base,
|
||
"GET",
|
||
f"/api/v1/attachments/{attachment['id']}{suffix}",
|
||
headers=bearer(eve),
|
||
)
|
||
print(f"outsider GET /attachments/{{id}}{suffix}: {status}")
|
||
ok = ok and status == 404
|
||
print(f"=== docker compose ps after\n{ps()}", flush=True)
|
||
restore_replicas(args.replicas)
|
||
print(f"replicas restored:\n{ps()}")
|
||
print("PASS" if ok else "FAIL")
|
||
return 0 if ok else 1
|
||
|
||
|
||
# --- лаба 5: сценарий защиты и состояние pipeline ------------------------------
|
||
|
||
|
||
def started_at(service: str) -> str:
|
||
ids = compose("ps", "-q", service, check=False).stdout.split()
|
||
if not ids:
|
||
return f"{service}: not running"
|
||
done = subprocess.run(
|
||
["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return "; ".join(line.strip().lstrip("/") for line in done.stdout.splitlines())
|
||
|
||
|
||
def jpeg_size(data: bytes) -> tuple:
|
||
"""Размеры JPEG по маркеру SOF (как tests/image_probe.py)."""
|
||
i = 2
|
||
while i + 9 < len(data):
|
||
if data[i] != 0xFF:
|
||
i += 1
|
||
continue
|
||
marker = data[i + 1]
|
||
length = int.from_bytes(data[i + 2 : i + 4], "big")
|
||
if marker in (0xC0, 0xC1, 0xC2):
|
||
height = int.from_bytes(data[i + 5 : i + 7], "big")
|
||
width = int.from_bytes(data[i + 7 : i + 9], "big")
|
||
return width, height
|
||
i += 2 + length
|
||
return (0, 0)
|
||
|
||
|
||
def defense(args: argparse.Namespace) -> int:
|
||
base = args.base_url
|
||
services = ("api", "postgres", "valkey", "s3", "rabbitmq", "worker", "relay")
|
||
print("$ docker compose stop worker")
|
||
compose("stop", "worker")
|
||
ok = True
|
||
try:
|
||
alice, bob = (register(base, label) for label in ("alice", "bob"))
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/chats",
|
||
headers=bearer(alice),
|
||
body={"title": "Защита", "member_ids": [alice["id"], bob["id"]]},
|
||
)
|
||
chat = json.loads(data)["id"]
|
||
content = FIXTURE.read_bytes()
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/attachments?filename=sample.png",
|
||
headers={**bearer(alice), "Content-Type": "image/png"},
|
||
raw=content,
|
||
)
|
||
attachment = json.loads(data)
|
||
print(f"worker stopped; upload -> {status} {attachment.get('status')}")
|
||
print(f"attachment_id={attachment.get('id')}")
|
||
ok = ok and status == 202
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/messages",
|
||
headers={**bearer(alice), "Idempotency-Key": uuid.uuid4().hex},
|
||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||
)
|
||
print(f"message with queued attachment -> {status}")
|
||
ok = ok and status == 201
|
||
status, _, _ = call(
|
||
base,
|
||
"GET",
|
||
f"/api/v1/attachments/{attachment['id']}/content",
|
||
headers=bearer(bob),
|
||
)
|
||
print(f"content while worker is stopped -> {status}")
|
||
ok = ok and status == 409
|
||
status, _, _ = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/messages",
|
||
headers={**bearer(bob), "Idempotency-Key": uuid.uuid4().hex},
|
||
body={"text": "текст без вложения"},
|
||
)
|
||
print(f"plain text message meanwhile -> {status}")
|
||
ok = ok and status == 201
|
||
print("StartedAt before restart:")
|
||
for service in services:
|
||
print(" " + started_at(service))
|
||
restart = subprocess.run(
|
||
["scripts/restart.sh"],
|
||
cwd=ROOT,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
print(f"$ scripts/restart.sh -> exit {restart.returncode}")
|
||
ok = ok and restart.returncode == 0
|
||
deadline = time.monotonic() + 90
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
if call(base, "GET", "/health/ready")[0] == 200:
|
||
break
|
||
except OSError:
|
||
pass
|
||
time.sleep(0.5)
|
||
meta = wait_status(base, bob, attachment["id"], ("queued", "processing"), 30)
|
||
print(f"after restart, worker still stopped: {meta['status']}")
|
||
finally:
|
||
print("$ docker compose start worker")
|
||
compose("up", "-d", "--no-deps", "worker")
|
||
meta = wait_status(base, bob, attachment["id"], ("ready", "failed"))
|
||
print(f"after worker start: {meta['status']} error_code={meta['error_code']}")
|
||
ok = ok and meta["status"] == "ready"
|
||
status, _, data = call(
|
||
base,
|
||
"GET",
|
||
f"/api/v1/attachments/{attachment['id']}/content",
|
||
headers=bearer(bob),
|
||
)
|
||
same = hashlib.sha256(data).hexdigest() == hashlib.sha256(content).hexdigest()
|
||
print(f"original -> {status}, sha256 {'== fixture' if same else '!= fixture'}")
|
||
ok = ok and status == 200 and same
|
||
status, _, data = call(
|
||
base,
|
||
"GET",
|
||
f"/api/v1/attachments/{attachment['id']}/content?variant=thumbnail",
|
||
headers=bearer(bob),
|
||
)
|
||
print(f"thumbnail -> {status}, JPEG {jpeg_size(data)}")
|
||
ok = ok and status == 200 and jpeg_size(data) == (256, 160)
|
||
print("StartedAt after:")
|
||
for service in services:
|
||
print(" " + started_at(service))
|
||
print("PASS" if ok else "FAIL")
|
||
return 0 if ok else 1
|
||
|
||
|
||
PIPELINE_SQL = (
|
||
"SELECT state, count(*), "
|
||
"coalesce(round(extract(epoch FROM now() - min(created_at))::numeric, 1), 0) "
|
||
"FROM attachment_jobs GROUP BY state ORDER BY state"
|
||
)
|
||
|
||
|
||
def pipeline(_args: argparse.Namespace) -> int:
|
||
db = _env_value("POSTGRES_DB", "messenger")
|
||
user = _env_value("POSTGRES_USER", "messenger")
|
||
done = compose(
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"psql",
|
||
"-U",
|
||
user,
|
||
"-d",
|
||
db,
|
||
"-At",
|
||
"-F",
|
||
" ",
|
||
"-c",
|
||
PIPELINE_SQL,
|
||
check=False,
|
||
)
|
||
print("state count oldest_s")
|
||
print(done.stdout.strip() or done.stderr.strip())
|
||
queues = compose(
|
||
"exec",
|
||
"-T",
|
||
"rabbitmq",
|
||
"rabbitmqctl",
|
||
"-q",
|
||
"list_queues",
|
||
"name",
|
||
"messages_ready",
|
||
"messages_unacknowledged",
|
||
"consumers",
|
||
check=False,
|
||
)
|
||
print("queue ready unacked consumers")
|
||
print(queues.stdout.strip() or queues.stderr.strip())
|
||
return done.returncode
|
||
|
||
|
||
# --- приватность bucket вне API ------------------------------------------------
|
||
|
||
|
||
def _env_value(name: str, default: str) -> str:
|
||
env_file = os.environ.get("COMPOSE_ENV_FILES") or str(ROOT / ".env")
|
||
path = Path(env_file.split(",")[0])
|
||
if path.exists():
|
||
for key, value in _assignments(path.read_text(encoding="utf-8")):
|
||
if key == name and value:
|
||
return value
|
||
return os.environ.get(name) or default
|
||
|
||
|
||
def _http_status(url: str, method: str = "GET", data: bytes | None = None) -> str:
|
||
request = urllib.request.Request(url, data=data, method=method)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=5) as response:
|
||
return str(response.status)
|
||
except urllib.error.HTTPError as error:
|
||
return str(error.code)
|
||
except OSError as error:
|
||
return type(error).__name__
|
||
|
||
|
||
def _network_status(path: str, method: str = "GET") -> str:
|
||
"""Анонимный запрос к API S3 из сети стенда (контейнер LB, bash /dev/tcp)."""
|
||
request = (
|
||
f"{method} {path} HTTP/1.0\\r\\nHost: s3\\r\\nContent-Length: 0\\r\\n\\r\\n"
|
||
)
|
||
script = f"exec 3<>/dev/tcp/s3/8333 && printf '{request}' >&3 && head -n 1 <&3"
|
||
done = compose("exec", "-T", "haproxy", "bash", "-c", script, check=False)
|
||
match = re.search(r"HTTP/1\.[01] (\d{3})", done.stdout)
|
||
return match.group(1) if match else "no answer"
|
||
|
||
|
||
def _port_open(port: str) -> bool:
|
||
script = f"timeout 2 bash -c 'exec 3<>/dev/tcp/s3/{port}'"
|
||
return (
|
||
compose("exec", "-T", "haproxy", "bash", "-c", script, check=False).returncode
|
||
== 0
|
||
)
|
||
|
||
|
||
def s3_check(_args: argparse.Namespace) -> int:
|
||
rows: list[tuple[str, str, str, str]] = []
|
||
print("$ docker compose run --rm --no-deps s3-bootstrap check")
|
||
check = compose("run", "--rm", "--no-deps", "s3-bootstrap", "check", check=False)
|
||
for line in check.stdout.splitlines():
|
||
parts = line.split("\t")
|
||
if line.startswith("s3-bootstrap: check") and len(parts) == 5:
|
||
rows.append(("bootstrap (loopback s3)", parts[1], parts[2], parts[3]))
|
||
elif line.strip():
|
||
print(line)
|
||
if check.returncode != 0:
|
||
print(check.stderr.strip())
|
||
bucket = _env_value("S3_BUCKET", "messenger-attachments")
|
||
host = f"http://127.0.0.1:{_env_value('S3_HOST_PORT', '8334')}"
|
||
probe = f"/{bucket}/.s3-check/probe"
|
||
try:
|
||
for name, url, method in (
|
||
("anonymous get object", host + probe, "GET"),
|
||
("anonymous head object", host + probe, "HEAD"),
|
||
("anonymous list objects", f"{host}/{bucket}?list-type=2", "GET"),
|
||
("anonymous list buckets", host + "/", "GET"),
|
||
):
|
||
rows.append(("host 127.0.0.1", name, "403", _http_status(url, method)))
|
||
rows.append(
|
||
(
|
||
"host 127.0.0.1",
|
||
"anonymous put object",
|
||
"403",
|
||
_http_status(f"{host}/{bucket}/.s3-check/anonymous", "PUT", b"x"),
|
||
)
|
||
)
|
||
for name, path, method in (
|
||
("anonymous get object", probe, "GET"),
|
||
("anonymous list objects", f"/{bucket}?list-type=2", "GET"),
|
||
("anonymous put object", f"/{bucket}/.s3-check/anonymous", "PUT"),
|
||
):
|
||
rows.append(
|
||
("stand network (haproxy)", name, "403", _network_status(path, method))
|
||
)
|
||
listening = compose(
|
||
"exec", "-T", "s3", "sh", "-c", "netstat -tln", check=False
|
||
).stdout
|
||
ports = sorted(
|
||
{
|
||
m.group(1)
|
||
for m in re.finditer(r"\s\S+:(\d+)\s+\S+:\*?\S*\s+LISTEN", listening)
|
||
},
|
||
key=int,
|
||
)
|
||
for port in ports:
|
||
expected = "open" if port in S3_NETWORK_PORTS else "closed"
|
||
got = "open" if _port_open(port) else "closed"
|
||
rows.append(("stand network (haproxy)", f"s3 port {port}", expected, got))
|
||
finally:
|
||
cleanup = compose(
|
||
"run", "--rm", "--no-deps", "s3-bootstrap", "cleanup", check=False
|
||
)
|
||
print(
|
||
"$ docker compose run --rm --no-deps s3-bootstrap cleanup -> exit",
|
||
cleanup.returncode,
|
||
)
|
||
failed = 0
|
||
print(f"{'where':<26} {'check':<46} {'expected':>8} {'got':>8} verdict")
|
||
for where, name, expected, got in rows:
|
||
verdict = "ok" if expected == got else "FAIL"
|
||
failed += verdict != "ok"
|
||
print(f"{where:<26} {name:<46} {expected:>8} {got:>8} {verdict}")
|
||
print(f"checks: {len(rows)}, failed: {failed}, bootstrap exit {check.returncode}")
|
||
return 1 if failed or check.returncode != 0 else 0
|
||
|
||
|
||
# --- лаба 6: телеметрия ----------------------------------------------------------
|
||
|
||
# метки метрик, в имени которых не должно быть ничего из этого
|
||
# (contracts/observability.md, «Не labels метрик»)
|
||
FORBIDDEN_LABEL_PARTS = (
|
||
"user",
|
||
"chat",
|
||
"message_id",
|
||
"attachment",
|
||
"request_id",
|
||
"url",
|
||
"path",
|
||
"filename",
|
||
"text",
|
||
)
|
||
UUID_RE = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
|
||
# сколько ждать сигналы: экспорт метрик 15 с + scrape 15 с + пакеты логов
|
||
SIGNAL_TIMEOUT = 120.0
|
||
|
||
|
||
class Grafana:
|
||
"""HTTP API хранилищ через прокси источников Grafana (uid из
|
||
infra/otel/grafana/datasources.yaml). Пароль admin — из .env, не
|
||
печатается."""
|
||
|
||
def __init__(self, port: str, password: str) -> None:
|
||
self.base = f"http://127.0.0.1:{port}"
|
||
token = base64.b64encode(f"admin:{password}".encode()).decode()
|
||
self.headers = {"Authorization": "Basic " + token}
|
||
|
||
def get(self, path: str, params: dict | None = None):
|
||
query = ("?" + urllib.parse.urlencode(params)) if params else ""
|
||
request = urllib.request.Request(self.base + path + query, headers=self.headers)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=15) as response:
|
||
return response.status, json.loads(response.read() or b"null")
|
||
except urllib.error.HTTPError as error:
|
||
return error.code, None
|
||
except (OSError, ValueError) as error:
|
||
return type(error).__name__, None
|
||
|
||
def source(self, uid: str, path: str, params: dict | None = None):
|
||
return self.get(f"/api/datasources/proxy/uid/{uid}{path}", params)
|
||
|
||
|
||
class Checks:
|
||
def __init__(self) -> None:
|
||
self.results: list[bool] = []
|
||
|
||
def add(self, ok: bool, name: str, detail: str = "") -> bool:
|
||
self.results.append(ok)
|
||
print(f"{'PASS' if ok else 'FAIL'} {name}" + (f": {detail}" if detail else ""))
|
||
return ok
|
||
|
||
|
||
def _poll(fetch, timeout: float = SIGNAL_TIMEOUT, pause: float = 3.0):
|
||
"""Повторять `fetch()`, пока он не вернёт непустое, или до срока."""
|
||
deadline = time.monotonic() + timeout
|
||
while True:
|
||
value = fetch()
|
||
if value or time.monotonic() >= deadline:
|
||
return value
|
||
time.sleep(pause)
|
||
|
||
|
||
def _tempo_find(grafana: Grafana, traceql: str, since: float) -> list[str]:
|
||
status, data = grafana.source(
|
||
"traces",
|
||
"/api/search",
|
||
{"q": traceql, "start": int(since) - 60, "end": int(time.time()) + 60},
|
||
)
|
||
if status != 200 or not data:
|
||
return []
|
||
# Tempo отдаёт id без ведущих нулей, а журнал — все 32 hex-знака
|
||
return [item["traceID"].zfill(32) for item in data.get("traces") or []]
|
||
|
||
|
||
def _tempo_trace(grafana: Grafana, trace_id: str) -> dict:
|
||
status, data = grafana.source("traces", f"/api/v2/traces/{trace_id}")
|
||
return data if status == 200 and data else {}
|
||
|
||
|
||
def _spans(trace: dict) -> list[tuple[str, str, dict]]:
|
||
"""(service.name, имя span, атрибуты) из ответа /api/v2/traces."""
|
||
found = []
|
||
body = trace.get("trace", trace)
|
||
for resource in body.get("resourceSpans") or body.get("batches") or []:
|
||
service = ""
|
||
for attr in resource.get("resource", {}).get("attributes", []):
|
||
if attr["key"] == "service.name":
|
||
service = attr["value"].get("stringValue", "")
|
||
for scope in resource.get("scopeSpans", []):
|
||
for span in scope.get("spans", []):
|
||
attrs = {
|
||
a["key"]: next(iter(a["value"].values()), None)
|
||
for a in span.get("attributes", [])
|
||
}
|
||
found.append((service, span.get("name", ""), attrs))
|
||
return found
|
||
|
||
|
||
def _loki_lines(grafana: Grafana, query: str, since: float, limit: int = 1000):
|
||
"""[(метки и structured metadata потока, строка)] за окно с `since`."""
|
||
status, data = grafana.source(
|
||
"logs",
|
||
"/loki/api/v1/query_range",
|
||
{
|
||
"query": query,
|
||
"start": int((since - 60) * 1e9),
|
||
"end": int((time.time() + 60) * 1e9),
|
||
"limit": limit,
|
||
"direction": "backward",
|
||
},
|
||
)
|
||
if status != 200 or not data:
|
||
return []
|
||
rows = []
|
||
for stream in data.get("data", {}).get("result", []):
|
||
for _, line in stream.get("values", []):
|
||
rows.append((stream.get("stream", {}), line))
|
||
return rows
|
||
|
||
|
||
def _prom(grafana: Grafana, expr: str) -> list:
|
||
status, data = grafana.source("metrics", "/api/v1/query", {"query": expr})
|
||
if status != 200 or not data:
|
||
return []
|
||
return data.get("data", {}).get("result", [])
|
||
|
||
|
||
def _prom_value(grafana: Grafana, expr: str) -> float:
|
||
result = _prom(grafana, expr)
|
||
return float(result[0]["value"][1]) if result else 0.0
|
||
|
||
|
||
def telemetry_check(args: argparse.Namespace) -> int:
|
||
base = args.base_url
|
||
checks = Checks()
|
||
password_env = _env_value("GRAFANA_ADMIN_PASSWORD", "")
|
||
if not password_env:
|
||
print("FAIL GRAFANA_ADMIN_PASSWORD is empty: run make env")
|
||
return 1
|
||
grafana = Grafana(_env_value("GRAFANA_HOST_PORT", "3180"), password_env)
|
||
started = time.time()
|
||
|
||
# 1. Grafana: provisioning из git
|
||
status, health = grafana.get("/api/health")
|
||
checks.add(status == 200, "grafana /api/health", f"status {status}")
|
||
status, sources = grafana.get("/api/datasources")
|
||
uids = sorted(item["uid"] for item in sources or [])
|
||
checks.add(
|
||
{"metrics", "logs", "traces"} <= set(uids), "datasources provisioned", str(uids)
|
||
)
|
||
status, board = grafana.get("/api/dashboards/uid/messenger-lab6")
|
||
meta = (board or {}).get("meta", {})
|
||
panels = (board or {}).get("dashboard", {}).get("panels", [])
|
||
checks.add(
|
||
status == 200 and meta.get("provisioned") is True,
|
||
"dashboard messenger-lab6 provisioned",
|
||
f"status {status}, panels {len(panels)}",
|
||
)
|
||
|
||
# 2. контролируемые запросы: свой X-Request-Id, известные пароль, токен,
|
||
# текст сообщения и имя файла — потом их не должно быть в телеметрии
|
||
marker = secrets.token_hex(4)
|
||
rid_me = f"tcheck-{marker}-me"
|
||
rid_upload = f"tcheck-{marker}-upload"
|
||
password = "Tc-" + secrets.token_urlsafe(18)
|
||
text_marker = f"telemetry body {secrets.token_hex(8)}"
|
||
filename = f"tcheck-{secrets.token_hex(6)}.png"
|
||
username = f"tcheck_{marker}"
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/users",
|
||
body={"username": username, "display_name": "Проверка", "password": password},
|
||
)
|
||
if status != 201:
|
||
print(f"FAIL register -> {status}")
|
||
return 1
|
||
user_id = json.loads(data)["id"]
|
||
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/auth/sessions",
|
||
headers={"Authorization": "Basic " + basic},
|
||
)
|
||
token = json.loads(data)["token"]
|
||
auth = {"Authorization": "Bearer " + token}
|
||
status, headers, _ = call(
|
||
base, "GET", "/api/v1/users/me", headers={**auth, "X-Request-Id": rid_me}
|
||
)
|
||
checks.add(
|
||
status == 200
|
||
and {k.lower(): v for k, v in headers.items()}.get("x-request-id") == rid_me,
|
||
"controlled request GET /api/v1/users/me",
|
||
f"status {status}, X-Request-Id {rid_me}",
|
||
)
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/chats",
|
||
headers=auth,
|
||
body={"title": "Телеметрия", "member_ids": [user_id]},
|
||
)
|
||
chat = json.loads(data)["id"]
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/attachments?filename={filename}",
|
||
headers={**auth, "Content-Type": "image/png", "X-Request-Id": rid_upload},
|
||
raw=FIXTURE.read_bytes(),
|
||
)
|
||
attachment = json.loads(data)["id"] if status == 202 else ""
|
||
checks.add(status == 202, "upload accepted", f"status {status}")
|
||
actor = {"id": user_id, "token": token}
|
||
meta_doc = (
|
||
wait_status(base, actor, attachment, ("ready", "failed")) if attachment else {}
|
||
)
|
||
checks.add(
|
||
meta_doc.get("status") == "ready",
|
||
"attachment ready",
|
||
str(meta_doc.get("status")),
|
||
)
|
||
status, _, _ = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/messages",
|
||
headers={**auth, "Idempotency-Key": f"tcheck-{marker}"},
|
||
body={"text": text_marker, "attachment_ids": [attachment]},
|
||
)
|
||
checks.add(status == 201, "message with attachment", f"status {status}")
|
||
|
||
# 3. трасса запроса в Tempo и связанный лог в Loki
|
||
api_traces = _poll(
|
||
lambda: _tempo_find(
|
||
grafana, f'{{ span.messenger.request_id = "{rid_me}" }}', started
|
||
)
|
||
)
|
||
checks.add(
|
||
len(api_traces) == 1, "API trace found by request id", ",".join(api_traces)
|
||
)
|
||
api_trace = _tempo_trace(grafana, api_traces[0]) if api_traces else {}
|
||
spans = _spans(api_trace)
|
||
server = [s for s in spans if s[1] == "GET /api/v1/users/me"]
|
||
checks.add(
|
||
bool(server)
|
||
and server[0][0] == "messenger-api"
|
||
and server[0][2].get("http.route") == "/api/v1/users/me",
|
||
"server span with route template",
|
||
f"{len(spans)} spans: " + ", ".join(sorted({s[1] for s in spans})),
|
||
)
|
||
# встроенная телеметрия FastAPI выключена: span только наши
|
||
foreign = sorted({s[1] for s in spans if s[0] != "messenger-api"})
|
||
checks.add(not foreign, "no spans from FastAPI native telemetry", str(foreign))
|
||
api_logs = _poll(
|
||
lambda: _loki_lines(
|
||
grafana,
|
||
f'{{service_name="messenger-api"}} | request_id="{rid_me}"',
|
||
started,
|
||
)
|
||
)
|
||
linked = [
|
||
row
|
||
for row in api_logs
|
||
if api_traces and row[0].get("trace_id") == api_traces[0]
|
||
]
|
||
checks.add(
|
||
bool(linked),
|
||
"API log record in Loki with the same trace_id",
|
||
linked[0][1][:100] if linked else f"{len(api_logs)} records",
|
||
)
|
||
stdout = compose(
|
||
"logs", "--no-log-prefix", "--since", "15m", "api", check=False
|
||
).stdout
|
||
stdout_lines = [line for line in stdout.splitlines() if f"[rid={rid_me}]" in line]
|
||
checks.add(
|
||
any(
|
||
api_traces and f"trace_id={api_traces[0]}" in line for line in stdout_lines
|
||
),
|
||
"API stdout line carries the same trace_id",
|
||
stdout_lines[0][:140] if stdout_lines else "no line",
|
||
)
|
||
upload_traces = _poll(
|
||
lambda: _tempo_find(
|
||
grafana, f'{{ span.messenger.request_id = "{rid_upload}" }}', started
|
||
)
|
||
)
|
||
upload_spans = (
|
||
{s[1] for s in _spans(_tempo_trace(grafana, upload_traces[0]))}
|
||
if upload_traces
|
||
else set()
|
||
)
|
||
checks.add(
|
||
{"POST /api/v1/chats/{chat_id}/attachments", "S3 put_object"} <= upload_spans
|
||
and any(name.startswith("postgresql ") for name in upload_spans),
|
||
"upload trace has PostgreSQL and S3 child spans",
|
||
", ".join(sorted(upload_spans)),
|
||
)
|
||
|
||
# 4. трасса задания worker и его лог
|
||
job_traces = _poll(
|
||
lambda: _tempo_find(
|
||
grafana,
|
||
f'{{ resource.service.name = "messenger-worker" && '
|
||
f'span.messenger.attachment.id = "{attachment}" }}',
|
||
started,
|
||
)
|
||
)
|
||
job_trace = _tempo_trace(grafana, job_traces[0]) if job_traces else {}
|
||
job_spans = _spans(job_trace)
|
||
consumer = [s for s in job_spans if s[1].startswith("process ")]
|
||
checks.add(
|
||
bool(consumer) and consumer[0][2].get("messenger.job.outcome") == "ready",
|
||
"worker consumer span with outcome ready",
|
||
", ".join(sorted({s[1] for s in job_spans})) or "not found",
|
||
)
|
||
worker_logs = _poll(
|
||
lambda: _loki_lines(
|
||
grafana,
|
||
f'{{service_name="messenger-worker"}} |= "{attachment}" '
|
||
'| event="attachment processed"',
|
||
started,
|
||
)
|
||
)
|
||
checks.add(
|
||
any(
|
||
job_traces and row[0].get("trace_id") == job_traces[0]
|
||
for row in worker_logs
|
||
),
|
||
"worker log 'attachment processed' with the job trace_id",
|
||
worker_logs[0][1][:100] if worker_logs else "no record",
|
||
)
|
||
|
||
# 5. метрики в Prometheus
|
||
me_count = _poll(
|
||
lambda: _prom_value(
|
||
grafana,
|
||
'sum(http_server_request_duration_seconds_count{job="messenger-api",'
|
||
'http_route="/api/v1/users/me",http_response_status_code="200"})',
|
||
)
|
||
)
|
||
checks.add(me_count > 0, "HTTP duration histogram (API)", f"count {me_count:g}")
|
||
ready_count = _poll(
|
||
lambda: _prom_value(
|
||
grafana,
|
||
'sum(messenger_job_duration_seconds_count{job="messenger-worker",'
|
||
'messenger_job_outcome="ready"})',
|
||
)
|
||
)
|
||
checks.add(
|
||
ready_count > 0,
|
||
"job duration histogram (worker, ready)",
|
||
f"count {ready_count:g}",
|
||
)
|
||
published = _poll(
|
||
lambda: _prom_value(
|
||
grafana,
|
||
'sum(messenger_relay_jobs_total{messenger_relay_action="published"})',
|
||
)
|
||
)
|
||
checks.add(published > 0, "relay published counter", f"{published:g}")
|
||
info = _prom(grafana, 'target_info{job=~"messenger-.+"}')
|
||
services = sorted(
|
||
{
|
||
f"{row['metric'].get('job')}@{row['metric'].get('service_version')}"
|
||
f"/{row['metric'].get('deployment_environment_name')}"
|
||
for row in info
|
||
}
|
||
)
|
||
checks.add(
|
||
all(item.endswith("/lab") and "@None" not in item for item in services)
|
||
and len(services) == 3,
|
||
"resource attributes (target_info)",
|
||
", ".join(services),
|
||
)
|
||
status, series = grafana.source(
|
||
"metrics",
|
||
"/api/v1/series",
|
||
{
|
||
"match[]": '{__name__=~"http_server_request_duration_seconds_count'
|
||
'|messenger_.+_count|messenger_.+_total", job=~"messenger-.+"}',
|
||
"start": int(started),
|
||
},
|
||
)
|
||
# второй инструмент той же метрики (например, встроенная телеметрия
|
||
# FastAPI) за последнюю минуту не прибавлял
|
||
duplicate = _prom_value(
|
||
grafana,
|
||
"sum(rate(http_server_request_duration_seconds_count"
|
||
'{otel_scope_name!="messenger"}[1m]))',
|
||
)
|
||
checks.add(
|
||
duplicate == 0, "HTTP histogram has a single source", f"rate {duplicate:g}"
|
||
)
|
||
bad = set()
|
||
for row in (series or {}).get("data", []):
|
||
for key, value in row.items():
|
||
if any(part in key for part in FORBIDDEN_LABEL_PARTS) or UUID_RE.search(
|
||
value
|
||
):
|
||
bad.add(f"{key}={value}")
|
||
checks.add(
|
||
status == 200 and not bad,
|
||
"metric labels without ids, raw URLs and filenames",
|
||
f"{len((series or {}).get('data', []))} series"
|
||
+ (", bad: " + ", ".join(sorted(bad)[:5]) if bad else ""),
|
||
)
|
||
|
||
# 6. секретов и тел в телеметрии нет: ищем у себя в выгрузке, а не
|
||
# запросом к Loki — иначе пароль попал бы в журнал запросов
|
||
secrets_list = {
|
||
"password": password,
|
||
"token": token,
|
||
"message text": text_marker,
|
||
"filename": filename,
|
||
}
|
||
logs = _loki_lines(grafana, '{service_name=~"messenger-.+"}', started, limit=5000)
|
||
traces_blob = json.dumps([api_trace, job_trace]) + "".join(
|
||
json.dumps(_tempo_trace(grafana, trace_id))
|
||
for trace_id in _tempo_find(
|
||
grafana, '{ resource.service.name =~ "messenger-.+" }', started
|
||
)[:20]
|
||
)
|
||
logs_blob = json.dumps(logs)
|
||
leaked = [
|
||
f"{name} in {where}"
|
||
for name, value in secrets_list.items()
|
||
for where, blob in (("logs", logs_blob), ("traces", traces_blob))
|
||
if value in blob
|
||
]
|
||
authorization = "authorization" in traces_blob.lower()
|
||
checks.add(
|
||
not leaked and not authorization and bool(logs),
|
||
"no password, token, message text, filename or Authorization in telemetry",
|
||
f"{len(logs)} log records, {len(traces_blob)} bytes of traces"
|
||
+ (", leaked: " + ", ".join(leaked) if leaked else "")
|
||
+ (", Authorization in traces" if authorization else ""),
|
||
)
|
||
|
||
passed = sum(checks.results)
|
||
verdict = "PASS" if all(checks.results) else "FAIL"
|
||
print(f"telemetry-check: {verdict} ({passed}/{len(checks.results)} checks)")
|
||
return 0 if verdict == "PASS" else 1
|
||
|
||
|
||
# --- уровень 4: цепочка трассы, alerts, отказ хранилища телеметрии -------------
|
||
|
||
|
||
def _psql(sql: str) -> str:
|
||
done = compose(
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"psql",
|
||
"-U",
|
||
_env_value("POSTGRES_USER", "messenger"),
|
||
"-d",
|
||
_env_value("POSTGRES_DB", "messenger"),
|
||
"-At",
|
||
"-F",
|
||
" ",
|
||
"-c",
|
||
sql,
|
||
check=False,
|
||
)
|
||
return done.stdout.strip()
|
||
|
||
|
||
def _grafana_or_none():
|
||
password = _env_value("GRAFANA_ADMIN_PASSWORD", "")
|
||
if not password:
|
||
print("FAIL GRAFANA_ADMIN_PASSWORD is empty: run make env")
|
||
return None
|
||
return Grafana(_env_value("GRAFANA_HOST_PORT", "3180"), password)
|
||
|
||
|
||
def _actor_with_chat(base: str, label: str):
|
||
actor = register(base, label)
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/chats",
|
||
headers=bearer(actor),
|
||
body={"title": "Уровень 4", "member_ids": [actor["id"]]},
|
||
)
|
||
if status != 201:
|
||
raise SystemExit(f"create chat -> {status}")
|
||
return actor, json.loads(data)["id"]
|
||
|
||
|
||
def _upload(base: str, actor: dict, chat: str):
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
f"/api/v1/chats/{chat}/attachments?filename=sample.png",
|
||
headers={**bearer(actor), "Content-Type": "image/png"},
|
||
raw=FIXTURE.read_bytes(),
|
||
)
|
||
return status, (json.loads(data)["id"] if status == 202 else "")
|
||
|
||
|
||
def _job_row(attachment: str) -> str:
|
||
"""attempt state last_error задания вложения (id проверен по шаблону)."""
|
||
if not UUID_RE.fullmatch(attachment):
|
||
return ""
|
||
return _psql(
|
||
"SELECT attempt, state, coalesce(last_error, '-') FROM attachment_jobs "
|
||
f"WHERE attachment_id = '{attachment}'"
|
||
)
|
||
|
||
|
||
def _trace_spans(trace: dict) -> list[dict]:
|
||
"""Span трассы Tempo с id, родителем, видом и атрибутами."""
|
||
found = []
|
||
body = trace.get("trace", trace)
|
||
for resource in body.get("resourceSpans") or body.get("batches") or []:
|
||
service = ""
|
||
for attr in resource.get("resource", {}).get("attributes", []):
|
||
if attr["key"] == "service.name":
|
||
service = attr["value"].get("stringValue", "")
|
||
for scope in resource.get("scopeSpans", []):
|
||
for span in scope.get("spans", []):
|
||
attrs = {
|
||
a["key"]: next(iter(a["value"].values()), None)
|
||
for a in span.get("attributes", [])
|
||
}
|
||
found.append(
|
||
{
|
||
"service": service,
|
||
"name": span.get("name", ""),
|
||
"id": span.get("spanId", ""),
|
||
"parent": span.get("parentSpanId", ""),
|
||
"kind": span.get("kind", ""),
|
||
"start": int(span.get("startTimeUnixNano", 0)),
|
||
"ms": (
|
||
int(span.get("endTimeUnixNano", 0))
|
||
- int(span.get("startTimeUnixNano", 0))
|
||
)
|
||
/ 1e6,
|
||
"error": span.get("status", {}).get("code")
|
||
== "STATUS_CODE_ERROR",
|
||
"attrs": attrs,
|
||
}
|
||
)
|
||
return found
|
||
|
||
|
||
def _print_tree(spans: list[dict]) -> None:
|
||
children: dict[str, list[dict]] = {}
|
||
ids = {span["id"] for span in spans}
|
||
for span in sorted(spans, key=lambda s: s["start"]):
|
||
key = span["parent"] if span["parent"] in ids else ""
|
||
children.setdefault(key, []).append(span)
|
||
|
||
def walk(parent: str, depth: int) -> None:
|
||
for span in children.get(parent, []):
|
||
attrs = span["attrs"]
|
||
extra = " ".join(
|
||
f"{key.split('.')[-1]}={attrs[key]}"
|
||
for key in ("messenger.job.attempt", "messenger.job.outcome")
|
||
if key in attrs
|
||
)
|
||
mark = " ERROR" if span["error"] else ""
|
||
print(
|
||
f" {' ' * depth}{span['service']}: {span['name']} "
|
||
f"[{span['kind'].replace('SPAN_KIND_', '').lower()}] "
|
||
f"{span['ms']:.1f} ms {extra}{mark}"
|
||
)
|
||
walk(span["id"], depth + 1)
|
||
|
||
walk("", 0)
|
||
|
||
|
||
def _descendants(spans: list[dict], root: str) -> list[dict]:
|
||
found, frontier = [], {root}
|
||
while frontier:
|
||
level = [span for span in spans if span["parent"] in frontier]
|
||
found.extend(level)
|
||
frontier = {span["id"] for span in level}
|
||
return found
|
||
|
||
|
||
def trace_chain(args: argparse.Namespace) -> int:
|
||
base = args.base_url
|
||
grafana = _grafana_or_none()
|
||
if grafana is None:
|
||
return 1
|
||
checks = Checks()
|
||
began = time.monotonic()
|
||
started = time.time()
|
||
actor, chat = _actor_with_chat(base, "trace")
|
||
compose("stop", "worker")
|
||
try:
|
||
status, attachment = _upload(base, actor, chat)
|
||
checks.add(status == 202, "upload accepted, worker stopped", f"status {status}")
|
||
if not attachment:
|
||
return 1
|
||
print(f"attachment_id={attachment}")
|
||
# первая попытка падает на чтении staging: S3 остановлен. После
|
||
# первого повтора worker снова останавливается, чтобы следующая
|
||
# попытка не сожгла бюджет, пока S3 поднимается
|
||
compose("stop", "s3")
|
||
try:
|
||
# `start` поднял бы и зависимость s3 (depends_on), --no-deps — нет
|
||
compose("up", "-d", "--no-deps", "worker")
|
||
|
||
def retried() -> str:
|
||
# queued с last_error — это retry_job; просто опубликованное
|
||
# тоже queued, но без ошибки
|
||
row = _job_row(attachment).split(" ", 2)
|
||
failed = len(row) == 3 and row[1] == "queued" and row[2] != "-"
|
||
return " ".join(row) if failed else ""
|
||
|
||
row = _poll(retried, timeout=60, pause=1)
|
||
checks.add(
|
||
bool(row),
|
||
"attempt 1 failed (S3 stopped), job back to queued",
|
||
row or "no retry",
|
||
)
|
||
compose("stop", "worker")
|
||
finally:
|
||
compose("up", "-d", "--wait", "s3")
|
||
finally:
|
||
compose("start", "worker")
|
||
meta = wait_status(base, actor, attachment, ("ready", "failed"), timeout=120)
|
||
checks.add(
|
||
meta["status"] == "ready", "attachment ready on a later attempt", meta["status"]
|
||
)
|
||
print(f"job: {_job_row(attachment)}")
|
||
|
||
query = f'{{span.messenger.attachment.id="{attachment}"}}'
|
||
|
||
def whole_trace():
|
||
ids = sorted(set(_tempo_find(grafana, query, started)))
|
||
if len(ids) != 1:
|
||
return None
|
||
spans = _trace_spans(_tempo_trace(grafana, ids[0]))
|
||
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
|
||
servers = [s for s in spans if s["kind"] == "SPAN_KIND_SERVER"]
|
||
ready = [
|
||
s for s in consumers if s["attrs"].get("messenger.job.outcome") == "ready"
|
||
]
|
||
if servers and ready and len(consumers) >= 2:
|
||
return ids[0], spans
|
||
return None
|
||
|
||
found = _poll(whole_trace, timeout=90)
|
||
ids = sorted(set(_tempo_find(grafana, query, started)))
|
||
checks.add(
|
||
bool(found), "one trace for the upload in Tempo", f"trace ids found: {len(ids)}"
|
||
)
|
||
if not found:
|
||
print(f"trace-chain: FAIL ({sum(checks.results)}/{len(checks.results)} checks)")
|
||
return 1
|
||
trace_id, spans = found
|
||
print(f"trace_id={trace_id} (Grafana → Explore → Tempo → {trace_id})")
|
||
_print_tree(spans)
|
||
by_id = {span["id"]: span for span in spans}
|
||
server = next(s for s in spans if s["kind"] == "SPAN_KIND_SERVER")
|
||
producers = [s for s in spans if s["kind"] == "SPAN_KIND_PRODUCER"]
|
||
consumers = [s for s in spans if s["kind"] == "SPAN_KIND_CONSUMER"]
|
||
checks.add(
|
||
server["service"] == "messenger-api"
|
||
and server["name"].startswith("POST /api/v1/chats/{chat_id}/attachments"),
|
||
"root: API server span of the upload",
|
||
server["name"],
|
||
)
|
||
checks.add(
|
||
len(producers) >= 2
|
||
and all(
|
||
p["service"] == "messenger-relay" and p["parent"] == server["id"]
|
||
for p in producers
|
||
),
|
||
"relay producer spans are children of the upload span",
|
||
f"{len(producers)} publish spans",
|
||
)
|
||
linked = [
|
||
c
|
||
for c in consumers
|
||
if by_id.get(c["parent"], {}).get("kind") == "SPAN_KIND_PRODUCER"
|
||
and by_id[c["parent"]]["attrs"].get("messenger.job.attempt")
|
||
== c["attrs"].get("messenger.job.attempt")
|
||
]
|
||
attempts = sorted(
|
||
int(c["attrs"].get("messenger.job.attempt") or 0) for c in consumers
|
||
)
|
||
checks.add(
|
||
len(linked) == len(consumers) >= 2,
|
||
"worker consumer spans are children of the producer of the same attempt",
|
||
f"attempts {attempts}",
|
||
)
|
||
first = min(consumers, key=lambda s: s["start"])
|
||
checks.add(
|
||
first["attrs"].get("messenger.job.outcome") == "retry",
|
||
"attempt 1 outcome retry in the same trace",
|
||
str(first["attrs"].get("messenger.job.outcome")),
|
||
)
|
||
last = next(
|
||
c for c in consumers if c["attrs"].get("messenger.job.outcome") == "ready"
|
||
)
|
||
below = {span["name"] for span in _descendants(spans, last["id"])}
|
||
checks.add(
|
||
"S3 put_object" in below and "postgresql transaction" in below,
|
||
"ready attempt reaches S3 and PostgreSQL",
|
||
", ".join(sorted(below)),
|
||
)
|
||
return _finish("trace-chain", checks, began)
|
||
|
||
|
||
def _finish(name: str, checks: Checks, began: float) -> int:
|
||
verdict = "PASS" if all(checks.results) else "FAIL"
|
||
passed, total = sum(checks.results), len(checks.results)
|
||
took = time.monotonic() - began
|
||
print(f"{name}: {verdict} ({passed}/{total} checks, {took:.0f} s)")
|
||
return 0 if verdict == "PASS" else 1
|
||
|
||
|
||
def _alerts(grafana: Grafana) -> dict[str, str]:
|
||
"""alertname → state (pending/firing) из Prometheus; нет в списке — resolved."""
|
||
status, data = grafana.source("metrics", "/api/v1/alerts")
|
||
if status != 200 or not data:
|
||
return {}
|
||
return {a["labels"]["alertname"]: a["state"] for a in data["data"]["alerts"]}
|
||
|
||
|
||
def _wait_alert(
|
||
grafana: Grafana, name: str, firing: bool, timeout: float
|
||
) -> float | None:
|
||
"""Секунды до firing (или до ухода из списка); None — не дождались."""
|
||
began = time.monotonic()
|
||
last = None
|
||
while time.monotonic() - began < timeout:
|
||
state = _alerts(grafana).get(name, "inactive")
|
||
if state != last:
|
||
print(f" {time.monotonic() - began:5.0f} s {name}: {state}")
|
||
last = state
|
||
if (state == "firing") == firing and (firing or state == "inactive"):
|
||
return time.monotonic() - began
|
||
time.sleep(5)
|
||
return None
|
||
|
||
|
||
class _Load:
|
||
"""Фоновая нагрузка GET /users/me: статусы ответов по фазам."""
|
||
|
||
def __init__(self, base: str, actor: dict, threads: int = 4) -> None:
|
||
import threading
|
||
|
||
self.base, self.actor = base, actor
|
||
self.statuses: Counter = Counter()
|
||
self.stop = threading.Event()
|
||
self.threads = [
|
||
threading.Thread(target=self._run, daemon=True) for _ in range(threads)
|
||
]
|
||
|
||
def _run(self) -> None:
|
||
while not self.stop.is_set():
|
||
try:
|
||
status, _, _ = call(
|
||
self.base, "GET", "/api/v1/users/me", headers=bearer(self.actor)
|
||
)
|
||
except OSError as error:
|
||
status = type(error).__name__
|
||
self.statuses[status] += 1
|
||
time.sleep(0.2)
|
||
|
||
def __enter__(self):
|
||
for thread in self.threads:
|
||
thread.start()
|
||
return self
|
||
|
||
def take(self) -> dict:
|
||
seen, self.statuses = dict(self.statuses), Counter()
|
||
return seen
|
||
|
||
def __exit__(self, *_exc) -> None:
|
||
self.stop.set()
|
||
for thread in self.threads:
|
||
thread.join(timeout=15)
|
||
|
||
|
||
LOCK_SQL = (
|
||
"BEGIN; LOCK TABLE sessions IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(600); COMMIT;"
|
||
)
|
||
|
||
|
||
def alerts_check(args: argparse.Namespace) -> int:
|
||
base = args.base_url
|
||
grafana = _grafana_or_none()
|
||
if grafana is None:
|
||
return 1
|
||
checks = Checks()
|
||
began = time.monotonic()
|
||
rules = grafana.source("metrics", "/api/v1/rules")[1] or {}
|
||
names = sorted(
|
||
rule["name"]
|
||
for group in rules.get("data", {}).get("groups", [])
|
||
for rule in group["rules"]
|
||
if rule.get("type") == "alerting"
|
||
)
|
||
checks.add(
|
||
{"MessengerHighErrorRate", "MessengerQueueLagGrowing"} <= set(names),
|
||
"alert rules loaded in Prometheus",
|
||
", ".join(names),
|
||
)
|
||
actor, chat = _actor_with_chat(base, "alerts")
|
||
|
||
# (а) worker остановлен → возраст задач растёт → queue lag
|
||
print("(a) worker stopped: oldest job age grows")
|
||
phase = time.monotonic()
|
||
compose("stop", "worker")
|
||
try:
|
||
uploads = [_upload(base, actor, chat) for _ in range(3)]
|
||
checks.add(
|
||
all(s == 202 for s, _ in uploads), "3 uploads accepted with worker stopped"
|
||
)
|
||
fired = _wait_alert(grafana, "MessengerQueueLagGrowing", True, 300)
|
||
depth = _prom_value(grafana, "sum(messenger_job_queue_depth)")
|
||
age = _prom_value(grafana, "max(messenger_job_queue_oldest_age_seconds)")
|
||
checks.add(
|
||
fired is not None,
|
||
"MessengerQueueLagGrowing firing",
|
||
f"after {fired or 0:.0f} s; backlog {depth:.0f} jobs, oldest {age:.0f} s",
|
||
)
|
||
finally:
|
||
compose("start", "worker")
|
||
resolved = _wait_alert(grafana, "MessengerQueueLagGrowing", False, 240)
|
||
after = _prom_value(grafana, "sum(messenger_job_queue_depth)")
|
||
checks.add(
|
||
resolved is not None and after < depth,
|
||
"worker started: backlog drained, alert resolved",
|
||
f"after {resolved or 0:.0f} s; backlog {depth:.0f} -> {after:.0f}",
|
||
)
|
||
print(f" phase (a): {time.monotonic() - phase:.0f} s")
|
||
|
||
# (б) таблица сессий под блокировкой: каждый запрос с токеном ждёт
|
||
# statement_timeout и получает 503 — «медленная база»
|
||
print("(b) sessions table locked: authenticated requests wait and get 503")
|
||
phase = time.monotonic()
|
||
with _Load(base, actor) as load:
|
||
time.sleep(10)
|
||
print(f" baseline statuses: {load.take()}")
|
||
locker = subprocess.Popen(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"exec",
|
||
"-T",
|
||
"postgres",
|
||
"psql",
|
||
"-U",
|
||
_env_value("POSTGRES_USER", "messenger"),
|
||
"-d",
|
||
_env_value("POSTGRES_DB", "messenger"),
|
||
"-c",
|
||
LOCK_SQL,
|
||
],
|
||
cwd=ROOT,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
try:
|
||
fired = _wait_alert(grafana, "MessengerHighErrorRate", True, 300)
|
||
ratio = _prom_value(grafana, "messenger:http_errors:ratio_rate2m")
|
||
waiting = _prom_value(
|
||
grafana,
|
||
'max(db_client_connection_pending_requests{job="messenger-api"})',
|
||
)
|
||
used = _prom_value(grafana, USED_EXPR)
|
||
checks.add(
|
||
fired is not None,
|
||
"MessengerHighErrorRate firing",
|
||
f"after {fired or 0:.0f} s; 5xx ratio {ratio:.2f}, "
|
||
f"pool used {used:.0f}, waiting {waiting:.0f}; "
|
||
f"statuses {load.take()}",
|
||
)
|
||
finally:
|
||
_psql(
|
||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||
"WHERE query LIKE '%pg_sleep(600)%' AND pid <> pg_backend_pid()"
|
||
)
|
||
locker.wait(timeout=30)
|
||
# запросы, начатые до снятия блокировки, ещё доходят с 503
|
||
time.sleep(8)
|
||
print(f" statuses around release: {load.take()}")
|
||
resolved = _wait_alert(grafana, "MessengerHighErrorRate", False, 300)
|
||
statuses = load.take()
|
||
checks.add(
|
||
resolved is not None and set(statuses) <= {200},
|
||
"lock released: errors stopped, alert resolved",
|
||
f"after {resolved or 0:.0f} s; statuses after release {statuses}",
|
||
)
|
||
print(f" phase (b): {time.monotonic() - phase:.0f} s")
|
||
return _finish("alerts-check", checks, began)
|
||
|
||
|
||
USED_EXPR = (
|
||
'sum(db_client_connection_count{job="messenger-api", '
|
||
'db_client_connection_state="used"})'
|
||
)
|
||
|
||
|
||
def _container_mib(service: str) -> float:
|
||
cid = compose("ps", "-q", service, check=False).stdout.strip()
|
||
if not cid:
|
||
return 0.0
|
||
done = subprocess.run(
|
||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", cid],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
used = (done.stdout.split("/")[0] or "0").strip()
|
||
number = float(re.sub(r"[^0-9.]", "", used) or 0)
|
||
return number * {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024}.get(
|
||
re.sub(r"[0-9.]", "", used), 1
|
||
)
|
||
|
||
|
||
# Collector 0.164 отдаёт свои счётчики без суффикса _total; enqueue_failed
|
||
# появляется только после первого отказа очереди
|
||
LOSS_EXPR = (
|
||
'(sum(otelcol_exporter_send_failed_spans{exporter="otlp_http/tempo"}) or vector(0))'
|
||
' + (sum(otelcol_exporter_enqueue_failed_spans{exporter="otlp_http/tempo"})'
|
||
" or vector(0))"
|
||
)
|
||
SENT_EXPR = 'sum(otelcol_exporter_sent_spans{exporter="otlp_http/tempo"})'
|
||
QUEUE_EXPR = 'sum(otelcol_exporter_queue_size{exporter="otlp_http/tempo"})'
|
||
|
||
|
||
def telemetry_outage(args: argparse.Namespace) -> int:
|
||
base = args.base_url
|
||
grafana = _grafana_or_none()
|
||
if grafana is None:
|
||
return 1
|
||
checks = Checks()
|
||
began = time.monotonic()
|
||
actor, chat = _actor_with_chat(base, "outage")
|
||
lost_before = _prom_value(grafana, LOSS_EXPR)
|
||
memory = _container_mib("otel-collector")
|
||
print(f"before: collector {memory:.0f} MiB, spans lost so far {lost_before:.0f}")
|
||
compose("stop", "tempo")
|
||
peak, queue_peak, uploads = 0.0, 0.0, []
|
||
try:
|
||
with _Load(base, actor, threads=2) as load:
|
||
deadline = time.monotonic() + args.seconds
|
||
while time.monotonic() < deadline:
|
||
uploads.append(_upload(base, actor, chat))
|
||
peak = max(peak, _container_mib("otel-collector"))
|
||
queue_peak = max(queue_peak, _prom_value(grafana, QUEUE_EXPR))
|
||
time.sleep(10)
|
||
statuses = load.take()
|
||
ready = [
|
||
wait_status(base, actor, a, ("ready", "failed"), timeout=90)["status"]
|
||
for s, a in uploads
|
||
if s == 202
|
||
]
|
||
checks.add(
|
||
set(statuses) <= {200},
|
||
f"API answers during {args.seconds} s without Tempo",
|
||
f"GET /users/me statuses {statuses}",
|
||
)
|
||
checks.add(
|
||
len(ready) == len(uploads) and set(ready) == {"ready"},
|
||
"uploads accepted and processed without Tempo",
|
||
f"{len(uploads)} uploads, ready {ready.count('ready')}",
|
||
)
|
||
limit = 256.0
|
||
checks.add(
|
||
0 < peak < limit,
|
||
"Collector memory bounded",
|
||
f"peak {peak:.0f} MiB of {limit:.0f} MiB limit; "
|
||
f"export queue peak {queue_peak:.0f} of 20000 items",
|
||
)
|
||
finally:
|
||
compose("start", "tempo")
|
||
# счётчики отказов Collector доходят до Prometheus за scrape
|
||
lost = _poll(
|
||
lambda: max(_prom_value(grafana, LOSS_EXPR) - lost_before, 0),
|
||
timeout=90,
|
||
pause=5,
|
||
)
|
||
checks.add(
|
||
bool(lost),
|
||
"lost spans are counted by the Collector",
|
||
f"{lost or 0:.0f} spans (send_failed + enqueue_failed, otlp_http/tempo)",
|
||
)
|
||
sent_before = _prom_value(grafana, SENT_EXPR)
|
||
recovered = _poll(
|
||
lambda: (
|
||
_prom_value(grafana, SENT_EXPR) > sent_before
|
||
and _prom_value(grafana, QUEUE_EXPR) < 100
|
||
),
|
||
timeout=120,
|
||
pause=5,
|
||
)
|
||
checks.add(
|
||
bool(recovered),
|
||
"Tempo back: export resumes, queue drained",
|
||
f"queue now {_prom_value(grafana, QUEUE_EXPR):.0f} items",
|
||
)
|
||
return _finish("telemetry-outage", checks, began)
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
commands = parser.add_subparsers(dest="command", required=True)
|
||
commands.add_parser("env")
|
||
one = commands.add_parser("instances")
|
||
one.add_argument("--base-url", default="http://localhost:8080")
|
||
one.add_argument("--requests", type=int, default=12)
|
||
commands.add_parser("lb-stats")
|
||
each = commands.add_parser("failover-each")
|
||
each.add_argument("--base-url", default="http://localhost:8080")
|
||
each.add_argument("--replicas", type=int, default=2)
|
||
each.add_argument("--mode", choices=("stop", "kill"), default="stop")
|
||
attachment = commands.add_parser("attachment-failover")
|
||
attachment.add_argument("--base-url", default="http://localhost:8080")
|
||
attachment.add_argument("--replicas", type=int, default=2)
|
||
attachment.add_argument("--mode", choices=("stop", "kill"), default="stop")
|
||
commands.add_parser("s3-check")
|
||
scenario = commands.add_parser("defense")
|
||
scenario.add_argument("--base-url", default="http://localhost:8080")
|
||
commands.add_parser("pipeline")
|
||
telemetry = commands.add_parser("telemetry-check")
|
||
telemetry.add_argument("--base-url", default="http://localhost:8080")
|
||
for name in ("trace-chain", "alerts-check", "telemetry-outage"):
|
||
level4 = commands.add_parser(name)
|
||
level4.add_argument("--base-url", default="http://localhost:8080")
|
||
if name == "telemetry-outage":
|
||
level4.add_argument("--seconds", type=int, default=120)
|
||
args = parser.parse_args()
|
||
handler = {
|
||
"env": env,
|
||
"instances": instances,
|
||
"lb-stats": lb_stats,
|
||
"failover-each": failover_each,
|
||
"attachment-failover": attachment_failover,
|
||
"s3-check": s3_check,
|
||
"defense": defense,
|
||
"pipeline": pipeline,
|
||
"telemetry-check": telemetry_check,
|
||
"trace-chain": trace_chain,
|
||
"alerts-check": alerts_check,
|
||
"telemetry-outage": telemetry_outage,
|
||
}
|
||
return handler[args.command](args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|