Author SHA1 Message Date
gytall 2b721844fe fix report.md 2026-11-20 17:18:20 +03:00
gytall 27a3a6895f created tasks for grade 4 2026-11-19 19:09:19 +03:00
gytall 52be9d282b reyalysed grade 3 2026-11-14 05:00:23 +03:00
38 changed files with 3261 additions and 194 deletions
+3 -9
View File
@@ -98,13 +98,7 @@ IMAGE_CPU_SECONDS=20
JOB_RETRY_MAX_SECONDS=120
FAILED_STAGING_RETENTION_SECONDS=604800
# Дополнительные предложения skeleton lab6; пока не подключены к перенесённому стенду.
DATABASE_URL_FILE=/run/secrets/database_url
CACHE_URL_FILE=/run/secrets/cache_url
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
OTEL_SERVICE_NAME=messenger-api
# Наблюдаемость: SDK экспортирует HTTP/protobuf в Collector.
OTEL_ENABLED=true
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
GRAFANA_HOST_PORT=13000
+3 -1
View File
@@ -32,7 +32,9 @@ coverage/
# Inspect git diff --cached: ignore rules are not a secret scanner.
# Перенесённые исключения lab5
venv/
.venv/
.pytest_cache/
.ruff_cache/
/docs/
.verification-error.log
+28 -5
View File
@@ -8,14 +8,17 @@ SERVICE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
COMPOSE = docker compose --env-file .env
LOCAL_COMPOSE = $(COMPOSE) -f compose.yaml -f compose.local.yml
TEST_PROJECT ?= messenger-test
TEST_LB_PORT ?= 28080
TEST_DB_PORT ?= $(DB_HOST_PORT)
TEST_REDIS_PORT ?= $(REDIS_HOST_PORT)
TEST_GRAFANA_PORT ?= 23000
GRAFANA_URL ?= http://localhost:$(or $(GRAFANA_HOST_PORT),13000)
TEST_BASE_URL = http://localhost:$(TEST_LB_PORT)
TEST_ENV = COMPOSE_PROJECT_NAME=$(TEST_PROJECT) COMPOSE_FILE=compose.yaml:compose.test.yaml \
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT)
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT) GRAFANA_HOST_PORT=$(TEST_GRAFANA_PORT)
TEST_COMPOSE = $(TEST_ENV) $(COMPOSE)
TEST_DB_PASSWORD = $(if $(wildcard secrets/db_password),$(file <secrets/db_password),$(DB_PASSWORD))
@@ -25,13 +28,14 @@ TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file <secrets/r
TEST_REDIS_DB ?= 15
export TEST_DATABASE_URL TEST_REDIS_HOST TEST_REDIS_PORT TEST_REDIS_PASSWORD TEST_REDIS_DB
.PHONY: help init up down restart ps logs migrate migration worker-stop worker-start \
.PHONY: help init up up-local down restart ps logs migrate migration worker-stop worker-start \
cleanup-dry-run cleanup-apply check validate test load test-unit test-pipeline \
test-nginx test-storage test-outbox test-jobs test-persistence test-failover \
test-suite test-stand-up test-stand-destroy
test-suite test-stand-up test-stand-destroy test-telemetry test-observability verify clean-stands
help:
@echo "Запуск: init | up | down | restart | ps | logs SERVICE=worker"
@echo "Локальные порты и веб-панели: up-local"
@echo "Обслуживание: migrate | migration m='описание' | worker-stop | worker-start"
@echo "Очистка объектов: cleanup-dry-run | cleanup-apply"
@echo "Проверки: check | validate | test | test-pipeline | test-unit PYTEST_ARGS=..."
@@ -39,12 +43,16 @@ help:
@echo "Сервисы: test-nginx | test-storage | test-outbox | test-jobs"
@echo "Отказы API (по запросу): test-persistence | test-failover"
@echo "Автономный прогон с удалением тестового стенда: test-suite"
@echo "Наблюдаемость: test-observability; полный прогон с очисткой: verify"
init:
$(PYTHON) scripts/init_local.py
up:
$(COMPOSE) up -d --build --wait --wait-timeout 120
$(COMPOSE) up -d --build --wait --wait-timeout 180
up-local:
$(LOCAL_COMPOSE) up -d --build --wait --wait-timeout 180
down:
$(COMPOSE) down
@@ -88,6 +96,21 @@ test:
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-telemetry:
$(PYTHON) -m pytest -q student-tests/test_telemetry.py
# Проверка на уже запущенном стенде; включает сценарии отказа и восстановления.
test-observability:
$(PYTHON) -m scripts.check_observability --base-url "$(BASE_URL)" --grafana-url "$(GRAFANA_URL)" --incidents --restart-telemetry
# Один прогон на отдельном стенде, удаление его контейнеров/сети/томов при завершении.
verify:
$(PYTHON) -m scripts.deployment verify
# Удаление основного и тестового стендов; .env/secrets сохраняются.
clean-stands:
$(PYTHON) -m scripts.deployment clean --apply
# Все сохранённые регрессионные тесты запускаются явно; PYTEST_ARGS выбирает подмножество.
test-unit:
$(TEST_ENV) $(PYTHON) -m pytest -q $(PYTEST_ARGS)
@@ -115,7 +138,7 @@ test-failover:
# Отдельные PostgreSQL/Redis нужны для локальных тестов; тестовая БД создаётся явно.
test-stand-up:
$(TEST_COMPOSE) up -d --build --wait --wait-timeout 120
$(TEST_COMPOSE) up -d --build --wait --wait-timeout 180
$(TEST_COMPOSE) exec -T postgres sh -c 'psql -U "$$POSTGRES_USER" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '\''$${POSTGRES_DB}_test'\''" | grep -q 1 || createdb -U "$$POSTGRES_USER" "$${POSTGRES_DB}_test"'
test-stand-destroy:
+92 -31
View File
@@ -1,46 +1,107 @@
# Отчёт — лабораторная 6
# Отчёт - лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор: Григорян Илья Мурадович, БПИ2303.
- Цель: оценка **4**
- Источники: OpenTelemetry SDK 1.45.1, FastAPI, SQLAlchemy 2.1.4,
[внутренние метрики Collector](https://opentelemetry.io/docs/collector/internal-telemetry/),
[контракт наблюдаемости](contracts/observability.md). Помощь Codex.
## Запуск из чистого clone
## Запуск
Перенесённый стенд lab5 и команды его запуска описаны в [RUNNING.md](RUNNING.md).
Это основа для lab6; телеметрия текущей лабораторной пока не реализована.
Python 3.12 и Docker Compose. Зависимости и образы закреплены в репозитории.
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make init
make up
make test
make down
```
## Архитектура и решения
API доступен на http://localhost:18081, дашборд — на
http://localhost:13000/d/messenger-overview.
Инициализация создаёт только отсутствующие настройки и секреты. Миграции и
настройка приватного bucket выполняются при запуске. `make down` сохраняет тома.
Из lab5 перенесена реализация уровня 4: API, миграции, PostgreSQL/Redis,
MinIO, RabbitMQ, outbox publisher, worker, очистка, crop/WebP, ограниченные
повторы с задержкой, failed registry и ручной replay, собственные тесты и
проверки отказов. Контракты, публичные проверки и задание lab6 сохранены.
Результаты проверок lab5 не считаются доказательствами прохождения lab6.
## Архитектура и трассировка
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Две реплики API за nginx. PostgreSQL хранит сообщения и состояние заданий,
Redis — сессии, MinIO — файлы, RabbitMQ доставляет задания worker.
Миграции и настройка MinIO выполняются при запуске, доступы передаются
через файловые секреты. Версии, проверки готовности и тома заданы в
[compose.yaml](compose.yaml).
## Доказательства критериев
API, publisher и worker отправляют телеметрию по OTLP в Collector.
Prometheus хранит метрики, Loki — структурные логи, Tempo — трассы.
Grafana читает три источника и автоматически загружает дашборд.
История хранится 72 часа в именованных томах.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
API сохраняет W3C `traceparent` и `tracestate` вместе с заданием. Publisher
создаёт дочерний producer span и передаёт контекст через заголовки RabbitMQ.
Каждая попытка получает свой consumer span в той же трассе. Успешная попытка
добавляет операции чтения и записи S3 и фиксации `ready` в PostgreSQL.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
HTTP-ответ и лог связаны по `X-Request-Id`, лог и трасса — по `trace_id`.
Метрики группируются по шаблону маршрута, методу, классу ответа, очереди,
типу и исходу задания. ID отдельных объектов используются для поиска в
логах, а тексты сообщений, секреты и содержимое файлов в телеметрию не передаются.
## Эксперименты и отказы
## Метрики и цели
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Что смотрим | Запрос или метрика | Единица |
| --- | --- | --- |
| Частота HTTP | `sum by (http_request_method,http_route)(rate(messenger_http_requests_total{http_route!~"/health/.*"}[1m]))` | запросы/с |
| Доля ошибок | Сумма rate для класса 5xx / сумма rate всех бизнес-запросов, без health | доля |
| HTTP p50/p95/p99 | `histogram_quantile(q, sum by(le)(rate(messenger_http_duration_seconds_bucket{http_route!~"/health/.*"}[1m])))` | с |
| Worker | `messenger_job_attempts_total` по outcome и `messenger_job_duration_seconds_bucket` | попытки, с |
| Очередь | `max by(queue_name)(messenger_queue_depth_ratio)`, `max by(queue_name)(messenger_queue_oldest_seconds)` | задания, с |
| Пул PostgreSQL | `sum by(service_name)(messenger_db_pool_used_ratio)` и аналогично `messenger_db_pool_waiting_ratio` | соединения, ожидающие |
| Загрузка worker | `sum(messenger_worker_busy_ratio) / sum(messenger_worker_capacity_ratio)` | доля |
| CPU и память | `rate(messenger_process_cpu_seconds_total[1m])`, `messenger_process_rss_bytes` | ядра, байты |
| До готовности вложения | `histogram_quantile(0.95, sum by(le)(rate(messenger_job_time_to_ready_seconds_bucket[5m])))` | с |
| Экспорт Collector | `otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity` | пакеты |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Глубина очереди — число заданий в состоянии `queued` в PostgreSQL,
включая отложенные повторы. Возраст считается от первого приёма.
Время до `ready` включает ожидание и повторы. Процентили вычисляются
по объединённой гистограмме всех реплик. Ряды глубины и пула с суффиксом
`_ratio` содержат абсолютные количества.
## Ограничения и следующие шаги
SLI доступности — доля бизнес-запросов без 5xx. SLO — ≥99% за скользящие
24 часа. Для проверки используйте отношение сумм `increase(...[24h])`,
исключив health из числителя и знаменателя. Окно без трафика не оценивается.
Короткие оповещения для стенда заданы в [alerts.yaml](infra/otel/alerts.yaml).
`MessengerHighErrorRate` срабатывает при >5% 5xx за 30 с с выдержкой 5 с.
`MessengerQueueLag` срабатывает, если очередь непуста и возраст старейшего
задания >15 с с выдержкой 5 с. Состояние доступно в Prometheus через
`ALERTS{alertstate="firing"}`. Внешняя доставка отключена.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## Диагностика инцидентов
| Инцидент | Как найти причину и подтвердить восстановление |
| --- | --- |
| PostgreSQL остановлен | Рост 5xx → лог по `X-Request-Id` → ERROR span `postgres.checkout` в связанной трассе. Оповещение вошло в firing через 32,4 с и снялось через 67,5 с от начала сценария. Готовность API восстановилась. |
| Worker остановлен | Растёт возраст очереди, новые попытки отсутствуют. Оповещение сработало через 28,6 с после приёма. После пяти временных ошибок и ручного replay шестая попытка завершилась `ready`, глубина вернулась к нулю, оповещение снялось. В трассе `ff8352e27270fbc16fea7216c3ff45c0` видны все попытки и операции S3/БД. |
| Loki остановлен | В Prometheus смотрим бизнес-запросы, RSS и экспортную очередь Collector. За 51 с записано 204 сообщения, потеряно 6 логов после повторов отправки. Пик очереди — 48 из 256 пакетов, RSS — 228–231 MiB. После запуска Loki найден новый связанный лог. |
Collector использует memory limiter (192 MiB, запас для всплесков 32 MiB),
пакеты до 256 записей, очереди по 256 пакетов для Loki и Tempo и повторы
до 30 с. Лимит контейнера — 256 MiB. Потери измерены по счётчикам Collector
для этого короткого отказа.
## Полный прогон
`make verify` запускает регрессию, smoke, нагрузку, сценарии отказов,
проверки сохранности и переключения реплик на отдельном стенде, затем
удаляет его контейнеры, сеть и тома.
Прогон 19.11.2026, 18:50–19:03 МСК — **PASS**. Регрессия: **206 тестов**
за 98,89 с. Нагрузка: 100 успешных сообщений, 0 ошибок, около 10 запросов/с.
Клиентские p50/p95/p99 — 18,0 / 39,6 / 46,0 мс. Проверены панели, связность
трасс и логов, приватность и сохранение истории после перезапуска.
Доказательства сохранены в [журнале проверок](evidence/verification.json)
и [результатах наблюдаемости](evidence/observability.json).
Стенд работает на одном Docker-хосте. Соблюдение SLO требует наблюдения
за полным 24-часовым окном.
+37 -65
View File
@@ -1,90 +1,62 @@
# Запуск перенесённой реализации lab5
# Запуск Messenger
В lab6 перенесена реализация lab5 для оценки 4: API-реплики за nginx,
PostgreSQL, Redis для сессий, MinIO для файлов, RabbitMQ, outbox publisher,
worker и очистка объектов. Сохранены миграции и собственные тесты.
OpenTelemetry, Collector и Grafana из задания lab6 пока не реализованы.
## Основной стенд
Нужны Python 3.12 и Docker с Compose v2. Версии Python-зависимостей закреплены
в `requirements.txt`, образы — в Dockerfile и Compose.
Нужны Python 3.12 и Docker Compose v2. Версии закреплены в requirements.txt и Compose.
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
make init
make check validate
make up
make test
```
`make init` создаёт отсутствующие `.env` и случайные файловые секреты в `secrets/`.
Секреты и данные из lab5 не копируются. Настройки берутся из `.env.example`;
дополнительные переменные skeleton lab6 в конце файла пока не подключены.
Миграции БД и настройка MinIO выполняются при запуске.
`make up` ожидает готовности сервисов. По умолчанию API доступен по
http://localhost:18081 (`LB_HOST_PORT`); `make test` и `make load` используют
этот порт. Его можно переопределить через `BASE_URL`.
`make init` создаёт только отсутствующие `.env` и случайные файловые секреты.
Миграции PostgreSQL и приватный bucket/политика MinIO создаются автоматически.
Основной проект — `messenger`, тестовый — `messenger-test`.
- API: http://localhost:18081 (`LB_HOST_PORT`).
- Grafana: http://localhost:13000/d/messenger-overview (`GRAFANA_HOST_PORT`).
- Просмотр Grafana — анонимный Viewer; admin password — `secrets/grafana_admin_password`.
- Внутренние API/OTLP/Prometheus/Loki/Tempo порты на хост не публикуются.
`OTEL_ENABLED` включает экспорт трёх сигналов, `OTEL_EXPORTER_OTLP_ENDPOINT` задаёт
Collector, `APP_ENV` — окружение resource attributes. Имена сервисов — messenger-api
и messenger-worker. Конфигурация — в `infra/otel/`, метрики/логи/трассы сохраняются в volumes.
```sh
make ps
make logs SERVICE=worker
make load
make down
make down # данные сохраняются
make clean-stands # удалить оба стенда, тома и образы приложения
```
`make down` сохраняет данные. Для **намеренного удаления всех учебных данных**:
`docker compose --env-file .env down --volumes`.
Compose-проект по умолчанию называется `messenger`, как в lab5: останавливайте
предыдущий стенд перед запуском либо используйте отдельный `COMPOSE_PROJECT_NAME`
и свободный `LB_HOST_PORT`. Одинаковое имя проекта подключает существующие volumes;
для независимого стенда lab6 задайте новое имя до первого запуска.
`.env`, `secrets/`, базовые образы и evidence при очистке сохраняются.
План удаления без применения: `.venv/bin/python -m scripts.deployment clean`.
## Перенесённые возможности уровня 4
## Проверки
Загрузка изображения асинхронная: API сохраняет исходник в staging MinIO и
задание/outbox в PostgreSQL, publisher отправляет задание в устойчивую очередь
RabbitMQ, worker сохраняет варианты и результат в БД, затем подтверждает доставку.
Сохранены original, JPEG thumbnail, центральный JPEG crop и WebP, ограничения
размера, пикселей, времени, памяти и CPU обработки.
`make verify` выполняет один последовательный прогон на одном тестовом стенде:
контракт/OpenAPI, полная регрессия с Redis restart, smoke, короткая нагрузка,
проверки телеметрии и двух отказов, persistence/failover. Очистка выполняется
в `finally`, включая ошибку. Результаты — два JSON в `evidence/`.
Не выполняйте одновременно команды, управляющие тем же стендом.
Повторы ограничены бюджетом, задержка растёт со случайным разбросом.
Идемпотентность, lease (временное право worker обрабатывать задание),
реестр неудачных заданий и ручной replay защищают обработку при повторной доставке.
Очистка сохраняет исходники активных заданий и учитывает сроки хранения ready/failed.
Отдельные проверки: `make test-telemetry` без Docker; `make test-observability`
на уже запущенном стенде (включает ограниченные отказы и восстановление).
`make test-unit` использует тестовую БД; сначала `make test-stand-up`, затем
`make test-stand-destroy`. Тестовые порты: API 28080, Grafana 23000,
PostgreSQL 25432, Redis 26379, только localhost.
## Задания и файлы
Загрузка асинхронная: staging/outbox → RabbitMQ → worker → original и JPEG/WebP
варианты. Повторы ограничены, готовность — после фиксации объектов и metadata.
```sh
docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed
docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored
```
`JOB_ID` и `REQUEST_UUID` замените UUID. Повтор replay с тем же request-id безопасен.
## Проверки
`make check` проверяет структуру контракта, `make validate` — OpenAPI.
`make test` запускает актуальные публичные тесты lab6 на основном стенде.
Собственные тесты lab5 находятся в `student-tests/`:
```sh
make test-stand-up
make test-unit
make test-pipeline
make test BASE_URL=http://localhost:28080
make test-stand-destroy
```
Тестовый стенд использует отдельный проект `messenger-test` и порт 28080,
а PostgreSQL/Redis публикуются только локально через `compose.test.yaml`.
`make test-stand-destroy` удаляет данные тестового стенда.
`make test-suite` автоматически поднимает этот стенд, проверяет pipeline,
публичный smoke, S3, outbox и очередь, затем удаляет его вместе с данными.
Полная регрессия запускается отдельно через `make test-unit`.
Проверки с намеренными отказами запускаются явно:
`make test-jobs`, `make test-outbox`, `make test-persistence`, `make test-failover`.
Они останавливают или перезапускают компоненты выбранного Compose-проекта.
Для собственных тестов с перезапусками контейнеров: `DOCKER_TESTS=1 make test-unit`.
Стенд рассчитан на один Docker-хост; репликация хранилищ и защита от потери диска
не реализованы.
Замените UUID; повтор replay с тем же request-id безопасен.
`make load` — унаследованный генератор без ограничения частоты, он может упереться
в nginx 50 запросов/с. Для штатной проверки используется 10 запросов/с на 10 секунд.
+25
View File
@@ -1,3 +1,6 @@
from contextlib import nullcontext
from sqlalchemy.pool import AsyncAdaptedQueuePool
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
@@ -8,9 +11,31 @@ from sqlalchemy.ext.asyncio import (
from app.core.config import Settings
class ObservedPool(AsyncAdaptedQueuePool):
telemetry = None
waiting = 0
def __init__(self, *args, pool_size, max_overflow, **kwargs):
self.capacity = pool_size + max_overflow if pool_size and max_overflow >= 0 else None
super().__init__(*args, pool_size=pool_size, max_overflow=max_overflow, **kwargs)
def _do_get(self):
waiting = self.capacity is not None and self.checkedout() >= self.capacity
self.waiting += int(waiting)
try:
return super()._do_get()
finally:
self.waiting -= int(waiting)
def connect(self):
with self.telemetry.span("postgres.checkout") if self.telemetry else nullcontext():
return super().connect()
def create_db_engine(settings: Settings) -> AsyncEngine:
return create_async_engine(
settings.get_database_url(),
poolclass=ObservedPool,
pool_size=settings.db_pool_size,
max_overflow=settings.db_max_overflow,
pool_timeout=settings.db_pool_timeout_seconds,
+29 -12
View File
@@ -10,6 +10,9 @@ import signal
from uuid import UUID, uuid4
from PIL import features
from opentelemetry.context import Context
from opentelemetry.trace import SpanKind
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from pydantic import ValidationError
from app.core.background import (
@@ -21,6 +24,7 @@ from app.core.background import (
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.core.telemetry import Telemetry
from app.domain.attachment_jobs import process_attachment_job
from app.errors.exceptions import ProcessingError
from app.schemas.job import JobMessage
@@ -67,6 +71,7 @@ async def consume_connected(
settings: JobSettings,
stop: asyncio.Event,
faults: Faults,
telemetry: Telemetry | None = None,
) -> None:
consumer = uuid4().hex
while not stop.is_set():
@@ -81,15 +86,22 @@ async def consume_connected(
await broker.reject_invalid(delivery.id)
continue
try:
async with factory() as session:
acknowledge = await process_attachment_job(
session,
storage,
message,
settings,
after_object=faults.after_object,
before_processing=faults.before_processing,
)
telemetry = telemetry or Telemetry("messenger-worker", enabled=False)
parent = TraceContextTextMapPropagator().extract(delivery.id.headers or {}, context=Context())
with telemetry.span(
"attachment.process.v1", context=parent, kind=SpanKind.CONSUMER,
attributes={"messaging.system": "rabbitmq", "job.type": message.type,
"job.attempt": message.attempt},
):
telemetry.event("job.received", job_id=str(message.job_id),
attachment_id=str(message.attachment_id))
async with factory() as session:
acknowledge = await process_attachment_job(
session, storage, message, settings,
after_object=faults.after_object,
before_processing=faults.before_processing,
telemetry=telemetry,
)
except ProcessingError:
await broker.reject_invalid(delivery.id)
else:
@@ -104,10 +116,10 @@ async def consume_connected(
await wait_or_stop(stop, settings.publisher_poll_seconds)
async def consume(factory, storage, settings, stop, faults) -> None:
async def consume(factory, storage, settings, stop, faults, telemetry=None) -> None:
broker = JobBroker(settings)
try:
await consume_connected(factory, storage, broker, settings, stop, faults)
await consume_connected(factory, storage, broker, settings, stop, faults, telemetry)
finally:
await broker.aclose()
@@ -115,19 +127,23 @@ async def consume(factory, storage, settings, stop, faults) -> None:
async def run_worker(faults: Faults) -> None:
settings = Settings()
jobs = JobSettings()
telemetry = Telemetry("messenger-worker")
logging.basicConfig(level=settings.log_level)
if not features.check("webp"):
logging.error("Pillow requires WebP support for the image worker")
raise RuntimeError("WebP codec is unavailable")
stop = shutdown_event()
engine = create_db_engine(settings)
telemetry.observe_pool(engine.pool)
telemetry.gauge("messenger.worker.busy", lambda: telemetry.busy)
telemetry.gauge("messenger.worker.capacity", lambda: jobs.worker_concurrency)
storage = None
try:
storage = create_s3_storage(settings.s3)
factory = create_session_factory(engine)
async with asyncio.TaskGroup() as group:
tasks = [
group.create_task(consume(factory, storage, jobs, stop, faults))
group.create_task(consume(factory, storage, jobs, stop, faults, telemetry))
for _ in range(jobs.worker_concurrency)
]
await stop.wait()
@@ -138,6 +154,7 @@ async def run_worker(faults: Faults) -> None:
if storage is not None:
await storage.aclose()
await engine.dispose()
await asyncio.to_thread(telemetry.shutdown)
def main() -> None:
+16 -1
View File
@@ -6,6 +6,7 @@ import json
import logging
import os
import signal
import time
from uuid import UUID
from app.core.background import (
@@ -19,6 +20,8 @@ from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs
from app.storage.broker import JobBroker
from app.storage.jobs import queued_backlog
from app.core.telemetry import Telemetry
def crash_before_publish(attachment_id: UUID) -> None:
@@ -36,6 +39,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
logging.basicConfig(level=settings.log_level)
stop = shutdown_event()
engine = create_db_engine(settings)
telemetry = Telemetry("messenger-publisher")
telemetry.observe_pool(engine.pool)
backlog = (0, 0.0)
labels = {"queue.name": jobs.broker_queue}
telemetry.gauge("messenger.queue.depth", lambda: backlog[0], attributes=labels)
telemetry.gauge("messenger.queue.oldest", lambda: backlog[1], unit="s", attributes=labels)
next_sample = 0.0
broker = None
try:
broker = JobBroker(jobs)
@@ -43,12 +53,16 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
while not stop.is_set():
published = False
try:
await broker.ensure_ready()
async with factory() as session:
if telemetry.enabled and time.monotonic() >= next_sample:
backlog = await queued_backlog(session)
next_sample = time.monotonic() + 2
await broker.ensure_ready()
await recover_expired_jobs(session, jobs)
published = await publish_next_job(
session,
broker,
telemetry=telemetry,
attachment_id=crash_attachment_id,
before_publish=(
crash_before_publish
@@ -64,6 +78,7 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
if broker is not None:
await broker.aclose()
await engine.dispose()
await asyncio.to_thread(telemetry.shutdown)
def main() -> None:
+197
View File
@@ -0,0 +1,197 @@
"""Явная OTLP-инструментация: экспортируются только выбранные технические поля."""
from contextlib import contextmanager
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import socket
import time
from opentelemetry import trace
from opentelemetry.context import Context
from opentelemetry._logs import SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.metrics import Observation
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace import SpanKind, Status, StatusCode
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
JOB_BUCKETS = (0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
class Telemetry:
def __init__(self, service: str, instance: str | None = None, *, enabled=None):
self.enabled = (os.getenv("OTEL_ENABLED", "false").lower() == "true"
if enabled is None else enabled)
self.service = service
self.busy = 0
if not self.enabled:
self.tracer = trace.NoOpTracerProvider().get_tracer("messenger")
return
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
resource = Resource({
"service.name": service,
"service.version": "1.6.0",
"service.instance.id": instance or socket.gethostname(),
"deployment.environment.name": os.getenv("APP_ENV", "development"),
})
self.traces = TracerProvider(resource=resource)
self.traces.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
))
self.tracer = self.traces.get_tracer("messenger", "1.6.0")
self.logs = LoggerProvider(resource=resource)
self.logs.add_log_record_processor(BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
))
self.logger = self.logs.get_logger("messenger", "1.6.0")
self.metrics = MeterProvider(resource=resource, metric_readers=[
PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
export_interval_millis=5000, export_timeout_millis=3000,
)
], views=[
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
])
meter = self.meter = self.metrics.get_meter("messenger", "1.6.0")
self.http_requests = meter.create_counter("messenger.http.requests", unit="1")
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
self.jobs = meter.create_counter("messenger.job.attempts", unit="1")
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
self.time_to_ready = meter.create_histogram("messenger.job.time_to_ready", unit="s")
self.gauge("messenger.process.cpu", time.process_time, unit="s", counter=True)
self.gauge("messenger.process.rss", resident_bytes, unit="By")
def gauge(self, name, sample, *, unit="1", attributes=None, counter=False):
if self.enabled:
create = (self.meter.create_observable_counter if counter
else self.meter.create_observable_gauge)
create(name, callbacks=[lambda _: [Observation(sample(), attributes)]], unit=unit)
def observe_pool(self, pool):
if self.enabled:
pool.telemetry = self
self.gauge("messenger.db.pool.used", pool.checkedout)
self.gauge("messenger.db.pool.waiting", lambda: pool.waiting)
@contextmanager
def span(self, name, **kwargs):
with self.tracer.start_as_current_span(
name, record_exception=False, set_status_on_exception=False, **kwargs,
) as span:
try:
yield span
except BaseException:
span.set_status(Status(StatusCode.ERROR))
raise
def event(self, event: str, *, error=False, **fields):
if not self.enabled:
return
context = trace.get_current_span().get_span_context()
body = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"severity": "ERROR" if error else "INFO",
"service_name": self.service, "event": event,
"trace_id": f"{context.trace_id:032x}", "span_id": f"{context.span_id:016x}",
**fields,
}
text = json.dumps(body, ensure_ascii=False)
# Не подключаем root logging: SDK/SQL-исключения могут содержать секреты.
self.logger.emit(body=text, severity_text=body["severity"],
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
attributes={"event": event})
print(text, flush=True)
@contextmanager
def job_attempt(self):
self.busy += 1
started = time.monotonic()
result = {"outcome": "success"}
try:
yield result
except BaseException:
result["outcome"] = "error"
raise
finally:
self.busy -= 1
if self.enabled:
attrs = {"job.type": "attachment.process.v1", "outcome": result["outcome"]}
elapsed = time.monotonic() - started
self.jobs.add(1, attrs)
self.job_duration.record(elapsed, attrs)
span = trace.get_current_span()
span.set_attribute("outcome", result["outcome"])
span.set_status(Status(StatusCode.ERROR if result["outcome"] == "error" else StatusCode.OK))
self.event("job.processed", error=result["outcome"] == "error",
job_type="attachment.process.v1", duration_seconds=elapsed, **result)
def shutdown(self):
if self.enabled:
self.metrics.shutdown(timeout_millis=5000)
self.logs.shutdown()
self.traces.shutdown()
def resident_bytes():
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
class HttpTelemetryMiddleware:
def __init__(self, app, telemetry: Telemetry):
self.app = app
self.telemetry = telemetry
async def __call__(self, scope, receive, send):
if scope["type"] != "http" or not self.telemetry.enabled:
return await self.app(scope, receive, send)
telemetry = self.telemetry
method = scope["method"]
if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}:
method = "OTHER"
started = time.monotonic()
status = 500
request_id = ""
with telemetry.tracer.start_as_current_span(
"HTTP", context=Context(), kind=SpanKind.SERVER,
record_exception=False, set_status_on_exception=False,
) as span:
async def observed_send(message):
nonlocal status, request_id
if message["type"] == "http.response.start":
status = message["status"]
headers = list(message.get("headers", []))
request_id = next((v.decode() for k, v in headers if k.lower() == b"x-request-id"), "")
headers.append((b"x-trace-id", f"{span.get_span_context().trace_id:032x}".encode()))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, observed_send)
finally:
request_id = request_id or scope.get("state", {}).get("request_id", "")
route = getattr(scope.get("route"), "path", "unmatched")
attrs = {"http.request.method": method, "http.route": route,
"http.response.status_class": f"{status // 100}xx"}
elapsed = time.monotonic() - started
telemetry.http_requests.add(1, attrs)
telemetry.http_duration.record(elapsed, attrs)
span.update_name(f"{method} {route}")
span.set_attributes({**attrs, "http.response.status_code": status})
span.set_status(Status(StatusCode.ERROR if status >= 500 else StatusCode.OK))
telemetry.event("http.request", error=status >= 500, method=method, route=route,
status_code=status, request_id=request_id, duration_seconds=elapsed)
+73 -49
View File
@@ -2,16 +2,21 @@
import asyncio
from collections.abc import Callable
from contextlib import nullcontext
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from uuid import UUID, uuid4
from minio.error import MinioException
from opentelemetry.context import Context
from opentelemetry.trace import SpanKind
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from sqlalchemy.ext.asyncio import AsyncSession
from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.jobs import JobSettings
from app.core.telemetry import Telemetry
from app.domain.job_retry import classify_failure, retry_delay
from app.domain.job_files import fetch_original, render_variants
from app.errors.exceptions import ProcessingError
@@ -42,6 +47,7 @@ async def publish_next_job(
*,
attachment_id: UUID | None = None,
before_publish: Callable[[UUID], None] | None = None,
telemetry: Telemetry | None = None,
) -> bool:
"""Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль."""
job = await outbox.next_unpublished_job(session, attachment_id=attachment_id)
@@ -49,15 +55,20 @@ async def publish_next_job(
if job is not None:
if before_publish is not None:
before_publish(job.attachment_id)
await broker.publish_job(
JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
parent = TraceContextTextMapPropagator().extract(job.trace_context or {}, context=Context())
with telemetry.span(
"rabbitmq.publish", context=parent, kind=SpanKind.PRODUCER,
attributes={"messaging.system": "rabbitmq", "job.attempt": job.attempt + 1},
) if telemetry else nullcontext():
await broker.publish_job(
JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
)
)
outbox.mark_published(job)
await session.commit()
return published
@@ -143,11 +154,13 @@ async def build_result(
storage: MinioStorage,
work: JobWork,
after_object: Callable[[UUID, str], None] | None = None,
telemetry: Telemetry | None = None,
) -> bool:
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
with TemporaryDirectory(prefix="messenger-job-") as directory:
source = Path(directory) / "original"
await fetch_original(storage, work, source)
with telemetry.span("s3.read_staging") if telemetry else nullcontext():
await fetch_original(storage, work, source)
result = await render_variants(storage, work, source, Path(directory))
row = await jobs.lock_job(session, work.job_id)
if row is None:
@@ -163,33 +176,39 @@ async def build_result(
return False
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
await storage.put_file(
work.original_key,
source,
size_bytes=work.size_bytes,
content_type=work.content_type,
sha256=work.sha256,
)
if after_object is not None:
after_object(work.attachment_id, "original")
for variant in result.variants:
with telemetry.span("s3.write_result") if telemetry else nullcontext():
await storage.put_file(
work.variant_key(variant.name),
Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes,
content_type=variant.content_type,
sha256=variant.sha256,
work.original_key,
source,
size_bytes=work.size_bytes,
content_type=work.content_type,
sha256=work.sha256,
)
if after_object is not None:
after_object(work.attachment_id, variant.name)
jobs.complete_job(
job,
attachment,
work,
[variant.model_dump() for variant in result.variants],
await jobs.database_time(session),
)
await session.commit()
after_object(work.attachment_id, "original")
for variant in result.variants:
await storage.put_file(
work.variant_key(variant.name),
Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes,
content_type=variant.content_type,
sha256=variant.sha256,
)
if after_object is not None:
after_object(work.attachment_id, variant.name)
with telemetry.span("postgres.commit_ready") if telemetry else nullcontext():
ready_at = await jobs.database_time(session)
accepted_at = job.created_at
jobs.complete_job(
job,
attachment,
work,
[variant.model_dump() for variant in result.variants],
ready_at,
)
await session.commit()
if telemetry and telemetry.enabled:
telemetry.time_to_ready.record(max(0, (ready_at - accepted_at).total_seconds()))
return True
@@ -201,26 +220,31 @@ async def process_attachment_job(
*,
after_object: Callable[[UUID, str], None] | None = None,
before_processing: Callable[[UUID], None] | None = None,
telemetry: Telemetry | None = None,
) -> bool:
claim = await claim_job(session, message, settings)
with telemetry.span("postgres.claim_job") if telemetry else nullcontext():
claim = await claim_job(session, message, settings)
acknowledge = claim.acknowledge
if claim.work is not None:
work = claim.work
try:
async with asyncio.timeout(settings.job_timeout_seconds):
if before_processing is not None:
before_processing(work.attachment_id)
acknowledge = await build_result(session, storage, work, after_object)
except (
ProcessingError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
) as error:
code, permanent = classify_failure(error)
await save_failure(session, work, settings, code, permanent=permanent)
acknowledge = True
with telemetry.job_attempt() if telemetry else nullcontext({}) as result:
try:
async with asyncio.timeout(settings.job_timeout_seconds):
if before_processing is not None:
before_processing(work.attachment_id)
acknowledge = await build_result(session, storage, work, after_object, telemetry)
result["outcome"] = "success" if acknowledge else "deferred"
except (
ProcessingError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
) as error:
code, permanent = classify_failure(error)
await save_failure(session, work, settings, code, permanent=permanent)
result.update(outcome="error", error_code=code)
acknowledge = True
# Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку.
return acknowledge
+6 -2
View File
@@ -167,6 +167,10 @@ async def dependency_unavailable_handler(
request_id,
type(exc).__name__,
)
telemetry = getattr(request.app.state, "telemetry", None)
if telemetry is not None:
telemetry.event("dependency.unavailable", error=True,
request_id=request_id, exception_type=type(exc).__name__)
error = unavailable()
return _build_response(
@@ -206,9 +210,9 @@ async def unhandled_error_handler(
request_id = _get_request_id(request)
logger.error(
"Unhandled error request_id=%s",
"Unhandled error request_id=%s type=%s",
request_id,
exc_info=exc,
type(exc).__name__,
)
error = internal_error()
+9 -1
View File
@@ -1,3 +1,4 @@
import asyncio
import logging
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
@@ -12,6 +13,7 @@ from app.core.lifecycle import GracefulServer, Lifecycle
from app.core.redis import create_redis_client
from app.core.request_capacity import acquire_request_slot, create_request_capacity
from app.core.request_id import RequestIdMiddleware
from app.core.telemetry import HttpTelemetryMiddleware, Telemetry
from app.endpoints.auth import router as auth_router
from app.endpoints.attachments import router as attachments_router
from app.endpoints.chats import router as chats_router
@@ -44,8 +46,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
await s3.aclose()
await redis.aclose()
await engine.dispose()
await asyncio.to_thread(telemetry.shutdown)
application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan)
application = FastAPI(title="Messenger", version="1.6.0", lifespan=lifespan)
application.state.settings = settings
application.state.engine = engine
@@ -57,11 +60,15 @@ def create_app(settings: Settings | None = None) -> FastAPI:
)
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
application.state.instance_id = resolve_instance_id(settings.instance_id)
telemetry = Telemetry("messenger-api", application.state.instance_id)
telemetry.observe_pool(engine.pool)
application.state.telemetry = telemetry
application.state.lifecycle = Lifecycle()
application.add_middleware(RequestIdMiddleware)
application.add_middleware(
InstanceIdMiddleware, instance_id=application.state.instance_id
)
application.add_middleware(HttpTelemetryMiddleware, telemetry=telemetry)
register_exception_handlers(application)
limited = [Depends(acquire_request_slot)]
@@ -83,6 +90,7 @@ def main() -> None:
host=settings.app_host,
port=settings.app_port,
log_level=settings.log_level.lower(),
access_log=False,
timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
)
GracefulServer(
+2 -1
View File
@@ -3,7 +3,7 @@
from datetime import datetime
from uuid import UUID, uuid4
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy import JSON, CheckConstraint, ForeignKey, Index, String, func
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
@@ -23,6 +23,7 @@ class AttachmentJob(Base):
ForeignKey("attachments.id"),
unique=True,
)
trace_context: Mapped[dict | None] = mapped_column(JSON, nullable=True)
staging_key: Mapped[str | None] = mapped_column(String(128))
pipeline_version: Mapped[int] = mapped_column(server_default="1")
attempt: Mapped[int] = mapped_column(server_default="0")
+4
View File
@@ -6,6 +6,7 @@ from dataclasses import dataclass
import aio_pika
from aio_pika.abc import AbstractIncomingMessage
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from app.core.jobs import JobSettings
from app.schemas.job import JobMessage
@@ -54,10 +55,13 @@ class JobBroker:
async def publish_job(self, message: JobMessage) -> None:
await self.ensure_ready()
headers = {}
TraceContextTextMapPropagator().inject(headers)
await self.channel.default_exchange.publish(
aio_pika.Message(
body=message.model_dump_json(exclude_none=True).encode(),
content_type="application/json",
headers=headers,
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=f"{message.job_id}:{message.attempt}",
),
+14 -1
View File
@@ -4,6 +4,7 @@ from dataclasses import dataclass
from datetime import datetime, timedelta
from uuid import UUID
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from sqlalchemy import and_, or_, func, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -35,7 +36,10 @@ class JobWork:
async def create_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
carrier = {}
TraceContextTextMapPropagator().inject(carrier)
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key,
trace_context=carrier or None)
session.add(job)
await session.flush()
return job.id
@@ -56,6 +60,15 @@ async def database_time(session: AsyncSession) -> datetime:
return await session.scalar(select(func.clock_timestamp()))
async def queued_backlog(session: AsyncSession) -> tuple[int, float]:
count, oldest = (await session.execute(
select(func.count(), func.min(AttachmentJob.created_at))
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "queued")
)).one()
return count, max(0, ((await database_time(session)) - oldest).total_seconds()) if oldest else 0.0
async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
+45
View File
@@ -0,0 +1,45 @@
# Локальный доступ к сервисам из VS Code и браузера.
# docker compose -f compose.yaml -f compose.local.yml up -d --wait
# Все опубликованные порты доступны только с этого компьютера.
services:
api:
# Docker выделяет каждой реплике отдельный свободный порт диапазона.
ports:
- "127.0.0.1:8081-8090:8080"
postgres:
ports:
- "127.0.0.1:${POSTGRES_LOCAL_PORT:-5432}:5432"
redis:
ports:
- "127.0.0.1:${REDIS_LOCAL_PORT:-6379}:6379"
minio:
command: ["server", "/data", "--console-address", "0.0.0.0:9001"]
environment:
MINIO_BROWSER: "on"
# Добавляем обычную сеть для доступа через опубликованные порты;
# внутренняя objects сохраняется для приложения и bootstrap.
networks: [default, objects]
ports:
- "127.0.0.1:${MINIO_LOCAL_PORT:-9000}:9000"
- "127.0.0.1:${MINIO_CONSOLE_LOCAL_PORT:-9001}:9001"
broker:
volumes:
- ./infra/broker/enabled_plugins.local:/etc/rabbitmq/enabled_plugins:ro
ports:
- "127.0.0.1:${BROKER_LOCAL_PORT:-5672}:5672"
- "127.0.0.1:${BROKER_MANAGEMENT_LOCAL_PORT:-15672}:15672"
prometheus:
ports:
- "127.0.0.1:${PROMETHEUS_LOCAL_PORT:-9090}:9090"
loki:
ports:
- "127.0.0.1:${LOKI_LOCAL_PORT:-3100}:3100"
tempo:
ports:
- "127.0.0.1:${TEMPO_LOCAL_PORT:-3200}:3200"
- "127.0.0.1:${TEMPO_OTLP_LOCAL_PORT:-14318}:4318"
otel-collector:
ports:
- "127.0.0.1:${OTEL_HTTP_LOCAL_PORT:-4318}:4318"
- "127.0.0.1:${OTEL_METRICS_LOCAL_PORT:-8889}:8889"
- "127.0.0.1:${OTEL_HEALTH_LOCAL_PORT:-13133}:13133"
+99
View File
@@ -11,6 +11,8 @@ x-base-env: &base-env
APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
OTEL_ENABLED: ${OTEL_ENABLED:-true}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
@@ -388,11 +390,106 @@ services:
cpus: "0.5"
memory: 128M
otel-collector:
image: otel/opentelemetry-collector-contrib:0.165.0
command: ["--config=/etc/otelcol/config.yaml"]
restart: unless-stopped
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
prometheus:
image: prom/prometheus:v3.16.0
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=3d"]
restart: unless-stopped
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheusdata:/prometheus
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
loki:
image: grafana/loki:3.7.10
command: ["-config.file=/etc/loki/config.yaml"]
restart: unless-stopped
volumes:
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
- lokidata:/loki
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
tempo:
image: grafana/tempo:2.9.0
user: "0:0"
command: ["-config.file=/etc/tempo/config.yaml"]
restart: unless-stopped
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
- tempodata:/var/tempo
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
grafana:
image: grafana/grafana:13.3.1
restart: unless-stopped
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
secrets:
- grafana_admin_password
ports:
- "127.0.0.1:${GRAFANA_HOST_PORT:-13000}:3000"
volumes:
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
healthcheck:
# Loki/Tempo/Collector не содержат shell/HTTP-клиент: проверяем их из Grafana.
test:
- CMD-SHELL
- wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health && wget -q -T 2 -O /dev/null http://loki:3100/ready && wget -q -T 2 -O /dev/null http://tempo:3200/ready && wget -q -T 2 -O /dev/null http://otel-collector:13133/
interval: 5s
timeout: 10s
retries: 36
deploy:
resources:
limits:
cpus: "0.5"
memory: 512M
volumes:
pgdata:
redisdata:
miniodata:
rabbitmqdata:
prometheusdata:
lokidata:
tempodata:
grafanadata:
networks:
default:
@@ -400,6 +497,8 @@ networks:
internal: true
secrets:
grafana_admin_password:
file: ./secrets/grafana_admin_password
broker_password:
file: ./secrets/broker_password
db_password:
+8 -3
View File
@@ -1,5 +1,10 @@
# Доказательства
# Результаты проверок
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
`make verify` заново собирает два файла одного прогона:
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
- `verification.json`: команды, время, результат регрессии, API/отказов и очистки.
- `observability.json`: нагрузка/CPU/RAM, метрики, корреляция логов и traces,
privacy, dashboard, инциденты и сохранность истории после restart.
Сохраняются только
синтетические технические данные; credentials, токены и тела сообщений исключены.
+888
View File
@@ -0,0 +1,888 @@
{
"date_utc": "2026-11-19T15:56:59.724115+00:00",
"project": "messenger-test",
"load": {
"date_utc": "2026-11-19T15:56:52.469419+00:00",
"workload": "synthetic POST messages; fixed arrival rate; no retries",
"duration_seconds": 10,
"elapsed_seconds": 9.923,
"arrival_rate": 10,
"concurrency_limit": 2,
"successful_requests": 100,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 10.078,
"successful_latency_ms": {
"p50": 18.036,
"p95": 39.623,
"p99": 45.954
},
"dataset": {
"users": 3,
"chats": 1,
"messages": 100
},
"resource_samples": [
{
"elapsed_seconds": 2.058,
"containers": [
{
"Name": "messenger-test-api-1",
"CPUPerc": "40.63%",
"MemUsage": "94.64MiB / 1.5GiB",
"MemPerc": "6.16%"
},
{
"Name": "messenger-test-api-2",
"CPUPerc": "5.96%",
"MemUsage": "87.06MiB / 1.5GiB",
"MemPerc": "5.67%"
},
{
"Name": "messenger-test-broker-1",
"CPUPerc": "86.17%",
"MemUsage": "148.3MiB / 768MiB",
"MemPerc": "19.31%"
},
{
"Name": "messenger-test-cleanup-1",
"CPUPerc": "0.00%",
"MemUsage": "74.44MiB / 256MiB",
"MemPerc": "29.08%"
},
{
"Name": "messenger-test-grafana-1",
"CPUPerc": "0.47%",
"MemUsage": "292.7MiB / 512MiB",
"MemPerc": "57.17%"
},
{
"Name": "messenger-test-loki-1",
"CPUPerc": "0.58%",
"MemUsage": "69.43MiB / 512MiB",
"MemPerc": "13.56%"
},
{
"Name": "messenger-test-minio-1",
"CPUPerc": "0.08%",
"MemUsage": "131.4MiB / 512MiB",
"MemPerc": "25.67%"
},
{
"Name": "messenger-test-nginx-1",
"CPUPerc": "0.27%",
"MemUsage": "5.359MiB / 128MiB",
"MemPerc": "4.19%"
},
{
"Name": "messenger-test-otel-collector-1",
"CPUPerc": "0.30%",
"MemUsage": "62.13MiB / 256MiB",
"MemPerc": "24.27%"
},
{
"Name": "messenger-test-postgres-1",
"CPUPerc": "1.54%",
"MemUsage": "107.4MiB / 512MiB",
"MemPerc": "20.99%"
},
{
"Name": "messenger-test-prometheus-1",
"CPUPerc": "0.14%",
"MemUsage": "37.91MiB / 512MiB",
"MemPerc": "7.41%"
},
{
"Name": "messenger-test-publisher-1",
"CPUPerc": "0.73%",
"MemUsage": "65.48MiB / 256MiB",
"MemPerc": "25.58%"
},
{
"Name": "messenger-test-redis-1",
"CPUPerc": "3.00%",
"MemUsage": "8.527MiB / 512MiB",
"MemPerc": "1.67%"
},
{
"Name": "messenger-test-tempo-1",
"CPUPerc": "0.30%",
"MemUsage": "318.4MiB / 512MiB",
"MemPerc": "62.19%"
},
{
"Name": "messenger-test-worker-1",
"CPUPerc": "0.19%",
"MemUsage": "73.34MiB / 2GiB",
"MemPerc": "3.58%"
}
]
},
{
"elapsed_seconds": 7.107,
"containers": [
{
"Name": "messenger-test-api-1",
"CPUPerc": "7.55%",
"MemUsage": "94.69MiB / 1.5GiB",
"MemPerc": "6.16%"
},
{
"Name": "messenger-test-api-2",
"CPUPerc": "39.26%",
"MemUsage": "87.07MiB / 1.5GiB",
"MemPerc": "5.67%"
},
{
"Name": "messenger-test-broker-1",
"CPUPerc": "15.48%",
"MemUsage": "189.4MiB / 768MiB",
"MemPerc": "24.66%"
},
{
"Name": "messenger-test-cleanup-1",
"CPUPerc": "0.00%",
"MemUsage": "74.44MiB / 256MiB",
"MemPerc": "29.08%"
},
{
"Name": "messenger-test-grafana-1",
"CPUPerc": "0.55%",
"MemUsage": "292.7MiB / 512MiB",
"MemPerc": "57.17%"
},
{
"Name": "messenger-test-loki-1",
"CPUPerc": "0.56%",
"MemUsage": "69.18MiB / 512MiB",
"MemPerc": "13.51%"
},
{
"Name": "messenger-test-minio-1",
"CPUPerc": "3.59%",
"MemUsage": "135.3MiB / 512MiB",
"MemPerc": "26.42%"
},
{
"Name": "messenger-test-nginx-1",
"CPUPerc": "0.25%",
"MemUsage": "4.82MiB / 128MiB",
"MemPerc": "3.77%"
},
{
"Name": "messenger-test-otel-collector-1",
"CPUPerc": "0.64%",
"MemUsage": "62.66MiB / 256MiB",
"MemPerc": "24.48%"
},
{
"Name": "messenger-test-postgres-1",
"CPUPerc": "7.61%",
"MemUsage": "107.5MiB / 512MiB",
"MemPerc": "20.99%"
},
{
"Name": "messenger-test-prometheus-1",
"CPUPerc": "0.21%",
"MemUsage": "38.25MiB / 512MiB",
"MemPerc": "7.47%"
},
{
"Name": "messenger-test-publisher-1",
"CPUPerc": "0.65%",
"MemUsage": "65.73MiB / 256MiB",
"MemPerc": "25.68%"
},
{
"Name": "messenger-test-redis-1",
"CPUPerc": "0.48%",
"MemUsage": "8.773MiB / 512MiB",
"MemPerc": "1.71%"
},
{
"Name": "messenger-test-tempo-1",
"CPUPerc": "0.28%",
"MemUsage": "320.2MiB / 512MiB",
"MemPerc": "62.53%"
},
{
"Name": "messenger-test-worker-1",
"CPUPerc": "0.01%",
"MemUsage": "73.34MiB / 2GiB",
"MemPerc": "3.58%"
}
]
}
],
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark."
},
"api_replicas": 2,
"dashboard_uid": "messenger-overview",
"api_correlation": {
"trace_id": "b07429169c91eb6e67d211959181ab10",
"span_id": "96318acd3dddff53",
"log": {
"timestamp": "2026-11-19T15:56:54.681885+00:00",
"severity": "INFO",
"service_name": "messenger-api",
"event": "http.request",
"trace_id": "b07429169c91eb6e67d211959181ab10",
"span_id": "96318acd3dddff53",
"method": "POST",
"route": "/api/v1/chats/{chat_id}/messages",
"status_code": 201,
"request_id": "171446ed-c27f-45e0-86b3-1b6e06420b4e",
"duration_seconds": 0.08245528700001614
},
"spans": [
{
"name": "POST /api/v1/chats/{chat_id}/messages",
"kind": "SPAN_KIND_SERVER",
"spanId": "ljGKzT3d/1M=",
"startTimeUnixNano": "1795103814599010618",
"endTimeUnixNano": "1795103814682418591",
"status": {
"code": "STATUS_CODE_OK"
}
}
]
},
"worker_correlation": {
"trace_id": "ad41b35e5e8b02c32f7a7b74d7b6278a",
"span_id": "ff9586c715f69b64",
"log": {
"timestamp": "2026-11-19T15:56:55.880853+00:00",
"severity": "INFO",
"service_name": "messenger-worker",
"event": "job.received",
"trace_id": "ad41b35e5e8b02c32f7a7b74d7b6278a",
"span_id": "ff9586c715f69b64",
"job_id": "ce2dd02a-7263-43b3-9cdb-1caf53fcb262",
"attachment_id": "d49c525e-6cab-4a88-b348-867cc4185499"
},
"spans": [
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"spanId": "/5WGxxX2m2Q=",
"startTimeUnixNano": "1795103815880827668",
"endTimeUnixNano": "1795103816744588122",
"status": {
"code": "STATUS_CODE_OK"
}
}
]
},
"metrics": {
"http_requests": [
{
"metric": {},
"value": [
1795103819.091,
"383"
]
}
],
"http_duration": [
{
"metric": {},
"value": [
1795103819.104,
"383"
]
}
],
"job_success": [
{
"metric": {},
"value": [
1795103819.119,
"1"
]
}
],
"job_duration": [
{
"metric": {},
"value": [
1795103819.132,
"1"
]
}
]
},
"upload_trace": {
"trace_id": "ad41b35e5e8b02c32f7a7b74d7b6278a",
"chain": [
{
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"span_id": "74b246705b53c840",
"parent_span_id": "",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "cf8900edf04666d0",
"parent_span_id": "74b246705b53c840",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "dc111544195865ab",
"parent_span_id": "74b246705b53c840",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "ea30143c06d7101a",
"parent_span_id": "74b246705b53c840",
"job": {}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "9414f5609a1615fc",
"parent_span_id": "74b246705b53c840",
"job": {
"job.attempt": {
"intValue": "1"
}
}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "ff9586c715f69b64",
"parent_span_id": "9414f5609a1615fc",
"job": {
"job.attempt": {
"intValue": "1"
},
"outcome": {
"stringValue": "success"
}
}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "3a99027f12c54d14",
"parent_span_id": "ff9586c715f69b64",
"job": {}
},
{
"name": "s3.write_result",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "7238fed2427ff2b5",
"parent_span_id": "ff9586c715f69b64",
"job": {}
},
{
"name": "postgres.commit_ready",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "6228546b59c97cc1",
"parent_span_id": "ff9586c715f69b64",
"job": {}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "d038f59546905b0d",
"parent_span_id": "ff9586c715f69b64",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "8ddfdeb23bfdca06",
"parent_span_id": "d038f59546905b0d",
"job": {}
},
{
"name": "s3.read_staging",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "87438655356f817f",
"parent_span_id": "ff9586c715f69b64",
"job": {}
}
]
},
"incidents": [
{
"scenario": "postgres_unavailable",
"http_status": 503,
"correlation": {
"trace_id": "020e820d68045ad3195e842244670b11",
"span_id": "d100199e5812f6e3",
"log": {
"timestamp": "2026-11-19T15:57:03.906932+00:00",
"severity": "ERROR",
"service_name": "messenger-api",
"event": "http.request",
"trace_id": "020e820d68045ad3195e842244670b11",
"span_id": "d100199e5812f6e3",
"method": "POST",
"route": "/api/v1/users",
"status_code": 503,
"request_id": "420eae93-41a6-41af-8f37-09e76b8ec997",
"duration_seconds": 3.004546160000018
},
"spans": [
{
"name": "POST /api/v1/users",
"kind": "SPAN_KIND_SERVER",
"spanId": "0QAZnlgS9uM=",
"startTimeUnixNano": "1795103820902285656",
"endTimeUnixNano": "1795103823907032467",
"status": {
"code": "STATUS_CODE_ERROR"
}
}
]
},
"diagnostic": {
"timestamp": "2026-11-19T15:57:03.905870+00:00",
"severity": "ERROR",
"service_name": "messenger-api",
"event": "dependency.unavailable",
"trace_id": "020e820d68045ad3195e842244670b11",
"span_id": "d100199e5812f6e3",
"request_id": "420eae93-41a6-41af-8f37-09e76b8ec997",
"exception_type": "TimeoutError"
},
"db_span": "postgres.checkout",
"alert": "MessengerHighErrorRate",
"firing_after_seconds": 32.361,
"resolved_after_seconds": 67.526
},
{
"scenario": "worker_stopped_then_transient_retry",
"alert": "MessengerQueueLag",
"queue": {
"depth": 1.0,
"oldest_seconds": 25.641574,
"firing_after_seconds": 28.639,
"resolved": true,
"final_depth": 0
},
"recovered_status": "ready",
"trace_id": "ff8352e27270fbc16fea7216c3ff45c0",
"chain": [
{
"name": "POST /api/v1/chats/{chat_id}/attachments",
"kind": "SPAN_KIND_SERVER",
"span_id": "a8daa5d1e858d802",
"parent_span_id": "",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "1bfe66add09278e5",
"parent_span_id": "a8daa5d1e858d802",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "cf5bdfb14aca0e03",
"parent_span_id": "a8daa5d1e858d802",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "fe476468bb06d3f2",
"parent_span_id": "a8daa5d1e858d802",
"job": {}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "07218fe96edd8a9f",
"parent_span_id": "a8daa5d1e858d802",
"job": {
"job.attempt": {
"intValue": "1"
}
}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "93984e580b7e86d5",
"parent_span_id": "a8daa5d1e858d802",
"job": {
"job.attempt": {
"intValue": "2"
}
}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "a458e41372613973",
"parent_span_id": "a8daa5d1e858d802",
"job": {
"job.attempt": {
"intValue": "3"
}
}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "bbf04d7c40b14fb3",
"parent_span_id": "a8daa5d1e858d802",
"job": {
"job.attempt": {
"intValue": "4"
}
}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "3c0af738c589e75e",
"parent_span_id": "07218fe96edd8a9f",
"job": {
"job.attempt": {
"intValue": "1"
},
"outcome": {
"stringValue": "error"
}
}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "36993e3e09207c51",
"parent_span_id": "3c0af738c589e75e",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "ea60348b52b97e80",
"parent_span_id": "36993e3e09207c51",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "a6ce9125f0f2b69a",
"parent_span_id": "3c0af738c589e75e",
"job": {}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "f7b2eab20244c154",
"parent_span_id": "93984e580b7e86d5",
"job": {
"job.attempt": {
"intValue": "2"
},
"outcome": {
"stringValue": "error"
}
}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "06c6bcc9ba7a345d",
"parent_span_id": "f7b2eab20244c154",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "83c7a52ecd57f72f",
"parent_span_id": "06c6bcc9ba7a345d",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "0310e2dd03331cb3",
"parent_span_id": "f7b2eab20244c154",
"job": {}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "f2c43df28f9d7a39",
"parent_span_id": "a458e41372613973",
"job": {
"job.attempt": {
"intValue": "3"
},
"outcome": {
"stringValue": "error"
}
}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "b81945a1dac3e54e",
"parent_span_id": "f2c43df28f9d7a39",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "206ffc2a7aaafca5",
"parent_span_id": "b81945a1dac3e54e",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "f6f62fbcc37d065f",
"parent_span_id": "f2c43df28f9d7a39",
"job": {}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "caf2f1cf5ab31318",
"parent_span_id": "bbf04d7c40b14fb3",
"job": {
"job.attempt": {
"intValue": "4"
},
"outcome": {
"stringValue": "error"
}
}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "0e0bdca52b68b3d7",
"parent_span_id": "caf2f1cf5ab31318",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "ff6de1ac78d07a28",
"parent_span_id": "0e0bdca52b68b3d7",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "2f459882491c3a7c",
"parent_span_id": "caf2f1cf5ab31318",
"job": {}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "0a79dcefe6527245",
"parent_span_id": "a8daa5d1e858d802",
"job": {
"job.attempt": {
"intValue": "5"
}
}
},
{
"name": "rabbitmq.publish",
"kind": "SPAN_KIND_PRODUCER",
"span_id": "a4e8cf406651958d",
"parent_span_id": "a8daa5d1e858d802",
"job": {
"job.attempt": {
"intValue": "6"
}
}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "6d709c8f5827479d",
"parent_span_id": "0a79dcefe6527245",
"job": {
"job.attempt": {
"intValue": "5"
},
"outcome": {
"stringValue": "error"
}
}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "0a5ed8ae1cdac4bd",
"parent_span_id": "6d709c8f5827479d",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "a6452306465ac351",
"parent_span_id": "0a5ed8ae1cdac4bd",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "d5c93140a00b63ab",
"parent_span_id": "6d709c8f5827479d",
"job": {}
},
{
"name": "attachment.process.v1",
"kind": "SPAN_KIND_CONSUMER",
"span_id": "45c2e49b6f6471bf",
"parent_span_id": "a4e8cf406651958d",
"job": {
"job.attempt": {
"intValue": "6"
},
"outcome": {
"stringValue": "success"
}
}
},
{
"name": "postgres.claim_job",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "6d995da536f4d248",
"parent_span_id": "45c2e49b6f6471bf",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "ab6ddaa6b67c5663",
"parent_span_id": "6d995da536f4d248",
"job": {}
},
{
"name": "s3.read_staging",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "ec1037446c0d5918",
"parent_span_id": "45c2e49b6f6471bf",
"job": {}
},
{
"name": "postgres.checkout",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "4059610a72cf3cd0",
"parent_span_id": "45c2e49b6f6471bf",
"job": {}
},
{
"name": "s3.write_result",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "741f62682fa8057f",
"parent_span_id": "45c2e49b6f6471bf",
"job": {}
},
{
"name": "postgres.commit_ready",
"kind": "SPAN_KIND_INTERNAL",
"span_id": "17c7adc7b583eeda",
"parent_span_id": "45c2e49b6f6471bf",
"job": {}
}
]
},
{
"scenario": "loki_unavailable",
"successful_messages": 204,
"dropped_after_export_failure": 6.0,
"dropped_at_enqueue": 0.0,
"samples": [
{
"seconds": 0.06,
"rss_bytes": 239079424.0,
"queued_batches": 0.0,
"capacity_batches": 256.0
},
{
"seconds": 5.14,
"rss_bytes": 239288320.0,
"queued_batches": 5.0,
"capacity_batches": 256.0
},
{
"seconds": 10.08,
"rss_bytes": 239292416.0,
"queued_batches": 10.0,
"capacity_batches": 256.0
},
{
"seconds": 15.12,
"rss_bytes": 240283648.0,
"queued_batches": 15.0,
"capacity_batches": 256.0
},
{
"seconds": 20.07,
"rss_bytes": 241635328.0,
"queued_batches": 20.0,
"capacity_batches": 256.0
},
{
"seconds": 25.09,
"rss_bytes": 241795072.0,
"queued_batches": 24.0,
"capacity_batches": 256.0
},
{
"seconds": 30.07,
"rss_bytes": 241836032.0,
"queued_batches": 29.0,
"capacity_batches": 256.0
},
{
"seconds": 35.09,
"rss_bytes": 241856512.0,
"queued_batches": 33.0,
"capacity_batches": 256.0
},
{
"seconds": 40.09,
"rss_bytes": 241860608.0,
"queued_batches": 38.0,
"capacity_batches": 256.0
},
{
"seconds": 45.09,
"rss_bytes": 242176000.0,
"queued_batches": 43.0,
"capacity_batches": 256.0
},
{
"seconds": 50.07,
"rss_bytes": 242384896.0,
"queued_batches": 48.0,
"capacity_batches": 256.0
}
],
"log_export_recovered": true
}
],
"privacy_checked_logs": 413,
"checks": {
"three_backends": true,
"dashboard_queries": true,
"server_consumer_traces": true,
"log_trace_correlation": true,
"low_cardinality": true,
"privacy": true,
"history_survives_restart": true
}
}
+140
View File
@@ -0,0 +1,140 @@
{
"date_utc": "2026-11-19T15:50:02.598982+00:00",
"checks": [
{
"name": "contract",
"command": [
".venv/bin/python",
"scripts/check_contract.py"
],
"exit_code": 0,
"duration_seconds": 0.112,
"result": "PASS"
},
{
"name": "openapi",
"command": [
".venv/bin/python",
"scripts/validate_openapi.py"
],
"exit_code": 0,
"duration_seconds": 1.068,
"result": "PASS"
},
{
"name": "compose",
"command": [
"docker",
"compose",
"--env-file",
".env",
"config",
"--quiet"
],
"exit_code": 0,
"duration_seconds": 0.113,
"result": "PASS"
},
{
"name": "startup",
"command": [
"make",
"--no-print-directory",
"test-stand-up"
],
"exit_code": 0,
"duration_seconds": 70.403,
"result": "PASS"
},
{
"name": "regression",
"command": [
"make",
"--no-print-directory",
"test-unit"
],
"exit_code": 0,
"duration_seconds": 322.597,
"result": "PASS",
"summary": "206 passed in 98.89s (0:01:38)"
},
{
"name": "api",
"command": [
".venv/bin/python",
"tests/smoke.py",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"duration_seconds": 4.112,
"result": "PASS"
},
{
"name": "observability",
"command": [
".venv/bin/python",
"-m",
"scripts.check_observability",
"--base-url",
"http://localhost:28080",
"--grafana-url",
"http://localhost:23000",
"--incidents",
"--restart-telemetry"
],
"exit_code": 0,
"duration_seconds": 292.686,
"result": "PASS"
},
{
"name": "persistence",
"command": [
".venv/bin/python",
"tests/resilience.py",
"--mode",
"persistence",
"--action-script",
"scripts/restart.sh",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"duration_seconds": 61.526,
"result": "PASS"
},
{
"name": "failover",
"command": [
".venv/bin/python",
"tests/resilience.py",
"--mode",
"failover",
"--action-script",
"scripts/stop-one.sh",
"--base-url",
"http://localhost:28080"
],
"exit_code": 0,
"duration_seconds": 19.039,
"result": "PASS"
}
],
"profile": {
"single_stack": true,
"load_duration_seconds": 10,
"load_requests_per_second": 10,
"load_concurrency": 2
},
"result": "PASS",
"cleanup": {
"project": "messenger-test",
"remaining": {
"containers": [],
"networks": [],
"volumes": []
},
"result": "PASS"
},
"finished_at_utc": "2026-11-19T16:03:05.425775+00:00"
}
+17 -4
View File
@@ -1,7 +1,20 @@
# Конфигурация наблюдаемости — реализовать
# Наблюдаемость Messenger
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
API, publisher и worker экспортируют OTLP HTTP/protobuf в Collector (:4318).
Метрики выдаются Prometheus на :8889, логи отправляются в native OTLP Loki,
traces — в Tempo. Grafana читает три хранилища через provisioned datasources.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
Конфиги backend, Collector, dashboard и provisioning находятся в этом каталоге.
Dashboard UID: `messenger-overview`. Метрики — в секундах и счётчиках попыток;
точные запросы/единицы находятся в JSON. Resource environment берётся из APP_ENV.
В Loki индексируются service.name и окружение; instance ID — structured metadata.
Строка лога содержит trace/span ID, ссылка TraceID открывает Tempo.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
История хранится в prometheusdata/lokidata/tempodata, retention 3 дня;
grafanadata содержит настройки Grafana. Внешний порт имеет только Grafana,
по умолчанию 127.0.0.1:13000. Её healthcheck проверяет также Loki, Tempo и Collector.
Бизнес-readiness API от этих сервисов не зависит.
`make test-observability` запускает контролируемые отказы на указанном стенде.
`make verify` создаёт отдельный проект и удаляет его ресурсы после прогона.
Правила Prometheus, запуск и порядок диагностики описаны в [REPORT.md](../../REPORT.md).
+18
View File
@@ -0,0 +1,18 @@
groups:
- name: messenger
interval: 5s
rules:
- alert: MessengerHighErrorRate
expr: |
sum(increase(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[30s]))
/ clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[30s])), 1) > 0.05
for: 5s
annotations:
summary: "Более 5% бизнес-запросов завершились HTTP 5xx за 30 секунд"
runbook: "REPORT.md#диагностика-инцидентов"
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_seconds) > 15 and max(messenger_queue_depth_ratio) > 0
for: 5s
annotations:
summary: "Старейшее ожидающее задание находится в очереди более 15 секунд"
runbook: "REPORT.md#диагностика-инцидентов"
+61
View File
@@ -0,0 +1,61 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 128
send_batch_max_size: 256
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 3s
sending_queue:
queue_size: 256
num_consumers: 2
retry_on_failure:
max_elapsed_time: 30s
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 3s
sending_queue:
queue_size: 256
num_consumers: 2
retry_on_failure:
max_elapsed_time: 30s
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
telemetry:
metrics:
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
extensions: [health_check]
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
@@ -0,0 +1,617 @@
{
"uid": "messenger-overview",
"title": "Messenger — наблюдаемость",
"schemaVersion": 39,
"version": 1,
"tags": [
"messenger",
"observability"
],
"timezone": "browser",
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "API: запросы в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "sum by (http_request_method, http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m]))",
"legendFormat": "{{http_request_method}} {{http_route}}"
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {}
},
{
"id": 2,
"title": "API: доля 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{http_response_status_class=\"5xx\",http_route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m])), 0.000001)",
"legendFormat": "5xx / все бизнес-запросы"
}
],
"description": "Ошибки сервера: 5xx / все запросы, кроме health. 4xx не считаются отказом сервера.",
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {}
},
{
"id": 3,
"title": "API: время ответа p50/p95/p99",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 9,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
"legendFormat": "p99"
}
],
"description": "Histogram в секундах, buckets суммируются по репликам.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 4,
"title": "Worker: попытки в секунду по исходу",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 9,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (rate(messenger_job_attempts_total[1m]))",
"legendFormat": "{{outcome}}"
}
],
"description": "success — результат записан; error — попытка не удалась; deferred — истёк lease. Дубли без обработки не считаются.",
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {}
},
{
"id": 5,
"title": "Worker: время обработки p95",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 18,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[1m])))",
"legendFormat": "{{outcome}}"
}
],
"description": "Одна фактическая попытка, включая S3, преобразования и фиксацию metadata.",
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 6,
"title": "Worker: обработанные попытки",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 18,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "sum by (outcome) (messenger_job_attempts_total)",
"legendFormat": "{{outcome}}"
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 7,
"title": "API: структурные логи",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 0,
"y": 27,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "{service_name=\"messenger-api\"} | json",
"queryType": "range"
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 8,
"title": "Worker: структурные логи",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"x": 12,
"y": 27,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "{service_name=\"messenger-worker\"} | json",
"queryType": "range"
}
],
"description": "",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 9,
"title": "Реальные server/consumer traces",
"type": "table",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"gridPos": {
"x": 0,
"y": 36,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"queryType": "traceqlSearch",
"query": "{ resource.service.name =~ \"messenger-api|messenger-worker\" }",
"limit": 20
}
],
"description": "Открыть trace в Explore; ссылка TraceID из строки Loki ведёт к той же трассе.",
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 10,
"title": "API: число 5xx бизнес-запросов",
"type": "stat",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 36,
"w": 12,
"h": 9
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_http_requests_total{http_response_status_class=\"5xx\",http_route!~\"/health/.*\"}) or vector(0)",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
},
"description": "Накопленное число 5xx: показывает и первую ошибку нового ряда, когда rate ещё не имеет двух разных точек."
},
{
"id": 11,
"title": "Очередь: ожидающие задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 45
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max by (queue_name) (messenger_queue_depth_ratio)",
"legendFormat": "Глубина {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 12,
"title": "Очередь: возраст старейшей задачи",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 45
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max by (queue_name) (messenger_queue_oldest_seconds)",
"legendFormat": "Возраст {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 13,
"title": "PostgreSQL: соединения и ожидающие",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 53
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_db_pool_used_ratio)",
"legendFormat": "Занято {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"refId": "B",
"expr": "sum by (service_name) (messenger_db_pool_waiting_ratio)",
"legendFormat": "Ожидают {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 14,
"title": "Worker: доля занятых слотов",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 53
},
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy_ratio) / sum(messenger_worker_capacity_ratio)",
"legendFormat": "Загрузка {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 15,
"title": "Процессы: CPU в ядрах",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 61
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
"legendFormat": "CPU {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 16,
"title": "Процессы: резидентная память",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 61
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
"legendFormat": "RSS {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 17,
"title": "Вложения: p95 времени до ready",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 69
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "p95 {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
},
{
"id": 18,
"title": "Collector: очереди экспорта",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 69
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"targets": [
{
"refId": "A",
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
"legendFormat": "Размер {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
},
{
"refId": "B",
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
"legendFormat": "Ёмкость {{service_name}}{{queue_name}}{{exporter}}",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
}
}
]
}
],
"annotations": {
"list": []
},
"templating": {
"list": []
}
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: Messenger
folder: Messenger
type: file
disableDeletion: true
editable: false
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,36 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
editable: false
jsonData:
derivedFields:
- name: TraceID
matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"'
datasourceUid: tempo
url: '$${__value.raw}'
urlDisplayLabel: Open trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-1m'
spanEndTimeShift: '1m'
tags: [{key: service.name, value: service_name}]
filterByTraceID: true
filterBySpanID: false
+39
View File
@@ -0,0 +1,39 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 72h
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
- action: structured_metadata
attributes: [service.version, service.instance.id]
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+11
View File
@@ -0,0 +1,11 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: messenger
static_configs:
- targets: [otel-collector:8889]
- job_name: collector
static_configs:
- targets: [otel-collector:8888]
rule_files: [/etc/prometheus/alerts.yaml]
+22
View File
@@ -0,0 +1,22 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 1m
compactor:
compaction:
block_retention: 72h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
@@ -0,0 +1,17 @@
"""Сохранить W3C context между HTTP-транзакцией и publisher."""
import sqlalchemy as sa
from alembic import op
revision = "b55d0c6819ab"
down_revision = "a62c950e18b3"
branch_labels = None
depends_on = None
def upgrade():
op.add_column("attachment_jobs", sa.Column("trace_context", sa.JSON(), nullable=True))
def downgrade():
op.drop_column("attachment_jobs", "trace_context")
+3
View File
@@ -21,6 +21,9 @@ MarkupSafe==3.0.4
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
opentelemetry-api==1.45.1
opentelemetry-sdk==1.45.1
opentelemetry-exporter-otlp-proto-http==1.45.1
opentelemetry-semantic-conventions==0.66b1
packaging==26.3
pathable==0.4.4
pluggy==1.7.0
+5 -1
View File
@@ -179,7 +179,7 @@ def crash(client, actor, chat, mode):
)
def retry_replay(client, actor, chat, permanent):
def retry_replay(client, actor, chat, permanent, on_failed=None, on_queued=None):
name = "messenger-retry-fault-" + uuid4().hex
samples = []
configuration = ComposeProject().config["services"]["worker"]["environment"]
@@ -188,6 +188,8 @@ def retry_replay(client, actor, chat, permanent):
try:
require_compose("stop", "worker")
attachment, original = client.attachment(actor, chat)
if on_queued is not None:
on_queued(attachment)
mode = "fail-permanent" if permanent else "fail-transient"
require_compose(
"run",
@@ -238,6 +240,8 @@ def retry_replay(client, actor, chat, permanent):
actor=actor,
expected=409,
)
if on_failed is not None:
on_failed(sample, name)
job_id = sample["job_id"]
request_id = str(uuid4())
listing = admin("list-failed")
+395
View File
@@ -0,0 +1,395 @@
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
В evidence сохраняются только агрегаты и выбранные технические поля.
"""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import math
import subprocess
import threading
from datetime import datetime, timezone
import json
from pathlib import Path
import time
import urllib.parse
import urllib.request
from uuid import uuid4
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, check
ROOT = Path(__file__).resolve().parents[1]
def eventually(operation, message, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
value = operation()
if value:
return value
except (OSError, ValueError, KeyError, AssertionError):
pass
time.sleep(1)
raise AssertionError(message)
class Grafana:
def __init__(self, base):
self.base = base.rstrip('/')
secret = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
self.authorization = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode()
def get(self, path, **params):
url = self.base + path
if params:
url += '?' + urllib.parse.urlencode(params)
request = urllib.request.Request(url, headers={'Authorization': self.authorization, 'Accept': 'application/json'})
with urllib.request.urlopen(request, timeout=10) as response:
return json.load(response)
def proxy(self, source, path, **params):
return self.get('/api/datasources/proxy/uid/' + source + path, **params)
def query(self, expression):
value = self.proxy('prometheus', '/api/v1/query', query=expression)
check(value['status'] == 'success', 'Prometheus query failed')
return value['data']['result']
def logs(self, selector, start):
value = self.proxy('loki', '/loki/api/v1/query_range', query=selector, start=start, end=time.time_ns(), limit=1000, direction='forward')
check(value['status'] == 'success', 'Loki query failed')
return [json.loads(line) for stream in value['data']['result'] for _, line, *_ in stream['values']]
def trace(self, trace_id):
return self.proxy('tempo', '/api/traces/' + trace_id)
def identifier_hex(value, byte_length):
if len(value) == byte_length * 2 and all(c in '0123456789abcdefABCDEF' for c in value):
return value.lower()
return base64.b64decode(value, validate=True).hex()
def spans_in(value):
found = []
if isinstance(value, dict):
for key, item in value.items():
if key == 'spans':
found.extend(item)
else:
found.extend(spans_in(item))
elif isinstance(value, list):
for item in value:
found.extend(spans_in(item))
return found
def correlation(grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None):
selector = '{service_name="' + service + '"} | json'
if service == 'messenger-api':
selector += ' | event="http.request"'
if trace_id:
selector += ' | trace_id="' + trace_id + '"'
if attachment_id:
selector += ' | attachment_id="' + attachment_id + '"'
if request_id:
selector += ' | request_id="' + request_id + '"'
records = eventually(lambda: grafana.logs(selector, start), 'No correlated log for ' + service)
record = records[0]
trace_id = record['trace_id']
def matching_trace():
candidate = grafana.trace(trace_id)
return candidate if any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans_in(candidate)) else None
trace = eventually(matching_trace, 'Matching span is missing in Tempo')
spans = [span for span in spans_in(trace) if identifier_hex(span.get('spanId', ''), 8) == record['span_id']]
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
check(any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans), 'Log span_id not found in real trace')
check(all(int(span['endTimeUnixNano']) > int(span['startTimeUnixNano']) for span in spans), 'Trace has no real duration')
return {'trace_id': trace_id, 'span_id': record['span_id'], 'log': record,
'spans': [{key: span.get(key) for key in ('name', 'kind', 'spanId', 'startTimeUnixNano', 'endTimeUnixNano', 'status')} for span in spans]}, trace
def run_load(args):
check(1 <= args.duration <= 300 and 1 <= args.rate <= 40 and 1 <= args.concurrency <= 16, 'Load profile outside bounds')
client = Client(args.base_url)
actor, _, _, chat = client.bootstrap()
project = ComposeProject()
container_ids = project.run('ps', '--quiet').splitlines()
samples = []
stop = threading.Event()
def sample():
while not stop.is_set():
result = subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*container_ids],capture_output=True,text=True,timeout=15)
if result.returncode == 0:
samples.append({'elapsed_seconds':round(time.monotonic()-started,3), 'containers':[
{k:row[k] for k in ('Name','CPUPerc','MemUsage','MemPerc')} for row in (json.loads(line) for line in result.stdout.splitlines())]})
stop.wait(3)
def write(index):
local = Client(args.base_url)
before = time.monotonic()
try:
local.message(actor,chat,'synthetic-observability-' + str(index))
return time.monotonic()-before
except Failure:
return None
started = time.monotonic()
sampler = threading.Thread(target=sample)
sampler.start()
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = []
for index in range(args.duration * args.rate):
time.sleep(max(0, started + index/args.rate - time.monotonic()))
futures.append(pool.submit(write,index))
results = [future.result() for future in futures]
elapsed = time.monotonic()-started
finally:
stop.set()
sampler.join()
latencies = sorted(value for value in results if value is not None)
failed = len(results)-len(latencies)
def percentile(q):
return round(latencies[max(0, math.ceil(q*len(latencies))-1)]*1000,3) if latencies else None
report = {'date_utc':datetime.now(timezone.utc).isoformat(), 'workload':'synthetic POST messages; fixed arrival rate; no retries',
'duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'arrival_rate':args.rate,'concurrency_limit':args.concurrency,
'successful_requests':len(latencies),'failed_requests':failed,'error_ratio':failed/len(results),
'successful_rps':round(len(latencies)/elapsed,3), 'successful_latency_ms':{name:percentile(q) for name,q in [('p50',.5),('p95',.95),('p99',.99)]},
'dataset':{'users':3,'chats':1,'messages':len(latencies)},'resource_samples':samples,
'limits':'Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.'}
check(bool(latencies) and failed == 0, 'Controlled load contains errors')
return report
def metric(grafana, expression):
rows = grafana.query(expression)
return float(rows[0]['value'][1]) if rows else None
def alert_firing(grafana, name):
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
def trace_chain(grafana, trace_id, *, retry=False):
def inspect():
spans = spans_in(grafana.trace(trace_id))
by_id = {identifier_hex(span['spanId'], 8): span for span in spans}
workers = [span for span in spans if span['name'] == 'attachment.process.v1'
and span.get('kind') in ('SPAN_KIND_CONSUMER', 5)]
for worker in workers:
publisher = by_id.get(identifier_hex(worker.get('parentSpanId', ''), 8), {})
if publisher.get('kind') not in ('SPAN_KIND_PRODUCER', 4):
return None
upload = by_id.get(identifier_hex(publisher.get('parentSpanId', ''), 8), {})
if upload.get('name') != 'POST /api/v1/chats/{chat_id}/attachments':
return None
names = {span['name'] for span in spans}
if not workers or not {'s3.read_staging', 's3.write_result', 'postgres.commit_ready'} <= names:
return None
if retry:
outcomes = set()
for span in workers:
attrs = {row['key']: row['value'] for row in span.get('attributes', [])}
outcomes.add((int(attrs['job.attempt']['intValue']), attrs.get('outcome', {}).get('stringValue')))
if not {(1, 'error'), (2, 'error')} <= outcomes or not any(n > 2 and outcome == 'success' for n, outcome in outcomes):
return None
return [{'name': span['name'], 'kind': span.get('kind'),
'span_id': identifier_hex(span['spanId'], 8),
'parent_span_id': identifier_hex(span.get('parentSpanId', ''), 8),
'job': {entry['key']: entry['value'] for entry in span.get('attributes', [])
if entry['key'] in {'job.attempt', 'outcome'}}} for span in spans]
return eventually(inspect, 'Missing upload -> producer -> consumer -> S3/DB chain')
def telemetry_outage(project, client, grafana, actor, chat):
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
baseline = [metric(grafana, expr) for expr in (failed, refused)]
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
samples = []
successful = 0
try:
project.run('stop', 'loki')
start = time.monotonic()
for second in range(51):
for index in range(4):
client.message(actor, chat, f'outage-{second}-{index}')
successful += 1
if second % 5 == 0:
samples.append({
'seconds': round(time.monotonic() - start, 2),
'rss_bytes': metric(grafana, rss),
'queued_batches': metric(grafana, 'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})'),
'capacity_batches': metric(grafana, 'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})'),
})
time.sleep(max(0, start + second + 1 - time.monotonic()))
losses = [metric(grafana, expr) - count for expr, count in zip((failed, refused), baseline)]
check(sum(losses) > 0, 'No measured Collector log loss during prolonged outage')
check(all(0 <= row['queued_batches'] <= row['capacity_batches'] == 256 for row in samples), 'Unexpected queue bound')
check(max(row['rss_bytes'] for row in samples) < 256 * 1024**2, 'Collector crossed container memory limit')
finally:
project.run('start', 'loki')
eventually(lambda: grafana.proxy('loki', '/loki/api/v1/labels').get('status') == 'success',
'Loki did not recover')
restored = client.message(actor, chat, 'telemetry-backend-restored')
correlation(grafana, 'messenger-api', time.time_ns() - 30 * 10**9,
trace_id=restored['headers']['X-Trace-Id'])
return {'scenario':'loki_unavailable', 'successful_messages': successful,
'dropped_after_export_failure': losses[0], 'dropped_at_enqueue': losses[1],
'samples': samples, 'log_export_recovered': True}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', required=True)
parser.add_argument('--grafana-url', default='http://localhost:13000')
parser.add_argument('--incidents', action='store_true')
parser.add_argument('--restart-telemetry', action='store_true')
parser.add_argument('--output', default='evidence/observability.json')
parser.add_argument('--duration', type=int, default=10)
parser.add_argument('--rate', type=int, default=10)
parser.add_argument('--concurrency', type=int, default=2)
args = parser.parse_args()
load = run_load(args)
start = time.time_ns()
project = ComposeProject()
client = Client(args.base_url, timeout=40)
aliases = {r.instance_id for r in project.replicas() if r.running}
check(len(aliases) >= 2, 'Need two real API containers')
check(client.call('GET', '/health/live')['headers'].get('X-Instance-Id') in aliases, 'Wrong Compose project for BASE_URL')
grafana = Grafana(args.grafana_url)
eventually(lambda: grafana.get('/api/health').get('database') == 'ok', 'Grafana is not ready')
sources = grafana.get('/api/datasources')
check({'prometheus', 'loki', 'tempo'} <= {source['uid'] for source in sources}, 'Missing provisioned datasource')
dashboard = grafana.get('/api/dashboards/uid/messenger-overview')['dashboard']
check(len(dashboard['panels']) >= 8, 'Dashboard is missing panels')
actor, bob, eve, chat = client.bootstrap()
sentinel = 'private-content-' + uuid4().hex
response = client.message(actor, chat, sentinel)
controlled_trace = response['headers'].get('X-Trace-Id')
check(bool(controlled_trace), 'No API trace identifier')
attachment, _ = client.attachment(actor, chat)
client.wait_ready(actor, attachment)
api, api_trace = correlation(grafana, 'messenger-api', start, trace_id=controlled_trace, request_id=response['headers']['X-Request-Id'])
worker, worker_trace = correlation(grafana, 'messenger-worker', start, attachment_id=attachment['id'])
accepted_chain = trace_chain(grafana, worker['trace_id'])
check(api['log']['request_id'] == response['headers']['X-Request-Id'], 'Request id correlation lost')
check(any(span['kind'] in ('SPAN_KIND_SERVER', 2) for span in api['spans']), 'No server span')
check(any(span['kind'] in ('SPAN_KIND_CONSUMER', 5) for span in worker['spans']), 'No consumer span')
processed = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | trace_id="' + worker['trace_id'] + '"', start), 'No worker result log')
check(processed[0]['outcome'] == 'success', 'Controlled worker did not succeed')
metrics = {}
expressions = {
'http_requests': 'sum(messenger_http_requests_total)',
'http_duration': 'sum(messenger_http_duration_seconds_count)',
'job_success': 'sum(messenger_job_attempts_total{outcome="success"})',
'job_duration': 'sum(messenger_job_duration_seconds_count)',
}
for name, expression in expressions.items():
metrics[name] = eventually(lambda expression=expression: grafana.query(expression), 'Missing metric: ' + expression)
check(float(metrics[name][0]['value'][1]) > 0, 'Empty metric: ' + name)
# Все PromQL из dashboard должны давать численные данные после traffic.
for panel in dashboard['panels']:
if panel['datasource']['uid'] == 'prometheus':
eventually(lambda panel=panel: all(grafana.query(target['expr']) for target in panel['targets']), 'Empty dashboard panel: ' + panel['title'])
forbidden_labels = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target','http_url'}
series = grafana.query('{__name__=~"messenger_.*"}')
check(not any(forbidden_labels & set(row['metric']) for row in series), 'High cardinality metric label')
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'project': project.name,
'load': load, 'api_replicas': len(aliases), 'dashboard_uid': dashboard['uid'],
'api_correlation': api, 'worker_correlation': worker, 'metrics': metrics, 'upload_trace': {'trace_id':worker['trace_id'], 'chain':accepted_chain}, 'incidents': []}
if args.incidents:
def fail_request():
return client.call('POST', '/api/v1/users', expected=503,
body={'username':'fault_' + uuid4().hex[:12],
'display_name':'Synthetic', 'password':'not-exported-' + uuid4().hex})
before = time.monotonic()
try:
project.run('stop', 'postgres')
failed = fail_request()
error_trace = failed['headers'].get('X-Trace-Id')
error, db_trace = correlation(grafana, 'messenger-api', start, trace_id=error_trace,
request_id=failed['headers']['X-Request-Id'])
diagnostics = eventually(lambda: grafana.logs(
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="' + error_trace + '"', start),
'No dependency diagnostic log')
checkout = next(span for span in spans_in(db_trace) if span['name'] == 'postgres.checkout'
and span.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2))
def db_alert():
for _ in range(4):
fail_request()
return alert_firing(grafana, 'MessengerHighErrorRate')
eventually(db_alert, 'Error rate alert did not fire', timeout=75)
report['incidents'].append({'scenario':'postgres_unavailable','http_status':503,'correlation':error,
'diagnostic':diagnostics[0], 'db_span':checkout['name'],
'alert':'MessengerHighErrorRate', 'firing_after_seconds':round(time.monotonic()-before,3)})
finally:
project.run('start', 'postgres')
eventually(lambda: _ready(client), 'API did not recover after PostgreSQL')
eventually(lambda: not alert_firing(grafana, 'MessengerHighErrorRate'), 'Error rate alert not resolved')
report['incidents'][-1]['resolved_after_seconds'] = round(time.monotonic() - before, 3)
from scripts.check_jobs import retry_replay
queue = {}
def observe_queue(_):
observed = time.monotonic()
eventually(lambda: alert_firing(grafana, 'MessengerQueueLag'), 'Queue lag alert did not fire')
queue.update(depth=metric(grafana, 'max(messenger_queue_depth_ratio)'),
oldest_seconds=metric(grafana, 'max(messenger_queue_oldest_seconds)'),
firing_after_seconds=round(time.monotonic()-observed,3))
incident = retry_replay(client, actor, chat, permanent=False, on_queued=observe_queue)
retried_worker, retry_trace = correlation(grafana, 'messenger-worker', start,
attachment_id=incident['attachment_id'])
chain = trace_chain(grafana, retried_worker['trace_id'], retry=True)
eventually(lambda: not alert_firing(grafana, 'MessengerQueueLag')
and metric(grafana, 'max(messenger_queue_depth_ratio)') == 0, 'Backlog did not drain')
queue.update(resolved=True, final_depth=0)
report['incidents'].append({'scenario':'worker_stopped_then_transient_retry',
'alert':'MessengerQueueLag', 'queue':queue, 'recovered_status':incident['after']['status'],
'trace_id':retried_worker['trace_id'], 'chain':chain})
report['incidents'].append(telemetry_outage(project, client, grafana, actor, chat))
else:
retry_trace = {}
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
private = [sentinel, *(person['token'] for person in (actor,bob,eve)), *(person['registration']['password'] for person in (actor,bob,eve))]
private.extend(path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file())
check(not any(secret and secret in exported for secret in private), 'Private value found in telemetry')
check(all(record.get('trace_id') and record.get('span_id') and record.get('timestamp') and record.get('severity') for record in logs), 'Incomplete structured logs')
report['privacy_checked_logs'] = len(logs)
report['checks'] = {'three_backends':True, 'dashboard_queries':True,'server_consumer_traces':True,'log_trace_correlation':True,'low_cardinality':True,'privacy':True}
if args.restart_telemetry:
historical_time = time.time()
expression = 'sum(messenger_http_requests_total)'
before = grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result']
project.run('restart', 'prometheus', 'loki', 'tempo', 'grafana')
after = eventually(lambda: grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result'], 'Prometheus history lost')
check(before == after, 'Historical metric changed after storage restart')
correlation(grafana, 'messenger-api', start, trace_id=api['trace_id'])
correlation(grafana, 'messenger-worker', start, trace_id=worker['trace_id'])
report['checks']['history_survives_restart'] = True
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
print('PASS: telemetry, dashboard, privacy, correlations' + ('; alerts and outage recovery verified' if args.incidents else ''))
def _ready(client):
return client.call('GET', '/health/ready', expected=[200,503])['status'] == 200
if __name__ == '__main__':
main()
+151
View File
@@ -71,3 +71,154 @@ class ComposeProject:
)
for row in map(json.loads, rows)
]
def cleanup_docker(*args, allow_failure=False):
result = subprocess.run(['docker',*args],cwd=ROOT,text=True,capture_output=True,timeout=120)
if result.returncode and not allow_failure:
raise RuntimeError('Docker cleanup failed: ' + ' '.join(args[:2]))
return result.stdout.strip()
def inventory(projects):
resources = {'containers':[], 'networks':[], 'volumes':[]}
for project in projects:
label = 'label=com.docker.compose.project=' + project
resources['containers'].extend(cleanup_docker('ps','-aq','--filter',label).splitlines())
resources['networks'].extend(cleanup_docker('network','ls','-q','--filter',label).splitlines())
resources['volumes'].extend(cleanup_docker('volume','ls','-q','--filter',label).splitlines())
return resources
def clean_stands(*, apply=False):
config = json.loads(cleanup_docker('compose','--env-file','.env','config','--format','json'))
projects = sorted({config['name'], os.getenv('MAIN_PROJECT', 'messenger'), os.getenv('TEST_PROJECT', 'messenger-test')})
resources = inventory(projects)
names = {service['image'] for service in config['services'].values() if 'build' in service}
image_ids = set()
for name in names:
image_ids.update(cleanup_docker('image','ls','-q',name).splitlines())
# У старых сборок API может уже не быть тега. Сверяем также команду и WORKDIR,
# чтобы не затронуть сторонние образы с обычным OCI revision label.
candidates = cleanup_docker('image','ls','-q','--filter','label=org.opencontainers.image.revision').splitlines()
for candidate in candidates:
image = json.loads(cleanup_docker('image','inspect',candidate))[0]
settings = image['Config']
if settings.get('Cmd') == ['python','-m','app'] and settings.get('WorkingDir') == '/app':
image_ids.add(candidate)
plan = {'projects':projects, 'resources':resources, 'application_image_ids':sorted(image_ids), 'local_settings_preserved':True}
if not apply:
return plan
if resources['containers']:
cleanup_docker('rm','-f','-v',*resources['containers'])
if resources['networks']:
cleanup_docker('network','rm',*resources['networks'])
if resources['volumes']:
cleanup_docker('volume','rm',*resources['volumes'])
if image_ids:
cleanup_docker('image','rm',*sorted(image_ids))
remaining = inventory(projects)
if any(remaining.values()):
raise RuntimeError('Cleanup incomplete')
plan['remaining'] = remaining
plan['application_images_remaining'] = [name for name in sorted(names) if cleanup_docker('image','ls','-q',name)]
if plan['application_images_remaining']:
raise RuntimeError('Application images remain')
plan['result'] = 'PASS'
return plan
def verify():
"""Один последовательный прогон на одном стенде с обязательной очисткой."""
from datetime import datetime, timezone
import re
import sys
import time
project_name = os.getenv('TEST_PROJECT', 'messenger-test')
if any(inventory([project_name]).values()):
raise RuntimeError(f'Choose an unused TEST_PROJECT; {project_name} already has resources')
os.environ.update(COMPOSE_PROJECT_NAME=project_name,
COMPOSE_FILE='compose.yaml:compose.test.yaml',
GRAFANA_HOST_PORT='23000', LB_HOST_PORT='28080',
DOCKER_TESTS='1', OTEL_ENABLED='true')
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'checks': [],
'profile': {'single_stack': True, 'load_duration_seconds': 10,
'load_requests_per_second': 10, 'load_concurrency': 2}}
destination = ROOT / 'evidence/verification.json'
destination.parent.mkdir(exist_ok=True)
def run(name, command, timeout=600, *, env=None):
print('RUN: ' + name, flush=True)
started = time.monotonic()
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=timeout,
env={**os.environ, **(env or {})})
executable = Path(command[0])
display_command = [str(executable.relative_to(ROOT)) if executable.is_relative_to(ROOT)
else command[0], *command[1:]]
entry = {'name': name, 'command': display_command, 'exit_code': result.returncode,
'duration_seconds': round(time.monotonic()-started, 3),
'result': 'PASS' if result.returncode == 0 else 'FAIL'}
if name == 'regression':
summary = re.search(r'\d+ passed[^\n]*', result.stdout)
if summary:
entry['summary'] = summary.group(0)
if result.returncode:
# Полные диагностические логи локальны; в evidence нет путей/credentials.
(ROOT / '.verification-error.log').write_text(result.stdout + result.stderr)
entry['diagnostics'] = '.verification-error.log (local, not committed)'
report['checks'].append(entry)
print(entry['result'] + ': ' + name, flush=True)
if result.returncode:
raise RuntimeError(name + ' failed; inspect local diagnostic log')
try:
run('contract', [sys.executable, 'scripts/check_contract.py'])
run('openapi', [sys.executable, 'scripts/validate_openapi.py'])
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
run('startup', ['make', '--no-print-directory', 'test-stand-up'], timeout=600)
# Локальные ASGI fixtures не находятся в сети Compose. Проверки SDK
# используют exporters в памяти, интеграционный стенд экспортирует OTLP.
run('regression', ['make', '--no-print-directory', 'test-unit'], env={'OTEL_ENABLED': 'false'})
run('api', [sys.executable, 'tests/smoke.py', '--base-url', 'http://localhost:28080'])
run('observability', [sys.executable, '-m', 'scripts.check_observability',
'--base-url', 'http://localhost:28080',
'--grafana-url', 'http://localhost:23000',
'--incidents', '--restart-telemetry'])
run('persistence', [sys.executable, 'tests/resilience.py', '--mode', 'persistence',
'--action-script', 'scripts/restart.sh', '--base-url', 'http://localhost:28080'])
run('failover', [sys.executable, 'tests/resilience.py', '--mode', 'failover',
'--action-script', 'scripts/stop-one.sh', '--base-url', 'http://localhost:28080'])
report['result'] = 'PASS'
except BaseException:
report['result'] = 'FAIL'
raise
finally:
try:
print('RUN: cleanup', flush=True)
ComposeProject().run('down', '--volumes', '--remove-orphans', timeout=120)
remaining = inventory([project_name])
if any(remaining.values()):
raise RuntimeError('Verification project cleanup incomplete')
report['cleanup'] = {'project': project_name, 'remaining': remaining, 'result': 'PASS'}
print('PASS: cleanup', flush=True)
finally:
report['finished_at_utc'] = datetime.now(timezone.utc).isoformat()
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
def main():
import argparse
parser = argparse.ArgumentParser(description='Проверка и очистка Compose-стендов')
parser.add_argument('command', choices=['verify', 'clean'])
parser.add_argument('--apply', action='store_true', help='Подтвердить удаление ресурсов')
args = parser.parse_args()
if args.command == 'verify':
verify()
else:
print(json.dumps(clean_stands(apply=args.apply), indent=2))
if __name__ == '__main__':
main()
+1
View File
@@ -15,6 +15,7 @@ def main() -> None:
directory = root / "secrets"
directory.mkdir(mode=0o700, exist_ok=True)
for name in (
"grafana_admin_password",
"db_password",
"redis_password",
"broker_password",
+12 -7
View File
@@ -24,14 +24,19 @@ wait_healthy() {
worker_running=$(compose ps --status running --quiet worker)
compose stop worker publisher cleanup
compose restart postgres redis minio broker
compose restart postgres redis minio broker prometheus loki tempo grafana otel-collector
wait_healthy postgres redis minio broker
compose restart api
wait_healthy api
compose restart nginx
wait_healthy nginx
compose start publisher cleanup
compose restart api nginx
wait_healthy api nginx
# Compose start также запускает завершённые migrate/bootstrap dependencies.
# Здесь нужны только ранее созданные фоновые контейнеры, без повторного bootstrap.
background_ids=$(compose ps --all --quiet publisher cleanup)
docker start $background_ids
# Сценарий с заранее остановленным worker сохраняет его остановленным.
if [ -n "$worker_running" ]; then
compose start worker
worker_ids=$(compose ps --all --quiet worker)
docker start $worker_ids
fi
# Пока восстанавливался бизнес-путь, telemetry backend тоже успели запуститься.
# Grafana healthcheck проверяет также готовность Loki, Tempo и Collector.
wait_healthy prometheus grafana
+2 -1
View File
@@ -65,7 +65,8 @@ async def test_unhandled_exception_is_hidden_from_client(
assert "test-secret" not in json.dumps(payload, ensure_ascii=False)
assert "Traceback" not in json.dumps(payload, ensure_ascii=False)
assert "Unhandled error request_id=" in caplog.text
assert "test-secret" in caplog.text
assert "test-secret" not in caplog.text
assert "RuntimeError" in caplog.text
assert_request_id(response)
+126
View File
@@ -0,0 +1,126 @@
"""Проверки приватности и исходов на реальных SDK exporters в памяти."""
import json
from fastapi import FastAPI
import httpx
from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.trace import SpanKind
import pytest
from app.core import telemetry as module
@pytest.fixture(autouse=True)
async def clean_db():
# Этот набор не использует БД, Redis и фикстуры приложения.
yield
@pytest.fixture
def telemetry(monkeypatch):
spans = InMemorySpanExporter()
logs = InMemoryLogRecordExporter()
metrics = InMemoryMetricReader()
monkeypatch.setattr(module, "OTLPSpanExporter", lambda **_: spans)
monkeypatch.setattr(module, "OTLPLogExporter", lambda **_: logs)
monkeypatch.setattr(module, "PeriodicExportingMetricReader", lambda *_, **__: metrics)
runtime = module.Telemetry("messenger-api", "unit", enabled=True)
yield runtime, spans, logs, metrics
runtime.shutdown()
async def test_http_template_correlation_and_private_data(telemetry):
runtime, spans, logs, metrics = telemetry
app = FastAPI()
@app.post("/items/{item_id}")
async def item(item_id: str):
return {"ok": True}
app.add_middleware(module.HttpTelemetryMiddleware, telemetry=runtime)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client:
# ASGI caller может иметь свой активный span, не относящийся к входящему HTTP.
with runtime.tracer.start_as_current_span("unrelated-task"):
response = await client.post("/items/private-id?token=private-query", json={"text":"private-body"}, headers={"Authorization":"Bearer private-token"})
await client.get("/arbitrary-private-path")
runtime.traces.force_flush()
runtime.logs.force_flush()
finished = spans.get_finished_spans()
assert finished[0].kind == SpanKind.SERVER
assert finished[0].name == "POST /items/{item_id}"
assert finished[1].attributes["http.route"] == "unmatched"
assert finished[0].context.trace_id != finished[1].context.trace_id
assert all(span.parent is None for span in finished[:2])
body = json.loads(logs.get_finished_logs()[0].log_record.body)
assert body["trace_id"] == response.headers["X-Trace-Id"]
assert int(body["span_id"], 16) == finished[0].context.span_id
data = metrics.get_metrics_data()
all_attributes = [dict(point.attributes) for resource in data.resource_metrics for scope in resource.scope_metrics for metric in scope.metrics for point in metric.data.data_points]
assert {"http.request.method":"POST", "http.route":"/items/{item_id}", "http.response.status_class":"2xx"} in all_attributes
exported = repr([(s.name, dict(s.attributes)) for s in finished]) + repr(all_attributes) + repr([str(log.log_record.body) for log in logs.get_finished_logs()])
for private in ("private-id", "private-query", "private-body", "private-token", "arbitrary-private-path"):
assert private not in exported
async def test_unhandled_error_preserves_correlation_without_exception_payload(telemetry):
from app.core.request_id import RequestIdMiddleware
from app.errors.handlers import register_exception_handlers
runtime, spans, logs, _ = telemetry
app = FastAPI()
@app.get("/explode")
async def explode():
raise RuntimeError("private-error-with-credentials")
app.add_middleware(RequestIdMiddleware)
app.add_middleware(module.HttpTelemetryMiddleware, telemetry=runtime)
register_exception_handlers(app)
async with httpx.AsyncClient(transport=httpx.ASGITransport(app, raise_app_exceptions=False), base_url="http://test") as client:
response = await client.get("/explode")
runtime.traces.force_flush()
runtime.logs.force_flush()
record = json.loads(logs.get_finished_logs()[0].log_record.body)
assert response.status_code == record["status_code"] == 500
assert response.headers["X-Request-Id"] == record["request_id"]
assert spans.get_finished_spans()[0].status.status_code.name == "ERROR"
assert "private-error-with-credentials" not in repr(record)
def test_job_success_and_error_record_real_outcomes(telemetry):
runtime, spans, logs, metrics = telemetry
for fail in (False, True):
try:
with runtime.tracer.start_as_current_span("attachment.process.v1", kind=SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False):
with runtime.job_attempt():
if fail:
raise RuntimeError("private-exception-payload")
except RuntimeError:
pass
runtime.traces.force_flush()
runtime.logs.force_flush()
finished = spans.get_finished_spans()
assert [s.attributes["outcome"] for s in finished] == ["success", "error"]
assert finished[1].status.status_code.name == "ERROR"
assert not finished[1].events
records = [json.loads(item.log_record.body) for item in logs.get_finished_logs()]
assert [record["outcome"] for record in records] == ["success", "error"]
assert "private-exception-payload" not in repr(records)
data = metrics.get_metrics_data()
counters = [point for resource in data.resource_metrics for scope in resource.scope_metrics for metric in scope.metrics if metric.name == "messenger.job.attempts" for point in metric.data.data_points]
assert {p.attributes["outcome"]:p.value for p in counters} == {"success":1,"error":1}
def test_disabled_telemetry_has_no_exporters():
runtime = module.Telemetry("messenger-api", enabled=False)
runtime.event("ignored")
runtime.shutdown()
assert not hasattr(runtime, "metrics")
def test_tempo_base64_and_hex_identifiers_match():
from scripts.check_observability import identifier_hex
assert identifier_hex("3X/51BqmrXA=", 8) == "dd7ff9d41aa6ad70"
assert identifier_hex("dd7ff9d41aa6ad70", 8) == "dd7ff9d41aa6ad70"