forked from mtusi2026/mtusi2026lab6
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2b721844fe | ||
|
|
27a3a6895f | ||
|
|
52be9d282b |
+3
-9
@@ -98,13 +98,7 @@ IMAGE_CPU_SECONDS=20
|
||||
JOB_RETRY_MAX_SECONDS=120
|
||||
FAILED_STAGING_RETENTION_SECONDS=604800
|
||||
|
||||
# Дополнительные предложения skeleton lab6; пока не подключены к перенесённому стенду.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Наблюдаемость: SDK экспортирует HTTP/protobuf в Collector.
|
||||
OTEL_ENABLED=true
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
GRAFANA_HOST_PORT=13000
|
||||
|
||||
+3
-1
@@ -32,7 +32,9 @@ coverage/
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
# Перенесённые исключения lab5
|
||||
venv/
|
||||
.venv/
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
/docs/
|
||||
|
||||
.verification-error.log
|
||||
|
||||
@@ -8,14 +8,17 @@ SERVICE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
COMPOSE = docker compose --env-file .env
|
||||
LOCAL_COMPOSE = $(COMPOSE) -f compose.yaml -f compose.local.yml
|
||||
|
||||
TEST_PROJECT ?= messenger-test
|
||||
TEST_LB_PORT ?= 28080
|
||||
TEST_DB_PORT ?= $(DB_HOST_PORT)
|
||||
TEST_REDIS_PORT ?= $(REDIS_HOST_PORT)
|
||||
TEST_GRAFANA_PORT ?= 23000
|
||||
GRAFANA_URL ?= http://localhost:$(or $(GRAFANA_HOST_PORT),13000)
|
||||
TEST_BASE_URL = http://localhost:$(TEST_LB_PORT)
|
||||
TEST_ENV = COMPOSE_PROJECT_NAME=$(TEST_PROJECT) COMPOSE_FILE=compose.yaml:compose.test.yaml \
|
||||
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT)
|
||||
LB_HOST_PORT=$(TEST_LB_PORT) DB_HOST_PORT=$(TEST_DB_PORT) REDIS_HOST_PORT=$(TEST_REDIS_PORT) GRAFANA_HOST_PORT=$(TEST_GRAFANA_PORT)
|
||||
TEST_COMPOSE = $(TEST_ENV) $(COMPOSE)
|
||||
|
||||
TEST_DB_PASSWORD = $(if $(wildcard secrets/db_password),$(file <secrets/db_password),$(DB_PASSWORD))
|
||||
@@ -25,13 +28,14 @@ TEST_REDIS_PASSWORD ?= $(if $(wildcard secrets/redis_password),$(file <secrets/r
|
||||
TEST_REDIS_DB ?= 15
|
||||
export TEST_DATABASE_URL TEST_REDIS_HOST TEST_REDIS_PORT TEST_REDIS_PASSWORD TEST_REDIS_DB
|
||||
|
||||
.PHONY: help init up down restart ps logs migrate migration worker-stop worker-start \
|
||||
.PHONY: help init up up-local down restart ps logs migrate migration worker-stop worker-start \
|
||||
cleanup-dry-run cleanup-apply check validate test load test-unit test-pipeline \
|
||||
test-nginx test-storage test-outbox test-jobs test-persistence test-failover \
|
||||
test-suite test-stand-up test-stand-destroy
|
||||
test-suite test-stand-up test-stand-destroy test-telemetry test-observability verify clean-stands
|
||||
|
||||
help:
|
||||
@echo "Запуск: init | up | down | restart | ps | logs SERVICE=worker"
|
||||
@echo "Локальные порты и веб-панели: up-local"
|
||||
@echo "Обслуживание: migrate | migration m='описание' | worker-stop | worker-start"
|
||||
@echo "Очистка объектов: cleanup-dry-run | cleanup-apply"
|
||||
@echo "Проверки: check | validate | test | test-pipeline | test-unit PYTEST_ARGS=..."
|
||||
@@ -39,12 +43,16 @@ help:
|
||||
@echo "Сервисы: test-nginx | test-storage | test-outbox | test-jobs"
|
||||
@echo "Отказы API (по запросу): test-persistence | test-failover"
|
||||
@echo "Автономный прогон с удалением тестового стенда: test-suite"
|
||||
@echo "Наблюдаемость: test-observability; полный прогон с очисткой: verify"
|
||||
|
||||
init:
|
||||
$(PYTHON) scripts/init_local.py
|
||||
|
||||
up:
|
||||
$(COMPOSE) up -d --build --wait --wait-timeout 120
|
||||
$(COMPOSE) up -d --build --wait --wait-timeout 180
|
||||
|
||||
up-local:
|
||||
$(LOCAL_COMPOSE) up -d --build --wait --wait-timeout 180
|
||||
|
||||
down:
|
||||
$(COMPOSE) down
|
||||
@@ -88,6 +96,21 @@ test:
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-telemetry:
|
||||
$(PYTHON) -m pytest -q student-tests/test_telemetry.py
|
||||
|
||||
# Проверка на уже запущенном стенде; включает сценарии отказа и восстановления.
|
||||
test-observability:
|
||||
$(PYTHON) -m scripts.check_observability --base-url "$(BASE_URL)" --grafana-url "$(GRAFANA_URL)" --incidents --restart-telemetry
|
||||
|
||||
# Один прогон на отдельном стенде, удаление его контейнеров/сети/томов при завершении.
|
||||
verify:
|
||||
$(PYTHON) -m scripts.deployment verify
|
||||
|
||||
# Удаление основного и тестового стендов; .env/secrets сохраняются.
|
||||
clean-stands:
|
||||
$(PYTHON) -m scripts.deployment clean --apply
|
||||
|
||||
# Все сохранённые регрессионные тесты запускаются явно; PYTEST_ARGS выбирает подмножество.
|
||||
test-unit:
|
||||
$(TEST_ENV) $(PYTHON) -m pytest -q $(PYTEST_ARGS)
|
||||
@@ -115,7 +138,7 @@ test-failover:
|
||||
|
||||
# Отдельные PostgreSQL/Redis нужны для локальных тестов; тестовая БД создаётся явно.
|
||||
test-stand-up:
|
||||
$(TEST_COMPOSE) up -d --build --wait --wait-timeout 120
|
||||
$(TEST_COMPOSE) up -d --build --wait --wait-timeout 180
|
||||
$(TEST_COMPOSE) exec -T postgres sh -c 'psql -U "$$POSTGRES_USER" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '\''$${POSTGRES_DB}_test'\''" | grep -q 1 || createdb -U "$$POSTGRES_USER" "$${POSTGRES_DB}_test"'
|
||||
|
||||
test-stand-destroy:
|
||||
|
||||
@@ -1,46 +1,107 @@
|
||||
# Отчёт — лабораторная 6
|
||||
# Отчёт - лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: Григорян Илья Мурадович, БПИ2303.
|
||||
- Цель: оценка **4**
|
||||
- Источники: OpenTelemetry SDK 1.45.1, FastAPI, SQLAlchemy 2.1.4,
|
||||
[внутренние метрики Collector](https://opentelemetry.io/docs/collector/internal-telemetry/),
|
||||
[контракт наблюдаемости](contracts/observability.md). Помощь Codex.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск
|
||||
|
||||
Перенесённый стенд lab5 и команды его запуска описаны в [RUNNING.md](RUNNING.md).
|
||||
Это основа для lab6; телеметрия текущей лабораторной пока не реализована.
|
||||
Python 3.12 и Docker Compose. Зависимости и образы закреплены в репозитории.
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
make init
|
||||
make up
|
||||
make test
|
||||
make down
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
API доступен на http://localhost:18081, дашборд — на
|
||||
http://localhost:13000/d/messenger-overview.
|
||||
Инициализация создаёт только отсутствующие настройки и секреты. Миграции и
|
||||
настройка приватного bucket выполняются при запуске. `make down` сохраняет тома.
|
||||
|
||||
Из lab5 перенесена реализация уровня 4: API, миграции, PostgreSQL/Redis,
|
||||
MinIO, RabbitMQ, outbox publisher, worker, очистка, crop/WebP, ограниченные
|
||||
повторы с задержкой, failed registry и ручной replay, собственные тесты и
|
||||
проверки отказов. Контракты, публичные проверки и задание lab6 сохранены.
|
||||
Результаты проверок lab5 не считаются доказательствами прохождения lab6.
|
||||
## Архитектура и трассировка
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Две реплики API за nginx. PostgreSQL хранит сообщения и состояние заданий,
|
||||
Redis — сессии, MinIO — файлы, RabbitMQ доставляет задания worker.
|
||||
Миграции и настройка MinIO выполняются при запуске, доступы передаются
|
||||
через файловые секреты. Версии, проверки готовности и тома заданы в
|
||||
[compose.yaml](compose.yaml).
|
||||
|
||||
## Доказательства критериев
|
||||
API, publisher и worker отправляют телеметрию по OTLP в Collector.
|
||||
Prometheus хранит метрики, Loki — структурные логи, Tempo — трассы.
|
||||
Grafana читает три источника и автоматически загружает дашборд.
|
||||
История хранится 72 часа в именованных томах.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
API сохраняет W3C `traceparent` и `tracestate` вместе с заданием. Publisher
|
||||
создаёт дочерний producer span и передаёт контекст через заголовки RabbitMQ.
|
||||
Каждая попытка получает свой consumer span в той же трассе. Успешная попытка
|
||||
добавляет операции чтения и записи S3 и фиксации `ready` в PostgreSQL.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
HTTP-ответ и лог связаны по `X-Request-Id`, лог и трасса — по `trace_id`.
|
||||
Метрики группируются по шаблону маршрута, методу, классу ответа, очереди,
|
||||
типу и исходу задания. ID отдельных объектов используются для поиска в
|
||||
логах, а тексты сообщений, секреты и содержимое файлов в телеметрию не передаются.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Метрики и цели
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
| Что смотрим | Запрос или метрика | Единица |
|
||||
| --- | --- | --- |
|
||||
| Частота HTTP | `sum by (http_request_method,http_route)(rate(messenger_http_requests_total{http_route!~"/health/.*"}[1m]))` | запросы/с |
|
||||
| Доля ошибок | Сумма rate для класса 5xx / сумма rate всех бизнес-запросов, без health | доля |
|
||||
| HTTP p50/p95/p99 | `histogram_quantile(q, sum by(le)(rate(messenger_http_duration_seconds_bucket{http_route!~"/health/.*"}[1m])))` | с |
|
||||
| Worker | `messenger_job_attempts_total` по outcome и `messenger_job_duration_seconds_bucket` | попытки, с |
|
||||
| Очередь | `max by(queue_name)(messenger_queue_depth_ratio)`, `max by(queue_name)(messenger_queue_oldest_seconds)` | задания, с |
|
||||
| Пул PostgreSQL | `sum by(service_name)(messenger_db_pool_used_ratio)` и аналогично `messenger_db_pool_waiting_ratio` | соединения, ожидающие |
|
||||
| Загрузка worker | `sum(messenger_worker_busy_ratio) / sum(messenger_worker_capacity_ratio)` | доля |
|
||||
| CPU и память | `rate(messenger_process_cpu_seconds_total[1m])`, `messenger_process_rss_bytes` | ядра, байты |
|
||||
| До готовности вложения | `histogram_quantile(0.95, sum by(le)(rate(messenger_job_time_to_ready_seconds_bucket[5m])))` | с |
|
||||
| Экспорт Collector | `otelcol_exporter_queue_size`, `otelcol_exporter_queue_capacity` | пакеты |
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Глубина очереди — число заданий в состоянии `queued` в PostgreSQL,
|
||||
включая отложенные повторы. Возраст считается от первого приёма.
|
||||
Время до `ready` включает ожидание и повторы. Процентили вычисляются
|
||||
по объединённой гистограмме всех реплик. Ряды глубины и пула с суффиксом
|
||||
`_ratio` содержат абсолютные количества.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
SLI доступности — доля бизнес-запросов без 5xx. SLO — ≥99% за скользящие
|
||||
24 часа. Для проверки используйте отношение сумм `increase(...[24h])`,
|
||||
исключив health из числителя и знаменателя. Окно без трафика не оценивается.
|
||||
Короткие оповещения для стенда заданы в [alerts.yaml](infra/otel/alerts.yaml).
|
||||
`MessengerHighErrorRate` срабатывает при >5% 5xx за 30 с с выдержкой 5 с.
|
||||
`MessengerQueueLag` срабатывает, если очередь непуста и возраст старейшего
|
||||
задания >15 с с выдержкой 5 с. Состояние доступно в Prometheus через
|
||||
`ALERTS{alertstate="firing"}`. Внешняя доставка отключена.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
## Диагностика инцидентов
|
||||
|
||||
| Инцидент | Как найти причину и подтвердить восстановление |
|
||||
| --- | --- |
|
||||
| PostgreSQL остановлен | Рост 5xx → лог по `X-Request-Id` → ERROR span `postgres.checkout` в связанной трассе. Оповещение вошло в firing через 32,4 с и снялось через 67,5 с от начала сценария. Готовность API восстановилась. |
|
||||
| Worker остановлен | Растёт возраст очереди, новые попытки отсутствуют. Оповещение сработало через 28,6 с после приёма. После пяти временных ошибок и ручного replay шестая попытка завершилась `ready`, глубина вернулась к нулю, оповещение снялось. В трассе `ff8352e27270fbc16fea7216c3ff45c0` видны все попытки и операции S3/БД. |
|
||||
| Loki остановлен | В Prometheus смотрим бизнес-запросы, RSS и экспортную очередь Collector. За 51 с записано 204 сообщения, потеряно 6 логов после повторов отправки. Пик очереди — 48 из 256 пакетов, RSS — 228–231 MiB. После запуска Loki найден новый связанный лог. |
|
||||
|
||||
Collector использует memory limiter (192 MiB, запас для всплесков 32 MiB),
|
||||
пакеты до 256 записей, очереди по 256 пакетов для Loki и Tempo и повторы
|
||||
до 30 с. Лимит контейнера — 256 MiB. Потери измерены по счётчикам Collector
|
||||
для этого короткого отказа.
|
||||
|
||||
## Полный прогон
|
||||
|
||||
`make verify` запускает регрессию, smoke, нагрузку, сценарии отказов,
|
||||
проверки сохранности и переключения реплик на отдельном стенде, затем
|
||||
удаляет его контейнеры, сеть и тома.
|
||||
|
||||
Прогон 19.11.2026, 18:50–19:03 МСК — **PASS**. Регрессия: **206 тестов**
|
||||
за 98,89 с. Нагрузка: 100 успешных сообщений, 0 ошибок, около 10 запросов/с.
|
||||
Клиентские p50/p95/p99 — 18,0 / 39,6 / 46,0 мс. Проверены панели, связность
|
||||
трасс и логов, приватность и сохранение истории после перезапуска.
|
||||
Доказательства сохранены в [журнале проверок](evidence/verification.json)
|
||||
и [результатах наблюдаемости](evidence/observability.json).
|
||||
|
||||
Стенд работает на одном Docker-хосте. Соблюдение SLO требует наблюдения
|
||||
за полным 24-часовым окном.
|
||||
|
||||
+37
-65
@@ -1,90 +1,62 @@
|
||||
# Запуск перенесённой реализации lab5
|
||||
# Запуск Messenger
|
||||
|
||||
В lab6 перенесена реализация lab5 для оценки 4: API-реплики за nginx,
|
||||
PostgreSQL, Redis для сессий, MinIO для файлов, RabbitMQ, outbox publisher,
|
||||
worker и очистка объектов. Сохранены миграции и собственные тесты.
|
||||
OpenTelemetry, Collector и Grafana из задания lab6 пока не реализованы.
|
||||
|
||||
## Основной стенд
|
||||
|
||||
Нужны Python 3.12 и Docker с Compose v2. Версии Python-зависимостей закреплены
|
||||
в `requirements.txt`, образы — в Dockerfile и Compose.
|
||||
Нужны Python 3.12 и Docker Compose v2. Версии закреплены в requirements.txt и Compose.
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
make init
|
||||
make check validate
|
||||
make up
|
||||
make test
|
||||
```
|
||||
|
||||
`make init` создаёт отсутствующие `.env` и случайные файловые секреты в `secrets/`.
|
||||
Секреты и данные из lab5 не копируются. Настройки берутся из `.env.example`;
|
||||
дополнительные переменные skeleton lab6 в конце файла пока не подключены.
|
||||
Миграции БД и настройка MinIO выполняются при запуске.
|
||||
`make up` ожидает готовности сервисов. По умолчанию API доступен по
|
||||
http://localhost:18081 (`LB_HOST_PORT`); `make test` и `make load` используют
|
||||
этот порт. Его можно переопределить через `BASE_URL`.
|
||||
`make init` создаёт только отсутствующие `.env` и случайные файловые секреты.
|
||||
Миграции PostgreSQL и приватный bucket/политика MinIO создаются автоматически.
|
||||
Основной проект — `messenger`, тестовый — `messenger-test`.
|
||||
|
||||
- API: http://localhost:18081 (`LB_HOST_PORT`).
|
||||
- Grafana: http://localhost:13000/d/messenger-overview (`GRAFANA_HOST_PORT`).
|
||||
- Просмотр Grafana — анонимный Viewer; admin password — `secrets/grafana_admin_password`.
|
||||
- Внутренние API/OTLP/Prometheus/Loki/Tempo порты на хост не публикуются.
|
||||
|
||||
`OTEL_ENABLED` включает экспорт трёх сигналов, `OTEL_EXPORTER_OTLP_ENDPOINT` задаёт
|
||||
Collector, `APP_ENV` — окружение resource attributes. Имена сервисов — messenger-api
|
||||
и messenger-worker. Конфигурация — в `infra/otel/`, метрики/логи/трассы сохраняются в volumes.
|
||||
|
||||
```sh
|
||||
make ps
|
||||
make logs SERVICE=worker
|
||||
make load
|
||||
make down
|
||||
make down # данные сохраняются
|
||||
make clean-stands # удалить оба стенда, тома и образы приложения
|
||||
```
|
||||
|
||||
`make down` сохраняет данные. Для **намеренного удаления всех учебных данных**:
|
||||
`docker compose --env-file .env down --volumes`.
|
||||
Compose-проект по умолчанию называется `messenger`, как в lab5: останавливайте
|
||||
предыдущий стенд перед запуском либо используйте отдельный `COMPOSE_PROJECT_NAME`
|
||||
и свободный `LB_HOST_PORT`. Одинаковое имя проекта подключает существующие volumes;
|
||||
для независимого стенда lab6 задайте новое имя до первого запуска.
|
||||
`.env`, `secrets/`, базовые образы и evidence при очистке сохраняются.
|
||||
План удаления без применения: `.venv/bin/python -m scripts.deployment clean`.
|
||||
|
||||
## Перенесённые возможности уровня 4
|
||||
## Проверки
|
||||
|
||||
Загрузка изображения асинхронная: API сохраняет исходник в staging MinIO и
|
||||
задание/outbox в PostgreSQL, publisher отправляет задание в устойчивую очередь
|
||||
RabbitMQ, worker сохраняет варианты и результат в БД, затем подтверждает доставку.
|
||||
Сохранены original, JPEG thumbnail, центральный JPEG crop и WebP, ограничения
|
||||
размера, пикселей, времени, памяти и CPU обработки.
|
||||
`make verify` выполняет один последовательный прогон на одном тестовом стенде:
|
||||
контракт/OpenAPI, полная регрессия с Redis restart, smoke, короткая нагрузка,
|
||||
проверки телеметрии и двух отказов, persistence/failover. Очистка выполняется
|
||||
в `finally`, включая ошибку. Результаты — два JSON в `evidence/`.
|
||||
Не выполняйте одновременно команды, управляющие тем же стендом.
|
||||
|
||||
Повторы ограничены бюджетом, задержка растёт со случайным разбросом.
|
||||
Идемпотентность, lease (временное право worker обрабатывать задание),
|
||||
реестр неудачных заданий и ручной replay защищают обработку при повторной доставке.
|
||||
Очистка сохраняет исходники активных заданий и учитывает сроки хранения ready/failed.
|
||||
Отдельные проверки: `make test-telemetry` без Docker; `make test-observability`
|
||||
на уже запущенном стенде (включает ограниченные отказы и восстановление).
|
||||
`make test-unit` использует тестовую БД; сначала `make test-stand-up`, затем
|
||||
`make test-stand-destroy`. Тестовые порты: API 28080, Grafana 23000,
|
||||
PostgreSQL 25432, Redis 26379, только localhost.
|
||||
|
||||
## Задания и файлы
|
||||
|
||||
Загрузка асинхронная: staging/outbox → RabbitMQ → worker → original и JPEG/WebP
|
||||
варианты. Повторы ограничены, готовность — после фиксации объектов и metadata.
|
||||
|
||||
```sh
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored
|
||||
```
|
||||
|
||||
`JOB_ID` и `REQUEST_UUID` замените UUID. Повтор replay с тем же request-id безопасен.
|
||||
|
||||
## Проверки
|
||||
|
||||
`make check` проверяет структуру контракта, `make validate` — OpenAPI.
|
||||
`make test` запускает актуальные публичные тесты lab6 на основном стенде.
|
||||
Собственные тесты lab5 находятся в `student-tests/`:
|
||||
|
||||
```sh
|
||||
make test-stand-up
|
||||
make test-unit
|
||||
make test-pipeline
|
||||
make test BASE_URL=http://localhost:28080
|
||||
make test-stand-destroy
|
||||
```
|
||||
|
||||
Тестовый стенд использует отдельный проект `messenger-test` и порт 28080,
|
||||
а PostgreSQL/Redis публикуются только локально через `compose.test.yaml`.
|
||||
`make test-stand-destroy` удаляет данные тестового стенда.
|
||||
`make test-suite` автоматически поднимает этот стенд, проверяет pipeline,
|
||||
публичный smoke, S3, outbox и очередь, затем удаляет его вместе с данными.
|
||||
Полная регрессия запускается отдельно через `make test-unit`.
|
||||
|
||||
Проверки с намеренными отказами запускаются явно:
|
||||
`make test-jobs`, `make test-outbox`, `make test-persistence`, `make test-failover`.
|
||||
Они останавливают или перезапускают компоненты выбранного Compose-проекта.
|
||||
Для собственных тестов с перезапусками контейнеров: `DOCKER_TESTS=1 make test-unit`.
|
||||
Стенд рассчитан на один Docker-хост; репликация хранилищ и защита от потери диска
|
||||
не реализованы.
|
||||
Замените UUID; повтор replay с тем же request-id безопасен.
|
||||
`make load` — унаследованный генератор без ограничения частоты, он может упереться
|
||||
в nginx 50 запросов/с. Для штатной проверки используется 10 запросов/с на 10 секунд.
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
from contextlib import nullcontext
|
||||
|
||||
from sqlalchemy.pool import AsyncAdaptedQueuePool
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncEngine,
|
||||
AsyncSession,
|
||||
@@ -8,9 +11,31 @@ from sqlalchemy.ext.asyncio import (
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
class ObservedPool(AsyncAdaptedQueuePool):
|
||||
telemetry = None
|
||||
waiting = 0
|
||||
|
||||
def __init__(self, *args, pool_size, max_overflow, **kwargs):
|
||||
self.capacity = pool_size + max_overflow if pool_size and max_overflow >= 0 else None
|
||||
super().__init__(*args, pool_size=pool_size, max_overflow=max_overflow, **kwargs)
|
||||
|
||||
def _do_get(self):
|
||||
waiting = self.capacity is not None and self.checkedout() >= self.capacity
|
||||
self.waiting += int(waiting)
|
||||
try:
|
||||
return super()._do_get()
|
||||
finally:
|
||||
self.waiting -= int(waiting)
|
||||
|
||||
def connect(self):
|
||||
with self.telemetry.span("postgres.checkout") if self.telemetry else nullcontext():
|
||||
return super().connect()
|
||||
|
||||
|
||||
def create_db_engine(settings: Settings) -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
settings.get_database_url(),
|
||||
poolclass=ObservedPool,
|
||||
pool_size=settings.db_pool_size,
|
||||
max_overflow=settings.db_max_overflow,
|
||||
pool_timeout=settings.db_pool_timeout_seconds,
|
||||
|
||||
+29
-12
@@ -10,6 +10,9 @@ import signal
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import features
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.background import (
|
||||
@@ -21,6 +24,7 @@ from app.core.background import (
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.core.telemetry import Telemetry
|
||||
from app.domain.attachment_jobs import process_attachment_job
|
||||
from app.errors.exceptions import ProcessingError
|
||||
from app.schemas.job import JobMessage
|
||||
@@ -67,6 +71,7 @@ async def consume_connected(
|
||||
settings: JobSettings,
|
||||
stop: asyncio.Event,
|
||||
faults: Faults,
|
||||
telemetry: Telemetry | None = None,
|
||||
) -> None:
|
||||
consumer = uuid4().hex
|
||||
while not stop.is_set():
|
||||
@@ -81,15 +86,22 @@ async def consume_connected(
|
||||
await broker.reject_invalid(delivery.id)
|
||||
continue
|
||||
try:
|
||||
async with factory() as session:
|
||||
acknowledge = await process_attachment_job(
|
||||
session,
|
||||
storage,
|
||||
message,
|
||||
settings,
|
||||
after_object=faults.after_object,
|
||||
before_processing=faults.before_processing,
|
||||
)
|
||||
telemetry = telemetry or Telemetry("messenger-worker", enabled=False)
|
||||
parent = TraceContextTextMapPropagator().extract(delivery.id.headers or {}, context=Context())
|
||||
with telemetry.span(
|
||||
"attachment.process.v1", context=parent, kind=SpanKind.CONSUMER,
|
||||
attributes={"messaging.system": "rabbitmq", "job.type": message.type,
|
||||
"job.attempt": message.attempt},
|
||||
):
|
||||
telemetry.event("job.received", job_id=str(message.job_id),
|
||||
attachment_id=str(message.attachment_id))
|
||||
async with factory() as session:
|
||||
acknowledge = await process_attachment_job(
|
||||
session, storage, message, settings,
|
||||
after_object=faults.after_object,
|
||||
before_processing=faults.before_processing,
|
||||
telemetry=telemetry,
|
||||
)
|
||||
except ProcessingError:
|
||||
await broker.reject_invalid(delivery.id)
|
||||
else:
|
||||
@@ -104,10 +116,10 @@ async def consume_connected(
|
||||
await wait_or_stop(stop, settings.publisher_poll_seconds)
|
||||
|
||||
|
||||
async def consume(factory, storage, settings, stop, faults) -> None:
|
||||
async def consume(factory, storage, settings, stop, faults, telemetry=None) -> None:
|
||||
broker = JobBroker(settings)
|
||||
try:
|
||||
await consume_connected(factory, storage, broker, settings, stop, faults)
|
||||
await consume_connected(factory, storage, broker, settings, stop, faults, telemetry)
|
||||
finally:
|
||||
await broker.aclose()
|
||||
|
||||
@@ -115,19 +127,23 @@ async def consume(factory, storage, settings, stop, faults) -> None:
|
||||
async def run_worker(faults: Faults) -> None:
|
||||
settings = Settings()
|
||||
jobs = JobSettings()
|
||||
telemetry = Telemetry("messenger-worker")
|
||||
logging.basicConfig(level=settings.log_level)
|
||||
if not features.check("webp"):
|
||||
logging.error("Pillow requires WebP support for the image worker")
|
||||
raise RuntimeError("WebP codec is unavailable")
|
||||
stop = shutdown_event()
|
||||
engine = create_db_engine(settings)
|
||||
telemetry.observe_pool(engine.pool)
|
||||
telemetry.gauge("messenger.worker.busy", lambda: telemetry.busy)
|
||||
telemetry.gauge("messenger.worker.capacity", lambda: jobs.worker_concurrency)
|
||||
storage = None
|
||||
try:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
factory = create_session_factory(engine)
|
||||
async with asyncio.TaskGroup() as group:
|
||||
tasks = [
|
||||
group.create_task(consume(factory, storage, jobs, stop, faults))
|
||||
group.create_task(consume(factory, storage, jobs, stop, faults, telemetry))
|
||||
for _ in range(jobs.worker_concurrency)
|
||||
]
|
||||
await stop.wait()
|
||||
@@ -138,6 +154,7 @@ async def run_worker(faults: Faults) -> None:
|
||||
if storage is not None:
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(telemetry.shutdown)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.background import (
|
||||
@@ -19,6 +20,8 @@ from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs
|
||||
from app.storage.broker import JobBroker
|
||||
from app.storage.jobs import queued_backlog
|
||||
from app.core.telemetry import Telemetry
|
||||
|
||||
|
||||
def crash_before_publish(attachment_id: UUID) -> None:
|
||||
@@ -36,6 +39,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
logging.basicConfig(level=settings.log_level)
|
||||
stop = shutdown_event()
|
||||
engine = create_db_engine(settings)
|
||||
telemetry = Telemetry("messenger-publisher")
|
||||
telemetry.observe_pool(engine.pool)
|
||||
backlog = (0, 0.0)
|
||||
labels = {"queue.name": jobs.broker_queue}
|
||||
telemetry.gauge("messenger.queue.depth", lambda: backlog[0], attributes=labels)
|
||||
telemetry.gauge("messenger.queue.oldest", lambda: backlog[1], unit="s", attributes=labels)
|
||||
next_sample = 0.0
|
||||
broker = None
|
||||
try:
|
||||
broker = JobBroker(jobs)
|
||||
@@ -43,12 +53,16 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
while not stop.is_set():
|
||||
published = False
|
||||
try:
|
||||
await broker.ensure_ready()
|
||||
async with factory() as session:
|
||||
if telemetry.enabled and time.monotonic() >= next_sample:
|
||||
backlog = await queued_backlog(session)
|
||||
next_sample = time.monotonic() + 2
|
||||
await broker.ensure_ready()
|
||||
await recover_expired_jobs(session, jobs)
|
||||
published = await publish_next_job(
|
||||
session,
|
||||
broker,
|
||||
telemetry=telemetry,
|
||||
attachment_id=crash_attachment_id,
|
||||
before_publish=(
|
||||
crash_before_publish
|
||||
@@ -64,6 +78,7 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
if broker is not None:
|
||||
await broker.aclose()
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(telemetry.shutdown)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Явная OTLP-инструментация: экспортируются только выбранные технические поля."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import time
|
||||
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry._logs import SeverityNumber
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.metrics import Observation
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
|
||||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
||||
JOB_BUCKETS = (0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
|
||||
|
||||
|
||||
class Telemetry:
|
||||
def __init__(self, service: str, instance: str | None = None, *, enabled=None):
|
||||
self.enabled = (os.getenv("OTEL_ENABLED", "false").lower() == "true"
|
||||
if enabled is None else enabled)
|
||||
self.service = service
|
||||
self.busy = 0
|
||||
if not self.enabled:
|
||||
self.tracer = trace.NoOpTracerProvider().get_tracer("messenger")
|
||||
return
|
||||
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
resource = Resource({
|
||||
"service.name": service,
|
||||
"service.version": "1.6.0",
|
||||
"service.instance.id": instance or socket.gethostname(),
|
||||
"deployment.environment.name": os.getenv("APP_ENV", "development"),
|
||||
})
|
||||
self.traces = TracerProvider(resource=resource)
|
||||
self.traces.add_span_processor(BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
|
||||
))
|
||||
self.tracer = self.traces.get_tracer("messenger", "1.6.0")
|
||||
self.logs = LoggerProvider(resource=resource)
|
||||
self.logs.add_log_record_processor(BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
|
||||
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
|
||||
))
|
||||
self.logger = self.logs.get_logger("messenger", "1.6.0")
|
||||
self.metrics = MeterProvider(resource=resource, metric_readers=[
|
||||
PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
|
||||
export_interval_millis=5000, export_timeout_millis=3000,
|
||||
)
|
||||
], views=[
|
||||
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
||||
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||||
])
|
||||
meter = self.meter = self.metrics.get_meter("messenger", "1.6.0")
|
||||
self.http_requests = meter.create_counter("messenger.http.requests", unit="1")
|
||||
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
|
||||
self.jobs = meter.create_counter("messenger.job.attempts", unit="1")
|
||||
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
||||
|
||||
self.time_to_ready = meter.create_histogram("messenger.job.time_to_ready", unit="s")
|
||||
self.gauge("messenger.process.cpu", time.process_time, unit="s", counter=True)
|
||||
self.gauge("messenger.process.rss", resident_bytes, unit="By")
|
||||
|
||||
def gauge(self, name, sample, *, unit="1", attributes=None, counter=False):
|
||||
if self.enabled:
|
||||
create = (self.meter.create_observable_counter if counter
|
||||
else self.meter.create_observable_gauge)
|
||||
create(name, callbacks=[lambda _: [Observation(sample(), attributes)]], unit=unit)
|
||||
|
||||
def observe_pool(self, pool):
|
||||
if self.enabled:
|
||||
pool.telemetry = self
|
||||
self.gauge("messenger.db.pool.used", pool.checkedout)
|
||||
self.gauge("messenger.db.pool.waiting", lambda: pool.waiting)
|
||||
|
||||
@contextmanager
|
||||
def span(self, name, **kwargs):
|
||||
with self.tracer.start_as_current_span(
|
||||
name, record_exception=False, set_status_on_exception=False, **kwargs,
|
||||
) as span:
|
||||
try:
|
||||
yield span
|
||||
except BaseException:
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
raise
|
||||
|
||||
def event(self, event: str, *, error=False, **fields):
|
||||
if not self.enabled:
|
||||
return
|
||||
context = trace.get_current_span().get_span_context()
|
||||
body = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"severity": "ERROR" if error else "INFO",
|
||||
"service_name": self.service, "event": event,
|
||||
"trace_id": f"{context.trace_id:032x}", "span_id": f"{context.span_id:016x}",
|
||||
**fields,
|
||||
}
|
||||
text = json.dumps(body, ensure_ascii=False)
|
||||
# Не подключаем root logging: SDK/SQL-исключения могут содержать секреты.
|
||||
self.logger.emit(body=text, severity_text=body["severity"],
|
||||
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
|
||||
attributes={"event": event})
|
||||
print(text, flush=True)
|
||||
|
||||
@contextmanager
|
||||
def job_attempt(self):
|
||||
self.busy += 1
|
||||
started = time.monotonic()
|
||||
result = {"outcome": "success"}
|
||||
try:
|
||||
yield result
|
||||
except BaseException:
|
||||
result["outcome"] = "error"
|
||||
raise
|
||||
finally:
|
||||
self.busy -= 1
|
||||
if self.enabled:
|
||||
attrs = {"job.type": "attachment.process.v1", "outcome": result["outcome"]}
|
||||
elapsed = time.monotonic() - started
|
||||
self.jobs.add(1, attrs)
|
||||
self.job_duration.record(elapsed, attrs)
|
||||
span = trace.get_current_span()
|
||||
span.set_attribute("outcome", result["outcome"])
|
||||
span.set_status(Status(StatusCode.ERROR if result["outcome"] == "error" else StatusCode.OK))
|
||||
self.event("job.processed", error=result["outcome"] == "error",
|
||||
job_type="attachment.process.v1", duration_seconds=elapsed, **result)
|
||||
|
||||
def shutdown(self):
|
||||
if self.enabled:
|
||||
self.metrics.shutdown(timeout_millis=5000)
|
||||
self.logs.shutdown()
|
||||
self.traces.shutdown()
|
||||
|
||||
|
||||
def resident_bytes():
|
||||
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
|
||||
|
||||
|
||||
class HttpTelemetryMiddleware:
|
||||
def __init__(self, app, telemetry: Telemetry):
|
||||
self.app = app
|
||||
self.telemetry = telemetry
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] != "http" or not self.telemetry.enabled:
|
||||
return await self.app(scope, receive, send)
|
||||
telemetry = self.telemetry
|
||||
method = scope["method"]
|
||||
if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}:
|
||||
method = "OTHER"
|
||||
started = time.monotonic()
|
||||
status = 500
|
||||
request_id = ""
|
||||
with telemetry.tracer.start_as_current_span(
|
||||
"HTTP", context=Context(), kind=SpanKind.SERVER,
|
||||
record_exception=False, set_status_on_exception=False,
|
||||
) as span:
|
||||
async def observed_send(message):
|
||||
nonlocal status, request_id
|
||||
if message["type"] == "http.response.start":
|
||||
status = message["status"]
|
||||
headers = list(message.get("headers", []))
|
||||
request_id = next((v.decode() for k, v in headers if k.lower() == b"x-request-id"), "")
|
||||
headers.append((b"x-trace-id", f"{span.get_span_context().trace_id:032x}".encode()))
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
try:
|
||||
await self.app(scope, receive, observed_send)
|
||||
finally:
|
||||
request_id = request_id or scope.get("state", {}).get("request_id", "")
|
||||
route = getattr(scope.get("route"), "path", "unmatched")
|
||||
attrs = {"http.request.method": method, "http.route": route,
|
||||
"http.response.status_class": f"{status // 100}xx"}
|
||||
elapsed = time.monotonic() - started
|
||||
telemetry.http_requests.add(1, attrs)
|
||||
telemetry.http_duration.record(elapsed, attrs)
|
||||
span.update_name(f"{method} {route}")
|
||||
span.set_attributes({**attrs, "http.response.status_code": status})
|
||||
span.set_status(Status(StatusCode.ERROR if status >= 500 else StatusCode.OK))
|
||||
telemetry.event("http.request", error=status >= 500, method=method, route=route,
|
||||
status_code=status, request_id=request_id, duration_seconds=elapsed)
|
||||
@@ -2,16 +2,21 @@
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from minio.error import MinioException
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.trace import SpanKind
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from urllib3.exceptions import HTTPError as StorageHTTPError
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.core.telemetry import Telemetry
|
||||
from app.domain.job_retry import classify_failure, retry_delay
|
||||
from app.domain.job_files import fetch_original, render_variants
|
||||
from app.errors.exceptions import ProcessingError
|
||||
@@ -42,6 +47,7 @@ async def publish_next_job(
|
||||
*,
|
||||
attachment_id: UUID | None = None,
|
||||
before_publish: Callable[[UUID], None] | None = None,
|
||||
telemetry: Telemetry | None = None,
|
||||
) -> bool:
|
||||
"""Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль."""
|
||||
job = await outbox.next_unpublished_job(session, attachment_id=attachment_id)
|
||||
@@ -49,15 +55,20 @@ async def publish_next_job(
|
||||
if job is not None:
|
||||
if before_publish is not None:
|
||||
before_publish(job.attachment_id)
|
||||
await broker.publish_job(
|
||||
JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
parent = TraceContextTextMapPropagator().extract(job.trace_context or {}, context=Context())
|
||||
with telemetry.span(
|
||||
"rabbitmq.publish", context=parent, kind=SpanKind.PRODUCER,
|
||||
attributes={"messaging.system": "rabbitmq", "job.attempt": job.attempt + 1},
|
||||
) if telemetry else nullcontext():
|
||||
await broker.publish_job(
|
||||
JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
)
|
||||
)
|
||||
outbox.mark_published(job)
|
||||
await session.commit()
|
||||
return published
|
||||
@@ -143,11 +154,13 @@ async def build_result(
|
||||
storage: MinioStorage,
|
||||
work: JobWork,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
telemetry: Telemetry | None = None,
|
||||
) -> bool:
|
||||
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
|
||||
with TemporaryDirectory(prefix="messenger-job-") as directory:
|
||||
source = Path(directory) / "original"
|
||||
await fetch_original(storage, work, source)
|
||||
with telemetry.span("s3.read_staging") if telemetry else nullcontext():
|
||||
await fetch_original(storage, work, source)
|
||||
result = await render_variants(storage, work, source, Path(directory))
|
||||
row = await jobs.lock_job(session, work.job_id)
|
||||
if row is None:
|
||||
@@ -163,33 +176,39 @@ async def build_result(
|
||||
return False
|
||||
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
|
||||
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
|
||||
await storage.put_file(
|
||||
work.original_key,
|
||||
source,
|
||||
size_bytes=work.size_bytes,
|
||||
content_type=work.content_type,
|
||||
sha256=work.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, "original")
|
||||
for variant in result.variants:
|
||||
with telemetry.span("s3.write_result") if telemetry else nullcontext():
|
||||
await storage.put_file(
|
||||
work.variant_key(variant.name),
|
||||
Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes,
|
||||
content_type=variant.content_type,
|
||||
sha256=variant.sha256,
|
||||
work.original_key,
|
||||
source,
|
||||
size_bytes=work.size_bytes,
|
||||
content_type=work.content_type,
|
||||
sha256=work.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, variant.name)
|
||||
jobs.complete_job(
|
||||
job,
|
||||
attachment,
|
||||
work,
|
||||
[variant.model_dump() for variant in result.variants],
|
||||
await jobs.database_time(session),
|
||||
)
|
||||
await session.commit()
|
||||
after_object(work.attachment_id, "original")
|
||||
for variant in result.variants:
|
||||
await storage.put_file(
|
||||
work.variant_key(variant.name),
|
||||
Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes,
|
||||
content_type=variant.content_type,
|
||||
sha256=variant.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, variant.name)
|
||||
with telemetry.span("postgres.commit_ready") if telemetry else nullcontext():
|
||||
ready_at = await jobs.database_time(session)
|
||||
accepted_at = job.created_at
|
||||
jobs.complete_job(
|
||||
job,
|
||||
attachment,
|
||||
work,
|
||||
[variant.model_dump() for variant in result.variants],
|
||||
ready_at,
|
||||
)
|
||||
await session.commit()
|
||||
if telemetry and telemetry.enabled:
|
||||
telemetry.time_to_ready.record(max(0, (ready_at - accepted_at).total_seconds()))
|
||||
return True
|
||||
|
||||
|
||||
@@ -201,26 +220,31 @@ async def process_attachment_job(
|
||||
*,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
before_processing: Callable[[UUID], None] | None = None,
|
||||
telemetry: Telemetry | None = None,
|
||||
) -> bool:
|
||||
claim = await claim_job(session, message, settings)
|
||||
with telemetry.span("postgres.claim_job") if telemetry else nullcontext():
|
||||
claim = await claim_job(session, message, settings)
|
||||
acknowledge = claim.acknowledge
|
||||
if claim.work is not None:
|
||||
work = claim.work
|
||||
try:
|
||||
async with asyncio.timeout(settings.job_timeout_seconds):
|
||||
if before_processing is not None:
|
||||
before_processing(work.attachment_id)
|
||||
acknowledge = await build_result(session, storage, work, after_object)
|
||||
except (
|
||||
ProcessingError,
|
||||
MinioException,
|
||||
StorageHTTPError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
) as error:
|
||||
code, permanent = classify_failure(error)
|
||||
await save_failure(session, work, settings, code, permanent=permanent)
|
||||
acknowledge = True
|
||||
with telemetry.job_attempt() if telemetry else nullcontext({}) as result:
|
||||
try:
|
||||
async with asyncio.timeout(settings.job_timeout_seconds):
|
||||
if before_processing is not None:
|
||||
before_processing(work.attachment_id)
|
||||
acknowledge = await build_result(session, storage, work, after_object, telemetry)
|
||||
result["outcome"] = "success" if acknowledge else "deferred"
|
||||
except (
|
||||
ProcessingError,
|
||||
MinioException,
|
||||
StorageHTTPError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
) as error:
|
||||
code, permanent = classify_failure(error)
|
||||
await save_failure(session, work, settings, code, permanent=permanent)
|
||||
result.update(outcome="error", error_code=code)
|
||||
acknowledge = True
|
||||
# Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку.
|
||||
return acknowledge
|
||||
|
||||
|
||||
@@ -167,6 +167,10 @@ async def dependency_unavailable_handler(
|
||||
request_id,
|
||||
type(exc).__name__,
|
||||
)
|
||||
telemetry = getattr(request.app.state, "telemetry", None)
|
||||
if telemetry is not None:
|
||||
telemetry.event("dependency.unavailable", error=True,
|
||||
request_id=request_id, exception_type=type(exc).__name__)
|
||||
|
||||
error = unavailable()
|
||||
return _build_response(
|
||||
@@ -206,9 +210,9 @@ async def unhandled_error_handler(
|
||||
request_id = _get_request_id(request)
|
||||
|
||||
logger.error(
|
||||
"Unhandled error request_id=%s",
|
||||
"Unhandled error request_id=%s type=%s",
|
||||
request_id,
|
||||
exc_info=exc,
|
||||
type(exc).__name__,
|
||||
)
|
||||
|
||||
error = internal_error()
|
||||
|
||||
+9
-1
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
@@ -12,6 +13,7 @@ from app.core.lifecycle import GracefulServer, Lifecycle
|
||||
from app.core.redis import create_redis_client
|
||||
from app.core.request_capacity import acquire_request_slot, create_request_capacity
|
||||
from app.core.request_id import RequestIdMiddleware
|
||||
from app.core.telemetry import HttpTelemetryMiddleware, Telemetry
|
||||
from app.endpoints.auth import router as auth_router
|
||||
from app.endpoints.attachments import router as attachments_router
|
||||
from app.endpoints.chats import router as chats_router
|
||||
@@ -44,8 +46,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
await s3.aclose()
|
||||
await redis.aclose()
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(telemetry.shutdown)
|
||||
|
||||
application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan)
|
||||
application = FastAPI(title="Messenger", version="1.6.0", lifespan=lifespan)
|
||||
|
||||
application.state.settings = settings
|
||||
application.state.engine = engine
|
||||
@@ -57,11 +60,15 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
)
|
||||
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
|
||||
application.state.instance_id = resolve_instance_id(settings.instance_id)
|
||||
telemetry = Telemetry("messenger-api", application.state.instance_id)
|
||||
telemetry.observe_pool(engine.pool)
|
||||
application.state.telemetry = telemetry
|
||||
application.state.lifecycle = Lifecycle()
|
||||
application.add_middleware(RequestIdMiddleware)
|
||||
application.add_middleware(
|
||||
InstanceIdMiddleware, instance_id=application.state.instance_id
|
||||
)
|
||||
application.add_middleware(HttpTelemetryMiddleware, telemetry=telemetry)
|
||||
register_exception_handlers(application)
|
||||
|
||||
limited = [Depends(acquire_request_slot)]
|
||||
@@ -83,6 +90,7 @@ def main() -> None:
|
||||
host=settings.app_host,
|
||||
port=settings.app_port,
|
||||
log_level=settings.log_level.lower(),
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
|
||||
)
|
||||
GracefulServer(
|
||||
|
||||
+2
-1
@@ -3,7 +3,7 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
|
||||
from sqlalchemy import JSON, CheckConstraint, ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
@@ -23,6 +23,7 @@ class AttachmentJob(Base):
|
||||
ForeignKey("attachments.id"),
|
||||
unique=True,
|
||||
)
|
||||
trace_context: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
staging_key: Mapped[str | None] = mapped_column(String(128))
|
||||
pipeline_version: Mapped[int] = mapped_column(server_default="1")
|
||||
attempt: Mapped[int] = mapped_column(server_default="0")
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass
|
||||
|
||||
import aio_pika
|
||||
from aio_pika.abc import AbstractIncomingMessage
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.schemas.job import JobMessage
|
||||
@@ -54,10 +55,13 @@ class JobBroker:
|
||||
|
||||
async def publish_job(self, message: JobMessage) -> None:
|
||||
await self.ensure_ready()
|
||||
headers = {}
|
||||
TraceContextTextMapPropagator().inject(headers)
|
||||
await self.channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=message.model_dump_json(exclude_none=True).encode(),
|
||||
content_type="application/json",
|
||||
headers=headers,
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=f"{message.job_id}:{message.attempt}",
|
||||
),
|
||||
|
||||
+14
-1
@@ -4,6 +4,7 @@ from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import UUID
|
||||
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from sqlalchemy import and_, or_, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
@@ -35,7 +36,10 @@ class JobWork:
|
||||
async def create_job(
|
||||
session: AsyncSession, attachment_id: UUID, staging_key: str
|
||||
) -> UUID:
|
||||
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
|
||||
carrier = {}
|
||||
TraceContextTextMapPropagator().inject(carrier)
|
||||
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key,
|
||||
trace_context=carrier or None)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
return job.id
|
||||
@@ -56,6 +60,15 @@ async def database_time(session: AsyncSession) -> datetime:
|
||||
return await session.scalar(select(func.clock_timestamp()))
|
||||
|
||||
|
||||
async def queued_backlog(session: AsyncSession) -> tuple[int, float]:
|
||||
count, oldest = (await session.execute(
|
||||
select(func.count(), func.min(AttachmentJob.created_at))
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "queued")
|
||||
)).one()
|
||||
return count, max(0, ((await database_time(session)) - oldest).total_seconds()) if oldest else 0.0
|
||||
|
||||
|
||||
async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Локальный доступ к сервисам из VS Code и браузера.
|
||||
# docker compose -f compose.yaml -f compose.local.yml up -d --wait
|
||||
# Все опубликованные порты доступны только с этого компьютера.
|
||||
services:
|
||||
api:
|
||||
# Docker выделяет каждой реплике отдельный свободный порт диапазона.
|
||||
ports:
|
||||
- "127.0.0.1:8081-8090:8080"
|
||||
postgres:
|
||||
ports:
|
||||
- "127.0.0.1:${POSTGRES_LOCAL_PORT:-5432}:5432"
|
||||
redis:
|
||||
ports:
|
||||
- "127.0.0.1:${REDIS_LOCAL_PORT:-6379}:6379"
|
||||
minio:
|
||||
command: ["server", "/data", "--console-address", "0.0.0.0:9001"]
|
||||
environment:
|
||||
MINIO_BROWSER: "on"
|
||||
# Добавляем обычную сеть для доступа через опубликованные порты;
|
||||
# внутренняя objects сохраняется для приложения и bootstrap.
|
||||
networks: [default, objects]
|
||||
ports:
|
||||
- "127.0.0.1:${MINIO_LOCAL_PORT:-9000}:9000"
|
||||
- "127.0.0.1:${MINIO_CONSOLE_LOCAL_PORT:-9001}:9001"
|
||||
broker:
|
||||
volumes:
|
||||
- ./infra/broker/enabled_plugins.local:/etc/rabbitmq/enabled_plugins:ro
|
||||
ports:
|
||||
- "127.0.0.1:${BROKER_LOCAL_PORT:-5672}:5672"
|
||||
- "127.0.0.1:${BROKER_MANAGEMENT_LOCAL_PORT:-15672}:15672"
|
||||
prometheus:
|
||||
ports:
|
||||
- "127.0.0.1:${PROMETHEUS_LOCAL_PORT:-9090}:9090"
|
||||
loki:
|
||||
ports:
|
||||
- "127.0.0.1:${LOKI_LOCAL_PORT:-3100}:3100"
|
||||
tempo:
|
||||
ports:
|
||||
- "127.0.0.1:${TEMPO_LOCAL_PORT:-3200}:3200"
|
||||
- "127.0.0.1:${TEMPO_OTLP_LOCAL_PORT:-14318}:4318"
|
||||
otel-collector:
|
||||
ports:
|
||||
- "127.0.0.1:${OTEL_HTTP_LOCAL_PORT:-4318}:4318"
|
||||
- "127.0.0.1:${OTEL_METRICS_LOCAL_PORT:-8889}:8889"
|
||||
- "127.0.0.1:${OTEL_HEALTH_LOCAL_PORT:-13133}:13133"
|
||||
@@ -11,6 +11,8 @@ x-base-env: &base-env
|
||||
APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
|
||||
APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
|
||||
APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
|
||||
OTEL_ENABLED: ${OTEL_ENABLED:-true}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||
LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
|
||||
MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
|
||||
MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
|
||||
@@ -388,11 +390,106 @@ services:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.165.0
|
||||
command: ["--config=/etc/otelcol/config.yaml"]
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.16.0
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=3d"]
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheusdata:/prometheus
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.10
|
||||
command: ["-config.file=/etc/loki/config.yaml"]
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
|
||||
- lokidata:/loki
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.9.0
|
||||
user: "0:0"
|
||||
command: ["-config.file=/etc/tempo/config.yaml"]
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:13.3.1
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
secrets:
|
||||
- grafana_admin_password
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_HOST_PORT:-13000}:3000"
|
||||
volumes:
|
||||
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
healthcheck:
|
||||
# Loki/Tempo/Collector не содержат shell/HTTP-клиент: проверяем их из Grafana.
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health && wget -q -T 2 -O /dev/null http://loki:3100/ready && wget -q -T 2 -O /dev/null http://tempo:3200/ready && wget -q -T 2 -O /dev/null http://otel-collector:13133/
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 36
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
miniodata:
|
||||
rabbitmqdata:
|
||||
prometheusdata:
|
||||
lokidata:
|
||||
tempodata:
|
||||
grafanadata:
|
||||
|
||||
networks:
|
||||
default:
|
||||
@@ -400,6 +497,8 @@ networks:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
grafana_admin_password:
|
||||
file: ./secrets/grafana_admin_password
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
db_password:
|
||||
|
||||
+8
-3
@@ -1,5 +1,10 @@
|
||||
# Доказательства
|
||||
# Результаты проверок
|
||||
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
`make verify` заново собирает два файла одного прогона:
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
- `verification.json`: команды, время, результат регрессии, API/отказов и очистки.
|
||||
- `observability.json`: нагрузка/CPU/RAM, метрики, корреляция логов и traces,
|
||||
privacy, dashboard, инциденты и сохранность истории после restart.
|
||||
|
||||
Сохраняются только
|
||||
синтетические технические данные; credentials, токены и тела сообщений исключены.
|
||||
|
||||
@@ -0,0 +1,888 @@
|
||||
{
|
||||
"date_utc": "2026-11-19T15:56:59.724115+00:00",
|
||||
"project": "messenger-test",
|
||||
"load": {
|
||||
"date_utc": "2026-11-19T15:56:52.469419+00:00",
|
||||
"workload": "synthetic POST messages; fixed arrival rate; no retries",
|
||||
"duration_seconds": 10,
|
||||
"elapsed_seconds": 9.923,
|
||||
"arrival_rate": 10,
|
||||
"concurrency_limit": 2,
|
||||
"successful_requests": 100,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 10.078,
|
||||
"successful_latency_ms": {
|
||||
"p50": 18.036,
|
||||
"p95": 39.623,
|
||||
"p99": 45.954
|
||||
},
|
||||
"dataset": {
|
||||
"users": 3,
|
||||
"chats": 1,
|
||||
"messages": 100
|
||||
},
|
||||
"resource_samples": [
|
||||
{
|
||||
"elapsed_seconds": 2.058,
|
||||
"containers": [
|
||||
{
|
||||
"Name": "messenger-test-api-1",
|
||||
"CPUPerc": "40.63%",
|
||||
"MemUsage": "94.64MiB / 1.5GiB",
|
||||
"MemPerc": "6.16%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-api-2",
|
||||
"CPUPerc": "5.96%",
|
||||
"MemUsage": "87.06MiB / 1.5GiB",
|
||||
"MemPerc": "5.67%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-broker-1",
|
||||
"CPUPerc": "86.17%",
|
||||
"MemUsage": "148.3MiB / 768MiB",
|
||||
"MemPerc": "19.31%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-cleanup-1",
|
||||
"CPUPerc": "0.00%",
|
||||
"MemUsage": "74.44MiB / 256MiB",
|
||||
"MemPerc": "29.08%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-grafana-1",
|
||||
"CPUPerc": "0.47%",
|
||||
"MemUsage": "292.7MiB / 512MiB",
|
||||
"MemPerc": "57.17%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-loki-1",
|
||||
"CPUPerc": "0.58%",
|
||||
"MemUsage": "69.43MiB / 512MiB",
|
||||
"MemPerc": "13.56%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-minio-1",
|
||||
"CPUPerc": "0.08%",
|
||||
"MemUsage": "131.4MiB / 512MiB",
|
||||
"MemPerc": "25.67%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-nginx-1",
|
||||
"CPUPerc": "0.27%",
|
||||
"MemUsage": "5.359MiB / 128MiB",
|
||||
"MemPerc": "4.19%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-otel-collector-1",
|
||||
"CPUPerc": "0.30%",
|
||||
"MemUsage": "62.13MiB / 256MiB",
|
||||
"MemPerc": "24.27%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-postgres-1",
|
||||
"CPUPerc": "1.54%",
|
||||
"MemUsage": "107.4MiB / 512MiB",
|
||||
"MemPerc": "20.99%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-prometheus-1",
|
||||
"CPUPerc": "0.14%",
|
||||
"MemUsage": "37.91MiB / 512MiB",
|
||||
"MemPerc": "7.41%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-publisher-1",
|
||||
"CPUPerc": "0.73%",
|
||||
"MemUsage": "65.48MiB / 256MiB",
|
||||
"MemPerc": "25.58%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-redis-1",
|
||||
"CPUPerc": "3.00%",
|
||||
"MemUsage": "8.527MiB / 512MiB",
|
||||
"MemPerc": "1.67%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-tempo-1",
|
||||
"CPUPerc": "0.30%",
|
||||
"MemUsage": "318.4MiB / 512MiB",
|
||||
"MemPerc": "62.19%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-worker-1",
|
||||
"CPUPerc": "0.19%",
|
||||
"MemUsage": "73.34MiB / 2GiB",
|
||||
"MemPerc": "3.58%"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 7.107,
|
||||
"containers": [
|
||||
{
|
||||
"Name": "messenger-test-api-1",
|
||||
"CPUPerc": "7.55%",
|
||||
"MemUsage": "94.69MiB / 1.5GiB",
|
||||
"MemPerc": "6.16%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-api-2",
|
||||
"CPUPerc": "39.26%",
|
||||
"MemUsage": "87.07MiB / 1.5GiB",
|
||||
"MemPerc": "5.67%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-broker-1",
|
||||
"CPUPerc": "15.48%",
|
||||
"MemUsage": "189.4MiB / 768MiB",
|
||||
"MemPerc": "24.66%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-cleanup-1",
|
||||
"CPUPerc": "0.00%",
|
||||
"MemUsage": "74.44MiB / 256MiB",
|
||||
"MemPerc": "29.08%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-grafana-1",
|
||||
"CPUPerc": "0.55%",
|
||||
"MemUsage": "292.7MiB / 512MiB",
|
||||
"MemPerc": "57.17%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-loki-1",
|
||||
"CPUPerc": "0.56%",
|
||||
"MemUsage": "69.18MiB / 512MiB",
|
||||
"MemPerc": "13.51%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-minio-1",
|
||||
"CPUPerc": "3.59%",
|
||||
"MemUsage": "135.3MiB / 512MiB",
|
||||
"MemPerc": "26.42%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-nginx-1",
|
||||
"CPUPerc": "0.25%",
|
||||
"MemUsage": "4.82MiB / 128MiB",
|
||||
"MemPerc": "3.77%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-otel-collector-1",
|
||||
"CPUPerc": "0.64%",
|
||||
"MemUsage": "62.66MiB / 256MiB",
|
||||
"MemPerc": "24.48%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-postgres-1",
|
||||
"CPUPerc": "7.61%",
|
||||
"MemUsage": "107.5MiB / 512MiB",
|
||||
"MemPerc": "20.99%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-prometheus-1",
|
||||
"CPUPerc": "0.21%",
|
||||
"MemUsage": "38.25MiB / 512MiB",
|
||||
"MemPerc": "7.47%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-publisher-1",
|
||||
"CPUPerc": "0.65%",
|
||||
"MemUsage": "65.73MiB / 256MiB",
|
||||
"MemPerc": "25.68%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-redis-1",
|
||||
"CPUPerc": "0.48%",
|
||||
"MemUsage": "8.773MiB / 512MiB",
|
||||
"MemPerc": "1.71%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-tempo-1",
|
||||
"CPUPerc": "0.28%",
|
||||
"MemUsage": "320.2MiB / 512MiB",
|
||||
"MemPerc": "62.53%"
|
||||
},
|
||||
{
|
||||
"Name": "messenger-test-worker-1",
|
||||
"CPUPerc": "0.01%",
|
||||
"MemUsage": "73.34MiB / 2GiB",
|
||||
"MemPerc": "3.58%"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark."
|
||||
},
|
||||
"api_replicas": 2,
|
||||
"dashboard_uid": "messenger-overview",
|
||||
"api_correlation": {
|
||||
"trace_id": "b07429169c91eb6e67d211959181ab10",
|
||||
"span_id": "96318acd3dddff53",
|
||||
"log": {
|
||||
"timestamp": "2026-11-19T15:56:54.681885+00:00",
|
||||
"severity": "INFO",
|
||||
"service_name": "messenger-api",
|
||||
"event": "http.request",
|
||||
"trace_id": "b07429169c91eb6e67d211959181ab10",
|
||||
"span_id": "96318acd3dddff53",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/messages",
|
||||
"status_code": 201,
|
||||
"request_id": "171446ed-c27f-45e0-86b3-1b6e06420b4e",
|
||||
"duration_seconds": 0.08245528700001614
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"name": "POST /api/v1/chats/{chat_id}/messages",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"spanId": "ljGKzT3d/1M=",
|
||||
"startTimeUnixNano": "1795103814599010618",
|
||||
"endTimeUnixNano": "1795103814682418591",
|
||||
"status": {
|
||||
"code": "STATUS_CODE_OK"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"worker_correlation": {
|
||||
"trace_id": "ad41b35e5e8b02c32f7a7b74d7b6278a",
|
||||
"span_id": "ff9586c715f69b64",
|
||||
"log": {
|
||||
"timestamp": "2026-11-19T15:56:55.880853+00:00",
|
||||
"severity": "INFO",
|
||||
"service_name": "messenger-worker",
|
||||
"event": "job.received",
|
||||
"trace_id": "ad41b35e5e8b02c32f7a7b74d7b6278a",
|
||||
"span_id": "ff9586c715f69b64",
|
||||
"job_id": "ce2dd02a-7263-43b3-9cdb-1caf53fcb262",
|
||||
"attachment_id": "d49c525e-6cab-4a88-b348-867cc4185499"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"spanId": "/5WGxxX2m2Q=",
|
||||
"startTimeUnixNano": "1795103815880827668",
|
||||
"endTimeUnixNano": "1795103816744588122",
|
||||
"status": {
|
||||
"code": "STATUS_CODE_OK"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"metrics": {
|
||||
"http_requests": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795103819.091,
|
||||
"383"
|
||||
]
|
||||
}
|
||||
],
|
||||
"http_duration": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795103819.104,
|
||||
"383"
|
||||
]
|
||||
}
|
||||
],
|
||||
"job_success": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795103819.119,
|
||||
"1"
|
||||
]
|
||||
}
|
||||
],
|
||||
"job_duration": [
|
||||
{
|
||||
"metric": {},
|
||||
"value": [
|
||||
1795103819.132,
|
||||
"1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"upload_trace": {
|
||||
"trace_id": "ad41b35e5e8b02c32f7a7b74d7b6278a",
|
||||
"chain": [
|
||||
{
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"span_id": "74b246705b53c840",
|
||||
"parent_span_id": "",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "cf8900edf04666d0",
|
||||
"parent_span_id": "74b246705b53c840",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "dc111544195865ab",
|
||||
"parent_span_id": "74b246705b53c840",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "ea30143c06d7101a",
|
||||
"parent_span_id": "74b246705b53c840",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "9414f5609a1615fc",
|
||||
"parent_span_id": "74b246705b53c840",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "ff9586c715f69b64",
|
||||
"parent_span_id": "9414f5609a1615fc",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "1"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "success"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "3a99027f12c54d14",
|
||||
"parent_span_id": "ff9586c715f69b64",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "s3.write_result",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "7238fed2427ff2b5",
|
||||
"parent_span_id": "ff9586c715f69b64",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.commit_ready",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "6228546b59c97cc1",
|
||||
"parent_span_id": "ff9586c715f69b64",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "d038f59546905b0d",
|
||||
"parent_span_id": "ff9586c715f69b64",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "8ddfdeb23bfdca06",
|
||||
"parent_span_id": "d038f59546905b0d",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "s3.read_staging",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "87438655356f817f",
|
||||
"parent_span_id": "ff9586c715f69b64",
|
||||
"job": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
"incidents": [
|
||||
{
|
||||
"scenario": "postgres_unavailable",
|
||||
"http_status": 503,
|
||||
"correlation": {
|
||||
"trace_id": "020e820d68045ad3195e842244670b11",
|
||||
"span_id": "d100199e5812f6e3",
|
||||
"log": {
|
||||
"timestamp": "2026-11-19T15:57:03.906932+00:00",
|
||||
"severity": "ERROR",
|
||||
"service_name": "messenger-api",
|
||||
"event": "http.request",
|
||||
"trace_id": "020e820d68045ad3195e842244670b11",
|
||||
"span_id": "d100199e5812f6e3",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/users",
|
||||
"status_code": 503,
|
||||
"request_id": "420eae93-41a6-41af-8f37-09e76b8ec997",
|
||||
"duration_seconds": 3.004546160000018
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"name": "POST /api/v1/users",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"spanId": "0QAZnlgS9uM=",
|
||||
"startTimeUnixNano": "1795103820902285656",
|
||||
"endTimeUnixNano": "1795103823907032467",
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"diagnostic": {
|
||||
"timestamp": "2026-11-19T15:57:03.905870+00:00",
|
||||
"severity": "ERROR",
|
||||
"service_name": "messenger-api",
|
||||
"event": "dependency.unavailable",
|
||||
"trace_id": "020e820d68045ad3195e842244670b11",
|
||||
"span_id": "d100199e5812f6e3",
|
||||
"request_id": "420eae93-41a6-41af-8f37-09e76b8ec997",
|
||||
"exception_type": "TimeoutError"
|
||||
},
|
||||
"db_span": "postgres.checkout",
|
||||
"alert": "MessengerHighErrorRate",
|
||||
"firing_after_seconds": 32.361,
|
||||
"resolved_after_seconds": 67.526
|
||||
},
|
||||
{
|
||||
"scenario": "worker_stopped_then_transient_retry",
|
||||
"alert": "MessengerQueueLag",
|
||||
"queue": {
|
||||
"depth": 1.0,
|
||||
"oldest_seconds": 25.641574,
|
||||
"firing_after_seconds": 28.639,
|
||||
"resolved": true,
|
||||
"final_depth": 0
|
||||
},
|
||||
"recovered_status": "ready",
|
||||
"trace_id": "ff8352e27270fbc16fea7216c3ff45c0",
|
||||
"chain": [
|
||||
{
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"span_id": "a8daa5d1e858d802",
|
||||
"parent_span_id": "",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "1bfe66add09278e5",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "cf5bdfb14aca0e03",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "fe476468bb06d3f2",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "07218fe96edd8a9f",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "1"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "93984e580b7e86d5",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "a458e41372613973",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "bbf04d7c40b14fb3",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "4"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "3c0af738c589e75e",
|
||||
"parent_span_id": "07218fe96edd8a9f",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "1"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "36993e3e09207c51",
|
||||
"parent_span_id": "3c0af738c589e75e",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "ea60348b52b97e80",
|
||||
"parent_span_id": "36993e3e09207c51",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "a6ce9125f0f2b69a",
|
||||
"parent_span_id": "3c0af738c589e75e",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "f7b2eab20244c154",
|
||||
"parent_span_id": "93984e580b7e86d5",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "2"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "06c6bcc9ba7a345d",
|
||||
"parent_span_id": "f7b2eab20244c154",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "83c7a52ecd57f72f",
|
||||
"parent_span_id": "06c6bcc9ba7a345d",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "0310e2dd03331cb3",
|
||||
"parent_span_id": "f7b2eab20244c154",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "f2c43df28f9d7a39",
|
||||
"parent_span_id": "a458e41372613973",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "3"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "b81945a1dac3e54e",
|
||||
"parent_span_id": "f2c43df28f9d7a39",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "206ffc2a7aaafca5",
|
||||
"parent_span_id": "b81945a1dac3e54e",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "f6f62fbcc37d065f",
|
||||
"parent_span_id": "f2c43df28f9d7a39",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "caf2f1cf5ab31318",
|
||||
"parent_span_id": "bbf04d7c40b14fb3",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "4"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "0e0bdca52b68b3d7",
|
||||
"parent_span_id": "caf2f1cf5ab31318",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "ff6de1ac78d07a28",
|
||||
"parent_span_id": "0e0bdca52b68b3d7",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "2f459882491c3a7c",
|
||||
"parent_span_id": "caf2f1cf5ab31318",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "0a79dcefe6527245",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "5"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq.publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"span_id": "a4e8cf406651958d",
|
||||
"parent_span_id": "a8daa5d1e858d802",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "6"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "6d709c8f5827479d",
|
||||
"parent_span_id": "0a79dcefe6527245",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "5"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "error"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "0a5ed8ae1cdac4bd",
|
||||
"parent_span_id": "6d709c8f5827479d",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "a6452306465ac351",
|
||||
"parent_span_id": "0a5ed8ae1cdac4bd",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "d5c93140a00b63ab",
|
||||
"parent_span_id": "6d709c8f5827479d",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "attachment.process.v1",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"span_id": "45c2e49b6f6471bf",
|
||||
"parent_span_id": "a4e8cf406651958d",
|
||||
"job": {
|
||||
"job.attempt": {
|
||||
"intValue": "6"
|
||||
},
|
||||
"outcome": {
|
||||
"stringValue": "success"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "postgres.claim_job",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "6d995da536f4d248",
|
||||
"parent_span_id": "45c2e49b6f6471bf",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "ab6ddaa6b67c5663",
|
||||
"parent_span_id": "6d995da536f4d248",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "s3.read_staging",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "ec1037446c0d5918",
|
||||
"parent_span_id": "45c2e49b6f6471bf",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.checkout",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "4059610a72cf3cd0",
|
||||
"parent_span_id": "45c2e49b6f6471bf",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "s3.write_result",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "741f62682fa8057f",
|
||||
"parent_span_id": "45c2e49b6f6471bf",
|
||||
"job": {}
|
||||
},
|
||||
{
|
||||
"name": "postgres.commit_ready",
|
||||
"kind": "SPAN_KIND_INTERNAL",
|
||||
"span_id": "17c7adc7b583eeda",
|
||||
"parent_span_id": "45c2e49b6f6471bf",
|
||||
"job": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"scenario": "loki_unavailable",
|
||||
"successful_messages": 204,
|
||||
"dropped_after_export_failure": 6.0,
|
||||
"dropped_at_enqueue": 0.0,
|
||||
"samples": [
|
||||
{
|
||||
"seconds": 0.06,
|
||||
"rss_bytes": 239079424.0,
|
||||
"queued_batches": 0.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 5.14,
|
||||
"rss_bytes": 239288320.0,
|
||||
"queued_batches": 5.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 10.08,
|
||||
"rss_bytes": 239292416.0,
|
||||
"queued_batches": 10.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 15.12,
|
||||
"rss_bytes": 240283648.0,
|
||||
"queued_batches": 15.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 20.07,
|
||||
"rss_bytes": 241635328.0,
|
||||
"queued_batches": 20.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 25.09,
|
||||
"rss_bytes": 241795072.0,
|
||||
"queued_batches": 24.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 30.07,
|
||||
"rss_bytes": 241836032.0,
|
||||
"queued_batches": 29.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 35.09,
|
||||
"rss_bytes": 241856512.0,
|
||||
"queued_batches": 33.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 40.09,
|
||||
"rss_bytes": 241860608.0,
|
||||
"queued_batches": 38.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 45.09,
|
||||
"rss_bytes": 242176000.0,
|
||||
"queued_batches": 43.0,
|
||||
"capacity_batches": 256.0
|
||||
},
|
||||
{
|
||||
"seconds": 50.07,
|
||||
"rss_bytes": 242384896.0,
|
||||
"queued_batches": 48.0,
|
||||
"capacity_batches": 256.0
|
||||
}
|
||||
],
|
||||
"log_export_recovered": true
|
||||
}
|
||||
],
|
||||
"privacy_checked_logs": 413,
|
||||
"checks": {
|
||||
"three_backends": true,
|
||||
"dashboard_queries": true,
|
||||
"server_consumer_traces": true,
|
||||
"log_trace_correlation": true,
|
||||
"low_cardinality": true,
|
||||
"privacy": true,
|
||||
"history_survives_restart": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"date_utc": "2026-11-19T15:50:02.598982+00:00",
|
||||
"checks": [
|
||||
{
|
||||
"name": "contract",
|
||||
"command": [
|
||||
".venv/bin/python",
|
||||
"scripts/check_contract.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 0.112,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "openapi",
|
||||
"command": [
|
||||
".venv/bin/python",
|
||||
"scripts/validate_openapi.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 1.068,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "compose",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"config",
|
||||
"--quiet"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 0.113,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "startup",
|
||||
"command": [
|
||||
"make",
|
||||
"--no-print-directory",
|
||||
"test-stand-up"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 70.403,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "regression",
|
||||
"command": [
|
||||
"make",
|
||||
"--no-print-directory",
|
||||
"test-unit"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 322.597,
|
||||
"result": "PASS",
|
||||
"summary": "206 passed in 98.89s (0:01:38)"
|
||||
},
|
||||
{
|
||||
"name": "api",
|
||||
"command": [
|
||||
".venv/bin/python",
|
||||
"tests/smoke.py",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 4.112,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "observability",
|
||||
"command": [
|
||||
".venv/bin/python",
|
||||
"-m",
|
||||
"scripts.check_observability",
|
||||
"--base-url",
|
||||
"http://localhost:28080",
|
||||
"--grafana-url",
|
||||
"http://localhost:23000",
|
||||
"--incidents",
|
||||
"--restart-telemetry"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 292.686,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "persistence",
|
||||
"command": [
|
||||
".venv/bin/python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"persistence",
|
||||
"--action-script",
|
||||
"scripts/restart.sh",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 61.526,
|
||||
"result": "PASS"
|
||||
},
|
||||
{
|
||||
"name": "failover",
|
||||
"command": [
|
||||
".venv/bin/python",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"failover",
|
||||
"--action-script",
|
||||
"scripts/stop-one.sh",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"duration_seconds": 19.039,
|
||||
"result": "PASS"
|
||||
}
|
||||
],
|
||||
"profile": {
|
||||
"single_stack": true,
|
||||
"load_duration_seconds": 10,
|
||||
"load_requests_per_second": 10,
|
||||
"load_concurrency": 2
|
||||
},
|
||||
"result": "PASS",
|
||||
"cleanup": {
|
||||
"project": "messenger-test",
|
||||
"remaining": {
|
||||
"containers": [],
|
||||
"networks": [],
|
||||
"volumes": []
|
||||
},
|
||||
"result": "PASS"
|
||||
},
|
||||
"finished_at_utc": "2026-11-19T16:03:05.425775+00:00"
|
||||
}
|
||||
+17
-4
@@ -1,7 +1,20 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Наблюдаемость Messenger
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
API, publisher и worker экспортируют OTLP HTTP/protobuf в Collector (:4318).
|
||||
Метрики выдаются Prometheus на :8889, логи отправляются в native OTLP Loki,
|
||||
traces — в Tempo. Grafana читает три хранилища через provisioned datasources.
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
Конфиги backend, Collector, dashboard и provisioning находятся в этом каталоге.
|
||||
Dashboard UID: `messenger-overview`. Метрики — в секундах и счётчиках попыток;
|
||||
точные запросы/единицы находятся в JSON. Resource environment берётся из APP_ENV.
|
||||
В Loki индексируются service.name и окружение; instance ID — structured metadata.
|
||||
Строка лога содержит trace/span ID, ссылка TraceID открывает Tempo.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
История хранится в prometheusdata/lokidata/tempodata, retention 3 дня;
|
||||
grafanadata содержит настройки Grafana. Внешний порт имеет только Grafana,
|
||||
по умолчанию 127.0.0.1:13000. Её healthcheck проверяет также Loki, Tempo и Collector.
|
||||
Бизнес-readiness API от этих сервисов не зависит.
|
||||
|
||||
`make test-observability` запускает контролируемые отказы на указанном стенде.
|
||||
`make verify` создаёт отдельный проект и удаляет его ресурсы после прогона.
|
||||
Правила Prometheus, запуск и порядок диагностики описаны в [REPORT.md](../../REPORT.md).
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: |
|
||||
sum(increase(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[30s]))
|
||||
/ clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[30s])), 1) > 0.05
|
||||
for: 5s
|
||||
annotations:
|
||||
summary: "Более 5% бизнес-запросов завершились HTTP 5xx за 30 секунд"
|
||||
runbook: "REPORT.md#диагностика-инцидентов"
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_seconds) > 15 and max(messenger_queue_depth_ratio) > 0
|
||||
for: 5s
|
||||
annotations:
|
||||
summary: "Старейшее ожидающее задание находится в очереди более 15 секунд"
|
||||
runbook: "REPORT.md#диагностика-инцидентов"
|
||||
@@ -0,0 +1,61 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 3s
|
||||
sending_queue:
|
||||
queue_size: 256
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
max_elapsed_time: 30s
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 3s
|
||||
sending_queue:
|
||||
queue_size: 256
|
||||
num_consumers: 2
|
||||
retry_on_failure:
|
||||
max_elapsed_time: 30s
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
@@ -0,0 +1,617 @@
|
||||
{
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger — наблюдаемость",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"tags": [
|
||||
"messenger",
|
||||
"observability"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API: запросы в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (http_request_method, http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m]))",
|
||||
"legendFormat": "{{http_request_method}} {{http_route}}"
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API: доля 5xx",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(messenger_http_requests_total{http_response_status_class=\"5xx\",http_route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m])), 0.000001)",
|
||||
"legendFormat": "5xx / все бизнес-запросы"
|
||||
}
|
||||
],
|
||||
"description": "Ошибки сервера: 5xx / все запросы, кроме health. 4xx не считаются отказом сервера.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API: время ответа p50/p95/p99",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 9,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"description": "Histogram в секундах, buckets суммируются по репликам.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Worker: попытки в секунду по исходу",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 9,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(messenger_job_attempts_total[1m]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"description": "success — результат записан; error — попытка не удалась; deferred — истёк lease. Дубли без обработки не считаются.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Worker: время обработки p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 18,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"description": "Одна фактическая попытка, включая S3, преобразования и фиксацию metadata.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker: обработанные попытки",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 18,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (messenger_job_attempts_total)",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "API: структурные логи",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 27,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=\"messenger-api\"} | json",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Worker: структурные логи",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 27,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=\"messenger-worker\"} | json",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Реальные server/consumer traces",
|
||||
"type": "table",
|
||||
"datasource": {
|
||||
"type": "tempo",
|
||||
"uid": "tempo"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 36,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"queryType": "traceqlSearch",
|
||||
"query": "{ resource.service.name =~ \"messenger-api|messenger-worker\" }",
|
||||
"limit": 20
|
||||
}
|
||||
],
|
||||
"description": "Открыть trace в Explore; ссылка TraceID из строки Loki ведёт к той же трассе.",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "API: число 5xx бизнес-запросов",
|
||||
"type": "stat",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 36,
|
||||
"w": 12,
|
||||
"h": 9
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_http_requests_total{http_response_status_class=\"5xx\",http_route!~\"/health/.*\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
},
|
||||
"description": "Накопленное число 5xx: показывает и первую ошибку нового ряда, когда rate ещё не имеет двух разных точек."
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Очередь: ожидающие задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 45
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max by (queue_name) (messenger_queue_depth_ratio)",
|
||||
"legendFormat": "Глубина {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Очередь: возраст старейшей задачи",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 45
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max by (queue_name) (messenger_queue_oldest_seconds)",
|
||||
"legendFormat": "Возраст {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "PostgreSQL: соединения и ожидающие",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 53
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_db_pool_used_ratio)",
|
||||
"legendFormat": "Занято {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by (service_name) (messenger_db_pool_waiting_ratio)",
|
||||
"legendFormat": "Ожидают {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Worker: доля занятых слотов",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 53
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy_ratio) / sum(messenger_worker_capacity_ratio)",
|
||||
"legendFormat": "Загрузка {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Процессы: CPU в ядрах",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 61
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "CPU {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Процессы: резидентная память",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 61
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
|
||||
"legendFormat": "RSS {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Вложения: p95 времени до ready",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 69
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
|
||||
"legendFormat": "p95 {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "Collector: очереди экспорта",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 69
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_size)",
|
||||
"legendFormat": "Размер {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
|
||||
"legendFormat": "Ёмкость {{service_name}}{{queue_name}}{{exporter}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"templating": {
|
||||
"list": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: true
|
||||
editable: false
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
urlDisplayLabel: Open trace
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-1m'
|
||||
spanEndTimeShift: '1m'
|
||||
tags: [{key: service.name, value: service_name}]
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
@@ -0,0 +1,39 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 72h
|
||||
otlp_config:
|
||||
resource_attributes:
|
||||
ignore_defaults: true
|
||||
attributes_config:
|
||||
- action: index_label
|
||||
attributes: [service.name, deployment.environment.name]
|
||||
- action: structured_metadata
|
||||
attributes: [service.version, service.instance.id]
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
@@ -0,0 +1,22 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
ingester:
|
||||
max_block_duration: 1m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 72h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Сохранить W3C context между HTTP-транзакцией и publisher."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "b55d0c6819ab"
|
||||
down_revision = "a62c950e18b3"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("attachment_jobs", sa.Column("trace_context", sa.JSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("attachment_jobs", "trace_context")
|
||||
@@ -21,6 +21,9 @@ MarkupSafe==3.0.4
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
opentelemetry-api==1.45.1
|
||||
opentelemetry-sdk==1.45.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.45.1
|
||||
opentelemetry-semantic-conventions==0.66b1
|
||||
packaging==26.3
|
||||
pathable==0.4.4
|
||||
pluggy==1.7.0
|
||||
|
||||
@@ -179,7 +179,7 @@ def crash(client, actor, chat, mode):
|
||||
)
|
||||
|
||||
|
||||
def retry_replay(client, actor, chat, permanent):
|
||||
def retry_replay(client, actor, chat, permanent, on_failed=None, on_queued=None):
|
||||
name = "messenger-retry-fault-" + uuid4().hex
|
||||
samples = []
|
||||
configuration = ComposeProject().config["services"]["worker"]["environment"]
|
||||
@@ -188,6 +188,8 @@ def retry_replay(client, actor, chat, permanent):
|
||||
try:
|
||||
require_compose("stop", "worker")
|
||||
attachment, original = client.attachment(actor, chat)
|
||||
if on_queued is not None:
|
||||
on_queued(attachment)
|
||||
mode = "fail-permanent" if permanent else "fail-transient"
|
||||
require_compose(
|
||||
"run",
|
||||
@@ -238,6 +240,8 @@ def retry_replay(client, actor, chat, permanent):
|
||||
actor=actor,
|
||||
expected=409,
|
||||
)
|
||||
if on_failed is not None:
|
||||
on_failed(sample, name)
|
||||
job_id = sample["job_id"]
|
||||
request_id = str(uuid4())
|
||||
listing = admin("list-failed")
|
||||
|
||||
@@ -0,0 +1,395 @@
|
||||
"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
|
||||
|
||||
--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
|
||||
В evidence сохраняются только агрегаты и выбранные технические поля.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import math
|
||||
import subprocess
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, Failure, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def eventually(operation, message, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
value = operation()
|
||||
if value:
|
||||
return value
|
||||
except (OSError, ValueError, KeyError, AssertionError):
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
class Grafana:
|
||||
def __init__(self, base):
|
||||
self.base = base.rstrip('/')
|
||||
secret = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
||||
self.authorization = 'Basic ' + base64.b64encode(('admin:' + secret).encode()).decode()
|
||||
|
||||
def get(self, path, **params):
|
||||
url = self.base + path
|
||||
if params:
|
||||
url += '?' + urllib.parse.urlencode(params)
|
||||
request = urllib.request.Request(url, headers={'Authorization': self.authorization, 'Accept': 'application/json'})
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
def proxy(self, source, path, **params):
|
||||
return self.get('/api/datasources/proxy/uid/' + source + path, **params)
|
||||
|
||||
def query(self, expression):
|
||||
value = self.proxy('prometheus', '/api/v1/query', query=expression)
|
||||
check(value['status'] == 'success', 'Prometheus query failed')
|
||||
return value['data']['result']
|
||||
|
||||
def logs(self, selector, start):
|
||||
value = self.proxy('loki', '/loki/api/v1/query_range', query=selector, start=start, end=time.time_ns(), limit=1000, direction='forward')
|
||||
check(value['status'] == 'success', 'Loki query failed')
|
||||
return [json.loads(line) for stream in value['data']['result'] for _, line, *_ in stream['values']]
|
||||
|
||||
def trace(self, trace_id):
|
||||
return self.proxy('tempo', '/api/traces/' + trace_id)
|
||||
|
||||
|
||||
def identifier_hex(value, byte_length):
|
||||
if len(value) == byte_length * 2 and all(c in '0123456789abcdefABCDEF' for c in value):
|
||||
return value.lower()
|
||||
return base64.b64decode(value, validate=True).hex()
|
||||
|
||||
|
||||
def spans_in(value):
|
||||
found = []
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if key == 'spans':
|
||||
found.extend(item)
|
||||
else:
|
||||
found.extend(spans_in(item))
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
found.extend(spans_in(item))
|
||||
return found
|
||||
|
||||
|
||||
def correlation(grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None):
|
||||
selector = '{service_name="' + service + '"} | json'
|
||||
if service == 'messenger-api':
|
||||
selector += ' | event="http.request"'
|
||||
if trace_id:
|
||||
selector += ' | trace_id="' + trace_id + '"'
|
||||
if attachment_id:
|
||||
selector += ' | attachment_id="' + attachment_id + '"'
|
||||
if request_id:
|
||||
selector += ' | request_id="' + request_id + '"'
|
||||
records = eventually(lambda: grafana.logs(selector, start), 'No correlated log for ' + service)
|
||||
record = records[0]
|
||||
trace_id = record['trace_id']
|
||||
def matching_trace():
|
||||
candidate = grafana.trace(trace_id)
|
||||
return candidate if any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans_in(candidate)) else None
|
||||
trace = eventually(matching_trace, 'Matching span is missing in Tempo')
|
||||
spans = [span for span in spans_in(trace) if identifier_hex(span.get('spanId', ''), 8) == record['span_id']]
|
||||
# Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
|
||||
check(any(identifier_hex(span.get('spanId', ''), 8) == record['span_id'] for span in spans), 'Log span_id not found in real trace')
|
||||
check(all(int(span['endTimeUnixNano']) > int(span['startTimeUnixNano']) for span in spans), 'Trace has no real duration')
|
||||
return {'trace_id': trace_id, 'span_id': record['span_id'], 'log': record,
|
||||
'spans': [{key: span.get(key) for key in ('name', 'kind', 'spanId', 'startTimeUnixNano', 'endTimeUnixNano', 'status')} for span in spans]}, trace
|
||||
|
||||
|
||||
def run_load(args):
|
||||
check(1 <= args.duration <= 300 and 1 <= args.rate <= 40 and 1 <= args.concurrency <= 16, 'Load profile outside bounds')
|
||||
client = Client(args.base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
project = ComposeProject()
|
||||
container_ids = project.run('ps', '--quiet').splitlines()
|
||||
samples = []
|
||||
stop = threading.Event()
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
result = subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*container_ids],capture_output=True,text=True,timeout=15)
|
||||
if result.returncode == 0:
|
||||
samples.append({'elapsed_seconds':round(time.monotonic()-started,3), 'containers':[
|
||||
{k:row[k] for k in ('Name','CPUPerc','MemUsage','MemPerc')} for row in (json.loads(line) for line in result.stdout.splitlines())]})
|
||||
stop.wait(3)
|
||||
|
||||
def write(index):
|
||||
local = Client(args.base_url)
|
||||
before = time.monotonic()
|
||||
try:
|
||||
local.message(actor,chat,'synthetic-observability-' + str(index))
|
||||
return time.monotonic()-before
|
||||
except Failure:
|
||||
return None
|
||||
|
||||
started = time.monotonic()
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = []
|
||||
for index in range(args.duration * args.rate):
|
||||
time.sleep(max(0, started + index/args.rate - time.monotonic()))
|
||||
futures.append(pool.submit(write,index))
|
||||
results = [future.result() for future in futures]
|
||||
elapsed = time.monotonic()-started
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
latencies = sorted(value for value in results if value is not None)
|
||||
failed = len(results)-len(latencies)
|
||||
def percentile(q):
|
||||
return round(latencies[max(0, math.ceil(q*len(latencies))-1)]*1000,3) if latencies else None
|
||||
report = {'date_utc':datetime.now(timezone.utc).isoformat(), 'workload':'synthetic POST messages; fixed arrival rate; no retries',
|
||||
'duration_seconds':args.duration,'elapsed_seconds':round(elapsed,3),'arrival_rate':args.rate,'concurrency_limit':args.concurrency,
|
||||
'successful_requests':len(latencies),'failed_requests':failed,'error_ratio':failed/len(results),
|
||||
'successful_rps':round(len(latencies)/elapsed,3), 'successful_latency_ms':{name:percentile(q) for name,q in [('p50',.5),('p95',.95),('p99',.99)]},
|
||||
'dataset':{'users':3,'chats':1,'messages':len(latencies)},'resource_samples':samples,
|
||||
'limits':'Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.'}
|
||||
check(bool(latencies) and failed == 0, 'Controlled load contains errors')
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def metric(grafana, expression):
|
||||
rows = grafana.query(expression)
|
||||
return float(rows[0]['value'][1]) if rows else None
|
||||
|
||||
|
||||
def alert_firing(grafana, name):
|
||||
return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
|
||||
|
||||
|
||||
def trace_chain(grafana, trace_id, *, retry=False):
|
||||
def inspect():
|
||||
spans = spans_in(grafana.trace(trace_id))
|
||||
by_id = {identifier_hex(span['spanId'], 8): span for span in spans}
|
||||
workers = [span for span in spans if span['name'] == 'attachment.process.v1'
|
||||
and span.get('kind') in ('SPAN_KIND_CONSUMER', 5)]
|
||||
for worker in workers:
|
||||
publisher = by_id.get(identifier_hex(worker.get('parentSpanId', ''), 8), {})
|
||||
if publisher.get('kind') not in ('SPAN_KIND_PRODUCER', 4):
|
||||
return None
|
||||
upload = by_id.get(identifier_hex(publisher.get('parentSpanId', ''), 8), {})
|
||||
if upload.get('name') != 'POST /api/v1/chats/{chat_id}/attachments':
|
||||
return None
|
||||
names = {span['name'] for span in spans}
|
||||
if not workers or not {'s3.read_staging', 's3.write_result', 'postgres.commit_ready'} <= names:
|
||||
return None
|
||||
if retry:
|
||||
outcomes = set()
|
||||
for span in workers:
|
||||
attrs = {row['key']: row['value'] for row in span.get('attributes', [])}
|
||||
outcomes.add((int(attrs['job.attempt']['intValue']), attrs.get('outcome', {}).get('stringValue')))
|
||||
if not {(1, 'error'), (2, 'error')} <= outcomes or not any(n > 2 and outcome == 'success' for n, outcome in outcomes):
|
||||
return None
|
||||
return [{'name': span['name'], 'kind': span.get('kind'),
|
||||
'span_id': identifier_hex(span['spanId'], 8),
|
||||
'parent_span_id': identifier_hex(span.get('parentSpanId', ''), 8),
|
||||
'job': {entry['key']: entry['value'] for entry in span.get('attributes', [])
|
||||
if entry['key'] in {'job.attempt', 'outcome'}}} for span in spans]
|
||||
return eventually(inspect, 'Missing upload -> producer -> consumer -> S3/DB chain')
|
||||
|
||||
|
||||
def telemetry_outage(project, client, grafana, actor, chat):
|
||||
failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
||||
refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
|
||||
baseline = [metric(grafana, expr) for expr in (failed, refused)]
|
||||
rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
|
||||
samples = []
|
||||
successful = 0
|
||||
try:
|
||||
project.run('stop', 'loki')
|
||||
start = time.monotonic()
|
||||
for second in range(51):
|
||||
for index in range(4):
|
||||
client.message(actor, chat, f'outage-{second}-{index}')
|
||||
successful += 1
|
||||
if second % 5 == 0:
|
||||
samples.append({
|
||||
'seconds': round(time.monotonic() - start, 2),
|
||||
'rss_bytes': metric(grafana, rss),
|
||||
'queued_batches': metric(grafana, 'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})'),
|
||||
'capacity_batches': metric(grafana, 'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})'),
|
||||
})
|
||||
time.sleep(max(0, start + second + 1 - time.monotonic()))
|
||||
losses = [metric(grafana, expr) - count for expr, count in zip((failed, refused), baseline)]
|
||||
check(sum(losses) > 0, 'No measured Collector log loss during prolonged outage')
|
||||
check(all(0 <= row['queued_batches'] <= row['capacity_batches'] == 256 for row in samples), 'Unexpected queue bound')
|
||||
check(max(row['rss_bytes'] for row in samples) < 256 * 1024**2, 'Collector crossed container memory limit')
|
||||
finally:
|
||||
project.run('start', 'loki')
|
||||
eventually(lambda: grafana.proxy('loki', '/loki/api/v1/labels').get('status') == 'success',
|
||||
'Loki did not recover')
|
||||
restored = client.message(actor, chat, 'telemetry-backend-restored')
|
||||
correlation(grafana, 'messenger-api', time.time_ns() - 30 * 10**9,
|
||||
trace_id=restored['headers']['X-Trace-Id'])
|
||||
return {'scenario':'loki_unavailable', 'successful_messages': successful,
|
||||
'dropped_after_export_failure': losses[0], 'dropped_at_enqueue': losses[1],
|
||||
'samples': samples, 'log_export_recovered': True}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', required=True)
|
||||
parser.add_argument('--grafana-url', default='http://localhost:13000')
|
||||
parser.add_argument('--incidents', action='store_true')
|
||||
parser.add_argument('--restart-telemetry', action='store_true')
|
||||
parser.add_argument('--output', default='evidence/observability.json')
|
||||
parser.add_argument('--duration', type=int, default=10)
|
||||
parser.add_argument('--rate', type=int, default=10)
|
||||
parser.add_argument('--concurrency', type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
load = run_load(args)
|
||||
start = time.time_ns()
|
||||
project = ComposeProject()
|
||||
client = Client(args.base_url, timeout=40)
|
||||
aliases = {r.instance_id for r in project.replicas() if r.running}
|
||||
check(len(aliases) >= 2, 'Need two real API containers')
|
||||
check(client.call('GET', '/health/live')['headers'].get('X-Instance-Id') in aliases, 'Wrong Compose project for BASE_URL')
|
||||
grafana = Grafana(args.grafana_url)
|
||||
eventually(lambda: grafana.get('/api/health').get('database') == 'ok', 'Grafana is not ready')
|
||||
sources = grafana.get('/api/datasources')
|
||||
check({'prometheus', 'loki', 'tempo'} <= {source['uid'] for source in sources}, 'Missing provisioned datasource')
|
||||
dashboard = grafana.get('/api/dashboards/uid/messenger-overview')['dashboard']
|
||||
check(len(dashboard['panels']) >= 8, 'Dashboard is missing panels')
|
||||
actor, bob, eve, chat = client.bootstrap()
|
||||
sentinel = 'private-content-' + uuid4().hex
|
||||
response = client.message(actor, chat, sentinel)
|
||||
controlled_trace = response['headers'].get('X-Trace-Id')
|
||||
check(bool(controlled_trace), 'No API trace identifier')
|
||||
attachment, _ = client.attachment(actor, chat)
|
||||
client.wait_ready(actor, attachment)
|
||||
api, api_trace = correlation(grafana, 'messenger-api', start, trace_id=controlled_trace, request_id=response['headers']['X-Request-Id'])
|
||||
worker, worker_trace = correlation(grafana, 'messenger-worker', start, attachment_id=attachment['id'])
|
||||
accepted_chain = trace_chain(grafana, worker['trace_id'])
|
||||
check(api['log']['request_id'] == response['headers']['X-Request-Id'], 'Request id correlation lost')
|
||||
check(any(span['kind'] in ('SPAN_KIND_SERVER', 2) for span in api['spans']), 'No server span')
|
||||
check(any(span['kind'] in ('SPAN_KIND_CONSUMER', 5) for span in worker['spans']), 'No consumer span')
|
||||
processed = eventually(lambda: grafana.logs('{service_name="messenger-worker"} | json | event="job.processed" | trace_id="' + worker['trace_id'] + '"', start), 'No worker result log')
|
||||
check(processed[0]['outcome'] == 'success', 'Controlled worker did not succeed')
|
||||
metrics = {}
|
||||
expressions = {
|
||||
'http_requests': 'sum(messenger_http_requests_total)',
|
||||
'http_duration': 'sum(messenger_http_duration_seconds_count)',
|
||||
'job_success': 'sum(messenger_job_attempts_total{outcome="success"})',
|
||||
'job_duration': 'sum(messenger_job_duration_seconds_count)',
|
||||
}
|
||||
for name, expression in expressions.items():
|
||||
metrics[name] = eventually(lambda expression=expression: grafana.query(expression), 'Missing metric: ' + expression)
|
||||
check(float(metrics[name][0]['value'][1]) > 0, 'Empty metric: ' + name)
|
||||
# Все PromQL из dashboard должны давать численные данные после traffic.
|
||||
for panel in dashboard['panels']:
|
||||
if panel['datasource']['uid'] == 'prometheus':
|
||||
eventually(lambda panel=panel: all(grafana.query(target['expr']) for target in panel['targets']), 'Empty dashboard panel: ' + panel['title'])
|
||||
forbidden_labels = {'user_id','chat_id','message_id','attachment_id','request_id','filename','url','http_target','http_url'}
|
||||
series = grafana.query('{__name__=~"messenger_.*"}')
|
||||
check(not any(forbidden_labels & set(row['metric']) for row in series), 'High cardinality metric label')
|
||||
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'project': project.name,
|
||||
'load': load, 'api_replicas': len(aliases), 'dashboard_uid': dashboard['uid'],
|
||||
'api_correlation': api, 'worker_correlation': worker, 'metrics': metrics, 'upload_trace': {'trace_id':worker['trace_id'], 'chain':accepted_chain}, 'incidents': []}
|
||||
if args.incidents:
|
||||
def fail_request():
|
||||
return client.call('POST', '/api/v1/users', expected=503,
|
||||
body={'username':'fault_' + uuid4().hex[:12],
|
||||
'display_name':'Synthetic', 'password':'not-exported-' + uuid4().hex})
|
||||
|
||||
before = time.monotonic()
|
||||
try:
|
||||
project.run('stop', 'postgres')
|
||||
failed = fail_request()
|
||||
error_trace = failed['headers'].get('X-Trace-Id')
|
||||
error, db_trace = correlation(grafana, 'messenger-api', start, trace_id=error_trace,
|
||||
request_id=failed['headers']['X-Request-Id'])
|
||||
diagnostics = eventually(lambda: grafana.logs(
|
||||
'{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="' + error_trace + '"', start),
|
||||
'No dependency diagnostic log')
|
||||
checkout = next(span for span in spans_in(db_trace) if span['name'] == 'postgres.checkout'
|
||||
and span.get('status', {}).get('code') in ('STATUS_CODE_ERROR', 2))
|
||||
|
||||
def db_alert():
|
||||
for _ in range(4):
|
||||
fail_request()
|
||||
return alert_firing(grafana, 'MessengerHighErrorRate')
|
||||
|
||||
eventually(db_alert, 'Error rate alert did not fire', timeout=75)
|
||||
report['incidents'].append({'scenario':'postgres_unavailable','http_status':503,'correlation':error,
|
||||
'diagnostic':diagnostics[0], 'db_span':checkout['name'],
|
||||
'alert':'MessengerHighErrorRate', 'firing_after_seconds':round(time.monotonic()-before,3)})
|
||||
finally:
|
||||
project.run('start', 'postgres')
|
||||
eventually(lambda: _ready(client), 'API did not recover after PostgreSQL')
|
||||
eventually(lambda: not alert_firing(grafana, 'MessengerHighErrorRate'), 'Error rate alert not resolved')
|
||||
report['incidents'][-1]['resolved_after_seconds'] = round(time.monotonic() - before, 3)
|
||||
|
||||
from scripts.check_jobs import retry_replay
|
||||
queue = {}
|
||||
|
||||
def observe_queue(_):
|
||||
observed = time.monotonic()
|
||||
eventually(lambda: alert_firing(grafana, 'MessengerQueueLag'), 'Queue lag alert did not fire')
|
||||
queue.update(depth=metric(grafana, 'max(messenger_queue_depth_ratio)'),
|
||||
oldest_seconds=metric(grafana, 'max(messenger_queue_oldest_seconds)'),
|
||||
firing_after_seconds=round(time.monotonic()-observed,3))
|
||||
|
||||
incident = retry_replay(client, actor, chat, permanent=False, on_queued=observe_queue)
|
||||
retried_worker, retry_trace = correlation(grafana, 'messenger-worker', start,
|
||||
attachment_id=incident['attachment_id'])
|
||||
chain = trace_chain(grafana, retried_worker['trace_id'], retry=True)
|
||||
eventually(lambda: not alert_firing(grafana, 'MessengerQueueLag')
|
||||
and metric(grafana, 'max(messenger_queue_depth_ratio)') == 0, 'Backlog did not drain')
|
||||
queue.update(resolved=True, final_depth=0)
|
||||
report['incidents'].append({'scenario':'worker_stopped_then_transient_retry',
|
||||
'alert':'MessengerQueueLag', 'queue':queue, 'recovered_status':incident['after']['status'],
|
||||
'trace_id':retried_worker['trace_id'], 'chain':chain})
|
||||
report['incidents'].append(telemetry_outage(project, client, grafana, actor, chat))
|
||||
else:
|
||||
retry_trace = {}
|
||||
# Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
|
||||
logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
|
||||
exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
|
||||
private = [sentinel, *(person['token'] for person in (actor,bob,eve)), *(person['registration']['password'] for person in (actor,bob,eve))]
|
||||
private.extend(path.read_text().strip() for path in (ROOT / 'secrets').iterdir() if path.is_file())
|
||||
check(not any(secret and secret in exported for secret in private), 'Private value found in telemetry')
|
||||
check(all(record.get('trace_id') and record.get('span_id') and record.get('timestamp') and record.get('severity') for record in logs), 'Incomplete structured logs')
|
||||
report['privacy_checked_logs'] = len(logs)
|
||||
report['checks'] = {'three_backends':True, 'dashboard_queries':True,'server_consumer_traces':True,'log_trace_correlation':True,'low_cardinality':True,'privacy':True}
|
||||
if args.restart_telemetry:
|
||||
historical_time = time.time()
|
||||
expression = 'sum(messenger_http_requests_total)'
|
||||
before = grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result']
|
||||
project.run('restart', 'prometheus', 'loki', 'tempo', 'grafana')
|
||||
after = eventually(lambda: grafana.proxy('prometheus', '/api/v1/query', query=expression, time=historical_time)['data']['result'], 'Prometheus history lost')
|
||||
check(before == after, 'Historical metric changed after storage restart')
|
||||
correlation(grafana, 'messenger-api', start, trace_id=api['trace_id'])
|
||||
correlation(grafana, 'messenger-worker', start, trace_id=worker['trace_id'])
|
||||
report['checks']['history_survives_restart'] = True
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS: telemetry, dashboard, privacy, correlations' + ('; alerts and outage recovery verified' if args.incidents else ''))
|
||||
|
||||
|
||||
def _ready(client):
|
||||
return client.call('GET', '/health/ready', expected=[200,503])['status'] == 200
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -71,3 +71,154 @@ class ComposeProject:
|
||||
)
|
||||
for row in map(json.loads, rows)
|
||||
]
|
||||
|
||||
|
||||
def cleanup_docker(*args, allow_failure=False):
|
||||
result = subprocess.run(['docker',*args],cwd=ROOT,text=True,capture_output=True,timeout=120)
|
||||
if result.returncode and not allow_failure:
|
||||
raise RuntimeError('Docker cleanup failed: ' + ' '.join(args[:2]))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def inventory(projects):
|
||||
resources = {'containers':[], 'networks':[], 'volumes':[]}
|
||||
for project in projects:
|
||||
label = 'label=com.docker.compose.project=' + project
|
||||
resources['containers'].extend(cleanup_docker('ps','-aq','--filter',label).splitlines())
|
||||
resources['networks'].extend(cleanup_docker('network','ls','-q','--filter',label).splitlines())
|
||||
resources['volumes'].extend(cleanup_docker('volume','ls','-q','--filter',label).splitlines())
|
||||
return resources
|
||||
|
||||
|
||||
|
||||
def clean_stands(*, apply=False):
|
||||
config = json.loads(cleanup_docker('compose','--env-file','.env','config','--format','json'))
|
||||
projects = sorted({config['name'], os.getenv('MAIN_PROJECT', 'messenger'), os.getenv('TEST_PROJECT', 'messenger-test')})
|
||||
resources = inventory(projects)
|
||||
names = {service['image'] for service in config['services'].values() if 'build' in service}
|
||||
image_ids = set()
|
||||
for name in names:
|
||||
image_ids.update(cleanup_docker('image','ls','-q',name).splitlines())
|
||||
# У старых сборок API может уже не быть тега. Сверяем также команду и WORKDIR,
|
||||
# чтобы не затронуть сторонние образы с обычным OCI revision label.
|
||||
candidates = cleanup_docker('image','ls','-q','--filter','label=org.opencontainers.image.revision').splitlines()
|
||||
for candidate in candidates:
|
||||
image = json.loads(cleanup_docker('image','inspect',candidate))[0]
|
||||
settings = image['Config']
|
||||
if settings.get('Cmd') == ['python','-m','app'] and settings.get('WorkingDir') == '/app':
|
||||
image_ids.add(candidate)
|
||||
plan = {'projects':projects, 'resources':resources, 'application_image_ids':sorted(image_ids), 'local_settings_preserved':True}
|
||||
if not apply:
|
||||
return plan
|
||||
if resources['containers']:
|
||||
cleanup_docker('rm','-f','-v',*resources['containers'])
|
||||
if resources['networks']:
|
||||
cleanup_docker('network','rm',*resources['networks'])
|
||||
if resources['volumes']:
|
||||
cleanup_docker('volume','rm',*resources['volumes'])
|
||||
if image_ids:
|
||||
cleanup_docker('image','rm',*sorted(image_ids))
|
||||
remaining = inventory(projects)
|
||||
if any(remaining.values()):
|
||||
raise RuntimeError('Cleanup incomplete')
|
||||
plan['remaining'] = remaining
|
||||
plan['application_images_remaining'] = [name for name in sorted(names) if cleanup_docker('image','ls','-q',name)]
|
||||
if plan['application_images_remaining']:
|
||||
raise RuntimeError('Application images remain')
|
||||
plan['result'] = 'PASS'
|
||||
return plan
|
||||
|
||||
|
||||
def verify():
|
||||
"""Один последовательный прогон на одном стенде с обязательной очисткой."""
|
||||
from datetime import datetime, timezone
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
project_name = os.getenv('TEST_PROJECT', 'messenger-test')
|
||||
if any(inventory([project_name]).values()):
|
||||
raise RuntimeError(f'Choose an unused TEST_PROJECT; {project_name} already has resources')
|
||||
os.environ.update(COMPOSE_PROJECT_NAME=project_name,
|
||||
COMPOSE_FILE='compose.yaml:compose.test.yaml',
|
||||
GRAFANA_HOST_PORT='23000', LB_HOST_PORT='28080',
|
||||
DOCKER_TESTS='1', OTEL_ENABLED='true')
|
||||
report = {'date_utc': datetime.now(timezone.utc).isoformat(), 'checks': [],
|
||||
'profile': {'single_stack': True, 'load_duration_seconds': 10,
|
||||
'load_requests_per_second': 10, 'load_concurrency': 2}}
|
||||
destination = ROOT / 'evidence/verification.json'
|
||||
destination.parent.mkdir(exist_ok=True)
|
||||
|
||||
def run(name, command, timeout=600, *, env=None):
|
||||
print('RUN: ' + name, flush=True)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=timeout,
|
||||
env={**os.environ, **(env or {})})
|
||||
executable = Path(command[0])
|
||||
display_command = [str(executable.relative_to(ROOT)) if executable.is_relative_to(ROOT)
|
||||
else command[0], *command[1:]]
|
||||
entry = {'name': name, 'command': display_command, 'exit_code': result.returncode,
|
||||
'duration_seconds': round(time.monotonic()-started, 3),
|
||||
'result': 'PASS' if result.returncode == 0 else 'FAIL'}
|
||||
if name == 'regression':
|
||||
summary = re.search(r'\d+ passed[^\n]*', result.stdout)
|
||||
if summary:
|
||||
entry['summary'] = summary.group(0)
|
||||
if result.returncode:
|
||||
# Полные диагностические логи локальны; в evidence нет путей/credentials.
|
||||
(ROOT / '.verification-error.log').write_text(result.stdout + result.stderr)
|
||||
entry['diagnostics'] = '.verification-error.log (local, not committed)'
|
||||
report['checks'].append(entry)
|
||||
print(entry['result'] + ': ' + name, flush=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError(name + ' failed; inspect local diagnostic log')
|
||||
|
||||
try:
|
||||
run('contract', [sys.executable, 'scripts/check_contract.py'])
|
||||
run('openapi', [sys.executable, 'scripts/validate_openapi.py'])
|
||||
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
|
||||
run('startup', ['make', '--no-print-directory', 'test-stand-up'], timeout=600)
|
||||
# Локальные ASGI fixtures не находятся в сети Compose. Проверки SDK
|
||||
# используют exporters в памяти, интеграционный стенд экспортирует OTLP.
|
||||
run('regression', ['make', '--no-print-directory', 'test-unit'], env={'OTEL_ENABLED': 'false'})
|
||||
run('api', [sys.executable, 'tests/smoke.py', '--base-url', 'http://localhost:28080'])
|
||||
run('observability', [sys.executable, '-m', 'scripts.check_observability',
|
||||
'--base-url', 'http://localhost:28080',
|
||||
'--grafana-url', 'http://localhost:23000',
|
||||
'--incidents', '--restart-telemetry'])
|
||||
run('persistence', [sys.executable, 'tests/resilience.py', '--mode', 'persistence',
|
||||
'--action-script', 'scripts/restart.sh', '--base-url', 'http://localhost:28080'])
|
||||
run('failover', [sys.executable, 'tests/resilience.py', '--mode', 'failover',
|
||||
'--action-script', 'scripts/stop-one.sh', '--base-url', 'http://localhost:28080'])
|
||||
report['result'] = 'PASS'
|
||||
except BaseException:
|
||||
report['result'] = 'FAIL'
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
print('RUN: cleanup', flush=True)
|
||||
ComposeProject().run('down', '--volumes', '--remove-orphans', timeout=120)
|
||||
remaining = inventory([project_name])
|
||||
if any(remaining.values()):
|
||||
raise RuntimeError('Verification project cleanup incomplete')
|
||||
report['cleanup'] = {'project': project_name, 'remaining': remaining, 'result': 'PASS'}
|
||||
print('PASS: cleanup', flush=True)
|
||||
finally:
|
||||
report['finished_at_utc'] = datetime.now(timezone.utc).isoformat()
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description='Проверка и очистка Compose-стендов')
|
||||
parser.add_argument('command', choices=['verify', 'clean'])
|
||||
parser.add_argument('--apply', action='store_true', help='Подтвердить удаление ресурсов')
|
||||
args = parser.parse_args()
|
||||
if args.command == 'verify':
|
||||
verify()
|
||||
else:
|
||||
print(json.dumps(clean_stands(apply=args.apply), indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -15,6 +15,7 @@ def main() -> None:
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
for name in (
|
||||
"grafana_admin_password",
|
||||
"db_password",
|
||||
"redis_password",
|
||||
"broker_password",
|
||||
|
||||
+12
-7
@@ -24,14 +24,19 @@ wait_healthy() {
|
||||
|
||||
worker_running=$(compose ps --status running --quiet worker)
|
||||
compose stop worker publisher cleanup
|
||||
compose restart postgres redis minio broker
|
||||
compose restart postgres redis minio broker prometheus loki tempo grafana otel-collector
|
||||
wait_healthy postgres redis minio broker
|
||||
compose restart api
|
||||
wait_healthy api
|
||||
compose restart nginx
|
||||
wait_healthy nginx
|
||||
compose start publisher cleanup
|
||||
compose restart api nginx
|
||||
wait_healthy api nginx
|
||||
# Compose start также запускает завершённые migrate/bootstrap dependencies.
|
||||
# Здесь нужны только ранее созданные фоновые контейнеры, без повторного bootstrap.
|
||||
background_ids=$(compose ps --all --quiet publisher cleanup)
|
||||
docker start $background_ids
|
||||
# Сценарий с заранее остановленным worker сохраняет его остановленным.
|
||||
if [ -n "$worker_running" ]; then
|
||||
compose start worker
|
||||
worker_ids=$(compose ps --all --quiet worker)
|
||||
docker start $worker_ids
|
||||
fi
|
||||
# Пока восстанавливался бизнес-путь, telemetry backend тоже успели запуститься.
|
||||
# Grafana healthcheck проверяет также готовность Loki, Tempo и Collector.
|
||||
wait_healthy prometheus grafana
|
||||
|
||||
@@ -65,7 +65,8 @@ async def test_unhandled_exception_is_hidden_from_client(
|
||||
assert "test-secret" not in json.dumps(payload, ensure_ascii=False)
|
||||
assert "Traceback" not in json.dumps(payload, ensure_ascii=False)
|
||||
assert "Unhandled error request_id=" in caplog.text
|
||||
assert "test-secret" in caplog.text
|
||||
assert "test-secret" not in caplog.text
|
||||
assert "RuntimeError" in caplog.text
|
||||
assert_request_id(response)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Проверки приватности и исходов на реальных SDK exporters в памяти."""
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import FastAPI
|
||||
import httpx
|
||||
from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import SpanKind
|
||||
import pytest
|
||||
|
||||
from app.core import telemetry as module
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def clean_db():
|
||||
# Этот набор не использует БД, Redis и фикстуры приложения.
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def telemetry(monkeypatch):
|
||||
spans = InMemorySpanExporter()
|
||||
logs = InMemoryLogRecordExporter()
|
||||
metrics = InMemoryMetricReader()
|
||||
monkeypatch.setattr(module, "OTLPSpanExporter", lambda **_: spans)
|
||||
monkeypatch.setattr(module, "OTLPLogExporter", lambda **_: logs)
|
||||
monkeypatch.setattr(module, "PeriodicExportingMetricReader", lambda *_, **__: metrics)
|
||||
runtime = module.Telemetry("messenger-api", "unit", enabled=True)
|
||||
yield runtime, spans, logs, metrics
|
||||
runtime.shutdown()
|
||||
|
||||
|
||||
async def test_http_template_correlation_and_private_data(telemetry):
|
||||
runtime, spans, logs, metrics = telemetry
|
||||
app = FastAPI()
|
||||
|
||||
@app.post("/items/{item_id}")
|
||||
async def item(item_id: str):
|
||||
return {"ok": True}
|
||||
|
||||
app.add_middleware(module.HttpTelemetryMiddleware, telemetry=runtime)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://test") as client:
|
||||
# ASGI caller может иметь свой активный span, не относящийся к входящему HTTP.
|
||||
with runtime.tracer.start_as_current_span("unrelated-task"):
|
||||
response = await client.post("/items/private-id?token=private-query", json={"text":"private-body"}, headers={"Authorization":"Bearer private-token"})
|
||||
await client.get("/arbitrary-private-path")
|
||||
runtime.traces.force_flush()
|
||||
runtime.logs.force_flush()
|
||||
finished = spans.get_finished_spans()
|
||||
assert finished[0].kind == SpanKind.SERVER
|
||||
assert finished[0].name == "POST /items/{item_id}"
|
||||
assert finished[1].attributes["http.route"] == "unmatched"
|
||||
assert finished[0].context.trace_id != finished[1].context.trace_id
|
||||
assert all(span.parent is None for span in finished[:2])
|
||||
body = json.loads(logs.get_finished_logs()[0].log_record.body)
|
||||
assert body["trace_id"] == response.headers["X-Trace-Id"]
|
||||
assert int(body["span_id"], 16) == finished[0].context.span_id
|
||||
data = metrics.get_metrics_data()
|
||||
all_attributes = [dict(point.attributes) for resource in data.resource_metrics for scope in resource.scope_metrics for metric in scope.metrics for point in metric.data.data_points]
|
||||
assert {"http.request.method":"POST", "http.route":"/items/{item_id}", "http.response.status_class":"2xx"} in all_attributes
|
||||
exported = repr([(s.name, dict(s.attributes)) for s in finished]) + repr(all_attributes) + repr([str(log.log_record.body) for log in logs.get_finished_logs()])
|
||||
for private in ("private-id", "private-query", "private-body", "private-token", "arbitrary-private-path"):
|
||||
assert private not in exported
|
||||
|
||||
|
||||
async def test_unhandled_error_preserves_correlation_without_exception_payload(telemetry):
|
||||
from app.core.request_id import RequestIdMiddleware
|
||||
from app.errors.handlers import register_exception_handlers
|
||||
runtime, spans, logs, _ = telemetry
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/explode")
|
||||
async def explode():
|
||||
raise RuntimeError("private-error-with-credentials")
|
||||
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
app.add_middleware(module.HttpTelemetryMiddleware, telemetry=runtime)
|
||||
register_exception_handlers(app)
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app, raise_app_exceptions=False), base_url="http://test") as client:
|
||||
response = await client.get("/explode")
|
||||
runtime.traces.force_flush()
|
||||
runtime.logs.force_flush()
|
||||
record = json.loads(logs.get_finished_logs()[0].log_record.body)
|
||||
assert response.status_code == record["status_code"] == 500
|
||||
assert response.headers["X-Request-Id"] == record["request_id"]
|
||||
assert spans.get_finished_spans()[0].status.status_code.name == "ERROR"
|
||||
assert "private-error-with-credentials" not in repr(record)
|
||||
|
||||
|
||||
def test_job_success_and_error_record_real_outcomes(telemetry):
|
||||
runtime, spans, logs, metrics = telemetry
|
||||
for fail in (False, True):
|
||||
try:
|
||||
with runtime.tracer.start_as_current_span("attachment.process.v1", kind=SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False):
|
||||
with runtime.job_attempt():
|
||||
if fail:
|
||||
raise RuntimeError("private-exception-payload")
|
||||
except RuntimeError:
|
||||
pass
|
||||
runtime.traces.force_flush()
|
||||
runtime.logs.force_flush()
|
||||
finished = spans.get_finished_spans()
|
||||
assert [s.attributes["outcome"] for s in finished] == ["success", "error"]
|
||||
assert finished[1].status.status_code.name == "ERROR"
|
||||
assert not finished[1].events
|
||||
records = [json.loads(item.log_record.body) for item in logs.get_finished_logs()]
|
||||
assert [record["outcome"] for record in records] == ["success", "error"]
|
||||
assert "private-exception-payload" not in repr(records)
|
||||
data = metrics.get_metrics_data()
|
||||
counters = [point for resource in data.resource_metrics for scope in resource.scope_metrics for metric in scope.metrics if metric.name == "messenger.job.attempts" for point in metric.data.data_points]
|
||||
assert {p.attributes["outcome"]:p.value for p in counters} == {"success":1,"error":1}
|
||||
|
||||
|
||||
def test_disabled_telemetry_has_no_exporters():
|
||||
runtime = module.Telemetry("messenger-api", enabled=False)
|
||||
runtime.event("ignored")
|
||||
runtime.shutdown()
|
||||
assert not hasattr(runtime, "metrics")
|
||||
|
||||
|
||||
def test_tempo_base64_and_hex_identifiers_match():
|
||||
from scripts.check_observability import identifier_hex
|
||||
assert identifier_hex("3X/51BqmrXA=", 8) == "dd7ff9d41aa6ad70"
|
||||
assert identifier_hex("dd7ff9d41aa6ad70", 8) == "dd7ff9d41aa6ad70"
|
||||
Reference in New Issue
Block a user