Собрать наблюдаемый стенд, чтобы причину деградации API и worker было видно по метрикам, логам и трассам.
This commit is contained in:
+18
-11
@@ -1,30 +1,37 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Local synthetic credentials for compose. Copy to .env; never commit .env.
|
||||
# The application reads DATABASE_URL, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY and BROKER_URL.
|
||||
# *_FILE variables from the course worksheet are accepted too when the direct variable is empty.
|
||||
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace-with-local-secret
|
||||
POSTGRES_DB=messenger
|
||||
S3_ACCESS_KEY=replace-with-local-secret
|
||||
S3_SECRET_KEY=replace-with-local-secret
|
||||
RABBITMQ_USER=messenger
|
||||
RABBITMQ_PASSWORD=replace-with-local-secret
|
||||
GRAFANA_ADMIN_PASSWORD=replace-with-local-secret
|
||||
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
DATABASE_URL=postgresql://messenger:replace-with-local-secret@postgres:5432/messenger
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=replace-with-local-secret
|
||||
S3_SECRET_ACCESS_KEY=replace-with-local-secret
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
BROKER_URL=amqp://messenger:replace-with-local-secret@rabbitmq:5672/%2F
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
## Лабораторная 6
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Казанцева Виктория / БПИ2301
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: 2, SLO и обнаружение инцидентов
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: мессенджер лабораторий 1–5 (сессии, keyset, идемпотентность, вложения, две API-реплики, PostgreSQL, SeaweedFS, RabbitMQ, worker) и наблюдаемость: OTLP в Collector, Prometheus, Loki, Tempo, Grafana provisioning, burn-rate алерты.
|
||||
Команды запуска из чистого clone: см. REPORT.md. Кратко: `python scripts/render-s3-config.py`, затем `docker compose --env-file .env up -d --build`.
|
||||
Результаты проверок и ссылка на REPORT.md: smoke и check_contract PASS; persistence и failover PASS; доказательства в REPORT.md и `evidence/`.
|
||||
Ограничения / известные ошибки: на Windows `.sh` хуки не исполняются напрямую; прогон шёл через Git Bash. Короткий и длинный эксперименты SLO шли подряд, поэтому детекция длинной деградации быстрее холодного старта.
|
||||
Источники: документация OpenTelemetry, Prometheus, Grafana.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] Проверки `scripts/check_contract.py` и `tests/smoke.py` (то же, что make check и make test) прошли; на Windows они запускались через `PYTHONUTF8=1 python`.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
@@ -28,5 +28,8 @@ coverage/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
faults/*
|
||||
!faults/README.md
|
||||
infra/seaweed/s3.json
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
FROM python:3.12.8-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
AWS_EC2_METADATA_DISABLED=true
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY app /app/app
|
||||
|
||||
USER app
|
||||
EXPOSE 8080 8081
|
||||
|
||||
CMD ["python", "-m", "app.main"]
|
||||
@@ -1,37 +1,121 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Казанцева Виктория / БПИ2301
|
||||
- Целевая оценка: 5, трек 2 (SLO и обнаружение инцидентов)
|
||||
- Commit/PR: не публиковался
|
||||
- Источники: OpenTelemetry Collector, Prometheus histograms, W3C Trace Context, Grafana provisioning.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Compose v2 и Python 3.9+ на хосте (проверки). Образы закреплены в `compose.yaml`, приложение собирается из этого репозитория.
|
||||
|
||||
## Архитектура и решения
|
||||
```
|
||||
python scripts/render-s3-config.py
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env build api-1
|
||||
docker compose --env-file .env up -d
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
`scripts/render-s3-config.py` читает `S3_ACCESS_KEY` и `S3_SECRET_KEY` из локального `.env` и пишет gitignored `infra/seaweed/s3.json`. Шаблон переменных — `.env.example`. Файл `.env` не коммитится.
|
||||
|
||||
Схема БД применяется дважды и одинаково: `app/schema.sql` монтируется в `docker-entrypoint-initdb.d` и повторно накатывается приложением под `pg_advisory_lock`. Бакет S3 создаёт API/worker при старте, если его ещё нет. Очередь `attachment.process.v1` объявляет приложение.
|
||||
|
||||
Готовность: `GET http://127.0.0.1:8080/health/ready` возвращает 200. На Windows публичные скрипты запускать с `PYTHONUTF8=1`, потому что `Path.read_text()` в проверках курса читает JSON в кодировке локали.
|
||||
|
||||
```
|
||||
python scripts/check_contract.py
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных томов только явно: `docker compose --env-file .env down -v`.
|
||||
|
||||
`scripts/restart.sh` перезапускает postgres, SeaweedFS, RabbitMQ, обе API-реплики и worker, ждёт их healthcheck и затем перечитывает upstream в nginx. Тома не удаляются. `scripts/stop-one.sh` останавливает только `api-1` или `api-2` из `TARGET_INSTANCE_ID`. На Windows эти `.sh` напрямую не исполняются (`WinError 193`); проверка ниже запускала их через Git Bash, не меняя `tests/resilience.py`.
|
||||
|
||||
## Архитектура
|
||||
|
||||
Бизнес-API лабораторий 1–5 не менялся. Две реплики `api-1` и `api-2` за nginx (`127.0.0.1:8080`, round-robin). PostgreSQL — источник истины для пользователей, сессий, чатов, сообщений, идемпотентности и outbox. SeaweedFS хранит байты вложений (`staging/`, `originals/`, `variants/`). RabbitMQ — durable очередь `attachment.process.v1`, persistent messages, confirm и ручной ack. Worker отдельно от API.
|
||||
|
||||
Запись вложения: проверка доступа, байты в staging, строка attachment и outbox в одной транзакции, затем publish. Сбой брокера после commit не отменяет `202 queued`: poller допубликует outbox. Worker идемпотентен, lease 60 с, повтор того же `traceparent`.
|
||||
|
||||
Наблюдаемость: API и worker шлют OTLP/HTTP protobuf на Collector (`:4318`). Collector: memory limiter 192 MiB, удаление URL/SQL/Authorization, batch, очередь exporter 200 и retry не дольше 30 с. Дальше Prometheus (scrape `:8889`), Loki (OTLP), Tempo (gRPC). Grafana только запрашивает эти три хранилища и не хранит сигналы сама. Дашборд `messenger-overview` и datasources поднимаются provisioning-ом из Git. Правила — `infra/prometheus/rules.yml`.
|
||||
|
||||
| Роль | Сервис | Образ |
|
||||
| --- | --- | --- |
|
||||
| API, worker | api-1, api-2, worker | локальная сборка `mtusi-lab6-messenger:1.6.0` на `python:3.12.8-slim-bookworm` |
|
||||
| Балансировщик | nginx | `nginx:1.27.4-alpine` |
|
||||
| БД | postgres | `postgres:16.6-alpine` |
|
||||
| S3 | s3 | `chrislusf/seaweedfs:4.51` |
|
||||
| Брокер | rabbitmq | `rabbitmq:3.13.7-management-alpine` |
|
||||
| Collector | otel-collector | `otel/opentelemetry-collector-contrib:0.120.0` |
|
||||
| Метрики | prometheus | `prom/prometheus:v2.55.1` |
|
||||
| Логи | loki | `grafana/loki:3.3.2` |
|
||||
| Трассы | tempo | `grafana/tempo:2.7.2` |
|
||||
| UI | grafana | `grafana/grafana:11.5.2` |
|
||||
|
||||
Prometheus хранит ряды и состояние алертов. Loki хранит структурные JSON-логи. Tempo хранит спаны. Grafana их показывает.
|
||||
|
||||
Метрики не используют `user_id`, `chat_id`, `message_id`, `attachment_id` как labels. Маршрут в метрике — шаблон. В логах есть `trace_id` и `span_id`. Пароли, токены, текст сообщений и байты картинок в telemetry не пишутся.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда | Наблюдение | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт skeleton | `PYTHONUTF8=1 python scripts/check_contract.py` | PASS | вывод в этой сессии |
|
||||
| Публичный smoke | `PYTHONUTF8=1 python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS после включённой telemetry | — |
|
||||
| Готовность не зависит от Grafana | `GET /health/ready` при остановленном Tempo | 200, бизнес-GET тоже 200 | `evidence/incidents.json` |
|
||||
| Метрики API и worker | PromQL после smoke | есть `http_server_requests_total`, `http_server_duration_seconds_bucket`, `messenger_jobs_total`, `messenger_job_duration_seconds_bucket`, глубина/возраст очереди, пул БД, busy/capacity/consumers | `scripts/probe_signals.py` |
|
||||
| Лог и трасса одного upload | Loki `job.ready` и Tempo по тому же `trace_id` | лог `attempt: 2` и трасса ниже | `evidence/retry-trace.json` |
|
||||
| Дашборд из Git | `GET /api/dashboards/uid/messenger-overview` | uid `messenger-overview`, Grafana 11.5.2 | provisioning в `infra/grafana` |
|
||||
| p50/p95/p99 из histogram | панели дашборда, `histogram_quantile` по сумме bucket | не среднее и не среднее p99 реплик | `infra/grafana/dashboards/messenger.json` |
|
||||
| W3C API → broker → worker, retry, S3/DB | один upload с `process_fail_remaining=1` | см. цепочку | `evidence/retry-trace.json` |
|
||||
| SLI/SLO и burn rate | правила Prometheus | семь alert rules загружены | `infra/prometheus/rules.yml` |
|
||||
| Короткий всплеск не открывает fast burn | `scripts/slo_experiment.py` | pending, не firing | `evidence/slo-track2.json` |
|
||||
| Длинная деградация | тот же прогон | fast burn firing через 51.6 с, resolved через 70.6 с | `evidence/slo-track2.json` |
|
||||
| Остановка worker | `scripts/incident_followup.py` | consumers firing 75.5 с, queue lag 105.7 с, оба resolved за 20.1 с | `evidence/incidents.json` |
|
||||
| Остановка Tempo | Tempo stop 40 с | 76× HTTP 200, failed spans 9→24, очередь 22, RAM collector 68.8 MiB / 320 MiB | `evidence/incidents.json` |
|
||||
| Сохранность | публичный persistence через `scripts/restart.sh` | PASS, сессия и вложение на месте | — |
|
||||
| Отказ одной реплики | публичный failover через `scripts/stop-one.sh` | 2 instance id до отказа, 77 успешных записей, 0 ошибок, восстановление 2.094 с | — |
|
||||
| Нагрузка | `python tests/load.py --duration 15 --concurrency 2` | 1247 успехов, 0 ошибок, 82.7 rps, p50 31 мс, p95/p99 47 мс | `evidence/load.json` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Цепочка одного upload, trace `cf5025ad9d9a2a248da8335a0c6caee2`:
|
||||
|
||||
## Эксперименты и отказы
|
||||
1. `POST /api/v1/chats/{chat_id}/attachments` — корень.
|
||||
2. `attachment.process publish`, parent = POST.
|
||||
3. Два `attachment.process consume` с одним и тем же parent = publish. Первый — внедрённый transient (`job.retry`, attempt 1). Второй — sibling, attempt 2, `job.ready`.
|
||||
4. У второго consume дети `s3.get`, два `s3.put` и `db.transaction`. У POST свои `db.transaction` и `s3.put` staging.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Перцентили на дашборде считаются как `histogram_quantile` от `sum by (le)` rate bucket. p99 реплик не усредняется: сначала складываются bucket, потом берётся квантиль.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
## SLI / SLO
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
SLI API: доля ответов бизнес-маршрутов, которые не 5xx. Знаменатель — все бизнес-запросы. `/health/*` и несовпавший маршрут исключены. 4xx не ошибка SLI. SLO: 99% за скользящие окна burn rate (бюджет 1%).
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Быстрое сжигание: и 1 мин, и 5 мин строго выше 14.4% дольше 30 с, при трафике больше 0.2 rps.
|
||||
- Медленное: и 5 мин, и 15 мин выше 6% дольше 1 мин.
|
||||
- Дополнительно: error rate 5 мин выше 5% дольше 1 мин; возраст старейшей job больше 30 с дольше 1 мин; число consumers по данным API равно 0 дольше 1 мин; p95 времени до ready выше 10 с.
|
||||
|
||||
Доставка наружу не настраивалась. Состояние — локальные ряды `ALERTS`.
|
||||
|
||||
## Трек 2
|
||||
|
||||
Гипотеза: при уже идущем успешном трафике всплеск около 10 с поднимает минутное окно и не поднимает пятиминутное выше 14.4%, поэтому fast burn не становится firing. Непрерывные 5xx дольше примерно 45 с поднимают оба коротких окна, и fast burn зажигается. Остановленный worker виден по возрасту очереди и нулю consumers, а не по error rate API. Остановленный Tempo не мешает принять сообщения, exporter упирается в очередь и лимит памяти.
|
||||
|
||||
Методика на этом хосте (16 ГБ RAM, лимиты контейнеров в `compose.yaml`): закрытый цикл `GET /api/v1/chats` около 2 rps. Прогрев 90 с дал 180 ответов 200 и ноль alerts. Затем 10 с с `http_error_ratio=1`: 20 ответов 500. Сразу после этого `ApiHighErrorRate` и `ApiErrorBudgetSlowBurn` были только pending, `ApiErrorBudgetFastBurn` не появился. Дальше те же 100% ошибок: fast burn, high error rate и slow burn стали firing через 51.6 с (104 ответа 500). После снятия fault fast burn ушёл через 70.6 с; high и slow оставались firing и к началу следующего прогона (около двух минут спустя) уже были inactive.
|
||||
|
||||
Ограничения измерения. Короткий всплеск и длинный прогон шли подряд, поэтому ошибки всплеска ещё лежали в пятиминутном окне и ускорили детект длинной деградации относительно «холодного» старта. p99 нагрузки 15 с на 2 потоках совпал с p95 (47 мс): выборка короткая, это закрытый цикл, не открытый поток. Время resolved для slow burn снято по факту «уже нет» к 21:49:48 UTC, без секундомера. `AttachmentReadyLatencyHigh` стал firing из-за job, которая ждала остановленный worker: это следствие того же инцидента, гистограмма 15 мин ещё держит медленное завершение.
|
||||
|
||||
## Два инцидента
|
||||
|
||||
Остановка worker. Сразу после stop API показывал `messenger_worker_consumers = 0` и растущий возраст очереди. `WorkerConsumersZero` firing на 75.5 с, `QueueLagHigh` на 105.7 с при возрасте около 98 с. После start оба alert исчезли за 20.1 с. Runbook: `docs/runbooks/queue-lag.md`.
|
||||
|
||||
Остановка Tempo на 40 с. Бизнес-GET: 76 ответов 200, ошибок нет. `otelcol_exporter_send_failed_spans` вырос с 9 до 24. Размер очереди exporter дошёл до 22 при лимите 200. Память collector 68.8 MiB при лимите контейнера 320 MiB и memory limiter 192 MiB. Сообщения не терялись. Runbook: `docs/runbooks/telemetry-backend.md`.
|
||||
|
||||
Как отличить медленную БД от медленного handler: `db_slow_ms` спит внутри спана `db.transaction`, `handler_slow_ms` спит в серверном спане и вне `db.transaction`.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Публичные `tests/resilience.py` на этом Windows запускают `.sh` как Win32 и получают `WinError 193`. Сами хуки прогнаны тем же тестом через Git Bash. На Linux `make test-persistence` и `make test-failover` вызывают скрипты напрямую.
|
||||
|
||||
Чтобы вообще скачать образы, с диска C: были удалены неиспользуемые образы других локальных проектов. Их тома не удалялись. Образы этих проектов при следующем запуске нужно скачать снова.
|
||||
|
||||
SeaweedFS выбран вместо MinIO: образы `minio/minio` с Docker Hub в этой среде не скачиваются. Контракт разрешает SeaweedFS.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger application for lab 6."""
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import faults, http, telemetry
|
||||
from app.errors import AppError
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
class _TooLarge(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def _read_body(receive, limit):
|
||||
chunks = []
|
||||
size = 0
|
||||
while True:
|
||||
message = await receive()
|
||||
if message["type"] == "http.disconnect":
|
||||
return None
|
||||
if message["type"] != "http.request":
|
||||
continue
|
||||
data = message.get("body", b"") or b""
|
||||
size += len(data)
|
||||
if size > limit:
|
||||
raise _TooLarge()
|
||||
chunks.append(data)
|
||||
if not message.get("more_body"):
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _too_large(scope, limit):
|
||||
for name, value in scope.get("headers", []):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
return int(value) > limit
|
||||
except ValueError:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def app(scope, receive, send):
|
||||
if scope["type"] == "lifespan":
|
||||
while True:
|
||||
message = await receive()
|
||||
if message["type"] == "lifespan.startup":
|
||||
telemetry.log_event("service.listening", port=8080)
|
||||
await send({"type": "lifespan.startup.complete"})
|
||||
elif message["type"] == "lifespan.shutdown":
|
||||
await send({"type": "lifespan.shutdown.complete"})
|
||||
return
|
||||
return
|
||||
if scope["type"] != "http":
|
||||
return
|
||||
method = scope.get("method", "GET")
|
||||
path = scope.get("path", "/")
|
||||
raw_headers = {}
|
||||
for name, value in scope.get("headers", []):
|
||||
raw_headers[name.decode("latin-1").lower()] = value.decode("latin-1")
|
||||
request_id = http.request_id_from(raw_headers)
|
||||
template, _name, _groups = http.match(method, path)
|
||||
limit = http.body_limit(path)
|
||||
started = time.perf_counter()
|
||||
status = 500
|
||||
response_headers = {}
|
||||
body = b""
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
f"{method} {template}",
|
||||
kind=SpanKind.SERVER,
|
||||
attributes={"http.request.method": method, "http.route": template},
|
||||
) as span:
|
||||
try:
|
||||
if _too_large(scope, limit):
|
||||
raise AppError(413, "payload_too_large", "Слишком большой объект")
|
||||
try:
|
||||
payload = await _read_body(receive, limit)
|
||||
except _TooLarge as exc:
|
||||
raise AppError(413, "payload_too_large", "Слишком большой объект") from exc
|
||||
if payload is None:
|
||||
return
|
||||
if method in {"POST", "PUT"} and path.rstrip("/").endswith("/attachments") and len(payload) > settings.max_upload:
|
||||
raise AppError(413, "payload_too_large", "Слишком большой объект")
|
||||
query = http.query_from(scope.get("query_string", b"").decode("ascii", "replace"))
|
||||
if path.startswith("/api/") and random.random() < faults.http_error_ratio():
|
||||
raise AppError(500, "internal_error", "Внутренняя ошибка")
|
||||
|
||||
def run():
|
||||
return http.handle(method, path, query, raw_headers, payload)
|
||||
|
||||
template, status, response_headers, body = await asyncio.to_thread(run)
|
||||
span.set_attribute("http.route", template)
|
||||
except Exception as exc:
|
||||
status, response_headers, body = http.error_response(exc, request_id)
|
||||
if not isinstance(exc, AppError):
|
||||
telemetry.log_event("internal_error", severity="ERROR", error_type=type(exc).__name__, request_id=request_id)
|
||||
if status >= 500:
|
||||
telemetry.mark_error(span)
|
||||
span.set_attribute("error.type", type(exc).__name__)
|
||||
duration = time.perf_counter() - started
|
||||
span.set_attribute("http.response.status_code", status)
|
||||
telemetry.record_http(method, template, status, duration)
|
||||
telemetry.log_event(
|
||||
"http.request.completed",
|
||||
request_id=request_id,
|
||||
http_route=template,
|
||||
http_method=method,
|
||||
status=status,
|
||||
duration_ms=int(duration * 1000),
|
||||
)
|
||||
header_list = [
|
||||
(b"x-request-id", request_id.encode("ascii")),
|
||||
(b"x-instance-id", settings.instance_id.encode("ascii")),
|
||||
]
|
||||
for key, value in response_headers.items():
|
||||
header_list.append((key.lower().encode("latin-1"), value.encode("latin-1")))
|
||||
await send({"type": "http.response.start", "status": status, "headers": header_list})
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
|
||||
|
||||
def new_request_id():
|
||||
return uuid.uuid4().hex
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from urllib.parse import quote, urlparse, unquote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
import pika
|
||||
|
||||
from app import telemetry
|
||||
from app.db import jsonb
|
||||
from app.settings import settings
|
||||
|
||||
QUEUE = "attachment.process.v1"
|
||||
|
||||
|
||||
def _params():
|
||||
parsed = urlparse(settings.broker_url)
|
||||
return pika.ConnectionParameters(
|
||||
host=parsed.hostname or "rabbitmq",
|
||||
port=parsed.port or 5672,
|
||||
virtual_host=unquote(parsed.path[1:] or "/"),
|
||||
credentials=pika.PlainCredentials(unquote(parsed.username or ""), unquote(parsed.password or "")),
|
||||
heartbeat=60,
|
||||
blocked_connection_timeout=2,
|
||||
socket_timeout=2,
|
||||
connection_attempts=1,
|
||||
retry_delay=0,
|
||||
)
|
||||
|
||||
|
||||
def _credentials():
|
||||
parsed = urlparse(settings.broker_url)
|
||||
return unquote(parsed.username or ""), unquote(parsed.password or ""), unquote(parsed.path[1:] or "/")
|
||||
|
||||
|
||||
def init():
|
||||
connection = pika.BlockingConnection(_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def ready():
|
||||
try:
|
||||
connection = pika.BlockingConnection(_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True, passive=True)
|
||||
finally:
|
||||
connection.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def insert_outbox(conn, payload, created_at):
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO outbox (job_id, attachment_id, payload, created_at)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
""",
|
||||
(uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at),
|
||||
)
|
||||
|
||||
|
||||
def publish_pending(conn, job_id=None):
|
||||
query = """
|
||||
SELECT job_id, payload
|
||||
FROM outbox
|
||||
WHERE published_at IS NULL
|
||||
"""
|
||||
args = []
|
||||
if job_id is not None:
|
||||
query += " AND job_id = %s"
|
||||
args.append(job_id)
|
||||
query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20"
|
||||
rows = conn.execute(query, args).fetchall()
|
||||
for row in rows:
|
||||
payload = row["payload"]
|
||||
if isinstance(payload, str):
|
||||
payload = json.loads(payload)
|
||||
_publish(payload)
|
||||
conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],))
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _publish(payload):
|
||||
carrier = {}
|
||||
if payload.get("traceparent"):
|
||||
carrier["traceparent"] = payload["traceparent"]
|
||||
if payload.get("tracestate"):
|
||||
carrier["tracestate"] = payload["tracestate"]
|
||||
parent = telemetry.extract_context(carrier)
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"attachment.process publish",
|
||||
context=parent,
|
||||
kind=telemetry.trace.SpanKind.PRODUCER,
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE,
|
||||
"messaging.operation.type": "publish",
|
||||
},
|
||||
):
|
||||
outgoing = telemetry.inject_carrier()
|
||||
body = dict(payload)
|
||||
if outgoing.get("traceparent"):
|
||||
body["traceparent"] = outgoing["traceparent"]
|
||||
if outgoing.get("tracestate"):
|
||||
body["tracestate"] = outgoing["tracestate"]
|
||||
_send(body, outgoing)
|
||||
telemetry.log_event(
|
||||
"job.published",
|
||||
job_id=str(payload.get("job_id")),
|
||||
attempt=int(payload.get("attempt") or 1),
|
||||
)
|
||||
|
||||
|
||||
def _send(payload, headers):
|
||||
connection = pika.BlockingConnection(_params())
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.confirm_delivery()
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=QUEUE,
|
||||
body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
headers=headers,
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def consumer_count():
|
||||
user, password, vhost = _credentials()
|
||||
url = (
|
||||
settings.rabbit_management_url.rstrip("/")
|
||||
+ "/api/queues/"
|
||||
+ quote(vhost, safe="")
|
||||
+ "/"
|
||||
+ quote(QUEUE, safe="")
|
||||
)
|
||||
token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii")
|
||||
request = Request(url, headers={"Authorization": "Basic " + token})
|
||||
with urlopen(request, timeout=2) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
return int(data.get("consumers") or 0)
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
import base64
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from app.errors import AppError, bad_request
|
||||
|
||||
USERNAME = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
def utcnow():
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def iso(value):
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=dt.timezone.utc)
|
||||
value = value.astimezone(dt.timezone.utc)
|
||||
return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z"
|
||||
|
||||
|
||||
def parse_iso(value):
|
||||
if not isinstance(value, str) or not value.endswith("Z"):
|
||||
raise bad_request()
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value[:-1] + "+00:00")
|
||||
except ValueError as exc:
|
||||
raise bad_request() from exc
|
||||
|
||||
|
||||
def parse_uuid(value):
|
||||
if not isinstance(value, str):
|
||||
raise bad_request()
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError as exc:
|
||||
raise bad_request() from exc
|
||||
|
||||
|
||||
def require_text(value, minimum, maximum, allow_blank=False):
|
||||
if not isinstance(value, str):
|
||||
raise bad_request()
|
||||
if "\x00" in value or any(ord(ch) < 32 or ord(ch) == 127 for ch in value):
|
||||
raise bad_request()
|
||||
if not allow_blank and not value.strip():
|
||||
raise bad_request()
|
||||
if not (minimum <= len(value) <= maximum):
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def canonical_message(text, attachment_ids):
|
||||
body = {"attachment_ids": list(attachment_ids), "text": text}
|
||||
raw = json.dumps(body, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
return hashlib.sha256(raw.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def encode_cursor(kind, resource_id, created_at, row_id):
|
||||
raw = f"{kind}|{resource_id}|{iso(created_at)}|{row_id}"
|
||||
return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(token, kind, resource_id):
|
||||
if not isinstance(token, str) or not token:
|
||||
raise bad_request()
|
||||
padded = token + ("=" * (-len(token) % 4))
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8")
|
||||
cursor_kind, cursor_resource, stamp, row_id = raw.split("|", 3)
|
||||
created_at = parse_iso(stamp)
|
||||
row_uuid = str(uuid.UUID(row_id))
|
||||
except (ValueError, UnicodeError, AppError) as exc:
|
||||
raise bad_request() from exc
|
||||
if cursor_kind != kind or cursor_resource != resource_id:
|
||||
raise bad_request()
|
||||
return created_at, row_uuid
|
||||
|
||||
|
||||
def fitted_size(width, height, box=256):
|
||||
if width < 1 or height < 1:
|
||||
raise ValueError("dimensions")
|
||||
if width <= box and height <= box:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return box, max(1, (height * box) // width)
|
||||
return max(1, (width * box) // height), box
|
||||
|
||||
|
||||
def detect_image(data, declared):
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
detected = "image/png"
|
||||
elif data.startswith(b"\xff\xd8\xff"):
|
||||
detected = "image/jpeg"
|
||||
else:
|
||||
return None
|
||||
if declared != detected:
|
||||
return None
|
||||
return detected
|
||||
|
||||
|
||||
def media_type(header):
|
||||
if not header or not isinstance(header, str):
|
||||
return ""
|
||||
return header.split(";", 1)[0].strip().lower()
|
||||
|
||||
|
||||
def loads_object(raw):
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeError as exc:
|
||||
raise bad_request() from exc
|
||||
|
||||
def pairs(items):
|
||||
obj = {}
|
||||
for key, value in items:
|
||||
if key in obj:
|
||||
raise ValueError("duplicate")
|
||||
obj[key] = value
|
||||
return obj
|
||||
|
||||
try:
|
||||
value = json.loads(text, object_pairs_hook=pairs)
|
||||
except (ValueError, UnicodeError) as exc:
|
||||
raise bad_request() from exc
|
||||
if not isinstance(value, dict):
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def expect_fields(data, allowed, required):
|
||||
if set(data) - set(allowed):
|
||||
raise bad_request()
|
||||
for key in required:
|
||||
if key not in data:
|
||||
raise bad_request()
|
||||
@@ -0,0 +1,98 @@
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
from app import telemetry
|
||||
from app.errors import AppError, unavailable
|
||||
from app.faults import db_fail, db_slow_seconds
|
||||
from app.settings import settings
|
||||
|
||||
pool = None
|
||||
|
||||
|
||||
def init():
|
||||
global pool
|
||||
pool = ConnectionPool(
|
||||
settings.database_url,
|
||||
min_size=1,
|
||||
max_size=settings.pool_max,
|
||||
timeout=2,
|
||||
max_waiting=16,
|
||||
check=ConnectionPool.check_connection,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"autocommit": False,
|
||||
"connect_timeout": 3,
|
||||
"options": "-c timezone=UTC -c statement_timeout=4000",
|
||||
},
|
||||
open=True,
|
||||
reconnect_timeout=2,
|
||||
)
|
||||
|
||||
|
||||
def migrate():
|
||||
script = Path(__file__).with_name("schema.sql").read_text(encoding="utf-8")
|
||||
statements = []
|
||||
buffer = []
|
||||
for line in script.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("--"):
|
||||
continue
|
||||
buffer.append(line)
|
||||
if stripped.endswith(";"):
|
||||
statements.append("\n".join(buffer))
|
||||
buffer = []
|
||||
with pool.connection() as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(861006)")
|
||||
for statement in statements:
|
||||
conn.execute(statement)
|
||||
conn.execute("SELECT pg_advisory_unlock(861006)")
|
||||
conn.commit()
|
||||
|
||||
|
||||
def jsonb(value):
|
||||
return Jsonb(value)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection():
|
||||
if db_fail():
|
||||
raise unavailable()
|
||||
delay = db_slow_seconds()
|
||||
started = time.perf_counter()
|
||||
span = telemetry.tracer().start_as_current_span(
|
||||
"db.transaction",
|
||||
attributes={"db.system": "postgresql"},
|
||||
)
|
||||
span.__enter__()
|
||||
try:
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
with pool.connection() as conn:
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
except AppError:
|
||||
raise
|
||||
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError) as exc:
|
||||
raise unavailable() from exc
|
||||
finally:
|
||||
span.__exit__(None, None, None)
|
||||
telemetry.record_db(time.perf_counter() - started)
|
||||
|
||||
|
||||
def stats():
|
||||
if pool is None:
|
||||
return 0, 0
|
||||
data = pool.get_stats()
|
||||
used = int(data.get("pool_size", 0)) - int(data.get("pool_available", 0))
|
||||
waiting = int(data.get("requests_waiting", 0))
|
||||
return max(used, 0), waiting
|
||||
+584
@@ -0,0 +1,584 @@
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
from argon2.low_level import Type
|
||||
from psycopg.errors import UniqueViolation
|
||||
|
||||
from app import broker, storage, telemetry
|
||||
from app.codec import (
|
||||
USERNAME,
|
||||
IDEMPOTENCY,
|
||||
canonical_message,
|
||||
decode_cursor,
|
||||
detect_image,
|
||||
encode_cursor,
|
||||
expect_fields,
|
||||
iso,
|
||||
parse_uuid,
|
||||
require_text,
|
||||
utcnow,
|
||||
)
|
||||
from app.db import connection, jsonb
|
||||
from app.errors import bad_request, conflict, not_found, too_large, unauthorized, unsupported
|
||||
from app.images import validate_image
|
||||
from app.settings import settings
|
||||
|
||||
BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER = 'Bearer realm="messenger"'
|
||||
_hasher = PasswordHasher(time_cost=1, memory_cost=8192, parallelism=1, hash_len=32, salt_len=16, type=Type.ID)
|
||||
_dummy = None
|
||||
|
||||
|
||||
def init_hasher():
|
||||
global _dummy
|
||||
_dummy = _hasher.hash("not-a-real-password")
|
||||
|
||||
|
||||
def _user_json(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _token_hash(token):
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def _password(value):
|
||||
if not isinstance(value, str) or "\x00" in value or not (12 <= len(value) <= 128):
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def as_uuid(value):
|
||||
if isinstance(value, uuid.UUID):
|
||||
return value
|
||||
return uuid.UUID(str(value))
|
||||
|
||||
|
||||
def _username(value):
|
||||
if not isinstance(value, str) or USERNAME.fullmatch(value) is None:
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def register(data):
|
||||
expect_fields(data, ["username", "display_name", "password"], ["username", "display_name", "password"])
|
||||
username = _username(data["username"])
|
||||
display_name = require_text(data["display_name"], 1, 100)
|
||||
password = _password(data["password"])
|
||||
password_hash = _hasher.hash(password)
|
||||
user_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
try:
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
""",
|
||||
(user_id, username, display_name, password_hash, created_at),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
except UniqueViolation as exc:
|
||||
raise conflict("Имя пользователя занято") from exc
|
||||
return 201, _user_json(row)
|
||||
|
||||
|
||||
def login(username, password):
|
||||
if not isinstance(username, str) or not isinstance(password, str) or "\x00" in username or "\x00" in password:
|
||||
raise unauthorized(BASIC)
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
stored = row["password_hash"] if row else _dummy
|
||||
try:
|
||||
_hasher.verify(stored, password)
|
||||
valid = row is not None
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
valid = False
|
||||
if not valid:
|
||||
raise unauthorized(BASIC)
|
||||
token = secrets_token()
|
||||
expires_at = utcnow() + dt.timedelta(seconds=settings.session_ttl)
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(_token_hash(token), row["id"], expires_at),
|
||||
)
|
||||
return 201, {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": iso(expires_at),
|
||||
"user": _user_json(row),
|
||||
}
|
||||
|
||||
|
||||
def secrets_token():
|
||||
import secrets
|
||||
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def current_user(token):
|
||||
if not token or len(token) < 32:
|
||||
raise unauthorized(BEARER)
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT u.id, u.username, u.display_name, u.created_at
|
||||
FROM sessions s
|
||||
JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > NOW()
|
||||
""",
|
||||
(_token_hash(token),),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise unauthorized(BEARER)
|
||||
return row
|
||||
|
||||
|
||||
def ready_check():
|
||||
with connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
if not storage.ready() or not broker.ready():
|
||||
raise unavailable()
|
||||
|
||||
|
||||
def revoke(token):
|
||||
user = current_user(token)
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE sessions SET revoked_at = NOW() WHERE token_hash = %s AND revoked_at IS NULL",
|
||||
(_token_hash(token),),
|
||||
)
|
||||
return user
|
||||
|
||||
|
||||
def create_chat(user, data):
|
||||
expect_fields(data, ["title", "member_ids"], ["title", "member_ids"])
|
||||
title = require_text(data["title"], 1, 100)
|
||||
raw_members = data["member_ids"]
|
||||
if not isinstance(raw_members, list) or not (1 <= len(raw_members) <= 100):
|
||||
raise bad_request()
|
||||
members = [parse_uuid(item) for item in raw_members]
|
||||
if len(set(members)) != len(members):
|
||||
raise bad_request()
|
||||
if str(user["id"]) not in members:
|
||||
raise bad_request()
|
||||
chat_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
with connection() as conn:
|
||||
found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", ([as_uuid(item) for item in members],)).fetchall()
|
||||
if len(found) != len(members):
|
||||
raise bad_request()
|
||||
conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
|
||||
(chat_id, title, created_at),
|
||||
)
|
||||
for position, member_id in enumerate(members):
|
||||
conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
|
||||
(chat_id, as_uuid(member_id), position),
|
||||
)
|
||||
return 201, {
|
||||
"id": str(chat_id),
|
||||
"title": title,
|
||||
"member_ids": members,
|
||||
"created_at": iso(created_at),
|
||||
}
|
||||
|
||||
|
||||
def _chat_json(conn, row):
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": [str(item["user_id"]) for item in members],
|
||||
"created_at": iso(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def list_chats(user, limit, cursor):
|
||||
created_at = None
|
||||
row_id = None
|
||||
if cursor is not None:
|
||||
created_at, row_id = decode_cursor(cursor, "chats", str(user["id"]))
|
||||
with connection() as conn:
|
||||
if created_at is None:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = %s
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(user["id"], limit + 1),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE m.user_id = %s AND (c.created_at, c.id) > (%s, %s)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(user["id"], created_at, as_uuid(row_id), limit + 1),
|
||||
).fetchall()
|
||||
extra = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
items = [_chat_json(conn, row) for row in rows]
|
||||
next_cursor = None
|
||||
if extra and rows:
|
||||
last = rows[-1]
|
||||
next_cursor = encode_cursor("chats", str(user["id"]), last["created_at"], str(last["id"]))
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
def get_chat(user, chat_id):
|
||||
with connection() as conn:
|
||||
row = _visible_chat(conn, user["id"], chat_id)
|
||||
return _chat_json(conn, row)
|
||||
|
||||
|
||||
def _visible_chat(conn, user_id, chat_id):
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE c.id = %s AND m.user_id = %s
|
||||
""",
|
||||
(as_uuid(chat_id), as_uuid(user_id)),
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
def _message_json(row, attachment_ids):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
"attachment_ids": attachment_ids,
|
||||
}
|
||||
|
||||
|
||||
def _attachment_ids(conn, message_id):
|
||||
rows = conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
|
||||
(message_id,),
|
||||
).fetchall()
|
||||
return [str(row["attachment_id"]) for row in rows]
|
||||
|
||||
|
||||
def create_message(user, chat_id, key, data, path):
|
||||
chat_uuid = as_uuid(chat_id)
|
||||
if IDEMPOTENCY.fullmatch(key or "") is None:
|
||||
raise bad_request()
|
||||
expect_fields(data, ["text", "attachment_ids"], ["text"])
|
||||
if not isinstance(data["text"], str) or "\x00" in data["text"] or len(data["text"]) > 4000:
|
||||
raise bad_request()
|
||||
text = data["text"]
|
||||
if "attachment_ids" in data:
|
||||
raw_ids = data["attachment_ids"]
|
||||
if not isinstance(raw_ids, list) or len(raw_ids) > 10:
|
||||
raise bad_request()
|
||||
attachment_ids = [parse_uuid(item) for item in raw_ids]
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise bad_request()
|
||||
else:
|
||||
attachment_ids = []
|
||||
if not text.strip() and not attachment_ids:
|
||||
raise bad_request()
|
||||
body_hash = canonical_message(text, attachment_ids)
|
||||
lock = int.from_bytes(hashlib.sha256(f"{user['id']}|POST|{path}|{key}".encode("utf-8")).digest()[:8], "big", signed=True)
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_id)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,))
|
||||
existing = conn.execute(
|
||||
"""
|
||||
SELECT body_hash, response_json, created_at
|
||||
FROM idempotency
|
||||
WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s
|
||||
""",
|
||||
(user["id"], path, key),
|
||||
).fetchone()
|
||||
if existing and existing["created_at"] >= utcnow() - dt.timedelta(seconds=settings.idempotency_ttl):
|
||||
if bytes(existing["body_hash"]) != body_hash:
|
||||
raise conflict("Ключ идемпотентности уже использован с другим телом")
|
||||
stored = existing["response_json"]
|
||||
if isinstance(stored, str):
|
||||
import json
|
||||
|
||||
stored = json.loads(stored)
|
||||
return 201, stored
|
||||
if existing:
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency WHERE user_id = %s AND method = 'POST' AND path = %s AND idem_key = %s",
|
||||
(user["id"], path, key),
|
||||
)
|
||||
if attachment_ids:
|
||||
rows = conn.execute(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s)",
|
||||
([as_uuid(item) for item in attachment_ids],),
|
||||
).fetchall()
|
||||
found = {str(row["id"]): row for row in rows}
|
||||
if len(found) != len(attachment_ids):
|
||||
raise not_found()
|
||||
for attachment_id in attachment_ids:
|
||||
row = found[attachment_id]
|
||||
if str(row["chat_id"]) != chat_id:
|
||||
raise not_found()
|
||||
if row["status"] == "failed":
|
||||
raise conflict("Нельзя прикрепить неуспешное вложение")
|
||||
message_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
|
||||
(message_id, chat_uuid, user["id"], text, created_at),
|
||||
)
|
||||
for position, attachment_id in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
(message_id, as_uuid(attachment_id), position),
|
||||
)
|
||||
stored = conn.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s",
|
||||
(message_id,),
|
||||
).fetchone()
|
||||
payload = _message_json(stored, attachment_ids)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO idempotency (user_id, method, path, idem_key, body_hash, status_code, response_json, created_at)
|
||||
VALUES (%s, 'POST', %s, %s, %s, 201, %s, %s)
|
||||
""",
|
||||
(user["id"], path, key, body_hash, jsonb(payload), created_at),
|
||||
)
|
||||
return 201, payload
|
||||
|
||||
|
||||
def list_messages(user, chat_id, limit, cursor):
|
||||
chat_uuid = as_uuid(chat_id)
|
||||
created_at = None
|
||||
row_id = None
|
||||
if cursor is not None:
|
||||
created_at, row_id = decode_cursor(cursor, "messages", chat_id)
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_uuid)
|
||||
if created_at is None:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
(chat_uuid, limit + 1),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s AND (created_at, id) > (%s, %s)
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
(chat_uuid, created_at, as_uuid(row_id), limit + 1),
|
||||
).fetchall()
|
||||
extra = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
items = [_message_json(row, _attachment_ids(conn, row["id"])) for row in rows]
|
||||
next_cursor = None
|
||||
if extra and rows:
|
||||
last = rows[-1]
|
||||
next_cursor = encode_cursor("messages", chat_id, last["created_at"], str(last["id"]))
|
||||
return {"items": items, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
def _variants(conn, attachment_id):
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT name, content_type, size_bytes, width, height, sha256
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = %s AND pipeline_version = 1
|
||||
ORDER BY name
|
||||
""",
|
||||
(attachment_id,),
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"name": row["name"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"width": row["width"],
|
||||
"height": row["height"],
|
||||
"sha256": row["sha256"],
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
|
||||
|
||||
def _attachment_json(row, variants):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": variants,
|
||||
"error_code": row["error_code"],
|
||||
"created_at": iso(row["created_at"]),
|
||||
"updated_at": iso(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
def upload(user, chat_id, filename, content_type, data):
|
||||
import hashlib as hl
|
||||
|
||||
if not data:
|
||||
raise unsupported()
|
||||
if len(data) > settings.max_upload:
|
||||
raise too_large()
|
||||
detected = detect_image(data, content_type)
|
||||
if not detected:
|
||||
raise unsupported()
|
||||
validate_image(data)
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_id)
|
||||
attachment_id = uuid.uuid4()
|
||||
created_at = utcnow()
|
||||
digest = hl.sha256(data).hexdigest()
|
||||
staging_key = f"staging/{attachment_id}"
|
||||
storage.put(staging_key, data, detected, "staging")
|
||||
carrier = telemetry.inject_carrier()
|
||||
job_id = uuid.uuid4()
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job_id),
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": iso(created_at),
|
||||
}
|
||||
if carrier.get("traceparent"):
|
||||
payload["traceparent"] = carrier["traceparent"]
|
||||
if carrier.get("tracestate"):
|
||||
payload["tracestate"] = carrier["tracestate"]
|
||||
row = {
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
"owner_id": user["id"],
|
||||
"filename": filename,
|
||||
"content_type": detected,
|
||||
"size_bytes": len(data),
|
||||
"sha256": digest,
|
||||
"status": "queued",
|
||||
"error_code": None,
|
||||
"created_at": created_at,
|
||||
"updated_at": created_at,
|
||||
}
|
||||
with connection() as conn:
|
||||
_visible_chat(conn, user["id"], chat_id)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status,
|
||||
error_code, staging_key, pipeline_version, attempt, created_at, updated_at
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, 1, 0, %s, %s)
|
||||
""",
|
||||
(
|
||||
attachment_id,
|
||||
as_uuid(chat_id),
|
||||
user["id"],
|
||||
filename,
|
||||
detected,
|
||||
len(data),
|
||||
digest,
|
||||
staging_key,
|
||||
created_at,
|
||||
created_at,
|
||||
),
|
||||
)
|
||||
broker.insert_outbox(conn, payload, created_at)
|
||||
try:
|
||||
with connection() as conn:
|
||||
broker.publish_pending(conn, job_id)
|
||||
except Exception:
|
||||
telemetry.log_event("job.publish_deferred", severity="WARNING", job_id=str(job_id))
|
||||
telemetry.log_event("attachment.accepted", attachment_id=str(attachment_id), size_bytes=len(data), content_type=detected)
|
||||
return 202, _attachment_json(row, [])
|
||||
|
||||
|
||||
def get_attachment(user, attachment_id):
|
||||
attachment_uuid = as_uuid(attachment_id)
|
||||
with connection() as conn:
|
||||
row = _visible_attachment(conn, user["id"], attachment_uuid)
|
||||
return _attachment_json(row, _variants(conn, attachment_uuid))
|
||||
|
||||
|
||||
def _visible_attachment(conn, user_id, attachment_id):
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id = %s", (as_uuid(attachment_id),)).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(row["chat_id"], user_id),
|
||||
).fetchone()
|
||||
if not member:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
def download(user, attachment_id, variant):
|
||||
attachment_uuid = as_uuid(attachment_id)
|
||||
if variant not in {"original", "thumbnail", "crop", "optimized"}:
|
||||
raise bad_request()
|
||||
with connection() as conn:
|
||||
row = _visible_attachment(conn, user["id"], attachment_uuid)
|
||||
if row["status"] != "ready":
|
||||
raise conflict("Вложение ещё не готово")
|
||||
if variant == "original":
|
||||
key = row["original_key"]
|
||||
content_type = row["content_type"]
|
||||
else:
|
||||
found = conn.execute(
|
||||
"""
|
||||
SELECT object_key, content_type
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = %s AND name = %s AND pipeline_version = 1
|
||||
""",
|
||||
(attachment_uuid, variant),
|
||||
).fetchone()
|
||||
if not found:
|
||||
raise not_found()
|
||||
key = found["object_key"]
|
||||
content_type = found["content_type"]
|
||||
return storage.get(key, variant), content_type
|
||||
@@ -0,0 +1,40 @@
|
||||
class AppError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def bad_request(message="Неверный запрос"):
|
||||
return AppError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(challenge):
|
||||
return AppError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Требуется аутентификация",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
def not_found():
|
||||
return AppError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
def conflict(message="Конфликт"):
|
||||
return AppError(409, "conflict", message)
|
||||
|
||||
|
||||
def too_large(message="Слишком большой объект"):
|
||||
return AppError(413, "payload_too_large", message)
|
||||
|
||||
|
||||
def unsupported(message="Неподдерживаемый тип изображения"):
|
||||
return AppError(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
def unavailable(message="Сервис временно недоступен"):
|
||||
return AppError(503, "unavailable", message)
|
||||
@@ -0,0 +1,73 @@
|
||||
from pathlib import Path
|
||||
|
||||
from app.settings import settings
|
||||
|
||||
|
||||
def _text(name):
|
||||
path = Path(settings.fault_dir) / name
|
||||
try:
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
except (FileNotFoundError, OSError):
|
||||
return ""
|
||||
|
||||
|
||||
def db_slow_seconds():
|
||||
raw = _text("db_slow_ms")
|
||||
if not raw:
|
||||
return 0.0
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return min(value, 10000.0) / 1000.0
|
||||
|
||||
|
||||
def db_fail():
|
||||
return _text("db_fail") == "1"
|
||||
|
||||
|
||||
def http_error_ratio():
|
||||
raw = _text("http_error_ratio")
|
||||
if not raw:
|
||||
return 0.0
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return min(value, 1.0)
|
||||
|
||||
|
||||
def handler_slow_seconds():
|
||||
raw = _text("handler_slow_ms")
|
||||
if not raw:
|
||||
return 0.0
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
if value <= 0:
|
||||
return 0.0
|
||||
return min(value, 10000.0) / 1000.0
|
||||
|
||||
|
||||
def consume_process_failure():
|
||||
path = Path(settings.fault_dir) / "process_fail_remaining"
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8").strip()
|
||||
except (FileNotFoundError, OSError):
|
||||
return False
|
||||
try:
|
||||
remaining = int(raw or "0")
|
||||
except ValueError:
|
||||
return False
|
||||
if remaining <= 0:
|
||||
return False
|
||||
try:
|
||||
path.write_text(str(remaining - 1), encoding="utf-8")
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
from app import domain, faults
|
||||
from app.codec import REQUEST_ID, loads_object, media_type, require_text
|
||||
from app.errors import AppError, bad_request, not_found, unauthorized
|
||||
from app.settings import settings
|
||||
|
||||
_ROUTES = [
|
||||
("GET", re.compile(r"^/health/live$"), "/health/live", "live"),
|
||||
("GET", re.compile(r"^/health/ready$"), "/health/ready", "ready"),
|
||||
("POST", re.compile(r"^/api/v1/users$"), "/api/v1/users", "register"),
|
||||
("GET", re.compile(r"^/api/v1/users/me$"), "/api/v1/users/me", "me"),
|
||||
("POST", re.compile(r"^/api/v1/auth/sessions$"), "/api/v1/auth/sessions", "login"),
|
||||
("DELETE", re.compile(r"^/api/v1/auth/sessions/current$"), "/api/v1/auth/sessions/current", "logout"),
|
||||
("POST", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "create_chat"),
|
||||
("GET", re.compile(r"^/api/v1/chats$"), "/api/v1/chats", "list_chats"),
|
||||
("GET", re.compile(r"^/api/v1/chats/([^/]+)$"), "/api/v1/chats/{chat_id}", "get_chat"),
|
||||
("POST", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "create_message"),
|
||||
("GET", re.compile(r"^/api/v1/chats/([^/]+)/messages$"), "/api/v1/chats/{chat_id}/messages", "list_messages"),
|
||||
("POST", re.compile(r"^/api/v1/chats/([^/]+)/attachments$"), "/api/v1/chats/{chat_id}/attachments", "upload"),
|
||||
("GET", re.compile(r"^/api/v1/attachments/([^/]+)$"), "/api/v1/attachments/{attachment_id}", "get_attachment"),
|
||||
("GET", re.compile(r"^/api/v1/attachments/([^/]+)/content$"), "/api/v1/attachments/{attachment_id}/content", "download"),
|
||||
]
|
||||
|
||||
|
||||
def match(method, path):
|
||||
for route_method, pattern, template, name in _ROUTES:
|
||||
found = pattern.fullmatch(path)
|
||||
if found and route_method == method:
|
||||
return template, name, found.groups()
|
||||
return "unmatched", "unmatched", ()
|
||||
|
||||
|
||||
def _single(query, name):
|
||||
values = query.get(name)
|
||||
if not values:
|
||||
return None
|
||||
if len(values) != 1 or values[0] == "":
|
||||
raise bad_request()
|
||||
return values[0]
|
||||
|
||||
|
||||
def _limit(query):
|
||||
raw = _single(query, "limit")
|
||||
if raw is None:
|
||||
return 50
|
||||
if re.fullmatch(r"[1-9][0-9]*", raw) is None:
|
||||
raise bad_request()
|
||||
value = int(raw)
|
||||
if value > 100:
|
||||
raise bad_request()
|
||||
return value
|
||||
|
||||
|
||||
def _json_body(headers, body):
|
||||
if media_type(headers.get("content-type", "")) != "application/json":
|
||||
raise bad_request()
|
||||
if not body:
|
||||
raise bad_request()
|
||||
return loads_object(body)
|
||||
|
||||
|
||||
def _bearer(headers):
|
||||
value = headers.get("authorization", "")
|
||||
if not value.lower().startswith("bearer "):
|
||||
raise unauthorized(domain.BEARER)
|
||||
token = value.split(" ", 1)[1].strip()
|
||||
return token
|
||||
|
||||
|
||||
def _basic(headers):
|
||||
import base64
|
||||
|
||||
value = headers.get("authorization", "")
|
||||
if not value.lower().startswith("basic "):
|
||||
raise unauthorized(domain.BASIC)
|
||||
try:
|
||||
decoded = base64.b64decode(value.split(" ", 1)[1].strip(), validate=True).decode("utf-8")
|
||||
except (ValueError, UnicodeError) as exc:
|
||||
raise unauthorized(domain.BASIC) from exc
|
||||
if ":" not in decoded:
|
||||
raise unauthorized(domain.BASIC)
|
||||
return decoded.split(":", 1)
|
||||
|
||||
|
||||
def _filename(query):
|
||||
raw = _single(query, "filename")
|
||||
if raw is None:
|
||||
raise bad_request()
|
||||
if len(raw) < 1 or len(raw) > 255 or not raw.strip():
|
||||
raise bad_request()
|
||||
if any(ord(ch) < 32 or ord(ch) == 127 or ch in "/\\" for ch in raw):
|
||||
raise bad_request()
|
||||
return raw
|
||||
|
||||
|
||||
def handle(method, path, query, headers, body):
|
||||
template, name, groups = match(method, path)
|
||||
if name == "unmatched":
|
||||
raise not_found()
|
||||
if name not in {"live", "ready"}:
|
||||
delay = faults.handler_slow_seconds()
|
||||
if delay:
|
||||
import time
|
||||
|
||||
time.sleep(delay)
|
||||
if name == "live":
|
||||
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
|
||||
if name == "ready":
|
||||
domain.ready_check()
|
||||
return template, 200, {"Content-Type": "application/json; charset=utf-8"}, _dump({"status": "ok"})
|
||||
if name == "register":
|
||||
status, payload = domain.register(_json_body(headers, body))
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "login":
|
||||
if body:
|
||||
raise bad_request()
|
||||
username, password = _basic(headers)
|
||||
status, payload = domain.login(username, password)
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "me":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
return template, 200, _json_headers(), _dump(domain._user_json(user))
|
||||
if name == "logout":
|
||||
domain.revoke(_bearer(headers))
|
||||
return template, 204, {}, b""
|
||||
if name == "create_chat":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
status, payload = domain.create_chat(user, _json_body(headers, body))
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "list_chats":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.list_chats(user, _limit(query), _single(query, "cursor"))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "get_chat":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.get_chat(user, _uuid_segment(groups[0]))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "create_message":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
chat_id = _uuid_segment(groups[0])
|
||||
status, payload = domain.create_message(
|
||||
user,
|
||||
chat_id,
|
||||
headers.get("idempotency-key"),
|
||||
_json_body(headers, body),
|
||||
path,
|
||||
)
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "list_messages":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.list_messages(user, _uuid_segment(groups[0]), _limit(query), _single(query, "cursor"))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "upload":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
content_type = media_type(headers.get("content-type", ""))
|
||||
status, payload = domain.upload(user, _uuid_segment(groups[0]), _filename(query), content_type, body)
|
||||
return template, status, _json_headers(), _dump(payload)
|
||||
if name == "get_attachment":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
payload = domain.get_attachment(user, _uuid_segment(groups[0]))
|
||||
return template, 200, _json_headers(), _dump(payload)
|
||||
if name == "download":
|
||||
user = domain.current_user(_bearer(headers))
|
||||
variant = _single(query, "variant") or "original"
|
||||
data, content_type = domain.download(user, _uuid_segment(groups[0]), variant)
|
||||
return template, 200, {"Content-Type": content_type}, data
|
||||
raise not_found()
|
||||
|
||||
|
||||
def _uuid_segment(value):
|
||||
from app.codec import parse_uuid
|
||||
|
||||
return parse_uuid(value)
|
||||
|
||||
|
||||
def _json_headers():
|
||||
return {"Content-Type": "application/json; charset=utf-8"}
|
||||
|
||||
|
||||
def _dump(payload):
|
||||
return json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def error_response(exc, request_id):
|
||||
if isinstance(exc, AppError):
|
||||
status = exc.status
|
||||
code = exc.code
|
||||
message = exc.message
|
||||
headers = dict(exc.headers)
|
||||
else:
|
||||
status = 500
|
||||
code = "internal_error"
|
||||
message = "Внутренняя ошибка"
|
||||
headers = {}
|
||||
headers["Content-Type"] = "application/json; charset=utf-8"
|
||||
body = _dump({"error": {"code": code, "message": message, "request_id": request_id}})
|
||||
return status, headers, body
|
||||
|
||||
|
||||
def request_id_from(headers):
|
||||
incoming = headers.get("x-request-id", "")
|
||||
if REQUEST_ID.fullmatch(incoming):
|
||||
return incoming
|
||||
import uuid
|
||||
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def query_from(raw):
|
||||
if not raw:
|
||||
return {}
|
||||
return parse_qs(raw, keep_blank_values=True, strict_parsing=False)
|
||||
|
||||
|
||||
def body_limit(path):
|
||||
if "/attachments" in path and path.rstrip("/").endswith("attachments"):
|
||||
return settings.max_upload + 1
|
||||
return 1_000_000
|
||||
@@ -0,0 +1,62 @@
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from app.codec import fitted_size
|
||||
from app.errors import AppError, too_large, unsupported
|
||||
from app.settings import settings
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = settings.max_pixels
|
||||
|
||||
|
||||
def _open(data):
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.verify()
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
width, height = image.size
|
||||
if width < 1 or height < 1 or width * height > settings.max_pixels:
|
||||
raise too_large()
|
||||
image.load()
|
||||
return image.copy()
|
||||
except AppError:
|
||||
raise
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise too_large() from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombWarning) as exc:
|
||||
raise unsupported() from exc
|
||||
|
||||
|
||||
def validate_image(data):
|
||||
_open(data).close()
|
||||
|
||||
|
||||
def _rgb(image):
|
||||
if image.mode == "RGB":
|
||||
return image
|
||||
if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info):
|
||||
rgba = image.convert("RGBA")
|
||||
background = Image.new("RGB", rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel("A"))
|
||||
return background
|
||||
return image.convert("RGB")
|
||||
|
||||
|
||||
def thumbnail_jpeg(data):
|
||||
image = _open(data)
|
||||
try:
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image = _rgb(image)
|
||||
target = fitted_size(image.width, image.height, 256)
|
||||
if (image.width, image.height) != target:
|
||||
image = image.resize(target, Image.Resampling.LANCZOS)
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="JPEG", quality=85, optimize=True)
|
||||
blob = buffer.getvalue()
|
||||
return blob, image.width, image.height
|
||||
except AppError:
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, ValueError) as exc:
|
||||
raise unsupported() from exc
|
||||
finally:
|
||||
image.close()
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import time
|
||||
|
||||
import uvicorn
|
||||
|
||||
from app import broker, db, domain, storage, telemetry
|
||||
from app.asgi import app
|
||||
from app.poller import start
|
||||
|
||||
|
||||
def _retry(label, action):
|
||||
last = None
|
||||
for _ in range(40):
|
||||
try:
|
||||
action()
|
||||
return
|
||||
except Exception as exc:
|
||||
last = exc
|
||||
telemetry.log_event("dependency.retry", severity="WARNING", dependency=label, error_type=type(exc).__name__)
|
||||
time.sleep(1)
|
||||
raise last
|
||||
|
||||
|
||||
def main():
|
||||
telemetry.init()
|
||||
_retry("postgres", db.init)
|
||||
db.migrate()
|
||||
_retry("s3", storage.init)
|
||||
_retry("broker", broker.init)
|
||||
domain.init_hasher()
|
||||
start()
|
||||
uvicorn.run(
|
||||
app,
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0"),
|
||||
port=int(os.environ.get("APP_PORT", "8080")),
|
||||
access_log=False,
|
||||
log_config={
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"loggers": {
|
||||
"uvicorn": {"level": "WARNING"},
|
||||
"uvicorn.error": {"level": "WARNING"},
|
||||
"uvicorn.access": {"level": "WARNING"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
import threading
|
||||
|
||||
from app import broker, db, telemetry
|
||||
|
||||
|
||||
def _once():
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS depth,
|
||||
EXTRACT(EPOCH FROM (NOW() - MIN(created_at))) AS age
|
||||
FROM attachments
|
||||
WHERE status IN ('queued', 'processing')
|
||||
"""
|
||||
).fetchone()
|
||||
depth = int(row["depth"] or 0)
|
||||
age = float(row["age"] or 0) if depth else 0.0
|
||||
used, waiting = db.stats()
|
||||
telemetry.update_runtime(queue_depth=depth, queue_age=age, db_up=1, pool_used=used, pool_waiting=waiting)
|
||||
except Exception:
|
||||
telemetry.update_runtime(db_up=0)
|
||||
try:
|
||||
telemetry.update_runtime(worker_consumers=broker.consumer_count())
|
||||
except Exception:
|
||||
telemetry.log_event("broker.stats_unavailable", severity="WARNING")
|
||||
telemetry.sample_process()
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
broker.publish_pending(conn)
|
||||
except Exception:
|
||||
telemetry.log_event("job.publish_deferred", severity="WARNING")
|
||||
|
||||
|
||||
def start():
|
||||
stop = threading.Event()
|
||||
|
||||
def loop():
|
||||
while not stop.wait(2):
|
||||
_once()
|
||||
|
||||
_once()
|
||||
threading.Thread(target=loop, name="poller", daemon=True).start()
|
||||
return stop
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
sender_id uuid NOT NULL REFERENCES users (id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS messages_order_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
owner_id uuid NOT NULL REFERENCES users (id),
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
error_code text,
|
||||
staging_key text NOT NULL,
|
||||
original_key text,
|
||||
pipeline_version integer NOT NULL DEFAULT 1,
|
||||
attempt integer NOT NULL DEFAULT 0,
|
||||
lease_until timestamptz,
|
||||
worker_id text,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
ready_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_open_idx ON attachments (created_at)
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
name text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
pipeline_version integer NOT NULL,
|
||||
PRIMARY KEY (attachment_id, name, pipeline_version)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idempotency (
|
||||
user_id uuid NOT NULL,
|
||||
method text NOT NULL,
|
||||
path text NOT NULL,
|
||||
idem_key text NOT NULL,
|
||||
body_hash bytea NOT NULL,
|
||||
status_code integer NOT NULL,
|
||||
response_json jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, path, idem_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
payload jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
published_at timestamptz
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (created_at)
|
||||
WHERE published_at IS NULL;
|
||||
@@ -0,0 +1,45 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _read(name, file_name=None, default=None):
|
||||
value = os.environ.get(name)
|
||||
if value:
|
||||
return value.strip()
|
||||
file_env = os.environ.get(file_name) if file_name else None
|
||||
if file_env:
|
||||
return Path(file_env).read_text(encoding="utf-8").strip()
|
||||
if default is not None:
|
||||
return default
|
||||
raise RuntimeError("missing configuration " + name)
|
||||
|
||||
|
||||
class Settings:
|
||||
def __init__(self):
|
||||
self.service_name = os.environ.get("OTEL_SERVICE_NAME", "messenger-api")
|
||||
self.service_version = os.environ.get("SERVICE_VERSION", "1.6.0")
|
||||
self.instance_id = os.environ.get("INSTANCE_ID", "api-local")
|
||||
self.environment = "lab"
|
||||
self.database_url = _read("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
self.session_ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
self.idempotency_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
self.s3_endpoint = os.environ.get("S3_ENDPOINT", "http://minio:9000")
|
||||
self.s3_region = os.environ.get("S3_REGION", "us-east-1")
|
||||
self.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
self.s3_access_key = _read("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
self.s3_secret_key = _read("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
self.broker_url = _read("BROKER_URL", "BROKER_URL_FILE")
|
||||
self.worker_concurrency = int(os.environ.get("WORKER_CONCURRENCY", "2"))
|
||||
self.job_max_attempts = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
||||
self.job_timeout = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
|
||||
self.staging_retention = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
|
||||
self.max_upload = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
|
||||
self.max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
self.pool_max = int(os.environ.get("DB_POOL_MAX", "8"))
|
||||
self.otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
||||
self.fault_dir = os.environ.get("FAULT_DIR", "/faults")
|
||||
self.rabbit_management_url = os.environ.get("RABBITMQ_MANAGEMENT_URL", "http://rabbitmq:15672")
|
||||
self.role = os.environ.get("APP_ROLE", "api")
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,92 @@
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app import telemetry
|
||||
from app.settings import settings
|
||||
|
||||
_client = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def init():
|
||||
global _client
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
aws_access_key_id=settings.s3_access_key,
|
||||
aws_secret_access_key=settings.s3_secret_key,
|
||||
region_name=settings.s3_region,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
try:
|
||||
_client.head_bucket(Bucket=settings.s3_bucket)
|
||||
return
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
try:
|
||||
_client.create_bucket(Bucket=settings.s3_bucket)
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
_warm()
|
||||
|
||||
|
||||
def _warm():
|
||||
"""Read one existing object so the first user download is not a cold volume open."""
|
||||
try:
|
||||
listed = _client.list_objects_v2(Bucket=settings.s3_bucket, MaxKeys=1)
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
return
|
||||
contents = listed.get("Contents") or []
|
||||
if not contents:
|
||||
return
|
||||
try:
|
||||
body = _client.get_object(Bucket=settings.s3_bucket, Key=contents[0]["Key"])["Body"]
|
||||
body.read()
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
return
|
||||
|
||||
|
||||
def ready():
|
||||
try:
|
||||
with _lock:
|
||||
_client.head_bucket(Bucket=settings.s3_bucket)
|
||||
return True
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def put(key, data, content_type, variant):
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"s3.put",
|
||||
attributes={"s3.operation": "put", "variant": variant},
|
||||
):
|
||||
with _lock:
|
||||
_client.put_object(Bucket=settings.s3_bucket, Key=key, Body=data, ContentType=content_type)
|
||||
|
||||
|
||||
def get(key, variant):
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"s3.get",
|
||||
attributes={"s3.operation": "get", "variant": variant},
|
||||
):
|
||||
with _lock:
|
||||
response = _client.get_object(Bucket=settings.s3_bucket, Key=key)
|
||||
return response["Body"].read()
|
||||
|
||||
|
||||
def delete(key):
|
||||
with _lock:
|
||||
_client.delete_object(Bucket=settings.s3_bucket, Key=key)
|
||||
@@ -0,0 +1,379 @@
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from opentelemetry import context as otel_context
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.propagate import extract, inject
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk._logs.export import LogExportResult
|
||||
from opentelemetry.sdk.metrics.export import MetricExportResult
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
from app.settings import settings
|
||||
|
||||
BANNED_FIELDS = {
|
||||
"password",
|
||||
"token",
|
||||
"authorization",
|
||||
"secret",
|
||||
"filename",
|
||||
"text",
|
||||
"body",
|
||||
"database_url",
|
||||
"broker_url",
|
||||
"image",
|
||||
}
|
||||
|
||||
_dropped = None
|
||||
_http_requests = None
|
||||
_http_duration = None
|
||||
_jobs = None
|
||||
_job_duration = None
|
||||
_ready_duration = None
|
||||
_db_duration = None
|
||||
_state = {
|
||||
"queue_depth": 0,
|
||||
"queue_age": 0.0,
|
||||
"db_up": 0,
|
||||
"pool_used": 0,
|
||||
"pool_waiting": 0,
|
||||
"worker_busy": 0,
|
||||
"worker_capacity": 0,
|
||||
"worker_consumers": 0,
|
||||
"cpu": 0.0,
|
||||
"memory": 0,
|
||||
}
|
||||
_cpu_last = None
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
class _GuardedExporter:
|
||||
def __init__(self, inner, signal):
|
||||
self.inner = inner
|
||||
self.signal = signal
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.inner, name)
|
||||
|
||||
def export(self, items, **kwargs):
|
||||
try:
|
||||
result = self.inner.export(items)
|
||||
except Exception:
|
||||
_count_drop(self.signal, len(items))
|
||||
return self._failure()
|
||||
if getattr(result, "name", None) != "SUCCESS":
|
||||
_count_drop(self.signal, len(items))
|
||||
return result
|
||||
|
||||
def _failure(self):
|
||||
if self.signal == "metrics":
|
||||
return MetricExportResult.FAILURE
|
||||
if self.signal == "logs":
|
||||
return LogExportResult.FAILURE
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
def shutdown(self, **kwargs):
|
||||
try:
|
||||
return self.inner.shutdown(**kwargs)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def force_flush(self, timeout_millis=3000):
|
||||
try:
|
||||
return self.inner.force_flush(timeout_millis)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class _RawFormatter(logging.Formatter):
|
||||
def format(self, record):
|
||||
return record.getMessage()
|
||||
|
||||
|
||||
def _count_drop(signal, count):
|
||||
if _dropped is not None and count:
|
||||
_dropped.add(count, {"signal": signal})
|
||||
|
||||
|
||||
def _resource():
|
||||
return Resource.create(
|
||||
{
|
||||
"service.name": settings.service_name,
|
||||
"service.version": settings.service_version,
|
||||
"service.instance.id": settings.instance_id,
|
||||
"deployment.environment.name": settings.environment,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _histogram(name, buckets):
|
||||
return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets))
|
||||
|
||||
|
||||
def init():
|
||||
global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration
|
||||
resource = _resource()
|
||||
endpoint = settings.otlp_endpoint
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(
|
||||
BatchSpanProcessor(
|
||||
_GuardedExporter(OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=5), "spans"),
|
||||
max_queue_size=256,
|
||||
schedule_delay_millis=2000,
|
||||
max_export_batch_size=64,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
)
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
reader = PeriodicExportingMetricReader(
|
||||
_GuardedExporter(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=5), "metrics"),
|
||||
export_interval_millis=5000,
|
||||
export_timeout_millis=4000,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[reader],
|
||||
views=[
|
||||
_histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]),
|
||||
_histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]),
|
||||
_histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]),
|
||||
_histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]),
|
||||
],
|
||||
)
|
||||
from opentelemetry import metrics
|
||||
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
meter = metrics.get_meter("messenger", settings.service_version)
|
||||
_dropped = meter.create_counter("messenger.telemetry.dropped", unit="1")
|
||||
_http_requests = meter.create_counter("http.server.requests", unit="1")
|
||||
_http_duration = meter.create_histogram("http.server.duration", unit="s")
|
||||
_jobs = meter.create_counter("messenger.jobs", unit="1")
|
||||
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
||||
_ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
|
||||
_db_duration = meter.create_histogram("messenger.db.duration", unit="s")
|
||||
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth])
|
||||
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s")
|
||||
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
|
||||
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
|
||||
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy])
|
||||
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity])
|
||||
meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers])
|
||||
meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up])
|
||||
meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu])
|
||||
meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By")
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(
|
||||
BatchLogRecordProcessor(
|
||||
_GuardedExporter(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=5), "logs"),
|
||||
max_queue_size=256,
|
||||
schedule_delay_millis=2000,
|
||||
max_export_batch_size=64,
|
||||
export_timeout_millis=5000,
|
||||
)
|
||||
)
|
||||
root = logging.getLogger()
|
||||
root.handlers.clear()
|
||||
root.setLevel(logging.INFO)
|
||||
stream = logging.StreamHandler()
|
||||
stream.setFormatter(_RawFormatter())
|
||||
root.addHandler(stream)
|
||||
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
||||
root.addHandler(otel_handler)
|
||||
for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access"):
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
quiet = logging.getLogger("opentelemetry")
|
||||
quiet.handlers.clear()
|
||||
quiet.propagate = False
|
||||
quiet.setLevel(logging.CRITICAL)
|
||||
log_event("service.started", role=settings.role)
|
||||
|
||||
|
||||
def tracer():
|
||||
return trace.get_tracer("messenger", settings.service_version)
|
||||
|
||||
|
||||
def current_ids():
|
||||
span = trace.get_current_span()
|
||||
ctx = span.get_span_context()
|
||||
if not ctx or not ctx.is_valid:
|
||||
return None, None
|
||||
return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
|
||||
|
||||
|
||||
def log_event(event, severity="INFO", **fields):
|
||||
trace_id, span_id = current_ids()
|
||||
payload = {
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z",
|
||||
"severity": severity,
|
||||
"service.name": settings.service_name,
|
||||
"event": event,
|
||||
"trace_id": trace_id,
|
||||
"span_id": span_id,
|
||||
}
|
||||
for key, value in fields.items():
|
||||
if key in BANNED_FIELDS or value is None:
|
||||
continue
|
||||
if isinstance(value, (str, int, float, bool)):
|
||||
payload[key] = value
|
||||
level = getattr(logging, severity, logging.INFO)
|
||||
logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False))
|
||||
|
||||
|
||||
def inject_carrier(carrier=None):
|
||||
target = {} if carrier is None else carrier
|
||||
inject(target)
|
||||
return target
|
||||
|
||||
|
||||
def extract_context(carrier):
|
||||
clean = {}
|
||||
for key, value in (carrier or {}).items():
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", "replace")
|
||||
if isinstance(value, str):
|
||||
clean[str(key).lower()] = value
|
||||
return extract(clean)
|
||||
|
||||
|
||||
def record_http(method, route, status, duration):
|
||||
if _http_requests is None:
|
||||
return
|
||||
klass = f"{int(status) // 100}xx"
|
||||
attributes = {
|
||||
"http.request.method": method,
|
||||
"http.route": route,
|
||||
"http.response.status_class": klass,
|
||||
}
|
||||
_http_requests.add(1, attributes)
|
||||
_http_duration.record(duration, attributes)
|
||||
|
||||
|
||||
def record_job(outcome, duration):
|
||||
attributes = {"job_type": "attachment.process.v1", "outcome": outcome}
|
||||
if _jobs is not None:
|
||||
_jobs.add(1, attributes)
|
||||
_job_duration.record(max(duration, 0.0), attributes)
|
||||
|
||||
|
||||
def record_ready(duration):
|
||||
if _ready_duration is not None:
|
||||
_ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"})
|
||||
|
||||
|
||||
def record_db(duration):
|
||||
if _db_duration is not None:
|
||||
_db_duration.record(max(duration, 0.0), {"db.system": "postgresql"})
|
||||
|
||||
|
||||
def update_runtime(**values):
|
||||
with _lock:
|
||||
_state.update(values)
|
||||
|
||||
|
||||
def note_worker(busy, capacity):
|
||||
update_runtime(worker_busy=busy, worker_capacity=capacity)
|
||||
|
||||
|
||||
def sample_process():
|
||||
global _cpu_last
|
||||
now_cpu = time.process_time()
|
||||
now_wall = time.monotonic()
|
||||
ratio = 0.0
|
||||
if _cpu_last is not None:
|
||||
dc = now_cpu - _cpu_last[0]
|
||||
dw = now_wall - _cpu_last[1]
|
||||
if dw > 0:
|
||||
ratio = dc / dw
|
||||
_cpu_last = (now_cpu, now_wall)
|
||||
memory = 0
|
||||
try:
|
||||
with open("/proc/self/statm", encoding="ascii") as handle:
|
||||
resident = int(handle.read().split()[1])
|
||||
memory = resident * os_page()
|
||||
except (OSError, IndexError, ValueError):
|
||||
memory = 0
|
||||
update_runtime(cpu=ratio, memory=memory)
|
||||
|
||||
|
||||
_page = None
|
||||
|
||||
|
||||
def os_page():
|
||||
global _page
|
||||
if _page is None:
|
||||
import os
|
||||
|
||||
_page = os.sysconf("SC_PAGE_SIZE")
|
||||
return _page
|
||||
|
||||
|
||||
def _obs(value, attributes=None):
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
return [Observation(value, attributes or {})]
|
||||
|
||||
|
||||
def _queue_depth(_options):
|
||||
return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _queue_age(_options):
|
||||
return _obs(_state["queue_age"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _pool_used(_options):
|
||||
return _obs(_state["pool_used"], {"pool": "postgres"})
|
||||
|
||||
|
||||
def _pool_waiting(_options):
|
||||
return _obs(_state["pool_waiting"], {"pool": "postgres"})
|
||||
|
||||
|
||||
def _worker_busy(_options):
|
||||
if settings.role != "worker":
|
||||
return []
|
||||
return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _worker_capacity(_options):
|
||||
if settings.role != "worker":
|
||||
return []
|
||||
return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _worker_consumers(_options):
|
||||
return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"})
|
||||
|
||||
|
||||
def _db_up(_options):
|
||||
return _obs(_state["db_up"])
|
||||
|
||||
|
||||
def _cpu(_options):
|
||||
return _obs(_state["cpu"])
|
||||
|
||||
|
||||
def _memory(_options):
|
||||
return _obs(_state["memory"])
|
||||
|
||||
|
||||
def mark_error(span):
|
||||
span.set_status(Status(StatusCode.ERROR))
|
||||
|
||||
|
||||
def attach(ctx):
|
||||
return otel_context.attach(ctx)
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import pika
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app import broker, db, storage, telemetry
|
||||
from app.broker import QUEUE, _params
|
||||
from app.codec import utcnow
|
||||
from app.errors import AppError
|
||||
from app.faults import consume_process_failure
|
||||
from app.images import thumbnail_jpeg
|
||||
from app.poller import start
|
||||
from app.settings import settings
|
||||
|
||||
_inflight = 0
|
||||
_inflight_lock = threading.Lock()
|
||||
_stop = threading.Event()
|
||||
|
||||
|
||||
class Transient(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _busy():
|
||||
global _inflight
|
||||
with _inflight_lock:
|
||||
_inflight += 1
|
||||
current = _inflight
|
||||
telemetry.note_worker(current, settings.worker_concurrency)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with _inflight_lock:
|
||||
_inflight -= 1
|
||||
current = _inflight
|
||||
telemetry.note_worker(current, settings.worker_concurrency)
|
||||
|
||||
|
||||
def _mark_failed(attachment_id, attempt, code):
|
||||
with db.connection() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = %s, lease_until = NULL, worker_id = NULL, updated_at = %s
|
||||
WHERE id = %s AND status = 'processing' AND attempt = %s
|
||||
""",
|
||||
(code, utcnow(), attachment_id, attempt),
|
||||
)
|
||||
telemetry.log_event("job.failed", attachment_id=str(attachment_id), attempt=attempt, error_code=code)
|
||||
|
||||
|
||||
def _release(attachment_id, attempt):
|
||||
with db.connection() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'queued', lease_until = NULL, worker_id = NULL, updated_at = %s
|
||||
WHERE id = %s AND status = 'processing' AND attempt = %s
|
||||
""",
|
||||
(utcnow(), attachment_id, attempt),
|
||||
)
|
||||
|
||||
|
||||
def process_delivery(body, headers):
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
attachment_id = uuid.UUID(str(payload["attachment_id"]))
|
||||
pipeline = int(payload["pipeline_version"])
|
||||
except (ValueError, UnicodeError, KeyError, TypeError):
|
||||
telemetry.log_event("job.poison", severity="ERROR")
|
||||
return "ack"
|
||||
carrier = {}
|
||||
for key, value in (headers or {}).items():
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8", "replace")
|
||||
if isinstance(value, str):
|
||||
carrier[str(key).lower()] = value
|
||||
if payload.get("traceparent"):
|
||||
carrier.setdefault("traceparent", payload["traceparent"])
|
||||
if payload.get("tracestate"):
|
||||
carrier.setdefault("tracestate", payload["tracestate"])
|
||||
parent = telemetry.extract_context(carrier)
|
||||
outcome = None
|
||||
with telemetry.tracer().start_as_current_span(
|
||||
"attachment.process consume",
|
||||
context=parent,
|
||||
kind=SpanKind.CONSUMER,
|
||||
attributes={
|
||||
"messaging.system": "rabbitmq",
|
||||
"messaging.destination.name": QUEUE,
|
||||
"messaging.operation.type": "consume",
|
||||
},
|
||||
) as span:
|
||||
try:
|
||||
action, outcome, attempt = _consume(payload, attachment_id, pipeline, span)
|
||||
except Exception:
|
||||
telemetry.mark_error(span)
|
||||
telemetry.log_event("job.failed", severity="ERROR", error_type="unexpected", attachment_id=str(attachment_id))
|
||||
action, outcome = "requeue", "retry"
|
||||
attempt = None
|
||||
if outcome:
|
||||
telemetry.record_job(outcome, time.perf_counter() - started)
|
||||
if attempt is not None:
|
||||
span.set_attribute("attempt", attempt)
|
||||
if outcome in {"failed", "retry"}:
|
||||
telemetry.mark_error(span)
|
||||
return action
|
||||
|
||||
|
||||
def _consume(payload, attachment_id, pipeline, span):
|
||||
now = utcnow()
|
||||
with db.connection() as conn:
|
||||
row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
|
||||
if not row or int(row["pipeline_version"]) != pipeline:
|
||||
return "ack", None, None
|
||||
if row["status"] == "ready":
|
||||
return "ack", None, int(row["attempt"])
|
||||
if row["status"] == "failed":
|
||||
return "ack", None, int(row["attempt"])
|
||||
if row["status"] == "processing" and row["lease_until"] and row["lease_until"] > now:
|
||||
held_attempt = int(row["attempt"])
|
||||
staging_key = None
|
||||
else:
|
||||
held_attempt = None
|
||||
attempt = int(row["attempt"]) + 1
|
||||
if attempt > settings.job_max_attempts:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'failed', error_code = 'retry_exhausted', lease_until = NULL, updated_at = %s
|
||||
WHERE id = %s
|
||||
""",
|
||||
(now, attachment_id),
|
||||
)
|
||||
return "ack", "failed", attempt
|
||||
lease = now + dt.timedelta(seconds=settings.job_timeout)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'processing', attempt = %s, lease_until = %s, worker_id = %s, updated_at = %s
|
||||
WHERE id = %s
|
||||
""",
|
||||
(attempt, lease, settings.instance_id, now, attachment_id),
|
||||
)
|
||||
staging_key = row["staging_key"]
|
||||
content_type = row["content_type"]
|
||||
expected_sha = row["sha256"]
|
||||
created_at = row["created_at"]
|
||||
if staging_key is None:
|
||||
span.add_event("lease_held")
|
||||
time.sleep(1)
|
||||
return "requeue", None, held_attempt
|
||||
span.set_attribute("attempt", attempt)
|
||||
telemetry.log_event("job.consumed", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
|
||||
try:
|
||||
if consume_process_failure():
|
||||
span.add_event("retry", {"reason": "injected"})
|
||||
raise Transient()
|
||||
data = storage.get(staging_key, "staging")
|
||||
if hashlib.sha256(data).hexdigest() != expected_sha:
|
||||
raise Permanent("invalid_image")
|
||||
thumb, width, height = thumbnail_jpeg(data)
|
||||
original_key = f"originals/{attachment_id}"
|
||||
thumb_key = f"variants/{attachment_id}/thumbnail"
|
||||
storage.put(original_key, data, content_type, "original")
|
||||
storage.put(thumb_key, thumb, "image/jpeg", "thumbnail")
|
||||
except Permanent as exc:
|
||||
_mark_failed(attachment_id, attempt, str(exc) or "processing_failed")
|
||||
return "ack", "failed", attempt
|
||||
except AppError as exc:
|
||||
code = "invalid_image" if exc.status in {413, 415} else "processing_failed"
|
||||
_mark_failed(attachment_id, attempt, code)
|
||||
return "ack", "failed", attempt
|
||||
except Transient:
|
||||
_release(attachment_id, attempt)
|
||||
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="transient")
|
||||
time.sleep(0.5)
|
||||
return "requeue", "retry", attempt
|
||||
except (BotoCoreError, ClientError, OSError, TimeoutError):
|
||||
_release(attachment_id, attempt)
|
||||
telemetry.log_event("job.retry", attachment_id=str(attachment_id), attempt=attempt, reason="storage")
|
||||
time.sleep(min(2 ** max(attempt - 1, 0), 8))
|
||||
return "requeue", "retry", attempt
|
||||
thumb_sha = hashlib.sha256(thumb).hexdigest()
|
||||
ready_at = utcnow()
|
||||
with db.connection() as conn:
|
||||
current = conn.execute("SELECT status, attempt FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone()
|
||||
if not current or current["status"] != "processing" or int(current["attempt"]) != attempt:
|
||||
return "ack", None, attempt
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO attachment_variants (
|
||||
attachment_id, name, content_type, size_bytes, width, height, sha256, object_key, pipeline_version
|
||||
) VALUES (%s, 'thumbnail', 'image/jpeg', %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (attachment_id, name, pipeline_version) DO UPDATE
|
||||
SET content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
width = EXCLUDED.width,
|
||||
height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256,
|
||||
object_key = EXCLUDED.object_key
|
||||
""",
|
||||
(attachment_id, len(thumb), width, height, thumb_sha, thumb_key, pipeline),
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE attachments
|
||||
SET status = 'ready', error_code = NULL, original_key = %s, ready_at = %s,
|
||||
lease_until = NULL, updated_at = %s
|
||||
WHERE id = %s AND status = 'processing' AND attempt = %s
|
||||
""",
|
||||
(original_key, ready_at, ready_at, attachment_id, attempt),
|
||||
)
|
||||
try:
|
||||
storage.delete(staging_key)
|
||||
except (BotoCoreError, ClientError, OSError):
|
||||
telemetry.log_event("staging.cleanup_deferred", severity="WARNING", attachment_id=str(attachment_id))
|
||||
telemetry.record_ready((ready_at - created_at).total_seconds())
|
||||
telemetry.log_event("job.ready", attachment_id=str(attachment_id), attempt=attempt, job_id=str(payload.get("job_id")))
|
||||
return "ack", "success", attempt
|
||||
|
||||
|
||||
def _serve(connection):
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.basic_qos(prefetch_count=settings.worker_concurrency)
|
||||
executor = ThreadPoolExecutor(max_workers=settings.worker_concurrency, thread_name_prefix="job")
|
||||
|
||||
def on_message(ch, method, properties, body):
|
||||
def work():
|
||||
with _busy():
|
||||
try:
|
||||
action = process_delivery(body, properties.headers if properties else None)
|
||||
except Exception:
|
||||
action = "requeue"
|
||||
|
||||
def finish():
|
||||
try:
|
||||
if action == "ack":
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||||
except Exception:
|
||||
telemetry.log_event("job.ack_failed", severity="ERROR")
|
||||
|
||||
connection.add_callback_threadsafe(finish)
|
||||
|
||||
executor.submit(work)
|
||||
|
||||
channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False)
|
||||
|
||||
def watch():
|
||||
_stop.wait()
|
||||
connection.add_callback_threadsafe(channel.stop_consuming)
|
||||
|
||||
threading.Thread(target=watch, name="stop-watch", daemon=True).start()
|
||||
channel.start_consuming()
|
||||
executor.shutdown(wait=True, cancel_futures=False)
|
||||
|
||||
|
||||
class _Health(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path.startswith("/health/live"):
|
||||
self._send(200)
|
||||
return
|
||||
if self.path.startswith("/health/ready"):
|
||||
try:
|
||||
with db.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
ok = storage.ready() and broker.ready()
|
||||
except Exception:
|
||||
ok = False
|
||||
self._send(200 if ok else 503)
|
||||
return
|
||||
self._send(404)
|
||||
|
||||
def _send(self, status):
|
||||
body = b'{"status":"ok"}' if status == 200 else b'{"status":"unavailable"}'
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
return
|
||||
|
||||
|
||||
def _health_server():
|
||||
server = ThreadingHTTPServer(("0.0.0.0", 8081), _Health)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
def _request_stop(signum, _frame):
|
||||
_stop.set()
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, _request_stop)
|
||||
signal.signal(signal.SIGINT, _request_stop)
|
||||
telemetry.init()
|
||||
from app.main import _retry
|
||||
|
||||
_retry("postgres", db.init)
|
||||
db.migrate()
|
||||
_retry("s3", storage.init)
|
||||
_retry("broker", broker.init)
|
||||
telemetry.note_worker(0, settings.worker_concurrency)
|
||||
start()
|
||||
threading.Thread(target=_health_server, name="health", daemon=True).start()
|
||||
while not _stop.is_set():
|
||||
connection = None
|
||||
try:
|
||||
connection = pika.BlockingConnection(_params())
|
||||
telemetry.update_runtime(worker_consumers=settings.worker_concurrency)
|
||||
_serve(connection)
|
||||
except Exception:
|
||||
if _stop.is_set():
|
||||
break
|
||||
telemetry.log_event("broker.reconnect", severity="WARNING", error_type="connection")
|
||||
time.sleep(1)
|
||||
finally:
|
||||
if connection is not None and connection.is_open:
|
||||
try:
|
||||
connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
name: mtusi-lab6
|
||||
|
||||
x-app-env: &app-env
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_KEY}
|
||||
BROKER_URL: amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@rabbitmq:5672/%2F
|
||||
RABBITMQ_MANAGEMENT_URL: http://rabbitmq:15672
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
DB_POOL_MAX: "8"
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
||||
SERVICE_VERSION: "1.6.0"
|
||||
FAULT_DIR: /faults
|
||||
AWS_EC2_METADATA_DISABLED: "true"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pg-data:/var/lib/postgresql/data
|
||||
- ./app/schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
command: ["server", "-dir=/data", "-ip=0.0.0.0", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.ip.bind=0.0.0.0", "-s3.config=/etc/seaweed/s3.json"]
|
||||
volumes:
|
||||
- s3-data:/data
|
||||
- ./infra/seaweed/s3.json:/etc/seaweed/s3.json:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "weed", "version"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
rabbitmq:
|
||||
image: rabbitmq:3.13.7-management-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER}
|
||||
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD}
|
||||
volumes:
|
||||
- rabbit-data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
api-1: &api
|
||||
build: .
|
||||
image: mtusi-lab6-messenger:1.6.0
|
||||
environment:
|
||||
<<: *app-env
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
INSTANCE_ID: api-1
|
||||
APP_ROLE: api
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=api-1
|
||||
volumes:
|
||||
- ./faults:/faults
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"]
|
||||
interval: 5s
|
||||
timeout: 4s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
api-2:
|
||||
<<: *api
|
||||
environment:
|
||||
<<: *app-env
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
INSTANCE_ID: api-2
|
||||
APP_ROLE: api
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=api-2
|
||||
|
||||
worker:
|
||||
build: .
|
||||
image: mtusi-lab6-messenger:1.6.0
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
INSTANCE_ID: worker-1
|
||||
APP_ROLE: worker
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab,service.version=1.6.0,service.instance.id=worker-1
|
||||
volumes:
|
||||
- ./faults:/faults
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=3)"]
|
||||
interval: 5s
|
||||
timeout: 4s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
nginx:
|
||||
image: nginx:1.27.4-alpine
|
||||
depends_on:
|
||||
api-1:
|
||||
condition: service_healthy
|
||||
api-2:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "127.0.0.1:8080:80"
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.120.0
|
||||
command: ["--config=/etc/otelcol/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
||||
depends_on:
|
||||
- tempo
|
||||
- loki
|
||||
- prometheus
|
||||
mem_limit: 320m
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.55.1
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=2d
|
||||
- --web.enable-lifecycle
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
ports:
|
||||
- "127.0.0.1:9090:9090"
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.3.2
|
||||
command: ["-config.file=/etc/loki/loki.yml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yml:/etc/loki/loki.yml:ro
|
||||
- loki-data:/loki
|
||||
ports:
|
||||
- "127.0.0.1:3100:3100"
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
command: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yml:/etc/tempo/tempo.yml:ro
|
||||
- tempo-data:/var/tempo
|
||||
ports:
|
||||
- "127.0.0.1:3200:3200"
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_SERVER_HTTP_PORT: "3000"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
mem_limit: 384m
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pg-data:
|
||||
s3-data:
|
||||
rabbit-data:
|
||||
prometheus-data:
|
||||
loki-data:
|
||||
tempo-data:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,18 @@
|
||||
# API error budget
|
||||
|
||||
SLO: 99% of business HTTP responses are not 5xx, measured on routes other than `/health/*`.
|
||||
4xx is a client outcome and is outside the numerator. The ratio is `5xx / all business requests`.
|
||||
|
||||
Windows used on this lab stand:
|
||||
|
||||
- fast burn: 1 minute and 5 minutes, both above 14.4% (14.4 times a 1% budget), `for` 30s, and at least 0.2 requests/s
|
||||
- slow burn: 5 minutes and 15 minutes, both above 6%, `for` 1 minute
|
||||
- simple page: 5 minute ratio above 5% for 1 minute
|
||||
|
||||
A short spike can move the 1 minute window without moving the 5 minute window enough, so the fast burn stays inactive. A sustained burst moves both.
|
||||
|
||||
1. Open Prometheus `ALERTS` or the Messenger dashboard panel "Prometheus alert state".
|
||||
2. If `ApiErrorBudgetFastBurn` or `ApiHighErrorRate` is firing, open a trace for a 5xx request.
|
||||
3. If the `db.transaction` span is the long or erroring child, follow `docs/runbooks/db-latency.md`.
|
||||
4. If the server span is long and the database span is short, the handler itself is slow (`handler_slow_ms` fault or application code).
|
||||
5. Remove the fault file and wait for the next evaluations. The alert leaves `firing` when the expression is false; `for` does not delay resolution.
|
||||
@@ -0,0 +1,15 @@
|
||||
# Slow or unavailable database
|
||||
|
||||
`messenger_db_up` is 0 when the process cannot run `SELECT 1`. Business calls then return 503 `unavailable`. That increases the API error ratio. Health `/health/live` stays 200. `/health/ready` becomes 503.
|
||||
|
||||
A slow database is different from a slow handler:
|
||||
|
||||
- slow database: the `db.transaction` span is long and the server span is about the same length
|
||||
- slow handler: `handler_slow` sits in the server span, and `db.transaction` stays short
|
||||
|
||||
`DatabaseUnready` fires when `min(messenger_db_up) == 0` for 30 seconds.
|
||||
|
||||
1. Compare one server span with its `db.transaction` child in Tempo.
|
||||
2. If the database span failed, check PostgreSQL. Grafana being down does not cause this alert.
|
||||
3. Restore PostgreSQL. `messenger_db_up` returns to 1 and the error-budget alerts resolve after the windows slide.
|
||||
4. The collector and Grafana are not on the readiness path.
|
||||
@@ -0,0 +1,14 @@
|
||||
# Queue lag and worker loss
|
||||
|
||||
`messenger_queue_oldest_age_seconds` is the age of the oldest attachment still `queued` or `processing`. It is not the duration of a completed job. `messenger_queue_depth` is how many such rows exist. Both are read from PostgreSQL by the API, so they keep moving when the worker process is stopped.
|
||||
|
||||
`messenger_worker_consumers` is the RabbitMQ consumer count. The API polls the management API, so the series becomes 0 instead of going stale.
|
||||
|
||||
Alert `QueueLagHigh`: max age above 30 seconds for 1 minute.
|
||||
Alert `WorkerConsumersZero`: max consumers equals 0 for 1 minute.
|
||||
|
||||
1. Confirm consumers are 0 or age is climbing on the Messenger dashboard.
|
||||
2. Check worker logs in Loki for `job.failed` or `broker.reconnect`. A clean stop has neither new `job.consumed` nor a database error.
|
||||
3. Start the worker again: `docker compose --env-file .env start worker`.
|
||||
4. Age returns to 0 after the backlog is acknowledged. Alerts resolve on the next evaluation.
|
||||
5. Do not use p95 time-to-ready alone: it only observes jobs that reached `ready`.
|
||||
@@ -0,0 +1,8 @@
|
||||
# Telemetry backend unavailable
|
||||
|
||||
The API and worker export OTLP on a background queue (256 spans or log records, 5 second timeout). The collector limits itself to 192 MiB, batches, and keeps a sending queue of 200 items with retry capped at 30 seconds. When Tempo, Loki, or Prometheus is down, exports fail and `messenger_telemetry_dropped_total` increases. Message acceptance does not wait on those exports.
|
||||
|
||||
1. Stop one backend, for example `docker compose --env-file .env stop tempo`.
|
||||
2. Send a few API requests. `/health/ready` stays 200 and messages still return 201.
|
||||
3. Watch collector memory. It should stay near its limit, not grow without bound.
|
||||
4. Start the backend again. New spans appear. Spans dropped during the outage are not replayed from the in-memory queue once it has overflowed.
|
||||
@@ -0,0 +1,11 @@
|
||||
# Time to ready
|
||||
|
||||
SLO: 95% of attachments that reach `ready` do so within 10 seconds of durable acceptance.
|
||||
The series is `messenger_attachment_time_to_ready_seconds`, a histogram in seconds.
|
||||
`AttachmentReadyLatencyHigh` uses `histogram_quantile(0.95, ...)` over 15 minutes and pages after 2 minutes above 10 seconds.
|
||||
|
||||
This SLO does not see jobs that never finish. Pair it with queue age.
|
||||
|
||||
1. If p95 is high and queue age is low, open a worker trace and compare `s3.put`, `s3.get`, and `db.transaction`.
|
||||
2. A retry span with `attempt > 1` and event `retry` means the first attempt was transient. The successful attempt should still contain S3 and the ready update.
|
||||
3. If p95 is quiet but users wait, read queue age. The worker is likely stopped or stuck before completion.
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"alerts_before": {
|
||||
"AttachmentReadyLatencyHigh": "pending"
|
||||
},
|
||||
"while_worker_stopped": {
|
||||
"first_firing": {
|
||||
"WorkerConsumersZero": 75.5,
|
||||
"QueueLagHigh": 105.7
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"t": 90.6,
|
||||
"alerts": {
|
||||
"QueueLagHigh": "pending",
|
||||
"AttachmentReadyLatencyHigh": "firing",
|
||||
"WorkerConsumersZero": "firing"
|
||||
},
|
||||
"age": "82.87327",
|
||||
"api_consumers": "0"
|
||||
},
|
||||
{
|
||||
"t": 95.6,
|
||||
"alerts": {
|
||||
"QueueLagHigh": "pending",
|
||||
"AttachmentReadyLatencyHigh": "firing",
|
||||
"WorkerConsumersZero": "firing"
|
||||
},
|
||||
"age": "88.417685",
|
||||
"api_consumers": "0"
|
||||
},
|
||||
{
|
||||
"t": 100.7,
|
||||
"alerts": {
|
||||
"QueueLagHigh": "pending",
|
||||
"AttachmentReadyLatencyHigh": "firing",
|
||||
"WorkerConsumersZero": "firing"
|
||||
},
|
||||
"age": "92.908538",
|
||||
"api_consumers": "0"
|
||||
},
|
||||
{
|
||||
"t": 105.7,
|
||||
"alerts": {
|
||||
"QueueLagHigh": "firing",
|
||||
"AttachmentReadyLatencyHigh": "firing",
|
||||
"WorkerConsumersZero": "firing"
|
||||
},
|
||||
"age": "98.453736",
|
||||
"api_consumers": "0"
|
||||
}
|
||||
]
|
||||
},
|
||||
"after_worker_start": {
|
||||
"cleared_after_seconds": {
|
||||
"QueueLagHigh": 20.1,
|
||||
"WorkerConsumersZero": 20.1
|
||||
},
|
||||
"alerts": {
|
||||
"AttachmentReadyLatencyHigh": "firing"
|
||||
}
|
||||
},
|
||||
"tempo_down_40s": {
|
||||
"api_results": {
|
||||
"200": 76
|
||||
},
|
||||
"failed_before": [
|
||||
"9"
|
||||
],
|
||||
"failed_after": [
|
||||
"24"
|
||||
],
|
||||
"queue_size": [
|
||||
"0",
|
||||
"22"
|
||||
],
|
||||
"collector_memory": "68.8MiB / 320MiB"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.078,
|
||||
"concurrency": 2,
|
||||
"successful_requests": 1247,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 82.703,
|
||||
"successful_latency_ms": {
|
||||
"p50": 31.0,
|
||||
"p95": 47.0,
|
||||
"p99": 47.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"trace_id": "cf5025ad9d9a2a248da8335a0c6caee2",
|
||||
"log": {
|
||||
"job.retry": {"attempt": 1, "same_trace": true},
|
||||
"job.ready": {"attempt": 2, "same_trace": true}
|
||||
},
|
||||
"chain": [
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"attachment.process publish parent=POST",
|
||||
"attachment.process consume (attempt 1, injected transient) parent=publish",
|
||||
"attachment.process consume (attempt 2) parent=publish",
|
||||
"s3.get and s3.put parent=second consume",
|
||||
"db.transaction parent=POST and parent=consume"
|
||||
],
|
||||
"note": "Both consumer spans share the producer span as parent. No message text, filename, or object key is stored here."
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
{
|
||||
"started": "2026-11-06T21:40:04Z",
|
||||
"retry_upload": {
|
||||
"final_status": "ready",
|
||||
"log": {
|
||||
"event": "job.ready",
|
||||
"attempt": 1,
|
||||
"trace_id": "40c8e289c22994a3c786030354799554",
|
||||
"span_id": "606017e0d8c6f6b5",
|
||||
"service": "messenger-worker"
|
||||
},
|
||||
"consume_spans": 1,
|
||||
"span_names": [
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"db.transaction",
|
||||
"db.transaction",
|
||||
"s3.put",
|
||||
"db.transaction",
|
||||
"db.transaction",
|
||||
"attachment.process publish",
|
||||
"attachment.process consume",
|
||||
"db.transaction",
|
||||
"s3.get",
|
||||
"s3.put",
|
||||
"s3.put",
|
||||
"db.transaction"
|
||||
],
|
||||
"parents_linked": true
|
||||
},
|
||||
"warmup": {
|
||||
"200": 180
|
||||
},
|
||||
"alerts_after_warmup": {},
|
||||
"short_spike": {
|
||||
"500": 20
|
||||
},
|
||||
"alerts_after_short_spike": {
|
||||
"ApiHighErrorRate": "pending",
|
||||
"ApiErrorBudgetSlowBurn": "pending"
|
||||
},
|
||||
"recovery_after_spike": {
|
||||
"200": 40
|
||||
},
|
||||
"fast_burn_detected_after_seconds": 51.6,
|
||||
"alerts_when_fast_burn_fired": {
|
||||
"ApiHighErrorRate": "firing",
|
||||
"ApiErrorBudgetSlowBurn": "firing",
|
||||
"ApiErrorBudgetFastBurn": "firing"
|
||||
},
|
||||
"sustained_counts_until_detection_or_timeout": {
|
||||
"500": 104
|
||||
},
|
||||
"fast_burn_resolved_after_seconds": 70.6,
|
||||
"alerts_after_resolve": {
|
||||
"ApiHighErrorRate": "firing",
|
||||
"ApiErrorBudgetSlowBurn": "firing"
|
||||
},
|
||||
"worker_stopped": {
|
||||
"consumers_firing_after_seconds": null,
|
||||
"queue_lag_firing_after_seconds": 0.0,
|
||||
"alerts": {
|
||||
"ApiHighErrorRate": "firing",
|
||||
"ApiErrorBudgetSlowBurn": "firing",
|
||||
"QueueLagHigh": "firing"
|
||||
}
|
||||
},
|
||||
"worker_restored": {
|
||||
"consumers_cleared_after_seconds": 0.0,
|
||||
"alerts": {
|
||||
"ApiHighErrorRate": "firing",
|
||||
"ApiErrorBudgetSlowBurn": "firing",
|
||||
"QueueLagHigh": "firing"
|
||||
}
|
||||
},
|
||||
"latency_note": "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span",
|
||||
"tempo_stopped": {
|
||||
"api_results": {
|
||||
"200": 16
|
||||
},
|
||||
"failed_spans_before": [
|
||||
"0"
|
||||
],
|
||||
"failed_spans_after": [
|
||||
"0"
|
||||
],
|
||||
"collector_memory": "67.75MiB / 320MiB"
|
||||
},
|
||||
"finished": "2026-11-06T21:48:01Z"
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
Local fault-injection files. They are not present by default, so the API behaves normally.
|
||||
|
||||
Write one of these names in this directory while the stack is running:
|
||||
|
||||
- `http_error_ratio` — float 0..1, fraction of `/api/` requests that return 500
|
||||
- `db_slow_ms` — delay added inside the `db.transaction` span
|
||||
- `db_fail` — `1` makes database calls return 503
|
||||
- `handler_slow_ms` — delay inside the HTTP server span and outside the database span
|
||||
- `process_fail_remaining` — positive integer, worker fails that many processing attempts once each
|
||||
|
||||
Do not put secrets here. The worker must be able to update `process_fail_remaining`.
|
||||
+18
-4
@@ -1,7 +1,21 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Топология — корневой `compose.yaml`. `compose.example.yaml` остаётся worksheet курса и не запускается.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
| Роль | Сервис | Образ | Порт на хосте | Том |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| PostgreSQL | postgres | postgres:16.6-alpine | нет | pg-data |
|
||||
| S3 | s3 | chrislusf/seaweedfs:4.51 | нет | s3-data, плюс gitignored `infra/seaweed/s3.json` |
|
||||
| Брокер | rabbitmq | rabbitmq:3.13.7-management-alpine | нет | rabbit-data |
|
||||
| API | api-1, api-2 | сборка из Dockerfile | нет | `./faults` |
|
||||
| Worker | worker | та же сборка | нет | `./faults` |
|
||||
| Балансировщик | nginx | nginx:1.27.4-alpine | 127.0.0.1:8080 | — |
|
||||
| Collector | otel-collector | otel/opentelemetry-collector-contrib:0.120.0 | нет | — |
|
||||
| Метрики | prometheus | prom/prometheus:v2.55.1 | 127.0.0.1:9090 | prometheus-data |
|
||||
| Логи | loki | grafana/loki:3.3.2 | 127.0.0.1:3100 | loki-data |
|
||||
| Трассы | tempo | grafana/tempo:2.7.2 | 127.0.0.1:3200 | tempo-data |
|
||||
| UI | grafana | grafana/grafana:11.5.2 | 127.0.0.1:3000 | grafana-data |
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
Секреты приходят из `.env` (БД, S3, RabbitMQ, пароль Grafana). Идентичность SeaweedFS генерируется `python scripts/render-s3-config.py` и не коммитится. Схема БД — `app/schema.sql`. Бакет и очередь создают приложения при старте.
|
||||
|
||||
Grafana не хранит метрики, логи и трассы. Datasources и дашборд `messenger-overview` импортируются из `infra/grafana/provisioning` и `infra/grafana/dashboards`. Правила алертов — `infra/prometheus/rules.yml`.
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
{
|
||||
"uid": "messenger-overview",
|
||||
"title": "Messenger observability",
|
||||
"timezone": "utc",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"tags": ["messenger", "lab6"],
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "text",
|
||||
"title": "Signal stores",
|
||||
"gridPos": {"h": 4, "w": 24, "x": 0, "y": 0},
|
||||
"options": {
|
||||
"mode": "markdown",
|
||||
"content": "Prometheus stores metrics (rates, histograms, gauges). Loki stores structured logs. Tempo stores traces. The collector is not a database: it receives OTLP, redacts, batches, and exports. Grafana only queries those stores. Percentiles use `histogram_quantile` on cumulative buckets, summed across API replicas before the quantile."
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "API RPS by route",
|
||||
"description": "Unit: requests per second. Sum of rates across replicas.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
|
||||
"fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line"}}, "overrides": []},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (http_route) (rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))",
|
||||
"legendFormat": "{{http_route}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "API error ratio (5xx / business)",
|
||||
"description": "Unit: fraction 0-1. Numerator is status class 5xx. Denominator is business routes, excluding /health and unmatched. 4xx is not an error for this SLI.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "messenger:api_error_ratio:rate5m", "legendFormat": "5m"},
|
||||
{"refId": "B", "expr": "messenger:api_error_ratio:rate1m", "legendFormat": "1m"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "API duration p50/p95/p99",
|
||||
"description": "Unit: seconds. histogram_quantile over the sum of bucket rates. Not an average of per-replica quantiles.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p95"},
|
||||
{"refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*|unmatched\"}[5m])))", "legendFormat": "p99"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Jobs by outcome",
|
||||
"description": "Unit: jobs per second. outcome is success, failed, or retry. job_type is attachment.process.v1.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
|
||||
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "sum by (outcome) (rate(messenger_jobs_total[1m]))", "legendFormat": "{{outcome}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Worker job duration p95",
|
||||
"description": "Unit: seconds, from the job histogram.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 20},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[5m])))", "legendFormat": "{{outcome}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Queue depth and oldest age",
|
||||
"description": "Depth is a count of queued+processing attachments. Age is seconds since the oldest of those rows was accepted. max() avoids double-counting the two API replicas.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 20},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "age"}, "properties": [{"id": "unit", "value": "s"}]}]},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max(messenger_queue_depth{queue=\"attachment.process.v1\"})", "legendFormat": "depth"},
|
||||
{"refId": "B", "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachment.process.v1\"})", "legendFormat": "age"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Time to ready p95",
|
||||
"description": "Unit: seconds from durable acceptance to ready. Completed jobs only.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 20},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m])))", "legendFormat": "p95"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "DB pool used and waiting",
|
||||
"description": "Unit: connections. Summed across processes. waiting is requests blocked on the pool.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 28},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "sum(messenger_db_pool_used)", "legendFormat": "used"},
|
||||
{"refId": "B", "expr": "sum(messenger_db_pool_waiting)", "legendFormat": "waiting"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Worker saturation and consumers",
|
||||
"description": "busy/capacity are counts inside the worker. consumers is the RabbitMQ consumer count observed by the API.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 28},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max(messenger_worker_busy)", "legendFormat": "busy"},
|
||||
{"refId": "B", "expr": "max(messenger_worker_capacity)", "legendFormat": "capacity"},
|
||||
{"refId": "C", "expr": "max(messenger_worker_consumers{service_name=\"messenger-api\"})", "legendFormat": "consumers"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Process CPU and memory",
|
||||
"description": "CPU unit is process CPU-seconds per wall second (can exceed 1). Memory unit is bytes of RSS.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 28},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "max by (service_name) (messenger_process_cpu_utilization)", "legendFormat": "cpu {{service_name}}"},
|
||||
{"refId": "B", "expr": "max by (service_name) (messenger_process_memory_bytes)", "legendFormat": "rss {{service_name}}"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "DB duration p95",
|
||||
"description": "Unit: seconds. Includes an injected db_slow fault. Compare with the server span to separate a slow database from a slow handler.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_db_duration_seconds_bucket[5m])))", "legendFormat": "db p95"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "table",
|
||||
"title": "Prometheus alert state",
|
||||
"description": "Local firing/pending state. No external notification channel is configured.",
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "ALERTS{alertstate=~\"firing|pending\"}", "instant": true, "format": "table"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "logs",
|
||||
"title": "API and worker logs",
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 44},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "{service_name=~\"messenger-api|messenger-worker\"}"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
orgId: 1
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,41 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 5s
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: "\"trace_id\":\"([0-9a-f]+)\""
|
||||
url: "$${__value.raw}"
|
||||
datasourceUid: tempo
|
||||
urlDisplayLabel: View trace
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
filterByTraceID: true
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
lokiSearch:
|
||||
datasourceUid: loki
|
||||
@@ -0,0 +1,39 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
volume_enabled: true
|
||||
reject_old_samples: false
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
max_query_series: 500
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,22 @@
|
||||
upstream messenger_api {
|
||||
server api-1:8080 max_fails=1 fail_timeout=5s;
|
||||
server api-2:8080 max_fails=1 fail_timeout=5s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
client_max_body_size 12m;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
|
||||
proxy_next_upstream_tries 2;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger_api;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection "";
|
||||
}
|
||||
}
|
||||
+12
-4
@@ -1,7 +1,15 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# Collector
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
`collector.yaml` принимает OTLP gRPC `:4317` и HTTP `:4318` только внутри сети Compose.
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
Процессоры на всех трёх pipeline: memory limiter (192 MiB, spike 48 MiB), удаление URL, SQL и заголовков Authorization/Cookie, batch.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
Экспорт:
|
||||
|
||||
- трассы — OTLP gRPC в Tempo, очередь 200, retry не дольше 30 с;
|
||||
- логи — OTLP HTTP в Loki `/otlp/v1/logs`, та же очередь и retry;
|
||||
- метрики — Prometheus exporter `:8889`, его скрейпит Prometheus.
|
||||
|
||||
Внутренние метрики Collector — `:8888`. Health — `:13133`. Эти порты наружу не опубликованы.
|
||||
|
||||
Если Tempo или Loki недоступны, очередь exporter ограничена 200, retry обрывается через 30 с, memory limiter не даёт процессу расти до лимита контейнера 320 MiB. Приложение при этом отвечает: экспорт не блокирует приём сообщений.
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
attributes/redact:
|
||||
actions:
|
||||
- key: url.full
|
||||
action: delete
|
||||
- key: url.query
|
||||
action: delete
|
||||
- key: url.path
|
||||
action: delete
|
||||
- key: http.url
|
||||
action: delete
|
||||
- key: http.target
|
||||
action: delete
|
||||
- key: db.statement
|
||||
action: delete
|
||||
- key: db.query.text
|
||||
action: delete
|
||||
- key: http.request.header.authorization
|
||||
action: delete
|
||||
- key: http.request.header.cookie
|
||||
action: delete
|
||||
batch:
|
||||
timeout: 2s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
timeout: 5s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 15s
|
||||
max_elapsed_time: 30s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
otlphttp/loki:
|
||||
logs_endpoint: http://loki:3100/otlp/v1/logs
|
||||
tls:
|
||||
insecure: true
|
||||
timeout: 5s
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 15s
|
||||
max_elapsed_time: 30s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
metric_expiration: 180s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
metrics:
|
||||
address: 0.0.0.0:8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/redact, batch]
|
||||
exporters: [otlp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/redact, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, attributes/redact, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,15 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
scrape_timeout: 4s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8889"]
|
||||
- job_name: otel-collector-internal
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
@@ -0,0 +1,96 @@
|
||||
groups:
|
||||
- name: messenger-sli
|
||||
interval: 5s
|
||||
rules:
|
||||
- record: messenger:api_requests:rate1m
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[1m]))
|
||||
- record: messenger:api_5xx:rate1m
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[1m]))
|
||||
- record: messenger:api_error_ratio:rate1m
|
||||
expr: messenger:api_5xx:rate1m / clamp_min(messenger:api_requests:rate1m, 1e-9)
|
||||
- record: messenger:api_requests:rate5m
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[5m]))
|
||||
- record: messenger:api_5xx:rate5m
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[5m]))
|
||||
- record: messenger:api_error_ratio:rate5m
|
||||
expr: messenger:api_5xx:rate5m / clamp_min(messenger:api_requests:rate5m, 1e-9)
|
||||
- record: messenger:api_requests:rate15m
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched"}[15m]))
|
||||
- record: messenger:api_5xx:rate15m
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",http_route!~"/health/.*|unmatched",http_response_status_class="5xx"}[15m]))
|
||||
- record: messenger:api_error_ratio:rate15m
|
||||
expr: messenger:api_5xx:rate15m / clamp_min(messenger:api_requests:rate15m, 1e-9)
|
||||
|
||||
- name: messenger-alerts
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: ApiHighErrorRate
|
||||
expr: messenger:api_error_ratio:rate5m > 0.05 and messenger:api_requests:rate5m > 0.2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
slo: api-availability
|
||||
annotations:
|
||||
summary: API business 5xx ratio is above 5%
|
||||
description: Numerator is 5xx on business routes. Denominator is all business requests. Health checks and 4xx are excluded. Window 5m, for 1m.
|
||||
runbook: docs/runbooks/api-error-budget.md
|
||||
- alert: ApiErrorBudgetFastBurn
|
||||
expr: messenger:api_error_ratio:rate1m > 0.144 and messenger:api_error_ratio:rate5m > 0.144 and messenger:api_requests:rate1m > 0.2
|
||||
for: 30s
|
||||
labels:
|
||||
severity: page
|
||||
slo: api-availability
|
||||
burn: fast
|
||||
annotations:
|
||||
summary: Fast multi-window burn of the 1% API error budget
|
||||
description: Both the 1m and 5m error ratios exceed 14.4% (14.4x a 1% budget). A sub-minute spike does not satisfy the 5m window when healthy traffic is present.
|
||||
runbook: docs/runbooks/api-error-budget.md
|
||||
- alert: ApiErrorBudgetSlowBurn
|
||||
expr: messenger:api_error_ratio:rate5m > 0.06 and messenger:api_error_ratio:rate15m > 0.06 and messenger:api_requests:rate5m > 0.2
|
||||
for: 1m
|
||||
labels:
|
||||
severity: ticket
|
||||
slo: api-availability
|
||||
burn: slow
|
||||
annotations:
|
||||
summary: Slow multi-window burn of the API error budget
|
||||
description: 5m and 15m error ratios exceed 6% (6x a 1% budget). This catches a long mild degradation that the fast burn may miss.
|
||||
runbook: docs/runbooks/api-error-budget.md
|
||||
- alert: QueueLagHigh
|
||||
expr: max(messenger_queue_oldest_age_seconds{queue="attachment.process.v1"}) > 30
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
slo: time-to-ready
|
||||
annotations:
|
||||
summary: Oldest attachment job is older than 30s
|
||||
description: Backlog age is not processing latency. It grows when the worker is stopped or cannot finish jobs. Unit is seconds.
|
||||
runbook: docs/runbooks/queue-lag.md
|
||||
- alert: WorkerConsumersZero
|
||||
expr: max(messenger_worker_consumers{service_name="messenger-api",queue="attachment.process.v1"}) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: RabbitMQ has no attachment consumers
|
||||
description: The API polls the RabbitMQ management API, so this series stays fresh after the worker process stops.
|
||||
runbook: docs/runbooks/queue-lag.md
|
||||
- alert: AttachmentReadyLatencyHigh
|
||||
expr: histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[15m]))) > 10
|
||||
for: 2m
|
||||
labels:
|
||||
severity: ticket
|
||||
slo: time-to-ready
|
||||
annotations:
|
||||
summary: p95 time to ready is above 10s
|
||||
description: Histogram of completed jobs only. A stalled queue does not move this SLO; use QueueLagHigh for that.
|
||||
runbook: docs/runbooks/time-to-ready.md
|
||||
- alert: DatabaseUnready
|
||||
expr: min(messenger_db_up) == 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: An API or worker process cannot use PostgreSQL
|
||||
description: Distinguish this from a slow handler by comparing the db.transaction span with the server span.
|
||||
runbook: docs/runbooks/db-latency.md
|
||||
@@ -0,0 +1,9 @@
|
||||
# SeaweedFS S3 identity
|
||||
|
||||
`s3.json` is generated locally and is not committed:
|
||||
|
||||
```
|
||||
python scripts/render-s3-config.py
|
||||
```
|
||||
|
||||
The script reads `S3_ACCESS_KEY` and `S3_SECRET_KEY` from `.env` and writes this directory's identity file. Compose mounts it read-only at `/etc/seaweed/s3.json`.
|
||||
@@ -0,0 +1,34 @@
|
||||
stream_over_http_enabled: true
|
||||
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
trace_idle_period: 10s
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 48h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/traces
|
||||
|
||||
querier:
|
||||
max_concurrent_queries: 4
|
||||
@@ -0,0 +1,10 @@
|
||||
starlette==0.45.3
|
||||
uvicorn==0.34.0
|
||||
psycopg[binary,pool]==3.2.6
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.37.38
|
||||
pika==1.3.2
|
||||
Pillow==11.1.0
|
||||
opentelemetry-api==1.29.0
|
||||
opentelemetry-sdk==1.29.0
|
||||
opentelemetry-exporter-otlp-proto-http==1.29.0
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Re-check worker-stop alerts and a Tempo outage after the rule fix."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
OUT = ROOT / "evidence" / "incidents.json"
|
||||
|
||||
|
||||
def prom(expr):
|
||||
url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)["data"]["result"]
|
||||
|
||||
|
||||
def alerts():
|
||||
return {
|
||||
item["metric"].get("alertname"): item["metric"].get("alertstate")
|
||||
for item in prom("ALERTS")
|
||||
}
|
||||
|
||||
|
||||
def reload_rules():
|
||||
request = urllib.request.Request("http://127.0.0.1:9090/-/reload", method="POST")
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return response.status
|
||||
|
||||
|
||||
def main():
|
||||
print("reload", reload_rules(), flush=True)
|
||||
client = Client("http://127.0.0.1:8080", timeout=8)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
report = {"alerts_before": alerts()}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
||||
client.attachment(alice, chat)
|
||||
started = time.monotonic()
|
||||
seen = {}
|
||||
samples = []
|
||||
while time.monotonic() - started < 130:
|
||||
state = alerts()
|
||||
age = prom('max(messenger_queue_oldest_age_seconds{service_name="messenger-api"})')
|
||||
consumers = prom('max(messenger_worker_consumers{service_name="messenger-api"})')
|
||||
sample = {
|
||||
"t": round(time.monotonic() - started, 1),
|
||||
"alerts": state,
|
||||
"age": age[0]["value"][1] if age else None,
|
||||
"api_consumers": consumers[0]["value"][1] if consumers else None,
|
||||
}
|
||||
samples.append(sample)
|
||||
for name in ("QueueLagHigh", "WorkerConsumersZero"):
|
||||
if name not in seen and state.get(name) == "firing":
|
||||
seen[name] = sample["t"]
|
||||
if len(seen) == 2:
|
||||
break
|
||||
time.sleep(5)
|
||||
report["while_worker_stopped"] = {"first_firing": seen, "samples": samples[-4:]}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
||||
cleared = {}
|
||||
restore_started = time.monotonic()
|
||||
while time.monotonic() - restore_started < 90:
|
||||
state = alerts()
|
||||
for name in ("QueueLagHigh", "WorkerConsumersZero"):
|
||||
if name not in cleared and name not in state:
|
||||
cleared[name] = round(time.monotonic() - restore_started, 1)
|
||||
if "QueueLagHigh" in cleared and "WorkerConsumersZero" in cleared:
|
||||
break
|
||||
time.sleep(5)
|
||||
report["after_worker_start"] = {"cleared_after_seconds": cleared, "alerts": alerts()}
|
||||
|
||||
before_failed = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
||||
counts = {}
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
time.sleep(0.5)
|
||||
time.sleep(5)
|
||||
memory = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
||||
text=True,
|
||||
).strip()
|
||||
queue = prom("otelcol_exporter_queue_size")
|
||||
failed = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
||||
report["tempo_down_40s"] = {
|
||||
"api_results": counts,
|
||||
"failed_before": [item["value"][1] for item in before_failed],
|
||||
"failed_after": [item["value"][1] for item in failed],
|
||||
"queue_size": [item["value"][1] for item in queue],
|
||||
"collector_memory": memory,
|
||||
}
|
||||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print("wrote", OUT, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
query = '{service_name="messenger-api"} |= "http.request.completed" |= "content"'
|
||||
url = "http://127.0.0.1:3100/loki/api/v1/query_range?query=" + urllib.parse.quote(query) + "&limit=15"
|
||||
data = json.load(urllib.request.urlopen(url, timeout=10))
|
||||
values = []
|
||||
for stream in data["data"]["result"]:
|
||||
values.extend(stream["values"])
|
||||
print("n", len(values))
|
||||
for _, line in values[-8:]:
|
||||
print(line[:500])
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Find the newest worker job.retry / job.ready lines and summarize that trace."""
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
def lines(event):
|
||||
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
||||
payload = json.load(
|
||||
urllib.request.urlopen(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(query)
|
||||
+ "&limit=20",
|
||||
timeout=10,
|
||||
)
|
||||
)
|
||||
found = []
|
||||
for stream in payload["data"]["result"]:
|
||||
for ts, line in stream["values"]:
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if body.get("event") == event:
|
||||
found.append((int(ts), body.get("attempt"), body.get("trace_id")))
|
||||
found.sort()
|
||||
return found[-5:]
|
||||
|
||||
|
||||
print("retry", lines("job.retry"))
|
||||
print("ready", lines("job.ready"))
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Print a short, secret-free snapshot of metrics, one log, and recent traces."""
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def query(expr):
|
||||
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
|
||||
|
||||
|
||||
names = get("http://127.0.0.1:9090/api/v1/label/__name__/values")["data"]
|
||||
app = [name for name in names if name.startswith("http_") or name.startswith("messenger_")]
|
||||
print("NAMES")
|
||||
print("\n".join(app))
|
||||
for expr in (
|
||||
"messenger_jobs_total",
|
||||
"messenger_queue_depth",
|
||||
"messenger_db_up",
|
||||
"messenger_worker_busy",
|
||||
"messenger_worker_consumers",
|
||||
):
|
||||
print(expr, query(expr))
|
||||
|
||||
log_query = '{service_name="messenger-worker"}'
|
||||
logs = get(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(log_query)
|
||||
+ "&limit=2"
|
||||
)
|
||||
streams = logs["data"]["result"]
|
||||
print("LOG_STREAMS", len(streams))
|
||||
if streams:
|
||||
print(streams[0]["values"][-1][1][:600])
|
||||
|
||||
tempo_query = '{resource.service.name="messenger-api"}'
|
||||
traces = get(
|
||||
"http://127.0.0.1:3200/api/search?q=" + urllib.parse.quote(tempo_query) + "&limit=5"
|
||||
)
|
||||
print("TRACES")
|
||||
for item in traces.get("traces", []):
|
||||
print(item.get("rootServiceName"), item.get("rootTraceName"), item.get("traceID"), item.get("durationMs"))
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Summarize one Tempo trace by id. Prints span names and parent links only."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
trace_id = sys.argv[1]
|
||||
payload = json.load(urllib.request.urlopen(f"http://127.0.0.1:3200/api/traces/{trace_id}", timeout=10))
|
||||
batches = payload.get("batches") or payload.get("resourceSpans") or []
|
||||
# Tempo may return OTLP JSON under batches or the classic trace format.
|
||||
if "trace" in payload:
|
||||
spans = payload["trace"].get("spans") or []
|
||||
for span in spans:
|
||||
print(span.get("name") or span.get("operationName"), span.get("spanID"), span.get("parentSpanID"))
|
||||
raise SystemExit
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if "name" in node and ("spanId" in node or "span_id" in node):
|
||||
print(
|
||||
node.get("name"),
|
||||
"id=" + str(node.get("spanId") or node.get("span_id")),
|
||||
"parent=" + str(node.get("parentSpanId") or node.get("parent_span_id") or "-"),
|
||||
)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Write the local SeaweedFS identity file from .env. The file is gitignored."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
env = {}
|
||||
for line in (root / ".env").read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
env[key] = value
|
||||
|
||||
out = root / "infra" / "seaweed" / "s3.json"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
document = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "app",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": env["S3_ACCESS_KEY"],
|
||||
"secretKey": env["S3_SECRET_KEY"],
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Write", "Tagging"],
|
||||
}
|
||||
]
|
||||
}
|
||||
out.write_text(json.dumps(document), encoding="utf-8")
|
||||
print("wrote infra/seaweed/s3.json")
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
wait_healthy() {
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
ok=1
|
||||
for svc in "$@"; do
|
||||
health=$(docker compose --env-file .env ps --format '{{.Service}} {{.Health}}' "$svc" | awk 'NR==1 {print $2}')
|
||||
if [ "$health" != "healthy" ]; then
|
||||
ok=0
|
||||
fi
|
||||
done
|
||||
if [ "$ok" -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "timed out waiting for healthy: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
docker compose --env-file .env restart -t 5 postgres s3 rabbitmq
|
||||
wait_healthy postgres s3 rabbitmq
|
||||
docker compose --env-file .env restart -t 5 api-1 api-2 worker
|
||||
wait_healthy api-1 api-2 worker
|
||||
docker compose --env-file .env restart -t 2 nginx
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Run the public resilience test on Windows by executing the shell hook with Git Bash.
|
||||
|
||||
The public test launches the hook directly. That works on Linux. This runner does not
|
||||
change tests/resilience.py; it only prefixes the same script with bash.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
import resilience # noqa: E402
|
||||
|
||||
BASH = r"C:\Program Files\Git\bin\bash.exe"
|
||||
_popen = resilience.subprocess.Popen
|
||||
|
||||
|
||||
def popen(args, **kwargs):
|
||||
if args and str(args[0]).endswith(".sh"):
|
||||
args = [BASH, str(args[0])]
|
||||
return _popen(args, **kwargs)
|
||||
|
||||
|
||||
resilience.subprocess.Popen = popen
|
||||
resilience.run(resilience.main)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Track 2 measurements. Writes evidence/slo-track2.json without tokens or message text."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
FAULTS = ROOT / "faults"
|
||||
OUT = ROOT / "evidence" / "slo-track2.json"
|
||||
|
||||
|
||||
def get(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def prom(expr):
|
||||
return get("http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr))["data"]["result"]
|
||||
|
||||
|
||||
def alert_states():
|
||||
found = {}
|
||||
for item in prom('ALERTS{alertname=~"Api.*|QueueLagHigh|WorkerConsumersZero|AttachmentReadyLatencyHigh|DatabaseUnready"}'):
|
||||
name = item["metric"].get("alertname")
|
||||
found[name] = item["metric"].get("alertstate")
|
||||
return found
|
||||
|
||||
|
||||
def write_fault(name, value):
|
||||
path = FAULTS / name
|
||||
if value is None:
|
||||
path.unlink(missing_ok=True)
|
||||
else:
|
||||
path.write_text(str(value), encoding="utf-8")
|
||||
|
||||
|
||||
def clear_faults():
|
||||
for name in ("http_error_ratio", "db_slow_ms", "db_fail", "handler_slow_ms", "process_fail_remaining"):
|
||||
write_fault(name, None)
|
||||
|
||||
|
||||
def drive(client, actor, seconds, rps):
|
||||
interval = 1.0 / rps
|
||||
deadline = time.monotonic() + seconds
|
||||
counts = {}
|
||||
while time.monotonic() < deadline:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = client.call("GET", "/api/v1/chats", [200, 500], actor=actor)
|
||||
status = result["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
sleep = interval - (time.monotonic() - started)
|
||||
if sleep > 0:
|
||||
time.sleep(sleep)
|
||||
return counts
|
||||
|
||||
|
||||
def wait_state(name, wanted, timeout):
|
||||
started = time.monotonic()
|
||||
last = {}
|
||||
while time.monotonic() - started < timeout:
|
||||
last = alert_states()
|
||||
if last.get(name) == wanted or (wanted is None and name not in last):
|
||||
return round(time.monotonic() - started, 1), last
|
||||
time.sleep(5)
|
||||
return None, last
|
||||
|
||||
|
||||
def span_summary(trace_id):
|
||||
payload = get(f"http://127.0.0.1:3200/api/traces/{trace_id}")
|
||||
spans = []
|
||||
|
||||
def walk(node):
|
||||
if isinstance(node, dict):
|
||||
if "name" in node and "spanId" in node:
|
||||
start = node.get("startTimeUnixNano")
|
||||
end = node.get("endTimeUnixNano")
|
||||
duration = None
|
||||
try:
|
||||
duration = round((int(end) - int(start)) / 1e6, 1)
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
spans.append(
|
||||
{
|
||||
"name": node.get("name"),
|
||||
"span": node.get("spanId"),
|
||||
"parent": node.get("parentSpanId") or "",
|
||||
"ms": duration,
|
||||
}
|
||||
)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
walk(payload)
|
||||
return spans
|
||||
|
||||
|
||||
def latest_worker_trace(event):
|
||||
query = '{service_name="messenger-worker"} |= "' + event + '"'
|
||||
payload = get(
|
||||
"http://127.0.0.1:3100/loki/api/v1/query_range?query="
|
||||
+ urllib.parse.quote(query)
|
||||
+ "&limit=5"
|
||||
)
|
||||
lines = []
|
||||
for stream in payload["data"]["result"]:
|
||||
for _, line in stream["values"]:
|
||||
try:
|
||||
body = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if body.get("event") == event:
|
||||
lines.append(body)
|
||||
if not lines:
|
||||
return None
|
||||
body = lines[-1]
|
||||
return {
|
||||
"event": body.get("event"),
|
||||
"attempt": body.get("attempt"),
|
||||
"trace_id": body.get("trace_id"),
|
||||
"span_id": body.get("span_id"),
|
||||
"service": body.get("service.name"),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
clear_faults()
|
||||
report = {"started": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
|
||||
client = Client("http://127.0.0.1:8080", timeout=8)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
print("bootstrap ok", flush=True)
|
||||
|
||||
write_fault("process_fail_remaining", "1")
|
||||
attachment, _content = client.attachment(alice, chat)
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=40)
|
||||
time.sleep(3)
|
||||
log = latest_worker_trace("job.ready")
|
||||
spans = []
|
||||
if log and log.get("trace_id"):
|
||||
try:
|
||||
spans = span_summary(log["trace_id"])
|
||||
except Exception as exc:
|
||||
report["retry_trace_error"] = type(exc).__name__
|
||||
report["retry_upload"] = {
|
||||
"final_status": ready["status"],
|
||||
"log": log,
|
||||
"consume_spans": sum(1 for span in spans if span["name"] == "attachment.process consume"),
|
||||
"span_names": [span["name"] for span in spans],
|
||||
"parents_linked": any(span["parent"] for span in spans),
|
||||
}
|
||||
print("retry", report["retry_upload"]["consume_spans"], report["retry_upload"]["log"], flush=True)
|
||||
|
||||
print("warmup", flush=True)
|
||||
report["warmup"] = drive(client, alice, 90, 2)
|
||||
report["alerts_after_warmup"] = alert_states()
|
||||
write_fault("http_error_ratio", "1")
|
||||
print("short spike", flush=True)
|
||||
report["short_spike"] = drive(client, alice, 10, 2)
|
||||
time.sleep(35)
|
||||
report["alerts_after_short_spike"] = alert_states()
|
||||
clear_faults()
|
||||
report["recovery_after_spike"] = drive(client, alice, 20, 2)
|
||||
|
||||
write_fault("http_error_ratio", "1")
|
||||
print("sustained", flush=True)
|
||||
sustained_started = time.monotonic()
|
||||
seen = None
|
||||
counts = {}
|
||||
deadline = time.monotonic() + 100
|
||||
interval = 0.5
|
||||
while time.monotonic() < deadline:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status = client.call("GET", "/api/v1/chats", [200, 500], actor=alice)["status"]
|
||||
except Exception:
|
||||
status = "error"
|
||||
counts[str(status)] = counts.get(str(status), 0) + 1
|
||||
states = alert_states()
|
||||
if seen is None and states.get("ApiErrorBudgetFastBurn") == "firing":
|
||||
seen = round(time.monotonic() - sustained_started, 1)
|
||||
report["fast_burn_detected_after_seconds"] = seen
|
||||
report["alerts_when_fast_burn_fired"] = states
|
||||
break
|
||||
sleep = interval - (time.monotonic() - started)
|
||||
if sleep > 0:
|
||||
time.sleep(sleep)
|
||||
report["sustained_counts_until_detection_or_timeout"] = counts
|
||||
if seen is None:
|
||||
report["fast_burn_detected_after_seconds"] = None
|
||||
report["alerts_after_sustained"] = alert_states()
|
||||
clear_faults()
|
||||
print("resolve", flush=True)
|
||||
resolve_started = time.monotonic()
|
||||
resolved = None
|
||||
resolve_deadline = time.monotonic() + 180
|
||||
while time.monotonic() < resolve_deadline:
|
||||
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
||||
if "ApiErrorBudgetFastBurn" not in alert_states():
|
||||
resolved = round(time.monotonic() - resolve_started, 1)
|
||||
break
|
||||
time.sleep(5)
|
||||
report["fast_burn_resolved_after_seconds"] = resolved
|
||||
report["alerts_after_resolve"] = alert_states()
|
||||
|
||||
print("worker stop", flush=True)
|
||||
import subprocess
|
||||
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "5", "worker"], cwd=ROOT)
|
||||
client.attachment(alice, chat)
|
||||
consumer_wait, consumer_state = wait_state("WorkerConsumersZero", "firing", 150)
|
||||
queue_wait, queue_state = wait_state("QueueLagHigh", "firing", 150)
|
||||
report["worker_stopped"] = {
|
||||
"consumers_firing_after_seconds": consumer_wait,
|
||||
"queue_lag_firing_after_seconds": queue_wait,
|
||||
"alerts": queue_state or consumer_state,
|
||||
}
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "worker"], cwd=ROOT)
|
||||
restored, restored_state = wait_state("WorkerConsumersZero", None, 120)
|
||||
report["worker_restored"] = {"consumers_cleared_after_seconds": restored, "alerts": restored_state}
|
||||
|
||||
print("latency split", flush=True)
|
||||
write_fault("db_slow_ms", "800")
|
||||
client.call("GET", "/api/v1/chats", [200, 500, 503], actor=alice)
|
||||
time.sleep(3)
|
||||
write_fault("db_slow_ms", None)
|
||||
write_fault("handler_slow_ms", "800")
|
||||
client.call("GET", "/api/v1/chats", [200, 500], actor=alice)
|
||||
time.sleep(3)
|
||||
write_fault("handler_slow_ms", None)
|
||||
report["latency_note"] = "db_slow_ms sleeps inside db.transaction; handler_slow_ms sleeps in the HTTP span and outside that span"
|
||||
|
||||
print("tempo down", flush=True)
|
||||
before = prom("otelcol_exporter_send_failed_spans")
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "tempo"], cwd=ROOT)
|
||||
tempo_counts = drive(client, alice, 8, 2)
|
||||
time.sleep(6)
|
||||
after = prom("otelcol_exporter_send_failed_spans")
|
||||
stats = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", "mtusi-lab6-otel-collector-1"],
|
||||
text=True,
|
||||
).strip()
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "tempo"], cwd=ROOT)
|
||||
report["tempo_stopped"] = {
|
||||
"api_results": tempo_counts,
|
||||
"failed_spans_before": [item["value"][1] for item in before],
|
||||
"failed_spans_after": [item["value"][1] for item in after],
|
||||
"collector_memory": stats,
|
||||
}
|
||||
clear_faults()
|
||||
report["finished"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print("wrote", OUT, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
clear_faults()
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1|api-2) ;;
|
||||
*) echo "unexpected instance" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user