Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
442595b37c | ||
|
|
0497b0c217 |
+19
-21
@@ -1,30 +1,28 @@
|
|||||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses.
|
||||||
APP_HOST=0.0.0.0
|
|
||||||
APP_PORT=8080
|
APP_PORT=8080
|
||||||
APP_ENV=lab
|
|
||||||
SESSION_TTL_SECONDS=3600
|
SESSION_TTL_SECONDS=3600
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
|
||||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
|
||||||
SESSION_STORE=postgres
|
|
||||||
CACHE_URL_FILE=/run/secrets/cache_url
|
|
||||||
API_REPLICAS=2
|
|
||||||
IDEMPOTENCY_TTL_SECONDS=86400
|
|
||||||
S3_ENDPOINT=http://s3:9000
|
|
||||||
S3_REGION=us-east-1
|
|
||||||
S3_BUCKET=messenger-attachments
|
|
||||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
|
||||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
|
||||||
MAX_UPLOAD_BYTES=10485760
|
MAX_UPLOAD_BYTES=10485760
|
||||||
MAX_IMAGE_PIXELS=40000000
|
MAX_IMAGE_PIXELS=40000000
|
||||||
BROKER_URL_FILE=/run/secrets/broker_url
|
MAX_PENDING_ATTACHMENTS=64
|
||||||
WORKER_CONCURRENCY=2
|
WORKER_CONCURRENCY=2
|
||||||
JOB_MAX_ATTEMPTS=5
|
JOB_MAX_ATTEMPTS=5
|
||||||
JOB_TIMEOUT_SECONDS=60
|
JOB_TIMEOUT_SECONDS=60
|
||||||
|
JOB_MEMORY_MB=768
|
||||||
STAGING_RETENTION_SECONDS=86400
|
STAGING_RETENTION_SECONDS=86400
|
||||||
OTEL_SERVICE_NAME=messenger-api
|
|
||||||
# Worker must set a distinct service.name.
|
# Choose credentials locally.
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
POSTGRES_DB=messenger
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
POSTGRES_USER=messenger
|
||||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
POSTGRES_PASSWORD=replace-with-local-password
|
||||||
|
CACHE_TTL_SECONDS=30
|
||||||
|
MAX_REQUEST_BYTES=65536
|
||||||
|
MAX_INFLIGHT=32
|
||||||
|
MINIO_ROOT_USER=local-admin
|
||||||
|
MINIO_ROOT_PASSWORD=replace-with-admin-password
|
||||||
|
S3_ACCESS_KEY_ID=messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY=replace-with-api-password
|
||||||
|
|
||||||
|
# Compose sets service.name separately for API, dispatcher and worker.
|
||||||
|
OTEL_SDK_DISABLED=false
|
||||||
|
GRAFANA_ADMIN_PASSWORD=replace-with-grafana-password
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# Shell scripts are mounted into Linux containers, including on Windows hosts.
|
||||||
|
*.sh text eol=lf
|
||||||
@@ -1,18 +1,32 @@
|
|||||||
## Лабораторная 6
|
## Лабораторная 6
|
||||||
|
|
||||||
Автор / группа: TODO
|
Автор / группа: Ромашко Владимир, БВТ2301.
|
||||||
Целевая оценка (3/4/5): TODO
|
|
||||||
Трек 5, если заявлен: TODO
|
|
||||||
|
|
||||||
Что реализовано и что перенесено: TODO
|
Целевая оценка (3/4/5): **4**.
|
||||||
Команды запуска из чистого clone: TODO
|
|
||||||
Результаты проверок и ссылка на REPORT.md: TODO
|
|
||||||
Ограничения / известные ошибки: TODO
|
|
||||||
Источники и использование ИИ: TODO
|
|
||||||
|
|
||||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
Что реализовано и что перенесено: приложение lab5 с API-репликами, БД, S3, брокером и worker сохранено. Добавлены OTLP metrics/logs/traces, Collector, Prometheus/Loki/Tempo, Grafana provisioning, сквозной trace context с retry, SLO и два алерта.
|
||||||
- [ ] Контракт и публичные тесты не ослаблены.
|
|
||||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
Команды запуска из чистого clone:
|
||||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
|
||||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
```sh
|
||||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
cp .env.example .env
|
||||||
|
# Задать локальные пароли в .env.
|
||||||
|
docker compose up -d --build
|
||||||
|
curl http://127.0.0.1:8080/health/ready # дождаться 200
|
||||||
|
python -X utf8 scripts/check_contract.py
|
||||||
|
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||||
|
python -X utf8 student-tests/observability.py --scenario all
|
||||||
|
```
|
||||||
|
|
||||||
|
Результаты проверок: check, публичный smoke и проверка сигналов — PASS; сценарии worker/БД/S3/Tempo пройдены, алерты firing → resolved. Нагрузка: 2389 запросов за 30s, без ошибок. [REPORT.md](../REPORT.md), [вывод](../evidence/verification.txt), [сценарии](../evidence/observability.json), [нагрузка](../evidence/load.json).
|
||||||
|
|
||||||
|
Ограничения / известные ошибки: один узел, без HA/TLS и внешней доставки алертов; при длительном отказе backend допустима потеря телеметрии. Блокирующих ошибок в проверенных сценариях не выявлено.
|
||||||
|
|
||||||
|
Источники и использование ИИ: контракт лабораторной, документация OpenTelemetry и Loki; Codex — реализация, тестирование и оформление. Ссылки в REPORT.md.
|
||||||
|
|
||||||
|
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||||
|
- [x] Контракт и публичные тесты не ослаблены.
|
||||||
|
- [x] make check и make test выполнены эквивалентными Python-командами; приложен реальный вывод.
|
||||||
|
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||||
|
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||||
|
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||||
|
|||||||
+24
@@ -0,0 +1,24 @@
|
|||||||
|
FROM python:3.12.8-slim-bookworm
|
||||||
|
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
|
PYTHONUNBUFFERED=1 \
|
||||||
|
APP_HOST=0.0.0.0 \
|
||||||
|
APP_PORT=8080
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY src ./src
|
||||||
|
COPY migrations ./migrations
|
||||||
|
|
||||||
|
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
||||||
|
USER appuser
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
|
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
|
||||||
|
|
||||||
|
STOPSIGNAL SIGTERM
|
||||||
|
CMD ["python", "-m", "src"]
|
||||||
@@ -1,37 +1,72 @@
|
|||||||
# Отчёт — лабораторная 6
|
# Отчёт — лабораторная 6
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор, группа: Ромашко Владимир, БВТ2301.
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: **4**.
|
||||||
- Commit/PR: TODO после публикации
|
- Источники: [контракт](contracts/observability.md), [OpenTelemetry](https://opentelemetry.io/docs/collector/configuration/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/). Библиотеки: FastAPI, SQLAlchemy, Redis, boto3, Pillow, OpenTelemetry SDK, psutil. Codex использовался для реализации, проверок и оформления.
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Нужны Docker с Compose и Python 3.9+ на хосте; свободные порты 8080, 3000, 9090, 3100, 3200. В контейнерах Python 3.12.8; PostgreSQL 17.4, Redis 7.4.2, Nginx 1.28.0; MinIO закреплён digest. Телеметрия: SDK 1.38.0 / semantic conventions 0.59b0, Collector 0.136.0, Prometheus 3.6.0, Loki 3.5.7, Tempo 2.9.0, Grafana 12.2.0.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD,
|
||||||
|
# S3_SECRET_ACCESS_KEY и GRAFANA_ADMIN_PASSWORD.
|
||||||
|
docker compose config --quiet
|
||||||
|
docker compose up -d --build
|
||||||
|
curl http://127.0.0.1:8080/health/ready # дождаться 200 {"status":"ok"}
|
||||||
|
python -X utf8 scripts/check_contract.py
|
||||||
|
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||||
|
python -X utf8 student-tests/observability.py --scenario all
|
||||||
|
python -X utf8 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||||
|
docker compose down # безопасная остановка, volumes остаются
|
||||||
|
```
|
||||||
|
|
||||||
|
Миграции (включая 008 с trace context) и создание S3 bucket выполняются автоматически. Grafana: http://localhost:3000, `admin`, пароль из `.env`; dashboard **Lab 6 / Messenger / Lab 6** импортируется автоматически. Python-команды check/smoke эквивалентны `make check` / `make test`; на проверявшемся Windows-хосте make отсутствует. Для **намеренного удаления всех учебных данных**: `docker compose down -v` (при проверках не выполнялось).
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
Из lab5 сохранены Nginx → две API-реплики, PostgreSQL, Redis для сессий/кеша, S3, durable Redis Streams, outbox/dispatcher и worker. PostgreSQL — источник истины для заданий; attachment и outbox фиксируются одной транзакцией после записи staging, ack выполняется после устойчивого результата.
|
||||||
|
|
||||||
|
Добавлены API/worker/dispatcher → OTLP → Collector → Prometheus (метрики), Loki (логи), Tempo (трассы) → Grafana. История хранится в volumes, retention 72 часа. W3C context сохраняется в outbox и переносится через broker в worker и image process; retry остаётся в той же трассе. Экспортируются шаблоны маршрутов и технические поля, без SQL-параметров, токенов и содержимого сообщений.
|
||||||
|
|
||||||
|
Метрики: `http_requests_total` и `jobs_processed_total` — счётчики; `http_duration_seconds`, `job_duration_seconds`, `attachment_time_to_ready_seconds` — histogram в секундах; `queue_depth`, `db_pool_used/waiting`, `worker_busy/capacity` — количества; `queue_oldest_age_seconds` — секунды; `container_memory_bytes` / `container_cpu_seconds_total` — память/CPU. Точные имена, labels и запросы: [описание сигналов](infra/otel/README.md#сигналы), [dashboard JSON](infra/grafana/dashboards/messenger.json).
|
||||||
|
|
||||||
|
Переменные `.env.example` подставляются в Compose: пароли — соответствующим сервисам, `MAX_*`, `JOB_*`, `WORKER_CONCURRENCY` — приложению, `APP_PORT` — Nginx. `OTEL_SDK_DISABLED=false` включает телеметрию; service.name и внутренние адреса заданы отдельно по ролям.
|
||||||
|
|
||||||
|
SLI = 1 − 5xx / все запросы без health за **30 минут**, SLO ≥ **99%**. `ApiHighErrorRate`: >5% ошибок за 2m, минимум 20 запросов, `for: 20s`. `QueueLagHigh`: возраст queued >30s при свежем sample, `for: 15s`. Правила и [инструкции диагностики](infra/otel/README.md#queue-lag) сохранены в репозитории.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
| Структура и публичный smoke | Команды check/smoke выше | PASS при включённой телеметрии | [verification.txt](evidence/verification.txt) |
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
| Минимум lab5 и общий стенд | `docker compose up -d --build`; smoke | API, upload 202, ready, оригинал и thumbnail работают; хранилища имеют volumes | [compose.yaml](compose.yaml), verification.txt |
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
| OTLP: метрики, логи, трассы API/worker | `observability.py --scenario smoke` | Все обязательные метрики найдены; связанные логи трёх сервисов | [observability.json: smoke](evidence/observability.json) |
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
| Импортируемый dashboard и безопасные атрибуты | Provisioning; сценарий smoke | 18 панелей, 3 datasource; проверены labels и отсутствие приватных входных данных | [provisioning](infra/grafana/provisioning), observability.json |
|
||||||
|
| Корреляция HTTP/job → log/trace | Сценарий smoke | Найдены request_id, trace_id и parent-связи до S3/DB | observability.json: smoke |
|
||||||
|
| W3C propagation, включая retry | `observability.py --scenario retry` | Попытки 1 и 2 в одной трассе, tracestate сохранён | observability.json: retry |
|
||||||
|
| p50/p95/p99, очередь, DB pool, worker, CPU/RAM, time-to-ready | Сценарий smoke; dashboard | Метрики существуют; percentile вычисляется по сумме histogram buckets | dashboard JSON, observability.json |
|
||||||
|
| SLI/SLO и два проверенных алерта | `--scenario queue`, `--scenario database` | Оба алерта firing → resolved после восстановления | [alerts.yaml](infra/otel/alerts.yaml), observability.json |
|
||||||
|
| Ограниченная буферизация и отказ telemetry backend | `--scenario backend` | API продолжает работу; очередь ≤32 batches, RSS ≤199.03 MiB | [collector.yaml](infra/otel/collector.yaml), observability.json |
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
Здесь `observability.py` — `python -X utf8 student-tests/observability.py`. Сценарии останавливают реальные контейнеры текущего Compose-проекта и восстанавливают их в `finally`; volumes не удаляют.
|
||||||
|
|
||||||
## Эксперименты и отказы
|
## Эксперименты и отказы
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
Проверки 16.11.2026. Гипотеза: телеметрия показывает причину отказа, а её backend не блокирует бизнес-операции. Docker Desktop: 8 CPU, 7.76 GiB RAM; лимиты API — по 1 CPU/512 MiB, worker — 2 CPU/2 GiB, Collector — 0.5 CPU/256 MiB.
|
||||||
|
|
||||||
|
Нагрузка: один прогон closed-loop POST сообщений, 4 клиента, 30 секунд, без отдельного прогрева и автоматических retry. **2389 успешных запросов, 0 ошибок, 79.219 RPS; p50/p95/p99 = 40.784/99.520/129.314 ms**. Исходная сводка: [load.json](evidence/load.json). Это локальная проверка, не исследование предельной производительности; CPU во время прогона отдельно не фиксировался, RSS Collector измерен в сценарии отказа ниже.
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| TODO | TODO | TODO | TODO |
|
| Остановить worker | Рост возраста очереди и QueueLagHigh | Depth=1, возраст 55s; producer есть, consumer появляется после запуска; затем ready, depth=0 | firing → resolved: 20.3s; принятое вложение сохранено |
|
||||||
|
| Остановить PostgreSQL | 503 и ApiHighErrorRate | Error ratio 42.6%; в trace `db.acquire`/`OperationalError` и связанный лог — отказ БД, а не медленный handler | После запуска запросы успешны; firing → resolved: 109.1s с учётом окна 2m |
|
||||||
|
| Временно остановить S3 | Retry без потери задания | Первая попытка failed, вторая ready, общий trace | Задание успешно завершено |
|
||||||
|
| Остановить Tempo на 60s | Приложение работает, буферы ограничены | 165 записей и обработка вложения успешны; очередь ≤32, RSS 198.52–199.02 MiB; после запуска новая трасса найдена | Потеря 808 принятых Collector spans, бизнес-операции успешны |
|
||||||
|
|
||||||
|
Порядок диагностики: dashboard → лог по request_id/trace_id → ошибочный или отсутствующий span зависимости → восстановление сервиса → проверка ready/успешных запросов и resolved. Для Tempo потеря рассчитана как Δaccepted − Δsent после опустошения очереди; повторы отправки не считаются отдельными потерянными spans. Подробные timestamps и ID — в [observability.json](evidence/observability.json).
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения и следующие шаги
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
Один локальный узел, без HA и TLS; алерты только локальные. SDK ограничивает очереди 512 записями, Collector использует memory limiter/batch и retry до 15s: длительный отказ может терять телеметрию. Потери до приёма Collector указанной дельтой не измеряются. История удаляется по retention; метрики не заменяют журнал транзакций. При недоступной БД API возвращает 503, после восстановления работает снова. Треки оценки 5 не реализовывались.
|
||||||
|
|||||||
+313
@@ -0,0 +1,313 @@
|
|||||||
|
x-api: &api
|
||||||
|
image: mtusi2026lab6-api:local
|
||||||
|
pull_policy: never
|
||||||
|
dns_opt:
|
||||||
|
- timeout:1
|
||||||
|
- attempts:1
|
||||||
|
environment: &app-env
|
||||||
|
APP_HOST: 0.0.0.0
|
||||||
|
APP_PORT: 8080
|
||||||
|
POSTGRES_HOST: postgres
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||||
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
|
||||||
|
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
|
||||||
|
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||||
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||||
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||||
|
MAX_PENDING_ATTACHMENTS: ${MAX_PENDING_ATTACHMENTS:-64}
|
||||||
|
BROKER_URL: redis://broker:6379/0
|
||||||
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||||
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||||
|
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||||
|
JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768}
|
||||||
|
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||||
|
OTEL_SERVICE_NAME: messenger-api
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||||
|
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||||
|
networks: [backend]
|
||||||
|
cpus: 1.0
|
||||||
|
mem_limit: 512m
|
||||||
|
stop_grace_period: 10s
|
||||||
|
|
||||||
|
services:
|
||||||
|
broker:
|
||||||
|
image: redis:7.4.2-bookworm
|
||||||
|
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction]
|
||||||
|
volumes:
|
||||||
|
- broker_data:/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, redis-cli, ping]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 256m
|
||||||
|
|
||||||
|
dispatcher:
|
||||||
|
<<: *api
|
||||||
|
environment:
|
||||||
|
<<: *app-env
|
||||||
|
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||||
|
command: [python, -m, src.jobs, dispatch]
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
broker:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: unless-stopped
|
||||||
|
mem_limit: 128m
|
||||||
|
|
||||||
|
worker:
|
||||||
|
<<: *api
|
||||||
|
environment:
|
||||||
|
<<: *app-env
|
||||||
|
OTEL_SERVICE_NAME: messenger-worker
|
||||||
|
command: [python, -m, src.worker]
|
||||||
|
init: true
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
broker:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
cpus: 2.0
|
||||||
|
mem_limit: 2g
|
||||||
|
pids_limit: 64
|
||||||
|
|
||||||
|
api:
|
||||||
|
<<: *api
|
||||||
|
build: .
|
||||||
|
deploy:
|
||||||
|
replicas: 2
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
start_period: 5s
|
||||||
|
retries: 2
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
<<: *api
|
||||||
|
environment:
|
||||||
|
<<: *app-env
|
||||||
|
OTEL_SDK_DISABLED: 'true'
|
||||||
|
command: [python, -m, src.database]
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
restart: 'no'
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
|
||||||
|
s3:
|
||||||
|
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
|
||||||
|
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||||
|
command: [server, /data, --console-address, '127.0.0.1:9001']
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||||
|
MINIO_BROWSER: 'off'
|
||||||
|
volumes:
|
||||||
|
- s3_data:/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 1.0
|
||||||
|
mem_limit: 512m
|
||||||
|
|
||||||
|
s3-init:
|
||||||
|
# The pinned server image also contains mc; no second image is needed.
|
||||||
|
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||||
|
entrypoint: [sh, /bootstrap.sh]
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||||
|
volumes:
|
||||||
|
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
|
||||||
|
- ./infra/s3-policy.json:/policy.json:ro
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
networks: [backend]
|
||||||
|
restart: 'no'
|
||||||
|
cpus: 0.25
|
||||||
|
mem_limit: 128m
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
<<: *api
|
||||||
|
environment:
|
||||||
|
<<: *app-env
|
||||||
|
OTEL_SDK_DISABLED: 'true'
|
||||||
|
command: [python, -m, src.cleanup]
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
s3-init:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
restart: unless-stopped
|
||||||
|
cpus: 0.25
|
||||||
|
mem_limit: 128m
|
||||||
|
|
||||||
|
nginx:
|
||||||
|
image: nginx:1.28.0-alpine
|
||||||
|
entrypoint: [sh, /etc/nginx/readiness.sh]
|
||||||
|
dns_opt:
|
||||||
|
- timeout:1
|
||||||
|
- attempts:1
|
||||||
|
ports:
|
||||||
|
- '127.0.0.1:${APP_PORT:-8080}:8080'
|
||||||
|
volumes:
|
||||||
|
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||||
|
- ./infra/readiness.sh:/etc/nginx/readiness.sh:ro
|
||||||
|
networks: [edge, backend]
|
||||||
|
depends_on:
|
||||||
|
api:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 3
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 128m
|
||||||
|
stop_grace_period: 10s
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
image: postgres:17.4-bookworm
|
||||||
|
environment:
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||||
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 1.0
|
||||||
|
mem_limit: 512m
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7.4.2-bookworm
|
||||||
|
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"]
|
||||||
|
volumes:
|
||||||
|
- redis_data:/data
|
||||||
|
networks: [backend]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 256m
|
||||||
|
|
||||||
|
otel-collector:
|
||||||
|
image: otel/opentelemetry-collector-contrib:0.136.0
|
||||||
|
command: [--config=/etc/otel/collector.yaml]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro
|
||||||
|
networks: [backend]
|
||||||
|
mem_limit: 256m
|
||||||
|
cpus: 0.5
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v3.6.0
|
||||||
|
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=3d]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
|
||||||
|
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||||
|
- prometheus_data:/prometheus
|
||||||
|
ports: ['127.0.0.1:9090:9090']
|
||||||
|
networks: [edge, backend]
|
||||||
|
mem_limit: 512m
|
||||||
|
cpus: 0.5
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:3.5.7
|
||||||
|
command: [-config.file=/etc/loki/local.yaml]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/loki.yaml:/etc/loki/local.yaml:ro
|
||||||
|
- loki_data:/loki
|
||||||
|
ports: ['127.0.0.1:3100:3100']
|
||||||
|
networks: [edge, backend]
|
||||||
|
mem_limit: 384m
|
||||||
|
cpus: 0.5
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
tempo:
|
||||||
|
image: grafana/tempo:2.9.0
|
||||||
|
command: [-config.file=/etc/tempo.yaml]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro
|
||||||
|
- tempo_data:/var/tempo
|
||||||
|
ports: ['127.0.0.1:3200:3200']
|
||||||
|
networks: [edge, backend]
|
||||||
|
mem_limit: 384m
|
||||||
|
cpus: 0.5
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:12.2.0
|
||||||
|
environment:
|
||||||
|
GF_SECURITY_ADMIN_USER: admin
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: 'false'
|
||||||
|
GF_ANALYTICS_REPORTING_ENABLED: 'false'
|
||||||
|
GF_ANALYTICS_CHECK_FOR_UPDATES: 'false'
|
||||||
|
volumes:
|
||||||
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
- grafana_data:/var/lib/grafana
|
||||||
|
ports: ['127.0.0.1:3000:3000']
|
||||||
|
networks: [edge, backend]
|
||||||
|
mem_limit: 384m
|
||||||
|
cpus: 0.5
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
networks:
|
||||||
|
edge:
|
||||||
|
backend:
|
||||||
|
internal: true
|
||||||
|
volumes:
|
||||||
|
broker_data:
|
||||||
|
postgres_data:
|
||||||
|
redis_data:
|
||||||
|
s3_data:
|
||||||
|
prometheus_data:
|
||||||
|
loki_data:
|
||||||
|
tempo_data:
|
||||||
|
grafana_data:
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"lab": 6,
|
||||||
|
"workload": "closed-loop POST messages; no automatic retries",
|
||||||
|
"requested_duration_seconds": 30.0,
|
||||||
|
"elapsed_seconds": 30.157,
|
||||||
|
"concurrency": 4,
|
||||||
|
"successful_requests": 2389,
|
||||||
|
"failed_requests": 0,
|
||||||
|
"error_ratio": 0.0,
|
||||||
|
"successful_rps": 79.219,
|
||||||
|
"successful_latency_ms": {
|
||||||
|
"p50": 40.784,
|
||||||
|
"p95": 99.52,
|
||||||
|
"p99": 129.314
|
||||||
|
},
|
||||||
|
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,5 @@
|
|||||||
|
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
Running smoke
|
||||||
|
PASS: smoke
|
||||||
|
PASS: Grafana provisioning (18 panels, 3 datasources), all dashboard queries, no active alerts; local secrets absent from Git candidates; assignment/contracts/public tests unchanged.
|
||||||
@@ -0,0 +1,716 @@
|
|||||||
|
{
|
||||||
|
"uid": "messenger-lab6",
|
||||||
|
"title": "Messenger / Lab 6",
|
||||||
|
"schemaVersion": 41,
|
||||||
|
"version": 1,
|
||||||
|
"refresh": "5s",
|
||||||
|
"time": {
|
||||||
|
"from": "now-30m",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"tags": [
|
||||||
|
"lab6"
|
||||||
|
],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"title": "API requests / second",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 0,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Business requests only; health probes are excluded.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "sum by (method,route) (rate(http_requests_total{service_name=\"messenger-api\"}[2m]))",
|
||||||
|
"legendFormat": "{{method}} {{route}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"title": "API server error ratio",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 0,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "5xx / all non-health requests, rolling 2m. 4xx are not server errors.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "messenger:http_error_ratio:2m",
|
||||||
|
"legendFormat": "5xx / all"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"title": "API duration p50 / p95 / p99",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 8,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Quantiles from merged histogram buckets across API replicas. Never averages of per-instance percentiles.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "C",
|
||||||
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"title": "Job attempts / second",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 8,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "One count per claimed attempt; failed includes retryable failures, skipped is a fenced attempt.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "sum by (outcome) (rate(jobs_processed_total[2m]))",
|
||||||
|
"legendFormat": "{{outcome}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "ops"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"title": "Worker processing duration p50 / p95 / p99",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 16,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Includes isolated image process and durable result check; excludes queue wait.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "histogram_quantile(0.5, sum by (le) (rate(job_duration_seconds_bucket[2m])))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(job_duration_seconds_bucket[2m])))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "C",
|
||||||
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(job_duration_seconds_bucket[2m])))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"title": "Time from accepted upload to ready",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 16,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Database timestamps: durable attachment creation to ready update. Includes queue wait and retries; observed on successful attempts.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "histogram_quantile(0.5, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))",
|
||||||
|
"legendFormat": "p50"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))",
|
||||||
|
"legendFormat": "p95"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "C",
|
||||||
|
"expr": "histogram_quantile(0.99, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))",
|
||||||
|
"legendFormat": "p99"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"title": "Durable queue depth",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 24,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "PostgreSQL queued attachments, including unpublished outbox/retries; excludes processing jobs.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "max(queue_depth)",
|
||||||
|
"legendFormat": "queued"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"title": "Oldest queued attachment age",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 24,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Age since acceptance. A stale sample is not zero; check queue sample age below.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "max(queue_oldest_age_seconds)",
|
||||||
|
"legendFormat": "age"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 9,
|
||||||
|
"title": "DB pool used / acquiring",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 32,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Per long-lived process. Acquiring includes connection creation and pool wait; max pool size is 10.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "db_pool_used",
|
||||||
|
"legendFormat": "{{service_name}} {{service_instance_id}} used"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "db_pool_waiting",
|
||||||
|
"legendFormat": "{{service_name}} {{service_instance_id}} acquiring"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"title": "Worker saturation",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 32,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Busy consumers / configured concurrency. Missing capacity after 30s indicates stopped worker.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "sum(worker_busy)",
|
||||||
|
"legendFormat": "busy"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "sum(worker_capacity)",
|
||||||
|
"legendFormat": "capacity"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"title": "Container memory (includes image children)",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 40,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "cgroup v2 memory.current. Includes child processes and charged cache.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "container_memory_bytes",
|
||||||
|
"legendFormat": "{{service_name}} {{service_instance_id}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "bytes"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"title": "Container CPU",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 40,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "cgroup v2 CPU seconds / second; 1.0 is one fully used CPU core.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "rate(container_cpu_seconds_total[2m])",
|
||||||
|
"legendFormat": "{{service_name}} {{service_instance_id}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 13,
|
||||||
|
"title": "API availability SLI / SLO",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 48,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Target: >=99% non-5xx business requests in a rolling 30m window. No traffic is not evidence that SLO is met.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "1 - ((sum(increase(http_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[30m])) or vector(0)) / clamp_min(sum(increase(http_requests_total{service_name=\"messenger-api\"}[30m])), 1))",
|
||||||
|
"legendFormat": "SLI over 30m"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "vector(0.99)",
|
||||||
|
"legendFormat": "SLO"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "percentunit"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 14,
|
||||||
|
"title": "Local alerts",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 48,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "1 means pending/firing. Series disappears after resolution. Rules and runbooks are stored in infra/otel.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "ALERTS{alertname=~\"ApiHighErrorRate|QueueLagHigh\"}",
|
||||||
|
"legendFormat": "{{alertname}} {{alertstate}}"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 15,
|
||||||
|
"title": "Collector memory",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 56,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "otelcol_process_memory_rss_bytes",
|
||||||
|
"legendFormat": "RSS"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "bytes"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 16,
|
||||||
|
"title": "Collector exporter queue / capacity",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 56,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Bounded queues: 32 batches per backend, up to 256 records per batch.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "otelcol_exporter_queue_size",
|
||||||
|
"legendFormat": "{{exporter}} queued"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "otelcol_exporter_queue_capacity",
|
||||||
|
"legendFormat": "{{exporter}} capacity"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 17,
|
||||||
|
"title": "Collector failed sends / rejected enqueue",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 64,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Send failures may include retries, not unique lost spans. Exact experiment loss is measured by accepted/sent deltas after draining.",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "rate(otelcol_exporter_send_failed_spans_total[2m])",
|
||||||
|
"legendFormat": "{{exporter}} span send failures"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "B",
|
||||||
|
"expr": "rate(otelcol_exporter_send_failed_log_records_total[2m])",
|
||||||
|
"legendFormat": "{{exporter}} log send failures"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"refId": "C",
|
||||||
|
"expr": "rate(otelcol_exporter_enqueue_failed_spans_total[2m])",
|
||||||
|
"legendFormat": "{{exporter}} span queue rejections"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "ops"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 18,
|
||||||
|
"title": "Queue sample age",
|
||||||
|
"type": "timeseries",
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 64,
|
||||||
|
"w": 12,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"description": "Above 15s: queue measurements are stale (e.g. DB outage).",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": "time() - max(queue_sample_timestamp_seconds)",
|
||||||
|
"legendFormat": "seconds since DB sample"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"links": [
|
||||||
|
{
|
||||||
|
"title": "Explore logs",
|
||||||
|
"url": "/explore?left=%7B%22datasource%22%3A%22loki%22%7D"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title": "Explore traces",
|
||||||
|
"url": "/explore?left=%7B%22datasource%22%3A%22tempo%22%7D"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
providers:
|
||||||
|
- name: messenger
|
||||||
|
folder: Lab 6
|
||||||
|
type: file
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
uid: prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
jsonData:
|
||||||
|
timeInterval: 5s
|
||||||
|
exemplarTraceIdDestinations:
|
||||||
|
- name: trace_id
|
||||||
|
datasourceUid: tempo
|
||||||
|
- name: Loki
|
||||||
|
uid: loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
jsonData:
|
||||||
|
derivedFields:
|
||||||
|
- name: trace_id
|
||||||
|
matcherType: label
|
||||||
|
matcherRegex: trace_id
|
||||||
|
datasourceUid: tempo
|
||||||
|
url: '$${__value.raw}'
|
||||||
|
urlDisplayLabel: View trace
|
||||||
|
- name: Tempo
|
||||||
|
uid: tempo
|
||||||
|
type: tempo
|
||||||
|
access: proxy
|
||||||
|
url: http://tempo:3200
|
||||||
|
jsonData:
|
||||||
|
tracesToLogsV2:
|
||||||
|
datasourceUid: loki
|
||||||
|
spanStartTimeShift: '-5m'
|
||||||
|
spanEndTimeShift: '5m'
|
||||||
|
tags: [{key: service.name, value: service_name}]
|
||||||
|
filterByTraceID: true
|
||||||
|
filterBySpanID: false
|
||||||
|
nodeGraph:
|
||||||
|
enabled: true
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
worker_processes 1;
|
||||||
|
worker_shutdown_timeout 8s;
|
||||||
|
events { worker_connections 1024; }
|
||||||
|
http {
|
||||||
|
upstream messenger {
|
||||||
|
zone messenger 64k;
|
||||||
|
include /etc/nginx/ready-backends.conf;
|
||||||
|
}
|
||||||
|
log_format upstream '$request_method $uri $status upstream=$upstream_addr '
|
||||||
|
'upstream_status=$upstream_status time=$request_time';
|
||||||
|
access_log /var/log/nginx/access.log upstream;
|
||||||
|
server {
|
||||||
|
listen 8080;
|
||||||
|
# API enforces the byte limit and returns the contracted JSON error.
|
||||||
|
client_max_body_size 0;
|
||||||
|
client_body_timeout 5s;
|
||||||
|
client_header_timeout 5s;
|
||||||
|
keepalive_timeout 5s;
|
||||||
|
location / {
|
||||||
|
proxy_pass http://messenger;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_request_buffering off;
|
||||||
|
proxy_buffering off;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header Connection close;
|
||||||
|
proxy_connect_timeout 1s;
|
||||||
|
proxy_send_timeout 35s;
|
||||||
|
proxy_read_timeout 120s;
|
||||||
|
# A 503 is an API response (for example, S3 is unavailable), so
|
||||||
|
# preserve it for the client instead of marking every replica down.
|
||||||
|
proxy_next_upstream error timeout http_502 http_504;
|
||||||
|
proxy_next_upstream_tries 2;
|
||||||
|
proxy_next_upstream_timeout 8s;
|
||||||
|
# Never transparently retry a POST already sent to an API.
|
||||||
|
# A client retries messages using the original Idempotency-Key.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+137
-4
@@ -1,7 +1,140 @@
|
|||||||
# Конфигурация наблюдаемости — реализовать
|
# Лаба 6: запуск и диагностика
|
||||||
|
|
||||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
# Задать локальные пароли в .env (в Git не добавлять).
|
||||||
|
docker compose up -d --build
|
||||||
|
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||||
|
python -X utf8 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||||
|
python -X utf8 student-tests/observability.py --scenario all
|
||||||
|
```
|
||||||
|
|
||||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
Последняя команда создаёт синтетические данные и временно останавливает **только
|
||||||
|
сервисы текущего Compose-проекта**. В `finally` запускает остановленные сервисы.
|
||||||
|
Не запускать параллельно с другими проверками этого стенда. Volumes не удаляются.
|
||||||
|
Санитизированные результаты сохраняются в `evidence/observability.json`.
|
||||||
|
|
||||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
Grafana: http://localhost:3000, пользователь `admin`, пароль `GRAFANA_ADMIN_PASSWORD`
|
||||||
|
из `.env`. Dashboard **Lab 6 / Messenger / Lab 6** и три datasource загружаются
|
||||||
|
автоматически. Prometheus: http://localhost:9090; Loki API: порт 3100; Tempo API:
|
||||||
|
3200. Все опубликованные порты слушают только loopback, OTLP наружу не опубликован.
|
||||||
|
|
||||||
|
Collector принимает OTLP/HTTP от API, dispatcher и worker. Prometheus хранит
|
||||||
|
метрики, Loki — логи, Tempo — трассы; Grafana обращается к этим хранилищам.
|
||||||
|
История сохраняется в именованных volumes, retention — 72 часа.
|
||||||
|
Версии закреплены в Compose и requirements.txt: OTel Python 1.38.0, semantic
|
||||||
|
conventions 0.59b0 (зависимость SDK); наши имена метрик перечислены ниже.
|
||||||
|
Это явная инструментация: auto-instrumentation, сбор SQL и HTTP headers отключены.
|
||||||
|
|
||||||
|
## Сигналы
|
||||||
|
|
||||||
|
| Prometheus name | Единицы и смысл |
|
||||||
|
| --- | --- |
|
||||||
|
| `http_requests_total` | Число бизнес-запросов; method, route template, status_class |
|
||||||
|
| `http_duration_seconds` | Histogram полного HTTP-запроса, включая отдачу stream |
|
||||||
|
| `jobs_processed_total` | Попытки обработки; job_type, outcome=ready/failed/skipped |
|
||||||
|
| `job_duration_seconds` | Histogram времени попытки, без ожидания в очереди |
|
||||||
|
| `attachment_time_to_ready_seconds` | Histogram от записи attachment до durable ready, включая retry |
|
||||||
|
| `queue_depth` | Количество queued в PostgreSQL, включая outbox и отложенные retry |
|
||||||
|
| `queue_oldest_age_seconds` | Возраст старейшего queued от момента приёма, не время обработки |
|
||||||
|
| `queue_sample_timestamp_seconds` | Unix-время последнего успешного опроса очереди |
|
||||||
|
| `db_pool_used`, `db_pool_waiting` | Занятые соединения / потоки, получающие соединение (ожидание + reconnect) |
|
||||||
|
| `worker_busy`, `worker_capacity` | Число занятых consumer / лимит concurrency |
|
||||||
|
| `process_memory_bytes`, `process_cpu_seconds_total` | RSS и CPU процесса |
|
||||||
|
| `container_memory_bytes`, `container_cpu_seconds_total` | cgroup v2, включая дочерние процессы обработки |
|
||||||
|
|
||||||
|
Histogram buckets, секунды: .005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10,
|
||||||
|
30, 60, 120, 300, +Inf. Dashboard содержит PromQL, единицы и описания каждого
|
||||||
|
графика. p50/p95/p99 считаются `histogram_quantile` по сумме buckets всех реплик.
|
||||||
|
Метрики очереди отдаёт только dispatcher; это не длина Redis Stream (в нём
|
||||||
|
ещё могут быть уже взятые сообщения). При недоступной БД sample становится
|
||||||
|
устаревшим; возраст sample выведен отдельно, QueueLagHigh не использует stale data.
|
||||||
|
|
||||||
|
Resource: service.name, service.version=1.6.0, service.instance.id=hostname,
|
||||||
|
deployment.environment.name=lab. API: messenger-api; worker и его дочерний
|
||||||
|
процесс: messenger-worker; publisher: messenger-dispatcher. Дочерний процесс
|
||||||
|
экспортирует только traces/logs, чтобы не создавать отдельные краткоживущие
|
||||||
|
ряды метрик. Parent измеряет длительность попытки и результат после COMMIT.
|
||||||
|
Метрики наблюдательные: авария после COMMIT до записи счётчика может потерять
|
||||||
|
измерение, но не результат обработки. Предметных ID в labels метрик нет.
|
||||||
|
|
||||||
|
## Связь сигналов
|
||||||
|
|
||||||
|
SERVER span API → traceparent/tracestate в transactional outbox → PRODUCER span
|
||||||
|
dispatcher → W3C context в JSON сообщения Redis → CONSUMER span worker → context
|
||||||
|
через окружение изолированного image process → S3/DB CLIENT spans. Retry публикует
|
||||||
|
новый producer/consumer в той же трассе, с `job.attempt`. Baggage не переносится.
|
||||||
|
Context сохранён в миграции 008, поэтому рестарт API не разрывает причинную связь.
|
||||||
|
|
||||||
|
Логи содержат timestamp/severity/service.name/event и trace_id/span_id активного
|
||||||
|
span. `http_request` содержит ограниченный `request_id`, совпадающий с
|
||||||
|
X-Request-Id ответа. Loki сохраняет технические поля как structured metadata;
|
||||||
|
индексирует только service.name и окружение. Для корреляции достаточно request_id,
|
||||||
|
trace_id и номера попытки: имена файлов и предметные ID не экспортируются.
|
||||||
|
|
||||||
|
В Grafana Explore / Loki:
|
||||||
|
|
||||||
|
```logql
|
||||||
|
{service_name="messenger-api"} | request_id="ЗНАЧЕНИЕ_X_REQUEST_ID"
|
||||||
|
{service_name=~"messenger-.*"} | trace_id="TRACE_ID"
|
||||||
|
```
|
||||||
|
|
||||||
|
Поле trace_id открывает Tempo; из Tempo доступен переход к логам. В трассе upload
|
||||||
|
видны `POST /api/v1/chats/{chat_id}/attachments`, `attachment-jobs publish`,
|
||||||
|
`attachment-jobs process`, `image.process`, `s3.GetObject`, `s3.PutObject`,
|
||||||
|
`db.SELECT`/`db.UPDATE`. Исключения содержат только тип, без message/stack/SQL.
|
||||||
|
|
||||||
|
## SLI/SLO и alerts
|
||||||
|
|
||||||
|
SLI доступности = 1 − число 5xx / число всех бизнес-запросов за скользящие **30 минут**.
|
||||||
|
SLO: **не менее 99%**. Health исключён из числителя и знаменателя; 4xx не считается
|
||||||
|
отказом сервера. Без запросов SLI не оцениваем. Это учебное окно, а не месячный SLO.
|
||||||
|
|
||||||
|
`ApiHighErrorRate`: 5xx > 5% за 2 минуты, минимум 20 запросов, `for: 20s`.
|
||||||
|
`QueueLagHigh`: queued > 0 и возраст > 30s, свежий sample, `for: 15s`.
|
||||||
|
Пороги короткие для воспроизводимой локальной демонстрации; алерты не являются
|
||||||
|
burn-rate исследованием. Rules вычисляет Prometheus; firing/pending видны в
|
||||||
|
http://localhost:9090/alerts и на dashboard. После восстановления alert пропадает
|
||||||
|
из instant `ALERTS`; история переходов остаётся в range query. Доставка наружу не настроена.
|
||||||
|
|
||||||
|
## Queue lag
|
||||||
|
|
||||||
|
1. Открыть queue depth/age и worker capacity/busy. Растущая очередь при отсутствии
|
||||||
|
capacity означает остановку worker; свежий sample подтверждает доступность БД.
|
||||||
|
2. В Loki найти `job_published`, затем открыть trace: producer есть, consumer ещё нет.
|
||||||
|
3. `docker compose start worker`; дождаться `job_processed` outcome=ready, depth=0.
|
||||||
|
4. Проверить, что QueueLagHigh больше не firing и в той же трассе появился consumer.
|
||||||
|
|
||||||
|
## API errors
|
||||||
|
|
||||||
|
1. При ApiHighErrorRate найти http_request со status_class=5xx и взять trace_id.
|
||||||
|
2. В Tempo проверить `db.acquire`/`db.SELECT`: error.type и длительность покажут
|
||||||
|
отказ подключения/БД. В Loki есть `db_error`/`operation_failed`. Медленный handler
|
||||||
|
без отказа зависимости не имел бы ошибочного DB span. Посмотреть db_pool_waiting.
|
||||||
|
3. Проверить PostgreSQL: `docker compose ps postgres`; восстановить
|
||||||
|
`docker compose start postgres`, дождаться readiness и успешного бизнес-запроса.
|
||||||
|
4. Через окно 2m error ratio снизится, alert перейдёт в resolved. Sample очереди
|
||||||
|
тоже должен стать свежим. Сценарий проверки сохраняет трассу ошибочного запроса.
|
||||||
|
|
||||||
|
## Недоступность telemetry backend
|
||||||
|
|
||||||
|
SDK: очереди по 512 spans/logs, batch 64, HTTP timeout 1s; metric export каждые 5s.
|
||||||
|
Отправка выполняется в фоновых потоках. Collector: memory limiter 192 MiB,
|
||||||
|
spike 48 MiB, container limit 256 MiB; batch ≤256 записей; отдельная очередь
|
||||||
|
32 batches / 2 consumers на Loki и Tempo; retry до 15s, timeout 2s, overflow
|
||||||
|
не блокирует приём. Конечные ограничения защищают память, но допускают потерю
|
||||||
|
телеметрии при длительном отказе. API и pipeline не зависят от готовности backend.
|
||||||
|
|
||||||
|
Сценарий `backend` останавливает Tempo, продолжает создавать сообщения и
|
||||||
|
обрабатывать изображения, снимает RSS/очередь Collector, затем запускает Tempo
|
||||||
|
и проверяет новую трассу. `send_failed_spans` может считать повторные попытки:
|
||||||
|
его нельзя выдавать за число уникально потерянных spans. Для отдельного окна
|
||||||
|
тест вычисляет accepted − sent после опустошения очереди (без рестартов Collector)
|
||||||
|
и сохраняет эти дельты вместе с наблюдаемой памятью. Это потери уже принятых
|
||||||
|
Collector spans; отброшенные до приёма SDK/receiver в этот счёт не входят.
|
||||||
|
Prometheus всё это время
|
||||||
|
работает, поэтому сохраняется история отказа и восстановления.
|
||||||
|
|
||||||
|
Первичные источники: [Collector](https://opentelemetry.io/docs/collector/configuration/),
|
||||||
|
[native Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
|
||||||
|
[bounded exporter queues](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.136.0/exporter/exporterhelper/README.md).
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
groups:
|
||||||
|
- name: messenger
|
||||||
|
rules:
|
||||||
|
- record: messenger:http_error_ratio:2m
|
||||||
|
expr: |
|
||||||
|
(sum(rate(http_requests_total{service_name="messenger-api",status_class="5xx"}[2m])) or vector(0))
|
||||||
|
/ clamp_min(sum(rate(http_requests_total{service_name="messenger-api"}[2m])), 0.001)
|
||||||
|
- alert: ApiHighErrorRate
|
||||||
|
expr: messenger:http_error_ratio:2m > 0.05 and sum(increase(http_requests_total{service_name="messenger-api"}[2m])) >= 20
|
||||||
|
for: 20s
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: 'API: more than 5% server errors over 2 minutes'
|
||||||
|
description: 'SLI is 5xx / all non-health requests. At least 20 requests in the window.'
|
||||||
|
runbook: 'infra/otel/README.md#api-errors'
|
||||||
|
- alert: QueueLagHigh
|
||||||
|
expr: |
|
||||||
|
max(queue_oldest_age_seconds) > 30
|
||||||
|
and max(queue_depth) > 0
|
||||||
|
and time() - max(queue_sample_timestamp_seconds) < 15
|
||||||
|
for: 15s
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
annotations:
|
||||||
|
summary: 'Oldest queued attachment waits more than 30 seconds'
|
||||||
|
description: 'Check worker capacity/busy and job errors; backlog is sampled by dispatcher.'
|
||||||
|
runbook: 'infra/otel/README.md#queue-lag'
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
processors:
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 192
|
||||||
|
spike_limit_mib: 48
|
||||||
|
batch:
|
||||||
|
timeout: 1s
|
||||||
|
send_batch_size: 128
|
||||||
|
send_batch_max_size: 256
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
prometheus:
|
||||||
|
endpoint: 0.0.0.0:8889
|
||||||
|
resource_to_telemetry_conversion:
|
||||||
|
enabled: true
|
||||||
|
enable_open_metrics: true
|
||||||
|
metric_expiration: 30s
|
||||||
|
translation_strategy: UnderscoreEscapingWithSuffixes
|
||||||
|
otlphttp/loki:
|
||||||
|
endpoint: http://loki:3100/otlp
|
||||||
|
timeout: 2s
|
||||||
|
sending_queue:
|
||||||
|
queue_size: 32
|
||||||
|
num_consumers: 2
|
||||||
|
block_on_overflow: false
|
||||||
|
retry_on_failure:
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 5s
|
||||||
|
max_elapsed_time: 15s
|
||||||
|
otlphttp/tempo:
|
||||||
|
endpoint: http://tempo:4318
|
||||||
|
timeout: 2s
|
||||||
|
sending_queue:
|
||||||
|
queue_size: 32
|
||||||
|
num_consumers: 2
|
||||||
|
block_on_overflow: false
|
||||||
|
retry_on_failure:
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 5s
|
||||||
|
max_elapsed_time: 15s
|
||||||
|
|
||||||
|
service:
|
||||||
|
telemetry:
|
||||||
|
metrics:
|
||||||
|
readers:
|
||||||
|
- pull:
|
||||||
|
exporter:
|
||||||
|
prometheus:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 8888
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [prometheus]
|
||||||
|
logs:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlphttp/loki]
|
||||||
|
traces:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlphttp/tempo]
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
common:
|
||||||
|
path_prefix: /loki
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2024-01-01
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
limits_config:
|
||||||
|
allow_structured_metadata: true
|
||||||
|
retention_period: 72h
|
||||||
|
otlp_config:
|
||||||
|
resource_attributes:
|
||||||
|
ignore_defaults: true
|
||||||
|
attributes_config:
|
||||||
|
- action: index_label
|
||||||
|
attributes: [service.name, deployment.environment.name]
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
retention_enabled: true
|
||||||
|
delete_request_store: filesystem
|
||||||
|
analytics:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 5s
|
||||||
|
evaluation_interval: 5s
|
||||||
|
rule_files:
|
||||||
|
- /etc/prometheus/alerts.yaml
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: messenger
|
||||||
|
static_configs:
|
||||||
|
- targets: ['otel-collector:8889']
|
||||||
|
- job_name: collector
|
||||||
|
static_configs:
|
||||||
|
- targets: ['otel-collector:8888']
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
server:
|
||||||
|
http_listen_port: 3200
|
||||||
|
distributor:
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
ingester:
|
||||||
|
trace_idle_period: 10s
|
||||||
|
max_block_duration: 5m
|
||||||
|
compactor:
|
||||||
|
compaction:
|
||||||
|
block_retention: 72h
|
||||||
|
storage:
|
||||||
|
trace:
|
||||||
|
backend: local
|
||||||
|
wal:
|
||||||
|
path: /var/tempo/wal
|
||||||
|
local:
|
||||||
|
path: /var/tempo/blocks
|
||||||
|
usage_report:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Only ready API addresses enter the upstream, including at nginx startup.
|
||||||
|
update_backends() {
|
||||||
|
next=/etc/nginx/ready-backends.conf.next
|
||||||
|
: > "$next"
|
||||||
|
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
|
||||||
|
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
|
||||||
|
echo "server $address:8080;" >> "$next"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
# If every running API temporarily loses a shared dependency (for example
|
||||||
|
# S3), retain the last addresses. The API can then return the contracted
|
||||||
|
# 503 instead of Nginx replacing it with a 502. At initial startup there
|
||||||
|
# is no previous healthy set, so use an inert placeholder.
|
||||||
|
if [ ! -s "$next" ]; then
|
||||||
|
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
|
||||||
|
rm -f "$next"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
echo 'server 127.0.0.1:9 down;' > "$next"
|
||||||
|
fi
|
||||||
|
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
|
||||||
|
mv "$next" /etc/nginx/ready-backends.conf
|
||||||
|
if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then
|
||||||
|
nginx -s reload
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
rm -f "$next"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
update_backends
|
||||||
|
(
|
||||||
|
while sleep 2; do
|
||||||
|
update_backends reload
|
||||||
|
done
|
||||||
|
) &
|
||||||
|
exec nginx -g 'daemon off;'
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
|
||||||
|
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||||
|
mc mb --ignore-existing local/messenger-attachments
|
||||||
|
mc anonymous set none local/messenger-attachments
|
||||||
|
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
|
||||||
|
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
|
||||||
|
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||||
|
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*",
|
||||||
|
"arn:aws:s3:::messenger-attachments/staging/*",
|
||||||
|
"arn:aws:s3:::messenger-attachments/variants/*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||||
|
password_hash text NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE chat_members (
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
PRIMARY KEY (chat_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||||
|
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
sender_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||||
|
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0;
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Additive migration: no rewrite of existing lab2 tables or messages.
|
||||||
|
CREATE TABLE message_idempotency (
|
||||||
|
actor_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||||
|
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
PRIMARY KEY (actor_id, chat_id, key)
|
||||||
|
);
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE attachments (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||||
|
owner_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
object_key text NOT NULL UNIQUE,
|
||||||
|
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||||
|
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||||
|
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||||
|
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
|
||||||
|
|
||||||
|
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||||
|
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||||
|
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||||
|
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
|
||||||
|
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
ALTER TABLE attachments
|
||||||
|
ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
ADD COLUMN status text NOT NULL DEFAULT 'ready'
|
||||||
|
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||||
|
ADD COLUMN staging_key text,
|
||||||
|
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
|
||||||
|
ADD COLUMN error_code text;
|
||||||
|
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||||
|
|
||||||
|
CREATE TABLE attachment_jobs (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||||
|
pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
|
||||||
|
attempt bigint NOT NULL DEFAULT 0,
|
||||||
|
failures bigint NOT NULL DEFAULT 0,
|
||||||
|
available_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
published_at timestamptz,
|
||||||
|
lease_until timestamptz,
|
||||||
|
lease_token uuid,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
replay_audit jsonb NOT NULL DEFAULT '[]'
|
||||||
|
);
|
||||||
|
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE attachment_jobs ADD COLUMN last_error text;
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz;
|
||||||
|
CREATE INDEX attachments_pending_idx ON attachments(status)
|
||||||
|
WHERE status IN ('queued', 'processing');
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}';
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
fastapi==0.115.6
|
||||||
|
uvicorn==0.34.0
|
||||||
|
pydantic==2.13.5
|
||||||
|
starlette==0.41.3
|
||||||
|
annotated-types==0.8.0
|
||||||
|
anyio==4.15.1
|
||||||
|
click==8.5.0
|
||||||
|
colorama==0.4.6; sys_platform == "win32"
|
||||||
|
h11==0.16.0
|
||||||
|
idna==3.20
|
||||||
|
pydantic_core==2.46.5
|
||||||
|
typing-inspection==0.4.4
|
||||||
|
typing_extensions==4.16.0
|
||||||
|
psycopg[binary]==3.2.6
|
||||||
|
SQLAlchemy==2.0.41
|
||||||
|
redis==5.2.1
|
||||||
|
argon2-cffi==23.1.0
|
||||||
|
boto3==1.40.70
|
||||||
|
Pillow==12.0.0
|
||||||
|
opentelemetry-api==1.38.0
|
||||||
|
opentelemetry-sdk==1.38.0
|
||||||
|
opentelemetry-exporter-otlp-proto-http==1.38.0
|
||||||
|
psutil==7.1.2
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
import uvicorn
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
uvicorn.run(
|
||||||
|
'src.main:app',
|
||||||
|
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||||
|
port=int(os.getenv('APP_PORT', '8080')),
|
||||||
|
workers=1,
|
||||||
|
access_log=False, # Structured logs use route templates, never query strings.
|
||||||
|
timeout_graceful_shutdown=5,
|
||||||
|
)
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
"""A shared admission limit for unfinished attachments across all API replicas."""
|
||||||
|
import os
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from .errors import APIError
|
||||||
|
from .models import Attachment
|
||||||
|
|
||||||
|
MAX_PENDING = int(os.getenv('MAX_PENDING_ATTACHMENTS', '64'))
|
||||||
|
if MAX_PENDING < 1:
|
||||||
|
raise ValueError('MAX_PENDING_ATTACHMENTS must be positive')
|
||||||
|
|
||||||
|
|
||||||
|
def reserve_capacity(session):
|
||||||
|
# Keep the lock until the new queued row commits. Counting without this lock
|
||||||
|
# would allow simultaneous uploads on different replicas to exceed the limit.
|
||||||
|
session.execute(select(func.pg_advisory_xact_lock(202605)))
|
||||||
|
count = session.scalar(select(func.count()).select_from(Attachment).where(
|
||||||
|
Attachment.status.in_(['queued', 'processing'])))
|
||||||
|
if count >= MAX_PENDING:
|
||||||
|
raise APIError(429, 'rate_limited', 'Очередь обработки заполнена; повторите загрузку позже')
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import os
|
||||||
|
import unicodedata
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
from .errors import APIError
|
||||||
|
from .admission import reserve_capacity
|
||||||
|
from .models import Attachment, AttachmentJob, ChatMember
|
||||||
|
from .object_storage import BUCKET, PREFIX, s3_client
|
||||||
|
from .store import ensure_uuid, serialize, store
|
||||||
|
from .telemetry import carrier
|
||||||
|
|
||||||
|
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
|
||||||
|
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
|
||||||
|
if MAX_BYTES < 1 or MAX_PIXELS < 1:
|
||||||
|
raise ValueError('Upload limits must be positive')
|
||||||
|
|
||||||
|
|
||||||
|
def validate_filename(filename):
|
||||||
|
if (not filename or not filename.strip() or len(filename) > 255
|
||||||
|
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверное имя файла')
|
||||||
|
|
||||||
|
|
||||||
|
def validate_image(source, content_type):
|
||||||
|
try:
|
||||||
|
source.seek(0)
|
||||||
|
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||||
|
if picture.width * picture.height > MAX_PIXELS:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
|
||||||
|
if Image.MIME[picture.format] != content_type:
|
||||||
|
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
|
||||||
|
picture.verify()
|
||||||
|
source.seek(0)
|
||||||
|
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||||
|
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
|
||||||
|
except Image.DecompressionBombError as exc:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
|
||||||
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||||
|
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
|
||||||
|
source.seek(0)
|
||||||
|
|
||||||
|
|
||||||
|
def lock_object(session, key):
|
||||||
|
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
|
||||||
|
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
|
||||||
|
|
||||||
|
|
||||||
|
def metadata(attachment):
|
||||||
|
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
|
||||||
|
'content_type', 'size_bytes', 'sha256', 'created_at',
|
||||||
|
'updated_at', 'status', 'error_code')
|
||||||
|
return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'}
|
||||||
|
for item in attachment.variants]}
|
||||||
|
|
||||||
|
|
||||||
|
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
|
||||||
|
validate_image(source, content_type)
|
||||||
|
attachment_id = str(uuid4())
|
||||||
|
key = PREFIX + attachment_id
|
||||||
|
staging_key = 'staging/' + attachment_id
|
||||||
|
with store.sessions.begin() as session:
|
||||||
|
store._chat(session, chat_id, actor_id)
|
||||||
|
reserve_capacity(session)
|
||||||
|
lock_object(session, staging_key)
|
||||||
|
s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source,
|
||||||
|
ContentLength=size, ContentType=content_type)
|
||||||
|
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
|
||||||
|
object_key=key, filename=filename, content_type=content_type,
|
||||||
|
size_bytes=size, sha256=digest, staging_key=staging_key)
|
||||||
|
session.add(attachment)
|
||||||
|
session.flush()
|
||||||
|
session.add(AttachmentJob(attachment_id=attachment_id, trace_context=carrier()))
|
||||||
|
result = metadata(attachment)
|
||||||
|
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def get_attachment(attachment_id, actor_id):
|
||||||
|
with store.sessions() as session:
|
||||||
|
attachment = session.scalar(select(Attachment).join(
|
||||||
|
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
|
||||||
|
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
|
||||||
|
if attachment is None:
|
||||||
|
raise APIError(404, 'not_found', 'Объект не найден')
|
||||||
|
return attachment
|
||||||
+68
@@ -0,0 +1,68 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from argon2 import PasswordHasher
|
||||||
|
from argon2.exceptions import VerificationError
|
||||||
|
from redis import Redis
|
||||||
|
|
||||||
|
from .errors import APIError
|
||||||
|
|
||||||
|
password_hasher = PasswordHasher()
|
||||||
|
dummy_hash = password_hasher.hash(secrets.token_urlsafe(32))
|
||||||
|
|
||||||
|
|
||||||
|
def verify_password(encoded, password):
|
||||||
|
try:
|
||||||
|
return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None
|
||||||
|
except VerificationError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def unauthorized():
|
||||||
|
return APIError(401, 'unauthorized', 'Нет действительной идентификации')
|
||||||
|
|
||||||
|
|
||||||
|
class Sessions:
|
||||||
|
def __init__(self):
|
||||||
|
self.client = Redis.from_url(
|
||||||
|
os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||||
|
socket_connect_timeout=1, socket_timeout=1, max_connections=20,
|
||||||
|
retry_on_timeout=False,
|
||||||
|
)
|
||||||
|
self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
|
||||||
|
if self.ttl < 1:
|
||||||
|
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def key(token):
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||||
|
raise unauthorized()
|
||||||
|
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
|
||||||
|
|
||||||
|
def create(self, user):
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl)
|
||||||
|
expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||||
|
self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}),
|
||||||
|
pxat=int(expires.timestamp() * 1000))
|
||||||
|
return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user}
|
||||||
|
|
||||||
|
def user_id(self, token):
|
||||||
|
raw = self.client.get(self.key(token))
|
||||||
|
if raw is None:
|
||||||
|
raise unauthorized()
|
||||||
|
session = json.loads(raw)
|
||||||
|
if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc):
|
||||||
|
raise unauthorized()
|
||||||
|
return session['user_id']
|
||||||
|
|
||||||
|
def delete(self, token):
|
||||||
|
if not self.client.delete(self.key(token)):
|
||||||
|
raise unauthorized()
|
||||||
|
|
||||||
|
|
||||||
|
sessions = Sessions()
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
"""History pages are disposable; PostgreSQL owns both data and their version."""
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
|
||||||
|
from .auth import sessions
|
||||||
|
|
||||||
|
|
||||||
|
class HistoryCache:
|
||||||
|
def __init__(self, client, ttl=30, prefix='history'):
|
||||||
|
self.client = client
|
||||||
|
self.ttl = ttl
|
||||||
|
self.prefix = prefix
|
||||||
|
if ttl < 1:
|
||||||
|
raise ValueError('CACHE_TTL_SECONDS must be positive')
|
||||||
|
|
||||||
|
def key(self, chat_id, version, limit, after):
|
||||||
|
page = json.dumps([limit, after], separators=(',', ':'))
|
||||||
|
digest = hashlib.sha256(page.encode()).hexdigest()
|
||||||
|
return f'{self.prefix}:{chat_id}:{version}:{digest}'
|
||||||
|
|
||||||
|
def get_or_load(self, chat_id, version, limit, after, load):
|
||||||
|
key = self.key(chat_id, version, limit, after)
|
||||||
|
try:
|
||||||
|
cached = self.client.get(key)
|
||||||
|
except RedisError:
|
||||||
|
return load()
|
||||||
|
if cached is not None:
|
||||||
|
return json.loads(cached)
|
||||||
|
page = load()
|
||||||
|
try:
|
||||||
|
self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl)
|
||||||
|
except RedisError:
|
||||||
|
pass
|
||||||
|
return page
|
||||||
|
|
||||||
|
|
||||||
|
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from .attachments import lock_object
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .models import Attachment
|
||||||
|
from .object_storage import BUCKET, s3_client
|
||||||
|
|
||||||
|
GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))))
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup(sessions, client, now=None):
|
||||||
|
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
||||||
|
deleted = 0
|
||||||
|
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET):
|
||||||
|
for obj in page.get('Contents', []):
|
||||||
|
key = obj['Key']
|
||||||
|
if not key.startswith(('staging/', 'originals/', 'variants/')):
|
||||||
|
continue
|
||||||
|
with sessions.begin() as session:
|
||||||
|
lock_object(session, key)
|
||||||
|
if key.startswith('staging/'):
|
||||||
|
attachment = session.scalar(select(Attachment).where(
|
||||||
|
Attachment.staging_key == key).with_for_update())
|
||||||
|
if attachment:
|
||||||
|
if attachment.status in ('queued', 'processing'):
|
||||||
|
continue
|
||||||
|
if attachment.status == 'failed' and attachment.updated_at >= cutoff:
|
||||||
|
continue
|
||||||
|
client.delete_object(Bucket=BUCKET, Key=key)
|
||||||
|
attachment.staging_key = None
|
||||||
|
deleted += 1
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
# All final keys belong to a server-generated attachment UUID.
|
||||||
|
from uuid import UUID
|
||||||
|
try:
|
||||||
|
attachment_id = str(UUID(key.split('/')[1]))
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
continue
|
||||||
|
attachment = session.get(Attachment, attachment_id, with_for_update=True)
|
||||||
|
if attachment:
|
||||||
|
if attachment.status != 'failed' or attachment.updated_at >= cutoff:
|
||||||
|
continue
|
||||||
|
client.delete_object(Bucket=BUCKET, Key=key)
|
||||||
|
deleted += 1
|
||||||
|
continue
|
||||||
|
# Recheck age after taking the same lock as the uploader.
|
||||||
|
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
||||||
|
continue
|
||||||
|
client.delete_object(Bucket=BUCKET, Key=key)
|
||||||
|
deleted += 1
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--once', action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
count = cleanup(sessionmaker(engine), s3_client())
|
||||||
|
logging.info('Cleanup completed: deleted=%d', count)
|
||||||
|
except Exception as exc:
|
||||||
|
logging.error('Cleanup deferred: %s', type(exc).__name__)
|
||||||
|
if args.once:
|
||||||
|
raise SystemExit(1) from None
|
||||||
|
if args.once:
|
||||||
|
break
|
||||||
|
time.sleep(60)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
+106
@@ -0,0 +1,106 @@
|
|||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, event, func, insert, select
|
||||||
|
from sqlalchemy.engine import URL, make_url
|
||||||
|
from sqlalchemy.pool import QueuePool
|
||||||
|
from threading import Lock
|
||||||
|
|
||||||
|
from . import telemetry as otel
|
||||||
|
|
||||||
|
from .models import SchemaMigration
|
||||||
|
|
||||||
|
|
||||||
|
def database_url():
|
||||||
|
if os.getenv('DATABASE_URL'):
|
||||||
|
url = make_url(os.environ['DATABASE_URL'])
|
||||||
|
if url.drivername in ('postgres', 'postgresql'):
|
||||||
|
url = url.set(drivername='postgresql+psycopg')
|
||||||
|
if url.drivername != 'postgresql+psycopg':
|
||||||
|
raise ValueError('DATABASE_URL must use PostgreSQL with psycopg')
|
||||||
|
return url
|
||||||
|
return URL.create(
|
||||||
|
'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'),
|
||||||
|
database=os.getenv('POSTGRES_DB', 'messenger'),
|
||||||
|
username=os.getenv('POSTGRES_USER', 'messenger'),
|
||||||
|
password=os.environ['POSTGRES_PASSWORD'],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ObservedPool(QueuePool):
|
||||||
|
"""Count threads acquiring a connection, including pool wait and reconnect time."""
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.waiting = 0
|
||||||
|
self.wait_lock = Lock()
|
||||||
|
|
||||||
|
def connect(self):
|
||||||
|
with self.wait_lock:
|
||||||
|
self.waiting += 1
|
||||||
|
try:
|
||||||
|
with otel.dependency_span('db.acquire', **{'db.system': 'postgresql'}):
|
||||||
|
return super().connect()
|
||||||
|
finally:
|
||||||
|
with self.wait_lock:
|
||||||
|
self.waiting -= 1
|
||||||
|
|
||||||
|
|
||||||
|
def create_db_engine():
|
||||||
|
engine = create_engine(
|
||||||
|
database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True,
|
||||||
|
poolclass=ObservedPool, hide_parameters=True,
|
||||||
|
connect_args={'connect_timeout': 2,
|
||||||
|
'options': '-c statement_timeout=3000 -c lock_timeout=2000'},
|
||||||
|
)
|
||||||
|
# Only operation names; SQL text, bound values and connection URLs are never exported.
|
||||||
|
@event.listens_for(engine, 'before_cursor_execute')
|
||||||
|
def before(conn, cursor, statement, parameters, context, executemany):
|
||||||
|
if not otel.trace.get_current_span().get_span_context().is_valid:
|
||||||
|
return
|
||||||
|
operation = statement.lstrip().split(None, 1)[0].upper() if statement.strip() else 'OTHER'
|
||||||
|
if operation not in {'SELECT', 'INSERT', 'UPDATE', 'DELETE'}:
|
||||||
|
operation = 'OTHER'
|
||||||
|
context.otel_span = otel.tracer.start_span('db.' + operation,
|
||||||
|
kind=otel.SpanKind.CLIENT, attributes={'db.system': 'postgresql', 'db.operation.name': operation})
|
||||||
|
|
||||||
|
@event.listens_for(engine, 'after_cursor_execute')
|
||||||
|
def after(conn, cursor, statement, parameters, context, executemany):
|
||||||
|
current = getattr(context, 'otel_span', None)
|
||||||
|
if current:
|
||||||
|
current.end()
|
||||||
|
|
||||||
|
@event.listens_for(engine, 'handle_error')
|
||||||
|
def error(context):
|
||||||
|
current = getattr(context.execution_context, 'otel_span', None)
|
||||||
|
if current:
|
||||||
|
current.set_status(otel.Status(otel.StatusCode.ERROR))
|
||||||
|
current.set_attribute('error.type', type(context.original_exception).__name__)
|
||||||
|
current.end()
|
||||||
|
otel.event('db_error', severity=otel.logging.ERROR, error_type=type(context.original_exception).__name__)
|
||||||
|
|
||||||
|
if os.getenv('OTEL_CHILD_PROCESS') != 'true':
|
||||||
|
otel.meter.create_observable_gauge('db_pool_used', callbacks=[
|
||||||
|
lambda _: [otel.metrics.Observation(engine.pool.checkedout())]])
|
||||||
|
otel.meter.create_observable_gauge('db_pool_waiting', callbacks=[
|
||||||
|
lambda _: [otel.metrics.Observation(engine.pool.waiting)]])
|
||||||
|
return engine
|
||||||
|
|
||||||
|
|
||||||
|
def migrate(engine):
|
||||||
|
with engine.begin() as conn:
|
||||||
|
conn.execute(select(func.pg_advisory_xact_lock(202602)))
|
||||||
|
SchemaMigration.__table__.create(conn, checkfirst=True)
|
||||||
|
applied = set(conn.scalars(select(SchemaMigration.version)))
|
||||||
|
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||||
|
if path.name not in applied:
|
||||||
|
conn.exec_driver_sql(path.read_text(encoding='utf-8'))
|
||||||
|
conn.execute(insert(SchemaMigration).values(version=path.name))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
migrate(engine)
|
||||||
|
print('Migrations applied')
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import Depends, Header, Query
|
||||||
|
|
||||||
|
from .auth import sessions, unauthorized
|
||||||
|
from .errors import APIError
|
||||||
|
from .store import store
|
||||||
|
|
||||||
|
|
||||||
|
def page_limit(limit: str = Query(default="50")) -> int:
|
||||||
|
if not limit.isascii() or not limit.isdecimal() or len(limit) > 3:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный limit")
|
||||||
|
number = int(limit)
|
||||||
|
if not 1 <= number <= 100:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный limit")
|
||||||
|
return number
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_token(authorization: Optional[str] = Header(default=None)) -> str:
|
||||||
|
parts = (authorization or '').split()
|
||||||
|
if len(parts) != 2 or parts[0].lower() != 'bearer':
|
||||||
|
raise unauthorized()
|
||||||
|
return parts[1]
|
||||||
|
|
||||||
|
|
||||||
|
def current_user(token: str = Depends(bearer_token)) -> dict:
|
||||||
|
user = store.get_user(sessions.user_id(token))
|
||||||
|
if user is None:
|
||||||
|
raise unauthorized()
|
||||||
|
return user
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from fastapi import Request
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
|
from .middleware import INSTANCE_ID
|
||||||
|
|
||||||
|
|
||||||
|
class APIError(Exception):
|
||||||
|
def __init__(self, status_code: int, code: str, message: str):
|
||||||
|
self.status_code = status_code
|
||||||
|
self.code = code
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse:
|
||||||
|
request_id = getattr(request.state, "request_id", "")
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status_code,
|
||||||
|
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
|
||||||
|
content={
|
||||||
|
"error": {
|
||||||
|
"code": code,
|
||||||
|
"message": message,
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def api_error_handler(request: Request, exc: APIError) -> JSONResponse:
|
||||||
|
response = error_response(request, exc.status_code, exc.code, exc.message)
|
||||||
|
if exc.status_code == 429:
|
||||||
|
response.headers['Retry-After'] = '1'
|
||||||
|
if exc.status_code == 401:
|
||||||
|
response.headers['WWW-Authenticate'] = (
|
||||||
|
'Basic realm="messenger", charset="UTF-8"'
|
||||||
|
if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||||
|
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
|
||||||
|
|
||||||
|
|
||||||
|
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||||
|
return error_response(request, 400, "invalid_request", "Неверный запрос")
|
||||||
|
|
||||||
|
|
||||||
|
def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||||
|
if exc.status_code == 404:
|
||||||
|
return error_response(request, 404, "not_found", "Объект не найден")
|
||||||
|
if exc.status_code == 405:
|
||||||
|
return error_response(request, 400, "invalid_request", "Метод не поддерживается")
|
||||||
|
else:
|
||||||
|
return error_response(request, exc.status_code, "invalid_request", "Неверный запрос")
|
||||||
|
|
||||||
|
|
||||||
|
def internal_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||||
|
return error_response(request, 500, "internal_error", "Внутренняя ошибка")
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC."""
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from .attachments import MAX_BYTES, MAX_PIXELS
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .jobs import now
|
||||||
|
from .models import Attachment, AttachmentJob
|
||||||
|
from .object_storage import BUCKET, s3_client
|
||||||
|
from . import telemetry as otel
|
||||||
|
|
||||||
|
|
||||||
|
def fit_size(size, bound):
|
||||||
|
width, height = size
|
||||||
|
if max(size) <= bound:
|
||||||
|
return size
|
||||||
|
return max(1, width * bound // max(size)), max(1, height * bound // max(size))
|
||||||
|
|
||||||
|
|
||||||
|
def variants(data):
|
||||||
|
with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original:
|
||||||
|
if original.width * original.height > MAX_PIXELS:
|
||||||
|
raise ValueError('pixel limit')
|
||||||
|
original.load()
|
||||||
|
oriented = ImageOps.exif_transpose(original)
|
||||||
|
rgba = oriented.convert('RGBA')
|
||||||
|
rgb = Image.new('RGB', rgba.size, 'white')
|
||||||
|
rgb.paste(rgba, mask=rgba.getchannel('A'))
|
||||||
|
rgba.close()
|
||||||
|
oriented.close()
|
||||||
|
rgb.info.clear()
|
||||||
|
try:
|
||||||
|
for name, image, fmt, mime, quality in (
|
||||||
|
('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
|
||||||
|
('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
|
||||||
|
('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80),
|
||||||
|
):
|
||||||
|
with image, io.BytesIO() as output:
|
||||||
|
image.save(output, format=fmt, quality=quality)
|
||||||
|
blob = output.getvalue()
|
||||||
|
yield {'name': name, 'content_type': mime, 'width': image.width,
|
||||||
|
'height': image.height, 'size_bytes': len(blob),
|
||||||
|
'sha256': hashlib.sha256(blob).hexdigest()}, blob
|
||||||
|
finally:
|
||||||
|
rgb.close()
|
||||||
|
|
||||||
|
|
||||||
|
def process(sessions, job_id, token):
|
||||||
|
with sessions() as session:
|
||||||
|
job = session.get(AttachmentJob, job_id)
|
||||||
|
if job.lease_token != token:
|
||||||
|
return
|
||||||
|
attachment = session.get(Attachment, job.attachment_id)
|
||||||
|
staging_key = attachment.staging_key
|
||||||
|
client = s3_client()
|
||||||
|
response = client.get_object(Bucket=BUCKET, Key=staging_key)
|
||||||
|
with response['Body'] as body:
|
||||||
|
data = body.read(MAX_BYTES + 1)
|
||||||
|
if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256:
|
||||||
|
raise ValueError('staging checksum mismatch')
|
||||||
|
try:
|
||||||
|
rendered = list(variants(data))
|
||||||
|
except OSError as exc:
|
||||||
|
raise ValueError('invalid image') from exc
|
||||||
|
# Job lock fences the final writes, cleanup and publication against lease recovery.
|
||||||
|
# Work before this transaction is CPU-only. Keys are stable across retries.
|
||||||
|
with sessions.begin() as session:
|
||||||
|
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||||
|
if job.lease_token != token or job.lease_until <= now():
|
||||||
|
return
|
||||||
|
current = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||||
|
if current.status != 'processing':
|
||||||
|
return
|
||||||
|
client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data,
|
||||||
|
ContentType=current.content_type)
|
||||||
|
saved = []
|
||||||
|
for meta, blob in rendered:
|
||||||
|
key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}'
|
||||||
|
client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type'])
|
||||||
|
saved.append({**meta, 'object_key': key})
|
||||||
|
# Test-only crash injection; never enabled by compose.yaml.
|
||||||
|
if os.getenv('JOB_FAULT') == 'after_objects':
|
||||||
|
os._exit(91)
|
||||||
|
current.variants = saved
|
||||||
|
current.status, current.error_code = 'ready', None
|
||||||
|
current.updated_at = now()
|
||||||
|
job.lease_token = job.lease_until = None
|
||||||
|
if os.getenv('JOB_FAULT') == 'after_commit':
|
||||||
|
os._exit(92)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# Linux worker: bounded address space as well as the container memory limit.
|
||||||
|
import resource
|
||||||
|
memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024
|
||||||
|
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||||
|
otel.setup(child=True)
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
with otel.span('image.process', context=otel.propagator.extract({
|
||||||
|
'traceparent': os.getenv('TRACEPARENT', ''), 'tracestate': os.getenv('TRACESTATE', '')})):
|
||||||
|
process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2])
|
||||||
|
otel.event('image_completed')
|
||||||
|
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError):
|
||||||
|
return 20
|
||||||
|
except MemoryError:
|
||||||
|
return 21
|
||||||
|
except Exception:
|
||||||
|
return 22 # No credentials or decoder/database internals in logs.
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
otel.shutdown()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
||||||
+199
@@ -0,0 +1,199 @@
|
|||||||
|
"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state."""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from redis import Redis
|
||||||
|
from redis.exceptions import ResponseError
|
||||||
|
from sqlalchemy import and_, or_, select, func
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .admission import reserve_capacity
|
||||||
|
from .models import Attachment, AttachmentJob
|
||||||
|
from .object_storage import BUCKET, s3_client
|
||||||
|
from . import telemetry as otel
|
||||||
|
|
||||||
|
STREAM = 'attachment-jobs'
|
||||||
|
GROUP = 'image-workers'
|
||||||
|
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
|
||||||
|
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60')))
|
||||||
|
LEASE_SECONDS = TIMEOUT + 15
|
||||||
|
|
||||||
|
|
||||||
|
def now():
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def broker():
|
||||||
|
return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
|
||||||
|
decode_responses=True, socket_connect_timeout=2, socket_timeout=5)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_group(client):
|
||||||
|
try:
|
||||||
|
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
|
||||||
|
except ResponseError as exc:
|
||||||
|
if not str(exc).startswith('BUSYGROUP'):
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def envelope(job):
|
||||||
|
return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1',
|
||||||
|
'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version,
|
||||||
|
'attempt': job.attempt + 1,
|
||||||
|
'created_at': job.created_at.isoformat().replace('+00:00', 'Z')}
|
||||||
|
|
||||||
|
|
||||||
|
def fail(session, job, attachment, code, permanent=False):
|
||||||
|
job.failures += 1
|
||||||
|
job.last_error = code
|
||||||
|
job.lease_token = job.lease_until = job.published_at = None
|
||||||
|
attachment.updated_at = now()
|
||||||
|
if permanent or job.failures >= MAX_ATTEMPTS:
|
||||||
|
attachment.status = 'failed'
|
||||||
|
attachment.error_code = code if permanent else 'retry_exhausted'
|
||||||
|
else:
|
||||||
|
attachment.status = 'queued'
|
||||||
|
attachment.error_code = None
|
||||||
|
delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5)
|
||||||
|
job.available_at = now() + timedelta(seconds=delay)
|
||||||
|
otel.event('job_retry' if attachment.status == 'queued' else 'job_failed',
|
||||||
|
attempt=job.attempt, outcome=attachment.status, error_code=code)
|
||||||
|
|
||||||
|
|
||||||
|
def dispatch_once(sessions, client):
|
||||||
|
# A crash after XADD but before COMMIT deliberately results in redelivery.
|
||||||
|
with sessions.begin() as session:
|
||||||
|
jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where(
|
||||||
|
or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)),
|
||||||
|
and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())),
|
||||||
|
AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at)
|
||||||
|
.with_for_update(skip_locked=True, of=AttachmentJob).limit(50)))
|
||||||
|
for job in jobs:
|
||||||
|
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||||
|
if attachment.status == 'processing':
|
||||||
|
if job.lease_until <= now():
|
||||||
|
fail(session, job, attachment, 'lease_expired')
|
||||||
|
continue
|
||||||
|
if job.published_at is None:
|
||||||
|
with otel.span('attachment-jobs publish', kind=otel.SpanKind.PRODUCER,
|
||||||
|
context=otel.propagator.extract(job.trace_context), attributes={
|
||||||
|
'messaging.system': 'redis', 'messaging.destination.name': STREAM,
|
||||||
|
'messaging.operation.type': 'publish', 'job.attempt': job.attempt + 1}):
|
||||||
|
client.xadd(STREAM, {'job': json.dumps({**envelope(job), **otel.carrier()})})
|
||||||
|
otel.event('job_published', attempt=job.attempt + 1)
|
||||||
|
job.published_at = now()
|
||||||
|
|
||||||
|
|
||||||
|
def sample_queue(sessions):
|
||||||
|
# Database is the durable backlog: includes outbox rows not yet in Redis and retries.
|
||||||
|
with sessions() as session:
|
||||||
|
depth, oldest = session.execute(select(func.count(), func.min(Attachment.created_at))
|
||||||
|
.select_from(Attachment).where(Attachment.status == 'queued')).one()
|
||||||
|
otel.queue_state.update(depth=depth, age=max(0, (now() - oldest).total_seconds()) if oldest else 0,
|
||||||
|
updated=time.time())
|
||||||
|
|
||||||
|
|
||||||
|
def claim(sessions, message):
|
||||||
|
if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1'
|
||||||
|
or message.get('pipeline_version') != 1):
|
||||||
|
raise ValueError('unsupported job envelope')
|
||||||
|
job_id = str(UUID(message['job_id']))
|
||||||
|
with sessions.begin() as session:
|
||||||
|
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||||
|
if job is None or job.attachment_id != message.get('attachment_id'):
|
||||||
|
raise ValueError('unknown job')
|
||||||
|
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||||
|
if attachment.status != 'queued' or job.available_at > now():
|
||||||
|
return None
|
||||||
|
job.attempt += 1
|
||||||
|
if job.first_started_at is None:
|
||||||
|
job.first_started_at = now()
|
||||||
|
job.lease_token = str(uuid4())
|
||||||
|
job.lease_until = now() + timedelta(seconds=LEASE_SECONDS)
|
||||||
|
attachment.status = 'processing'
|
||||||
|
attachment.updated_at = now()
|
||||||
|
return job.id, job.lease_token
|
||||||
|
|
||||||
|
|
||||||
|
def record_failure(sessions, job_id, token, code, permanent=False):
|
||||||
|
with sessions.begin() as session:
|
||||||
|
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||||
|
if job.lease_token == token:
|
||||||
|
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||||
|
fail(session, job, attachment, code, permanent)
|
||||||
|
|
||||||
|
|
||||||
|
def replay(sessions, job_id, reason):
|
||||||
|
if not reason.strip() or len(reason) > 200:
|
||||||
|
raise ValueError('Give a nonempty reason of at most 200 characters')
|
||||||
|
with sessions.begin() as session:
|
||||||
|
job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True)
|
||||||
|
if job is None:
|
||||||
|
raise ValueError('Job not found')
|
||||||
|
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||||
|
if attachment.status != 'failed':
|
||||||
|
raise ValueError('Replay is allowed only for failed jobs')
|
||||||
|
if not attachment.staging_key:
|
||||||
|
raise ValueError('Staging retention expired; upload the file again')
|
||||||
|
s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key)
|
||||||
|
reserve_capacity(session)
|
||||||
|
job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason,
|
||||||
|
'after_attempt': job.attempt,
|
||||||
|
'last_error': job.last_error}]
|
||||||
|
job.failures = 0
|
||||||
|
job.available_at = now()
|
||||||
|
job.published_at = job.lease_until = job.lease_token = None
|
||||||
|
attachment.status, attachment.error_code = 'queued', None
|
||||||
|
attachment.updated_at = now()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
sub = parser.add_subparsers(dest='command', required=True)
|
||||||
|
sub.add_parser('dispatch')
|
||||||
|
sub.add_parser('failed')
|
||||||
|
retry = sub.add_parser('replay')
|
||||||
|
retry.add_argument('job_id')
|
||||||
|
retry.add_argument('--reason', required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
otel.setup()
|
||||||
|
engine = create_db_engine()
|
||||||
|
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||||
|
try:
|
||||||
|
if args.command == 'failed':
|
||||||
|
with sessions() as session:
|
||||||
|
for job, att in session.execute(select(AttachmentJob, Attachment).join(
|
||||||
|
Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||||
|
.where(Attachment.status == 'failed')):
|
||||||
|
print(json.dumps({'job_id': job.id, 'attachment_id': att.id,
|
||||||
|
'attempt': job.attempt, 'error_code': att.error_code,
|
||||||
|
'last_error': job.last_error,
|
||||||
|
'replay_available': bool(att.staging_key)}))
|
||||||
|
elif args.command == 'replay':
|
||||||
|
replay(sessions, args.job_id, args.reason)
|
||||||
|
print('Job queued for replay')
|
||||||
|
else:
|
||||||
|
client = broker()
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
ensure_group(client)
|
||||||
|
dispatch_once(sessions, client)
|
||||||
|
sample_queue(sessions)
|
||||||
|
except Exception as exc:
|
||||||
|
otel.event('dispatch_deferred', error_type=type(exc).__name__)
|
||||||
|
time.sleep(1)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
otel.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from starlette.requests import Request
|
||||||
|
|
||||||
|
from .errors import error_response
|
||||||
|
from .middleware import is_upload
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Limits:
|
||||||
|
max_users: int = 1000
|
||||||
|
max_chats: int = 1000
|
||||||
|
max_messages: int = 10000
|
||||||
|
max_text_bytes: int = 16 * 1024 * 1024
|
||||||
|
max_request_bytes: int = 64 * 1024
|
||||||
|
max_inflight: int = 32
|
||||||
|
message_rate: int = 1000
|
||||||
|
message_burst: int = 1000
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls):
|
||||||
|
values = {}
|
||||||
|
for name, default in cls().__dict__.items():
|
||||||
|
value = int(os.getenv(name.upper(), str(default)))
|
||||||
|
if value < 1:
|
||||||
|
raise ValueError(f'{name.upper()} must be positive')
|
||||||
|
values[name] = value
|
||||||
|
return cls(**values)
|
||||||
|
|
||||||
|
|
||||||
|
limits = Limits.from_env()
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceLimitMiddleware:
|
||||||
|
def __init__(self, app, settings=limits):
|
||||||
|
self.app = app
|
||||||
|
self.settings = settings
|
||||||
|
self.active = 0
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
|
||||||
|
request = Request(scope)
|
||||||
|
|
||||||
|
async def reject(status, code, message):
|
||||||
|
response = error_response(request, status, code, message)
|
||||||
|
if status == 503:
|
||||||
|
response.headers['Retry-After'] = '1'
|
||||||
|
await response(scope, receive, send)
|
||||||
|
|
||||||
|
if self.active >= self.settings.max_inflight:
|
||||||
|
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
|
||||||
|
self.active += 1
|
||||||
|
try:
|
||||||
|
# The upload endpoint authenticates and bounds its stream itself.
|
||||||
|
if is_upload(scope['method'], scope['path']):
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
length = request.headers.get('content-length')
|
||||||
|
if length is not None:
|
||||||
|
try:
|
||||||
|
size = int(length)
|
||||||
|
if size < 0:
|
||||||
|
raise ValueError()
|
||||||
|
except ValueError:
|
||||||
|
return await reject(400, 'invalid_request', 'Неверный Content-Length')
|
||||||
|
if size > self.settings.max_request_bytes:
|
||||||
|
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
|
||||||
|
|
||||||
|
body = bytearray()
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(5):
|
||||||
|
while True:
|
||||||
|
message = await receive()
|
||||||
|
if message['type'] == 'http.disconnect':
|
||||||
|
return
|
||||||
|
chunk = message.get('body', b'')
|
||||||
|
if len(body) + len(chunk) > self.settings.max_request_bytes:
|
||||||
|
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
|
||||||
|
body.extend(chunk)
|
||||||
|
if not message.get('more_body', False):
|
||||||
|
break
|
||||||
|
except TimeoutError:
|
||||||
|
return await reject(503, 'unavailable', 'Истекло время чтения запроса')
|
||||||
|
|
||||||
|
delivered = False
|
||||||
|
|
||||||
|
async def buffered_receive():
|
||||||
|
nonlocal delivered
|
||||||
|
if not delivered:
|
||||||
|
delivered = True
|
||||||
|
return {'type': 'http.request', 'body': bytes(body), 'more_body': False}
|
||||||
|
return await receive()
|
||||||
|
|
||||||
|
await self.app(scope, buffered_receive, send)
|
||||||
|
finally:
|
||||||
|
self.active -= 1
|
||||||
+55
@@ -0,0 +1,55 @@
|
|||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from starlette.exceptions import HTTPException
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
from botocore.exceptions import BotoCoreError, ClientError
|
||||||
|
|
||||||
|
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .store import store
|
||||||
|
from .auth import sessions
|
||||||
|
from .middleware import RequestIdMiddleware
|
||||||
|
from .limits import ResourceLimitMiddleware
|
||||||
|
from .routers.health import router as health_router
|
||||||
|
from .routers.users import router as users_router
|
||||||
|
from .routers.chats import router as chats_router
|
||||||
|
from .routers.auth import router as auth_router
|
||||||
|
from .routers.attachments import router as attachments_router
|
||||||
|
from . import telemetry
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app):
|
||||||
|
telemetry.setup()
|
||||||
|
telemetry.seed_http(app.routes)
|
||||||
|
engine = create_db_engine()
|
||||||
|
try:
|
||||||
|
store.sessions = sessionmaker(engine, expire_on_commit=False)
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
sessions.client.close()
|
||||||
|
telemetry.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
app = FastAPI(title="MTUSI Messenger", lifespan=lifespan)
|
||||||
|
app.add_middleware(ResourceLimitMiddleware)
|
||||||
|
app.add_middleware(RequestIdMiddleware)
|
||||||
|
app.add_exception_handler(APIError, api_error_handler)
|
||||||
|
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||||
|
app.add_exception_handler(HTTPException, http_error_handler)
|
||||||
|
app.add_exception_handler(Exception, internal_error_handler)
|
||||||
|
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
|
||||||
|
app.add_exception_handler(error, dependency_error_handler)
|
||||||
|
|
||||||
|
app.include_router(health_router)
|
||||||
|
app.include_router(users_router)
|
||||||
|
app.include_router(chats_router)
|
||||||
|
app.include_router(auth_router)
|
||||||
|
app.include_router(attachments_router)
|
||||||
|
|
||||||
|
# Outside FastAPI's error middleware: includes generated 500 responses and full streams.
|
||||||
|
app = telemetry.HttpTelemetry(app)
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
from uuid import uuid4
|
||||||
|
from socket import gethostname
|
||||||
|
from starlette.responses import JSONResponse
|
||||||
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
from starlette.requests import Request
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def is_upload(method, path):
|
||||||
|
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
|
||||||
|
|
||||||
|
INSTANCE_ID = gethostname()
|
||||||
|
|
||||||
|
|
||||||
|
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||||
|
async def dispatch(self, request: Request, call_next):
|
||||||
|
request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4())
|
||||||
|
request.state.request_id = request_id
|
||||||
|
|
||||||
|
if (request.method in {"POST", "PUT", "PATCH"}
|
||||||
|
and request.url.path.startswith("/api/")
|
||||||
|
and not is_upload(request.method, request.url.path)
|
||||||
|
and request.url.path != '/api/v1/auth/sessions'):
|
||||||
|
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||||
|
if content_type != "application/json":
|
||||||
|
response = JSONResponse(
|
||||||
|
status_code=415,
|
||||||
|
content={
|
||||||
|
"error": {
|
||||||
|
"code": "unsupported_media_type",
|
||||||
|
"message": "Ожидается Content-Type application/json",
|
||||||
|
"request_id": request_id,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||||
|
return response
|
||||||
|
|
||||||
|
response = await call_next(request)
|
||||||
|
response.headers["X-Request-Id"] = request_id
|
||||||
|
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||||
|
return response
|
||||||
+109
@@ -0,0 +1,109 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
|
||||||
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||||
|
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
|
||||||
|
|
||||||
|
|
||||||
|
class Base(DeclarativeBase):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class User(Base):
|
||||||
|
__tablename__ = 'users'
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"),
|
||||||
|
CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'),
|
||||||
|
)
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
username: Mapped[str] = mapped_column(Text, unique=True)
|
||||||
|
display_name: Mapped[str] = mapped_column(Text)
|
||||||
|
password_hash: Mapped[str] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
class Chat(Base):
|
||||||
|
__tablename__ = 'chats'
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint('char_length(title) BETWEEN 1 AND 100'),
|
||||||
|
Index('chats_order_idx', 'created_at', 'id'),
|
||||||
|
)
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
title: Mapped[str] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
history_version: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||||
|
|
||||||
|
|
||||||
|
class ChatMember(Base):
|
||||||
|
__tablename__ = 'chat_members'
|
||||||
|
__table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),)
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||||
|
user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||||
|
|
||||||
|
|
||||||
|
class Message(Base):
|
||||||
|
__tablename__ = 'messages'
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint('char_length(text) <= 4000'),
|
||||||
|
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
|
||||||
|
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||||
|
)
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||||
|
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||||
|
text: Mapped[str] = mapped_column(Text)
|
||||||
|
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
|
||||||
|
|
||||||
|
class SchemaMigration(Base):
|
||||||
|
__tablename__ = 'schema_migrations'
|
||||||
|
version: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||||
|
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class Attachment(Base):
|
||||||
|
__tablename__ = 'attachments'
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||||
|
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||||
|
object_key: Mapped[str] = mapped_column(Text, unique=True)
|
||||||
|
filename: Mapped[str] = mapped_column(Text)
|
||||||
|
content_type: Mapped[str] = mapped_column(Text)
|
||||||
|
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||||
|
sha256: Mapped[str] = mapped_column(Text)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
status: Mapped[str] = mapped_column(Text, server_default='queued')
|
||||||
|
staging_key: Mapped[str | None] = mapped_column(Text)
|
||||||
|
variants: Mapped[list] = mapped_column(JSONB, server_default='[]')
|
||||||
|
error_code: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentJob(Base):
|
||||||
|
"""The job row is also the transactional outbox and failed-job registry."""
|
||||||
|
__tablename__ = 'attachment_jobs'
|
||||||
|
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||||
|
attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True)
|
||||||
|
pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1')
|
||||||
|
attempt: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||||
|
failures: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||||
|
last_error: Mapped[str | None] = mapped_column(Text)
|
||||||
|
first_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
|
replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]')
|
||||||
|
trace_context: Mapped[dict] = mapped_column(JSONB, server_default='{}')
|
||||||
|
|
||||||
|
|
||||||
|
class IdempotencyRecord(Base):
|
||||||
|
__tablename__ = 'message_idempotency'
|
||||||
|
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||||
|
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||||
|
key: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||||
|
message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE'))
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import os
|
||||||
|
from functools import lru_cache
|
||||||
|
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
from .telemetry import dependency_span
|
||||||
|
|
||||||
|
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||||
|
PREFIX = 'originals/'
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def s3_client():
|
||||||
|
client = boto3.client(
|
||||||
|
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||||
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
||||||
|
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
|
||||||
|
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
|
||||||
|
config=Config(connect_timeout=2, read_timeout=5,
|
||||||
|
retries={'total_max_attempts': 1},
|
||||||
|
s3={'addressing_style': 'path'},
|
||||||
|
request_checksum_calculation='when_required',
|
||||||
|
response_checksum_validation='when_required'),
|
||||||
|
)
|
||||||
|
# Instrument SDK operations without object keys, signed URLs, headers or bytes.
|
||||||
|
original = client._make_api_call
|
||||||
|
|
||||||
|
def call(operation, parameters):
|
||||||
|
with dependency_span('s3.' + operation, **{'rpc.system': 'aws-api', 'rpc.service': 'S3',
|
||||||
|
'rpc.method': operation}):
|
||||||
|
return original(operation, parameters)
|
||||||
|
client._make_api_call = call
|
||||||
|
return client
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
from tempfile import TemporaryFile
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query, Request
|
||||||
|
from starlette.background import BackgroundTask
|
||||||
|
from starlette.concurrency import run_in_threadpool
|
||||||
|
from starlette.requests import ClientDisconnect
|
||||||
|
from starlette.responses import StreamingResponse
|
||||||
|
|
||||||
|
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
|
||||||
|
from ..dependencies import current_user
|
||||||
|
from ..errors import APIError
|
||||||
|
from ..object_storage import BUCKET, s3_client
|
||||||
|
from ..store import ensure_uuid, store
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||||
|
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
|
||||||
|
decode_slot = asyncio.Semaphore(1)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/chats/{chat_id}/attachments', status_code=202)
|
||||||
|
async def upload(chat_id: str, request: Request, filename: str = Query(...),
|
||||||
|
user: dict = Depends(current_user)):
|
||||||
|
chat_id = ensure_uuid(chat_id)
|
||||||
|
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
|
||||||
|
validate_filename(filename)
|
||||||
|
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
|
||||||
|
if content_type not in ('image/png', 'image/jpeg'):
|
||||||
|
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
|
||||||
|
length = request.headers.get('content-length')
|
||||||
|
if length is not None:
|
||||||
|
try:
|
||||||
|
length = int(length)
|
||||||
|
if length < 0:
|
||||||
|
raise ValueError()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
|
||||||
|
if length > MAX_BYTES:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||||
|
size = 0
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with TemporaryFile() as source:
|
||||||
|
try:
|
||||||
|
async with asyncio.timeout(30):
|
||||||
|
async for chunk in request.stream():
|
||||||
|
size += len(chunk)
|
||||||
|
if size > MAX_BYTES:
|
||||||
|
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||||
|
digest.update(chunk)
|
||||||
|
await run_in_threadpool(source.write, chunk)
|
||||||
|
except (TimeoutError, ClientDisconnect) as exc:
|
||||||
|
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
|
||||||
|
if length is not None and size != length:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
|
||||||
|
async with decode_slot:
|
||||||
|
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
|
||||||
|
filename, content_type, size, digest.hexdigest())
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/attachments/{attachment_id}')
|
||||||
|
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
|
||||||
|
return metadata(get_attachment(attachment_id, user['id']))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/attachments/{attachment_id}/content')
|
||||||
|
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
|
||||||
|
attachment = get_attachment(attachment_id, user['id'])
|
||||||
|
if attachment.status != 'ready':
|
||||||
|
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||||
|
key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes
|
||||||
|
if variant != 'original':
|
||||||
|
item = next((v for v in attachment.variants if v['name'] == variant), None)
|
||||||
|
if item is None:
|
||||||
|
raise APIError(404, 'not_found', 'Вариант не найден')
|
||||||
|
key, content_type, size = item['object_key'], item['content_type'], item['size_bytes']
|
||||||
|
body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body']
|
||||||
|
|
||||||
|
def chunks():
|
||||||
|
try:
|
||||||
|
yield from body.iter_chunks(chunk_size=64 * 1024)
|
||||||
|
finally:
|
||||||
|
body.close()
|
||||||
|
|
||||||
|
return StreamingResponse(chunks(), media_type=content_type,
|
||||||
|
headers={'Content-Length': str(size),
|
||||||
|
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
|
||||||
|
'X-Content-Type-Options': 'nosniff'},
|
||||||
|
background=BackgroundTask(body.close))
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import base64
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, Request, Response
|
||||||
|
from starlette.concurrency import run_in_threadpool
|
||||||
|
|
||||||
|
from ..auth import sessions, unauthorized
|
||||||
|
from ..dependencies import bearer_token, current_user
|
||||||
|
from ..errors import APIError
|
||||||
|
from ..store import store
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('', status_code=201)
|
||||||
|
async def create_session(request: Request, authorization: str = Header(default='')):
|
||||||
|
if await request.body():
|
||||||
|
raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса')
|
||||||
|
try:
|
||||||
|
scheme, encoded = authorization.split()
|
||||||
|
if scheme.lower() != 'basic':
|
||||||
|
raise ValueError()
|
||||||
|
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||||
|
if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username:
|
||||||
|
raise ValueError()
|
||||||
|
except (ValueError, UnicodeError):
|
||||||
|
raise unauthorized() from None
|
||||||
|
user = await run_in_threadpool(store.login, username, password)
|
||||||
|
return await run_in_threadpool(sessions.create, user)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete('/current', status_code=204)
|
||||||
|
def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)):
|
||||||
|
sessions.delete(token)
|
||||||
|
return Response(status_code=204)
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
from typing import Optional
|
||||||
|
import re
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, Query, status
|
||||||
|
from ..errors import APIError
|
||||||
|
|
||||||
|
from ..dependencies import current_user, page_limit
|
||||||
|
from ..schemas.chat import Chat, ChatPage, CreateChat
|
||||||
|
from ..schemas.message import CreateMessage, Message, MessagePage
|
||||||
|
from ..store import store
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED)
|
||||||
|
def create_chat(payload: CreateChat, user: dict = Depends(current_user)):
|
||||||
|
return store.create_chat(user["id"], payload.title, payload.member_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", response_model=ChatPage)
|
||||||
|
def list_chats(
|
||||||
|
limit: int = Depends(page_limit),
|
||||||
|
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||||
|
user: dict = Depends(current_user),
|
||||||
|
):
|
||||||
|
return store.list_chats(user["id"], limit, cursor)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{chat_id}", response_model=Chat)
|
||||||
|
def get_chat(chat_id: str, user: dict = Depends(current_user)):
|
||||||
|
return store.get_chat_for_user(chat_id, user["id"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED)
|
||||||
|
def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user),
|
||||||
|
idempotency_key: Optional[str] = Header(default=None)):
|
||||||
|
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
|
||||||
|
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
|
||||||
|
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{chat_id}/messages", response_model=MessagePage)
|
||||||
|
def list_messages(
|
||||||
|
chat_id: str,
|
||||||
|
limit: int = Depends(page_limit),
|
||||||
|
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||||
|
user: dict = Depends(current_user),
|
||||||
|
):
|
||||||
|
return store.list_messages(chat_id, user["id"], limit, cursor)
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
from fastapi import APIRouter
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..schemas.common import Health
|
||||||
|
from ..auth import sessions
|
||||||
|
from ..store import store
|
||||||
|
from ..object_storage import BUCKET, s3_client
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/health')
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/live", response_model=Health)
|
||||||
|
def health_live():
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/ready", response_model=Health)
|
||||||
|
def health_ready():
|
||||||
|
with store.sessions() as session:
|
||||||
|
session.execute(select(1))
|
||||||
|
sessions.client.ping()
|
||||||
|
s3_client().head_bucket(Bucket=BUCKET)
|
||||||
|
return {"status": "ok"}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
from fastapi import APIRouter, Depends, status
|
||||||
|
|
||||||
|
from ..dependencies import current_user
|
||||||
|
from ..schemas.user import CreateUser, User
|
||||||
|
from ..store import store
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("", response_model=User, status_code=status.HTTP_201_CREATED)
|
||||||
|
def create_user(payload: CreateUser):
|
||||||
|
return store.create_user(payload.username, payload.display_name, payload.password)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/me", response_model=User)
|
||||||
|
def get_current_user(user: dict = Depends(current_user)):
|
||||||
|
return user
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from pydantic import Field, StrictStr, field_validator
|
||||||
|
|
||||||
|
from .common import ContractModel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class CreateChat(ContractModel):
|
||||||
|
title: StrictStr = Field(min_length=1, max_length=100)
|
||||||
|
member_ids: List[StrictStr] = Field(min_length=1, max_length=100)
|
||||||
|
|
||||||
|
@field_validator("title")
|
||||||
|
def validate_title(cls, value):
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("blank title")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("member_ids")
|
||||||
|
def validate_member_ids(cls, value):
|
||||||
|
if len(value) != len(set(value)):
|
||||||
|
raise ValueError("duplicate member_ids")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Chat(ContractModel):
|
||||||
|
id: str
|
||||||
|
title: str
|
||||||
|
member_ids: List[str]
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class ChatPage(ContractModel):
|
||||||
|
items: List[Chat]
|
||||||
|
next_cursor: Optional[str]
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from pydantic import BaseModel, ConfigDict, field_validator
|
||||||
|
|
||||||
|
|
||||||
|
class ContractModel(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
@field_validator('*')
|
||||||
|
def validate_text_encoding(cls, value, info):
|
||||||
|
if isinstance(value, str):
|
||||||
|
value.encode('utf-8')
|
||||||
|
if info.field_name != 'password' and '\x00' in value:
|
||||||
|
raise ValueError('NUL is not supported in PostgreSQL text')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Health(ContractModel):
|
||||||
|
status: str
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from pydantic import Field, StrictStr, field_validator, model_validator
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from .common import ContractModel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMessage(ContractModel):
|
||||||
|
text: StrictStr = Field(max_length=4000)
|
||||||
|
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
|
||||||
|
|
||||||
|
@field_validator('attachment_ids')
|
||||||
|
def validate_ids(cls, value):
|
||||||
|
ids = [str(UUID(item)) for item in value]
|
||||||
|
if len(ids) != len(set(ids)):
|
||||||
|
raise ValueError('duplicate attachments')
|
||||||
|
return ids
|
||||||
|
|
||||||
|
@model_validator(mode='after')
|
||||||
|
def validate_content(self):
|
||||||
|
if not self.text.strip() and not self.attachment_ids:
|
||||||
|
raise ValueError('empty message')
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class Message(ContractModel):
|
||||||
|
id: str
|
||||||
|
chat_id: str
|
||||||
|
sender_id: str
|
||||||
|
text: str
|
||||||
|
attachment_ids: List[str]
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class MessagePage(ContractModel):
|
||||||
|
items: List[Message]
|
||||||
|
next_cursor: Optional[str]
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from pydantic import Field, StrictStr, field_validator
|
||||||
|
|
||||||
|
from .common import ContractModel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUser(ContractModel):
|
||||||
|
username: StrictStr
|
||||||
|
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||||
|
password: StrictStr = Field(min_length=12, max_length=128)
|
||||||
|
|
||||||
|
@field_validator("username")
|
||||||
|
def validate_username(cls, value):
|
||||||
|
if not USERNAME_RE.fullmatch(value):
|
||||||
|
raise ValueError("invalid username")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("display_name")
|
||||||
|
def validate_display_name(cls, value):
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("blank display_name")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class User(ContractModel):
|
||||||
|
id: str
|
||||||
|
username: str
|
||||||
|
display_name: str = Field(min_length=1, max_length=100)
|
||||||
|
created_at: str
|
||||||
+229
@@ -0,0 +1,229 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Tuple
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, tuple_, update
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
|
from .auth import password_hasher, verify_password, unauthorized
|
||||||
|
from .errors import APIError
|
||||||
|
from .cache import history_cache
|
||||||
|
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_uuid(value):
|
||||||
|
try:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError()
|
||||||
|
return str(UUID(value))
|
||||||
|
except (TypeError, ValueError) as exc:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверный UUID') from exc
|
||||||
|
|
||||||
|
|
||||||
|
def serialize(model, *fields):
|
||||||
|
if model is None:
|
||||||
|
return None
|
||||||
|
result = {}
|
||||||
|
for field in fields:
|
||||||
|
value = getattr(model, field)
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||||
|
result[field] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
|
||||||
|
CHAT_FIELDS = ('id', 'title', 'created_at')
|
||||||
|
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
|
||||||
|
|
||||||
|
|
||||||
|
def after_condition(model, after):
|
||||||
|
return tuple_(model.created_at, model.id) > (
|
||||||
|
datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1])
|
||||||
|
|
||||||
|
|
||||||
|
def history_query(chat_id, limit, after):
|
||||||
|
query = select(Message).where(Message.chat_id == chat_id)
|
||||||
|
if after:
|
||||||
|
query = query.where(after_condition(Message, after))
|
||||||
|
return query.order_by(Message.created_at, Message.id).limit(limit + 1)
|
||||||
|
|
||||||
|
|
||||||
|
class PostgresStore:
|
||||||
|
def __init__(self, cache=history_cache):
|
||||||
|
self.sessions = None
|
||||||
|
self.cache = cache
|
||||||
|
|
||||||
|
def create_user(self, username, display_name, password):
|
||||||
|
user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password))
|
||||||
|
try:
|
||||||
|
with self.sessions.begin() as session:
|
||||||
|
session.add(user)
|
||||||
|
session.flush()
|
||||||
|
result = serialize(user, *USER_FIELDS)
|
||||||
|
return result
|
||||||
|
except IntegrityError as exc:
|
||||||
|
if getattr(exc.orig, 'sqlstate', None) == '23505':
|
||||||
|
raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc
|
||||||
|
raise
|
||||||
|
|
||||||
|
def get_user(self, user_id):
|
||||||
|
with self.sessions() as session:
|
||||||
|
return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS)
|
||||||
|
|
||||||
|
def login(self, username, password):
|
||||||
|
with self.sessions() as session:
|
||||||
|
user = session.scalar(select(User).where(User.username == username))
|
||||||
|
encoded = user.password_hash if user else None
|
||||||
|
result = serialize(user, *USER_FIELDS)
|
||||||
|
if not verify_password(encoded, password):
|
||||||
|
raise unauthorized()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def create_chat(self, actor_id, title, member_ids):
|
||||||
|
members = [ensure_uuid(member) for member in member_ids]
|
||||||
|
if len(members) != len(set(members)) or actor_id not in members:
|
||||||
|
raise APIError(400, 'invalid_request', 'Неверный список участников')
|
||||||
|
try:
|
||||||
|
with self.sessions.begin() as session:
|
||||||
|
chat = Chat(title=title)
|
||||||
|
session.add(chat)
|
||||||
|
session.flush()
|
||||||
|
session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members])
|
||||||
|
result = serialize(chat, *CHAT_FIELDS)
|
||||||
|
result['member_ids'] = sorted(members)
|
||||||
|
return result
|
||||||
|
except IntegrityError as exc:
|
||||||
|
if getattr(exc.orig, 'sqlstate', None) == '23503':
|
||||||
|
raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc
|
||||||
|
raise
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _chat(session, chat_id, actor_id):
|
||||||
|
chat = session.scalar(select(Chat).join(ChatMember).where(
|
||||||
|
Chat.id == chat_id, ChatMember.user_id == actor_id))
|
||||||
|
if chat is None:
|
||||||
|
raise APIError(404, 'not_found', 'Объект не найден')
|
||||||
|
return chat
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _members(session, chat):
|
||||||
|
result = serialize(chat, *CHAT_FIELDS)
|
||||||
|
result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where(
|
||||||
|
ChatMember.chat_id == chat.id).order_by(ChatMember.user_id)))
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get_chat_for_user(self, chat_id, actor_id):
|
||||||
|
with self.sessions() as session:
|
||||||
|
return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id))
|
||||||
|
|
||||||
|
def list_chats(self, actor_id, limit, cursor):
|
||||||
|
after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None
|
||||||
|
query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id)
|
||||||
|
if after:
|
||||||
|
query = query.where(after_condition(Chat, after))
|
||||||
|
query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1)
|
||||||
|
with self.sessions() as session:
|
||||||
|
chats = list(session.scalars(query))
|
||||||
|
items = [self._members(session, chat) for chat in chats[:limit]]
|
||||||
|
return self._page(items, len(chats) > limit, 'chats', actor_id)
|
||||||
|
|
||||||
|
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
|
||||||
|
chat_id = ensure_uuid(chat_id)
|
||||||
|
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
|
||||||
|
with self.sessions.begin() as session:
|
||||||
|
self._chat(session, chat_id, actor_id)
|
||||||
|
if attachment_ids:
|
||||||
|
found = list(session.scalars(select(Attachment).where(
|
||||||
|
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)
|
||||||
|
.order_by(Attachment.id).with_for_update()))
|
||||||
|
if len(found) != len(attachment_ids):
|
||||||
|
raise APIError(404, 'not_found', 'Объект не найден')
|
||||||
|
if idempotency_key is not None:
|
||||||
|
# Serialize writers to this chat before looking up the key.
|
||||||
|
# The same row is updated below for history cache invalidation.
|
||||||
|
session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update())
|
||||||
|
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
|
||||||
|
if record is not None:
|
||||||
|
previous = session.get(Message, record.message_id)
|
||||||
|
if previous.text != text or previous.attachment_ids != attachment_ids:
|
||||||
|
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||||
|
return serialize(previous, *MESSAGE_FIELDS)
|
||||||
|
if attachment_ids and any(item.status == 'failed' for item in found):
|
||||||
|
raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой')
|
||||||
|
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||||
|
history_version=Chat.history_version + 1))
|
||||||
|
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
|
||||||
|
session.add(message)
|
||||||
|
session.flush()
|
||||||
|
if idempotency_key is not None:
|
||||||
|
session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id,
|
||||||
|
key=idempotency_key, message_id=message.id))
|
||||||
|
result = serialize(message, *MESSAGE_FIELDS)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def list_messages(self, chat_id, actor_id, limit, cursor):
|
||||||
|
chat_id = ensure_uuid(chat_id)
|
||||||
|
after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None
|
||||||
|
with self.sessions() as session:
|
||||||
|
version = self._chat(session, chat_id, actor_id).history_version
|
||||||
|
return self.cache.get_or_load(
|
||||||
|
chat_id, version, limit, after,
|
||||||
|
lambda: self._load_messages(chat_id, limit, after),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _load_messages(self, chat_id, limit, after):
|
||||||
|
with self.sessions() as session:
|
||||||
|
messages = list(session.scalars(history_query(chat_id, limit, after)))
|
||||||
|
items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]]
|
||||||
|
return self._page(items, len(messages) > limit, 'messages', chat_id)
|
||||||
|
|
||||||
|
def _page(self, items, has_more, resource, owner_id):
|
||||||
|
cursor = None
|
||||||
|
if has_more:
|
||||||
|
last = items[-1]
|
||||||
|
cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id'])
|
||||||
|
return {'items': items, 'next_cursor': cursor}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str:
|
||||||
|
payload = {
|
||||||
|
"resource": resource,
|
||||||
|
"owner_id": owner_id,
|
||||||
|
"created_at": created_at,
|
||||||
|
"id": item_id,
|
||||||
|
}
|
||||||
|
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]:
|
||||||
|
try:
|
||||||
|
padded = cursor + "=" * (-len(cursor) % 4)
|
||||||
|
payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
|
||||||
|
except (ValueError, UnicodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor") from exc
|
||||||
|
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||||
|
|
||||||
|
if payload.get("resource") != resource or payload.get("owner_id") != owner_id:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||||
|
|
||||||
|
item_id = ensure_uuid(payload.get("id"))
|
||||||
|
created_at = payload.get("created_at")
|
||||||
|
if not isinstance(created_at, str) or not created_at:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||||
|
if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc:
|
||||||
|
raise ValueError("Expected UTC timestamp")
|
||||||
|
created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise APIError(400, "invalid_request", "Неверный cursor") from exc
|
||||||
|
return created_at, item_id
|
||||||
|
|
||||||
|
|
||||||
|
store = PostgresStore()
|
||||||
@@ -0,0 +1,206 @@
|
|||||||
|
"""Small, explicit OTel instrumentation. Only allowlisted technical fields leave the app."""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import time
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
from opentelemetry import metrics, trace
|
||||||
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||||
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider, Histogram
|
||||||
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||||
|
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||||
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||||
|
|
||||||
|
tracer = trace.get_tracer('messenger', '1.6.0')
|
||||||
|
meter = metrics.get_meter('messenger', '1.6.0')
|
||||||
|
propagator = TraceContextTextMapPropagator() # No baggage: it may contain user data.
|
||||||
|
logger = logging.getLogger('messenger.events')
|
||||||
|
logger.propagate = False
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
providers = []
|
||||||
|
LOG_FIELDS = {'method', 'route', 'status_class', 'status', 'duration_seconds', 'request_id',
|
||||||
|
'operation', 'error_type', 'attempt', 'outcome', 'error_code', 'job_type'}
|
||||||
|
http_requests = meter.create_counter('http_requests')
|
||||||
|
http_duration = meter.create_histogram('http_duration', unit='s')
|
||||||
|
jobs_processed = meter.create_counter('jobs_processed')
|
||||||
|
job_duration = meter.create_histogram('job_duration', unit='s')
|
||||||
|
time_to_ready = meter.create_histogram('attachment_time_to_ready', unit='s')
|
||||||
|
worker_busy = meter.create_up_down_counter('worker_busy')
|
||||||
|
queue_state = {'depth': 0, 'age': 0, 'updated': 0}
|
||||||
|
|
||||||
|
|
||||||
|
class JsonFormatter(logging.Formatter):
|
||||||
|
def format(self, record):
|
||||||
|
context = trace.get_current_span().get_span_context()
|
||||||
|
return json.dumps({
|
||||||
|
'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
|
||||||
|
'severity': record.levelname, 'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'),
|
||||||
|
'event': record.getMessage(),
|
||||||
|
'trace_id': format(context.trace_id, '032x') if context.is_valid else '',
|
||||||
|
'span_id': format(context.span_id, '016x') if context.is_valid else '',
|
||||||
|
**{key: record.__dict__[key] for key in LOG_FIELDS if key in record.__dict__},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def event(name, *, severity=logging.INFO, **fields):
|
||||||
|
# Call sites pass fixed event names, safe enums/numbers and technical correlation IDs.
|
||||||
|
logger.log(severity, name, extra={'event': name, **{k: v for k, v in fields.items() if k in LOG_FIELDS}})
|
||||||
|
|
||||||
|
|
||||||
|
def setup(*, child=False):
|
||||||
|
if providers or os.getenv('OTEL_SDK_DISABLED', 'false').lower() == 'true':
|
||||||
|
return
|
||||||
|
resource = Resource({
|
||||||
|
'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'),
|
||||||
|
'service.version': '1.6.0', 'service.instance.id': socket.gethostname(),
|
||||||
|
'deployment.environment.name': 'lab',
|
||||||
|
})
|
||||||
|
endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
|
||||||
|
traces = TracerProvider(resource=resource)
|
||||||
|
traces.add_span_processor(BatchSpanProcessor(
|
||||||
|
OTLPSpanExporter(endpoint=endpoint + '/v1/traces', timeout=1),
|
||||||
|
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000))
|
||||||
|
trace.set_tracer_provider(traces)
|
||||||
|
logs = LoggerProvider(resource=resource)
|
||||||
|
logs.add_log_record_processor(BatchLogRecordProcessor(
|
||||||
|
OTLPLogExporter(endpoint=endpoint + '/v1/logs', timeout=1),
|
||||||
|
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000))
|
||||||
|
console = logging.StreamHandler()
|
||||||
|
console.setFormatter(JsonFormatter())
|
||||||
|
logger.addHandler(console)
|
||||||
|
logger.addHandler(LoggingHandler(logger_provider=logs))
|
||||||
|
providers.extend([traces, logs])
|
||||||
|
if child:
|
||||||
|
return # Short-lived image processes emit spans/logs; parent owns worker metrics.
|
||||||
|
reader = PeriodicExportingMetricReader(
|
||||||
|
OTLPMetricExporter(endpoint=endpoint + '/v1/metrics', timeout=1),
|
||||||
|
export_interval_millis=5000, export_timeout_millis=2000)
|
||||||
|
buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 30, 60, 120, 300]
|
||||||
|
metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
|
||||||
|
View(instrument_type=Histogram,
|
||||||
|
aggregation=ExplicitBucketHistogramAggregation(buckets))])
|
||||||
|
metrics.set_meter_provider(metric_provider)
|
||||||
|
providers.append(metric_provider)
|
||||||
|
process = psutil.Process()
|
||||||
|
meter.create_observable_gauge('process_memory', unit='By', callbacks=[
|
||||||
|
lambda _: [metrics.Observation(process.memory_info().rss)]])
|
||||||
|
meter.create_observable_counter('process_cpu', unit='s', callbacks=[
|
||||||
|
lambda _: [metrics.Observation(sum(process.cpu_times()[:2]))]])
|
||||||
|
cgroup = Path('/sys/fs/cgroup')
|
||||||
|
if (cgroup / 'memory.current').exists():
|
||||||
|
meter.create_observable_gauge('container_memory', unit='By', callbacks=[
|
||||||
|
lambda _: [metrics.Observation(int((cgroup / 'memory.current').read_text()))]])
|
||||||
|
def cpu_usage(_):
|
||||||
|
values = dict(line.split() for line in (cgroup / 'cpu.stat').read_text().splitlines())
|
||||||
|
return [metrics.Observation(int(values['usage_usec']) / 1_000_000)]
|
||||||
|
meter.create_observable_counter('container_cpu', unit='s', callbacks=[cpu_usage])
|
||||||
|
if os.getenv('OTEL_SERVICE_NAME') == 'messenger-worker':
|
||||||
|
capacity = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
|
||||||
|
meter.create_observable_gauge('worker_capacity', callbacks=[
|
||||||
|
lambda _: [metrics.Observation(capacity)]])
|
||||||
|
worker_busy.add(0)
|
||||||
|
if os.getenv('OTEL_SERVICE_NAME') == 'messenger-dispatcher':
|
||||||
|
for name, key, unit in [('queue_depth', 'depth', ''), ('queue_oldest_age', 'age', 's'),
|
||||||
|
('queue_sample_timestamp', 'updated', 's')]:
|
||||||
|
meter.create_observable_gauge(name, unit=unit, callbacks=[
|
||||||
|
lambda _, key=key: [metrics.Observation(queue_state[key], {'queue': 'attachment-jobs'})]])
|
||||||
|
|
||||||
|
|
||||||
|
def shutdown():
|
||||||
|
for provider in reversed(providers):
|
||||||
|
provider.shutdown()
|
||||||
|
providers.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def seed_http(routes):
|
||||||
|
# Export zero before the first failure: Prometheus rate needs a prior sample.
|
||||||
|
for route in routes:
|
||||||
|
if route.path.startswith('/health/'):
|
||||||
|
continue
|
||||||
|
for method in route.methods or ():
|
||||||
|
for status in ('2xx', '3xx', '4xx', '5xx'):
|
||||||
|
http_requests.add(0, {'method': method, 'route': route.path, 'status_class': status})
|
||||||
|
|
||||||
|
|
||||||
|
def carrier():
|
||||||
|
result = {}
|
||||||
|
propagator.inject(result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def span(name, *, kind=SpanKind.INTERNAL, context=None, attributes=None):
|
||||||
|
# Never export exception messages/stack traces (SQL errors can embed parameters).
|
||||||
|
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attributes,
|
||||||
|
record_exception=False, set_status_on_exception=False) as current:
|
||||||
|
try:
|
||||||
|
yield current
|
||||||
|
except Exception as exc:
|
||||||
|
current.set_status(Status(StatusCode.ERROR))
|
||||||
|
current.set_attribute('error.type', type(exc).__name__)
|
||||||
|
event('operation_failed', severity=logging.ERROR, operation=name, error_type=type(exc).__name__)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def dependency_span(name, **attributes):
|
||||||
|
# Background health probes and queue polling must not create disconnected trace noise.
|
||||||
|
if not trace.get_current_span().get_span_context().is_valid:
|
||||||
|
yield
|
||||||
|
else:
|
||||||
|
with span(name, kind=SpanKind.CLIENT, attributes=attributes):
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
class HttpTelemetry:
|
||||||
|
def __init__(self, app):
|
||||||
|
self.app = app
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
headers = {k.decode('latin1'): v.decode('latin1') for k, v in scope['headers']
|
||||||
|
if k in (b'traceparent', b'tracestate')}
|
||||||
|
method = scope['method'] if scope['method'] in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER'
|
||||||
|
status = 500
|
||||||
|
started = time.monotonic()
|
||||||
|
with span('HTTP ' + method, kind=SpanKind.SERVER, context=propagator.extract(headers)) as current:
|
||||||
|
async def response(message):
|
||||||
|
nonlocal status
|
||||||
|
if message['type'] == 'http.response.start':
|
||||||
|
status = message['status']
|
||||||
|
await send(message)
|
||||||
|
try:
|
||||||
|
await self.app(scope, receive, response)
|
||||||
|
finally:
|
||||||
|
route = getattr(scope.get('route'), 'path', 'unmatched')
|
||||||
|
labels = {'method': method, 'route': route, 'status_class': f'{status // 100}xx'}
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
http_requests.add(1, labels)
|
||||||
|
http_duration.record(elapsed, labels)
|
||||||
|
current.update_name(method + ' ' + route)
|
||||||
|
current.set_attributes({'http.request.method': method, 'http.route': route,
|
||||||
|
'http.response.status_code': status})
|
||||||
|
if status >= 500:
|
||||||
|
current.set_status(Status(StatusCode.ERROR))
|
||||||
|
request_id = scope.get('state', {}).get('request_id', '')
|
||||||
|
if not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', request_id):
|
||||||
|
request_id = ''
|
||||||
|
current.set_attribute('request_id', request_id)
|
||||||
|
event('http_request', severity=logging.ERROR if status >= 500 else logging.INFO,
|
||||||
|
**labels, status=status, duration_seconds=elapsed,
|
||||||
|
request_id=request_id)
|
||||||
+122
@@ -0,0 +1,122 @@
|
|||||||
|
"""Bounded consumers, one unacknowledged message per consumer, hard process timeout."""
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from .database import create_db_engine
|
||||||
|
from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure
|
||||||
|
from .models import Attachment, AttachmentJob
|
||||||
|
from . import telemetry as otel
|
||||||
|
|
||||||
|
|
||||||
|
def handle(sessions, client, message_id, fields):
|
||||||
|
try:
|
||||||
|
message = json.loads(fields['job'])
|
||||||
|
if not isinstance(message, dict):
|
||||||
|
raise ValueError('invalid envelope')
|
||||||
|
except (ValueError, KeyError, TypeError):
|
||||||
|
otel.event('invalid_job')
|
||||||
|
acknowledge(client, message_id)
|
||||||
|
return
|
||||||
|
with otel.span('attachment-jobs process', kind=otel.SpanKind.CONSUMER,
|
||||||
|
context=otel.propagator.extract(message), attributes={
|
||||||
|
'messaging.system': 'redis', 'messaging.destination.name': STREAM,
|
||||||
|
'messaging.operation.type': 'process', 'job.attempt': message.get('attempt', 0)}):
|
||||||
|
process_message(sessions, client, message_id, message)
|
||||||
|
|
||||||
|
|
||||||
|
def process_message(sessions, client, message_id, message):
|
||||||
|
try:
|
||||||
|
claimed = claim(sessions, message)
|
||||||
|
except (ValueError, KeyError, TypeError):
|
||||||
|
otel.event('invalid_job')
|
||||||
|
claimed = None
|
||||||
|
if claimed:
|
||||||
|
job_id, token = claimed
|
||||||
|
started = time.monotonic()
|
||||||
|
outcome = 'failed'
|
||||||
|
otel.worker_busy.add(1)
|
||||||
|
try:
|
||||||
|
environment = {**os.environ, 'OTEL_CHILD_PROCESS': 'true'}
|
||||||
|
environment.update({key.upper(): value for key, value in otel.carrier().items()})
|
||||||
|
try:
|
||||||
|
result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token],
|
||||||
|
timeout=TIMEOUT, check=False, env=environment)
|
||||||
|
code = result.returncode
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
code = 23
|
||||||
|
if code in (91, 92):
|
||||||
|
# Simulate a lost worker before ack: leave the pending entry and lease.
|
||||||
|
return
|
||||||
|
if code:
|
||||||
|
record_failure(sessions, job_id, token,
|
||||||
|
{20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'}
|
||||||
|
.get(code, 'processing_failed'), permanent=code in (20, 21))
|
||||||
|
otel.trace.get_current_span().set_status(otel.Status(otel.StatusCode.ERROR))
|
||||||
|
else:
|
||||||
|
with sessions() as session:
|
||||||
|
job = session.get(AttachmentJob, job_id)
|
||||||
|
attachment = session.get(Attachment, job.attachment_id)
|
||||||
|
outcome = 'ready' if attachment.status == 'ready' else 'skipped'
|
||||||
|
if outcome == 'ready':
|
||||||
|
otel.time_to_ready.record((attachment.updated_at - attachment.created_at).total_seconds(),
|
||||||
|
{'job_type': 'attachment.process.v1'})
|
||||||
|
finally:
|
||||||
|
otel.worker_busy.add(-1)
|
||||||
|
labels = {'job_type': 'attachment.process.v1', 'outcome': outcome}
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
otel.jobs_processed.add(1, labels)
|
||||||
|
otel.job_duration.record(elapsed, labels)
|
||||||
|
otel.event('job_processed', severity=logging.ERROR if outcome == 'failed' else logging.INFO,
|
||||||
|
**labels, duration_seconds=elapsed)
|
||||||
|
# Only after a durable ready/failed/retry (or an existing durable lease).
|
||||||
|
acknowledge(client, message_id)
|
||||||
|
|
||||||
|
|
||||||
|
def acknowledge(client, message_id):
|
||||||
|
with client.pipeline(transaction=True) as pipe:
|
||||||
|
pipe.xack(STREAM, GROUP, message_id)
|
||||||
|
pipe.xdel(STREAM, message_id)
|
||||||
|
pipe.execute()
|
||||||
|
|
||||||
|
|
||||||
|
def consume(sessions):
|
||||||
|
client = broker()
|
||||||
|
consumer = str(uuid4())
|
||||||
|
cursor = '0-0'
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
ensure_group(client)
|
||||||
|
pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000,
|
||||||
|
start_id=cursor, count=1)
|
||||||
|
cursor, messages = pending[0], pending[1]
|
||||||
|
if not messages:
|
||||||
|
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
|
||||||
|
messages = batches[0][1] if batches else []
|
||||||
|
for message_id, fields in messages:
|
||||||
|
handle(sessions, client, message_id, fields)
|
||||||
|
except Exception as exc:
|
||||||
|
otel.event('worker_deferred', error_type=type(exc).__name__)
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
otel.setup()
|
||||||
|
engine = create_db_engine()
|
||||||
|
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||||
|
concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
|
||||||
|
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||||
|
for task in [pool.submit(consume, sessions) for _ in range(concurrency)]:
|
||||||
|
task.result()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,290 @@
|
|||||||
|
"""Lab 6 acceptance: real signals, two incidents, retry propagation and bounded outage.
|
||||||
|
|
||||||
|
Uses only the standard library. Stops services only in this Compose project and
|
||||||
|
always starts them again; never deletes volumes. Run separately from other tests.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / 'tests'))
|
||||||
|
from client import Client, check
|
||||||
|
|
||||||
|
PROM = 'http://127.0.0.1:9090'
|
||||||
|
LOKI = 'http://127.0.0.1:3100'
|
||||||
|
TEMPO = 'http://127.0.0.1:3200'
|
||||||
|
|
||||||
|
|
||||||
|
def stamp():
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def get(url, **params):
|
||||||
|
if params:
|
||||||
|
url += '?' + urllib.parse.urlencode(params)
|
||||||
|
with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=10) as r:
|
||||||
|
return json.load(r)
|
||||||
|
|
||||||
|
|
||||||
|
def query(expr):
|
||||||
|
data = get(PROM + '/api/v1/query', query=expr)
|
||||||
|
check(data['status'] == 'success', 'PromQL failed')
|
||||||
|
return data['data']['result']
|
||||||
|
|
||||||
|
|
||||||
|
def number(expr):
|
||||||
|
values = query(expr)
|
||||||
|
return float(values[0]['value'][1]) if values else 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def wait(label, fn, timeout=90):
|
||||||
|
end = time.monotonic() + timeout
|
||||||
|
last = None
|
||||||
|
while time.monotonic() < end:
|
||||||
|
try:
|
||||||
|
value = fn()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
except (urllib.error.URLError, TimeoutError, AssertionError) as exc:
|
||||||
|
last = type(exc).__name__
|
||||||
|
time.sleep(2)
|
||||||
|
raise AssertionError(f'Timeout: {label} ({last})')
|
||||||
|
|
||||||
|
|
||||||
|
def compose(*args):
|
||||||
|
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True,
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90)
|
||||||
|
|
||||||
|
|
||||||
|
def trace_data(trace_id):
|
||||||
|
data = get(TEMPO + '/api/traces/' + trace_id)
|
||||||
|
spans = []
|
||||||
|
for batch in data.get('batches', data.get('resourceSpans', [])):
|
||||||
|
resource = {a['key']: a['value'].get('stringValue') for a in batch.get('resource', {}).get('attributes', [])}
|
||||||
|
for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])):
|
||||||
|
for span in scope.get('spans', []):
|
||||||
|
spans.append({**span, 'service': resource.get('service.name')})
|
||||||
|
return data, spans
|
||||||
|
|
||||||
|
|
||||||
|
def logs(trace_id):
|
||||||
|
return get(LOKI + '/loki/api/v1/query_range',
|
||||||
|
query='{service_name=~"messenger-.*"} | trace_id="' + trace_id + '"',
|
||||||
|
start=str(time.time_ns() - 3_600_000_000_000), limit=1000)['data']['result']
|
||||||
|
|
||||||
|
|
||||||
|
def attrs(span):
|
||||||
|
return {a['key']: next(iter(a['value'].values())) for a in span.get('attributes', [])}
|
||||||
|
|
||||||
|
|
||||||
|
def pipeline(trace_id, *, retry=False):
|
||||||
|
data, spans = trace_data(trace_id)
|
||||||
|
names = {s['name'] for s in spans}
|
||||||
|
required = {'POST /api/v1/chats/{chat_id}/attachments', 'attachment-jobs publish',
|
||||||
|
'attachment-jobs process', 'image.process', 's3.GetObject', 's3.PutObject', 'db.UPDATE'}
|
||||||
|
if not required <= names:
|
||||||
|
return None
|
||||||
|
by_id = {s['spanId']: s for s in spans}
|
||||||
|
for name, parent_name in [('attachment-jobs publish', 'POST /api/v1/chats/{chat_id}/attachments'),
|
||||||
|
('attachment-jobs process', 'attachment-jobs publish'),
|
||||||
|
('image.process', 'attachment-jobs process')]:
|
||||||
|
for s in (s for s in spans if s['name'] == name):
|
||||||
|
check(by_id.get(s.get('parentSpanId'), {}).get('name') == parent_name,
|
||||||
|
'Broken trace parent for ' + name)
|
||||||
|
check(s.get('traceState') == 'lab=check', 'W3C tracestate lost at ' + name)
|
||||||
|
attempts = sorted({int(attrs(s)['job.attempt']) for s in spans
|
||||||
|
if s['name'] == 'attachment-jobs process'})
|
||||||
|
if retry and len(attempts) < 2:
|
||||||
|
return None
|
||||||
|
records = logs(trace_id)
|
||||||
|
services = {r['stream'].get('service_name') for r in records}
|
||||||
|
if not {'messenger-api', 'messenger-worker', 'messenger-dispatcher'} <= services:
|
||||||
|
return None
|
||||||
|
return {'trace_id': trace_id, 'span_count': len(spans), 'span_names': sorted(names),
|
||||||
|
'log_services': sorted(services), 'attempts': attempts, 'parent_chain_verified': True,
|
||||||
|
'tracestate_preserved': True,
|
||||||
|
'_raw': json.dumps([data, records])}
|
||||||
|
|
||||||
|
|
||||||
|
class Scenario:
|
||||||
|
def __init__(self, base_url):
|
||||||
|
self.client = Client(base_url)
|
||||||
|
self.alice, _, _, self.chat = self.client.bootstrap()
|
||||||
|
self.private_markers = [self.alice['token'], self.alice['registration']['password']]
|
||||||
|
|
||||||
|
def upload(self):
|
||||||
|
trace_id, request_id = uuid.uuid4().hex, str(uuid.uuid4())
|
||||||
|
filename = 'private_filename_' + uuid.uuid4().hex + '.png'
|
||||||
|
self.private_markers.append(filename)
|
||||||
|
result = self.client.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=' + filename,
|
||||||
|
202, actor=self.alice, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
|
||||||
|
headers={'Content-Type': 'image/png', 'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01',
|
||||||
|
'tracestate': 'lab=check', 'X-Request-Id': request_id}, schema='Attachment')
|
||||||
|
check(result['headers']['X-Request-Id'] == request_id, 'Request ID changed')
|
||||||
|
return result['json'], trace_id, request_id
|
||||||
|
|
||||||
|
def check_pipeline(self, trace_id, request_id, retry=False):
|
||||||
|
result = wait('pipeline spans/logs', lambda: pipeline(trace_id, retry=retry))
|
||||||
|
raw = result.pop('_raw')
|
||||||
|
check(all(value not in raw for value in self.private_markers), 'Private input leaked into telemetry')
|
||||||
|
check(request_id in raw, 'Request ID is missing from trace/log correlation')
|
||||||
|
result['request_id'] = request_id
|
||||||
|
result['private_inputs_absent'] = True
|
||||||
|
return result
|
||||||
|
|
||||||
|
def business(self, expected=200, trace_id=None):
|
||||||
|
headers = {'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01'} if trace_id else {}
|
||||||
|
return self.client.call('GET', '/api/v1/chats/' + self.chat['id'], expected,
|
||||||
|
actor=self.alice, headers=headers, schema='Chat' if expected == 200 else None)
|
||||||
|
|
||||||
|
def smoke(self):
|
||||||
|
att, trace_id, request_id = self.upload()
|
||||||
|
self.client.wait_ready(self.alice, att)
|
||||||
|
result = self.check_pipeline(trace_id, request_id)
|
||||||
|
names = ['http_requests_total', 'http_duration_seconds_bucket', 'jobs_processed_total',
|
||||||
|
'job_duration_seconds_bucket', 'attachment_time_to_ready_seconds_bucket',
|
||||||
|
'queue_depth', 'queue_oldest_age_seconds', 'db_pool_used', 'db_pool_waiting',
|
||||||
|
'worker_busy', 'worker_capacity', 'container_cpu_seconds_total', 'container_memory_bytes']
|
||||||
|
for name in names:
|
||||||
|
wait(name, lambda name=name: query(name))
|
||||||
|
series = get(PROM + '/api/v1/series', **{'match[]': '{job="messenger"}', 'start': str(time.time() - 300)})['data']
|
||||||
|
forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url'}
|
||||||
|
check(all(not forbidden.intersection(row) for row in series), 'High-cardinality metric label')
|
||||||
|
check(all(self.chat['id'] not in json.dumps(row) for row in series), 'Raw route ID in metrics')
|
||||||
|
result['metrics_present'] = names
|
||||||
|
return result
|
||||||
|
|
||||||
|
def queue(self):
|
||||||
|
result = {'started_at': stamp()}
|
||||||
|
compose('stop', 'worker')
|
||||||
|
try:
|
||||||
|
att, trace_id, request_id = self.upload()
|
||||||
|
self.client.message(self.alice, self.chat, 'queued attachment', attachments=[att['id']])
|
||||||
|
wait('QueueLagHigh firing', lambda: number('ALERTS{alertname="QueueLagHigh",alertstate="firing"}') == 1)
|
||||||
|
result.update(firing_at=stamp(), depth=number('max(queue_depth)'),
|
||||||
|
oldest_age_seconds=number('max(queue_oldest_age_seconds)'))
|
||||||
|
finally:
|
||||||
|
compose('start', 'worker')
|
||||||
|
ready = self.client.wait_ready(self.alice, att)
|
||||||
|
wait('queue alert resolved', lambda: not query('ALERTS{alertname="QueueLagHigh"}'))
|
||||||
|
result.update(resolved_at=stamp(), ready=ready['status'], depth_after=number('max(queue_depth)'),
|
||||||
|
pipeline=self.check_pipeline(trace_id, request_id))
|
||||||
|
return result
|
||||||
|
|
||||||
|
def database(self):
|
||||||
|
result = {'started_at': stamp()}
|
||||||
|
trace_id = uuid.uuid4().hex
|
||||||
|
compose('stop', 'postgres')
|
||||||
|
try:
|
||||||
|
errors = 0
|
||||||
|
def load_until_firing():
|
||||||
|
nonlocal errors
|
||||||
|
self.business(expected=503, trace_id=trace_id if errors == 0 else None)
|
||||||
|
errors += 1
|
||||||
|
return number('ALERTS{alertname="ApiHighErrorRate",alertstate="firing"}') == 1
|
||||||
|
wait('ApiHighErrorRate firing under sustained traffic', load_until_firing, timeout=120)
|
||||||
|
result['failed_requests'] = errors
|
||||||
|
result.update(firing_at=stamp(), error_ratio=number('messenger:http_error_ratio:2m'))
|
||||||
|
def failed_trace():
|
||||||
|
_, spans = trace_data(trace_id)
|
||||||
|
failed = [s for s in spans if s['name'].startswith('db.')
|
||||||
|
and s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')]
|
||||||
|
return [{'name': s['name'], 'error_type': attrs(s).get('error.type'),
|
||||||
|
'duration_seconds': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9}
|
||||||
|
for s in failed]
|
||||||
|
result['db_error_spans'] = wait('failed DB span', failed_trace)
|
||||||
|
result['trace_id'] = trace_id
|
||||||
|
result['correlated_log_streams'] = len(wait('DB error logs', lambda: logs(trace_id)))
|
||||||
|
finally:
|
||||||
|
compose('start', 'postgres')
|
||||||
|
wait('API recovered', self.business)
|
||||||
|
for _ in range(10):
|
||||||
|
self.business()
|
||||||
|
wait('API alert resolved', lambda: not query('ALERTS{alertname="ApiHighErrorRate"}'), timeout=160)
|
||||||
|
result['resolved_at'] = stamp()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def retry(self):
|
||||||
|
failures = number('sum(jobs_processed_total{outcome="failed"})')
|
||||||
|
compose('stop', 'worker')
|
||||||
|
try:
|
||||||
|
att, trace_id, request_id = self.upload()
|
||||||
|
compose('stop', 's3')
|
||||||
|
# `compose start worker` can also start its dependency graph, including S3.
|
||||||
|
compose('up', '-d', '--no-deps', '--no-recreate', 'worker')
|
||||||
|
wait('failed processing attempt', lambda: number('sum(jobs_processed_total{outcome="failed"})') > failures, timeout=40)
|
||||||
|
finally:
|
||||||
|
compose('start', 's3', 'worker')
|
||||||
|
wait('S3/API ready', lambda: self.client.call('GET', '/health/ready', schema='Health'))
|
||||||
|
self.client.wait_ready(self.alice, att)
|
||||||
|
return self.check_pipeline(trace_id, request_id, retry=True)
|
||||||
|
|
||||||
|
def backend(self):
|
||||||
|
accepted = 'sum(otelcol_receiver_accepted_spans_total)'
|
||||||
|
sent = 'sum(otelcol_exporter_sent_spans_total{exporter="otlphttp/tempo"})'
|
||||||
|
queue = 'sum(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})'
|
||||||
|
# A quiet interval on both ends makes accepted-minus-sent a loss count,
|
||||||
|
# not a count of failed export attempts or currently buffered spans.
|
||||||
|
time.sleep(8)
|
||||||
|
wait('trace queue drained', lambda: number(queue) == 0)
|
||||||
|
before = {'accepted': number(accepted), 'sent': number(sent)}
|
||||||
|
result = {'started_at': stamp(), 'before': before}
|
||||||
|
samples, writes = [], 0
|
||||||
|
compose('stop', 'tempo')
|
||||||
|
try:
|
||||||
|
end = time.monotonic() + 60
|
||||||
|
att, _, _ = self.upload()
|
||||||
|
self.client.wait_ready(self.alice, att)
|
||||||
|
while time.monotonic() < end:
|
||||||
|
body = 'private_message_' + uuid.uuid4().hex
|
||||||
|
self.private_markers.append(body)
|
||||||
|
self.client.message(self.alice, self.chat, body)
|
||||||
|
writes += 1
|
||||||
|
samples.append({'at': stamp(), 'collector_rss_bytes': number('otelcol_process_memory_rss_bytes'),
|
||||||
|
'queue_batches': number(queue)})
|
||||||
|
time.sleep(0.3)
|
||||||
|
check(writes > 10, 'Not enough writes during telemetry outage')
|
||||||
|
check(max(s['queue_batches'] for s in samples) <= 32, 'Collector queue is unbounded')
|
||||||
|
check(0 < max(s['collector_rss_bytes'] for s in samples) < 256 * 1024 * 1024,
|
||||||
|
'Collector exceeded its memory budget')
|
||||||
|
finally:
|
||||||
|
compose('start', 'tempo')
|
||||||
|
wait('trace queue drained', lambda: number(queue) == 0)
|
||||||
|
time.sleep(10) # Include final Prometheus scrape after drain.
|
||||||
|
after = {'accepted': number(accepted), 'sent': number(sent)}
|
||||||
|
lost = (after['accepted'] - before['accepted']) - (after['sent'] - before['sent'])
|
||||||
|
check(lost >= 0, 'Loss window was not quiescent; rerun backend separately')
|
||||||
|
result.update(after=after, lost_accepted_spans=lost, successful_writes=writes,
|
||||||
|
attachment_ready_during_outage=True, samples=samples, recovered_at=stamp())
|
||||||
|
att, trace_id, request_id = self.upload()
|
||||||
|
self.client.wait_ready(self.alice, att)
|
||||||
|
result['recovered_pipeline'] = self.check_pipeline(trace_id, request_id)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument('--scenario', choices=['smoke', 'queue', 'database', 'retry', 'backend', 'all'], default='smoke')
|
||||||
|
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||||
|
args = parser.parse_args()
|
||||||
|
scenario = Scenario(args.base_url)
|
||||||
|
path = ROOT / 'evidence/observability.json'
|
||||||
|
evidence = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {}
|
||||||
|
for name in (['smoke', 'queue', 'database', 'retry', 'backend'] if args.scenario == 'all' else [args.scenario]):
|
||||||
|
print('Running ' + name, flush=True)
|
||||||
|
evidence[name] = {'checked_at': stamp(), **getattr(scenario, name)()}
|
||||||
|
path.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
|
||||||
|
print('PASS: ' + name, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user