forked from mtusi2026/mtusi2026lab6
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c81dab8d95 | ||
|
|
67039fd30d | ||
|
|
9e6428de05 | ||
|
|
58f3d4c457 |
+37
-14
@@ -1,30 +1,53 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Local synthetic lab environment only. Copy to .env and replace every change-me value.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
LOG_LEVEL=info
|
||||
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-postgres
|
||||
POSTGRES_DB=messenger
|
||||
DATABASE_URL=postgresql+asyncpg://messenger:change-me-postgres@postgres:5432/messenger
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
MINIO_ROOT_USER=local-minio-admin
|
||||
MINIO_ROOT_PASSWORD=change-me-minio-root
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=change-me-minio-api
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
|
||||
BROKER_USER=messenger
|
||||
BROKER_PASSWORD=change-me-rabbitmq
|
||||
BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=90
|
||||
RETRY_BASE_SECONDS=2
|
||||
RETRY_MAX_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
THUMBNAIL_JPEG_QUALITY=85
|
||||
CROP_JPEG_QUALITY=85
|
||||
OPTIMIZED_WEBP_QUALITY=82
|
||||
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
OTEL_SERVICE_VERSION=1.6.0
|
||||
OTEL_DEPLOYMENT_ENVIRONMENT=lab
|
||||
OTEL_SDK_DISABLED=false
|
||||
OTEL_EXPORTER_OTLP_TIMEOUT=2
|
||||
OTEL_BSP_MAX_QUEUE_SIZE=2048
|
||||
OTEL_BLRP_MAX_QUEUE_SIZE=2048
|
||||
OTEL_METRIC_EXPORT_INTERVAL=5000
|
||||
|
||||
GRAFANA_PORT=3000
|
||||
GRAFANA_ADMIN_USER=admin
|
||||
GRAFANA_ADMIN_PASSWORD=change-me-grafana
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
FROM python:3.13.15-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install -r requirements.txt \
|
||||
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
|
||||
COPY app ./app
|
||||
COPY alembic ./alembic
|
||||
COPY alembic.ini .
|
||||
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"]
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"]
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM python:3.13.15-slim
|
||||
|
||||
WORKDIR /tests
|
||||
COPY requirements.txt requirements-dev.txt ./
|
||||
RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \
|
||||
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests
|
||||
|
||||
COPY lab.json .
|
||||
COPY app ./app
|
||||
COPY contracts ./contracts
|
||||
COPY tests ./tests
|
||||
RUN chown -R tests:tests /tests
|
||||
|
||||
USER 10001:10001
|
||||
CMD ["python", "-m", "pytest", "-q"]
|
||||
@@ -3,11 +3,12 @@ BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
OBSERVABILITY_COMPOSE = docker compose -f compose.yaml -f compose.observability-tests.yaml
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test test-observability test-retry-trace test-alert-worker test-alert-database test-telemetry-outage load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "check: offline structure; validate: full OpenAPI; test: public API; test-observability: Grafana/Prometheus/Loki/Tempo"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -18,6 +19,21 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-observability:
|
||||
$(OBSERVABILITY_COMPOSE) run --rm observability-test
|
||||
|
||||
test-retry-trace:
|
||||
$(PYTHON) tests/retry_trace.py --base-url "$(BASE_URL)"
|
||||
|
||||
test-alert-worker:
|
||||
$(PYTHON) tests/alert_scenarios.py worker --base-url "$(BASE_URL)"
|
||||
|
||||
test-alert-database:
|
||||
$(PYTHON) tests/alert_scenarios.py database --base-url "$(BASE_URL)"
|
||||
|
||||
test-telemetry-outage:
|
||||
$(PYTHON) tests/telemetry_outage.py --base-url "$(BASE_URL)"
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,37 +1,207 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Сергей Возжин, БПИ2301.
|
||||
- Целевая оценка: **4**. Трек оценки 5 не заявлен.
|
||||
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/61
|
||||
- Использованы Python 3.13, FastAPI, OpenTelemetry Python 1.31.1,
|
||||
OTel Collector Contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2,
|
||||
Tempo 2.7.2 и Grafana 11.6.0. Для планирования и реализации применялся ИИ.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose v2, GNU Make и Python 3.9+ для host-side тестов.
|
||||
Все образы и Python-зависимости закреплены в репозитории.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# Заменить все change-me значения согласованными локальными секретами.
|
||||
# Пароли в DATABASE_URL/BROKER_URL должны совпадать с POSTGRES_*/BROKER_*.
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait
|
||||
make check
|
||||
make test BASE_URL=http://localhost:8080
|
||||
make test-observability
|
||||
```
|
||||
|
||||
Grafana доступна только локально на `http://127.0.0.1:3000`. Dashboard,
|
||||
datasources и alerts создаются provisioning-механизмом автоматически. Миграции
|
||||
Alembic выполняет одноразовый сервис `migrate`; MinIO bucket и технического
|
||||
пользователя создаёт `s3-bootstrap`.
|
||||
|
||||
Безопасная остановка без удаления данных:
|
||||
|
||||
```sh
|
||||
docker compose down
|
||||
```
|
||||
|
||||
Намеренное удаление всех **локальных учебных** данных выполняется отдельно и
|
||||
необратимо:
|
||||
|
||||
```sh
|
||||
docker compose down --volumes
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
client -> HAProxy -> API x2 -> PostgreSQL / Redis / S3
|
||||
|
|
||||
+-> transactional outbox -> publisher -> RabbitMQ -> worker -> S3 + PostgreSQL
|
||||
|
||||
API / publisher / worker -> OTLP HTTP -> OTel Collector
|
||||
|-> Prometheus exporter <- Prometheus
|
||||
|-> OTLP logs -> Loki
|
||||
`-> OTLP traces -> Tempo
|
||||
Prometheus + Loki + Tempo <- provisioned datasources <- Grafana
|
||||
```
|
||||
|
||||
Из ЛР 5 без изменения бизнес-API перенесены две API-реплики, HAProxy,
|
||||
PostgreSQL, Redis sessions/idempotency, MinIO S3, RabbitMQ, transactional
|
||||
outbox, publisher, worker и обработка изображений. PostgreSQL остаётся источником
|
||||
истины для attachments/jobs/outbox. Durable acceptance атомарно создаёт attachment,
|
||||
job и outbox row; публикация и обработка повторяемы.
|
||||
|
||||
В outbox добавлены nullable `traceparent` и `tracestate`. API сохраняет W3C context
|
||||
в той же транзакции, publisher восстанавливает его, создаёт PRODUCER span и передаёт
|
||||
новый context в allowlisted AMQP headers. Worker извлекает headers и создаёт CONSUMER
|
||||
span. В одной трассе видны `messenger-api`, `messenger-publisher`,
|
||||
`messenger-worker`, SQL и ручные S3 CLIENT spans. Retry создаёт новую публикацию с
|
||||
текущим context; повторная обработка остаётся наблюдаемой и идемпотентной.
|
||||
|
||||
SDK использует bounded batch queues по 2048 элементов и timeout 2 с. Collector:
|
||||
`memory_limiter` 192 MiB со spike 48 MiB, `batch` до 1024 элементов, exporter queues
|
||||
по 500 элементов, два consumer, exponential retry не дольше 30 с. Поэтому отказ
|
||||
Loki/Tempo приводит к ограниченной потере telemetry, а не блокировке API или
|
||||
неограниченному росту RAM.
|
||||
Collector self-metrics экспортируются через `readers.pull` на `:8888`;
|
||||
Prometheus сохраняет RSS, размеры очередей и `send_failed` counters.
|
||||
|
||||
Секреты и пользовательское содержимое не экспортируются. `safe_extra` отбрасывает
|
||||
пароли, tokens, Authorization, secrets, message body, image bytes и signed URL.
|
||||
Метрики содержат только низкокардинальные route templates, method/status class,
|
||||
job type/outcome и queue name; UUID и filename labels отсутствуют. Технические IDs
|
||||
допустимы только в logs/spans для синтетических данных.
|
||||
|
||||
## Сигналы, dashboard и единицы
|
||||
|
||||
| Контракт | Реальная серия | Единица / основные labels |
|
||||
| --- | --- | --- |
|
||||
| HTTP requests | `http_server_request_count_total` | requests; method, route template, status class |
|
||||
| HTTP duration | `http_server_request_duration_seconds` | seconds, histogram |
|
||||
| Jobs outcome | `messenger_jobs_processed_total` | jobs; job type, outcome |
|
||||
| Job duration | `messenger_job_processing_duration_seconds` | seconds, histogram |
|
||||
| Time to ready | `messenger_attachment_time_to_ready_seconds` | seconds, histogram |
|
||||
| Queue | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | jobs / seconds |
|
||||
| DB pool | `messenger_db_pool_used`, `messenger_db_pool_waiting` | connections / waiters |
|
||||
| Worker | `messenger_worker_busy`, `messenger_worker_capacity` | slots |
|
||||
| Process | `process_cpu_time_seconds_total`, `process_memory_rss_bytes` | CPU seconds / bytes |
|
||||
|
||||
Dashboard UID `messenger-observability` содержит 12 panels: RPS, 5xx ratio,
|
||||
HTTP p50/p95/p99, jobs по outcome, worker p50/p95/p99, end-to-end p50/p95/p99,
|
||||
queue depth/oldest, DB pool, worker saturation, CPU, RSS и correlated logs.
|
||||
Percentiles вычисляются из объединённых buckets всех реплик, например:
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.99,
|
||||
sum by (le) (rate(http_server_request_duration_seconds_bucket{
|
||||
service_name="messenger-api",http_route!~"/health/.*"}[5m])))
|
||||
```
|
||||
|
||||
Границы HTTP histogram: 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms,
|
||||
1 s, 2.5 s, 5 s. Worker: 50 ms…60 s. Time-to-ready: 100 ms…120 s.
|
||||
Средние значения и средние p99 отдельных реплик не используются.
|
||||
|
||||
## SLI, SLO и alerts
|
||||
|
||||
API availability SLI: доля бизнес-запросов со status `<500` среди всех
|
||||
бизнес-запросов; `/health/live` и `/health/ready` исключены. Учебный SLO — не менее
|
||||
99% good requests в rolling window 30 минут. 4xx входят в denominator, но не в
|
||||
server-error numerator.
|
||||
|
||||
Pipeline SLI: доля durably accepted attachments, достигших `ready` не позднее
|
||||
60 секунд. Учебный SLO — не менее 95% за rolling 30 минут при доступном worker.
|
||||
|
||||
Provisioned rules:
|
||||
|
||||
- `ApiHighServerErrorRate`: 5xx / all business requests за 2 минуты > 5%, `for: 1m`;
|
||||
- `QueueOldestJobGrowing`: oldest job > 20 s при depth > 0, `for: 15s`.
|
||||
|
||||
Пороги и короткие окна выбраны для воспроизводимой учебной демонстрации, а не как
|
||||
production defaults. У каждого правила есть dashboard/panel link и локальный runbook
|
||||
в `infra/grafana/runbooks/`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Offline contract | `make check` | PASS | консоль и Git diff |
|
||||
| Публичный API с telemetry | `make test` | `PASS: lab6 public API scenario` | `tests/smoke.py` |
|
||||
| Dashboard/alerts/3 backends | `make test-observability` | PASS: dashboard, alerts, metrics, logs, cross-process trace | `tests/observability_smoke.py` |
|
||||
| API → publisher → worker | тот же smoke после upload | один `trace_id`, services API/publisher/worker | Loki + Tempo API assertion |
|
||||
| Retry в одной трассе | `make test-retry-trace` | 2 PRODUCER + 2 CONSUMER, SQL/S3, attachment ready | `tests/retry_trace.py` |
|
||||
| Worker outage | `make test-alert-worker` | Alerting 61.8 s, Normal 15.5 s после запуска | `tests/alert_scenarios.py` |
|
||||
| PostgreSQL outage | `make test-alert-database` | Alerting/Normal + linked log/failed PostgreSQL span | `tests/alert_scenarios.py` |
|
||||
| Telemetry outage | `make test-telemetry-outage` | 49.5 s, smoke PASS, bounded RSS, loss counters | `tests/telemetry_outage.py` |
|
||||
| Конфиги | `docker compose config --quiet`; JSON parse; Grafana logs | provisioning завершён без error | `infra/` и `compose.yaml` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Повторная проверка выполнена 2026-11-20: host Apple M4 Pro, 24 GiB RAM;
|
||||
Docker Desktop VM — 12 CPU и 9.70 GiB RAM. Каждый Compose-сервис дополнительно
|
||||
ограничен `cpus`/`mem_limit` из `compose.yaml`. Load evidence создал
|
||||
25 638 синтетических сообщений за 120 с с concurrency 8.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Разбор контролируемых инцидентов
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Временная ошибка S3 и retry
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Worker был остановлен до upload, затем S3 выключен только на первую
|
||||
попытку. Worker сохранил `job.retry_scheduled`, после восстановления S3
|
||||
то же attachment дошло до `ready`. Трасса
|
||||
`e8e2aff13b7bdb632c9cec5007dcc6d4` содержит два PRODUCER и два CONSUMER
|
||||
span, а также SQL/S3 CLIENT spans. Это доказывает перенос context
|
||||
через retry outbox event.
|
||||
|
||||
### Worker недоступен
|
||||
|
||||
Гипотеза: API продолжит durable acceptance, RabbitMQ и PostgreSQL сохранят работу,
|
||||
но `queue_depth` и `queue_oldest_age_seconds` вырастут. Сценарий создал отдельные
|
||||
синтетические users/chat, остановил только worker, загрузил PNG и опрашивал состояние
|
||||
Grafana rule. Alert перешёл в `Alerting` за 61.8 с. После `docker compose start worker`
|
||||
attachment стал ready, backlog уменьшился, rule стал `Normal` через 15.5 с. Потери
|
||||
бизнес-данных: 0. Диагноз по runbook: HTTP остаётся доступным, jobs не завершаются,
|
||||
worker saturation отсутствует, oldest age растёт — причина на consumer-side.
|
||||
|
||||
### PostgreSQL недоступен
|
||||
|
||||
Гипотеза: прямые business requests к API дадут 5xx, DB spans/логи укажут connection
|
||||
failure, а queue-only alert не обязан срабатывать. PostgreSQL остановлен, отдельный
|
||||
контейнер создавал synthetic `POST /api/v1/users` каждые 250 ms. Error-rate alert
|
||||
стал `Alerting` за 72.2 с. PostgreSQL запущен в `finally`, readiness восстановилась,
|
||||
rule вернулся в `Normal` через 113.1 с — старые 5xx должны были выйти из 2-минутного
|
||||
окна. Потери подтверждённых данных: 0; неуспешные запросы клиент может повторить.
|
||||
Связанный log и trace `34f96d6fc934f7c54075c44a7b39c176` содержат
|
||||
socket connection error и `connect` CLIENT span с `db.system=postgresql` и
|
||||
`STATUS_CODE_ERROR`. Это отделяет отказ БД от просто медленного HTTP handler.
|
||||
|
||||
### Loki и Tempo недоступны
|
||||
|
||||
Оба backend остановлены одновременно на 49.5 с, что дольше настроенного
|
||||
30-секундного retry. Полный бизнес-smoke прошёл, Collector остался running.
|
||||
RSS Collector изменился с 194.6 до пиковых 198.0 MiB при container limit
|
||||
256 MiB. Internal counters зафиксировали потерю 34 log records и 30 spans.
|
||||
Порог `memory_limiter=192 MiB` — мягкий порог отказа/сброса данных, а не
|
||||
обещание RSS ниже 192 MiB; жёсткий предел задан Compose-лимитом 256 MiB.
|
||||
Это ожидаемая bounded telemetry loss: наблюдаемость не является частью
|
||||
business commit.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Стенд single-node: PostgreSQL, RabbitMQ, MinIO и telemetry backends не имеют HA.
|
||||
- Sampling всегда 100%; исследование стоимости и tail sampling относится к оценке 5.
|
||||
- Grafana notification delivery не настроена: по заданию достаточно локальных
|
||||
firing/resolved states.
|
||||
- Queue depth берётся из durable job table, поэтому отражает бизнес-backlog, а не
|
||||
точное число RabbitMQ messages в каждую микросекунду.
|
||||
- Collector Prometheus exporter хранит только текущее состояние; долговременную
|
||||
историю сохраняет Prometheus volume. Loki и Tempo используют локальные volumes.
|
||||
- Для production нужны TLS/mTLS OTLP, secrets manager, HA backends, remote storage,
|
||||
более длинные SLO windows и проверенные notification routes.
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
sqlalchemy.url = placeholder://configured-at-runtime
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import pool
|
||||
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||
|
||||
from app.config import load_settings
|
||||
from app.db.postgres import Base
|
||||
from app import models # noqa: F401 - imports ORM models into Base.metadata
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def _url() -> str:
|
||||
return load_settings().database_url
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'})
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def _run_migrations(connection) -> None:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
async def _run_migrations_async() -> None:
|
||||
configuration = config.get_section(config.config_ini_section, {})
|
||||
configuration['sqlalchemy.url'] = _url()
|
||||
connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool)
|
||||
async with connectable.connect() as connection:
|
||||
await connection.run_sync(_run_migrations)
|
||||
await connectable.dispose()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
asyncio.run(_run_migrations_async())
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
branch_labels = ${repr(branch_labels)}
|
||||
depends_on = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Initial PostgreSQL schema for lab 2.
|
||||
|
||||
Revision ID: 20261019_0001
|
||||
Revises:
|
||||
Create Date: 2026-10-19
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261019_0001'
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
'users',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('username', sa.String(length=32), nullable=False),
|
||||
sa.Column('display_name', sa.String(length=100), nullable=False),
|
||||
sa.Column('password_hash', sa.Text(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('username'),
|
||||
)
|
||||
op.create_table(
|
||||
'chats',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('title', sa.String(length=100), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_table(
|
||||
'chat_members',
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('user_id', sa.Uuid(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('chat_id', 'user_id'),
|
||||
)
|
||||
op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id'])
|
||||
op.create_table(
|
||||
'messages',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('sender_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('text', sa.String(length=4000), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['sender_id'], ['users.id']),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id'])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('messages_history_idx', table_name='messages')
|
||||
op.drop_table('messages')
|
||||
op.drop_index('chat_members_user_chat_idx', table_name='chat_members')
|
||||
op.drop_table('chat_members')
|
||||
op.drop_table('chats')
|
||||
op.drop_table('users')
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Add shared message idempotency for lab 3.
|
||||
|
||||
Revision ID: 20261023_0002
|
||||
Revises: 20261019_0001
|
||||
Create Date: 2026-10-23
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261023_0002'
|
||||
down_revision = '20261019_0001'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
'messages',
|
||||
sa.Column('idempotency_key', sa.String(length=128), nullable=True),
|
||||
)
|
||||
op.create_unique_constraint(
|
||||
'messages_sender_chat_idempotency_key',
|
||||
'messages',
|
||||
['sender_id', 'chat_id', 'idempotency_key'],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(
|
||||
'messages_sender_chat_idempotency_key',
|
||||
'messages',
|
||||
type_='unique',
|
||||
)
|
||||
op.drop_column('messages', 'idempotency_key')
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Add S3 attachment metadata and message links for lab 4.
|
||||
|
||||
Revision ID: 20261030_0003
|
||||
Revises: 20261023_0002
|
||||
Create Date: 2026-10-30
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261030_0003'
|
||||
down_revision = '20261023_0002'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
'attachments',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('owner_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('filename', sa.String(length=255), nullable=False),
|
||||
sa.Column('object_key', sa.String(length=255), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=32), nullable=False),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('width', sa.Integer(), nullable=False),
|
||||
sa.Column('height', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), nullable=False),
|
||||
sa.Column('error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['owner_id'], ['users.id']),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('object_key', name='attachments_object_key_key'),
|
||||
)
|
||||
op.create_index(
|
||||
'attachments_chat_created_idx',
|
||||
'attachments',
|
||||
['chat_id', 'created_at', 'id'],
|
||||
)
|
||||
op.create_table(
|
||||
'message_attachments',
|
||||
sa.Column('message_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('position', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('message_id', 'attachment_id'),
|
||||
sa.UniqueConstraint(
|
||||
'message_id',
|
||||
'position',
|
||||
name='message_attachments_message_position_key',
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'message_attachments_attachment_idx',
|
||||
'message_attachments',
|
||||
['attachment_id'],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('message_attachments_attachment_idx', table_name='message_attachments')
|
||||
op.drop_table('message_attachments')
|
||||
op.drop_index('attachments_chat_created_idx', table_name='attachments')
|
||||
op.drop_table('attachments')
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Add asynchronous attachment jobs, variants and transactional outbox.
|
||||
|
||||
Revision ID: 20261116_0004
|
||||
Revises: 20261030_0003
|
||||
Create Date: 2026-11-16
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261116_0004'
|
||||
down_revision = '20261030_0003'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column('attachments', 'object_key', new_column_name='original_key')
|
||||
op.alter_column('attachments', 'original_key', nullable=True)
|
||||
op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True))
|
||||
op.add_column(
|
||||
'attachments',
|
||||
sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'),
|
||||
)
|
||||
op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key'])
|
||||
|
||||
op.create_table(
|
||||
'attachment_variants',
|
||||
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('pipeline_version', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=16), nullable=False),
|
||||
sa.Column('object_key', sa.String(length=255), nullable=False),
|
||||
sa.Column('content_type', sa.String(length=32), nullable=False),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||
sa.Column('width', sa.Integer(), nullable=False),
|
||||
sa.Column('height', sa.Integer(), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'),
|
||||
sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
'attachment_jobs',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('pipeline_version', sa.Integer(), nullable=False),
|
||||
sa.Column('state', sa.String(length=16), nullable=False),
|
||||
sa.Column('attempt', sa.Integer(), nullable=False),
|
||||
sa.Column('max_attempts', sa.Integer(), nullable=False),
|
||||
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('lease_owner', sa.String(length=128), nullable=True),
|
||||
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('queued', 'processing', 'ready', 'failed')",
|
||||
name='attachment_job_state_check',
|
||||
),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint(
|
||||
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'attachment_jobs_lease_idx',
|
||||
'attachment_jobs',
|
||||
['state', 'lease_until'],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
'outbox_events',
|
||||
sa.Column('id', sa.Uuid(), nullable=False),
|
||||
sa.Column('event_type', sa.String(length=64), nullable=False),
|
||||
sa.Column('job_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('claim_owner', sa.String(length=128), nullable=True),
|
||||
sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'),
|
||||
sa.Column('last_error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index(
|
||||
'outbox_events_pending_idx',
|
||||
'outbox_events',
|
||||
['published_at', 'available_at'],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
'job_events',
|
||||
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
|
||||
sa.Column('job_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('kind', sa.String(length=32), nullable=False),
|
||||
sa.Column('attempt', sa.Integer(), nullable=False),
|
||||
sa.Column('error_code', sa.String(length=64), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
)
|
||||
op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at'])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('job_events_job_created_idx', table_name='job_events')
|
||||
op.drop_table('job_events')
|
||||
op.drop_index('outbox_events_pending_idx', table_name='outbox_events')
|
||||
op.drop_table('outbox_events')
|
||||
op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs')
|
||||
op.drop_table('attachment_jobs')
|
||||
op.drop_table('attachment_variants')
|
||||
op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique')
|
||||
op.drop_column('attachments', 'pipeline_version')
|
||||
op.drop_column('attachments', 'staging_key')
|
||||
op.alter_column('attachments', 'original_key', nullable=False)
|
||||
op.alter_column('attachments', 'original_key', new_column_name='object_key')
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Persist W3C trace context with outbox events.
|
||||
|
||||
Revision ID: 20261117_0005
|
||||
Revises: 20261116_0004
|
||||
Create Date: 2026-11-17
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '20261117_0005'
|
||||
down_revision = '20261116_0004'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('outbox_events', sa.Column('traceparent', sa.String(length=55), nullable=True))
|
||||
op.add_column('outbox_events', sa.Column('tracestate', sa.String(length=512), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('outbox_events', 'tracestate')
|
||||
op.drop_column('outbox_events', 'traceparent')
|
||||
@@ -0,0 +1 @@
|
||||
"""Мессенджер на PostgreSQL и Redis, лабораторная 2."""
|
||||
@@ -0,0 +1 @@
|
||||
"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры."""
|
||||
@@ -0,0 +1,28 @@
|
||||
import base64
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from ..domain.errors import Invalid
|
||||
from ..domain.models import Keyset
|
||||
from .timefmt import from_rfc3339, to_rfc3339
|
||||
|
||||
MAX_LENGTH = 512
|
||||
|
||||
|
||||
def encode(scope: str, position: Keyset) -> str:
|
||||
payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)}
|
||||
raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
|
||||
return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode(scope: str, value: str) -> Keyset:
|
||||
if not value or len(value) > MAX_LENGTH:
|
||||
raise Invalid('cursor недействителен для этого запроса')
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
|
||||
payload = json.loads(raw)
|
||||
if not isinstance(payload, dict) or payload.get('s') != scope:
|
||||
raise ValueError('scope mismatch')
|
||||
return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i']))
|
||||
except Exception:
|
||||
raise Invalid('cursor недействителен для этого запроса') from None
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from fastapi import Depends, HTTPException, Request, Security, status
|
||||
|
||||
from ..db.redis import RedisSessionStore
|
||||
from ..domain.errors import Unauthorized
|
||||
from ..domain.models import User
|
||||
from ..services.chats import ChatService
|
||||
from ..services.attachments import AttachmentService
|
||||
from ..services.messages import MessageService
|
||||
from ..services.users import UserService
|
||||
from .security import bearer_auth
|
||||
|
||||
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Services:
|
||||
attachments: AttachmentService
|
||||
users: UserService
|
||||
chats: ChatService
|
||||
messages: MessageService
|
||||
sessions: RedisSessionStore
|
||||
|
||||
|
||||
def services(request: Request) -> Services:
|
||||
return request.app.state.services
|
||||
|
||||
|
||||
def user_service(container: Services = Depends(services)) -> UserService:
|
||||
return container.users
|
||||
|
||||
|
||||
def chat_service(container: Services = Depends(services)) -> ChatService:
|
||||
return container.chats
|
||||
|
||||
|
||||
def message_service(container: Services = Depends(services)) -> MessageService:
|
||||
return container.messages
|
||||
|
||||
|
||||
def attachment_service(container: Services = Depends(services)) -> AttachmentService:
|
||||
return container.attachments
|
||||
|
||||
|
||||
def session_store(container: Services = Depends(services)) -> RedisSessionStore:
|
||||
return container.sessions
|
||||
|
||||
|
||||
def _bearer_error() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail='требуется действующая Bearer-сессия',
|
||||
headers={'WWW-Authenticate': _BEARER_CHALLENGE},
|
||||
)
|
||||
|
||||
|
||||
async def bearer_token(authorization: str | None = Security(bearer_auth)) -> str:
|
||||
if not authorization:
|
||||
raise _bearer_error()
|
||||
scheme, separator, token = authorization.partition(' ')
|
||||
if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token:
|
||||
raise _bearer_error()
|
||||
return token
|
||||
|
||||
|
||||
async def current_user(
|
||||
token: str = Depends(bearer_token),
|
||||
container: Services = Depends(services),
|
||||
) -> User:
|
||||
user_id = await container.sessions.get_user_id(token)
|
||||
if user_id is None:
|
||||
raise _bearer_error()
|
||||
try:
|
||||
return await container.users.identify(user_id)
|
||||
except Unauthorized:
|
||||
raise _bearer_error() from None
|
||||
@@ -0,0 +1,107 @@
|
||||
import logging
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from ..domain.errors import (
|
||||
Conflict,
|
||||
DomainError,
|
||||
Invalid,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
Unauthorized,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
|
||||
logger = logging.getLogger('app.error')
|
||||
|
||||
CODE_BY_STATUS = {
|
||||
400: 'invalid_request',
|
||||
401: 'unauthorized',
|
||||
404: 'not_found',
|
||||
409: 'conflict',
|
||||
413: 'payload_too_large',
|
||||
415: 'unsupported_media_type',
|
||||
429: 'rate_limited',
|
||||
500: 'internal_error',
|
||||
503: 'unavailable',
|
||||
}
|
||||
|
||||
STATUS_BY_DOMAIN = {
|
||||
Invalid: 400,
|
||||
Unauthorized: 401,
|
||||
NotFound: 404,
|
||||
Conflict: 409,
|
||||
PayloadTooLarge: 413,
|
||||
UnsupportedMediaType: 415,
|
||||
Unavailable: 503,
|
||||
}
|
||||
|
||||
|
||||
def code_for(status: int) -> str:
|
||||
if status in CODE_BY_STATUS:
|
||||
return CODE_BY_STATUS[status]
|
||||
return 'invalid_request' if status < 500 else 'internal_error'
|
||||
|
||||
|
||||
def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}},
|
||||
)
|
||||
|
||||
|
||||
def _domain_status(error: DomainError) -> int:
|
||||
for kind, status in STATUS_BY_DOMAIN.items():
|
||||
if isinstance(error, kind):
|
||||
return status
|
||||
return 500
|
||||
|
||||
|
||||
def _describe(error: RequestValidationError) -> str:
|
||||
"""Краткое указание на пол, без stack trace и внутреннего устройства"""
|
||||
for item in error.errors():
|
||||
if item.get('type') == 'json_invalid':
|
||||
return 'тело запроса не является корректным JSON'
|
||||
location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body')
|
||||
return f'некорректное поле {location}' if location else 'некорректный запрос'
|
||||
return 'некорректный запрос'
|
||||
|
||||
|
||||
def request_id(request) -> str:
|
||||
return getattr(request.state, 'request_id', 'unknown')
|
||||
|
||||
|
||||
def install(app: FastAPI) -> None:
|
||||
@app.exception_handler(DomainError)
|
||||
async def _on_domain(request, error: DomainError) -> JSONResponse:
|
||||
return error_response(_domain_status(error), error.message, request_id(request))
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _on_validation(request, error: RequestValidationError) -> JSONResponse:
|
||||
return error_response(400, _describe(error), request_id(request))
|
||||
|
||||
@app.exception_handler(SQLAlchemyError)
|
||||
async def _on_database(request, error: SQLAlchemyError) -> JSONResponse:
|
||||
logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__)
|
||||
return error_response(503, 'база данных недоступна', request_id(request))
|
||||
|
||||
@app.exception_handler(OSError)
|
||||
async def _on_storage_socket(request, error: OSError) -> JSONResponse:
|
||||
logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__)
|
||||
return error_response(503, 'обязательное хранилище недоступно', request_id(request))
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def _on_http(request, error: StarletteHTTPException) -> JSONResponse:
|
||||
message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса'
|
||||
response = error_response(error.status_code, message, request_id(request))
|
||||
for name, value in (error.headers or {}).items():
|
||||
response.headers[name] = value
|
||||
return response
|
||||
|
||||
|
||||
__all__ = ['code_for', 'error_response', 'install']
|
||||
@@ -0,0 +1,83 @@
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from opentelemetry import trace
|
||||
from starlette.datastructures import MutableHeaders
|
||||
|
||||
from ..observability import get_metrics
|
||||
from ..observability.logging import safe_extra
|
||||
from .errors import error_response
|
||||
|
||||
logger = logging.getLogger('app')
|
||||
|
||||
|
||||
class RequestContext:
|
||||
def __init__(self, app) -> None:
|
||||
self.app = app
|
||||
self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname()
|
||||
|
||||
async def __call__(self, scope, receive, send) -> None:
|
||||
if scope['type'] != 'http':
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
request_id = uuid4().hex
|
||||
scope.setdefault('state', {})['request_id'] = request_id
|
||||
is_health = scope.get('path') in {'/health/live', '/health/ready'}
|
||||
started_at = time.perf_counter()
|
||||
started = False
|
||||
status_code = 500
|
||||
|
||||
async def send_wrapper(message) -> None:
|
||||
nonlocal started, status_code
|
||||
if message['type'] == 'http.response.start':
|
||||
started = True
|
||||
status_code = message['status']
|
||||
headers = MutableHeaders(scope=message)
|
||||
headers['X-Request-Id'] = request_id
|
||||
headers['X-Instance-Id'] = self.instance_id
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, send_wrapper)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
'unhandled HTTP error',
|
||||
extra=safe_extra('http.request.failed', request_id=request_id),
|
||||
)
|
||||
if started:
|
||||
raise
|
||||
response = error_response(500, 'внутренняя ошибка', request_id)
|
||||
await response(scope, receive, send_wrapper)
|
||||
finally:
|
||||
duration = time.perf_counter() - started_at
|
||||
route = getattr(scope.get('route'), 'path', None) or 'unmatched'
|
||||
method = scope.get('method', 'UNKNOWN')
|
||||
status_class = f'{status_code // 100}xx'
|
||||
span = trace.get_current_span()
|
||||
if span.is_recording():
|
||||
span.set_attribute('request.id', request_id)
|
||||
span.set_attribute('http.route', route)
|
||||
if not is_health:
|
||||
attributes = {
|
||||
'http.request.method': method,
|
||||
'http.route': route,
|
||||
'http.response.status_class': status_class,
|
||||
}
|
||||
instruments = get_metrics()
|
||||
instruments.http_requests.add(1, attributes)
|
||||
instruments.http_duration.record(duration, attributes)
|
||||
logger.info(
|
||||
'HTTP request completed',
|
||||
extra=safe_extra(
|
||||
'http.request.completed',
|
||||
request_id=request_id,
|
||||
http_method=method,
|
||||
http_route=route,
|
||||
http_status_code=status_code,
|
||||
duration_seconds=round(duration, 6),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,3 @@
|
||||
from . import auth, chats, health, messages, users
|
||||
|
||||
__all__ = ['auth', 'chats', 'health', 'messages', 'users']
|
||||
@@ -0,0 +1,79 @@
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.attachments import AttachmentService
|
||||
from ..deps import attachment_service, current_user
|
||||
from ..schemas import AttachmentOut
|
||||
|
||||
router = APIRouter(tags=['attachments'])
|
||||
|
||||
|
||||
@router.post(
|
||||
'/api/v1/chats/{chat_id}/attachments',
|
||||
status_code=202,
|
||||
response_model=AttachmentOut,
|
||||
summary='Принять PNG/JPEG на асинхронную обработку',
|
||||
openapi_extra={
|
||||
'requestBody': {
|
||||
'required': True,
|
||||
'description': 'Сырые байты изображения, не multipart/form-data.',
|
||||
'content': {
|
||||
'image/png': {
|
||||
'schema': {'type': 'string', 'format': 'binary'},
|
||||
},
|
||||
'image/jpeg': {
|
||||
'schema': {'type': 'string', 'format': 'binary'},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
async def upload(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
content_type: Annotated[str | None, Header(alias='Content-Type')] = None,
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> AttachmentOut:
|
||||
attachment = await attachments.upload(
|
||||
actor,
|
||||
chat_id,
|
||||
filename,
|
||||
content_type or '',
|
||||
request.stream(),
|
||||
)
|
||||
return AttachmentOut.from_domain(attachment)
|
||||
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut)
|
||||
async def metadata(
|
||||
attachment_id: UUID,
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> AttachmentOut:
|
||||
return AttachmentOut.from_domain(await attachments.get(actor, attachment_id))
|
||||
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}/content')
|
||||
async def content(
|
||||
attachment_id: UUID,
|
||||
variant: str = Query('original'),
|
||||
actor: User = Depends(current_user),
|
||||
attachments: AttachmentService = Depends(attachment_service),
|
||||
) -> StreamingResponse:
|
||||
attachment, stream, content_type, size_bytes = await attachments.content(
|
||||
actor, attachment_id, variant
|
||||
)
|
||||
return StreamingResponse(
|
||||
stream.chunks(),
|
||||
media_type=content_type,
|
||||
headers={
|
||||
'Content-Length': str(size_bytes),
|
||||
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from binascii import Error as Base64Error
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Security, status
|
||||
|
||||
from ...auth.tokens import create_token
|
||||
from ...db.redis import RedisSessionStore
|
||||
from ...domain.models import User
|
||||
from ...services.users import UserService
|
||||
from ..deps import bearer_token, current_user, session_store, user_service
|
||||
from ..security import basic_auth
|
||||
from ..schemas import SessionOut, UserOut
|
||||
from ..timefmt import to_rfc3339
|
||||
|
||||
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||
|
||||
_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
|
||||
|
||||
def _unauthorized_basic() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail='неверные учётные данные',
|
||||
headers={'WWW-Authenticate': _BASIC_CHALLENGE},
|
||||
)
|
||||
|
||||
|
||||
def _parse_basic(authorization: str | None) -> tuple[str, str]:
|
||||
if not authorization:
|
||||
raise _unauthorized_basic()
|
||||
scheme, separator, encoded = authorization.partition(' ')
|
||||
if scheme.lower() != 'basic' or not separator or not encoded:
|
||||
raise _unauthorized_basic()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
|
||||
except (Base64Error, UnicodeDecodeError):
|
||||
raise _unauthorized_basic() from None
|
||||
username, separator, password = decoded.partition(':')
|
||||
if not separator or not username:
|
||||
raise _unauthorized_basic()
|
||||
return username, password
|
||||
|
||||
|
||||
@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut)
|
||||
async def create_session(
|
||||
authorization: str | None = Security(basic_auth),
|
||||
users: UserService = Depends(user_service),
|
||||
sessions: RedisSessionStore = Depends(session_store),
|
||||
) -> SessionOut:
|
||||
username, password = _parse_basic(authorization)
|
||||
user = await users.authenticate(username, password)
|
||||
if user is None:
|
||||
raise _unauthorized_basic()
|
||||
token = create_token()
|
||||
expires_at = await sessions.create(user.id, token)
|
||||
return SessionOut(
|
||||
token=token,
|
||||
token_type='Bearer',
|
||||
expires_at=to_rfc3339(expires_at),
|
||||
user=UserOut.from_domain(user),
|
||||
)
|
||||
|
||||
|
||||
@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None)
|
||||
async def revoke_session(
|
||||
token: str = Depends(bearer_token),
|
||||
_: User = Depends(current_user),
|
||||
sessions: RedisSessionStore = Depends(session_store),
|
||||
) -> None:
|
||||
await sessions.revoke(token)
|
||||
@@ -0,0 +1,49 @@
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.chats import ChatService
|
||||
from .. import cursors
|
||||
from ..deps import chat_service, current_user
|
||||
from ..schemas import (
|
||||
LIMIT_DEFAULT,
|
||||
LIMIT_MAX,
|
||||
LIMIT_MIN,
|
||||
ChatOut,
|
||||
ChatPageOut,
|
||||
CreateChat,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix='/api/v1/chats', tags=['chats'])
|
||||
|
||||
|
||||
@router.post('', status_code=201, response_model=ChatOut)
|
||||
async def create(
|
||||
body: CreateChat,
|
||||
actor: User = Depends(current_user),
|
||||
chats: ChatService = Depends(chat_service),
|
||||
) -> ChatOut:
|
||||
return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids))
|
||||
|
||||
|
||||
@router.get('', response_model=ChatPageOut)
|
||||
async def page(
|
||||
actor: User = Depends(current_user),
|
||||
chats: ChatService = Depends(chat_service),
|
||||
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
|
||||
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
|
||||
) -> ChatPageOut:
|
||||
scope = f'chats:{actor.id}'
|
||||
after = cursors.decode(scope, cursor) if cursor is not None else None
|
||||
return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope)
|
||||
|
||||
|
||||
@router.get('/{chat_id}', response_model=ChatOut)
|
||||
async def get(
|
||||
chat_id: UUID,
|
||||
actor: User = Depends(current_user),
|
||||
chats: ChatService = Depends(chat_service),
|
||||
) -> ChatOut:
|
||||
return ChatOut.from_domain(await chats.get(actor, chat_id))
|
||||
@@ -0,0 +1,29 @@
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from redis.exceptions import RedisError
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from ...domain.errors import Unavailable
|
||||
|
||||
from ..schemas import HealthOut
|
||||
|
||||
router = APIRouter(tags=['health'])
|
||||
|
||||
|
||||
@router.get('/health/live', response_model=HealthOut)
|
||||
async def live() -> HealthOut:
|
||||
return HealthOut(status='ok')
|
||||
|
||||
|
||||
@router.get('/health/ready', response_model=HealthOut)
|
||||
async def ready(request: Request) -> HealthOut:
|
||||
if not request.app.state.ready:
|
||||
raise HTTPException(status_code=503, detail='приложение не готово принимать трафик')
|
||||
try:
|
||||
async with request.app.state.engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
await request.app.state.redis.ping()
|
||||
await request.app.state.storage.ping()
|
||||
except (SQLAlchemyError, RedisError, Unavailable, OSError):
|
||||
raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None
|
||||
return HealthOut(status='ok')
|
||||
@@ -0,0 +1,53 @@
|
||||
from typing import Annotated, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.messages import MessageService
|
||||
from .. import cursors
|
||||
from ..deps import current_user, message_service
|
||||
from ..schemas import (
|
||||
LIMIT_DEFAULT,
|
||||
LIMIT_MAX,
|
||||
LIMIT_MIN,
|
||||
CreateMessage,
|
||||
MessageOut,
|
||||
MessagePageOut,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages'])
|
||||
|
||||
|
||||
@router.post('', status_code=201, response_model=MessageOut)
|
||||
async def send(
|
||||
chat_id: UUID,
|
||||
body: CreateMessage,
|
||||
idempotency_key: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias='Idempotency-Key',
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r'^[A-Za-z0-9._:-]+$',
|
||||
),
|
||||
],
|
||||
actor: User = Depends(current_user),
|
||||
messages: MessageService = Depends(message_service),
|
||||
) -> MessageOut:
|
||||
return MessageOut.from_domain(
|
||||
await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key)
|
||||
)
|
||||
|
||||
|
||||
@router.get('', response_model=MessagePageOut)
|
||||
async def page(
|
||||
chat_id: UUID,
|
||||
actor: User = Depends(current_user),
|
||||
messages: MessageService = Depends(message_service),
|
||||
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
|
||||
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
|
||||
) -> MessagePageOut:
|
||||
scope = f'messages:{chat_id}'
|
||||
after = cursors.decode(scope, cursor) if cursor is not None else None
|
||||
return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope)
|
||||
@@ -0,0 +1,18 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from ...domain.models import User
|
||||
from ...services.users import UserService
|
||||
from ..deps import current_user, user_service
|
||||
from ..schemas import CreateUser, UserOut
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['users'])
|
||||
|
||||
|
||||
@router.post('/users', status_code=201, response_model=UserOut)
|
||||
async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut:
|
||||
return UserOut.from_domain(await users.register(body.username, body.display_name, body.password))
|
||||
|
||||
|
||||
@router.get('/users/me', response_model=UserOut)
|
||||
async def me(actor: User = Depends(current_user)) -> UserOut:
|
||||
return UserOut.from_domain(actor)
|
||||
@@ -0,0 +1,209 @@
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from ..domain.models import Attachment, Chat, Message, Page, User
|
||||
from . import cursors
|
||||
from .timefmt import to_rfc3339
|
||||
|
||||
NAME_MAX = 100
|
||||
TEXT_MAX = 4000
|
||||
MEMBERS_MAX = 100
|
||||
LIMIT_MIN = 1
|
||||
LIMIT_MAX = 100
|
||||
LIMIT_DEFAULT = 50
|
||||
|
||||
|
||||
def not_blank(value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError('строка не может состоять только из пробельных символов')
|
||||
return value
|
||||
|
||||
|
||||
class Strict(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
|
||||
class HealthOut(Strict):
|
||||
status: str = 'ok'
|
||||
|
||||
|
||||
class CreateUser(Strict):
|
||||
username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=NAME_MAX)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('display_name')
|
||||
@classmethod
|
||||
def _display_name(cls, value: str) -> str:
|
||||
return not_blank(value)
|
||||
|
||||
|
||||
class UserOut(Strict):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, user: User) -> 'UserOut':
|
||||
return cls(
|
||||
id=user.id,
|
||||
username=user.username,
|
||||
display_name=user.display_name,
|
||||
created_at=to_rfc3339(user.created_at),
|
||||
)
|
||||
|
||||
|
||||
class SessionOut(Strict):
|
||||
token: str = Field(min_length=32)
|
||||
token_type: str = 'Bearer'
|
||||
expires_at: str
|
||||
user: UserOut
|
||||
|
||||
|
||||
class CreateChat(Strict):
|
||||
title: str = Field(min_length=1, max_length=NAME_MAX)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX)
|
||||
|
||||
@field_validator('title')
|
||||
@classmethod
|
||||
def _title(cls, value: str) -> str:
|
||||
return not_blank(value)
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def _members_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('member_ids не должен содержать дубликаты')
|
||||
return value
|
||||
|
||||
|
||||
class ChatOut(Strict):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: str
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, chat: Chat) -> 'ChatOut':
|
||||
return cls(
|
||||
id=chat.id,
|
||||
title=chat.title,
|
||||
member_ids=list(chat.member_ids),
|
||||
created_at=to_rfc3339(chat.created_at),
|
||||
)
|
||||
|
||||
|
||||
class CreateMessage(Strict):
|
||||
text: str = Field(max_length=TEXT_MAX)
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def _attachments_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('attachment_ids не должен содержать дубликаты')
|
||||
return value
|
||||
|
||||
@model_validator(mode='after')
|
||||
def _content_present(self) -> 'CreateMessage':
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('сообщение должно содержать текст или вложение')
|
||||
return self
|
||||
|
||||
|
||||
class MessageOut(Strict):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: list[UUID]
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, message: Message) -> 'MessageOut':
|
||||
return cls(
|
||||
id=message.id,
|
||||
chat_id=message.chat_id,
|
||||
sender_id=message.sender_id,
|
||||
text=message.text,
|
||||
created_at=to_rfc3339(message.created_at),
|
||||
attachment_ids=list(message.attachment_ids),
|
||||
)
|
||||
|
||||
|
||||
class VariantOut(Strict):
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class AttachmentOut(Strict):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
variants: list[VariantOut] = Field(default_factory=list)
|
||||
error_code: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, attachment: Attachment) -> 'AttachmentOut':
|
||||
return cls(
|
||||
id=attachment.id,
|
||||
chat_id=attachment.chat_id,
|
||||
owner_id=attachment.owner_id,
|
||||
filename=attachment.filename,
|
||||
content_type=attachment.content_type,
|
||||
size_bytes=attachment.size_bytes,
|
||||
sha256=attachment.sha256,
|
||||
status=attachment.status,
|
||||
variants=[
|
||||
VariantOut(
|
||||
name=item.name,
|
||||
content_type=item.content_type,
|
||||
size_bytes=item.size_bytes,
|
||||
width=item.width,
|
||||
height=item.height,
|
||||
sha256=item.sha256,
|
||||
)
|
||||
for item in attachment.variants
|
||||
],
|
||||
error_code=attachment.error_code,
|
||||
created_at=to_rfc3339(attachment.created_at),
|
||||
updated_at=to_rfc3339(attachment.updated_at),
|
||||
)
|
||||
|
||||
|
||||
class ChatPageOut(Strict):
|
||||
items: list[ChatOut]
|
||||
next_cursor: Optional[str]
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut':
|
||||
return cls(
|
||||
items=[ChatOut.from_domain(chat) for chat in page.items],
|
||||
next_cursor=cursors.encode(scope, page.next) if page.next else None,
|
||||
)
|
||||
|
||||
|
||||
class MessagePageOut(Strict):
|
||||
items: list[MessageOut]
|
||||
next_cursor: Optional[str]
|
||||
|
||||
@classmethod
|
||||
def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut':
|
||||
return cls(
|
||||
items=[MessageOut.from_domain(message) for message in page.items],
|
||||
next_cursor=cursors.encode(scope, page.next) if page.next else None,
|
||||
)
|
||||
@@ -0,0 +1,29 @@
|
||||
"""OpenAPI security-схемы с сохранением собственного разбора Authorization."""
|
||||
|
||||
from fastapi import Request
|
||||
from fastapi.openapi.models import HTTPBase as HTTPBaseModel
|
||||
from fastapi.openapi.models import HTTPBearer as HTTPBearerModel
|
||||
from fastapi.security.base import SecurityBase
|
||||
|
||||
|
||||
class RawHTTPAuthorization(SecurityBase):
|
||||
"""Объявляет HTTP security scheme и возвращает исходный Authorization header."""
|
||||
|
||||
def __init__(self, model, scheme_name: str) -> None:
|
||||
self.model = model
|
||||
self.scheme_name = scheme_name
|
||||
|
||||
async def __call__(self, request: Request) -> str | None:
|
||||
"""Возвращает заголовок без разбора, чтобы сохранить текущую валидацию API."""
|
||||
|
||||
return request.headers.get('Authorization')
|
||||
|
||||
|
||||
basic_auth = RawHTTPAuthorization(
|
||||
HTTPBaseModel(scheme='basic'),
|
||||
scheme_name='basicAuth',
|
||||
)
|
||||
bearer_auth = RawHTTPAuthorization(
|
||||
HTTPBearerModel(scheme='bearer', bearerFormat='opaque session token'),
|
||||
scheme_name='bearerAuth',
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def to_rfc3339(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def from_rfc3339(value: str) -> datetime:
|
||||
if not isinstance(value, str) or not value.endswith('Z'):
|
||||
raise ValueError('timestamp must end with Z')
|
||||
return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc)
|
||||
@@ -0,0 +1,4 @@
|
||||
from .passwords import hash_password, password_matches
|
||||
from .tokens import create_token
|
||||
|
||||
__all__ = ['create_token', 'hash_password', 'password_matches']
|
||||
@@ -0,0 +1,15 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, VerifyMismatchError
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def password_matches(stored_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(stored_hash, password)
|
||||
except (VerifyMismatchError, VerificationError):
|
||||
return False
|
||||
@@ -0,0 +1,11 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
|
||||
|
||||
def create_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
return f'session:{digest}'
|
||||
@@ -0,0 +1,3 @@
|
||||
from .settings import Settings, load_settings
|
||||
|
||||
__all__ = ['Settings', 'load_settings']
|
||||
@@ -0,0 +1,85 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _read_value(name: str, file_name: str) -> str:
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
return value
|
||||
path = os.getenv(file_name)
|
||||
if path:
|
||||
return Path(path).read_text(encoding='utf-8').strip()
|
||||
raise RuntimeError(f'не задана настройка {name} или {file_name}')
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Settings:
|
||||
database_url: str
|
||||
redis_url: str
|
||||
session_ttl_seconds: int
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
broker_url: str
|
||||
worker_concurrency: int
|
||||
worker_prefetch: int
|
||||
job_max_attempts: int
|
||||
job_timeout_seconds: int
|
||||
job_lease_seconds: int
|
||||
retry_base_seconds: int
|
||||
retry_max_seconds: int
|
||||
staging_retention_seconds: int
|
||||
thumbnail_jpeg_quality: int
|
||||
crop_jpeg_quality: int
|
||||
optimized_webp_quality: int
|
||||
|
||||
|
||||
def _positive_int(name: str, default: str) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, default))
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f'{name} должен быть целым числом') from exc
|
||||
if value <= 0:
|
||||
raise RuntimeError(f'{name} должен быть больше нуля')
|
||||
return value
|
||||
|
||||
|
||||
def _quality(name: str, default: str) -> int:
|
||||
value = _positive_int(name, default)
|
||||
if value > 100:
|
||||
raise RuntimeError(f'{name} должен быть не больше 100')
|
||||
return value
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
|
||||
redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'),
|
||||
session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'),
|
||||
s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
s3_region=os.getenv('S3_REGION', 'us-east-1'),
|
||||
s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'),
|
||||
s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'),
|
||||
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
|
||||
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
|
||||
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
|
||||
broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'),
|
||||
worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'),
|
||||
worker_prefetch=_positive_int('WORKER_PREFETCH', '2'),
|
||||
job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'),
|
||||
job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'),
|
||||
job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'),
|
||||
retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'),
|
||||
retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'),
|
||||
staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'),
|
||||
thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'),
|
||||
crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'),
|
||||
optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'),
|
||||
)
|
||||
@@ -0,0 +1,4 @@
|
||||
from .postgres import Base, create_engine, create_session_factory
|
||||
from .redis import RedisSessionStore, create_redis
|
||||
|
||||
__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory']
|
||||
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from ..observability import get_metrics
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def create_engine(url: str) -> AsyncEngine:
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
pool_size=5,
|
||||
max_overflow=5,
|
||||
pool_timeout=5,
|
||||
pool_pre_ping=True,
|
||||
connect_args={'server_settings': {'statement_timeout': '5000'}},
|
||||
)
|
||||
instruments = get_metrics()
|
||||
attributes = {'pool.name': 'postgres'}
|
||||
|
||||
def update_pool_metrics(*_args) -> None:
|
||||
pool = engine.sync_engine.pool
|
||||
instruments.db_pool_used.set(float(pool.checkedout()), attributes)
|
||||
waiting = len(getattr(getattr(pool, '_pool', None), '_getters', ()))
|
||||
instruments.db_pool_waiting.set(float(waiting), attributes)
|
||||
|
||||
event.listen(engine.sync_engine, 'checkout', update_pool_metrics)
|
||||
event.listen(engine.sync_engine, 'checkin', update_pool_metrics)
|
||||
instruments.db_pool_used.set(0, attributes)
|
||||
instruments.db_pool_waiting.set(0, attributes)
|
||||
return engine
|
||||
|
||||
|
||||
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(engine, expire_on_commit=False)
|
||||
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from ..auth.tokens import session_key
|
||||
from ..domain.errors import Unavailable
|
||||
|
||||
|
||||
def create_redis(url: str) -> Redis:
|
||||
return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3)
|
||||
|
||||
|
||||
class RedisSessionStore:
|
||||
def __init__(self, client: Redis, ttl_seconds: int) -> None:
|
||||
self._client = client
|
||||
self._ttl_seconds = ttl_seconds
|
||||
|
||||
async def create(self, user_id: UUID, token: str) -> datetime:
|
||||
try:
|
||||
await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds)
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds)
|
||||
|
||||
async def get_user_id(self, token: str) -> UUID | None:
|
||||
try:
|
||||
value = await self._client.get(session_key(token))
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return UUID(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def revoke(self, token: str) -> None:
|
||||
try:
|
||||
await self._client.delete(session_key(token))
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
|
||||
async def ping(self) -> None:
|
||||
try:
|
||||
await self._client.ping()
|
||||
except RedisError as exc:
|
||||
raise Unavailable('хранилище сессий недоступно') from exc
|
||||
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class DomainError(Exception):
|
||||
"""Сообщение попадает клиенту, поэтому не содержит внутренних деталей."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
|
||||
|
||||
class Invalid(DomainError):
|
||||
"""Запрос нарушает правила предметной области."""
|
||||
|
||||
|
||||
class Unauthorized(DomainError):
|
||||
"""Нет действительной идентификации."""
|
||||
|
||||
|
||||
class NotFound(DomainError):
|
||||
"""Объекта нет либо он недоступен вызывающему: снаружи это неразличимо."""
|
||||
|
||||
|
||||
class Conflict(DomainError):
|
||||
"""Состояние не позволяет выполнить операцию."""
|
||||
|
||||
|
||||
class Unavailable(DomainError):
|
||||
"""Обязательная внешняя зависимость временно недоступна."""
|
||||
|
||||
|
||||
class PayloadTooLarge(DomainError):
|
||||
"""Тело запроса или декодированное содержимое превышает лимит."""
|
||||
|
||||
|
||||
class UnsupportedMediaType(DomainError):
|
||||
"""Формат содержимого не поддерживается или не соответствует MIME."""
|
||||
@@ -0,0 +1,116 @@
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class User:
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Chat:
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: tuple[UUID, ...]
|
||||
created_at: datetime
|
||||
|
||||
def has_member(self, user_id: UUID) -> bool:
|
||||
return user_id in self.member_ids
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Message:
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: tuple[UUID, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
original_key: str | None
|
||||
staging_key: str | None
|
||||
pipeline_version: int
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
width: int
|
||||
height: int
|
||||
status: str
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
variants: tuple['AttachmentVariant', ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentVariant:
|
||||
name: str
|
||||
object_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentJob:
|
||||
id: UUID
|
||||
attachment_id: UUID
|
||||
pipeline_version: int
|
||||
state: str
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
next_attempt_at: datetime
|
||||
lease_owner: str | None
|
||||
lease_until: datetime | None
|
||||
last_error_code: str | None
|
||||
replay_count: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OutboxEvent:
|
||||
id: UUID
|
||||
job_id: UUID
|
||||
payload: dict
|
||||
available_at: datetime
|
||||
traceparent: str | None = None
|
||||
tracestate: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Keyset:
|
||||
created_at: datetime
|
||||
id: UUID
|
||||
|
||||
@property
|
||||
def order(self) -> tuple[datetime, int]:
|
||||
return (self.created_at, self.id.int)
|
||||
|
||||
@classmethod
|
||||
def from_item(cls, item) -> 'Keyset':
|
||||
return cls(created_at=item.created_at, id=item.id)
|
||||
|
||||
|
||||
def order_key(item) -> tuple[datetime, int]:
|
||||
return (item.created_at, item.id.int)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Page:
|
||||
items: tuple
|
||||
next: Optional[Keyset]
|
||||
@@ -0,0 +1 @@
|
||||
"""Durable attachment job publisher and worker."""
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import aio_pika
|
||||
from aio_pika import DeliveryMode, ExchangeType
|
||||
|
||||
MAIN_EXCHANGE = 'attachments'
|
||||
MAIN_QUEUE = 'attachments.process'
|
||||
ROUTING_KEY = 'attachment.process.v1'
|
||||
DEAD_EXCHANGE = 'attachments.dead'
|
||||
DEAD_QUEUE = 'attachments.failed'
|
||||
logger = logging.getLogger('app.broker')
|
||||
|
||||
|
||||
async def connect(url: str) -> aio_pika.RobustConnection:
|
||||
delay = 0.5
|
||||
while True:
|
||||
try:
|
||||
return await aio_pika.connect_robust(url, timeout=5)
|
||||
except (OSError, aio_pika.exceptions.AMQPException):
|
||||
logger.warning('broker is not ready; reconnecting in %.1fs', delay)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 2, 5.0)
|
||||
|
||||
|
||||
async def topology(channel: aio_pika.abc.AbstractChannel):
|
||||
dead_exchange = await channel.declare_exchange(
|
||||
DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True
|
||||
)
|
||||
dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True)
|
||||
await dead_queue.bind(dead_exchange, ROUTING_KEY)
|
||||
exchange = await channel.declare_exchange(
|
||||
MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True
|
||||
)
|
||||
queue = await channel.declare_queue(
|
||||
MAIN_QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
'x-dead-letter-exchange': DEAD_EXCHANGE,
|
||||
'x-dead-letter-routing-key': ROUTING_KEY,
|
||||
},
|
||||
)
|
||||
await queue.bind(exchange, ROUTING_KEY)
|
||||
return exchange, queue
|
||||
|
||||
|
||||
def message(body: bytes, event_id: str, headers: dict[str, str] | None = None) -> aio_pika.Message:
|
||||
return aio_pika.Message(
|
||||
body=body,
|
||||
content_type='application/json',
|
||||
delivery_mode=DeliveryMode.PERSISTENT,
|
||||
message_id=event_id,
|
||||
type=ROUTING_KEY,
|
||||
headers=headers or {},
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
|
||||
class PermanentImageError(Exception):
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeneratedVariant:
|
||||
name: str
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
content: bytes
|
||||
|
||||
|
||||
def _jpeg_image(image: Image.Image) -> Image.Image:
|
||||
if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info:
|
||||
rgba = image.convert('RGBA')
|
||||
background = Image.new('RGBA', rgba.size, 'white')
|
||||
background.alpha_composite(rgba)
|
||||
return background.convert('RGB')
|
||||
return image.convert('RGB')
|
||||
|
||||
|
||||
def _encode(image: Image.Image, fmt: str, **options) -> bytes:
|
||||
output = BytesIO()
|
||||
image.save(output, format=fmt, **options)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant:
|
||||
return GeneratedVariant(
|
||||
name=name,
|
||||
content_type=content_type,
|
||||
width=image.width,
|
||||
height=image.height,
|
||||
sha256=hashlib.sha256(content).hexdigest(),
|
||||
content=content,
|
||||
)
|
||||
|
||||
|
||||
def process_image(
|
||||
source: bytes,
|
||||
declared_type: str,
|
||||
expected_sha256: str,
|
||||
max_pixels: int,
|
||||
thumbnail_quality: int,
|
||||
crop_quality: int,
|
||||
webp_quality: int,
|
||||
) -> tuple[GeneratedVariant, ...]:
|
||||
if hashlib.sha256(source).hexdigest() != expected_sha256:
|
||||
raise PermanentImageError('checksum_mismatch')
|
||||
try:
|
||||
with Image.open(BytesIO(source)) as opened:
|
||||
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '')
|
||||
if actual != declared_type:
|
||||
raise PermanentImageError('unsupported_media_type')
|
||||
if opened.width * opened.height > max_pixels:
|
||||
raise PermanentImageError('image_too_large')
|
||||
opened.load()
|
||||
oriented = ImageOps.exif_transpose(opened)
|
||||
base = oriented.copy()
|
||||
except PermanentImageError:
|
||||
raise
|
||||
except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error:
|
||||
raise PermanentImageError('invalid_image') from error
|
||||
|
||||
thumbnail = _jpeg_image(base.copy())
|
||||
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
thumbnail_bytes = _encode(
|
||||
thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True
|
||||
)
|
||||
|
||||
crop_source = _jpeg_image(base.copy())
|
||||
crop = ImageOps.fit(
|
||||
crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||
)
|
||||
crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True)
|
||||
|
||||
optimized = base.copy()
|
||||
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6)
|
||||
|
||||
return (
|
||||
_variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes),
|
||||
_variant('crop', 'image/jpeg', crop, crop_bytes),
|
||||
_variant('optimized', 'image/webp', optimized, optimized_bytes),
|
||||
)
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from ..config import load_settings
|
||||
from ..observability import get_metrics
|
||||
from ..observability.logging import safe_extra
|
||||
from ..observability.propagation import extract_trace_context, inject_trace_context
|
||||
from ..repositories import JobRepository, OutboxRepository
|
||||
from .broker import ROUTING_KEY, connect, message, topology
|
||||
from .runtime import configure_logging, dependencies
|
||||
|
||||
logger = logging.getLogger('app.publisher')
|
||||
tracer = trace.get_tracer('messenger.publisher', '1.6.0')
|
||||
|
||||
|
||||
async def run(once: bool = False) -> None:
|
||||
settings = load_settings()
|
||||
engine, sessions, _storage = dependencies(settings)
|
||||
outbox = OutboxRepository(sessions)
|
||||
jobs = JobRepository(sessions)
|
||||
instruments = get_metrics()
|
||||
queue_attributes = {'messaging.destination.name': 'attachments.process'}
|
||||
owner = f'{socket.gethostname()}:{os.getpid()}'
|
||||
connection = await connect(settings.broker_url)
|
||||
try:
|
||||
channel = await connection.channel(publisher_confirms=True)
|
||||
exchange, _queue = await topology(channel)
|
||||
while True:
|
||||
now = datetime.now(timezone.utc)
|
||||
await jobs.recover_expired(now)
|
||||
depth, oldest_age = await jobs.queue_stats(now)
|
||||
instruments.queue_depth.set(depth, queue_attributes)
|
||||
instruments.queue_oldest_age.set(oldest_age, queue_attributes)
|
||||
events = await outbox.claim(owner, now, lease_seconds=30)
|
||||
for event in events:
|
||||
try:
|
||||
parent = extract_trace_context({
|
||||
'traceparent': event.traceparent,
|
||||
'tracestate': event.tracestate,
|
||||
})
|
||||
with tracer.start_as_current_span(
|
||||
'attachments publish',
|
||||
context=parent,
|
||||
kind=SpanKind.PRODUCER,
|
||||
attributes={
|
||||
'messaging.system': 'rabbitmq',
|
||||
'messaging.destination.name': 'attachments.process',
|
||||
'messaging.operation.name': 'publish',
|
||||
'messaging.operation.type': 'send',
|
||||
'job.id': str(event.job_id),
|
||||
},
|
||||
):
|
||||
body = json.dumps(event.payload, separators=(',', ':')).encode()
|
||||
headers = inject_trace_context()
|
||||
await exchange.publish(
|
||||
message(body, str(event.id), headers),
|
||||
routing_key=ROUTING_KEY,
|
||||
mandatory=True,
|
||||
)
|
||||
logger.info(
|
||||
'outbox event published',
|
||||
extra=safe_extra(
|
||||
'outbox.published', event_id=str(event.id),
|
||||
job_id=str(event.job_id),
|
||||
),
|
||||
)
|
||||
if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1':
|
||||
os._exit(91)
|
||||
await outbox.published(event.id, owner, datetime.now(timezone.utc))
|
||||
except Exception:
|
||||
logger.exception(
|
||||
'outbox publish failed',
|
||||
extra=safe_extra(
|
||||
'outbox.publish_failed', event_id=str(event.id),
|
||||
job_id=str(event.job_id),
|
||||
),
|
||||
)
|
||||
await outbox.failed(
|
||||
event.id, owner, 'broker_unavailable', datetime.now(timezone.utc)
|
||||
)
|
||||
if once:
|
||||
return
|
||||
await asyncio.sleep(0.5 if events else 1.0)
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--once', action='store_true')
|
||||
args = parser.parse_args()
|
||||
telemetry = configure_logging('messenger-publisher')
|
||||
try:
|
||||
asyncio.run(run(args.once))
|
||||
finally:
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from ..config import Settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..observability import initialize_observability
|
||||
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
def configure_logging(service_name: str):
|
||||
return initialize_observability(service_name)
|
||||
|
||||
|
||||
def dependencies(settings: Settings):
|
||||
engine = create_engine(settings.database_url)
|
||||
SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
endpoint=settings.s3_endpoint,
|
||||
access_key=settings.s3_access_key_id,
|
||||
secret_key=settings.s3_secret_access_key,
|
||||
bucket=settings.s3_bucket,
|
||||
region=settings.s3_region,
|
||||
)
|
||||
return engine, sessions, storage
|
||||
@@ -0,0 +1,390 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
import aio_pika
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from ..config import Settings, load_settings
|
||||
from ..domain.errors import Unavailable
|
||||
from ..domain.models import AttachmentVariant
|
||||
from ..observability import get_metrics
|
||||
from ..observability.logging import safe_extra
|
||||
from ..observability.propagation import extract_trace_context
|
||||
from ..repositories import JobRepository
|
||||
from .broker import connect, topology
|
||||
from .pipeline import GeneratedVariant, PermanentImageError, process_image
|
||||
from .runtime import configure_logging, dependencies
|
||||
|
||||
logger = logging.getLogger('app.worker')
|
||||
tracer = trace.get_tracer('messenger.worker', '1.6.0')
|
||||
|
||||
|
||||
class ProcessingTimeout(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None:
|
||||
try:
|
||||
if memory_bytes:
|
||||
try:
|
||||
import resource
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
||||
except (ImportError, OSError, ValueError):
|
||||
pass
|
||||
connection.send(('ok', process_image(*arguments)))
|
||||
except PermanentImageError as error:
|
||||
connection.send(('permanent', error.code))
|
||||
except MemoryError:
|
||||
connection.send(('temporary', 'processing_memory_limit'))
|
||||
except BaseException:
|
||||
connection.send(('temporary', 'processing_failed'))
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]:
|
||||
context = multiprocessing.get_context('spawn')
|
||||
receiver, sender = context.Pipe(duplex=False)
|
||||
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
|
||||
process.start()
|
||||
sender.close()
|
||||
if not receiver.poll(timeout):
|
||||
process.terminate()
|
||||
process.join(5)
|
||||
receiver.close()
|
||||
raise ProcessingTimeout('processing_timeout')
|
||||
try:
|
||||
kind, result = receiver.recv()
|
||||
except EOFError as error:
|
||||
process.join(5)
|
||||
receiver.close()
|
||||
raise ProcessingTimeout('processing_failed') from error
|
||||
receiver.close()
|
||||
process.join(5)
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join(5)
|
||||
raise ProcessingTimeout('processing_failed')
|
||||
if kind == 'ok':
|
||||
return result
|
||||
if kind == 'permanent':
|
||||
raise PermanentImageError(result)
|
||||
raise ProcessingTimeout(result)
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None:
|
||||
self.settings = settings
|
||||
self.jobs = jobs
|
||||
self.storage = storage
|
||||
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
|
||||
self.semaphore = asyncio.Semaphore(settings.worker_concurrency)
|
||||
self.metrics = get_metrics()
|
||||
self.metric_attributes = {'job.type': 'attachment.process.v1'}
|
||||
self.metrics.worker_busy.set(0, self.metric_attributes)
|
||||
self.metrics.worker_capacity.set(settings.worker_concurrency, self.metric_attributes)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _busy_slot(self):
|
||||
async with self.semaphore:
|
||||
self.metrics.worker_busy.add(1, self.metric_attributes)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.metrics.worker_busy.add(-1, self.metric_attributes)
|
||||
|
||||
def _delay(self, attempt: int) -> float:
|
||||
base = min(
|
||||
self.settings.retry_max_seconds,
|
||||
self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)),
|
||||
)
|
||||
return base + random.uniform(0, base)
|
||||
|
||||
async def _handle_failure(
|
||||
self,
|
||||
incoming: aio_pika.IncomingMessage,
|
||||
job_id: UUID,
|
||||
attempt: int,
|
||||
code: str,
|
||||
permanent: bool,
|
||||
) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
available_at = now + timedelta(seconds=self._delay(attempt))
|
||||
try:
|
||||
state = await self.jobs.retry_or_fail(
|
||||
job_id,
|
||||
self.worker_id,
|
||||
code,
|
||||
permanent,
|
||||
available_at,
|
||||
now,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception('failed to persist job failure job_id=%s', job_id)
|
||||
await incoming.nack(requeue=True)
|
||||
return
|
||||
if state == 'failed':
|
||||
logger.error(
|
||||
'job reached terminal failure',
|
||||
extra=safe_extra(
|
||||
'job.failed', job_id=str(job_id), attempt=attempt, error_code=code
|
||||
),
|
||||
)
|
||||
await incoming.reject(requeue=False)
|
||||
else:
|
||||
logger.warning(
|
||||
'job retry scheduled',
|
||||
extra=safe_extra(
|
||||
'job.retry_scheduled', job_id=str(job_id), attempt=attempt,
|
||||
error_code=code,
|
||||
),
|
||||
)
|
||||
await incoming.ack()
|
||||
|
||||
async def handle(self, incoming: aio_pika.IncomingMessage) -> None:
|
||||
context = extract_trace_context(incoming.headers)
|
||||
job_id: UUID | None = None
|
||||
before = None
|
||||
try:
|
||||
preview = json.loads(incoming.body)
|
||||
job_id = UUID(preview['job_id'])
|
||||
before = await self.jobs.state(job_id)
|
||||
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||
pass
|
||||
started_at = time.perf_counter()
|
||||
attributes = {
|
||||
'messaging.system': 'rabbitmq',
|
||||
'messaging.destination.name': 'attachments.process',
|
||||
'messaging.operation.name': 'process',
|
||||
'messaging.operation.type': 'process',
|
||||
}
|
||||
if job_id is not None:
|
||||
attributes['job.id'] = str(job_id)
|
||||
with tracer.start_as_current_span(
|
||||
'attachments process', context=context, kind=SpanKind.CONSUMER,
|
||||
attributes=attributes,
|
||||
) as span:
|
||||
await self._handle_message(incoming)
|
||||
if before is None or before.state != 'queued' or job_id is None:
|
||||
return
|
||||
after = await self.jobs.state(job_id)
|
||||
if after is None or after.state not in {'queued', 'ready', 'failed'}:
|
||||
return
|
||||
if after.state == 'queued' and after.attempt == before.attempt:
|
||||
return
|
||||
outcome = 'retry' if after.state == 'queued' else after.state
|
||||
duration = time.perf_counter() - started_at
|
||||
metric_attributes = {**self.metric_attributes, 'outcome': outcome}
|
||||
self.metrics.jobs_processed.add(1, metric_attributes)
|
||||
self.metrics.job_duration.record(duration, metric_attributes)
|
||||
span.set_attribute('job.outcome', outcome)
|
||||
span.set_attribute('job.attempt', before.attempt)
|
||||
if outcome == 'ready':
|
||||
ready_seconds = max(
|
||||
0.0, (datetime.now(timezone.utc) - after.created_at).total_seconds()
|
||||
)
|
||||
self.metrics.time_to_ready.record(
|
||||
ready_seconds,
|
||||
{'pipeline.version': str(after.pipeline_version), 'outcome': 'ready'},
|
||||
)
|
||||
|
||||
async def _handle_message(self, incoming: aio_pika.IncomingMessage) -> None:
|
||||
async with self._busy_slot():
|
||||
try:
|
||||
payload = json.loads(incoming.body)
|
||||
if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1':
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
job_id = UUID(payload['job_id'])
|
||||
attachment_id = UUID(payload['attachment_id'])
|
||||
attempt = int(payload['attempt'])
|
||||
pipeline_version = int(payload['pipeline_version'])
|
||||
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
|
||||
current = await self.jobs.state(job_id)
|
||||
if current is None:
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version:
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
if current.state == 'ready':
|
||||
if await self.jobs.ready_effect_complete(job_id):
|
||||
await incoming.ack()
|
||||
else:
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
if current.state == 'failed':
|
||||
await incoming.reject(requeue=False)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
claimed = await self.jobs.claim(
|
||||
job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds
|
||||
)
|
||||
if claimed is None:
|
||||
await incoming.ack()
|
||||
return
|
||||
job, attachment = claimed
|
||||
if os.getenv('FAULT_WORKER_PERMANENT') == '1':
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'fault_injected', True
|
||||
)
|
||||
os._exit(94)
|
||||
if not attachment.staging_key:
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'staging_missing', True
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
source = await self.storage.read(
|
||||
attachment.staging_key, self.settings.max_upload_bytes
|
||||
)
|
||||
arguments = (
|
||||
source,
|
||||
attachment.content_type,
|
||||
attachment.sha256,
|
||||
self.settings.max_image_pixels,
|
||||
self.settings.thumbnail_jpeg_quality,
|
||||
self.settings.crop_jpeg_quality,
|
||||
self.settings.optimized_webp_quality,
|
||||
)
|
||||
generated = await asyncio.to_thread(
|
||||
_run_isolated,
|
||||
arguments,
|
||||
self.settings.job_timeout_seconds,
|
||||
512 * 1024 * 1024,
|
||||
)
|
||||
original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original'
|
||||
await self.storage.put_bytes(original_key, source, attachment.content_type)
|
||||
variants: list[AttachmentVariant] = []
|
||||
suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'}
|
||||
for item in generated:
|
||||
key = (
|
||||
f'variants/{attachment.id}/v{job.pipeline_version}/'
|
||||
f'{item.name}.{suffixes[item.name]}'
|
||||
)
|
||||
await self.storage.put_bytes(key, item.content, item.content_type)
|
||||
variants.append(
|
||||
AttachmentVariant(
|
||||
name=item.name,
|
||||
object_key=key,
|
||||
content_type=item.content_type,
|
||||
size_bytes=len(item.content),
|
||||
width=item.width,
|
||||
height=item.height,
|
||||
sha256=item.sha256,
|
||||
)
|
||||
)
|
||||
if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1':
|
||||
os._exit(92)
|
||||
completed = await self.jobs.complete(
|
||||
job.id,
|
||||
self.worker_id,
|
||||
original_key,
|
||||
tuple(variants),
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
if not completed:
|
||||
await incoming.nack(requeue=True)
|
||||
return
|
||||
if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1':
|
||||
os._exit(93)
|
||||
try:
|
||||
await self.storage.remove(attachment.staging_key)
|
||||
except Unavailable:
|
||||
logger.warning(
|
||||
'staging cleanup deferred',
|
||||
extra=safe_extra(
|
||||
'staging.cleanup_deferred', attachment_id=str(attachment.id)
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
'job completed',
|
||||
extra=safe_extra(
|
||||
'job.completed', job_id=str(job.id),
|
||||
attachment_id=str(attachment.id), attempt=job.attempt,
|
||||
),
|
||||
)
|
||||
await incoming.ack()
|
||||
except PermanentImageError as error:
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, error.code, True
|
||||
)
|
||||
except ProcessingTimeout as error:
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, str(error), False
|
||||
)
|
||||
except (Unavailable, OSError, ValueError):
|
||||
logger.exception(
|
||||
'temporary processing failure',
|
||||
extra=safe_extra(
|
||||
'job.processing_failed', job_id=str(job.id),
|
||||
error_code='storage_unavailable',
|
||||
),
|
||||
)
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'storage_unavailable', False
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
'unexpected processing failure',
|
||||
extra=safe_extra(
|
||||
'job.processing_failed', job_id=str(job.id),
|
||||
error_code='processing_failed',
|
||||
),
|
||||
)
|
||||
await self._handle_failure(
|
||||
incoming, job.id, job.attempt, 'processing_failed', False
|
||||
)
|
||||
|
||||
|
||||
async def run() -> None:
|
||||
settings = load_settings()
|
||||
engine, sessions, storage = dependencies(settings)
|
||||
jobs = JobRepository(sessions)
|
||||
worker = Worker(settings, jobs, storage)
|
||||
connection = await connect(settings.broker_url)
|
||||
try:
|
||||
channel = await connection.channel()
|
||||
await channel.set_qos(prefetch_count=settings.worker_prefetch)
|
||||
_exchange, queue = await topology(channel)
|
||||
await queue.consume(worker.handle, no_ack=False)
|
||||
logger.info(
|
||||
'worker started id=%s concurrency=%d prefetch=%d',
|
||||
worker.worker_id,
|
||||
settings.worker_concurrency,
|
||||
settings.worker_prefetch,
|
||||
)
|
||||
await asyncio.Future()
|
||||
finally:
|
||||
await connection.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
telemetry = configure_logging('messenger-worker')
|
||||
try:
|
||||
asyncio.run(run())
|
||||
finally:
|
||||
telemetry.shutdown()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
"""Сборка FastAPI приложения и подключение обязательных хранилищ."""
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
|
||||
from sqlalchemy import text
|
||||
|
||||
from .api import errors
|
||||
from .api.deps import Services
|
||||
from .api.middleware import RequestContext
|
||||
from .api.routers import attachments, auth, chats, health, messages, users
|
||||
from .config import load_settings
|
||||
from .db import RedisSessionStore, create_engine, create_redis, create_session_factory
|
||||
from .observability import get_metrics, initialize_observability
|
||||
from .observability.logging import safe_extra
|
||||
from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository
|
||||
from .services.attachments import AttachmentService
|
||||
from .services.chats import ChatService
|
||||
from .services.messages import MessageService
|
||||
from .services.users import UserService
|
||||
from .storage import S3Storage
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger('app')
|
||||
|
||||
DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.'
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
telemetry = initialize_observability('messenger-api')
|
||||
get_metrics()
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(application: FastAPI):
|
||||
application.state.ready = False
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine)
|
||||
redis = create_redis(settings.redis_url)
|
||||
storage = S3Storage(
|
||||
endpoint=settings.s3_endpoint,
|
||||
access_key=settings.s3_access_key_id,
|
||||
secret_key=settings.s3_secret_access_key,
|
||||
bucket=settings.s3_bucket,
|
||||
region=settings.s3_region,
|
||||
)
|
||||
try:
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
session_store = RedisSessionStore(redis, settings.session_ttl_seconds)
|
||||
await session_store.ping()
|
||||
await storage.ping()
|
||||
session_factory = create_session_factory(engine)
|
||||
user_repository = UserRepository(session_factory)
|
||||
chat_repository = ChatRepository(session_factory)
|
||||
message_repository = MessageRepository(session_factory)
|
||||
attachment_repository = AttachmentRepository(session_factory)
|
||||
application.state.services = Services(
|
||||
attachments=AttachmentService(
|
||||
attachment_repository,
|
||||
storage,
|
||||
settings.max_upload_bytes,
|
||||
settings.max_image_pixels,
|
||||
settings.job_max_attempts,
|
||||
),
|
||||
users=UserService(user_repository),
|
||||
chats=ChatService(chat_repository),
|
||||
messages=MessageService(message_repository),
|
||||
sessions=session_store,
|
||||
)
|
||||
application.state.engine = engine
|
||||
application.state.redis = redis
|
||||
application.state.storage = storage
|
||||
application.state.ready = True
|
||||
logger.info(
|
||||
'application started',
|
||||
extra=safe_extra(
|
||||
'api.startup', storage='postgres+s3', redis_sessions=True,
|
||||
async_attachments=True,
|
||||
),
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
application.state.ready = False
|
||||
await redis.aclose()
|
||||
await engine.dispose()
|
||||
logger.info('application stopped', extra=safe_extra('api.shutdown'))
|
||||
telemetry.shutdown()
|
||||
|
||||
api = FastAPI(
|
||||
title='MTUSI Messenger. LR 6',
|
||||
version='1.6.0',
|
||||
description=DESCRIPTION,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
api.router.redirect_slashes = False
|
||||
api.state.ready = False
|
||||
api.add_middleware(RequestContext)
|
||||
errors.install(api)
|
||||
for router in (
|
||||
health.router,
|
||||
users.router,
|
||||
auth.router,
|
||||
chats.router,
|
||||
messages.router,
|
||||
attachments.router,
|
||||
):
|
||||
api.include_router(router)
|
||||
FastAPIInstrumentor.instrument_app(
|
||||
api,
|
||||
excluded_urls='/health/live,/health/ready',
|
||||
)
|
||||
return api
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,18 @@
|
||||
from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow
|
||||
from .chat import ChatMemberRow, ChatRow
|
||||
from .message import MessageRow
|
||||
from .user import UserRow
|
||||
from .job import AttachmentJobRow, JobEventRow, OutboxEventRow
|
||||
|
||||
__all__ = [
|
||||
'AttachmentRow',
|
||||
'AttachmentVariantRow',
|
||||
'AttachmentJobRow',
|
||||
'ChatMemberRow',
|
||||
'ChatRow',
|
||||
'MessageAttachmentRow',
|
||||
'MessageRow',
|
||||
'UserRow',
|
||||
'JobEventRow',
|
||||
'OutboxEventRow',
|
||||
]
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class AttachmentRow(Base):
|
||||
__tablename__ = 'attachments'
|
||||
__table_args__ = (
|
||||
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
|
||||
UniqueConstraint('original_key', name='attachments_object_key_key'),
|
||||
UniqueConstraint('staging_key', name='attachments_staging_key_key'),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
original_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class AttachmentVariantRow(Base):
|
||||
__tablename__ = 'attachment_variants'
|
||||
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class MessageAttachmentRow(Base):
|
||||
__tablename__ = 'message_attachments'
|
||||
__table_args__ = (
|
||||
UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'),
|
||||
Index('message_attachments_attachment_idx', 'attachment_id'),
|
||||
)
|
||||
|
||||
message_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class ChatRow(Base):
|
||||
__tablename__ = 'chats'
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
title: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class ChatMemberRow(Base):
|
||||
__tablename__ = 'chat_members'
|
||||
__table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),)
|
||||
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||
user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class AttachmentJobRow(Base):
|
||||
__tablename__ = 'attachment_jobs'
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
|
||||
),
|
||||
Index('attachment_jobs_lease_idx', 'state', 'lease_until'),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class OutboxEventRow(Base):
|
||||
__tablename__ = 'outbox_events'
|
||||
__table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
job_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
|
||||
)
|
||||
payload: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||
traceparent: Mapped[str | None] = mapped_column(String(55), nullable=True)
|
||||
tracestate: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class JobEventRow(Base):
|
||||
__tablename__ = 'job_events'
|
||||
__table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
|
||||
job_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
|
||||
)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class MessageRow(Base):
|
||||
__tablename__ = 'messages'
|
||||
__table_args__ = (
|
||||
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||
UniqueConstraint(
|
||||
'sender_id',
|
||||
'chat_id',
|
||||
'idempotency_key',
|
||||
name='messages_sender_chat_idempotency_key',
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||
sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||
text: Mapped[str] = mapped_column(String(4000), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import DateTime, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import Uuid
|
||||
|
||||
from ..db.postgres import Base
|
||||
|
||||
|
||||
class UserRow(Base):
|
||||
__tablename__ = 'users'
|
||||
|
||||
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False)
|
||||
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
@@ -0,0 +1,9 @@
|
||||
from .metrics import Metrics, get_metrics
|
||||
from .runtime import ObservabilityRuntime, initialize_observability
|
||||
|
||||
__all__ = [
|
||||
'Metrics',
|
||||
'ObservabilityRuntime',
|
||||
'get_metrics',
|
||||
'initialize_observability',
|
||||
]
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
def _positive_int(name: str, default: str) -> int:
|
||||
try:
|
||||
value = int(os.getenv(name, default))
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f'{name} должен быть целым числом') from exc
|
||||
if value <= 0:
|
||||
raise RuntimeError(f'{name} должен быть больше нуля')
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObservabilitySettings:
|
||||
enabled: bool
|
||||
endpoint: str
|
||||
service_name: str
|
||||
service_version: str
|
||||
environment: str
|
||||
export_timeout_seconds: int
|
||||
span_queue_size: int
|
||||
log_queue_size: int
|
||||
metric_export_interval_ms: int
|
||||
|
||||
|
||||
def load_observability_settings(default_service_name: str) -> ObservabilitySettings:
|
||||
disabled = os.getenv('OTEL_SDK_DISABLED', 'false').strip().lower()
|
||||
if disabled not in {'true', 'false'}:
|
||||
raise RuntimeError('OTEL_SDK_DISABLED должен быть true или false')
|
||||
endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318')
|
||||
return ObservabilitySettings(
|
||||
enabled=disabled != 'true',
|
||||
endpoint=endpoint.rstrip('/'),
|
||||
service_name=os.getenv('OTEL_SERVICE_NAME', default_service_name),
|
||||
service_version=os.getenv('OTEL_SERVICE_VERSION', '1.6.0'),
|
||||
environment=os.getenv('OTEL_DEPLOYMENT_ENVIRONMENT', 'lab'),
|
||||
export_timeout_seconds=_positive_int('OTEL_EXPORTER_OTLP_TIMEOUT', '2'),
|
||||
span_queue_size=_positive_int('OTEL_BSP_MAX_QUEUE_SIZE', '2048'),
|
||||
log_queue_size=_positive_int('OTEL_BLRP_MAX_QUEUE_SIZE', '2048'),
|
||||
metric_export_interval_ms=_positive_int('OTEL_METRIC_EXPORT_INTERVAL', '5000'),
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from opentelemetry import trace
|
||||
|
||||
|
||||
_STANDARD_FIELDS = {
|
||||
'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 'funcName',
|
||||
'levelname', 'levelno', 'lineno', 'module', 'msecs', 'message', 'msg', 'name',
|
||||
'pathname', 'process', 'processName', 'relativeCreated', 'stack_info', 'thread',
|
||||
'threadName', 'taskName',
|
||||
}
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, object] = {
|
||||
'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
|
||||
'severity': record.levelname,
|
||||
'logger': record.name,
|
||||
'message': record.getMessage(),
|
||||
'event': getattr(record, 'event', 'log.message'),
|
||||
}
|
||||
span_context = trace.get_current_span().get_span_context()
|
||||
if span_context.is_valid:
|
||||
payload['trace_id'] = format(span_context.trace_id, '032x')
|
||||
payload['span_id'] = format(span_context.span_id, '016x')
|
||||
for key, value in record.__dict__.items():
|
||||
if key in _STANDARD_FIELDS or key.startswith('_'):
|
||||
continue
|
||||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||
payload[key] = value
|
||||
if record.exc_info:
|
||||
payload['exception.type'] = record.exc_info[0].__name__
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(',', ':'))
|
||||
|
||||
|
||||
def safe_extra(event: str, **attributes: object) -> dict[str, object]:
|
||||
blocked = {
|
||||
'authorization', 'password', 'token', 'session_token', 'message_body',
|
||||
'image_bytes', 's3_secret_access_key', 'signed_url',
|
||||
}
|
||||
extra: dict[str, object] = {'event': event}
|
||||
for key, value in attributes.items():
|
||||
normalized = key.lower()
|
||||
if normalized in blocked or any(part in normalized for part in ('password', 'secret')):
|
||||
continue
|
||||
extra[key] = value
|
||||
return extra
|
||||
@@ -0,0 +1,122 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Mapping
|
||||
|
||||
import psutil
|
||||
from opentelemetry import metrics
|
||||
from opentelemetry.metrics import Observation
|
||||
|
||||
|
||||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0)
|
||||
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
|
||||
READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0)
|
||||
|
||||
|
||||
class GaugeState:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._values: dict[tuple[tuple[str, str], ...], float] = {}
|
||||
|
||||
def set(self, value: float, attributes: Mapping[str, str] | None = None) -> None:
|
||||
key = tuple(sorted((attributes or {}).items()))
|
||||
with self._lock:
|
||||
self._values[key] = float(value)
|
||||
|
||||
def add(self, amount: float, attributes: Mapping[str, str] | None = None) -> None:
|
||||
key = tuple(sorted((attributes or {}).items()))
|
||||
with self._lock:
|
||||
self._values[key] = self._values.get(key, 0.0) + amount
|
||||
|
||||
def observations(self, _options=None):
|
||||
with self._lock:
|
||||
snapshot = tuple(self._values.items())
|
||||
return [Observation(value, dict(attributes)) for attributes, value in snapshot]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Metrics:
|
||||
http_requests: object
|
||||
http_duration: object
|
||||
jobs_processed: object
|
||||
job_duration: object
|
||||
time_to_ready: object
|
||||
queue_depth: GaugeState
|
||||
queue_oldest_age: GaugeState
|
||||
worker_busy: GaugeState
|
||||
worker_capacity: GaugeState
|
||||
db_pool_used: GaugeState
|
||||
db_pool_waiting: GaugeState
|
||||
|
||||
|
||||
_metrics: Metrics | None = None
|
||||
|
||||
|
||||
def _process_rss(_options=None):
|
||||
process = psutil.Process(os.getpid())
|
||||
return [Observation(process.memory_info().rss)]
|
||||
|
||||
|
||||
def _process_cpu(_options=None):
|
||||
process = psutil.Process(os.getpid())
|
||||
times = process.cpu_times()
|
||||
return [Observation(times.user + times.system)]
|
||||
|
||||
|
||||
def get_metrics() -> Metrics:
|
||||
global _metrics
|
||||
if _metrics is not None:
|
||||
return _metrics
|
||||
meter = metrics.get_meter('messenger.observability', '1.6.0')
|
||||
queue_depth = GaugeState()
|
||||
queue_oldest_age = GaugeState()
|
||||
worker_busy = GaugeState()
|
||||
worker_capacity = GaugeState()
|
||||
db_pool_used = GaugeState()
|
||||
db_pool_waiting = GaugeState()
|
||||
meter.create_observable_gauge(
|
||||
'messenger.queue.depth', callbacks=[queue_depth.observations], unit='{job}'
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
'messenger.queue.oldest.age', callbacks=[queue_oldest_age.observations], unit='s'
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
'messenger.worker.busy', callbacks=[worker_busy.observations], unit='{worker}'
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
'messenger.worker.capacity', callbacks=[worker_capacity.observations], unit='{worker}'
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
'messenger.db.pool.used', callbacks=[db_pool_used.observations], unit='{connection}'
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
'messenger.db.pool.waiting', callbacks=[db_pool_waiting.observations], unit='{request}'
|
||||
)
|
||||
meter.create_observable_gauge('process.memory.rss', callbacks=[_process_rss], unit='By')
|
||||
meter.create_observable_counter('process.cpu.time', callbacks=[_process_cpu], unit='s')
|
||||
_metrics = Metrics(
|
||||
http_requests=meter.create_counter(
|
||||
'http.server.request.count', unit='{request}', description='HTTP requests'
|
||||
),
|
||||
http_duration=meter.create_histogram(
|
||||
'http.server.request.duration', unit='s', description='HTTP request duration'
|
||||
),
|
||||
jobs_processed=meter.create_counter(
|
||||
'messenger.jobs.processed', unit='{job}', description='Finished job attempts'
|
||||
),
|
||||
job_duration=meter.create_histogram(
|
||||
'messenger.job.processing.duration', unit='s', description='Job processing duration'
|
||||
),
|
||||
time_to_ready=meter.create_histogram(
|
||||
'messenger.attachment.time_to_ready', unit='s', description='Durable accept to ready'
|
||||
),
|
||||
queue_depth=queue_depth,
|
||||
queue_oldest_age=queue_oldest_age,
|
||||
worker_busy=worker_busy,
|
||||
worker_capacity=worker_capacity,
|
||||
db_pool_used=db_pool_used,
|
||||
db_pool_waiting=db_pool_waiting,
|
||||
)
|
||||
return _metrics
|
||||
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
from opentelemetry import propagate
|
||||
|
||||
|
||||
_ALLOWED = {'traceparent', 'tracestate'}
|
||||
|
||||
|
||||
def inject_trace_context(context=None) -> dict[str, str]:
|
||||
carrier: dict[str, str] = {}
|
||||
propagate.inject(carrier, context=context)
|
||||
return {key: str(value) for key, value in carrier.items() if key in _ALLOWED}
|
||||
|
||||
|
||||
def extract_trace_context(carrier: Mapping[str, object] | None):
|
||||
normalized: dict[str, str] = {}
|
||||
for key, value in (carrier or {}).items():
|
||||
lowered = str(key).lower()
|
||||
if lowered not in _ALLOWED or value is None:
|
||||
continue
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode('ascii', errors='ignore')
|
||||
normalized[lowered] = str(value)
|
||||
return propagate.extract(normalized)
|
||||
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
|
||||
from .config import load_observability_settings
|
||||
from .logging import JsonFormatter
|
||||
from .metrics import HTTP_BUCKETS, JOB_BUCKETS, READY_BUCKETS
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ObservabilityRuntime:
|
||||
tracer_provider: TracerProvider | None = None
|
||||
meter_provider: MeterProvider | None = None
|
||||
logger_provider: LoggerProvider | None = None
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.logger_provider is not None:
|
||||
self.logger_provider.shutdown()
|
||||
if self.meter_provider is not None:
|
||||
self.meter_provider.shutdown()
|
||||
if self.tracer_provider is not None:
|
||||
self.tracer_provider.shutdown()
|
||||
|
||||
|
||||
_runtime: ObservabilityRuntime | None = None
|
||||
|
||||
|
||||
def initialize_observability(default_service_name: str) -> ObservabilityRuntime:
|
||||
global _runtime
|
||||
if _runtime is not None:
|
||||
return _runtime
|
||||
settings = load_observability_settings(default_service_name)
|
||||
console = logging.StreamHandler(sys.stdout)
|
||||
console.setFormatter(JsonFormatter())
|
||||
handlers: list[logging.Handler] = [console]
|
||||
runtime = ObservabilityRuntime()
|
||||
if settings.enabled:
|
||||
resource = Resource.create({
|
||||
'service.name': settings.service_name,
|
||||
'service.version': settings.service_version,
|
||||
'service.instance.id': os.getenv(
|
||||
'INSTANCE_ID', f'{socket.gethostname()}:{os.getpid()}'
|
||||
),
|
||||
'deployment.environment.name': settings.environment,
|
||||
})
|
||||
timeout = settings.export_timeout_seconds
|
||||
tracer_provider = TracerProvider(resource=resource)
|
||||
tracer_provider.add_span_processor(BatchSpanProcessor(
|
||||
OTLPSpanExporter(endpoint=f'{settings.endpoint}/v1/traces', timeout=timeout),
|
||||
max_queue_size=settings.span_queue_size,
|
||||
max_export_batch_size=min(512, settings.span_queue_size),
|
||||
schedule_delay_millis=1000,
|
||||
export_timeout_millis=timeout * 1000,
|
||||
))
|
||||
trace.set_tracer_provider(tracer_provider)
|
||||
|
||||
metric_reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=f'{settings.endpoint}/v1/metrics', timeout=timeout),
|
||||
export_interval_millis=settings.metric_export_interval_ms,
|
||||
export_timeout_millis=timeout * 1000,
|
||||
)
|
||||
meter_provider = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[metric_reader],
|
||||
views=[
|
||||
View(
|
||||
instrument_name='http.server.request.duration',
|
||||
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
|
||||
),
|
||||
View(
|
||||
instrument_name='messenger.job.processing.duration',
|
||||
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
|
||||
),
|
||||
View(
|
||||
instrument_name='messenger.attachment.time_to_ready',
|
||||
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS),
|
||||
),
|
||||
],
|
||||
)
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
|
||||
logger_provider = LoggerProvider(resource=resource)
|
||||
logger_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=f'{settings.endpoint}/v1/logs', timeout=timeout),
|
||||
max_queue_size=settings.log_queue_size,
|
||||
max_export_batch_size=min(512, settings.log_queue_size),
|
||||
schedule_delay_millis=1000,
|
||||
export_timeout_millis=timeout * 1000,
|
||||
))
|
||||
set_logger_provider(logger_provider)
|
||||
otlp_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
|
||||
otlp_handler.setFormatter(JsonFormatter())
|
||||
handlers.append(otlp_handler)
|
||||
runtime = ObservabilityRuntime(tracer_provider, meter_provider, logger_provider)
|
||||
logging.basicConfig(
|
||||
level=os.getenv('LOG_LEVEL', 'info').upper(), handlers=handlers, force=True
|
||||
)
|
||||
_runtime = runtime
|
||||
return runtime
|
||||
@@ -0,0 +1,10 @@
|
||||
from .attachments import AttachmentRepository
|
||||
from .chats import ChatRepository
|
||||
from .messages import MessageRepository
|
||||
from .users import UserRepository
|
||||
from .jobs import JobRepository, OutboxRepository
|
||||
|
||||
__all__ = [
|
||||
'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository',
|
||||
'OutboxRepository', 'UserRepository',
|
||||
]
|
||||
@@ -0,0 +1,177 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.models import Attachment, AttachmentVariant
|
||||
from ..models.attachment import AttachmentRow, AttachmentVariantRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.job import AttachmentJobRow, OutboxEventRow
|
||||
|
||||
|
||||
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict:
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'job_id': str(job_id),
|
||||
'type': 'attachment.process.v1',
|
||||
'attachment_id': str(attachment_id),
|
||||
'pipeline_version': pipeline_version,
|
||||
'attempt': attempt,
|
||||
'created_at': now.isoformat().replace('+00:00', 'Z'),
|
||||
}
|
||||
|
||||
|
||||
class AttachmentRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _variant(row: AttachmentVariantRow) -> AttachmentVariant:
|
||||
return AttachmentVariant(
|
||||
name=row.name,
|
||||
object_key=row.object_key,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
width=row.width,
|
||||
height=row.height,
|
||||
sha256=row.sha256,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _to_domain(
|
||||
cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = ()
|
||||
) -> Attachment:
|
||||
return Attachment(
|
||||
id=row.id,
|
||||
chat_id=row.chat_id,
|
||||
owner_id=row.owner_id,
|
||||
filename=row.filename,
|
||||
original_key=row.original_key,
|
||||
staging_key=row.staging_key,
|
||||
pipeline_version=row.pipeline_version,
|
||||
content_type=row.content_type,
|
||||
size_bytes=row.size_bytes,
|
||||
sha256=row.sha256,
|
||||
width=row.width,
|
||||
height=row.height,
|
||||
status=row.status,
|
||||
error_code=row.error_code,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
variants=tuple(cls._variant(item) for item in variants),
|
||||
)
|
||||
|
||||
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
|
||||
async with self._sessions() as session:
|
||||
found = await session.scalar(
|
||||
select(ChatRow.id)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
return found is not None
|
||||
|
||||
async def create_queued(
|
||||
self,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
job_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
staging_key: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
max_attempts: int,
|
||||
now: datetime,
|
||||
traceparent: str | None = None,
|
||||
tracestate: str | None = None,
|
||||
) -> Attachment:
|
||||
row = AttachmentRow(
|
||||
id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=owner_id,
|
||||
filename=filename,
|
||||
original_key=None,
|
||||
staging_key=staging_key,
|
||||
pipeline_version=1,
|
||||
content_type=content_type,
|
||||
size_bytes=size_bytes,
|
||||
sha256=sha256,
|
||||
width=width,
|
||||
height=height,
|
||||
status='queued',
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
job = AttachmentJobRow(
|
||||
id=job_id,
|
||||
attachment_id=attachment_id,
|
||||
pipeline_version=1,
|
||||
state='queued',
|
||||
attempt=1,
|
||||
max_attempts=max_attempts,
|
||||
next_attempt_at=now,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
last_error_code=None,
|
||||
replay_count=0,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
outbox = OutboxEventRow(
|
||||
id=uuid4(),
|
||||
event_type='attachment.process.v1',
|
||||
job_id=job_id,
|
||||
payload=job_payload(job_id, attachment_id, 1, 1, now),
|
||||
traceparent=traceparent,
|
||||
tracestate=tracestate,
|
||||
available_at=now,
|
||||
published_at=None,
|
||||
claim_owner=None,
|
||||
claim_until=None,
|
||||
publish_attempts=0,
|
||||
last_error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
session.add_all((row, job, outbox))
|
||||
await session.flush()
|
||||
return self._to_domain(row)
|
||||
|
||||
async def _variants(
|
||||
self, session: AsyncSession, attachment_id: UUID, pipeline_version: int
|
||||
) -> tuple[AttachmentVariantRow, ...]:
|
||||
rows = await session.scalars(
|
||||
select(AttachmentVariantRow)
|
||||
.where(
|
||||
AttachmentVariantRow.attachment_id == attachment_id,
|
||||
AttachmentVariantRow.pipeline_version == pipeline_version,
|
||||
)
|
||||
.order_by(AttachmentVariantRow.name)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.scalar(
|
||||
select(AttachmentRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id)
|
||||
.where(
|
||||
AttachmentRow.id == attachment_id,
|
||||
ChatMemberRow.user_id == user_id,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return self._to_domain(
|
||||
row, await self._variants(session, row.id, row.pipeline_version)
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.errors import Invalid
|
||||
from ..domain.models import Chat, Keyset
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.user import UserRow
|
||||
|
||||
|
||||
class ChatRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat:
|
||||
return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at)
|
||||
|
||||
@staticmethod
|
||||
async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]:
|
||||
result = await session.scalars(
|
||||
select(ChatMemberRow.user_id)
|
||||
.where(ChatMemberRow.chat_id == chat_id)
|
||||
.order_by(ChatMemberRow.user_id)
|
||||
)
|
||||
return tuple(result)
|
||||
|
||||
async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids))))
|
||||
if len(found) != len(member_ids):
|
||||
raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}')
|
||||
row = ChatRow(id=chat_id, title=title, created_at=created_at)
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids)
|
||||
await session.flush()
|
||||
return self._to_domain(row, member_ids)
|
||||
|
||||
async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.scalar(
|
||||
select(ChatRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return self._to_domain(row, await self._members(session, row.id))
|
||||
|
||||
async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]:
|
||||
async with self._sessions() as session:
|
||||
query = (
|
||||
select(ChatRow)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatMemberRow.user_id == user_id)
|
||||
.order_by(ChatRow.created_at, ChatRow.id)
|
||||
.limit(limit)
|
||||
)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id))
|
||||
rows = list((await session.scalars(query)).all())
|
||||
return [self._to_domain(row, await self._members(session, row.id)) for row in rows]
|
||||
@@ -0,0 +1,388 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent
|
||||
from ..models.attachment import AttachmentRow, AttachmentVariantRow
|
||||
from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow
|
||||
from ..observability.propagation import inject_trace_context
|
||||
from .attachments import AttachmentRepository, job_payload
|
||||
|
||||
|
||||
class JobRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _job(row: AttachmentJobRow) -> AttachmentJob:
|
||||
return AttachmentJob(
|
||||
id=row.id,
|
||||
attachment_id=row.attachment_id,
|
||||
pipeline_version=row.pipeline_version,
|
||||
state=row.state,
|
||||
attempt=row.attempt,
|
||||
max_attempts=row.max_attempts,
|
||||
next_attempt_at=row.next_attempt_at,
|
||||
lease_owner=row.lease_owner,
|
||||
lease_until=row.lease_until,
|
||||
last_error_code=row.last_error_code,
|
||||
replay_count=row.replay_count,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
)
|
||||
|
||||
async def claim(
|
||||
self,
|
||||
job_id: UUID,
|
||||
attempt: int,
|
||||
worker_id: str,
|
||||
now: datetime,
|
||||
lease_seconds: int,
|
||||
) -> tuple[AttachmentJob, Attachment] | None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
row = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if row is None or row.state != 'queued' or row.attempt != attempt:
|
||||
return None
|
||||
attachment_row = await session.get(AttachmentRow, row.attachment_id)
|
||||
if attachment_row is None:
|
||||
return None
|
||||
row.state = 'processing'
|
||||
row.lease_owner = worker_id
|
||||
row.lease_until = now + timedelta(seconds=lease_seconds)
|
||||
row.updated_at = now
|
||||
attachment_row.status = 'processing'
|
||||
attachment_row.updated_at = now
|
||||
session.add(
|
||||
JobEventRow(
|
||||
job_id=row.id,
|
||||
kind='claimed',
|
||||
attempt=row.attempt,
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
return self._job(row), AttachmentRepository._to_domain(attachment_row)
|
||||
|
||||
async def state(self, job_id: UUID) -> AttachmentJob | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.get(AttachmentJobRow, job_id)
|
||||
return self._job(row) if row is not None else None
|
||||
|
||||
async def id_for_attachment(self, attachment_id: UUID) -> UUID | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
select(AttachmentJobRow.id).where(
|
||||
AttachmentJobRow.attachment_id == attachment_id
|
||||
)
|
||||
)
|
||||
|
||||
async def failed_staging_key(self, job_id: UUID) -> str | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(
|
||||
select(AttachmentRow.staging_key)
|
||||
.join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id)
|
||||
.where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed')
|
||||
)
|
||||
|
||||
async def ready_effect_complete(self, job_id: UUID) -> bool:
|
||||
async with self._sessions() as session:
|
||||
job = await session.get(AttachmentJobRow, job_id)
|
||||
if job is None or job.state != 'ready':
|
||||
return False
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
if attachment is None or not attachment.original_key:
|
||||
return False
|
||||
names = set(
|
||||
await session.scalars(
|
||||
select(AttachmentVariantRow.name).where(
|
||||
AttachmentVariantRow.attachment_id == job.attachment_id,
|
||||
AttachmentVariantRow.pipeline_version == job.pipeline_version,
|
||||
)
|
||||
)
|
||||
)
|
||||
return names == {'thumbnail', 'crop', 'optimized'}
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
job_id: UUID,
|
||||
worker_id: str,
|
||||
original_key: str,
|
||||
variants: tuple[AttachmentVariant, ...],
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
job = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None or job.state == 'ready':
|
||||
return job is not None
|
||||
if job.state != 'processing' or job.lease_owner != worker_id:
|
||||
return False
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
if attachment is None:
|
||||
return False
|
||||
for variant in variants:
|
||||
statement = (
|
||||
insert(AttachmentVariantRow)
|
||||
.values(
|
||||
attachment_id=attachment.id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
name=variant.name,
|
||||
object_key=variant.object_key,
|
||||
content_type=variant.content_type,
|
||||
size_bytes=variant.size_bytes,
|
||||
width=variant.width,
|
||||
height=variant.height,
|
||||
sha256=variant.sha256,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
.on_conflict_do_update(
|
||||
index_elements=[
|
||||
AttachmentVariantRow.attachment_id,
|
||||
AttachmentVariantRow.pipeline_version,
|
||||
AttachmentVariantRow.name,
|
||||
],
|
||||
set_={
|
||||
'object_key': variant.object_key,
|
||||
'content_type': variant.content_type,
|
||||
'size_bytes': variant.size_bytes,
|
||||
'width': variant.width,
|
||||
'height': variant.height,
|
||||
'sha256': variant.sha256,
|
||||
'updated_at': now,
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.execute(statement)
|
||||
attachment.original_key = original_key
|
||||
attachment.status = 'ready'
|
||||
attachment.error_code = None
|
||||
attachment.updated_at = now
|
||||
job.state = 'ready'
|
||||
job.lease_owner = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = None
|
||||
job.updated_at = now
|
||||
session.add(
|
||||
JobEventRow(
|
||||
job_id=job.id,
|
||||
kind='ready',
|
||||
attempt=job.attempt,
|
||||
error_code=None,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
async def retry_or_fail(
|
||||
self,
|
||||
job_id: UUID,
|
||||
worker_id: str | None,
|
||||
error_code: str,
|
||||
permanent: bool,
|
||||
available_at: datetime,
|
||||
now: datetime,
|
||||
) -> str:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
job = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None or job.state in {'ready', 'failed'}:
|
||||
return job.state if job is not None else 'missing'
|
||||
if job.state != 'processing':
|
||||
return job.state
|
||||
if worker_id is not None:
|
||||
if job.lease_owner != worker_id:
|
||||
return job.state
|
||||
elif job.lease_until is None or job.lease_until >= now:
|
||||
return job.state
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
exhausted = job.attempt >= job.max_attempts
|
||||
if permanent or exhausted:
|
||||
job.state = 'failed'
|
||||
attachment.status = 'failed'
|
||||
attachment.error_code = error_code if permanent else 'retry_exhausted'
|
||||
kind = 'failed'
|
||||
else:
|
||||
job.attempt += 1
|
||||
job.state = 'queued'
|
||||
job.next_attempt_at = available_at
|
||||
attachment.status = 'queued'
|
||||
attachment.error_code = None
|
||||
trace_context = inject_trace_context()
|
||||
session.add(
|
||||
OutboxEventRow(
|
||||
id=uuid4(),
|
||||
event_type='attachment.process.v1',
|
||||
job_id=job.id,
|
||||
payload=job_payload(
|
||||
job.id,
|
||||
job.attachment_id,
|
||||
job.pipeline_version,
|
||||
job.attempt,
|
||||
now,
|
||||
),
|
||||
traceparent=trace_context.get('traceparent'),
|
||||
tracestate=trace_context.get('tracestate'),
|
||||
available_at=available_at,
|
||||
published_at=None,
|
||||
claim_owner=None,
|
||||
claim_until=None,
|
||||
publish_attempts=0,
|
||||
last_error_code=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
kind = 'retry_scheduled'
|
||||
job.last_error_code = error_code
|
||||
job.lease_owner = None
|
||||
job.lease_until = None
|
||||
job.updated_at = now
|
||||
attachment.updated_at = now
|
||||
session.add(
|
||||
JobEventRow(
|
||||
job_id=job.id,
|
||||
kind=kind,
|
||||
attempt=job.attempt,
|
||||
error_code=error_code,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
return job.state
|
||||
|
||||
async def queue_stats(self, now: datetime) -> tuple[int, float]:
|
||||
async with self._sessions() as session:
|
||||
depth, oldest = (
|
||||
await session.execute(
|
||||
select(func.count(AttachmentJobRow.id), func.min(AttachmentJobRow.created_at))
|
||||
.where(AttachmentJobRow.state.in_(('queued', 'processing')))
|
||||
)
|
||||
).one()
|
||||
age = max(0.0, (now - oldest).total_seconds()) if oldest is not None else 0.0
|
||||
return int(depth or 0), age
|
||||
|
||||
async def recover_expired(self, now: datetime) -> int:
|
||||
async with self._sessions() as session:
|
||||
ids = tuple(
|
||||
await session.scalars(
|
||||
select(AttachmentJobRow.id).where(
|
||||
AttachmentJobRow.state == 'processing',
|
||||
AttachmentJobRow.lease_until < now,
|
||||
)
|
||||
)
|
||||
)
|
||||
recovered = 0
|
||||
for job_id in ids:
|
||||
state = await self.retry_or_fail(
|
||||
job_id, None, 'lease_expired', False, now, now
|
||||
)
|
||||
recovered += state in {'queued', 'failed'}
|
||||
return recovered
|
||||
|
||||
async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
job = await session.scalar(
|
||||
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||
)
|
||||
if job is None or job.state != 'failed':
|
||||
return False
|
||||
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||
if attachment is None or not attachment.staging_key:
|
||||
return False
|
||||
if dry_run:
|
||||
return True
|
||||
job.state = 'queued'
|
||||
job.attempt += 1
|
||||
job.max_attempts = job.attempt + 4
|
||||
job.replay_count += 1
|
||||
job.last_error_code = None
|
||||
job.next_attempt_at = now
|
||||
job.updated_at = now
|
||||
attachment.status = 'queued'
|
||||
attachment.error_code = None
|
||||
attachment.updated_at = now
|
||||
session.add_all(
|
||||
(
|
||||
OutboxEventRow(
|
||||
id=uuid4(), event_type='attachment.process.v1', job_id=job.id,
|
||||
payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now),
|
||||
traceparent=None, tracestate=None,
|
||||
available_at=now, published_at=None, claim_owner=None, claim_until=None,
|
||||
publish_attempts=0, last_error_code=None, created_at=now, updated_at=now,
|
||||
),
|
||||
JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
class OutboxRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
rows = tuple(
|
||||
await session.scalars(
|
||||
select(OutboxEventRow)
|
||||
.where(
|
||||
OutboxEventRow.published_at.is_(None),
|
||||
OutboxEventRow.available_at <= now,
|
||||
or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now),
|
||||
)
|
||||
.order_by(OutboxEventRow.available_at, OutboxEventRow.created_at)
|
||||
.limit(limit)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
)
|
||||
until = now + timedelta(seconds=lease_seconds)
|
||||
for row in rows:
|
||||
row.claim_owner = owner
|
||||
row.claim_until = until
|
||||
row.publish_attempts += 1
|
||||
row.updated_at = now
|
||||
return tuple(
|
||||
OutboxEvent(
|
||||
row.id, row.job_id, row.payload, row.available_at,
|
||||
row.traceparent, row.tracestate,
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
|
||||
async def published(self, event_id: UUID, owner: str, now: datetime) -> None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(
|
||||
update(OutboxEventRow)
|
||||
.where(
|
||||
OutboxEventRow.id == event_id,
|
||||
OutboxEventRow.claim_owner == owner,
|
||||
OutboxEventRow.published_at.is_(None),
|
||||
)
|
||||
.values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now)
|
||||
)
|
||||
|
||||
async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(
|
||||
update(OutboxEventRow)
|
||||
.where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner)
|
||||
.values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now)
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.errors import Conflict, NotFound
|
||||
from ..domain.models import Keyset, Message
|
||||
from ..models.attachment import AttachmentRow, MessageAttachmentRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.message import MessageRow
|
||||
|
||||
|
||||
class MessageRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message:
|
||||
return Message(
|
||||
id=row.id,
|
||||
chat_id=row.chat_id,
|
||||
sender_id=row.sender_id,
|
||||
text=row.text,
|
||||
created_at=row.created_at,
|
||||
attachment_ids=tuple(attachment_ids),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]:
|
||||
result = await session.scalars(
|
||||
select(MessageAttachmentRow.attachment_id)
|
||||
.where(MessageAttachmentRow.message_id == message_id)
|
||||
.order_by(MessageAttachmentRow.position)
|
||||
)
|
||||
return tuple(result)
|
||||
|
||||
@staticmethod
|
||||
async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None:
|
||||
return await session.scalar(
|
||||
select(ChatRow.id)
|
||||
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||
)
|
||||
|
||||
async def create_for_member(
|
||||
self,
|
||||
message_id: UUID,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
attachment_ids: Sequence[UUID],
|
||||
created_at: datetime,
|
||||
idempotency_key: str,
|
||||
) -> Message | None:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id)
|
||||
if accessible_chat_id is None:
|
||||
return None
|
||||
|
||||
if attachment_ids:
|
||||
found = {
|
||||
attachment_id: status
|
||||
for attachment_id, status in (
|
||||
await session.execute(
|
||||
select(AttachmentRow.id, AttachmentRow.status).where(
|
||||
AttachmentRow.id.in_(attachment_ids),
|
||||
AttachmentRow.chat_id == accessible_chat_id,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
}
|
||||
if set(found) != set(attachment_ids):
|
||||
raise NotFound('вложение не найдено')
|
||||
if any(status == 'failed' for status in found.values()):
|
||||
raise Conflict('неуспешное вложение нельзя прикрепить')
|
||||
|
||||
statement = (
|
||||
insert(MessageRow)
|
||||
.values(
|
||||
id=message_id,
|
||||
chat_id=accessible_chat_id,
|
||||
sender_id=sender_id,
|
||||
text=text,
|
||||
created_at=created_at,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
.on_conflict_do_nothing(
|
||||
index_elements=[
|
||||
MessageRow.sender_id,
|
||||
MessageRow.chat_id,
|
||||
MessageRow.idempotency_key,
|
||||
]
|
||||
)
|
||||
.returning(MessageRow.id)
|
||||
)
|
||||
created_id = await session.scalar(statement)
|
||||
if created_id is not None:
|
||||
row = await session.get(MessageRow, created_id)
|
||||
session.add_all(
|
||||
MessageAttachmentRow(
|
||||
message_id=created_id,
|
||||
attachment_id=attachment_id,
|
||||
position=position,
|
||||
)
|
||||
for position, attachment_id in enumerate(attachment_ids)
|
||||
)
|
||||
await session.flush()
|
||||
result_attachment_ids = tuple(attachment_ids)
|
||||
else:
|
||||
row = await session.scalar(
|
||||
select(MessageRow).where(
|
||||
MessageRow.sender_id == sender_id,
|
||||
MessageRow.chat_id == accessible_chat_id,
|
||||
MessageRow.idempotency_key == idempotency_key,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
raise RuntimeError('результат идемпотентной операции не найден')
|
||||
result_attachment_ids = await self._attachment_ids(session, row.id)
|
||||
if row.text != text or result_attachment_ids != tuple(attachment_ids):
|
||||
raise Conflict('Idempotency-Key уже использован с другим сообщением')
|
||||
return self._to_domain(row, result_attachment_ids)
|
||||
|
||||
async def page_for_member(
|
||||
self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int
|
||||
) -> list[Message] | None:
|
||||
async with self._sessions() as session:
|
||||
accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id)
|
||||
if accessible_chat_id is None:
|
||||
return None
|
||||
query = (
|
||||
select(MessageRow)
|
||||
.where(MessageRow.chat_id == accessible_chat_id)
|
||||
.order_by(MessageRow.created_at, MessageRow.id)
|
||||
.limit(limit)
|
||||
)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id))
|
||||
rows = list((await session.scalars(query)).all())
|
||||
return [
|
||||
self._to_domain(row, await self._attachment_ids(session, row.id))
|
||||
for row in rows
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..domain.errors import Conflict
|
||||
from ..domain.models import User
|
||||
from ..models.user import UserRow
|
||||
|
||||
|
||||
class UserRepository:
|
||||
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sessions = sessions
|
||||
|
||||
@staticmethod
|
||||
def _to_domain(row: UserRow) -> User:
|
||||
return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at)
|
||||
|
||||
async def register(
|
||||
self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime
|
||||
) -> User:
|
||||
row = UserRow(
|
||||
id=user_id,
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=password_hash,
|
||||
created_at=created_at,
|
||||
)
|
||||
try:
|
||||
async with self._sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
except IntegrityError as exc:
|
||||
raise Conflict('username уже занят') from exc
|
||||
return self._to_domain(row)
|
||||
|
||||
async def get_by_id(self, user_id: UUID) -> User | None:
|
||||
async with self._sessions() as session:
|
||||
row = await session.get(UserRow, user_id)
|
||||
return self._to_domain(row) if row is not None else None
|
||||
|
||||
async def get_credentials(self, username: str) -> UserRow | None:
|
||||
async with self._sessions() as session:
|
||||
return await session.scalar(select(UserRow).where(UserRow.username == username))
|
||||
|
||||
@classmethod
|
||||
def to_domain(cls, row: UserRow) -> User:
|
||||
return cls._to_domain(row)
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from tempfile import SpooledTemporaryFile
|
||||
from typing import AsyncIterator, BinaryIO
|
||||
import unicodedata
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from ..domain.errors import (
|
||||
Conflict,
|
||||
Invalid,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
from ..domain.models import Attachment, User
|
||||
from ..observability.logging import safe_extra
|
||||
from ..observability.propagation import inject_trace_context
|
||||
from ..repositories import AttachmentRepository
|
||||
from ..storage import ObjectStream, S3Storage
|
||||
|
||||
logger = logging.getLogger('app.attachments')
|
||||
|
||||
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
|
||||
|
||||
|
||||
def _validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255:
|
||||
raise Invalid('filename должен содержать от 1 до 255 символов')
|
||||
if any(
|
||||
character in {'/', '\\'} or unicodedata.category(character) == 'Cc'
|
||||
for character in filename
|
||||
):
|
||||
raise Invalid('filename содержит запрещённые символы')
|
||||
|
||||
|
||||
def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]:
|
||||
try:
|
||||
source.seek(0)
|
||||
with Image.open(source) as image:
|
||||
actual_type = MIME_BY_FORMAT.get(image.format or '')
|
||||
if actual_type is None or actual_type != declared_type:
|
||||
raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG')
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise UnsupportedMediaType('изображение имеет неверные размеры')
|
||||
if width * height > max_pixels:
|
||||
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей')
|
||||
image.verify()
|
||||
except Image.DecompressionBombError as error:
|
||||
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error
|
||||
except (UnidentifiedImageError, OSError, SyntaxError) as error:
|
||||
raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error
|
||||
finally:
|
||||
source.seek(0)
|
||||
return actual_type, width, height
|
||||
|
||||
|
||||
class AttachmentService:
|
||||
def __init__(
|
||||
self,
|
||||
repository: AttachmentRepository,
|
||||
storage: S3Storage,
|
||||
max_upload_bytes: int,
|
||||
max_image_pixels: int,
|
||||
job_max_attempts: int,
|
||||
) -> None:
|
||||
self._repository = repository
|
||||
self._storage = storage
|
||||
self._max_upload_bytes = max_upload_bytes
|
||||
self._max_image_pixels = max_image_pixels
|
||||
self._job_max_attempts = job_max_attempts
|
||||
|
||||
async def upload(
|
||||
self,
|
||||
actor: User,
|
||||
chat_id: UUID,
|
||||
filename: str,
|
||||
declared_type: str,
|
||||
chunks: AsyncIterator[bytes],
|
||||
) -> Attachment:
|
||||
_validate_filename(filename)
|
||||
if declared_type not in MIME_BY_FORMAT.values():
|
||||
raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg')
|
||||
if not await self._repository.chat_exists_for_member(chat_id, actor.id):
|
||||
raise NotFound('чат не найден')
|
||||
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source:
|
||||
async for chunk in chunks:
|
||||
size += len(chunk)
|
||||
if size > self._max_upload_bytes:
|
||||
raise PayloadTooLarge('файл превышает лимит 10 MiB')
|
||||
digest.update(chunk)
|
||||
source.write(chunk)
|
||||
if size == 0:
|
||||
raise UnsupportedMediaType('пустой файл не является изображением')
|
||||
|
||||
actual_type, width, height = await asyncio.to_thread(
|
||||
_inspect_image,
|
||||
source,
|
||||
declared_type,
|
||||
self._max_image_pixels,
|
||||
)
|
||||
attachment_id = uuid4()
|
||||
job_id = uuid4()
|
||||
staging_key = f'staging/{attachment_id}/original'
|
||||
await self._storage.put(staging_key, source, size, actual_type)
|
||||
now = datetime.now(timezone.utc)
|
||||
trace_context = inject_trace_context()
|
||||
try:
|
||||
attachment = await self._repository.create_queued(
|
||||
attachment_id=attachment_id,
|
||||
job_id=job_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=actor.id,
|
||||
filename=filename,
|
||||
staging_key=staging_key,
|
||||
content_type=actual_type,
|
||||
size_bytes=size,
|
||||
sha256=digest.hexdigest(),
|
||||
width=width,
|
||||
height=height,
|
||||
max_attempts=self._job_max_attempts,
|
||||
now=now,
|
||||
traceparent=trace_context.get('traceparent'),
|
||||
tracestate=trace_context.get('tracestate'),
|
||||
)
|
||||
logger.info(
|
||||
'attachment accepted for asynchronous processing',
|
||||
extra=safe_extra(
|
||||
'attachment.accepted', attachment_id=str(attachment_id),
|
||||
job_id=str(job_id), pipeline_version=1,
|
||||
),
|
||||
)
|
||||
return attachment
|
||||
except Exception:
|
||||
try:
|
||||
await self._storage.remove(staging_key)
|
||||
except Unavailable:
|
||||
logger.warning(
|
||||
'failed to compensate staging object',
|
||||
extra=safe_extra(
|
||||
'staging.cleanup_deferred', attachment_id=str(attachment_id)
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
|
||||
attachment = await self._repository.get_for_member(attachment_id, actor.id)
|
||||
if attachment is None:
|
||||
raise NotFound('вложение не найдено')
|
||||
return attachment
|
||||
|
||||
async def content(
|
||||
self, actor: User, attachment_id: UUID, variant: str
|
||||
) -> tuple[Attachment, ObjectStream, str, int]:
|
||||
attachment = await self.get(actor, attachment_id)
|
||||
if attachment.status != 'ready':
|
||||
raise Conflict('вложение ещё не готово')
|
||||
if variant == 'original':
|
||||
key = attachment.original_key
|
||||
content_type = attachment.content_type
|
||||
size_bytes = attachment.size_bytes
|
||||
else:
|
||||
found = next((item for item in attachment.variants if item.name == variant), None)
|
||||
if found is None:
|
||||
raise NotFound('вариант вложения не найден')
|
||||
key = found.object_key
|
||||
content_type = found.content_type
|
||||
size_bytes = found.size_bytes
|
||||
if key is None:
|
||||
raise Unavailable('объект вложения отсутствует в метаданных')
|
||||
try:
|
||||
stream = await self._storage.get(key)
|
||||
except FileNotFoundError as error:
|
||||
raise Unavailable('объект вложения отсутствует в хранилище') from error
|
||||
return attachment, stream, content_type, size_bytes
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Sequence
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..domain.errors import Invalid, NotFound
|
||||
from ..domain.models import Chat, Keyset, Page, User
|
||||
from ..repositories import ChatRepository
|
||||
|
||||
|
||||
class ChatService:
|
||||
def __init__(self, repository: ChatRepository) -> None:
|
||||
self._repository = repository
|
||||
|
||||
async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat:
|
||||
if actor.id not in member_ids:
|
||||
raise Invalid('вызывающий должен входить в member_ids')
|
||||
return await self._repository.create(
|
||||
chat_id=uuid4(),
|
||||
title=title,
|
||||
member_ids=member_ids,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
async def get(self, actor: User, chat_id: UUID) -> Chat:
|
||||
chat = await self._repository.get_for_member(chat_id, actor.id)
|
||||
if chat is None:
|
||||
raise NotFound('чат не найден')
|
||||
return chat
|
||||
|
||||
async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page:
|
||||
rows = await self._repository.page_for_member(actor.id, after, limit + 1)
|
||||
items = tuple(rows[:limit])
|
||||
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Sequence
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..domain.errors import NotFound
|
||||
from ..domain.models import Keyset, Message, Page, User
|
||||
from ..repositories import MessageRepository
|
||||
|
||||
|
||||
class MessageService:
|
||||
def __init__(self, repository: MessageRepository) -> None:
|
||||
self._repository = repository
|
||||
|
||||
async def send(
|
||||
self,
|
||||
actor: User,
|
||||
chat_id: UUID,
|
||||
text: str,
|
||||
attachment_ids: Sequence[UUID],
|
||||
idempotency_key: str,
|
||||
) -> Message:
|
||||
message = await self._repository.create_for_member(
|
||||
message_id=uuid4(),
|
||||
chat_id=chat_id,
|
||||
sender_id=actor.id,
|
||||
text=text,
|
||||
attachment_ids=attachment_ids,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
if message is None:
|
||||
raise NotFound('чат не найден')
|
||||
return message
|
||||
|
||||
async def history(
|
||||
self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int
|
||||
) -> Page:
|
||||
rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1)
|
||||
if rows is None:
|
||||
raise NotFound('чат не найден')
|
||||
items = tuple(rows[:limit])
|
||||
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from ..auth.passwords import hash_password, password_matches
|
||||
from ..domain.errors import Unauthorized
|
||||
from ..domain.models import User
|
||||
from ..repositories import UserRepository
|
||||
|
||||
|
||||
class UserService:
|
||||
def __init__(self, repository: UserRepository) -> None:
|
||||
self._repository = repository
|
||||
|
||||
async def register(self, username: str, display_name: str, password: str) -> User:
|
||||
return await self._repository.register(
|
||||
user_id=uuid4(),
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=hash_password(password),
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
async def identify(self, user_id: UUID) -> User:
|
||||
user = await self._repository.get_by_id(user_id)
|
||||
if user is None:
|
||||
raise Unauthorized('требуется действующая сессия')
|
||||
return user
|
||||
|
||||
async def authenticate(self, username: str, password: str) -> User | None:
|
||||
row = await self._repository.get_credentials(username)
|
||||
if row is None or not password_matches(row.password_hash, password):
|
||||
return None
|
||||
return self._repository.to_domain(row)
|
||||
@@ -0,0 +1,3 @@
|
||||
from .s3 import ObjectStream, S3Storage, StoredObject
|
||||
|
||||
__all__ = ['ObjectStream', 'S3Storage', 'StoredObject']
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import certifi
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator
|
||||
from io import BytesIO
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from minio import Minio
|
||||
from minio.error import InvalidResponseError, S3Error
|
||||
from urllib3 import PoolManager, Retry, Timeout
|
||||
from urllib3.exceptions import HTTPError
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from ..domain.errors import Unavailable
|
||||
|
||||
tracer = trace.get_tracer('messenger.s3', '1.6.0')
|
||||
|
||||
|
||||
def _unavailable(error: Exception) -> Unavailable:
|
||||
return Unavailable('объектное хранилище недоступно')
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ObjectStream:
|
||||
response: object
|
||||
|
||||
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
|
||||
try:
|
||||
while chunk := self.response.read(size):
|
||||
yield chunk
|
||||
finally:
|
||||
self.response.close()
|
||||
self.response.release_conn()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StoredObject:
|
||||
key: str
|
||||
last_modified: datetime
|
||||
|
||||
|
||||
class S3Storage:
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
bucket: str,
|
||||
region: str,
|
||||
) -> None:
|
||||
parsed = urlparse(endpoint)
|
||||
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
|
||||
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
|
||||
self._client = Minio(
|
||||
parsed.netloc,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
secure=parsed.scheme == 'https',
|
||||
region=region,
|
||||
http_client=PoolManager(
|
||||
maxsize=8,
|
||||
timeout=Timeout(connect=2, read=5),
|
||||
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
|
||||
cert_reqs='CERT_REQUIRED',
|
||||
ca_certs=certifi.where(),
|
||||
),
|
||||
)
|
||||
self.bucket = bucket
|
||||
|
||||
@staticmethod
|
||||
def _translate(error: Exception) -> None:
|
||||
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
|
||||
raise FileNotFoundError from error
|
||||
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
|
||||
raise _unavailable(error) from error
|
||||
raise error
|
||||
|
||||
async def ping(self) -> None:
|
||||
try:
|
||||
with tracer.start_as_current_span(
|
||||
's3 bucket exists', kind=SpanKind.CLIENT,
|
||||
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||
):
|
||||
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
if not exists:
|
||||
raise Unavailable('bucket объектного хранилища не создан')
|
||||
|
||||
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
|
||||
source.seek(0)
|
||||
try:
|
||||
with tracer.start_as_current_span(
|
||||
's3 put object', kind=SpanKind.CLIENT,
|
||||
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||
):
|
||||
await asyncio.to_thread(
|
||||
self._client.put_object,
|
||||
self.bucket,
|
||||
key,
|
||||
source,
|
||||
length,
|
||||
content_type,
|
||||
)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
|
||||
async def get(self, key: str) -> ObjectStream:
|
||||
try:
|
||||
with tracer.start_as_current_span(
|
||||
's3 get object', kind=SpanKind.CLIENT,
|
||||
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||
):
|
||||
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
return ObjectStream(response)
|
||||
|
||||
async def read(self, key: str, max_bytes: int) -> bytes:
|
||||
stream = await self.get(key)
|
||||
result = bytearray()
|
||||
for chunk in stream.chunks():
|
||||
result.extend(chunk)
|
||||
if len(result) > max_bytes:
|
||||
raise ValueError('stored object exceeds configured byte limit')
|
||||
return bytes(result)
|
||||
|
||||
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
|
||||
await self.put(key, BytesIO(content), len(content), content_type)
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
try:
|
||||
with tracer.start_as_current_span(
|
||||
's3 stat object', kind=SpanKind.CLIENT,
|
||||
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||
):
|
||||
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
|
||||
return True
|
||||
except Exception as error:
|
||||
try:
|
||||
self._translate(error)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
raise
|
||||
|
||||
async def remove(self, key: str) -> None:
|
||||
try:
|
||||
with tracer.start_as_current_span(
|
||||
's3 remove object', kind=SpanKind.CLIENT,
|
||||
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||
):
|
||||
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
|
||||
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
|
||||
def collect() -> tuple[StoredObject, ...]:
|
||||
return tuple(
|
||||
StoredObject(item.object_name, item.last_modified)
|
||||
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
||||
)
|
||||
|
||||
try:
|
||||
return await asyncio.to_thread(collect)
|
||||
except Exception as error:
|
||||
self._translate(error)
|
||||
@@ -0,0 +1 @@
|
||||
"""Операторские CLI-инструменты приложения."""
|
||||
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
async def cleanup(apply: bool, grace_seconds: int) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
try:
|
||||
sessions = create_session_factory(engine)
|
||||
async with sessions() as session:
|
||||
known = set(
|
||||
key
|
||||
for key in await session.scalars(select(AttachmentRow.original_key))
|
||||
if key is not None
|
||||
)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
removed = 0
|
||||
for item in await storage.list('originals/'):
|
||||
if item.key in known or item.last_modified > cutoff:
|
||||
continue
|
||||
action = 'DELETE' if apply else 'WOULD_DELETE'
|
||||
print(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||
if apply:
|
||||
await storage.remove(item.key)
|
||||
removed += 1
|
||||
print(
|
||||
f'summary mode={"apply" if apply else "dry-run"} '
|
||||
f'known={len(known)} removed={removed} grace_seconds={grace_seconds}'
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata')
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--dry-run', action='store_true')
|
||||
mode.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error('--grace-seconds must be at least 60')
|
||||
raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
async def cleanup(apply: bool, retention_seconds: int) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds)
|
||||
try:
|
||||
async with sessions() as session:
|
||||
active = set(
|
||||
key
|
||||
for key in await session.scalars(
|
||||
select(AttachmentRow.staging_key).where(
|
||||
AttachmentRow.staging_key.is_not(None),
|
||||
AttachmentRow.status.in_(('queued', 'processing')),
|
||||
)
|
||||
)
|
||||
if key is not None
|
||||
)
|
||||
removed = 0
|
||||
for item in await storage.list('staging/'):
|
||||
if item.key in active or item.last_modified > cutoff:
|
||||
continue
|
||||
print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}')
|
||||
if apply:
|
||||
await storage.remove(item.key)
|
||||
removed += 1
|
||||
print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}')
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Clean inactive durable staging objects')
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--dry-run', action='store_true')
|
||||
mode.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--retention-seconds', type=int)
|
||||
args = parser.parse_args()
|
||||
settings = load_settings()
|
||||
retention = args.retention_seconds or settings.staging_retention_seconds
|
||||
if retention < 60:
|
||||
parser.error('retention must be at least 60 seconds')
|
||||
raise SystemExit(asyncio.run(cleanup(args.apply, retention)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,250 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import tracemalloc
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Callable, Sequence
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..storage import S3Storage, StoredObject
|
||||
|
||||
MISSING_ERROR_CODE = 'original_missing'
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentSnapshot:
|
||||
id: UUID
|
||||
object_key: str
|
||||
status: str
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReconcilePlan:
|
||||
orphan: tuple[StoredObject, ...]
|
||||
fresh_orphan: tuple[StoredObject, ...]
|
||||
missing: tuple[AttachmentSnapshot, ...]
|
||||
fresh_missing: tuple[AttachmentSnapshot, ...]
|
||||
correct: int
|
||||
|
||||
|
||||
def build_plan(
|
||||
attachments: Sequence[AttachmentSnapshot],
|
||||
objects: Sequence[StoredObject],
|
||||
cutoff: datetime,
|
||||
) -> ReconcilePlan:
|
||||
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
|
||||
known_keys = {attachment.object_key for attachment in attachments}
|
||||
object_keys = {item.key for item in objects}
|
||||
|
||||
unknown_objects = [item for item in objects if item.key not in known_keys]
|
||||
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
|
||||
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
|
||||
|
||||
ready_without_object = [
|
||||
attachment
|
||||
for attachment in attachments
|
||||
if attachment.status == 'ready' and attachment.object_key not in object_keys
|
||||
]
|
||||
missing = tuple(
|
||||
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
|
||||
)
|
||||
fresh_missing = tuple(
|
||||
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
|
||||
)
|
||||
correct = sum(attachment.object_key in object_keys for attachment in attachments)
|
||||
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
|
||||
|
||||
|
||||
async def _load_attachments(
|
||||
sessions: async_sessionmaker[AsyncSession],
|
||||
) -> tuple[AttachmentSnapshot, ...]:
|
||||
async with sessions() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
AttachmentRow.id,
|
||||
AttachmentRow.original_key,
|
||||
AttachmentRow.status,
|
||||
AttachmentRow.updated_at,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return tuple(AttachmentSnapshot(*row) for row in rows)
|
||||
|
||||
|
||||
async def _metadata_key_exists(
|
||||
sessions: async_sessionmaker[AsyncSession], object_key: str
|
||||
) -> bool:
|
||||
async with sessions() as session:
|
||||
attachment_id = await session.scalar(
|
||||
select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1)
|
||||
)
|
||||
return attachment_id is not None
|
||||
|
||||
|
||||
async def reconcile(
|
||||
sessions: async_sessionmaker[AsyncSession],
|
||||
storage: S3Storage,
|
||||
*,
|
||||
apply: bool,
|
||||
grace_seconds: int,
|
||||
now: datetime | None = None,
|
||||
output: Callable[[str], None] = print,
|
||||
) -> dict[str, int | float | str]:
|
||||
started = time.perf_counter()
|
||||
tracemalloc.start()
|
||||
current_time = now or datetime.now(timezone.utc)
|
||||
cutoff = current_time - timedelta(seconds=grace_seconds)
|
||||
|
||||
attachments = await _load_attachments(sessions)
|
||||
objects = await storage.list('originals/')
|
||||
plan = build_plan(attachments, objects, cutoff)
|
||||
|
||||
for item in plan.fresh_orphan:
|
||||
output(
|
||||
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
|
||||
)
|
||||
for attachment in plan.fresh_missing:
|
||||
output(
|
||||
'SKIP_FRESH_MISSING '
|
||||
f'attachment_id={attachment.id} key={attachment.object_key} '
|
||||
f'updated={attachment.updated_at.isoformat()}'
|
||||
)
|
||||
|
||||
removed = 0
|
||||
race_protected = 0
|
||||
for item in plan.orphan:
|
||||
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
|
||||
if not apply:
|
||||
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||
continue
|
||||
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
|
||||
# metadata immediately before a destructive action so a row created after
|
||||
# the initial snapshot protects its object.
|
||||
if await _metadata_key_exists(sessions, item.key):
|
||||
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
|
||||
race_protected += 1
|
||||
continue
|
||||
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||
await storage.remove(item.key)
|
||||
removed += 1
|
||||
|
||||
marked_failed = 0
|
||||
for attachment in plan.missing:
|
||||
if not apply:
|
||||
output(
|
||||
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
||||
)
|
||||
continue
|
||||
|
||||
# The object may have been restored after list_objects(). Do not turn
|
||||
# healthy metadata into failed based only on the earlier snapshot.
|
||||
if await storage.exists(attachment.object_key):
|
||||
output(
|
||||
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key}'
|
||||
)
|
||||
race_protected += 1
|
||||
continue
|
||||
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
update(AttachmentRow)
|
||||
.where(
|
||||
AttachmentRow.id == attachment.id,
|
||||
AttachmentRow.original_key == attachment.object_key,
|
||||
AttachmentRow.status == 'ready',
|
||||
AttachmentRow.updated_at == attachment.updated_at,
|
||||
)
|
||||
.values(
|
||||
status='failed',
|
||||
error_code=MISSING_ERROR_CODE,
|
||||
updated_at=current_time,
|
||||
)
|
||||
)
|
||||
changed = result.rowcount or 0
|
||||
if changed:
|
||||
output(
|
||||
f'MARK_MISSING attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
||||
)
|
||||
marked_failed += changed
|
||||
else:
|
||||
output(
|
||||
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
|
||||
f'key={attachment.object_key}'
|
||||
)
|
||||
race_protected += 1
|
||||
|
||||
_, peak_bytes = tracemalloc.get_traced_memory()
|
||||
tracemalloc.stop()
|
||||
summary: dict[str, int | float | str] = {
|
||||
'mode': 'apply' if apply else 'dry-run',
|
||||
'metadata': len(attachments),
|
||||
'objects': len(objects),
|
||||
'correct': plan.correct,
|
||||
'orphan': len(plan.orphan),
|
||||
'missing': len(plan.missing),
|
||||
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
|
||||
'removed': removed,
|
||||
'marked_failed': marked_failed,
|
||||
'race_protected': race_protected,
|
||||
'grace_seconds': grace_seconds,
|
||||
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
|
||||
'peak_memory_bytes': peak_bytes,
|
||||
}
|
||||
output(f'summary {json.dumps(summary, sort_keys=True)}')
|
||||
return summary
|
||||
|
||||
|
||||
async def run(apply: bool, grace_seconds: int) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
try:
|
||||
await reconcile(
|
||||
create_session_factory(engine),
|
||||
storage,
|
||||
apply=apply,
|
||||
grace_seconds=grace_seconds,
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument('--dry-run', action='store_true')
|
||||
mode.add_argument('--apply', action='store_true')
|
||||
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error('--grace-seconds must be at least 60')
|
||||
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..repositories import JobRepository
|
||||
from ..storage import S3Storage
|
||||
|
||||
|
||||
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
try:
|
||||
repository = JobRepository(create_session_factory(engine))
|
||||
if job_id is None:
|
||||
assert attachment_id is not None
|
||||
job_id = await repository.id_for_attachment(attachment_id)
|
||||
if job_id is None:
|
||||
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
|
||||
return 2
|
||||
staging_key = await repository.failed_staging_key(job_id)
|
||||
if staging_key is None:
|
||||
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||
return 2
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
if not await storage.exists(staging_key):
|
||||
print(f'REJECTED job_id={job_id} reason=staging_object_missing')
|
||||
return 2
|
||||
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
|
||||
if not accepted:
|
||||
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||
return 2
|
||||
print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}')
|
||||
return 0
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description='Safely replay one failed attachment job')
|
||||
target = parser.add_mutually_exclusive_group(required=True)
|
||||
target.add_argument('--job-id', type=UUID)
|
||||
target.add_argument('--attachment-id', type=UUID)
|
||||
parser.add_argument('--dry-run', action='store_true')
|
||||
args = parser.parse_args()
|
||||
raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from io import BytesIO
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from sqlalchemy import delete, insert
|
||||
|
||||
from ..config import load_settings
|
||||
from ..db import create_engine, create_session_factory
|
||||
from ..models.attachment import AttachmentRow
|
||||
from ..models.chat import ChatMemberRow, ChatRow
|
||||
from ..models.user import UserRow
|
||||
from ..storage import S3Storage
|
||||
|
||||
USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user')
|
||||
CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat')
|
||||
OBJECT_PREFIX = 'originals/reconciler-benchmark/'
|
||||
WORKERS = 16
|
||||
|
||||
|
||||
async def _put_objects(
|
||||
storage: S3Storage,
|
||||
keys: list[str],
|
||||
) -> None:
|
||||
for offset in range(0, len(keys), WORKERS):
|
||||
batch = keys[offset : offset + WORKERS]
|
||||
await asyncio.gather(
|
||||
*(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch)
|
||||
)
|
||||
|
||||
|
||||
async def prepare(size: int) -> None:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
old = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||
rows = [
|
||||
{
|
||||
'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'),
|
||||
'chat_id': CHAT_ID,
|
||||
'owner_id': USER_ID,
|
||||
'filename': f'benchmark-{index}.png',
|
||||
'original_key': f'{OBJECT_PREFIX}{index}',
|
||||
'staging_key': None,
|
||||
'pipeline_version': 1,
|
||||
'content_type': 'image/png',
|
||||
'size_bytes': 1,
|
||||
'sha256': '0' * 64,
|
||||
'width': 1,
|
||||
'height': 1,
|
||||
'status': 'ready',
|
||||
'error_code': None,
|
||||
'created_at': old,
|
||||
'updated_at': old,
|
||||
}
|
||||
for index in range(size)
|
||||
]
|
||||
try:
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
session.add(UserRow(
|
||||
id=USER_ID,
|
||||
username=f'bench-{USER_ID.hex[:16]}',
|
||||
display_name='Reconciler benchmark',
|
||||
password_hash='not-used',
|
||||
created_at=old,
|
||||
))
|
||||
session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old))
|
||||
await session.flush()
|
||||
session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID))
|
||||
await session.flush()
|
||||
for offset in range(0, size, 500):
|
||||
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
|
||||
|
||||
await _put_objects(storage, [row['original_key'] for row in rows])
|
||||
print(f'prepared metadata={size} objects={size}')
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def cleanup() -> None:
|
||||
settings = load_settings()
|
||||
engine = create_engine(settings.database_url)
|
||||
sessions = create_session_factory(engine)
|
||||
storage = S3Storage(
|
||||
settings.s3_endpoint,
|
||||
settings.s3_access_key_id,
|
||||
settings.s3_secret_access_key,
|
||||
settings.s3_bucket,
|
||||
settings.s3_region,
|
||||
)
|
||||
try:
|
||||
objects = await storage.list(OBJECT_PREFIX)
|
||||
keys = [item.key for item in objects]
|
||||
for offset in range(0, len(keys), WORKERS):
|
||||
await asyncio.gather(
|
||||
*(storage.remove(key) for key in keys[offset : offset + WORKERS])
|
||||
)
|
||||
|
||||
async with sessions() as session:
|
||||
async with session.begin():
|
||||
await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID))
|
||||
await session.execute(delete(UserRow).where(UserRow.id == USER_ID))
|
||||
print(f'cleaned objects={len(keys)}')
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks'
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest='command', required=True)
|
||||
prepare_parser = subparsers.add_parser('prepare')
|
||||
prepare_parser.add_argument('--size', type=int, required=True)
|
||||
subparsers.add_parser('cleanup')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == 'prepare':
|
||||
if args.size < 1 or args.size > 10000:
|
||||
parser.error('--size must be between 1 and 10000')
|
||||
asyncio.run(prepare(args.size))
|
||||
else:
|
||||
asyncio.run(cleanup())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,14 @@
|
||||
services:
|
||||
observability-test:
|
||||
image: python:3.13.15-slim
|
||||
working_dir: /tests
|
||||
command: ["python", "observability_smoke.py", "--grafana-url", "http://grafana:3000"]
|
||||
environment:
|
||||
GRAFANA_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GRAFANA_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana}
|
||||
volumes:
|
||||
- ./tests:/tests:ro
|
||||
depends_on:
|
||||
grafana:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
@@ -0,0 +1,198 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger_test
|
||||
POSTGRES_PASSWORD: messenger_test
|
||||
POSTGRES_DB: messenger_test
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger_test
|
||||
RABBITMQ_DEFAULT_PASS: messenger_test
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||
user: "0"
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: test-minio-admin
|
||||
MINIO_ROOT_PASSWORD: test-minio-root-password
|
||||
S3_BUCKET: messenger-attachments
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
s3-bootstrap:
|
||||
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing local/"$$S3_BUCKET"
|
||||
mc anonymous set none local/"$$S3_BUCKET"
|
||||
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
|
||||
mc admin policy create local messenger-api /tmp/s3-api-policy.json
|
||||
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
|
||||
environment:
|
||||
MINIO_ROOT_USER: test-minio-admin
|
||||
MINIO_ROOT_PASSWORD: test-minio-root-password
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
volumes:
|
||||
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build: .
|
||||
ports:
|
||||
- "127.0.0.1:18080:8080"
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
MAX_UPLOAD_BYTES: 10485760
|
||||
MAX_IMAGE_PIXELS: 40000000
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
JOB_MAX_ATTEMPTS: 5
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
publisher:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.publisher"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.worker"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
WORKER_CONCURRENCY: 2
|
||||
WORKER_PREFETCH: 2
|
||||
JOB_MAX_ATTEMPTS: 5
|
||||
JOB_TIMEOUT_SECONDS: 60
|
||||
JOB_LEASE_SECONDS: 90
|
||||
RETRY_BASE_SECONDS: 1
|
||||
RETRY_MAX_SECONDS: 4
|
||||
STAGING_RETENTION_SECONDS: 86400
|
||||
THUMBNAIL_JPEG_QUALITY: 85
|
||||
CROP_JPEG_QUALITY: 85
|
||||
OPTIMIZED_WEBP_QUALITY: 82
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
tests:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.test
|
||||
environment:
|
||||
BASE_URL: http://api:8080
|
||||
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: 3600
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: messenger-api
|
||||
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
x-otel-environment: &otel-environment
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-2}
|
||||
OTEL_SERVICE_VERSION: ${OTEL_SERVICE_VERSION:-1.6.0}
|
||||
OTEL_DEPLOYMENT_ENVIRONMENT: ${OTEL_DEPLOYMENT_ENVIRONMENT:-lab}
|
||||
OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048}
|
||||
OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048}
|
||||
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-5000}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.4-alpine
|
||||
cpus: "1.0"
|
||||
mem_limit: 512m
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
cpus: "0.25"
|
||||
mem_limit: 128m
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
cpus: "0.5"
|
||||
mem_limit: 384m
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq}
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
|
||||
s3:
|
||||
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||
user: "0"
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
|
||||
s3-bootstrap:
|
||||
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
|
||||
cpus: "0.25"
|
||||
mem_limit: 128m
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||
mc mb --ignore-existing local/"$$S3_BUCKET"
|
||||
mc anonymous set none local/"$$S3_BUCKET"
|
||||
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
|
||||
mc admin policy create local messenger-api /tmp/s3-api-policy.json
|
||||
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
volumes:
|
||||
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build: .
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment:
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build: .
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
environment:
|
||||
<<: *otel-environment
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
expose:
|
||||
- "8080"
|
||||
deploy:
|
||||
replicas: 2
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
stop_grace_period: 10s
|
||||
|
||||
publisher:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.publisher"]
|
||||
cpus: "0.25"
|
||||
mem_limit: 192m
|
||||
environment:
|
||||
<<: *otel-environment
|
||||
OTEL_SERVICE_NAME: messenger-publisher
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
|
||||
worker:
|
||||
build: .
|
||||
command: ["python", "-m", "app.jobs.worker"]
|
||||
cpus: "1.0"
|
||||
mem_limit: 768m
|
||||
environment:
|
||||
<<: *otel-environment
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
REDIS_URL: ${REDIS_URL}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||
RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2}
|
||||
RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85}
|
||||
CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85}
|
||||
OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 70s
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine
|
||||
cpus: "0.25"
|
||||
mem_limit: 128m
|
||||
ports:
|
||||
- "${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
stop_grace_period: 5s
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
cpus: "0.5"
|
||||
mem_limit: 256m
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
depends_on:
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
cpus: "0.5"
|
||||
mem_limit: 384m
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=1GB
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus_data:/prometheus
|
||||
depends_on:
|
||||
otel-collector:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.4.2
|
||||
user: "0"
|
||||
cpus: "0.5"
|
||||
mem_limit: 384m
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
user: "0"
|
||||
cpus: "0.5"
|
||||
mem_limit: 384m
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.6.0
|
||||
cpus: "0.5"
|
||||
mem_limit: 384m
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- ./infra/grafana/runbooks:/var/lib/grafana/runbooks:ro
|
||||
- grafana_data:/var/lib/grafana
|
||||
depends_on:
|
||||
prometheus:
|
||||
condition: service_started
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
broker_data:
|
||||
prometheus_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
grafana_data:
|
||||
@@ -0,0 +1,37 @@
|
||||
# Результаты повторной проверки 2026-11-20
|
||||
|
||||
Стенд: Apple M4 Pro, 24 GiB RAM; Docker Desktop VM 12 CPU / 9.70 GiB;
|
||||
лимиты сервисов из `compose.yaml`, только synthetic data.
|
||||
|
||||
```text
|
||||
make test
|
||||
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
make test-observability
|
||||
PASS: dashboard, alerts, metrics, correlated logs and cross-process trace
|
||||
|
||||
make test-retry-trace
|
||||
{"scenario":"retry_trace","attachment_id":"7ee0da1b-0c50-4d2e-9bf8-e2c68efb2ce4",
|
||||
"trace_id":"e8e2aff13b7bdb632c9cec5007dcc6d4","producer_spans":2,
|
||||
"consumer_spans":2,"result":"PASS"}
|
||||
|
||||
make test-alert-worker
|
||||
{"scenario":"worker_outage","firing_state":"Alerting","detect_seconds":61.8,
|
||||
"resolved_state":"Normal","recover_seconds":15.5,"result":"PASS"}
|
||||
|
||||
make test-alert-database
|
||||
{"scenario":"database_outage","firing_state":"Alerting","detect_seconds":72.2,
|
||||
"resolved_state":"Normal","recover_seconds":113.1,
|
||||
"trace_id":"34f96d6fc934f7c54075c44a7b39c176","db_span":"connect",
|
||||
"db_system":"postgresql","result":"PASS"}
|
||||
|
||||
make test-telemetry-outage
|
||||
{"scenario":"telemetry_backend_outage","business_smoke":"PASS",
|
||||
"collector":"running","duration_seconds":49.5,"rss_before_mib":194.6,
|
||||
"rss_peak_mib":198.0,"memory_limit_mib":256,"failed_log_records":34,
|
||||
"failed_spans":30,"result":"PASS"}
|
||||
```
|
||||
|
||||
Worker incident использовал attachment `a88a0cfd-bbda-4151-8584-de27154a8dcb`.
|
||||
Это идентификатор только локальных синтетических данных; credentials/tokens/body в
|
||||
доказательство не сохранялись.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"implementation_commit": "9e6428d",
|
||||
"host": "Apple M4 Pro, 24 GiB RAM",
|
||||
"docker_vm": "12 CPU, 9.70 GiB RAM; per-service limits from compose.yaml",
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 120.0,
|
||||
"elapsed_seconds": 120.084,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 25638,
|
||||
"generated_messages": 25638,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 213.501,
|
||||
"successful_latency_ms": {
|
||||
"p50": 17.158,
|
||||
"p95": 99.774,
|
||||
"p99": 171.24
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Business requests per second; health routes are excluded.",
|
||||
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 0},
|
||||
"id": 1,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "sum by (http_route, http_request_method) (rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[$__rate_interval]))", "legendFormat": "{{http_request_method}} {{http_route}}", "refId": "A"}],
|
||||
"title": "API RPS",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "5xx business requests divided by all business requests over 2 minutes.",
|
||||
"fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 0},
|
||||
"id": 2,
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}},
|
||||
"targets": [{"expr": "(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\",http_response_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[2m])), 0.001)) or vector(0)", "legendFormat": "5xx ratio", "refId": "A"}],
|
||||
"title": "API server error ratio",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Percentiles are computed from aggregate histogram buckets across API replicas.",
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 0},
|
||||
"id": 3,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
],
|
||||
"title": "API duration p50 / p95 / p99",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8},
|
||||
"id": 4,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", "legendFormat": "{{outcome}}", "refId": "A"}],
|
||||
"title": "Jobs by outcome",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 8},
|
||||
"id": 5,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
],
|
||||
"title": "Worker duration p50 / p95 / p99",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Time from durable acceptance to ready commit.",
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 8},
|
||||
"id": 6,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||
],
|
||||
"title": "End-to-end time to ready",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"description": "Logical non-terminal jobs from PostgreSQL and age of the oldest one.",
|
||||
"fieldConfig": {"defaults": {}, "overrides": [{"matcher": {"id": "byName", "options": "oldest age"}, "properties": [{"id": "unit", "value": "s"}]}]},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16},
|
||||
"id": 7,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "max(messenger_queue_depth{messaging_destination_name=\"attachments.process\"})", "legendFormat": "depth", "refId": "A"},
|
||||
{"expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process\"})", "legendFormat": "oldest age", "refId": "B"}
|
||||
],
|
||||
"title": "Queue depth and oldest job",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 16},
|
||||
"id": 8,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [
|
||||
{"expr": "messenger_db_pool_used", "legendFormat": "used {{service_name}} {{service_instance_id}}", "refId": "A"},
|
||||
{"expr": "messenger_db_pool_waiting", "legendFormat": "waiting {{service_name}} {{service_instance_id}}", "refId": "B"}
|
||||
],
|
||||
"title": "PostgreSQL pool used / waiting",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 16},
|
||||
"id": 9,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "saturation", "refId": "A"}],
|
||||
"title": "Worker saturation",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
|
||||
"id": 10,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "rate(process_cpu_time_seconds_total[5m])", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}],
|
||||
"title": "Process CPU",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
|
||||
"id": 11,
|
||||
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||
"targets": [{"expr": "process_memory_rss_bytes", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}],
|
||||
"title": "Process RSS",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
|
||||
"id": 12,
|
||||
"options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true},
|
||||
"targets": [{"expr": "{service_name=~\"$service\"} | json | http_route!~\"/health/.*\" | line_format \"{{.severity}} {{.service_name}} {{.event}}{{if .http_method}} {{.http_method}} {{.http_route}} status={{.http_status_code}} duration={{.duration_seconds}}{{end}} trace_id={{.trace_id}} {{.message}}\"", "refId": "A"}],
|
||||
"title": "Correlated application logs",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "5s",
|
||||
"schemaVersion": 39,
|
||||
"tags": ["mtusi", "lab6", "opentelemetry"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {"selected": false, "text": ".*", "value": ".*"},
|
||||
"datasource": {"type": "loki", "uid": "loki"},
|
||||
"definition": "label_values(service_name)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Service",
|
||||
"multi": true,
|
||||
"name": "service",
|
||||
"options": [],
|
||||
"query": "label_values(service_name)",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {"from": "now-30m", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Messenger Observability",
|
||||
"uid": "messenger-observability",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: Messenger SLO alerts
|
||||
folder: MTUSI Lab 6
|
||||
interval: 10s
|
||||
rules:
|
||||
- uid: messenger-api-high-error-rate
|
||||
title: ApiHighServerErrorRate
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 120
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: '(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*",http_response_status_class="5xx"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*"}[2m])), 0.001)) or vector(0)'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
relativeTimeRange:
|
||||
from: 0
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params: [0.05]
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: [C]
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
refId: C
|
||||
type: threshold
|
||||
dashboardUid: messenger-observability
|
||||
panelId: 2
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 1m
|
||||
annotations:
|
||||
summary: Более 5% бизнес-запросов завершаются 5xx.
|
||||
runbook_url: /var/lib/grafana/runbooks/api-high-error-rate.md
|
||||
labels:
|
||||
service: messenger-api
|
||||
severity: warning
|
||||
isPaused: false
|
||||
- uid: messenger-queue-oldest-growing
|
||||
title: QueueOldestJobGrowing
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 60
|
||||
to: 0
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
datasource:
|
||||
type: prometheus
|
||||
uid: prometheus
|
||||
editorMode: code
|
||||
expr: '(max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process"}) > 20 and max(messenger_queue_depth{messaging_destination_name="attachments.process"}) > 0) or vector(0)'
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
relativeTimeRange:
|
||||
from: 0
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params: [0.5]
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: [C]
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
refId: C
|
||||
type: threshold
|
||||
dashboardUid: messenger-observability
|
||||
panelId: 7
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 15s
|
||||
annotations:
|
||||
summary: Возраст старейшей незавершённой job превысил 20 секунд.
|
||||
runbook_url: /var/lib/grafana/runbooks/queue-oldest-job.md
|
||||
labels:
|
||||
service: messenger-worker
|
||||
severity: warning
|
||||
isPaused: false
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Messenger
|
||||
orgId: 1
|
||||
folder: MTUSI Lab 6
|
||||
type: file
|
||||
disableDeletion: true
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,47 @@
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Prometheus
|
||||
orgId: 1
|
||||
- name: Loki
|
||||
orgId: 1
|
||||
- name: Tempo
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherRegex: 'trace[_]?id["=: ]+([0-9a-f]{32})'
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -1m
|
||||
spanEndTimeShift: 1m
|
||||
filterByTraceID: true
|
||||
filterBySpanID: false
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# ApiHighServerErrorRate
|
||||
|
||||
Alert использует только 5xx бизнес-запросов и исключает `/health/*`. Клиентские 4xx
|
||||
не считаются отказом сервера.
|
||||
|
||||
1. Открыть панели API error ratio и DB pool.
|
||||
2. В Tempo найти медленные/error server spans за окно alert.
|
||||
3. Сравнить длительность server span с дочерними PostgreSQL/S3 spans.
|
||||
4. В Loki отфильтровать `{service_name="messenger-api"} | trace_id="..."`.
|
||||
5. Проверить readiness зависимостей, не удаляя volumes и не перезапуская весь стенд.
|
||||
|
||||
После восстановления убедиться, что новые запросы успешны, error ratio падает и alert
|
||||
переходит в Normal. Короткий единичный 5xx может не сработать из-за окна и `for: 1m`.
|
||||
@@ -0,0 +1,13 @@
|
||||
# QueueOldestJobGrowing
|
||||
|
||||
Alert означает, что в PostgreSQL остаётся хотя бы одна `queued`/`processing` job старше
|
||||
20 секунд. Это возраст backlog, а не duration одного consumer span.
|
||||
|
||||
1. Проверить queue depth, oldest age и worker busy/capacity.
|
||||
2. В Loki открыть `{service_name="messenger-worker"}` и проверить retry/failure events.
|
||||
3. По trace ID перейти в Tempo и проверить consumer, S3 и PostgreSQL spans.
|
||||
4. Восстановить worker или отказавшую зависимость, не создавая новую attachment job.
|
||||
5. Убедиться, что исходная job стала `ready`, backlog уменьшился и alert resolved.
|
||||
|
||||
Порог учебный и рассчитан на локальную демонстрацию; для production его связывают с
|
||||
реальным SLO времени до ready.
|
||||
@@ -0,0 +1,31 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout check 2s
|
||||
|
||||
resolvers docker
|
||||
nameserver docker_dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api_backend
|
||||
|
||||
backend api_backend
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 2 api:8080
|
||||
@@ -0,0 +1,41 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
storage_config:
|
||||
filesystem:
|
||||
directory: /loki/chunks
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,76 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 48
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 512
|
||||
send_batch_max_size: 1024
|
||||
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:8889
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 500
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 500
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 30s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
metrics:
|
||||
level: detailed
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
@@ -0,0 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-application-metrics
|
||||
static_configs:
|
||||
- targets: [otel-collector:8889]
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::BUCKET_NAME"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::BUCKET_NAME/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,16 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn[standard]==0.34.2
|
||||
SQLAlchemy[asyncio]==2.0.40
|
||||
asyncpg==0.30.0
|
||||
alembic==1.15.2
|
||||
redis==5.2.1
|
||||
argon2-cffi==25.1.0
|
||||
minio==7.2.20
|
||||
Pillow==13.0.0
|
||||
aio-pika==9.5.5
|
||||
opentelemetry-api==1.31.1
|
||||
opentelemetry-sdk==1.31.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.31.1
|
||||
opentelemetry-instrumentation-fastapi==0.52b1
|
||||
opentelemetry-instrumentation-sqlalchemy==0.52b1
|
||||
psutil==7.0.0
|
||||
+28
-1
@@ -1,6 +1,11 @@
|
||||
# Публичные проверки
|
||||
|
||||
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
|
||||
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; публичные
|
||||
проверки не создают готовое приложение и сами не управляют Docker. Отказные проверки
|
||||
lab6 ниже управляют только явно указанными Compose-сервисами. `make check` можно
|
||||
выполнить на исходном skeleton, `make test` — только после реализации API. Полная
|
||||
OpenAPI-валидация отдельно: `make validate` после установки
|
||||
`requirements-dev.txt` в venv.
|
||||
|
||||
```sh
|
||||
make check
|
||||
@@ -13,6 +18,28 @@ python3 tests/smoke.py --base-url http://localhost:8080
|
||||
|
||||
Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются.
|
||||
|
||||
## Наблюдаемость и контролируемые отказы, lab6
|
||||
|
||||
При запущенном Compose-стенде доступны дополнительные проверки реализации:
|
||||
|
||||
```sh
|
||||
make test-observability # dashboard, alerts, Prometheus, Loki и Tempo
|
||||
make test-retry-trace # временная S3-ошибка и retry в одной трассе
|
||||
make test-alert-worker # временно останавливает и восстанавливает worker
|
||||
make test-alert-database # временно останавливает и восстанавливает PostgreSQL
|
||||
make test-telemetry-outage # временно останавливает и восстанавливает Loki/Tempo
|
||||
```
|
||||
|
||||
Отказные сценарии изменяют состояние только перечисленных контейнеров и всегда
|
||||
выполняют восстановление в `finally`. Используются уникальные синтетические данные;
|
||||
секреты и tokens не печатаются. Retry-тест временно отключает S3 и проверяет
|
||||
две producer/consumer попытки в одной трассе. Worker-тест ждёт `QueueOldestJobGrowing` в
|
||||
`Alerting`, затем `Normal`; DB-тест аналогично проверяет `ApiHighServerErrorRate`.
|
||||
Тест telemetry outage держит backends выключенными более 30 с и фиксирует RSS
|
||||
Collector и `send_failed` counters.
|
||||
Окна alerts намеренно делают эти два теста не мгновенными. Запускайте их
|
||||
последовательно, а не параллельно.
|
||||
|
||||
## Проверка сохранности, lab2+
|
||||
|
||||
Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Воспроизводит ограниченные отказы worker и PostgreSQL для проверки alerts."""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
from client import Client
|
||||
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def compose(*args: str, capture: bool = False):
|
||||
"""Запускает Docker Compose из корня проекта и при необходимости захватывает вывод."""
|
||||
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
text=True,
|
||||
capture_output=capture,
|
||||
)
|
||||
|
||||
|
||||
def grafana_rules() -> dict:
|
||||
"""Получает текущие состояния provisioned alert rules через API Grafana."""
|
||||
|
||||
user = os.getenv("GRAFANA_ADMIN_USER", "admin")
|
||||
password = os.getenv("GRAFANA_ADMIN_PASSWORD", "change-me-grafana")
|
||||
token = base64.b64encode(f"{user}:{password}".encode()).decode()
|
||||
result = compose(
|
||||
"exec", "-T", "grafana", "wget", "-qO-",
|
||||
"--header", f"Authorization: Basic {token}",
|
||||
"http://127.0.0.1:3000/api/prometheus/grafana/api/v1/rules",
|
||||
capture=True,
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def internal_json(url: str) -> dict:
|
||||
"""Читает JSON из внутреннего observability-сервиса через Grafana-контейнер."""
|
||||
|
||||
result = compose(
|
||||
"exec", "-T", "grafana", "wget", "-qO-", url, capture=True
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def database_failure_evidence(start_ns: int) -> dict[str, str]:
|
||||
"""Находит связанный лог и ошибочный PostgreSQL CLIENT span."""
|
||||
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
query = urllib.parse.urlencode({
|
||||
"query": (
|
||||
'{service_name="messenger-api"} | json '
|
||||
'| logger="app.error" | message=~".*(database|socket).*"'
|
||||
),
|
||||
"start": str(start_ns),
|
||||
"end": str(time.time_ns()),
|
||||
"direction": "backward",
|
||||
"limit": "20",
|
||||
})
|
||||
loki = internal_json(
|
||||
"http://loki:3100/loki/api/v1/query_range?" + query
|
||||
)
|
||||
streams = loki.get("data", {}).get("result", [])
|
||||
if not streams:
|
||||
time.sleep(2)
|
||||
continue
|
||||
entry = streams[0]["values"][0]
|
||||
metadata = entry[2] if len(entry) > 2 else {}
|
||||
body = json.loads(entry[1])
|
||||
trace_id = metadata.get("trace_id") or body.get("trace_id")
|
||||
if not trace_id:
|
||||
time.sleep(2)
|
||||
continue
|
||||
trace = internal_json(f"http://tempo:3200/api/traces/{trace_id}")
|
||||
for batch in trace.get("batches", []):
|
||||
scopes = batch.get(
|
||||
"scopeSpans", batch.get("instrumentationLibrarySpans", [])
|
||||
)
|
||||
for scope in scopes:
|
||||
for span in scope.get("spans", []):
|
||||
attributes = {
|
||||
item["key"]: next(iter(item.get("value", {}).values()), "")
|
||||
for item in span.get("attributes", [])
|
||||
}
|
||||
if (
|
||||
span.get("kind") == "SPAN_KIND_CLIENT"
|
||||
and attributes.get("db.system") == "postgresql"
|
||||
and span.get("status", {}).get("code") == "STATUS_CODE_ERROR"
|
||||
):
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"log_message": body.get("message", ""),
|
||||
"db_span": span.get("name", ""),
|
||||
"db_system": "postgresql",
|
||||
}
|
||||
time.sleep(2)
|
||||
raise RuntimeError("linked PostgreSQL error log/trace was not found")
|
||||
|
||||
|
||||
def rule_state(title: str) -> str:
|
||||
"""Находит alert по заголовку и возвращает его текущее состояние."""
|
||||
|
||||
groups = grafana_rules().get("data", {}).get("groups", [])
|
||||
for group in groups:
|
||||
for rule in group.get("rules", []):
|
||||
if rule.get("name") == title:
|
||||
alerts = rule.get("alerts", [])
|
||||
return alerts[0].get("state", rule.get("state", "unknown")) if alerts else rule.get("state", "unknown")
|
||||
raise RuntimeError(f"Grafana rule not found: {title}")
|
||||
|
||||
|
||||
def wait_state(title: str, expected: set[str], timeout: int) -> tuple[str, float]:
|
||||
"""Ожидает одно из требуемых состояний alert и возвращает время ожидания."""
|
||||
|
||||
started = time.monotonic()
|
||||
deadline = started + timeout
|
||||
last = "unknown"
|
||||
while time.monotonic() < deadline:
|
||||
last = rule_state(title)
|
||||
if last.lower() in expected:
|
||||
return last, time.monotonic() - started
|
||||
time.sleep(5)
|
||||
raise RuntimeError(f"{title}: expected {sorted(expected)}, last state {last}")
|
||||
|
||||
|
||||
def worker_scenario(base_url: str):
|
||||
"""Останавливает worker и проверяет переход queue alert в Alerting и Normal."""
|
||||
|
||||
client = Client(base_url, timeout=10)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
firing, detect_seconds = wait_state(
|
||||
"QueueOldestJobGrowing", {"alerting", "firing"}, 100
|
||||
)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
ready = client.wait_ready(bob, attachment, wait_seconds=90)
|
||||
normal, recover_seconds = wait_state(
|
||||
"QueueOldestJobGrowing", {"normal", "inactive"}, 90
|
||||
)
|
||||
print(json.dumps({
|
||||
"scenario": "worker_outage",
|
||||
"attachment_id": ready["id"],
|
||||
"firing_state": firing,
|
||||
"detect_seconds": round(detect_seconds, 1),
|
||||
"resolved_state": normal,
|
||||
"recover_seconds": round(recover_seconds, 1),
|
||||
"result": "PASS",
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
def database_scenario(base_url: str):
|
||||
"""Останавливает PostgreSQL и проверяет firing/resolved для alert по 5xx."""
|
||||
|
||||
Client(base_url).call("GET", "/health/ready", schema="Health")
|
||||
incident_started_ns = time.time_ns()
|
||||
compose("stop", "postgres")
|
||||
load = subprocess.Popen(
|
||||
[
|
||||
"docker", "compose", "-f", "compose.yaml", "-f",
|
||||
"compose.observability-tests.yaml", "run", "--rm",
|
||||
"observability-test", "python", "database_error_load.py",
|
||||
"--base-url", "http://api:8080", "--duration", "150",
|
||||
],
|
||||
cwd=ROOT,
|
||||
)
|
||||
evidence = None
|
||||
try:
|
||||
firing, detect_seconds = wait_state(
|
||||
"ApiHighServerErrorRate", {"alerting", "firing"}, 130
|
||||
)
|
||||
evidence = database_failure_evidence(incident_started_ns)
|
||||
finally:
|
||||
load.terminate()
|
||||
try:
|
||||
load.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
load.kill()
|
||||
compose("start", "postgres")
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
Client(base_url).call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(3)
|
||||
else:
|
||||
raise RuntimeError("API readiness did not recover after PostgreSQL restart")
|
||||
normal, recover_seconds = wait_state(
|
||||
"ApiHighServerErrorRate", {"normal", "inactive"}, 180
|
||||
)
|
||||
print(json.dumps({
|
||||
"scenario": "database_outage",
|
||||
"firing_state": firing,
|
||||
"detect_seconds": round(detect_seconds, 1),
|
||||
"resolved_state": normal,
|
||||
"recover_seconds": round(recover_seconds, 1),
|
||||
"trace_id": evidence["trace_id"],
|
||||
"log_message": evidence["log_message"],
|
||||
"db_span": evidence["db_span"],
|
||||
"db_system": evidence["db_system"],
|
||||
"result": "PASS",
|
||||
}, ensure_ascii=False))
|
||||
|
||||
|
||||
def main():
|
||||
"""Разбирает аргументы и запускает выбранный отказный сценарий."""
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("scenario", choices=("worker", "database"))
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
args = parser.parse_args()
|
||||
if args.scenario == "worker":
|
||||
worker_scenario(args.base_url)
|
||||
else:
|
||||
database_scenario(args.base_url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Создаёт ограниченный поток DB-зависимых запросов для проверки alert по 5xx."""
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
def main():
|
||||
"""Отправляет synthetic POST-запросы заданное время и считает ошибки."""
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://api:8080")
|
||||
parser.add_argument("--duration", type=int, default=120)
|
||||
args = parser.parse_args()
|
||||
deadline = time.monotonic() + args.duration
|
||||
attempts = errors = 0
|
||||
while time.monotonic() < deadline:
|
||||
suffix = uuid.uuid4().hex[:12]
|
||||
body = json.dumps({
|
||||
"username": "incident_" + suffix,
|
||||
"display_name": "Synthetic incident",
|
||||
"password": "Synthetic-test-only-" + suffix,
|
||||
}).encode()
|
||||
request = urllib.request.Request(
|
||||
args.base_url.rstrip("/") + "/api/v1/users",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
attempts += 1
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=3) as response:
|
||||
response.read()
|
||||
errors += response.status >= 500
|
||||
except urllib.error.HTTPError as error:
|
||||
error.read()
|
||||
errors += error.code >= 500
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
errors += 1
|
||||
time.sleep(0.25)
|
||||
print(f"database incident load: attempts={attempts} errors={errors}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user