forked from mtusi2026/mtusi2026lab6
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c81dab8d95 | ||
|
|
67039fd30d | ||
|
|
9e6428de05 | ||
|
|
58f3d4c457 |
+37
-14
@@ -1,30 +1,53 @@
|
|||||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
# Local synthetic lab environment only. Copy to .env and replace every change-me value.
|
||||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
|
||||||
APP_HOST=0.0.0.0
|
APP_HOST=0.0.0.0
|
||||||
APP_PORT=8080
|
APP_PORT=8080
|
||||||
APP_ENV=lab
|
APP_ENV=lab
|
||||||
SESSION_TTL_SECONDS=3600
|
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
|
||||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
|
||||||
SESSION_STORE=postgres
|
|
||||||
CACHE_URL_FILE=/run/secrets/cache_url
|
|
||||||
API_REPLICAS=2
|
API_REPLICAS=2
|
||||||
|
LOG_LEVEL=info
|
||||||
|
|
||||||
|
POSTGRES_USER=messenger
|
||||||
|
POSTGRES_PASSWORD=change-me-postgres
|
||||||
|
POSTGRES_DB=messenger
|
||||||
|
DATABASE_URL=postgresql+asyncpg://messenger:change-me-postgres@postgres:5432/messenger
|
||||||
|
REDIS_URL=redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS=3600
|
||||||
IDEMPOTENCY_TTL_SECONDS=86400
|
IDEMPOTENCY_TTL_SECONDS=86400
|
||||||
|
|
||||||
|
MINIO_ROOT_USER=local-minio-admin
|
||||||
|
MINIO_ROOT_PASSWORD=change-me-minio-root
|
||||||
S3_ENDPOINT=http://s3:9000
|
S3_ENDPOINT=http://s3:9000
|
||||||
S3_REGION=us-east-1
|
S3_REGION=us-east-1
|
||||||
S3_BUCKET=messenger-attachments
|
S3_BUCKET=messenger-attachments
|
||||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
S3_ACCESS_KEY_ID=messenger-api
|
||||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
S3_SECRET_ACCESS_KEY=change-me-minio-api
|
||||||
MAX_UPLOAD_BYTES=10485760
|
MAX_UPLOAD_BYTES=10485760
|
||||||
MAX_IMAGE_PIXELS=40000000
|
MAX_IMAGE_PIXELS=40000000
|
||||||
BROKER_URL_FILE=/run/secrets/broker_url
|
|
||||||
|
BROKER_USER=messenger
|
||||||
|
BROKER_PASSWORD=change-me-rabbitmq
|
||||||
|
BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/
|
||||||
WORKER_CONCURRENCY=2
|
WORKER_CONCURRENCY=2
|
||||||
|
WORKER_PREFETCH=2
|
||||||
JOB_MAX_ATTEMPTS=5
|
JOB_MAX_ATTEMPTS=5
|
||||||
JOB_TIMEOUT_SECONDS=60
|
JOB_TIMEOUT_SECONDS=60
|
||||||
|
JOB_LEASE_SECONDS=90
|
||||||
|
RETRY_BASE_SECONDS=2
|
||||||
|
RETRY_MAX_SECONDS=60
|
||||||
STAGING_RETENTION_SECONDS=86400
|
STAGING_RETENTION_SECONDS=86400
|
||||||
OTEL_SERVICE_NAME=messenger-api
|
THUMBNAIL_JPEG_QUALITY=85
|
||||||
# Worker must set a distinct service.name.
|
CROP_JPEG_QUALITY=85
|
||||||
|
OPTIMIZED_WEBP_QUALITY=82
|
||||||
|
|
||||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
OTEL_SERVICE_VERSION=1.6.0
|
||||||
|
OTEL_DEPLOYMENT_ENVIRONMENT=lab
|
||||||
|
OTEL_SDK_DISABLED=false
|
||||||
|
OTEL_EXPORTER_OTLP_TIMEOUT=2
|
||||||
|
OTEL_BSP_MAX_QUEUE_SIZE=2048
|
||||||
|
OTEL_BLRP_MAX_QUEUE_SIZE=2048
|
||||||
|
OTEL_METRIC_EXPORT_INTERVAL=5000
|
||||||
|
|
||||||
|
GRAFANA_PORT=3000
|
||||||
|
GRAFANA_ADMIN_USER=admin
|
||||||
|
GRAFANA_ADMIN_PASSWORD=change-me-grafana
|
||||||
|
|||||||
+17
@@ -0,0 +1,17 @@
|
|||||||
|
FROM python:3.13.15-slim
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN python -m pip install -r requirements.txt \
|
||||||
|
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app
|
||||||
|
COPY app ./app
|
||||||
|
COPY alembic ./alembic
|
||||||
|
COPY alembic.ini .
|
||||||
|
|
||||||
|
USER 10001:10001
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||||
|
CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"]
|
||||||
|
|
||||||
|
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"]
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
FROM python:3.13.15-slim
|
||||||
|
|
||||||
|
WORKDIR /tests
|
||||||
|
COPY requirements.txt requirements-dev.txt ./
|
||||||
|
RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \
|
||||||
|
&& useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests
|
||||||
|
|
||||||
|
COPY lab.json .
|
||||||
|
COPY app ./app
|
||||||
|
COPY contracts ./contracts
|
||||||
|
COPY tests ./tests
|
||||||
|
RUN chown -R tests:tests /tests
|
||||||
|
|
||||||
|
USER 10001:10001
|
||||||
|
CMD ["python", "-m", "pytest", "-q"]
|
||||||
@@ -3,11 +3,12 @@ BASE_URL ?= http://localhost:8080
|
|||||||
CA_FILE ?=
|
CA_FILE ?=
|
||||||
ACTION_SCRIPT ?=
|
ACTION_SCRIPT ?=
|
||||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||||
|
OBSERVABILITY_COMPOSE = docker compose -f compose.yaml -f compose.observability-tests.yaml
|
||||||
|
|
||||||
.PHONY: help check validate test load test-persistence test-failover
|
.PHONY: help check validate test test-observability test-retry-trace test-alert-worker test-alert-database test-telemetry-outage load test-persistence test-failover
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
@echo "check: offline structure; validate: full OpenAPI; test: public API; test-observability: Grafana/Prometheus/Loki/Tempo"
|
||||||
|
|
||||||
check:
|
check:
|
||||||
$(PYTHON) scripts/check_contract.py
|
$(PYTHON) scripts/check_contract.py
|
||||||
@@ -18,6 +19,21 @@ validate:
|
|||||||
test:
|
test:
|
||||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
test-observability:
|
||||||
|
$(OBSERVABILITY_COMPOSE) run --rm observability-test
|
||||||
|
|
||||||
|
test-retry-trace:
|
||||||
|
$(PYTHON) tests/retry_trace.py --base-url "$(BASE_URL)"
|
||||||
|
|
||||||
|
test-alert-worker:
|
||||||
|
$(PYTHON) tests/alert_scenarios.py worker --base-url "$(BASE_URL)"
|
||||||
|
|
||||||
|
test-alert-database:
|
||||||
|
$(PYTHON) tests/alert_scenarios.py database --base-url "$(BASE_URL)"
|
||||||
|
|
||||||
|
test-telemetry-outage:
|
||||||
|
$(PYTHON) tests/telemetry_outage.py --base-url "$(BASE_URL)"
|
||||||
|
|
||||||
load:
|
load:
|
||||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||||
|
|
||||||
|
|||||||
@@ -1,37 +1,207 @@
|
|||||||
# Отчёт — лабораторная 6
|
# Отчёт — лабораторная 6
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор, группа: Сергей Возжин, БПИ2301.
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: **4**. Трек оценки 5 не заявлен.
|
||||||
- Commit/PR: TODO после публикации
|
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/61
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- Использованы Python 3.13, FastAPI, OpenTelemetry Python 1.31.1,
|
||||||
|
OTel Collector Contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2,
|
||||||
|
Tempo 2.7.2 и Grafana 11.6.0. Для планирования и реализации применялся ИИ.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Нужны Docker с Compose v2, GNU Make и Python 3.9+ для host-side тестов.
|
||||||
|
Все образы и Python-зависимости закреплены в репозитории.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
# Заменить все change-me значения согласованными локальными секретами.
|
||||||
|
# Пароли в DATABASE_URL/BROKER_URL должны совпадать с POSTGRES_*/BROKER_*.
|
||||||
|
docker compose config --quiet
|
||||||
|
docker compose up -d --build --wait
|
||||||
|
make check
|
||||||
|
make test BASE_URL=http://localhost:8080
|
||||||
|
make test-observability
|
||||||
|
```
|
||||||
|
|
||||||
|
Grafana доступна только локально на `http://127.0.0.1:3000`. Dashboard,
|
||||||
|
datasources и alerts создаются provisioning-механизмом автоматически. Миграции
|
||||||
|
Alembic выполняет одноразовый сервис `migrate`; MinIO bucket и технического
|
||||||
|
пользователя создаёт `s3-bootstrap`.
|
||||||
|
|
||||||
|
Безопасная остановка без удаления данных:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose down
|
||||||
|
```
|
||||||
|
|
||||||
|
Намеренное удаление всех **локальных учебных** данных выполняется отдельно и
|
||||||
|
необратимо:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose down --volumes
|
||||||
|
```
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
```text
|
||||||
|
client -> HAProxy -> API x2 -> PostgreSQL / Redis / S3
|
||||||
|
|
|
||||||
|
+-> transactional outbox -> publisher -> RabbitMQ -> worker -> S3 + PostgreSQL
|
||||||
|
|
||||||
|
API / publisher / worker -> OTLP HTTP -> OTel Collector
|
||||||
|
|-> Prometheus exporter <- Prometheus
|
||||||
|
|-> OTLP logs -> Loki
|
||||||
|
`-> OTLP traces -> Tempo
|
||||||
|
Prometheus + Loki + Tempo <- provisioned datasources <- Grafana
|
||||||
|
```
|
||||||
|
|
||||||
|
Из ЛР 5 без изменения бизнес-API перенесены две API-реплики, HAProxy,
|
||||||
|
PostgreSQL, Redis sessions/idempotency, MinIO S3, RabbitMQ, transactional
|
||||||
|
outbox, publisher, worker и обработка изображений. PostgreSQL остаётся источником
|
||||||
|
истины для attachments/jobs/outbox. Durable acceptance атомарно создаёт attachment,
|
||||||
|
job и outbox row; публикация и обработка повторяемы.
|
||||||
|
|
||||||
|
В outbox добавлены nullable `traceparent` и `tracestate`. API сохраняет W3C context
|
||||||
|
в той же транзакции, publisher восстанавливает его, создаёт PRODUCER span и передаёт
|
||||||
|
новый context в allowlisted AMQP headers. Worker извлекает headers и создаёт CONSUMER
|
||||||
|
span. В одной трассе видны `messenger-api`, `messenger-publisher`,
|
||||||
|
`messenger-worker`, SQL и ручные S3 CLIENT spans. Retry создаёт новую публикацию с
|
||||||
|
текущим context; повторная обработка остаётся наблюдаемой и идемпотентной.
|
||||||
|
|
||||||
|
SDK использует bounded batch queues по 2048 элементов и timeout 2 с. Collector:
|
||||||
|
`memory_limiter` 192 MiB со spike 48 MiB, `batch` до 1024 элементов, exporter queues
|
||||||
|
по 500 элементов, два consumer, exponential retry не дольше 30 с. Поэтому отказ
|
||||||
|
Loki/Tempo приводит к ограниченной потере telemetry, а не блокировке API или
|
||||||
|
неограниченному росту RAM.
|
||||||
|
Collector self-metrics экспортируются через `readers.pull` на `:8888`;
|
||||||
|
Prometheus сохраняет RSS, размеры очередей и `send_failed` counters.
|
||||||
|
|
||||||
|
Секреты и пользовательское содержимое не экспортируются. `safe_extra` отбрасывает
|
||||||
|
пароли, tokens, Authorization, secrets, message body, image bytes и signed URL.
|
||||||
|
Метрики содержат только низкокардинальные route templates, method/status class,
|
||||||
|
job type/outcome и queue name; UUID и filename labels отсутствуют. Технические IDs
|
||||||
|
допустимы только в logs/spans для синтетических данных.
|
||||||
|
|
||||||
|
## Сигналы, dashboard и единицы
|
||||||
|
|
||||||
|
| Контракт | Реальная серия | Единица / основные labels |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| HTTP requests | `http_server_request_count_total` | requests; method, route template, status class |
|
||||||
|
| HTTP duration | `http_server_request_duration_seconds` | seconds, histogram |
|
||||||
|
| Jobs outcome | `messenger_jobs_processed_total` | jobs; job type, outcome |
|
||||||
|
| Job duration | `messenger_job_processing_duration_seconds` | seconds, histogram |
|
||||||
|
| Time to ready | `messenger_attachment_time_to_ready_seconds` | seconds, histogram |
|
||||||
|
| Queue | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | jobs / seconds |
|
||||||
|
| DB pool | `messenger_db_pool_used`, `messenger_db_pool_waiting` | connections / waiters |
|
||||||
|
| Worker | `messenger_worker_busy`, `messenger_worker_capacity` | slots |
|
||||||
|
| Process | `process_cpu_time_seconds_total`, `process_memory_rss_bytes` | CPU seconds / bytes |
|
||||||
|
|
||||||
|
Dashboard UID `messenger-observability` содержит 12 panels: RPS, 5xx ratio,
|
||||||
|
HTTP p50/p95/p99, jobs по outcome, worker p50/p95/p99, end-to-end p50/p95/p99,
|
||||||
|
queue depth/oldest, DB pool, worker saturation, CPU, RSS и correlated logs.
|
||||||
|
Percentiles вычисляются из объединённых buckets всех реплик, например:
|
||||||
|
|
||||||
|
```promql
|
||||||
|
histogram_quantile(0.99,
|
||||||
|
sum by (le) (rate(http_server_request_duration_seconds_bucket{
|
||||||
|
service_name="messenger-api",http_route!~"/health/.*"}[5m])))
|
||||||
|
```
|
||||||
|
|
||||||
|
Границы HTTP histogram: 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms,
|
||||||
|
1 s, 2.5 s, 5 s. Worker: 50 ms…60 s. Time-to-ready: 100 ms…120 s.
|
||||||
|
Средние значения и средние p99 отдельных реплик не используются.
|
||||||
|
|
||||||
|
## SLI, SLO и alerts
|
||||||
|
|
||||||
|
API availability SLI: доля бизнес-запросов со status `<500` среди всех
|
||||||
|
бизнес-запросов; `/health/live` и `/health/ready` исключены. Учебный SLO — не менее
|
||||||
|
99% good requests в rolling window 30 минут. 4xx входят в denominator, но не в
|
||||||
|
server-error numerator.
|
||||||
|
|
||||||
|
Pipeline SLI: доля durably accepted attachments, достигших `ready` не позднее
|
||||||
|
60 секунд. Учебный SLO — не менее 95% за rolling 30 минут при доступном worker.
|
||||||
|
|
||||||
|
Provisioned rules:
|
||||||
|
|
||||||
|
- `ApiHighServerErrorRate`: 5xx / all business requests за 2 минуты > 5%, `for: 1m`;
|
||||||
|
- `QueueOldestJobGrowing`: oldest job > 20 s при depth > 0, `for: 15s`.
|
||||||
|
|
||||||
|
Пороги и короткие окна выбраны для воспроизводимой учебной демонстрации, а не как
|
||||||
|
production defaults. У каждого правила есть dashboard/panel link и локальный runbook
|
||||||
|
в `infra/grafana/runbooks/`.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
| Offline contract | `make check` | PASS | консоль и Git diff |
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
| Публичный API с telemetry | `make test` | `PASS: lab6 public API scenario` | `tests/smoke.py` |
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
| Dashboard/alerts/3 backends | `make test-observability` | PASS: dashboard, alerts, metrics, logs, cross-process trace | `tests/observability_smoke.py` |
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
| API → publisher → worker | тот же smoke после upload | один `trace_id`, services API/publisher/worker | Loki + Tempo API assertion |
|
||||||
|
| Retry в одной трассе | `make test-retry-trace` | 2 PRODUCER + 2 CONSUMER, SQL/S3, attachment ready | `tests/retry_trace.py` |
|
||||||
|
| Worker outage | `make test-alert-worker` | Alerting 61.8 s, Normal 15.5 s после запуска | `tests/alert_scenarios.py` |
|
||||||
|
| PostgreSQL outage | `make test-alert-database` | Alerting/Normal + linked log/failed PostgreSQL span | `tests/alert_scenarios.py` |
|
||||||
|
| Telemetry outage | `make test-telemetry-outage` | 49.5 s, smoke PASS, bounded RSS, loss counters | `tests/telemetry_outage.py` |
|
||||||
|
| Конфиги | `docker compose config --quiet`; JSON parse; Grafana logs | provisioning завершён без error | `infra/` и `compose.yaml` |
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
Повторная проверка выполнена 2026-11-20: host Apple M4 Pro, 24 GiB RAM;
|
||||||
|
Docker Desktop VM — 12 CPU и 9.70 GiB RAM. Каждый Compose-сервис дополнительно
|
||||||
|
ограничен `cpus`/`mem_limit` из `compose.yaml`. Load evidence создал
|
||||||
|
25 638 синтетических сообщений за 120 с с concurrency 8.
|
||||||
|
|
||||||
## Эксперименты и отказы
|
## Разбор контролируемых инцидентов
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
### Временная ошибка S3 и retry
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
Worker был остановлен до upload, затем S3 выключен только на первую
|
||||||
| --- | --- | --- | --- |
|
попытку. Worker сохранил `job.retry_scheduled`, после восстановления S3
|
||||||
| TODO | TODO | TODO | TODO |
|
то же attachment дошло до `ready`. Трасса
|
||||||
|
`e8e2aff13b7bdb632c9cec5007dcc6d4` содержит два PRODUCER и два CONSUMER
|
||||||
|
span, а также SQL/S3 CLIENT spans. Это доказывает перенос context
|
||||||
|
через retry outbox event.
|
||||||
|
|
||||||
|
### Worker недоступен
|
||||||
|
|
||||||
|
Гипотеза: API продолжит durable acceptance, RabbitMQ и PostgreSQL сохранят работу,
|
||||||
|
но `queue_depth` и `queue_oldest_age_seconds` вырастут. Сценарий создал отдельные
|
||||||
|
синтетические users/chat, остановил только worker, загрузил PNG и опрашивал состояние
|
||||||
|
Grafana rule. Alert перешёл в `Alerting` за 61.8 с. После `docker compose start worker`
|
||||||
|
attachment стал ready, backlog уменьшился, rule стал `Normal` через 15.5 с. Потери
|
||||||
|
бизнес-данных: 0. Диагноз по runbook: HTTP остаётся доступным, jobs не завершаются,
|
||||||
|
worker saturation отсутствует, oldest age растёт — причина на consumer-side.
|
||||||
|
|
||||||
|
### PostgreSQL недоступен
|
||||||
|
|
||||||
|
Гипотеза: прямые business requests к API дадут 5xx, DB spans/логи укажут connection
|
||||||
|
failure, а queue-only alert не обязан срабатывать. PostgreSQL остановлен, отдельный
|
||||||
|
контейнер создавал synthetic `POST /api/v1/users` каждые 250 ms. Error-rate alert
|
||||||
|
стал `Alerting` за 72.2 с. PostgreSQL запущен в `finally`, readiness восстановилась,
|
||||||
|
rule вернулся в `Normal` через 113.1 с — старые 5xx должны были выйти из 2-минутного
|
||||||
|
окна. Потери подтверждённых данных: 0; неуспешные запросы клиент может повторить.
|
||||||
|
Связанный log и trace `34f96d6fc934f7c54075c44a7b39c176` содержат
|
||||||
|
socket connection error и `connect` CLIENT span с `db.system=postgresql` и
|
||||||
|
`STATUS_CODE_ERROR`. Это отделяет отказ БД от просто медленного HTTP handler.
|
||||||
|
|
||||||
|
### Loki и Tempo недоступны
|
||||||
|
|
||||||
|
Оба backend остановлены одновременно на 49.5 с, что дольше настроенного
|
||||||
|
30-секундного retry. Полный бизнес-smoke прошёл, Collector остался running.
|
||||||
|
RSS Collector изменился с 194.6 до пиковых 198.0 MiB при container limit
|
||||||
|
256 MiB. Internal counters зафиксировали потерю 34 log records и 30 spans.
|
||||||
|
Порог `memory_limiter=192 MiB` — мягкий порог отказа/сброса данных, а не
|
||||||
|
обещание RSS ниже 192 MiB; жёсткий предел задан Compose-лимитом 256 MiB.
|
||||||
|
Это ожидаемая bounded telemetry loss: наблюдаемость не является частью
|
||||||
|
business commit.
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения и следующие шаги
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
- Стенд single-node: PostgreSQL, RabbitMQ, MinIO и telemetry backends не имеют HA.
|
||||||
|
- Sampling всегда 100%; исследование стоимости и tail sampling относится к оценке 5.
|
||||||
|
- Grafana notification delivery не настроена: по заданию достаточно локальных
|
||||||
|
firing/resolved states.
|
||||||
|
- Queue depth берётся из durable job table, поэтому отражает бизнес-backlog, а не
|
||||||
|
точное число RabbitMQ messages в каждую микросекунду.
|
||||||
|
- Collector Prometheus exporter хранит только текущее состояние; долговременную
|
||||||
|
историю сохраняет Prometheus volume. Loki и Tempo используют локальные volumes.
|
||||||
|
- Для production нужны TLS/mTLS OTLP, secrets manager, HA backends, remote storage,
|
||||||
|
более длинные SLO windows и проверенные notification routes.
|
||||||
|
|||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
[alembic]
|
||||||
|
script_location = alembic
|
||||||
|
prepend_sys_path = .
|
||||||
|
sqlalchemy.url = placeholder://configured-at-runtime
|
||||||
|
|
||||||
|
[loggers]
|
||||||
|
keys = root,sqlalchemy,alembic
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = console
|
||||||
|
|
||||||
|
[formatters]
|
||||||
|
keys = generic
|
||||||
|
|
||||||
|
[logger_root]
|
||||||
|
level = WARN
|
||||||
|
handlers = console
|
||||||
|
|
||||||
|
[logger_sqlalchemy]
|
||||||
|
level = WARN
|
||||||
|
handlers =
|
||||||
|
qualname = sqlalchemy.engine
|
||||||
|
|
||||||
|
[logger_alembic]
|
||||||
|
level = INFO
|
||||||
|
handlers =
|
||||||
|
qualname = alembic
|
||||||
|
|
||||||
|
[handler_console]
|
||||||
|
class = StreamHandler
|
||||||
|
args = (sys.stderr,)
|
||||||
|
level = NOTSET
|
||||||
|
formatter = generic
|
||||||
|
|
||||||
|
[formatter_generic]
|
||||||
|
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from logging.config import fileConfig
|
||||||
|
|
||||||
|
from alembic import context
|
||||||
|
from sqlalchemy import pool
|
||||||
|
from sqlalchemy.ext.asyncio import async_engine_from_config
|
||||||
|
|
||||||
|
from app.config import load_settings
|
||||||
|
from app.db.postgres import Base
|
||||||
|
from app import models # noqa: F401 - imports ORM models into Base.metadata
|
||||||
|
|
||||||
|
config = context.config
|
||||||
|
if config.config_file_name is not None:
|
||||||
|
fileConfig(config.config_file_name)
|
||||||
|
|
||||||
|
target_metadata = Base.metadata
|
||||||
|
|
||||||
|
|
||||||
|
def _url() -> str:
|
||||||
|
return load_settings().database_url
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_offline() -> None:
|
||||||
|
context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'})
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_migrations(connection) -> None:
|
||||||
|
context.configure(connection=connection, target_metadata=target_metadata)
|
||||||
|
with context.begin_transaction():
|
||||||
|
context.run_migrations()
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_migrations_async() -> None:
|
||||||
|
configuration = config.get_section(config.config_ini_section, {})
|
||||||
|
configuration['sqlalchemy.url'] = _url()
|
||||||
|
connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool)
|
||||||
|
async with connectable.connect() as connection:
|
||||||
|
await connection.run_sync(_run_migrations)
|
||||||
|
await connectable.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def run_migrations_online() -> None:
|
||||||
|
asyncio.run(_run_migrations_async())
|
||||||
|
|
||||||
|
|
||||||
|
if context.is_offline_mode():
|
||||||
|
run_migrations_offline()
|
||||||
|
else:
|
||||||
|
run_migrations_online()
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision | comma,n}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
branch_labels = ${repr(branch_labels)}
|
||||||
|
depends_on = ${repr(depends_on)}
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
${downgrades if downgrades else "pass"}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""Initial PostgreSQL schema for lab 2.
|
||||||
|
|
||||||
|
Revision ID: 20261019_0001
|
||||||
|
Revises:
|
||||||
|
Create Date: 2026-10-19
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = '20261019_0001'
|
||||||
|
down_revision = None
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
'users',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('username', sa.String(length=32), nullable=False),
|
||||||
|
sa.Column('display_name', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('password_hash', sa.Text(), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
sa.UniqueConstraint('username'),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
'chats',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('title', sa.String(length=100), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
'chat_members',
|
||||||
|
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('user_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('chat_id', 'user_id'),
|
||||||
|
)
|
||||||
|
op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id'])
|
||||||
|
op.create_table(
|
||||||
|
'messages',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('sender_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('text', sa.String(length=4000), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['sender_id'], ['users.id']),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
)
|
||||||
|
op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id'])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index('messages_history_idx', table_name='messages')
|
||||||
|
op.drop_table('messages')
|
||||||
|
op.drop_index('chat_members_user_chat_idx', table_name='chat_members')
|
||||||
|
op.drop_table('chat_members')
|
||||||
|
op.drop_table('chats')
|
||||||
|
op.drop_table('users')
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""Add shared message idempotency for lab 3.
|
||||||
|
|
||||||
|
Revision ID: 20261023_0002
|
||||||
|
Revises: 20261019_0001
|
||||||
|
Create Date: 2026-10-23
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = '20261023_0002'
|
||||||
|
down_revision = '20261019_0001'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
'messages',
|
||||||
|
sa.Column('idempotency_key', sa.String(length=128), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_unique_constraint(
|
||||||
|
'messages_sender_chat_idempotency_key',
|
||||||
|
'messages',
|
||||||
|
['sender_id', 'chat_id', 'idempotency_key'],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint(
|
||||||
|
'messages_sender_chat_idempotency_key',
|
||||||
|
'messages',
|
||||||
|
type_='unique',
|
||||||
|
)
|
||||||
|
op.drop_column('messages', 'idempotency_key')
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Add S3 attachment metadata and message links for lab 4.
|
||||||
|
|
||||||
|
Revision ID: 20261030_0003
|
||||||
|
Revises: 20261023_0002
|
||||||
|
Create Date: 2026-10-30
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = '20261030_0003'
|
||||||
|
down_revision = '20261023_0002'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
'attachments',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('owner_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('filename', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('object_key', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('content_type', sa.String(length=32), nullable=False),
|
||||||
|
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||||
|
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('width', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('height', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('status', sa.String(length=16), nullable=False),
|
||||||
|
sa.Column('error_code', sa.String(length=64), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['owner_id'], ['users.id']),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
sa.UniqueConstraint('object_key', name='attachments_object_key_key'),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
'attachments_chat_created_idx',
|
||||||
|
'attachments',
|
||||||
|
['chat_id', 'created_at', 'id'],
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
'message_attachments',
|
||||||
|
sa.Column('message_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('position', sa.Integer(), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||||
|
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('message_id', 'attachment_id'),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
'message_id',
|
||||||
|
'position',
|
||||||
|
name='message_attachments_message_position_key',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
'message_attachments_attachment_idx',
|
||||||
|
'message_attachments',
|
||||||
|
['attachment_id'],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index('message_attachments_attachment_idx', table_name='message_attachments')
|
||||||
|
op.drop_table('message_attachments')
|
||||||
|
op.drop_index('attachments_chat_created_idx', table_name='attachments')
|
||||||
|
op.drop_table('attachments')
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
"""Add asynchronous attachment jobs, variants and transactional outbox.
|
||||||
|
|
||||||
|
Revision ID: 20261116_0004
|
||||||
|
Revises: 20261030_0003
|
||||||
|
Create Date: 2026-11-16
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = '20261116_0004'
|
||||||
|
down_revision = '20261030_0003'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.alter_column('attachments', 'object_key', new_column_name='original_key')
|
||||||
|
op.alter_column('attachments', 'original_key', nullable=True)
|
||||||
|
op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True))
|
||||||
|
op.add_column(
|
||||||
|
'attachments',
|
||||||
|
sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'),
|
||||||
|
)
|
||||||
|
op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key'])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
'attachment_variants',
|
||||||
|
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('pipeline_version', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('name', sa.String(length=16), nullable=False),
|
||||||
|
sa.Column('object_key', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('content_type', sa.String(length=32), nullable=False),
|
||||||
|
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||||
|
sa.Column('width', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('height', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'),
|
||||||
|
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'),
|
||||||
|
sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'),
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
'attachment_jobs',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('attachment_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('pipeline_version', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('state', sa.String(length=16), nullable=False),
|
||||||
|
sa.Column('attempt', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('max_attempts', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('lease_owner', sa.String(length=128), nullable=True),
|
||||||
|
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('last_error_code', sa.String(length=64), nullable=True),
|
||||||
|
sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"state IN ('queued', 'processing', 'ready', 'failed')",
|
||||||
|
name='attachment_job_state_check',
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
'attachment_jobs_lease_idx',
|
||||||
|
'attachment_jobs',
|
||||||
|
['state', 'lease_until'],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
'outbox_events',
|
||||||
|
sa.Column('id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('event_type', sa.String(length=64), nullable=False),
|
||||||
|
sa.Column('job_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('payload', sa.JSON(), nullable=False),
|
||||||
|
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('claim_owner', sa.String(length=128), nullable=True),
|
||||||
|
sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'),
|
||||||
|
sa.Column('last_error_code', sa.String(length=64), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
'outbox_events_pending_idx',
|
||||||
|
'outbox_events',
|
||||||
|
['published_at', 'available_at'],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
'job_events',
|
||||||
|
sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column('job_id', sa.Uuid(), nullable=False),
|
||||||
|
sa.Column('kind', sa.String(length=32), nullable=False),
|
||||||
|
sa.Column('attempt', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('error_code', sa.String(length=64), nullable=True),
|
||||||
|
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
)
|
||||||
|
op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at'])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index('job_events_job_created_idx', table_name='job_events')
|
||||||
|
op.drop_table('job_events')
|
||||||
|
op.drop_index('outbox_events_pending_idx', table_name='outbox_events')
|
||||||
|
op.drop_table('outbox_events')
|
||||||
|
op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs')
|
||||||
|
op.drop_table('attachment_jobs')
|
||||||
|
op.drop_table('attachment_variants')
|
||||||
|
op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique')
|
||||||
|
op.drop_column('attachments', 'pipeline_version')
|
||||||
|
op.drop_column('attachments', 'staging_key')
|
||||||
|
op.alter_column('attachments', 'original_key', nullable=False)
|
||||||
|
op.alter_column('attachments', 'original_key', new_column_name='object_key')
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""Persist W3C trace context with outbox events.
|
||||||
|
|
||||||
|
Revision ID: 20261117_0005
|
||||||
|
Revises: 20261116_0004
|
||||||
|
Create Date: 2026-11-17
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = '20261117_0005'
|
||||||
|
down_revision = '20261116_0004'
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column('outbox_events', sa.Column('traceparent', sa.String(length=55), nullable=True))
|
||||||
|
op.add_column('outbox_events', sa.Column('tracestate', sa.String(length=512), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column('outbox_events', 'tracestate')
|
||||||
|
op.drop_column('outbox_events', 'traceparent')
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Мессенджер на PostgreSQL и Redis, лабораторная 2."""
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры."""
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from ..domain.errors import Invalid
|
||||||
|
from ..domain.models import Keyset
|
||||||
|
from .timefmt import from_rfc3339, to_rfc3339
|
||||||
|
|
||||||
|
MAX_LENGTH = 512
|
||||||
|
|
||||||
|
|
||||||
|
def encode(scope: str, position: Keyset) -> str:
|
||||||
|
payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)}
|
||||||
|
raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
|
||||||
|
return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=')
|
||||||
|
|
||||||
|
|
||||||
|
def decode(scope: str, value: str) -> Keyset:
|
||||||
|
if not value or len(value) > MAX_LENGTH:
|
||||||
|
raise Invalid('cursor недействителен для этого запроса')
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
|
||||||
|
payload = json.loads(raw)
|
||||||
|
if not isinstance(payload, dict) or payload.get('s') != scope:
|
||||||
|
raise ValueError('scope mismatch')
|
||||||
|
return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i']))
|
||||||
|
except Exception:
|
||||||
|
raise Invalid('cursor недействителен для этого запроса') from None
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from fastapi import Depends, HTTPException, Request, Security, status
|
||||||
|
|
||||||
|
from ..db.redis import RedisSessionStore
|
||||||
|
from ..domain.errors import Unauthorized
|
||||||
|
from ..domain.models import User
|
||||||
|
from ..services.chats import ChatService
|
||||||
|
from ..services.attachments import AttachmentService
|
||||||
|
from ..services.messages import MessageService
|
||||||
|
from ..services.users import UserService
|
||||||
|
from .security import bearer_auth
|
||||||
|
|
||||||
|
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Services:
|
||||||
|
attachments: AttachmentService
|
||||||
|
users: UserService
|
||||||
|
chats: ChatService
|
||||||
|
messages: MessageService
|
||||||
|
sessions: RedisSessionStore
|
||||||
|
|
||||||
|
|
||||||
|
def services(request: Request) -> Services:
|
||||||
|
return request.app.state.services
|
||||||
|
|
||||||
|
|
||||||
|
def user_service(container: Services = Depends(services)) -> UserService:
|
||||||
|
return container.users
|
||||||
|
|
||||||
|
|
||||||
|
def chat_service(container: Services = Depends(services)) -> ChatService:
|
||||||
|
return container.chats
|
||||||
|
|
||||||
|
|
||||||
|
def message_service(container: Services = Depends(services)) -> MessageService:
|
||||||
|
return container.messages
|
||||||
|
|
||||||
|
|
||||||
|
def attachment_service(container: Services = Depends(services)) -> AttachmentService:
|
||||||
|
return container.attachments
|
||||||
|
|
||||||
|
|
||||||
|
def session_store(container: Services = Depends(services)) -> RedisSessionStore:
|
||||||
|
return container.sessions
|
||||||
|
|
||||||
|
|
||||||
|
def _bearer_error() -> HTTPException:
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
detail='требуется действующая Bearer-сессия',
|
||||||
|
headers={'WWW-Authenticate': _BEARER_CHALLENGE},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def bearer_token(authorization: str | None = Security(bearer_auth)) -> str:
|
||||||
|
if not authorization:
|
||||||
|
raise _bearer_error()
|
||||||
|
scheme, separator, token = authorization.partition(' ')
|
||||||
|
if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token:
|
||||||
|
raise _bearer_error()
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
async def current_user(
|
||||||
|
token: str = Depends(bearer_token),
|
||||||
|
container: Services = Depends(services),
|
||||||
|
) -> User:
|
||||||
|
user_id = await container.sessions.get_user_id(token)
|
||||||
|
if user_id is None:
|
||||||
|
raise _bearer_error()
|
||||||
|
try:
|
||||||
|
return await container.users.identify(user_id)
|
||||||
|
except Unauthorized:
|
||||||
|
raise _bearer_error() from None
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||||
|
|
||||||
|
from ..domain.errors import (
|
||||||
|
Conflict,
|
||||||
|
DomainError,
|
||||||
|
Invalid,
|
||||||
|
NotFound,
|
||||||
|
PayloadTooLarge,
|
||||||
|
Unauthorized,
|
||||||
|
Unavailable,
|
||||||
|
UnsupportedMediaType,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger('app.error')
|
||||||
|
|
||||||
|
CODE_BY_STATUS = {
|
||||||
|
400: 'invalid_request',
|
||||||
|
401: 'unauthorized',
|
||||||
|
404: 'not_found',
|
||||||
|
409: 'conflict',
|
||||||
|
413: 'payload_too_large',
|
||||||
|
415: 'unsupported_media_type',
|
||||||
|
429: 'rate_limited',
|
||||||
|
500: 'internal_error',
|
||||||
|
503: 'unavailable',
|
||||||
|
}
|
||||||
|
|
||||||
|
STATUS_BY_DOMAIN = {
|
||||||
|
Invalid: 400,
|
||||||
|
Unauthorized: 401,
|
||||||
|
NotFound: 404,
|
||||||
|
Conflict: 409,
|
||||||
|
PayloadTooLarge: 413,
|
||||||
|
UnsupportedMediaType: 415,
|
||||||
|
Unavailable: 503,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def code_for(status: int) -> str:
|
||||||
|
if status in CODE_BY_STATUS:
|
||||||
|
return CODE_BY_STATUS[status]
|
||||||
|
return 'invalid_request' if status < 500 else 'internal_error'
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse:
|
||||||
|
return JSONResponse(
|
||||||
|
status_code=status,
|
||||||
|
content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _domain_status(error: DomainError) -> int:
|
||||||
|
for kind, status in STATUS_BY_DOMAIN.items():
|
||||||
|
if isinstance(error, kind):
|
||||||
|
return status
|
||||||
|
return 500
|
||||||
|
|
||||||
|
|
||||||
|
def _describe(error: RequestValidationError) -> str:
|
||||||
|
"""Краткое указание на пол, без stack trace и внутреннего устройства"""
|
||||||
|
for item in error.errors():
|
||||||
|
if item.get('type') == 'json_invalid':
|
||||||
|
return 'тело запроса не является корректным JSON'
|
||||||
|
location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body')
|
||||||
|
return f'некорректное поле {location}' if location else 'некорректный запрос'
|
||||||
|
return 'некорректный запрос'
|
||||||
|
|
||||||
|
|
||||||
|
def request_id(request) -> str:
|
||||||
|
return getattr(request.state, 'request_id', 'unknown')
|
||||||
|
|
||||||
|
|
||||||
|
def install(app: FastAPI) -> None:
|
||||||
|
@app.exception_handler(DomainError)
|
||||||
|
async def _on_domain(request, error: DomainError) -> JSONResponse:
|
||||||
|
return error_response(_domain_status(error), error.message, request_id(request))
|
||||||
|
|
||||||
|
@app.exception_handler(RequestValidationError)
|
||||||
|
async def _on_validation(request, error: RequestValidationError) -> JSONResponse:
|
||||||
|
return error_response(400, _describe(error), request_id(request))
|
||||||
|
|
||||||
|
@app.exception_handler(SQLAlchemyError)
|
||||||
|
async def _on_database(request, error: SQLAlchemyError) -> JSONResponse:
|
||||||
|
logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__)
|
||||||
|
return error_response(503, 'база данных недоступна', request_id(request))
|
||||||
|
|
||||||
|
@app.exception_handler(OSError)
|
||||||
|
async def _on_storage_socket(request, error: OSError) -> JSONResponse:
|
||||||
|
logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__)
|
||||||
|
return error_response(503, 'обязательное хранилище недоступно', request_id(request))
|
||||||
|
|
||||||
|
@app.exception_handler(StarletteHTTPException)
|
||||||
|
async def _on_http(request, error: StarletteHTTPException) -> JSONResponse:
|
||||||
|
message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса'
|
||||||
|
response = error_response(error.status_code, message, request_id(request))
|
||||||
|
for name, value in (error.headers or {}).items():
|
||||||
|
response.headers[name] = value
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ['code_for', 'error_response', 'install']
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import time
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from opentelemetry import trace
|
||||||
|
from starlette.datastructures import MutableHeaders
|
||||||
|
|
||||||
|
from ..observability import get_metrics
|
||||||
|
from ..observability.logging import safe_extra
|
||||||
|
from .errors import error_response
|
||||||
|
|
||||||
|
logger = logging.getLogger('app')
|
||||||
|
|
||||||
|
|
||||||
|
class RequestContext:
|
||||||
|
def __init__(self, app) -> None:
|
||||||
|
self.app = app
|
||||||
|
self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname()
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send) -> None:
|
||||||
|
if scope['type'] != 'http':
|
||||||
|
await self.app(scope, receive, send)
|
||||||
|
return
|
||||||
|
|
||||||
|
request_id = uuid4().hex
|
||||||
|
scope.setdefault('state', {})['request_id'] = request_id
|
||||||
|
is_health = scope.get('path') in {'/health/live', '/health/ready'}
|
||||||
|
started_at = time.perf_counter()
|
||||||
|
started = False
|
||||||
|
status_code = 500
|
||||||
|
|
||||||
|
async def send_wrapper(message) -> None:
|
||||||
|
nonlocal started, status_code
|
||||||
|
if message['type'] == 'http.response.start':
|
||||||
|
started = True
|
||||||
|
status_code = message['status']
|
||||||
|
headers = MutableHeaders(scope=message)
|
||||||
|
headers['X-Request-Id'] = request_id
|
||||||
|
headers['X-Instance-Id'] = self.instance_id
|
||||||
|
await send(message)
|
||||||
|
|
||||||
|
try:
|
||||||
|
await self.app(scope, receive, send_wrapper)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
'unhandled HTTP error',
|
||||||
|
extra=safe_extra('http.request.failed', request_id=request_id),
|
||||||
|
)
|
||||||
|
if started:
|
||||||
|
raise
|
||||||
|
response = error_response(500, 'внутренняя ошибка', request_id)
|
||||||
|
await response(scope, receive, send_wrapper)
|
||||||
|
finally:
|
||||||
|
duration = time.perf_counter() - started_at
|
||||||
|
route = getattr(scope.get('route'), 'path', None) or 'unmatched'
|
||||||
|
method = scope.get('method', 'UNKNOWN')
|
||||||
|
status_class = f'{status_code // 100}xx'
|
||||||
|
span = trace.get_current_span()
|
||||||
|
if span.is_recording():
|
||||||
|
span.set_attribute('request.id', request_id)
|
||||||
|
span.set_attribute('http.route', route)
|
||||||
|
if not is_health:
|
||||||
|
attributes = {
|
||||||
|
'http.request.method': method,
|
||||||
|
'http.route': route,
|
||||||
|
'http.response.status_class': status_class,
|
||||||
|
}
|
||||||
|
instruments = get_metrics()
|
||||||
|
instruments.http_requests.add(1, attributes)
|
||||||
|
instruments.http_duration.record(duration, attributes)
|
||||||
|
logger.info(
|
||||||
|
'HTTP request completed',
|
||||||
|
extra=safe_extra(
|
||||||
|
'http.request.completed',
|
||||||
|
request_id=request_id,
|
||||||
|
http_method=method,
|
||||||
|
http_route=route,
|
||||||
|
http_status_code=status_code,
|
||||||
|
duration_seconds=round(duration, 6),
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from . import auth, chats, health, messages, users
|
||||||
|
|
||||||
|
__all__ = ['auth', 'chats', 'health', 'messages', 'users']
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
from typing import Annotated
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, Query, Request
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
|
||||||
|
from ...domain.models import User
|
||||||
|
from ...services.attachments import AttachmentService
|
||||||
|
from ..deps import attachment_service, current_user
|
||||||
|
from ..schemas import AttachmentOut
|
||||||
|
|
||||||
|
router = APIRouter(tags=['attachments'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
'/api/v1/chats/{chat_id}/attachments',
|
||||||
|
status_code=202,
|
||||||
|
response_model=AttachmentOut,
|
||||||
|
summary='Принять PNG/JPEG на асинхронную обработку',
|
||||||
|
openapi_extra={
|
||||||
|
'requestBody': {
|
||||||
|
'required': True,
|
||||||
|
'description': 'Сырые байты изображения, не multipart/form-data.',
|
||||||
|
'content': {
|
||||||
|
'image/png': {
|
||||||
|
'schema': {'type': 'string', 'format': 'binary'},
|
||||||
|
},
|
||||||
|
'image/jpeg': {
|
||||||
|
'schema': {'type': 'string', 'format': 'binary'},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def upload(
|
||||||
|
chat_id: UUID,
|
||||||
|
request: Request,
|
||||||
|
filename: str = Query(..., min_length=1, max_length=255),
|
||||||
|
content_type: Annotated[str | None, Header(alias='Content-Type')] = None,
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
attachments: AttachmentService = Depends(attachment_service),
|
||||||
|
) -> AttachmentOut:
|
||||||
|
attachment = await attachments.upload(
|
||||||
|
actor,
|
||||||
|
chat_id,
|
||||||
|
filename,
|
||||||
|
content_type or '',
|
||||||
|
request.stream(),
|
||||||
|
)
|
||||||
|
return AttachmentOut.from_domain(attachment)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut)
|
||||||
|
async def metadata(
|
||||||
|
attachment_id: UUID,
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
attachments: AttachmentService = Depends(attachment_service),
|
||||||
|
) -> AttachmentOut:
|
||||||
|
return AttachmentOut.from_domain(await attachments.get(actor, attachment_id))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/api/v1/attachments/{attachment_id}/content')
|
||||||
|
async def content(
|
||||||
|
attachment_id: UUID,
|
||||||
|
variant: str = Query('original'),
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
attachments: AttachmentService = Depends(attachment_service),
|
||||||
|
) -> StreamingResponse:
|
||||||
|
attachment, stream, content_type, size_bytes = await attachments.content(
|
||||||
|
actor, attachment_id, variant
|
||||||
|
)
|
||||||
|
return StreamingResponse(
|
||||||
|
stream.chunks(),
|
||||||
|
media_type=content_type,
|
||||||
|
headers={
|
||||||
|
'Content-Length': str(size_bytes),
|
||||||
|
'Content-Disposition': f'inline; filename="attachment-{attachment.id}"',
|
||||||
|
},
|
||||||
|
)
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
from binascii import Error as Base64Error
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Security, status
|
||||||
|
|
||||||
|
from ...auth.tokens import create_token
|
||||||
|
from ...db.redis import RedisSessionStore
|
||||||
|
from ...domain.models import User
|
||||||
|
from ...services.users import UserService
|
||||||
|
from ..deps import bearer_token, current_user, session_store, user_service
|
||||||
|
from ..security import basic_auth
|
||||||
|
from ..schemas import SessionOut, UserOut
|
||||||
|
from ..timefmt import to_rfc3339
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||||
|
|
||||||
|
_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||||
|
|
||||||
|
|
||||||
|
def _unauthorized_basic() -> HTTPException:
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||||
|
detail='неверные учётные данные',
|
||||||
|
headers={'WWW-Authenticate': _BASIC_CHALLENGE},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_basic(authorization: str | None) -> tuple[str, str]:
|
||||||
|
if not authorization:
|
||||||
|
raise _unauthorized_basic()
|
||||||
|
scheme, separator, encoded = authorization.partition(' ')
|
||||||
|
if scheme.lower() != 'basic' or not separator or not encoded:
|
||||||
|
raise _unauthorized_basic()
|
||||||
|
try:
|
||||||
|
decoded = base64.b64decode(encoded, validate=True).decode('utf-8')
|
||||||
|
except (Base64Error, UnicodeDecodeError):
|
||||||
|
raise _unauthorized_basic() from None
|
||||||
|
username, separator, password = decoded.partition(':')
|
||||||
|
if not separator or not username:
|
||||||
|
raise _unauthorized_basic()
|
||||||
|
return username, password
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut)
|
||||||
|
async def create_session(
|
||||||
|
authorization: str | None = Security(basic_auth),
|
||||||
|
users: UserService = Depends(user_service),
|
||||||
|
sessions: RedisSessionStore = Depends(session_store),
|
||||||
|
) -> SessionOut:
|
||||||
|
username, password = _parse_basic(authorization)
|
||||||
|
user = await users.authenticate(username, password)
|
||||||
|
if user is None:
|
||||||
|
raise _unauthorized_basic()
|
||||||
|
token = create_token()
|
||||||
|
expires_at = await sessions.create(user.id, token)
|
||||||
|
return SessionOut(
|
||||||
|
token=token,
|
||||||
|
token_type='Bearer',
|
||||||
|
expires_at=to_rfc3339(expires_at),
|
||||||
|
user=UserOut.from_domain(user),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None)
|
||||||
|
async def revoke_session(
|
||||||
|
token: str = Depends(bearer_token),
|
||||||
|
_: User = Depends(current_user),
|
||||||
|
sessions: RedisSessionStore = Depends(session_store),
|
||||||
|
) -> None:
|
||||||
|
await sessions.revoke(token)
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
from typing import Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Query
|
||||||
|
|
||||||
|
from ...domain.models import User
|
||||||
|
from ...services.chats import ChatService
|
||||||
|
from .. import cursors
|
||||||
|
from ..deps import chat_service, current_user
|
||||||
|
from ..schemas import (
|
||||||
|
LIMIT_DEFAULT,
|
||||||
|
LIMIT_MAX,
|
||||||
|
LIMIT_MIN,
|
||||||
|
ChatOut,
|
||||||
|
ChatPageOut,
|
||||||
|
CreateChat,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1/chats', tags=['chats'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('', status_code=201, response_model=ChatOut)
|
||||||
|
async def create(
|
||||||
|
body: CreateChat,
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
chats: ChatService = Depends(chat_service),
|
||||||
|
) -> ChatOut:
|
||||||
|
return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('', response_model=ChatPageOut)
|
||||||
|
async def page(
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
chats: ChatService = Depends(chat_service),
|
||||||
|
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
|
||||||
|
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
|
||||||
|
) -> ChatPageOut:
|
||||||
|
scope = f'chats:{actor.id}'
|
||||||
|
after = cursors.decode(scope, cursor) if cursor is not None else None
|
||||||
|
return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/{chat_id}', response_model=ChatOut)
|
||||||
|
async def get(
|
||||||
|
chat_id: UUID,
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
chats: ChatService = Depends(chat_service),
|
||||||
|
) -> ChatOut:
|
||||||
|
return ChatOut.from_domain(await chats.get(actor, chat_id))
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
from fastapi import APIRouter, HTTPException, Request
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.exc import SQLAlchemyError
|
||||||
|
|
||||||
|
from ...domain.errors import Unavailable
|
||||||
|
|
||||||
|
from ..schemas import HealthOut
|
||||||
|
|
||||||
|
router = APIRouter(tags=['health'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/health/live', response_model=HealthOut)
|
||||||
|
async def live() -> HealthOut:
|
||||||
|
return HealthOut(status='ok')
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/health/ready', response_model=HealthOut)
|
||||||
|
async def ready(request: Request) -> HealthOut:
|
||||||
|
if not request.app.state.ready:
|
||||||
|
raise HTTPException(status_code=503, detail='приложение не готово принимать трафик')
|
||||||
|
try:
|
||||||
|
async with request.app.state.engine.connect() as connection:
|
||||||
|
await connection.execute(text('SELECT 1'))
|
||||||
|
await request.app.state.redis.ping()
|
||||||
|
await request.app.state.storage.ping()
|
||||||
|
except (SQLAlchemyError, RedisError, Unavailable, OSError):
|
||||||
|
raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None
|
||||||
|
return HealthOut(status='ok')
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
from typing import Annotated, Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, Header, Query
|
||||||
|
|
||||||
|
from ...domain.models import User
|
||||||
|
from ...services.messages import MessageService
|
||||||
|
from .. import cursors
|
||||||
|
from ..deps import current_user, message_service
|
||||||
|
from ..schemas import (
|
||||||
|
LIMIT_DEFAULT,
|
||||||
|
LIMIT_MAX,
|
||||||
|
LIMIT_MIN,
|
||||||
|
CreateMessage,
|
||||||
|
MessageOut,
|
||||||
|
MessagePageOut,
|
||||||
|
)
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('', status_code=201, response_model=MessageOut)
|
||||||
|
async def send(
|
||||||
|
chat_id: UUID,
|
||||||
|
body: CreateMessage,
|
||||||
|
idempotency_key: Annotated[
|
||||||
|
str,
|
||||||
|
Header(
|
||||||
|
alias='Idempotency-Key',
|
||||||
|
min_length=1,
|
||||||
|
max_length=128,
|
||||||
|
pattern=r'^[A-Za-z0-9._:-]+$',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
messages: MessageService = Depends(message_service),
|
||||||
|
) -> MessageOut:
|
||||||
|
return MessageOut.from_domain(
|
||||||
|
await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('', response_model=MessagePageOut)
|
||||||
|
async def page(
|
||||||
|
chat_id: UUID,
|
||||||
|
actor: User = Depends(current_user),
|
||||||
|
messages: MessageService = Depends(message_service),
|
||||||
|
limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX),
|
||||||
|
cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH),
|
||||||
|
) -> MessagePageOut:
|
||||||
|
scope = f'messages:{chat_id}'
|
||||||
|
after = cursors.decode(scope, cursor) if cursor is not None else None
|
||||||
|
return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope)
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
|
||||||
|
from ...domain.models import User
|
||||||
|
from ...services.users import UserService
|
||||||
|
from ..deps import current_user, user_service
|
||||||
|
from ..schemas import CreateUser, UserOut
|
||||||
|
|
||||||
|
router = APIRouter(prefix='/api/v1', tags=['users'])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post('/users', status_code=201, response_model=UserOut)
|
||||||
|
async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut:
|
||||||
|
return UserOut.from_domain(await users.register(body.username, body.display_name, body.password))
|
||||||
|
|
||||||
|
|
||||||
|
@router.get('/users/me', response_model=UserOut)
|
||||||
|
async def me(actor: User = Depends(current_user)) -> UserOut:
|
||||||
|
return UserOut.from_domain(actor)
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
from typing import Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||||
|
|
||||||
|
from ..domain.models import Attachment, Chat, Message, Page, User
|
||||||
|
from . import cursors
|
||||||
|
from .timefmt import to_rfc3339
|
||||||
|
|
||||||
|
NAME_MAX = 100
|
||||||
|
TEXT_MAX = 4000
|
||||||
|
MEMBERS_MAX = 100
|
||||||
|
LIMIT_MIN = 1
|
||||||
|
LIMIT_MAX = 100
|
||||||
|
LIMIT_DEFAULT = 50
|
||||||
|
|
||||||
|
|
||||||
|
def not_blank(value: str) -> str:
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError('строка не может состоять только из пробельных символов')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Strict(BaseModel):
|
||||||
|
model_config = ConfigDict(extra='forbid')
|
||||||
|
|
||||||
|
|
||||||
|
class HealthOut(Strict):
|
||||||
|
status: str = 'ok'
|
||||||
|
|
||||||
|
|
||||||
|
class CreateUser(Strict):
|
||||||
|
username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$')
|
||||||
|
display_name: str = Field(min_length=1, max_length=NAME_MAX)
|
||||||
|
password: str = Field(min_length=12, max_length=128)
|
||||||
|
|
||||||
|
@field_validator('display_name')
|
||||||
|
@classmethod
|
||||||
|
def _display_name(cls, value: str) -> str:
|
||||||
|
return not_blank(value)
|
||||||
|
|
||||||
|
|
||||||
|
class UserOut(Strict):
|
||||||
|
id: UUID
|
||||||
|
username: str
|
||||||
|
display_name: str
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_domain(cls, user: User) -> 'UserOut':
|
||||||
|
return cls(
|
||||||
|
id=user.id,
|
||||||
|
username=user.username,
|
||||||
|
display_name=user.display_name,
|
||||||
|
created_at=to_rfc3339(user.created_at),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SessionOut(Strict):
|
||||||
|
token: str = Field(min_length=32)
|
||||||
|
token_type: str = 'Bearer'
|
||||||
|
expires_at: str
|
||||||
|
user: UserOut
|
||||||
|
|
||||||
|
|
||||||
|
class CreateChat(Strict):
|
||||||
|
title: str = Field(min_length=1, max_length=NAME_MAX)
|
||||||
|
member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX)
|
||||||
|
|
||||||
|
@field_validator('title')
|
||||||
|
@classmethod
|
||||||
|
def _title(cls, value: str) -> str:
|
||||||
|
return not_blank(value)
|
||||||
|
|
||||||
|
@field_validator('member_ids')
|
||||||
|
@classmethod
|
||||||
|
def _members_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||||
|
if len(set(value)) != len(value):
|
||||||
|
raise ValueError('member_ids не должен содержать дубликаты')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class ChatOut(Strict):
|
||||||
|
id: UUID
|
||||||
|
title: str
|
||||||
|
member_ids: list[UUID]
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_domain(cls, chat: Chat) -> 'ChatOut':
|
||||||
|
return cls(
|
||||||
|
id=chat.id,
|
||||||
|
title=chat.title,
|
||||||
|
member_ids=list(chat.member_ids),
|
||||||
|
created_at=to_rfc3339(chat.created_at),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CreateMessage(Strict):
|
||||||
|
text: str = Field(max_length=TEXT_MAX)
|
||||||
|
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||||
|
|
||||||
|
@field_validator('attachment_ids')
|
||||||
|
@classmethod
|
||||||
|
def _attachments_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||||
|
if len(set(value)) != len(value):
|
||||||
|
raise ValueError('attachment_ids не должен содержать дубликаты')
|
||||||
|
return value
|
||||||
|
|
||||||
|
@model_validator(mode='after')
|
||||||
|
def _content_present(self) -> 'CreateMessage':
|
||||||
|
if not self.text.strip() and not self.attachment_ids:
|
||||||
|
raise ValueError('сообщение должно содержать текст или вложение')
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class MessageOut(Strict):
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
sender_id: UUID
|
||||||
|
text: str
|
||||||
|
created_at: str
|
||||||
|
attachment_ids: list[UUID]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_domain(cls, message: Message) -> 'MessageOut':
|
||||||
|
return cls(
|
||||||
|
id=message.id,
|
||||||
|
chat_id=message.chat_id,
|
||||||
|
sender_id=message.sender_id,
|
||||||
|
text=message.text,
|
||||||
|
created_at=to_rfc3339(message.created_at),
|
||||||
|
attachment_ids=list(message.attachment_ids),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class VariantOut(Strict):
|
||||||
|
name: str
|
||||||
|
content_type: str
|
||||||
|
size_bytes: int
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentOut(Strict):
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
owner_id: UUID
|
||||||
|
filename: str
|
||||||
|
content_type: str
|
||||||
|
size_bytes: int
|
||||||
|
sha256: str
|
||||||
|
status: str
|
||||||
|
variants: list[VariantOut] = Field(default_factory=list)
|
||||||
|
error_code: str | None
|
||||||
|
created_at: str
|
||||||
|
updated_at: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_domain(cls, attachment: Attachment) -> 'AttachmentOut':
|
||||||
|
return cls(
|
||||||
|
id=attachment.id,
|
||||||
|
chat_id=attachment.chat_id,
|
||||||
|
owner_id=attachment.owner_id,
|
||||||
|
filename=attachment.filename,
|
||||||
|
content_type=attachment.content_type,
|
||||||
|
size_bytes=attachment.size_bytes,
|
||||||
|
sha256=attachment.sha256,
|
||||||
|
status=attachment.status,
|
||||||
|
variants=[
|
||||||
|
VariantOut(
|
||||||
|
name=item.name,
|
||||||
|
content_type=item.content_type,
|
||||||
|
size_bytes=item.size_bytes,
|
||||||
|
width=item.width,
|
||||||
|
height=item.height,
|
||||||
|
sha256=item.sha256,
|
||||||
|
)
|
||||||
|
for item in attachment.variants
|
||||||
|
],
|
||||||
|
error_code=attachment.error_code,
|
||||||
|
created_at=to_rfc3339(attachment.created_at),
|
||||||
|
updated_at=to_rfc3339(attachment.updated_at),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ChatPageOut(Strict):
|
||||||
|
items: list[ChatOut]
|
||||||
|
next_cursor: Optional[str]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut':
|
||||||
|
return cls(
|
||||||
|
items=[ChatOut.from_domain(chat) for chat in page.items],
|
||||||
|
next_cursor=cursors.encode(scope, page.next) if page.next else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MessagePageOut(Strict):
|
||||||
|
items: list[MessageOut]
|
||||||
|
next_cursor: Optional[str]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut':
|
||||||
|
return cls(
|
||||||
|
items=[MessageOut.from_domain(message) for message in page.items],
|
||||||
|
next_cursor=cursors.encode(scope, page.next) if page.next else None,
|
||||||
|
)
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""OpenAPI security-схемы с сохранением собственного разбора Authorization."""
|
||||||
|
|
||||||
|
from fastapi import Request
|
||||||
|
from fastapi.openapi.models import HTTPBase as HTTPBaseModel
|
||||||
|
from fastapi.openapi.models import HTTPBearer as HTTPBearerModel
|
||||||
|
from fastapi.security.base import SecurityBase
|
||||||
|
|
||||||
|
|
||||||
|
class RawHTTPAuthorization(SecurityBase):
|
||||||
|
"""Объявляет HTTP security scheme и возвращает исходный Authorization header."""
|
||||||
|
|
||||||
|
def __init__(self, model, scheme_name: str) -> None:
|
||||||
|
self.model = model
|
||||||
|
self.scheme_name = scheme_name
|
||||||
|
|
||||||
|
async def __call__(self, request: Request) -> str | None:
|
||||||
|
"""Возвращает заголовок без разбора, чтобы сохранить текущую валидацию API."""
|
||||||
|
|
||||||
|
return request.headers.get('Authorization')
|
||||||
|
|
||||||
|
|
||||||
|
basic_auth = RawHTTPAuthorization(
|
||||||
|
HTTPBaseModel(scheme='basic'),
|
||||||
|
scheme_name='basicAuth',
|
||||||
|
)
|
||||||
|
bearer_auth = RawHTTPAuthorization(
|
||||||
|
HTTPBearerModel(scheme='bearer', bearerFormat='opaque session token'),
|
||||||
|
scheme_name='bearerAuth',
|
||||||
|
)
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
|
||||||
|
def to_rfc3339(value: datetime) -> str:
|
||||||
|
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||||
|
|
||||||
|
|
||||||
|
def from_rfc3339(value: str) -> datetime:
|
||||||
|
if not isinstance(value, str) or not value.endswith('Z'):
|
||||||
|
raise ValueError('timestamp must end with Z')
|
||||||
|
return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc)
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
from .passwords import hash_password, password_matches
|
||||||
|
from .tokens import create_token
|
||||||
|
|
||||||
|
__all__ = ['create_token', 'hash_password', 'password_matches']
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
from argon2 import PasswordHasher
|
||||||
|
from argon2.exceptions import VerificationError, VerifyMismatchError
|
||||||
|
|
||||||
|
_hasher = PasswordHasher()
|
||||||
|
|
||||||
|
|
||||||
|
def hash_password(password: str) -> str:
|
||||||
|
return _hasher.hash(password)
|
||||||
|
|
||||||
|
|
||||||
|
def password_matches(stored_hash: str, password: str) -> bool:
|
||||||
|
try:
|
||||||
|
return _hasher.verify(stored_hash, password)
|
||||||
|
except (VerifyMismatchError, VerificationError):
|
||||||
|
return False
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
import hashlib
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
|
||||||
|
def create_token() -> str:
|
||||||
|
return secrets.token_urlsafe(32)
|
||||||
|
|
||||||
|
|
||||||
|
def session_key(token: str) -> str:
|
||||||
|
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||||
|
return f'session:{digest}'
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from .settings import Settings, load_settings
|
||||||
|
|
||||||
|
__all__ = ['Settings', 'load_settings']
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _read_value(name: str, file_name: str) -> str:
|
||||||
|
value = os.getenv(name)
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
path = os.getenv(file_name)
|
||||||
|
if path:
|
||||||
|
return Path(path).read_text(encoding='utf-8').strip()
|
||||||
|
raise RuntimeError(f'не задана настройка {name} или {file_name}')
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Settings:
|
||||||
|
database_url: str
|
||||||
|
redis_url: str
|
||||||
|
session_ttl_seconds: int
|
||||||
|
s3_endpoint: str
|
||||||
|
s3_region: str
|
||||||
|
s3_bucket: str
|
||||||
|
s3_access_key_id: str
|
||||||
|
s3_secret_access_key: str
|
||||||
|
max_upload_bytes: int
|
||||||
|
max_image_pixels: int
|
||||||
|
broker_url: str
|
||||||
|
worker_concurrency: int
|
||||||
|
worker_prefetch: int
|
||||||
|
job_max_attempts: int
|
||||||
|
job_timeout_seconds: int
|
||||||
|
job_lease_seconds: int
|
||||||
|
retry_base_seconds: int
|
||||||
|
retry_max_seconds: int
|
||||||
|
staging_retention_seconds: int
|
||||||
|
thumbnail_jpeg_quality: int
|
||||||
|
crop_jpeg_quality: int
|
||||||
|
optimized_webp_quality: int
|
||||||
|
|
||||||
|
|
||||||
|
def _positive_int(name: str, default: str) -> int:
|
||||||
|
try:
|
||||||
|
value = int(os.getenv(name, default))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f'{name} должен быть целым числом') from exc
|
||||||
|
if value <= 0:
|
||||||
|
raise RuntimeError(f'{name} должен быть больше нуля')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _quality(name: str, default: str) -> int:
|
||||||
|
value = _positive_int(name, default)
|
||||||
|
if value > 100:
|
||||||
|
raise RuntimeError(f'{name} должен быть не больше 100')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def load_settings() -> Settings:
|
||||||
|
return Settings(
|
||||||
|
database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'),
|
||||||
|
redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'),
|
||||||
|
session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'),
|
||||||
|
s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||||
|
s3_region=os.getenv('S3_REGION', 'us-east-1'),
|
||||||
|
s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'),
|
||||||
|
s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'),
|
||||||
|
s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'),
|
||||||
|
max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'),
|
||||||
|
max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'),
|
||||||
|
broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'),
|
||||||
|
worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'),
|
||||||
|
worker_prefetch=_positive_int('WORKER_PREFETCH', '2'),
|
||||||
|
job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'),
|
||||||
|
job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'),
|
||||||
|
job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'),
|
||||||
|
retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'),
|
||||||
|
retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'),
|
||||||
|
staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'),
|
||||||
|
thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'),
|
||||||
|
crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'),
|
||||||
|
optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'),
|
||||||
|
)
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
from .postgres import Base, create_engine, create_session_factory
|
||||||
|
from .redis import RedisSessionStore, create_redis
|
||||||
|
|
||||||
|
__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory']
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from sqlalchemy import event
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
from sqlalchemy.orm import DeclarativeBase
|
||||||
|
|
||||||
|
from ..observability import get_metrics
|
||||||
|
|
||||||
|
|
||||||
|
class Base(DeclarativeBase):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def create_engine(url: str) -> AsyncEngine:
|
||||||
|
engine = create_async_engine(
|
||||||
|
url,
|
||||||
|
pool_size=5,
|
||||||
|
max_overflow=5,
|
||||||
|
pool_timeout=5,
|
||||||
|
pool_pre_ping=True,
|
||||||
|
connect_args={'server_settings': {'statement_timeout': '5000'}},
|
||||||
|
)
|
||||||
|
instruments = get_metrics()
|
||||||
|
attributes = {'pool.name': 'postgres'}
|
||||||
|
|
||||||
|
def update_pool_metrics(*_args) -> None:
|
||||||
|
pool = engine.sync_engine.pool
|
||||||
|
instruments.db_pool_used.set(float(pool.checkedout()), attributes)
|
||||||
|
waiting = len(getattr(getattr(pool, '_pool', None), '_getters', ()))
|
||||||
|
instruments.db_pool_waiting.set(float(waiting), attributes)
|
||||||
|
|
||||||
|
event.listen(engine.sync_engine, 'checkout', update_pool_metrics)
|
||||||
|
event.listen(engine.sync_engine, 'checkin', update_pool_metrics)
|
||||||
|
instruments.db_pool_used.set(0, attributes)
|
||||||
|
instruments.db_pool_waiting.set(0, attributes)
|
||||||
|
return engine
|
||||||
|
|
||||||
|
|
||||||
|
def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
|
||||||
|
return async_sessionmaker(engine, expire_on_commit=False)
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from redis.asyncio import Redis
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
|
||||||
|
from ..auth.tokens import session_key
|
||||||
|
from ..domain.errors import Unavailable
|
||||||
|
|
||||||
|
|
||||||
|
def create_redis(url: str) -> Redis:
|
||||||
|
return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3)
|
||||||
|
|
||||||
|
|
||||||
|
class RedisSessionStore:
|
||||||
|
def __init__(self, client: Redis, ttl_seconds: int) -> None:
|
||||||
|
self._client = client
|
||||||
|
self._ttl_seconds = ttl_seconds
|
||||||
|
|
||||||
|
async def create(self, user_id: UUID, token: str) -> datetime:
|
||||||
|
try:
|
||||||
|
await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds)
|
||||||
|
except RedisError as exc:
|
||||||
|
raise Unavailable('хранилище сессий недоступно') from exc
|
||||||
|
return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds)
|
||||||
|
|
||||||
|
async def get_user_id(self, token: str) -> UUID | None:
|
||||||
|
try:
|
||||||
|
value = await self._client.get(session_key(token))
|
||||||
|
except RedisError as exc:
|
||||||
|
raise Unavailable('хранилище сессий недоступно') from exc
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return UUID(value)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def revoke(self, token: str) -> None:
|
||||||
|
try:
|
||||||
|
await self._client.delete(session_key(token))
|
||||||
|
except RedisError as exc:
|
||||||
|
raise Unavailable('хранилище сессий недоступно') from exc
|
||||||
|
|
||||||
|
async def ping(self) -> None:
|
||||||
|
try:
|
||||||
|
await self._client.ping()
|
||||||
|
except RedisError as exc:
|
||||||
|
raise Unavailable('хранилище сессий недоступно') from exc
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
class DomainError(Exception):
|
||||||
|
"""Сообщение попадает клиенту, поэтому не содержит внутренних деталей."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
|
||||||
|
class Invalid(DomainError):
|
||||||
|
"""Запрос нарушает правила предметной области."""
|
||||||
|
|
||||||
|
|
||||||
|
class Unauthorized(DomainError):
|
||||||
|
"""Нет действительной идентификации."""
|
||||||
|
|
||||||
|
|
||||||
|
class NotFound(DomainError):
|
||||||
|
"""Объекта нет либо он недоступен вызывающему: снаружи это неразличимо."""
|
||||||
|
|
||||||
|
|
||||||
|
class Conflict(DomainError):
|
||||||
|
"""Состояние не позволяет выполнить операцию."""
|
||||||
|
|
||||||
|
|
||||||
|
class Unavailable(DomainError):
|
||||||
|
"""Обязательная внешняя зависимость временно недоступна."""
|
||||||
|
|
||||||
|
|
||||||
|
class PayloadTooLarge(DomainError):
|
||||||
|
"""Тело запроса или декодированное содержимое превышает лимит."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnsupportedMediaType(DomainError):
|
||||||
|
"""Формат содержимого не поддерживается или не соответствует MIME."""
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class User:
|
||||||
|
id: UUID
|
||||||
|
username: str
|
||||||
|
display_name: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Chat:
|
||||||
|
id: UUID
|
||||||
|
title: str
|
||||||
|
member_ids: tuple[UUID, ...]
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
def has_member(self, user_id: UUID) -> bool:
|
||||||
|
return user_id in self.member_ids
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Message:
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
sender_id: UUID
|
||||||
|
text: str
|
||||||
|
created_at: datetime
|
||||||
|
attachment_ids: tuple[UUID, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Attachment:
|
||||||
|
id: UUID
|
||||||
|
chat_id: UUID
|
||||||
|
owner_id: UUID
|
||||||
|
filename: str
|
||||||
|
original_key: str | None
|
||||||
|
staging_key: str | None
|
||||||
|
pipeline_version: int
|
||||||
|
content_type: str
|
||||||
|
size_bytes: int
|
||||||
|
sha256: str
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
status: str
|
||||||
|
error_code: str | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
variants: tuple['AttachmentVariant', ...] = ()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class AttachmentVariant:
|
||||||
|
name: str
|
||||||
|
object_key: str
|
||||||
|
content_type: str
|
||||||
|
size_bytes: int
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class AttachmentJob:
|
||||||
|
id: UUID
|
||||||
|
attachment_id: UUID
|
||||||
|
pipeline_version: int
|
||||||
|
state: str
|
||||||
|
attempt: int
|
||||||
|
max_attempts: int
|
||||||
|
next_attempt_at: datetime
|
||||||
|
lease_owner: str | None
|
||||||
|
lease_until: datetime | None
|
||||||
|
last_error_code: str | None
|
||||||
|
replay_count: int
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class OutboxEvent:
|
||||||
|
id: UUID
|
||||||
|
job_id: UUID
|
||||||
|
payload: dict
|
||||||
|
available_at: datetime
|
||||||
|
traceparent: str | None = None
|
||||||
|
tracestate: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Keyset:
|
||||||
|
created_at: datetime
|
||||||
|
id: UUID
|
||||||
|
|
||||||
|
@property
|
||||||
|
def order(self) -> tuple[datetime, int]:
|
||||||
|
return (self.created_at, self.id.int)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_item(cls, item) -> 'Keyset':
|
||||||
|
return cls(created_at=item.created_at, id=item.id)
|
||||||
|
|
||||||
|
|
||||||
|
def order_key(item) -> tuple[datetime, int]:
|
||||||
|
return (item.created_at, item.id.int)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Page:
|
||||||
|
items: tuple
|
||||||
|
next: Optional[Keyset]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Durable attachment job publisher and worker."""
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import aio_pika
|
||||||
|
from aio_pika import DeliveryMode, ExchangeType
|
||||||
|
|
||||||
|
MAIN_EXCHANGE = 'attachments'
|
||||||
|
MAIN_QUEUE = 'attachments.process'
|
||||||
|
ROUTING_KEY = 'attachment.process.v1'
|
||||||
|
DEAD_EXCHANGE = 'attachments.dead'
|
||||||
|
DEAD_QUEUE = 'attachments.failed'
|
||||||
|
logger = logging.getLogger('app.broker')
|
||||||
|
|
||||||
|
|
||||||
|
async def connect(url: str) -> aio_pika.RobustConnection:
|
||||||
|
delay = 0.5
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
return await aio_pika.connect_robust(url, timeout=5)
|
||||||
|
except (OSError, aio_pika.exceptions.AMQPException):
|
||||||
|
logger.warning('broker is not ready; reconnecting in %.1fs', delay)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
delay = min(delay * 2, 5.0)
|
||||||
|
|
||||||
|
|
||||||
|
async def topology(channel: aio_pika.abc.AbstractChannel):
|
||||||
|
dead_exchange = await channel.declare_exchange(
|
||||||
|
DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True
|
||||||
|
)
|
||||||
|
dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True)
|
||||||
|
await dead_queue.bind(dead_exchange, ROUTING_KEY)
|
||||||
|
exchange = await channel.declare_exchange(
|
||||||
|
MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True
|
||||||
|
)
|
||||||
|
queue = await channel.declare_queue(
|
||||||
|
MAIN_QUEUE,
|
||||||
|
durable=True,
|
||||||
|
arguments={
|
||||||
|
'x-dead-letter-exchange': DEAD_EXCHANGE,
|
||||||
|
'x-dead-letter-routing-key': ROUTING_KEY,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await queue.bind(exchange, ROUTING_KEY)
|
||||||
|
return exchange, queue
|
||||||
|
|
||||||
|
|
||||||
|
def message(body: bytes, event_id: str, headers: dict[str, str] | None = None) -> aio_pika.Message:
|
||||||
|
return aio_pika.Message(
|
||||||
|
body=body,
|
||||||
|
content_type='application/json',
|
||||||
|
delivery_mode=DeliveryMode.PERSISTENT,
|
||||||
|
message_id=event_id,
|
||||||
|
type=ROUTING_KEY,
|
||||||
|
headers=headers or {},
|
||||||
|
)
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||||
|
|
||||||
|
|
||||||
|
class PermanentImageError(Exception):
|
||||||
|
def __init__(self, code: str) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class GeneratedVariant:
|
||||||
|
name: str
|
||||||
|
content_type: str
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
sha256: str
|
||||||
|
content: bytes
|
||||||
|
|
||||||
|
|
||||||
|
def _jpeg_image(image: Image.Image) -> Image.Image:
|
||||||
|
if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info:
|
||||||
|
rgba = image.convert('RGBA')
|
||||||
|
background = Image.new('RGBA', rgba.size, 'white')
|
||||||
|
background.alpha_composite(rgba)
|
||||||
|
return background.convert('RGB')
|
||||||
|
return image.convert('RGB')
|
||||||
|
|
||||||
|
|
||||||
|
def _encode(image: Image.Image, fmt: str, **options) -> bytes:
|
||||||
|
output = BytesIO()
|
||||||
|
image.save(output, format=fmt, **options)
|
||||||
|
return output.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant:
|
||||||
|
return GeneratedVariant(
|
||||||
|
name=name,
|
||||||
|
content_type=content_type,
|
||||||
|
width=image.width,
|
||||||
|
height=image.height,
|
||||||
|
sha256=hashlib.sha256(content).hexdigest(),
|
||||||
|
content=content,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def process_image(
|
||||||
|
source: bytes,
|
||||||
|
declared_type: str,
|
||||||
|
expected_sha256: str,
|
||||||
|
max_pixels: int,
|
||||||
|
thumbnail_quality: int,
|
||||||
|
crop_quality: int,
|
||||||
|
webp_quality: int,
|
||||||
|
) -> tuple[GeneratedVariant, ...]:
|
||||||
|
if hashlib.sha256(source).hexdigest() != expected_sha256:
|
||||||
|
raise PermanentImageError('checksum_mismatch')
|
||||||
|
try:
|
||||||
|
with Image.open(BytesIO(source)) as opened:
|
||||||
|
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '')
|
||||||
|
if actual != declared_type:
|
||||||
|
raise PermanentImageError('unsupported_media_type')
|
||||||
|
if opened.width * opened.height > max_pixels:
|
||||||
|
raise PermanentImageError('image_too_large')
|
||||||
|
opened.load()
|
||||||
|
oriented = ImageOps.exif_transpose(opened)
|
||||||
|
base = oriented.copy()
|
||||||
|
except PermanentImageError:
|
||||||
|
raise
|
||||||
|
except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error:
|
||||||
|
raise PermanentImageError('invalid_image') from error
|
||||||
|
|
||||||
|
thumbnail = _jpeg_image(base.copy())
|
||||||
|
thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||||
|
thumbnail_bytes = _encode(
|
||||||
|
thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True
|
||||||
|
)
|
||||||
|
|
||||||
|
crop_source = _jpeg_image(base.copy())
|
||||||
|
crop = ImageOps.fit(
|
||||||
|
crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||||
|
)
|
||||||
|
crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True)
|
||||||
|
|
||||||
|
optimized = base.copy()
|
||||||
|
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||||
|
optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6)
|
||||||
|
|
||||||
|
return (
|
||||||
|
_variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes),
|
||||||
|
_variant('crop', 'image/jpeg', crop, crop_bytes),
|
||||||
|
_variant('optimized', 'image/webp', optimized, optimized_bytes),
|
||||||
|
)
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from opentelemetry import trace
|
||||||
|
from opentelemetry.trace import SpanKind
|
||||||
|
|
||||||
|
from ..config import load_settings
|
||||||
|
from ..observability import get_metrics
|
||||||
|
from ..observability.logging import safe_extra
|
||||||
|
from ..observability.propagation import extract_trace_context, inject_trace_context
|
||||||
|
from ..repositories import JobRepository, OutboxRepository
|
||||||
|
from .broker import ROUTING_KEY, connect, message, topology
|
||||||
|
from .runtime import configure_logging, dependencies
|
||||||
|
|
||||||
|
logger = logging.getLogger('app.publisher')
|
||||||
|
tracer = trace.get_tracer('messenger.publisher', '1.6.0')
|
||||||
|
|
||||||
|
|
||||||
|
async def run(once: bool = False) -> None:
|
||||||
|
settings = load_settings()
|
||||||
|
engine, sessions, _storage = dependencies(settings)
|
||||||
|
outbox = OutboxRepository(sessions)
|
||||||
|
jobs = JobRepository(sessions)
|
||||||
|
instruments = get_metrics()
|
||||||
|
queue_attributes = {'messaging.destination.name': 'attachments.process'}
|
||||||
|
owner = f'{socket.gethostname()}:{os.getpid()}'
|
||||||
|
connection = await connect(settings.broker_url)
|
||||||
|
try:
|
||||||
|
channel = await connection.channel(publisher_confirms=True)
|
||||||
|
exchange, _queue = await topology(channel)
|
||||||
|
while True:
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
await jobs.recover_expired(now)
|
||||||
|
depth, oldest_age = await jobs.queue_stats(now)
|
||||||
|
instruments.queue_depth.set(depth, queue_attributes)
|
||||||
|
instruments.queue_oldest_age.set(oldest_age, queue_attributes)
|
||||||
|
events = await outbox.claim(owner, now, lease_seconds=30)
|
||||||
|
for event in events:
|
||||||
|
try:
|
||||||
|
parent = extract_trace_context({
|
||||||
|
'traceparent': event.traceparent,
|
||||||
|
'tracestate': event.tracestate,
|
||||||
|
})
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
'attachments publish',
|
||||||
|
context=parent,
|
||||||
|
kind=SpanKind.PRODUCER,
|
||||||
|
attributes={
|
||||||
|
'messaging.system': 'rabbitmq',
|
||||||
|
'messaging.destination.name': 'attachments.process',
|
||||||
|
'messaging.operation.name': 'publish',
|
||||||
|
'messaging.operation.type': 'send',
|
||||||
|
'job.id': str(event.job_id),
|
||||||
|
},
|
||||||
|
):
|
||||||
|
body = json.dumps(event.payload, separators=(',', ':')).encode()
|
||||||
|
headers = inject_trace_context()
|
||||||
|
await exchange.publish(
|
||||||
|
message(body, str(event.id), headers),
|
||||||
|
routing_key=ROUTING_KEY,
|
||||||
|
mandatory=True,
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
'outbox event published',
|
||||||
|
extra=safe_extra(
|
||||||
|
'outbox.published', event_id=str(event.id),
|
||||||
|
job_id=str(event.job_id),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1':
|
||||||
|
os._exit(91)
|
||||||
|
await outbox.published(event.id, owner, datetime.now(timezone.utc))
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
'outbox publish failed',
|
||||||
|
extra=safe_extra(
|
||||||
|
'outbox.publish_failed', event_id=str(event.id),
|
||||||
|
job_id=str(event.job_id),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await outbox.failed(
|
||||||
|
event.id, owner, 'broker_unavailable', datetime.now(timezone.utc)
|
||||||
|
)
|
||||||
|
if once:
|
||||||
|
return
|
||||||
|
await asyncio.sleep(0.5 if events else 1.0)
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--once', action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
telemetry = configure_logging('messenger-publisher')
|
||||||
|
try:
|
||||||
|
asyncio.run(run(args.once))
|
||||||
|
finally:
|
||||||
|
telemetry.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ..config import Settings
|
||||||
|
from ..db import create_engine, create_session_factory
|
||||||
|
from ..observability import initialize_observability
|
||||||
|
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
|
||||||
|
from ..storage import S3Storage
|
||||||
|
|
||||||
|
|
||||||
|
def configure_logging(service_name: str):
|
||||||
|
return initialize_observability(service_name)
|
||||||
|
|
||||||
|
|
||||||
|
def dependencies(settings: Settings):
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine)
|
||||||
|
sessions = create_session_factory(engine)
|
||||||
|
storage = S3Storage(
|
||||||
|
endpoint=settings.s3_endpoint,
|
||||||
|
access_key=settings.s3_access_key_id,
|
||||||
|
secret_key=settings.s3_secret_access_key,
|
||||||
|
bucket=settings.s3_bucket,
|
||||||
|
region=settings.s3_region,
|
||||||
|
)
|
||||||
|
return engine, sessions, storage
|
||||||
@@ -0,0 +1,390 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import multiprocessing
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import socket
|
||||||
|
import time
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
import aio_pika
|
||||||
|
from opentelemetry import trace
|
||||||
|
from opentelemetry.trace import SpanKind
|
||||||
|
|
||||||
|
from ..config import Settings, load_settings
|
||||||
|
from ..domain.errors import Unavailable
|
||||||
|
from ..domain.models import AttachmentVariant
|
||||||
|
from ..observability import get_metrics
|
||||||
|
from ..observability.logging import safe_extra
|
||||||
|
from ..observability.propagation import extract_trace_context
|
||||||
|
from ..repositories import JobRepository
|
||||||
|
from .broker import connect, topology
|
||||||
|
from .pipeline import GeneratedVariant, PermanentImageError, process_image
|
||||||
|
from .runtime import configure_logging, dependencies
|
||||||
|
|
||||||
|
logger = logging.getLogger('app.worker')
|
||||||
|
tracer = trace.get_tracer('messenger.worker', '1.6.0')
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessingTimeout(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None:
|
||||||
|
try:
|
||||||
|
if memory_bytes:
|
||||||
|
try:
|
||||||
|
import resource
|
||||||
|
|
||||||
|
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
||||||
|
except (ImportError, OSError, ValueError):
|
||||||
|
pass
|
||||||
|
connection.send(('ok', process_image(*arguments)))
|
||||||
|
except PermanentImageError as error:
|
||||||
|
connection.send(('permanent', error.code))
|
||||||
|
except MemoryError:
|
||||||
|
connection.send(('temporary', 'processing_memory_limit'))
|
||||||
|
except BaseException:
|
||||||
|
connection.send(('temporary', 'processing_failed'))
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]:
|
||||||
|
context = multiprocessing.get_context('spawn')
|
||||||
|
receiver, sender = context.Pipe(duplex=False)
|
||||||
|
process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes))
|
||||||
|
process.start()
|
||||||
|
sender.close()
|
||||||
|
if not receiver.poll(timeout):
|
||||||
|
process.terminate()
|
||||||
|
process.join(5)
|
||||||
|
receiver.close()
|
||||||
|
raise ProcessingTimeout('processing_timeout')
|
||||||
|
try:
|
||||||
|
kind, result = receiver.recv()
|
||||||
|
except EOFError as error:
|
||||||
|
process.join(5)
|
||||||
|
receiver.close()
|
||||||
|
raise ProcessingTimeout('processing_failed') from error
|
||||||
|
receiver.close()
|
||||||
|
process.join(5)
|
||||||
|
if process.is_alive():
|
||||||
|
process.terminate()
|
||||||
|
process.join(5)
|
||||||
|
raise ProcessingTimeout('processing_failed')
|
||||||
|
if kind == 'ok':
|
||||||
|
return result
|
||||||
|
if kind == 'permanent':
|
||||||
|
raise PermanentImageError(result)
|
||||||
|
raise ProcessingTimeout(result)
|
||||||
|
|
||||||
|
|
||||||
|
class Worker:
|
||||||
|
def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None:
|
||||||
|
self.settings = settings
|
||||||
|
self.jobs = jobs
|
||||||
|
self.storage = storage
|
||||||
|
self.worker_id = f'{socket.gethostname()}:{os.getpid()}'
|
||||||
|
self.semaphore = asyncio.Semaphore(settings.worker_concurrency)
|
||||||
|
self.metrics = get_metrics()
|
||||||
|
self.metric_attributes = {'job.type': 'attachment.process.v1'}
|
||||||
|
self.metrics.worker_busy.set(0, self.metric_attributes)
|
||||||
|
self.metrics.worker_capacity.set(settings.worker_concurrency, self.metric_attributes)
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _busy_slot(self):
|
||||||
|
async with self.semaphore:
|
||||||
|
self.metrics.worker_busy.add(1, self.metric_attributes)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
self.metrics.worker_busy.add(-1, self.metric_attributes)
|
||||||
|
|
||||||
|
def _delay(self, attempt: int) -> float:
|
||||||
|
base = min(
|
||||||
|
self.settings.retry_max_seconds,
|
||||||
|
self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)),
|
||||||
|
)
|
||||||
|
return base + random.uniform(0, base)
|
||||||
|
|
||||||
|
async def _handle_failure(
|
||||||
|
self,
|
||||||
|
incoming: aio_pika.IncomingMessage,
|
||||||
|
job_id: UUID,
|
||||||
|
attempt: int,
|
||||||
|
code: str,
|
||||||
|
permanent: bool,
|
||||||
|
) -> None:
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
available_at = now + timedelta(seconds=self._delay(attempt))
|
||||||
|
try:
|
||||||
|
state = await self.jobs.retry_or_fail(
|
||||||
|
job_id,
|
||||||
|
self.worker_id,
|
||||||
|
code,
|
||||||
|
permanent,
|
||||||
|
available_at,
|
||||||
|
now,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception('failed to persist job failure job_id=%s', job_id)
|
||||||
|
await incoming.nack(requeue=True)
|
||||||
|
return
|
||||||
|
if state == 'failed':
|
||||||
|
logger.error(
|
||||||
|
'job reached terminal failure',
|
||||||
|
extra=safe_extra(
|
||||||
|
'job.failed', job_id=str(job_id), attempt=attempt, error_code=code
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
'job retry scheduled',
|
||||||
|
extra=safe_extra(
|
||||||
|
'job.retry_scheduled', job_id=str(job_id), attempt=attempt,
|
||||||
|
error_code=code,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await incoming.ack()
|
||||||
|
|
||||||
|
async def handle(self, incoming: aio_pika.IncomingMessage) -> None:
|
||||||
|
context = extract_trace_context(incoming.headers)
|
||||||
|
job_id: UUID | None = None
|
||||||
|
before = None
|
||||||
|
try:
|
||||||
|
preview = json.loads(incoming.body)
|
||||||
|
job_id = UUID(preview['job_id'])
|
||||||
|
before = await self.jobs.state(job_id)
|
||||||
|
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||||
|
pass
|
||||||
|
started_at = time.perf_counter()
|
||||||
|
attributes = {
|
||||||
|
'messaging.system': 'rabbitmq',
|
||||||
|
'messaging.destination.name': 'attachments.process',
|
||||||
|
'messaging.operation.name': 'process',
|
||||||
|
'messaging.operation.type': 'process',
|
||||||
|
}
|
||||||
|
if job_id is not None:
|
||||||
|
attributes['job.id'] = str(job_id)
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
'attachments process', context=context, kind=SpanKind.CONSUMER,
|
||||||
|
attributes=attributes,
|
||||||
|
) as span:
|
||||||
|
await self._handle_message(incoming)
|
||||||
|
if before is None or before.state != 'queued' or job_id is None:
|
||||||
|
return
|
||||||
|
after = await self.jobs.state(job_id)
|
||||||
|
if after is None or after.state not in {'queued', 'ready', 'failed'}:
|
||||||
|
return
|
||||||
|
if after.state == 'queued' and after.attempt == before.attempt:
|
||||||
|
return
|
||||||
|
outcome = 'retry' if after.state == 'queued' else after.state
|
||||||
|
duration = time.perf_counter() - started_at
|
||||||
|
metric_attributes = {**self.metric_attributes, 'outcome': outcome}
|
||||||
|
self.metrics.jobs_processed.add(1, metric_attributes)
|
||||||
|
self.metrics.job_duration.record(duration, metric_attributes)
|
||||||
|
span.set_attribute('job.outcome', outcome)
|
||||||
|
span.set_attribute('job.attempt', before.attempt)
|
||||||
|
if outcome == 'ready':
|
||||||
|
ready_seconds = max(
|
||||||
|
0.0, (datetime.now(timezone.utc) - after.created_at).total_seconds()
|
||||||
|
)
|
||||||
|
self.metrics.time_to_ready.record(
|
||||||
|
ready_seconds,
|
||||||
|
{'pipeline.version': str(after.pipeline_version), 'outcome': 'ready'},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _handle_message(self, incoming: aio_pika.IncomingMessage) -> None:
|
||||||
|
async with self._busy_slot():
|
||||||
|
try:
|
||||||
|
payload = json.loads(incoming.body)
|
||||||
|
if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1':
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
return
|
||||||
|
job_id = UUID(payload['job_id'])
|
||||||
|
attachment_id = UUID(payload['attachment_id'])
|
||||||
|
attempt = int(payload['attempt'])
|
||||||
|
pipeline_version = int(payload['pipeline_version'])
|
||||||
|
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
return
|
||||||
|
|
||||||
|
current = await self.jobs.state(job_id)
|
||||||
|
if current is None:
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
return
|
||||||
|
if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version:
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
return
|
||||||
|
if current.state == 'ready':
|
||||||
|
if await self.jobs.ready_effect_complete(job_id):
|
||||||
|
await incoming.ack()
|
||||||
|
else:
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
return
|
||||||
|
if current.state == 'failed':
|
||||||
|
await incoming.reject(requeue=False)
|
||||||
|
return
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
claimed = await self.jobs.claim(
|
||||||
|
job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds
|
||||||
|
)
|
||||||
|
if claimed is None:
|
||||||
|
await incoming.ack()
|
||||||
|
return
|
||||||
|
job, attachment = claimed
|
||||||
|
if os.getenv('FAULT_WORKER_PERMANENT') == '1':
|
||||||
|
await self._handle_failure(
|
||||||
|
incoming, job.id, job.attempt, 'fault_injected', True
|
||||||
|
)
|
||||||
|
os._exit(94)
|
||||||
|
if not attachment.staging_key:
|
||||||
|
await self._handle_failure(
|
||||||
|
incoming, job.id, job.attempt, 'staging_missing', True
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
source = await self.storage.read(
|
||||||
|
attachment.staging_key, self.settings.max_upload_bytes
|
||||||
|
)
|
||||||
|
arguments = (
|
||||||
|
source,
|
||||||
|
attachment.content_type,
|
||||||
|
attachment.sha256,
|
||||||
|
self.settings.max_image_pixels,
|
||||||
|
self.settings.thumbnail_jpeg_quality,
|
||||||
|
self.settings.crop_jpeg_quality,
|
||||||
|
self.settings.optimized_webp_quality,
|
||||||
|
)
|
||||||
|
generated = await asyncio.to_thread(
|
||||||
|
_run_isolated,
|
||||||
|
arguments,
|
||||||
|
self.settings.job_timeout_seconds,
|
||||||
|
512 * 1024 * 1024,
|
||||||
|
)
|
||||||
|
original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original'
|
||||||
|
await self.storage.put_bytes(original_key, source, attachment.content_type)
|
||||||
|
variants: list[AttachmentVariant] = []
|
||||||
|
suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'}
|
||||||
|
for item in generated:
|
||||||
|
key = (
|
||||||
|
f'variants/{attachment.id}/v{job.pipeline_version}/'
|
||||||
|
f'{item.name}.{suffixes[item.name]}'
|
||||||
|
)
|
||||||
|
await self.storage.put_bytes(key, item.content, item.content_type)
|
||||||
|
variants.append(
|
||||||
|
AttachmentVariant(
|
||||||
|
name=item.name,
|
||||||
|
object_key=key,
|
||||||
|
content_type=item.content_type,
|
||||||
|
size_bytes=len(item.content),
|
||||||
|
width=item.width,
|
||||||
|
height=item.height,
|
||||||
|
sha256=item.sha256,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1':
|
||||||
|
os._exit(92)
|
||||||
|
completed = await self.jobs.complete(
|
||||||
|
job.id,
|
||||||
|
self.worker_id,
|
||||||
|
original_key,
|
||||||
|
tuple(variants),
|
||||||
|
datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
if not completed:
|
||||||
|
await incoming.nack(requeue=True)
|
||||||
|
return
|
||||||
|
if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1':
|
||||||
|
os._exit(93)
|
||||||
|
try:
|
||||||
|
await self.storage.remove(attachment.staging_key)
|
||||||
|
except Unavailable:
|
||||||
|
logger.warning(
|
||||||
|
'staging cleanup deferred',
|
||||||
|
extra=safe_extra(
|
||||||
|
'staging.cleanup_deferred', attachment_id=str(attachment.id)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
'job completed',
|
||||||
|
extra=safe_extra(
|
||||||
|
'job.completed', job_id=str(job.id),
|
||||||
|
attachment_id=str(attachment.id), attempt=job.attempt,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await incoming.ack()
|
||||||
|
except PermanentImageError as error:
|
||||||
|
await self._handle_failure(
|
||||||
|
incoming, job.id, job.attempt, error.code, True
|
||||||
|
)
|
||||||
|
except ProcessingTimeout as error:
|
||||||
|
await self._handle_failure(
|
||||||
|
incoming, job.id, job.attempt, str(error), False
|
||||||
|
)
|
||||||
|
except (Unavailable, OSError, ValueError):
|
||||||
|
logger.exception(
|
||||||
|
'temporary processing failure',
|
||||||
|
extra=safe_extra(
|
||||||
|
'job.processing_failed', job_id=str(job.id),
|
||||||
|
error_code='storage_unavailable',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await self._handle_failure(
|
||||||
|
incoming, job.id, job.attempt, 'storage_unavailable', False
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.exception(
|
||||||
|
'unexpected processing failure',
|
||||||
|
extra=safe_extra(
|
||||||
|
'job.processing_failed', job_id=str(job.id),
|
||||||
|
error_code='processing_failed',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
await self._handle_failure(
|
||||||
|
incoming, job.id, job.attempt, 'processing_failed', False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def run() -> None:
|
||||||
|
settings = load_settings()
|
||||||
|
engine, sessions, storage = dependencies(settings)
|
||||||
|
jobs = JobRepository(sessions)
|
||||||
|
worker = Worker(settings, jobs, storage)
|
||||||
|
connection = await connect(settings.broker_url)
|
||||||
|
try:
|
||||||
|
channel = await connection.channel()
|
||||||
|
await channel.set_qos(prefetch_count=settings.worker_prefetch)
|
||||||
|
_exchange, queue = await topology(channel)
|
||||||
|
await queue.consume(worker.handle, no_ack=False)
|
||||||
|
logger.info(
|
||||||
|
'worker started id=%s concurrency=%d prefetch=%d',
|
||||||
|
worker.worker_id,
|
||||||
|
settings.worker_concurrency,
|
||||||
|
settings.worker_prefetch,
|
||||||
|
)
|
||||||
|
await asyncio.Future()
|
||||||
|
finally:
|
||||||
|
await connection.close()
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
telemetry = configure_logging('messenger-worker')
|
||||||
|
try:
|
||||||
|
asyncio.run(run())
|
||||||
|
finally:
|
||||||
|
telemetry.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
+120
@@ -0,0 +1,120 @@
|
|||||||
|
"""Сборка FastAPI приложения и подключение обязательных хранилищ."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||||
|
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from .api import errors
|
||||||
|
from .api.deps import Services
|
||||||
|
from .api.middleware import RequestContext
|
||||||
|
from .api.routers import attachments, auth, chats, health, messages, users
|
||||||
|
from .config import load_settings
|
||||||
|
from .db import RedisSessionStore, create_engine, create_redis, create_session_factory
|
||||||
|
from .observability import get_metrics, initialize_observability
|
||||||
|
from .observability.logging import safe_extra
|
||||||
|
from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository
|
||||||
|
from .services.attachments import AttachmentService
|
||||||
|
from .services.chats import ChatService
|
||||||
|
from .services.messages import MessageService
|
||||||
|
from .services.users import UserService
|
||||||
|
from .storage import S3Storage
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger('app')
|
||||||
|
|
||||||
|
DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.'
|
||||||
|
|
||||||
|
|
||||||
|
def create_app() -> FastAPI:
|
||||||
|
telemetry = initialize_observability('messenger-api')
|
||||||
|
get_metrics()
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(application: FastAPI):
|
||||||
|
application.state.ready = False
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine)
|
||||||
|
redis = create_redis(settings.redis_url)
|
||||||
|
storage = S3Storage(
|
||||||
|
endpoint=settings.s3_endpoint,
|
||||||
|
access_key=settings.s3_access_key_id,
|
||||||
|
secret_key=settings.s3_secret_access_key,
|
||||||
|
bucket=settings.s3_bucket,
|
||||||
|
region=settings.s3_region,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
async with engine.connect() as connection:
|
||||||
|
await connection.execute(text('SELECT 1'))
|
||||||
|
session_store = RedisSessionStore(redis, settings.session_ttl_seconds)
|
||||||
|
await session_store.ping()
|
||||||
|
await storage.ping()
|
||||||
|
session_factory = create_session_factory(engine)
|
||||||
|
user_repository = UserRepository(session_factory)
|
||||||
|
chat_repository = ChatRepository(session_factory)
|
||||||
|
message_repository = MessageRepository(session_factory)
|
||||||
|
attachment_repository = AttachmentRepository(session_factory)
|
||||||
|
application.state.services = Services(
|
||||||
|
attachments=AttachmentService(
|
||||||
|
attachment_repository,
|
||||||
|
storage,
|
||||||
|
settings.max_upload_bytes,
|
||||||
|
settings.max_image_pixels,
|
||||||
|
settings.job_max_attempts,
|
||||||
|
),
|
||||||
|
users=UserService(user_repository),
|
||||||
|
chats=ChatService(chat_repository),
|
||||||
|
messages=MessageService(message_repository),
|
||||||
|
sessions=session_store,
|
||||||
|
)
|
||||||
|
application.state.engine = engine
|
||||||
|
application.state.redis = redis
|
||||||
|
application.state.storage = storage
|
||||||
|
application.state.ready = True
|
||||||
|
logger.info(
|
||||||
|
'application started',
|
||||||
|
extra=safe_extra(
|
||||||
|
'api.startup', storage='postgres+s3', redis_sessions=True,
|
||||||
|
async_attachments=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
application.state.ready = False
|
||||||
|
await redis.aclose()
|
||||||
|
await engine.dispose()
|
||||||
|
logger.info('application stopped', extra=safe_extra('api.shutdown'))
|
||||||
|
telemetry.shutdown()
|
||||||
|
|
||||||
|
api = FastAPI(
|
||||||
|
title='MTUSI Messenger. LR 6',
|
||||||
|
version='1.6.0',
|
||||||
|
description=DESCRIPTION,
|
||||||
|
lifespan=lifespan,
|
||||||
|
)
|
||||||
|
api.router.redirect_slashes = False
|
||||||
|
api.state.ready = False
|
||||||
|
api.add_middleware(RequestContext)
|
||||||
|
errors.install(api)
|
||||||
|
for router in (
|
||||||
|
health.router,
|
||||||
|
users.router,
|
||||||
|
auth.router,
|
||||||
|
chats.router,
|
||||||
|
messages.router,
|
||||||
|
attachments.router,
|
||||||
|
):
|
||||||
|
api.include_router(router)
|
||||||
|
FastAPIInstrumentor.instrument_app(
|
||||||
|
api,
|
||||||
|
excluded_urls='/health/live,/health/ready',
|
||||||
|
)
|
||||||
|
return api
|
||||||
|
|
||||||
|
|
||||||
|
app = create_app()
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow
|
||||||
|
from .chat import ChatMemberRow, ChatRow
|
||||||
|
from .message import MessageRow
|
||||||
|
from .user import UserRow
|
||||||
|
from .job import AttachmentJobRow, JobEventRow, OutboxEventRow
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'AttachmentRow',
|
||||||
|
'AttachmentVariantRow',
|
||||||
|
'AttachmentJobRow',
|
||||||
|
'ChatMemberRow',
|
||||||
|
'ChatRow',
|
||||||
|
'MessageAttachmentRow',
|
||||||
|
'MessageRow',
|
||||||
|
'UserRow',
|
||||||
|
'JobEventRow',
|
||||||
|
'OutboxEventRow',
|
||||||
|
]
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import Uuid
|
||||||
|
|
||||||
|
from ..db.postgres import Base
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentRow(Base):
|
||||||
|
__tablename__ = 'attachments'
|
||||||
|
__table_args__ = (
|
||||||
|
Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'),
|
||||||
|
UniqueConstraint('original_key', name='attachments_object_key_key'),
|
||||||
|
UniqueConstraint('staging_key', name='attachments_staging_key_key'),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||||
|
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||||
|
owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||||
|
filename: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
original_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||||
|
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||||
|
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
status: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentVariantRow(Base):
|
||||||
|
__tablename__ = 'attachment_variants'
|
||||||
|
|
||||||
|
attachment_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
|
||||||
|
)
|
||||||
|
pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
name: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||||
|
object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||||
|
content_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||||
|
width: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
height: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class MessageAttachmentRow(Base):
|
||||||
|
__tablename__ = 'message_attachments'
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'),
|
||||||
|
Index('message_attachments_attachment_idx', 'attachment_id'),
|
||||||
|
)
|
||||||
|
|
||||||
|
message_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True
|
||||||
|
)
|
||||||
|
attachment_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True
|
||||||
|
)
|
||||||
|
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, ForeignKey, Index, String
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import Uuid
|
||||||
|
|
||||||
|
from ..db.postgres import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ChatRow(Base):
|
||||||
|
__tablename__ = 'chats'
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||||
|
title: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class ChatMemberRow(Base):
|
||||||
|
__tablename__ = 'chat_members'
|
||||||
|
__table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),)
|
||||||
|
|
||||||
|
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||||
|
user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True)
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import Uuid
|
||||||
|
|
||||||
|
from ..db.postgres import Base
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentJobRow(Base):
|
||||||
|
__tablename__ = 'attachment_jobs'
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key'
|
||||||
|
),
|
||||||
|
Index('attachment_jobs_lease_idx', 'state', 'lease_until'),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||||
|
attachment_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False
|
||||||
|
)
|
||||||
|
pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
state: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class OutboxEventRow(Base):
|
||||||
|
__tablename__ = 'outbox_events'
|
||||||
|
__table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||||
|
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
job_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
|
||||||
|
)
|
||||||
|
payload: Mapped[dict] = mapped_column(JSON, nullable=False)
|
||||||
|
traceparent: Mapped[str | None] = mapped_column(String(55), nullable=True)
|
||||||
|
tracestate: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||||
|
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
|
claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class JobEventRow(Base):
|
||||||
|
__tablename__ = 'job_events'
|
||||||
|
__table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
|
||||||
|
job_id: Mapped[UUID] = mapped_column(
|
||||||
|
ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False
|
||||||
|
)
|
||||||
|
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
attempt: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import Uuid
|
||||||
|
|
||||||
|
from ..db.postgres import Base
|
||||||
|
|
||||||
|
|
||||||
|
class MessageRow(Base):
|
||||||
|
__tablename__ = 'messages'
|
||||||
|
__table_args__ = (
|
||||||
|
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||||
|
UniqueConstraint(
|
||||||
|
'sender_id',
|
||||||
|
'chat_id',
|
||||||
|
'idempotency_key',
|
||||||
|
name='messages_sender_chat_idempotency_key',
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||||
|
chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False)
|
||||||
|
sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False)
|
||||||
|
text: Mapped[str] = mapped_column(String(4000), nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, String, Text
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import Uuid
|
||||||
|
|
||||||
|
from ..db.postgres import Base
|
||||||
|
|
||||||
|
|
||||||
|
class UserRow(Base):
|
||||||
|
__tablename__ = 'users'
|
||||||
|
|
||||||
|
id: Mapped[UUID] = mapped_column(Uuid, primary_key=True)
|
||||||
|
username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False)
|
||||||
|
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||||
|
password_hash: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
from .metrics import Metrics, get_metrics
|
||||||
|
from .runtime import ObservabilityRuntime, initialize_observability
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'Metrics',
|
||||||
|
'ObservabilityRuntime',
|
||||||
|
'get_metrics',
|
||||||
|
'initialize_observability',
|
||||||
|
]
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
def _positive_int(name: str, default: str) -> int:
|
||||||
|
try:
|
||||||
|
value = int(os.getenv(name, default))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f'{name} должен быть целым числом') from exc
|
||||||
|
if value <= 0:
|
||||||
|
raise RuntimeError(f'{name} должен быть больше нуля')
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ObservabilitySettings:
|
||||||
|
enabled: bool
|
||||||
|
endpoint: str
|
||||||
|
service_name: str
|
||||||
|
service_version: str
|
||||||
|
environment: str
|
||||||
|
export_timeout_seconds: int
|
||||||
|
span_queue_size: int
|
||||||
|
log_queue_size: int
|
||||||
|
metric_export_interval_ms: int
|
||||||
|
|
||||||
|
|
||||||
|
def load_observability_settings(default_service_name: str) -> ObservabilitySettings:
|
||||||
|
disabled = os.getenv('OTEL_SDK_DISABLED', 'false').strip().lower()
|
||||||
|
if disabled not in {'true', 'false'}:
|
||||||
|
raise RuntimeError('OTEL_SDK_DISABLED должен быть true или false')
|
||||||
|
endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318')
|
||||||
|
return ObservabilitySettings(
|
||||||
|
enabled=disabled != 'true',
|
||||||
|
endpoint=endpoint.rstrip('/'),
|
||||||
|
service_name=os.getenv('OTEL_SERVICE_NAME', default_service_name),
|
||||||
|
service_version=os.getenv('OTEL_SERVICE_VERSION', '1.6.0'),
|
||||||
|
environment=os.getenv('OTEL_DEPLOYMENT_ENVIRONMENT', 'lab'),
|
||||||
|
export_timeout_seconds=_positive_int('OTEL_EXPORTER_OTLP_TIMEOUT', '2'),
|
||||||
|
span_queue_size=_positive_int('OTEL_BSP_MAX_QUEUE_SIZE', '2048'),
|
||||||
|
log_queue_size=_positive_int('OTEL_BLRP_MAX_QUEUE_SIZE', '2048'),
|
||||||
|
metric_export_interval_ms=_positive_int('OTEL_METRIC_EXPORT_INTERVAL', '5000'),
|
||||||
|
)
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from opentelemetry import trace
|
||||||
|
|
||||||
|
|
||||||
|
_STANDARD_FIELDS = {
|
||||||
|
'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 'funcName',
|
||||||
|
'levelname', 'levelno', 'lineno', 'module', 'msecs', 'message', 'msg', 'name',
|
||||||
|
'pathname', 'process', 'processName', 'relativeCreated', 'stack_info', 'thread',
|
||||||
|
'threadName', 'taskName',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class JsonFormatter(logging.Formatter):
|
||||||
|
def format(self, record: logging.LogRecord) -> str:
|
||||||
|
payload: dict[str, object] = {
|
||||||
|
'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
|
||||||
|
'severity': record.levelname,
|
||||||
|
'logger': record.name,
|
||||||
|
'message': record.getMessage(),
|
||||||
|
'event': getattr(record, 'event', 'log.message'),
|
||||||
|
}
|
||||||
|
span_context = trace.get_current_span().get_span_context()
|
||||||
|
if span_context.is_valid:
|
||||||
|
payload['trace_id'] = format(span_context.trace_id, '032x')
|
||||||
|
payload['span_id'] = format(span_context.span_id, '016x')
|
||||||
|
for key, value in record.__dict__.items():
|
||||||
|
if key in _STANDARD_FIELDS or key.startswith('_'):
|
||||||
|
continue
|
||||||
|
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||||
|
payload[key] = value
|
||||||
|
if record.exc_info:
|
||||||
|
payload['exception.type'] = record.exc_info[0].__name__
|
||||||
|
return json.dumps(payload, ensure_ascii=False, separators=(',', ':'))
|
||||||
|
|
||||||
|
|
||||||
|
def safe_extra(event: str, **attributes: object) -> dict[str, object]:
|
||||||
|
blocked = {
|
||||||
|
'authorization', 'password', 'token', 'session_token', 'message_body',
|
||||||
|
'image_bytes', 's3_secret_access_key', 'signed_url',
|
||||||
|
}
|
||||||
|
extra: dict[str, object] = {'event': event}
|
||||||
|
for key, value in attributes.items():
|
||||||
|
normalized = key.lower()
|
||||||
|
if normalized in blocked or any(part in normalized for part in ('password', 'secret')):
|
||||||
|
continue
|
||||||
|
extra[key] = value
|
||||||
|
return extra
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Mapping
|
||||||
|
|
||||||
|
import psutil
|
||||||
|
from opentelemetry import metrics
|
||||||
|
from opentelemetry.metrics import Observation
|
||||||
|
|
||||||
|
|
||||||
|
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0)
|
||||||
|
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0)
|
||||||
|
READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0)
|
||||||
|
|
||||||
|
|
||||||
|
class GaugeState:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._values: dict[tuple[tuple[str, str], ...], float] = {}
|
||||||
|
|
||||||
|
def set(self, value: float, attributes: Mapping[str, str] | None = None) -> None:
|
||||||
|
key = tuple(sorted((attributes or {}).items()))
|
||||||
|
with self._lock:
|
||||||
|
self._values[key] = float(value)
|
||||||
|
|
||||||
|
def add(self, amount: float, attributes: Mapping[str, str] | None = None) -> None:
|
||||||
|
key = tuple(sorted((attributes or {}).items()))
|
||||||
|
with self._lock:
|
||||||
|
self._values[key] = self._values.get(key, 0.0) + amount
|
||||||
|
|
||||||
|
def observations(self, _options=None):
|
||||||
|
with self._lock:
|
||||||
|
snapshot = tuple(self._values.items())
|
||||||
|
return [Observation(value, dict(attributes)) for attributes, value in snapshot]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class Metrics:
|
||||||
|
http_requests: object
|
||||||
|
http_duration: object
|
||||||
|
jobs_processed: object
|
||||||
|
job_duration: object
|
||||||
|
time_to_ready: object
|
||||||
|
queue_depth: GaugeState
|
||||||
|
queue_oldest_age: GaugeState
|
||||||
|
worker_busy: GaugeState
|
||||||
|
worker_capacity: GaugeState
|
||||||
|
db_pool_used: GaugeState
|
||||||
|
db_pool_waiting: GaugeState
|
||||||
|
|
||||||
|
|
||||||
|
_metrics: Metrics | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _process_rss(_options=None):
|
||||||
|
process = psutil.Process(os.getpid())
|
||||||
|
return [Observation(process.memory_info().rss)]
|
||||||
|
|
||||||
|
|
||||||
|
def _process_cpu(_options=None):
|
||||||
|
process = psutil.Process(os.getpid())
|
||||||
|
times = process.cpu_times()
|
||||||
|
return [Observation(times.user + times.system)]
|
||||||
|
|
||||||
|
|
||||||
|
def get_metrics() -> Metrics:
|
||||||
|
global _metrics
|
||||||
|
if _metrics is not None:
|
||||||
|
return _metrics
|
||||||
|
meter = metrics.get_meter('messenger.observability', '1.6.0')
|
||||||
|
queue_depth = GaugeState()
|
||||||
|
queue_oldest_age = GaugeState()
|
||||||
|
worker_busy = GaugeState()
|
||||||
|
worker_capacity = GaugeState()
|
||||||
|
db_pool_used = GaugeState()
|
||||||
|
db_pool_waiting = GaugeState()
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
'messenger.queue.depth', callbacks=[queue_depth.observations], unit='{job}'
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
'messenger.queue.oldest.age', callbacks=[queue_oldest_age.observations], unit='s'
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
'messenger.worker.busy', callbacks=[worker_busy.observations], unit='{worker}'
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
'messenger.worker.capacity', callbacks=[worker_capacity.observations], unit='{worker}'
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
'messenger.db.pool.used', callbacks=[db_pool_used.observations], unit='{connection}'
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge(
|
||||||
|
'messenger.db.pool.waiting', callbacks=[db_pool_waiting.observations], unit='{request}'
|
||||||
|
)
|
||||||
|
meter.create_observable_gauge('process.memory.rss', callbacks=[_process_rss], unit='By')
|
||||||
|
meter.create_observable_counter('process.cpu.time', callbacks=[_process_cpu], unit='s')
|
||||||
|
_metrics = Metrics(
|
||||||
|
http_requests=meter.create_counter(
|
||||||
|
'http.server.request.count', unit='{request}', description='HTTP requests'
|
||||||
|
),
|
||||||
|
http_duration=meter.create_histogram(
|
||||||
|
'http.server.request.duration', unit='s', description='HTTP request duration'
|
||||||
|
),
|
||||||
|
jobs_processed=meter.create_counter(
|
||||||
|
'messenger.jobs.processed', unit='{job}', description='Finished job attempts'
|
||||||
|
),
|
||||||
|
job_duration=meter.create_histogram(
|
||||||
|
'messenger.job.processing.duration', unit='s', description='Job processing duration'
|
||||||
|
),
|
||||||
|
time_to_ready=meter.create_histogram(
|
||||||
|
'messenger.attachment.time_to_ready', unit='s', description='Durable accept to ready'
|
||||||
|
),
|
||||||
|
queue_depth=queue_depth,
|
||||||
|
queue_oldest_age=queue_oldest_age,
|
||||||
|
worker_busy=worker_busy,
|
||||||
|
worker_capacity=worker_capacity,
|
||||||
|
db_pool_used=db_pool_used,
|
||||||
|
db_pool_waiting=db_pool_waiting,
|
||||||
|
)
|
||||||
|
return _metrics
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
|
||||||
|
from opentelemetry import propagate
|
||||||
|
|
||||||
|
|
||||||
|
_ALLOWED = {'traceparent', 'tracestate'}
|
||||||
|
|
||||||
|
|
||||||
|
def inject_trace_context(context=None) -> dict[str, str]:
|
||||||
|
carrier: dict[str, str] = {}
|
||||||
|
propagate.inject(carrier, context=context)
|
||||||
|
return {key: str(value) for key, value in carrier.items() if key in _ALLOWED}
|
||||||
|
|
||||||
|
|
||||||
|
def extract_trace_context(carrier: Mapping[str, object] | None):
|
||||||
|
normalized: dict[str, str] = {}
|
||||||
|
for key, value in (carrier or {}).items():
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if lowered not in _ALLOWED or value is None:
|
||||||
|
continue
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
value = value.decode('ascii', errors='ignore')
|
||||||
|
normalized[lowered] = str(value)
|
||||||
|
return propagate.extract(normalized)
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from opentelemetry import metrics, trace
|
||||||
|
from opentelemetry._logs import set_logger_provider
|
||||||
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||||
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||||
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||||
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||||
|
from opentelemetry.sdk.metrics import MeterProvider
|
||||||
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||||
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||||||
|
from opentelemetry.sdk.resources import Resource
|
||||||
|
from opentelemetry.sdk.trace import TracerProvider
|
||||||
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||||
|
|
||||||
|
from .config import load_observability_settings
|
||||||
|
from .logging import JsonFormatter
|
||||||
|
from .metrics import HTTP_BUCKETS, JOB_BUCKETS, READY_BUCKETS
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ObservabilityRuntime:
|
||||||
|
tracer_provider: TracerProvider | None = None
|
||||||
|
meter_provider: MeterProvider | None = None
|
||||||
|
logger_provider: LoggerProvider | None = None
|
||||||
|
|
||||||
|
def shutdown(self) -> None:
|
||||||
|
if self.logger_provider is not None:
|
||||||
|
self.logger_provider.shutdown()
|
||||||
|
if self.meter_provider is not None:
|
||||||
|
self.meter_provider.shutdown()
|
||||||
|
if self.tracer_provider is not None:
|
||||||
|
self.tracer_provider.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
_runtime: ObservabilityRuntime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def initialize_observability(default_service_name: str) -> ObservabilityRuntime:
|
||||||
|
global _runtime
|
||||||
|
if _runtime is not None:
|
||||||
|
return _runtime
|
||||||
|
settings = load_observability_settings(default_service_name)
|
||||||
|
console = logging.StreamHandler(sys.stdout)
|
||||||
|
console.setFormatter(JsonFormatter())
|
||||||
|
handlers: list[logging.Handler] = [console]
|
||||||
|
runtime = ObservabilityRuntime()
|
||||||
|
if settings.enabled:
|
||||||
|
resource = Resource.create({
|
||||||
|
'service.name': settings.service_name,
|
||||||
|
'service.version': settings.service_version,
|
||||||
|
'service.instance.id': os.getenv(
|
||||||
|
'INSTANCE_ID', f'{socket.gethostname()}:{os.getpid()}'
|
||||||
|
),
|
||||||
|
'deployment.environment.name': settings.environment,
|
||||||
|
})
|
||||||
|
timeout = settings.export_timeout_seconds
|
||||||
|
tracer_provider = TracerProvider(resource=resource)
|
||||||
|
tracer_provider.add_span_processor(BatchSpanProcessor(
|
||||||
|
OTLPSpanExporter(endpoint=f'{settings.endpoint}/v1/traces', timeout=timeout),
|
||||||
|
max_queue_size=settings.span_queue_size,
|
||||||
|
max_export_batch_size=min(512, settings.span_queue_size),
|
||||||
|
schedule_delay_millis=1000,
|
||||||
|
export_timeout_millis=timeout * 1000,
|
||||||
|
))
|
||||||
|
trace.set_tracer_provider(tracer_provider)
|
||||||
|
|
||||||
|
metric_reader = PeriodicExportingMetricReader(
|
||||||
|
OTLPMetricExporter(endpoint=f'{settings.endpoint}/v1/metrics', timeout=timeout),
|
||||||
|
export_interval_millis=settings.metric_export_interval_ms,
|
||||||
|
export_timeout_millis=timeout * 1000,
|
||||||
|
)
|
||||||
|
meter_provider = MeterProvider(
|
||||||
|
resource=resource,
|
||||||
|
metric_readers=[metric_reader],
|
||||||
|
views=[
|
||||||
|
View(
|
||||||
|
instrument_name='http.server.request.duration',
|
||||||
|
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
|
||||||
|
),
|
||||||
|
View(
|
||||||
|
instrument_name='messenger.job.processing.duration',
|
||||||
|
aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
|
||||||
|
),
|
||||||
|
View(
|
||||||
|
instrument_name='messenger.attachment.time_to_ready',
|
||||||
|
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
metrics.set_meter_provider(meter_provider)
|
||||||
|
|
||||||
|
logger_provider = LoggerProvider(resource=resource)
|
||||||
|
logger_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||||
|
OTLPLogExporter(endpoint=f'{settings.endpoint}/v1/logs', timeout=timeout),
|
||||||
|
max_queue_size=settings.log_queue_size,
|
||||||
|
max_export_batch_size=min(512, settings.log_queue_size),
|
||||||
|
schedule_delay_millis=1000,
|
||||||
|
export_timeout_millis=timeout * 1000,
|
||||||
|
))
|
||||||
|
set_logger_provider(logger_provider)
|
||||||
|
otlp_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider)
|
||||||
|
otlp_handler.setFormatter(JsonFormatter())
|
||||||
|
handlers.append(otlp_handler)
|
||||||
|
runtime = ObservabilityRuntime(tracer_provider, meter_provider, logger_provider)
|
||||||
|
logging.basicConfig(
|
||||||
|
level=os.getenv('LOG_LEVEL', 'info').upper(), handlers=handlers, force=True
|
||||||
|
)
|
||||||
|
_runtime = runtime
|
||||||
|
return runtime
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
from .attachments import AttachmentRepository
|
||||||
|
from .chats import ChatRepository
|
||||||
|
from .messages import MessageRepository
|
||||||
|
from .users import UserRepository
|
||||||
|
from .jobs import JobRepository, OutboxRepository
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository',
|
||||||
|
'OutboxRepository', 'UserRepository',
|
||||||
|
]
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from ..domain.models import Attachment, AttachmentVariant
|
||||||
|
from ..models.attachment import AttachmentRow, AttachmentVariantRow
|
||||||
|
from ..models.chat import ChatMemberRow, ChatRow
|
||||||
|
from ..models.job import AttachmentJobRow, OutboxEventRow
|
||||||
|
|
||||||
|
|
||||||
|
def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict:
|
||||||
|
return {
|
||||||
|
'schema_version': 1,
|
||||||
|
'job_id': str(job_id),
|
||||||
|
'type': 'attachment.process.v1',
|
||||||
|
'attachment_id': str(attachment_id),
|
||||||
|
'pipeline_version': pipeline_version,
|
||||||
|
'attempt': attempt,
|
||||||
|
'created_at': now.isoformat().replace('+00:00', 'Z'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentRepository:
|
||||||
|
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||||
|
self._sessions = sessions
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _variant(row: AttachmentVariantRow) -> AttachmentVariant:
|
||||||
|
return AttachmentVariant(
|
||||||
|
name=row.name,
|
||||||
|
object_key=row.object_key,
|
||||||
|
content_type=row.content_type,
|
||||||
|
size_bytes=row.size_bytes,
|
||||||
|
width=row.width,
|
||||||
|
height=row.height,
|
||||||
|
sha256=row.sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _to_domain(
|
||||||
|
cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = ()
|
||||||
|
) -> Attachment:
|
||||||
|
return Attachment(
|
||||||
|
id=row.id,
|
||||||
|
chat_id=row.chat_id,
|
||||||
|
owner_id=row.owner_id,
|
||||||
|
filename=row.filename,
|
||||||
|
original_key=row.original_key,
|
||||||
|
staging_key=row.staging_key,
|
||||||
|
pipeline_version=row.pipeline_version,
|
||||||
|
content_type=row.content_type,
|
||||||
|
size_bytes=row.size_bytes,
|
||||||
|
sha256=row.sha256,
|
||||||
|
width=row.width,
|
||||||
|
height=row.height,
|
||||||
|
status=row.status,
|
||||||
|
error_code=row.error_code,
|
||||||
|
created_at=row.created_at,
|
||||||
|
updated_at=row.updated_at,
|
||||||
|
variants=tuple(cls._variant(item) for item in variants),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
found = await session.scalar(
|
||||||
|
select(ChatRow.id)
|
||||||
|
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||||
|
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||||
|
)
|
||||||
|
return found is not None
|
||||||
|
|
||||||
|
async def create_queued(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
attachment_id: UUID,
|
||||||
|
job_id: UUID,
|
||||||
|
chat_id: UUID,
|
||||||
|
owner_id: UUID,
|
||||||
|
filename: str,
|
||||||
|
staging_key: str,
|
||||||
|
content_type: str,
|
||||||
|
size_bytes: int,
|
||||||
|
sha256: str,
|
||||||
|
width: int,
|
||||||
|
height: int,
|
||||||
|
max_attempts: int,
|
||||||
|
now: datetime,
|
||||||
|
traceparent: str | None = None,
|
||||||
|
tracestate: str | None = None,
|
||||||
|
) -> Attachment:
|
||||||
|
row = AttachmentRow(
|
||||||
|
id=attachment_id,
|
||||||
|
chat_id=chat_id,
|
||||||
|
owner_id=owner_id,
|
||||||
|
filename=filename,
|
||||||
|
original_key=None,
|
||||||
|
staging_key=staging_key,
|
||||||
|
pipeline_version=1,
|
||||||
|
content_type=content_type,
|
||||||
|
size_bytes=size_bytes,
|
||||||
|
sha256=sha256,
|
||||||
|
width=width,
|
||||||
|
height=height,
|
||||||
|
status='queued',
|
||||||
|
error_code=None,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
)
|
||||||
|
job = AttachmentJobRow(
|
||||||
|
id=job_id,
|
||||||
|
attachment_id=attachment_id,
|
||||||
|
pipeline_version=1,
|
||||||
|
state='queued',
|
||||||
|
attempt=1,
|
||||||
|
max_attempts=max_attempts,
|
||||||
|
next_attempt_at=now,
|
||||||
|
lease_owner=None,
|
||||||
|
lease_until=None,
|
||||||
|
last_error_code=None,
|
||||||
|
replay_count=0,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
)
|
||||||
|
outbox = OutboxEventRow(
|
||||||
|
id=uuid4(),
|
||||||
|
event_type='attachment.process.v1',
|
||||||
|
job_id=job_id,
|
||||||
|
payload=job_payload(job_id, attachment_id, 1, 1, now),
|
||||||
|
traceparent=traceparent,
|
||||||
|
tracestate=tracestate,
|
||||||
|
available_at=now,
|
||||||
|
published_at=None,
|
||||||
|
claim_owner=None,
|
||||||
|
claim_until=None,
|
||||||
|
publish_attempts=0,
|
||||||
|
last_error_code=None,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
)
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
session.add_all((row, job, outbox))
|
||||||
|
await session.flush()
|
||||||
|
return self._to_domain(row)
|
||||||
|
|
||||||
|
async def _variants(
|
||||||
|
self, session: AsyncSession, attachment_id: UUID, pipeline_version: int
|
||||||
|
) -> tuple[AttachmentVariantRow, ...]:
|
||||||
|
rows = await session.scalars(
|
||||||
|
select(AttachmentVariantRow)
|
||||||
|
.where(
|
||||||
|
AttachmentVariantRow.attachment_id == attachment_id,
|
||||||
|
AttachmentVariantRow.pipeline_version == pipeline_version,
|
||||||
|
)
|
||||||
|
.order_by(AttachmentVariantRow.name)
|
||||||
|
)
|
||||||
|
return tuple(rows)
|
||||||
|
|
||||||
|
async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
row = await session.scalar(
|
||||||
|
select(AttachmentRow)
|
||||||
|
.join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id)
|
||||||
|
.where(
|
||||||
|
AttachmentRow.id == attachment_id,
|
||||||
|
ChatMemberRow.user_id == user_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
return self._to_domain(
|
||||||
|
row, await self._variants(session, row.id, row.pipeline_version)
|
||||||
|
)
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Sequence
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, tuple_
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from ..domain.errors import Invalid
|
||||||
|
from ..domain.models import Chat, Keyset
|
||||||
|
from ..models.chat import ChatMemberRow, ChatRow
|
||||||
|
from ..models.user import UserRow
|
||||||
|
|
||||||
|
|
||||||
|
class ChatRepository:
|
||||||
|
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||||
|
self._sessions = sessions
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat:
|
||||||
|
return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]:
|
||||||
|
result = await session.scalars(
|
||||||
|
select(ChatMemberRow.user_id)
|
||||||
|
.where(ChatMemberRow.chat_id == chat_id)
|
||||||
|
.order_by(ChatMemberRow.user_id)
|
||||||
|
)
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids))))
|
||||||
|
if len(found) != len(member_ids):
|
||||||
|
raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}')
|
||||||
|
row = ChatRow(id=chat_id, title=title, created_at=created_at)
|
||||||
|
session.add(row)
|
||||||
|
await session.flush()
|
||||||
|
session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids)
|
||||||
|
await session.flush()
|
||||||
|
return self._to_domain(row, member_ids)
|
||||||
|
|
||||||
|
async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
row = await session.scalar(
|
||||||
|
select(ChatRow)
|
||||||
|
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||||
|
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
return self._to_domain(row, await self._members(session, row.id))
|
||||||
|
|
||||||
|
async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
query = (
|
||||||
|
select(ChatRow)
|
||||||
|
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||||
|
.where(ChatMemberRow.user_id == user_id)
|
||||||
|
.order_by(ChatRow.created_at, ChatRow.id)
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
if after is not None:
|
||||||
|
query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id))
|
||||||
|
rows = list((await session.scalars(query)).all())
|
||||||
|
return [self._to_domain(row, await self._members(session, row.id)) for row in rows]
|
||||||
@@ -0,0 +1,388 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import func, or_, select, update
|
||||||
|
from sqlalchemy.dialects.postgresql import insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent
|
||||||
|
from ..models.attachment import AttachmentRow, AttachmentVariantRow
|
||||||
|
from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow
|
||||||
|
from ..observability.propagation import inject_trace_context
|
||||||
|
from .attachments import AttachmentRepository, job_payload
|
||||||
|
|
||||||
|
|
||||||
|
class JobRepository:
|
||||||
|
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||||
|
self._sessions = sessions
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _job(row: AttachmentJobRow) -> AttachmentJob:
|
||||||
|
return AttachmentJob(
|
||||||
|
id=row.id,
|
||||||
|
attachment_id=row.attachment_id,
|
||||||
|
pipeline_version=row.pipeline_version,
|
||||||
|
state=row.state,
|
||||||
|
attempt=row.attempt,
|
||||||
|
max_attempts=row.max_attempts,
|
||||||
|
next_attempt_at=row.next_attempt_at,
|
||||||
|
lease_owner=row.lease_owner,
|
||||||
|
lease_until=row.lease_until,
|
||||||
|
last_error_code=row.last_error_code,
|
||||||
|
replay_count=row.replay_count,
|
||||||
|
created_at=row.created_at,
|
||||||
|
updated_at=row.updated_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def claim(
|
||||||
|
self,
|
||||||
|
job_id: UUID,
|
||||||
|
attempt: int,
|
||||||
|
worker_id: str,
|
||||||
|
now: datetime,
|
||||||
|
lease_seconds: int,
|
||||||
|
) -> tuple[AttachmentJob, Attachment] | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
row = await session.scalar(
|
||||||
|
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||||
|
)
|
||||||
|
if row is None or row.state != 'queued' or row.attempt != attempt:
|
||||||
|
return None
|
||||||
|
attachment_row = await session.get(AttachmentRow, row.attachment_id)
|
||||||
|
if attachment_row is None:
|
||||||
|
return None
|
||||||
|
row.state = 'processing'
|
||||||
|
row.lease_owner = worker_id
|
||||||
|
row.lease_until = now + timedelta(seconds=lease_seconds)
|
||||||
|
row.updated_at = now
|
||||||
|
attachment_row.status = 'processing'
|
||||||
|
attachment_row.updated_at = now
|
||||||
|
session.add(
|
||||||
|
JobEventRow(
|
||||||
|
job_id=row.id,
|
||||||
|
kind='claimed',
|
||||||
|
attempt=row.attempt,
|
||||||
|
error_code=None,
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
return self._job(row), AttachmentRepository._to_domain(attachment_row)
|
||||||
|
|
||||||
|
async def state(self, job_id: UUID) -> AttachmentJob | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
row = await session.get(AttachmentJobRow, job_id)
|
||||||
|
return self._job(row) if row is not None else None
|
||||||
|
|
||||||
|
async def id_for_attachment(self, attachment_id: UUID) -> UUID | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
return await session.scalar(
|
||||||
|
select(AttachmentJobRow.id).where(
|
||||||
|
AttachmentJobRow.attachment_id == attachment_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def failed_staging_key(self, job_id: UUID) -> str | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
return await session.scalar(
|
||||||
|
select(AttachmentRow.staging_key)
|
||||||
|
.join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id)
|
||||||
|
.where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed')
|
||||||
|
)
|
||||||
|
|
||||||
|
async def ready_effect_complete(self, job_id: UUID) -> bool:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
job = await session.get(AttachmentJobRow, job_id)
|
||||||
|
if job is None or job.state != 'ready':
|
||||||
|
return False
|
||||||
|
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||||
|
if attachment is None or not attachment.original_key:
|
||||||
|
return False
|
||||||
|
names = set(
|
||||||
|
await session.scalars(
|
||||||
|
select(AttachmentVariantRow.name).where(
|
||||||
|
AttachmentVariantRow.attachment_id == job.attachment_id,
|
||||||
|
AttachmentVariantRow.pipeline_version == job.pipeline_version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return names == {'thumbnail', 'crop', 'optimized'}
|
||||||
|
|
||||||
|
async def complete(
|
||||||
|
self,
|
||||||
|
job_id: UUID,
|
||||||
|
worker_id: str,
|
||||||
|
original_key: str,
|
||||||
|
variants: tuple[AttachmentVariant, ...],
|
||||||
|
now: datetime,
|
||||||
|
) -> bool:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
job = await session.scalar(
|
||||||
|
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||||
|
)
|
||||||
|
if job is None or job.state == 'ready':
|
||||||
|
return job is not None
|
||||||
|
if job.state != 'processing' or job.lease_owner != worker_id:
|
||||||
|
return False
|
||||||
|
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||||
|
if attachment is None:
|
||||||
|
return False
|
||||||
|
for variant in variants:
|
||||||
|
statement = (
|
||||||
|
insert(AttachmentVariantRow)
|
||||||
|
.values(
|
||||||
|
attachment_id=attachment.id,
|
||||||
|
pipeline_version=job.pipeline_version,
|
||||||
|
name=variant.name,
|
||||||
|
object_key=variant.object_key,
|
||||||
|
content_type=variant.content_type,
|
||||||
|
size_bytes=variant.size_bytes,
|
||||||
|
width=variant.width,
|
||||||
|
height=variant.height,
|
||||||
|
sha256=variant.sha256,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
)
|
||||||
|
.on_conflict_do_update(
|
||||||
|
index_elements=[
|
||||||
|
AttachmentVariantRow.attachment_id,
|
||||||
|
AttachmentVariantRow.pipeline_version,
|
||||||
|
AttachmentVariantRow.name,
|
||||||
|
],
|
||||||
|
set_={
|
||||||
|
'object_key': variant.object_key,
|
||||||
|
'content_type': variant.content_type,
|
||||||
|
'size_bytes': variant.size_bytes,
|
||||||
|
'width': variant.width,
|
||||||
|
'height': variant.height,
|
||||||
|
'sha256': variant.sha256,
|
||||||
|
'updated_at': now,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.execute(statement)
|
||||||
|
attachment.original_key = original_key
|
||||||
|
attachment.status = 'ready'
|
||||||
|
attachment.error_code = None
|
||||||
|
attachment.updated_at = now
|
||||||
|
job.state = 'ready'
|
||||||
|
job.lease_owner = None
|
||||||
|
job.lease_until = None
|
||||||
|
job.last_error_code = None
|
||||||
|
job.updated_at = now
|
||||||
|
session.add(
|
||||||
|
JobEventRow(
|
||||||
|
job_id=job.id,
|
||||||
|
kind='ready',
|
||||||
|
attempt=job.attempt,
|
||||||
|
error_code=None,
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def retry_or_fail(
|
||||||
|
self,
|
||||||
|
job_id: UUID,
|
||||||
|
worker_id: str | None,
|
||||||
|
error_code: str,
|
||||||
|
permanent: bool,
|
||||||
|
available_at: datetime,
|
||||||
|
now: datetime,
|
||||||
|
) -> str:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
job = await session.scalar(
|
||||||
|
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||||
|
)
|
||||||
|
if job is None or job.state in {'ready', 'failed'}:
|
||||||
|
return job.state if job is not None else 'missing'
|
||||||
|
if job.state != 'processing':
|
||||||
|
return job.state
|
||||||
|
if worker_id is not None:
|
||||||
|
if job.lease_owner != worker_id:
|
||||||
|
return job.state
|
||||||
|
elif job.lease_until is None or job.lease_until >= now:
|
||||||
|
return job.state
|
||||||
|
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||||
|
exhausted = job.attempt >= job.max_attempts
|
||||||
|
if permanent or exhausted:
|
||||||
|
job.state = 'failed'
|
||||||
|
attachment.status = 'failed'
|
||||||
|
attachment.error_code = error_code if permanent else 'retry_exhausted'
|
||||||
|
kind = 'failed'
|
||||||
|
else:
|
||||||
|
job.attempt += 1
|
||||||
|
job.state = 'queued'
|
||||||
|
job.next_attempt_at = available_at
|
||||||
|
attachment.status = 'queued'
|
||||||
|
attachment.error_code = None
|
||||||
|
trace_context = inject_trace_context()
|
||||||
|
session.add(
|
||||||
|
OutboxEventRow(
|
||||||
|
id=uuid4(),
|
||||||
|
event_type='attachment.process.v1',
|
||||||
|
job_id=job.id,
|
||||||
|
payload=job_payload(
|
||||||
|
job.id,
|
||||||
|
job.attachment_id,
|
||||||
|
job.pipeline_version,
|
||||||
|
job.attempt,
|
||||||
|
now,
|
||||||
|
),
|
||||||
|
traceparent=trace_context.get('traceparent'),
|
||||||
|
tracestate=trace_context.get('tracestate'),
|
||||||
|
available_at=available_at,
|
||||||
|
published_at=None,
|
||||||
|
claim_owner=None,
|
||||||
|
claim_until=None,
|
||||||
|
publish_attempts=0,
|
||||||
|
last_error_code=None,
|
||||||
|
created_at=now,
|
||||||
|
updated_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
kind = 'retry_scheduled'
|
||||||
|
job.last_error_code = error_code
|
||||||
|
job.lease_owner = None
|
||||||
|
job.lease_until = None
|
||||||
|
job.updated_at = now
|
||||||
|
attachment.updated_at = now
|
||||||
|
session.add(
|
||||||
|
JobEventRow(
|
||||||
|
job_id=job.id,
|
||||||
|
kind=kind,
|
||||||
|
attempt=job.attempt,
|
||||||
|
error_code=error_code,
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
return job.state
|
||||||
|
|
||||||
|
async def queue_stats(self, now: datetime) -> tuple[int, float]:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
depth, oldest = (
|
||||||
|
await session.execute(
|
||||||
|
select(func.count(AttachmentJobRow.id), func.min(AttachmentJobRow.created_at))
|
||||||
|
.where(AttachmentJobRow.state.in_(('queued', 'processing')))
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
age = max(0.0, (now - oldest).total_seconds()) if oldest is not None else 0.0
|
||||||
|
return int(depth or 0), age
|
||||||
|
|
||||||
|
async def recover_expired(self, now: datetime) -> int:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
ids = tuple(
|
||||||
|
await session.scalars(
|
||||||
|
select(AttachmentJobRow.id).where(
|
||||||
|
AttachmentJobRow.state == 'processing',
|
||||||
|
AttachmentJobRow.lease_until < now,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
recovered = 0
|
||||||
|
for job_id in ids:
|
||||||
|
state = await self.retry_or_fail(
|
||||||
|
job_id, None, 'lease_expired', False, now, now
|
||||||
|
)
|
||||||
|
recovered += state in {'queued', 'failed'}
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
job = await session.scalar(
|
||||||
|
select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update()
|
||||||
|
)
|
||||||
|
if job is None or job.state != 'failed':
|
||||||
|
return False
|
||||||
|
attachment = await session.get(AttachmentRow, job.attachment_id)
|
||||||
|
if attachment is None or not attachment.staging_key:
|
||||||
|
return False
|
||||||
|
if dry_run:
|
||||||
|
return True
|
||||||
|
job.state = 'queued'
|
||||||
|
job.attempt += 1
|
||||||
|
job.max_attempts = job.attempt + 4
|
||||||
|
job.replay_count += 1
|
||||||
|
job.last_error_code = None
|
||||||
|
job.next_attempt_at = now
|
||||||
|
job.updated_at = now
|
||||||
|
attachment.status = 'queued'
|
||||||
|
attachment.error_code = None
|
||||||
|
attachment.updated_at = now
|
||||||
|
session.add_all(
|
||||||
|
(
|
||||||
|
OutboxEventRow(
|
||||||
|
id=uuid4(), event_type='attachment.process.v1', job_id=job.id,
|
||||||
|
payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now),
|
||||||
|
traceparent=None, tracestate=None,
|
||||||
|
available_at=now, published_at=None, claim_owner=None, claim_until=None,
|
||||||
|
publish_attempts=0, last_error_code=None, created_at=now, updated_at=now,
|
||||||
|
),
|
||||||
|
JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class OutboxRepository:
|
||||||
|
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||||
|
self._sessions = sessions
|
||||||
|
|
||||||
|
async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
rows = tuple(
|
||||||
|
await session.scalars(
|
||||||
|
select(OutboxEventRow)
|
||||||
|
.where(
|
||||||
|
OutboxEventRow.published_at.is_(None),
|
||||||
|
OutboxEventRow.available_at <= now,
|
||||||
|
or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now),
|
||||||
|
)
|
||||||
|
.order_by(OutboxEventRow.available_at, OutboxEventRow.created_at)
|
||||||
|
.limit(limit)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
until = now + timedelta(seconds=lease_seconds)
|
||||||
|
for row in rows:
|
||||||
|
row.claim_owner = owner
|
||||||
|
row.claim_until = until
|
||||||
|
row.publish_attempts += 1
|
||||||
|
row.updated_at = now
|
||||||
|
return tuple(
|
||||||
|
OutboxEvent(
|
||||||
|
row.id, row.job_id, row.payload, row.available_at,
|
||||||
|
row.traceparent, row.tracestate,
|
||||||
|
)
|
||||||
|
for row in rows
|
||||||
|
)
|
||||||
|
|
||||||
|
async def published(self, event_id: UUID, owner: str, now: datetime) -> None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
await session.execute(
|
||||||
|
update(OutboxEventRow)
|
||||||
|
.where(
|
||||||
|
OutboxEventRow.id == event_id,
|
||||||
|
OutboxEventRow.claim_owner == owner,
|
||||||
|
OutboxEventRow.published_at.is_(None),
|
||||||
|
)
|
||||||
|
.values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
await session.execute(
|
||||||
|
update(OutboxEventRow)
|
||||||
|
.where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner)
|
||||||
|
.values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now)
|
||||||
|
)
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Sequence
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, tuple_
|
||||||
|
from sqlalchemy.dialects.postgresql import insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from ..domain.errors import Conflict, NotFound
|
||||||
|
from ..domain.models import Keyset, Message
|
||||||
|
from ..models.attachment import AttachmentRow, MessageAttachmentRow
|
||||||
|
from ..models.chat import ChatMemberRow, ChatRow
|
||||||
|
from ..models.message import MessageRow
|
||||||
|
|
||||||
|
|
||||||
|
class MessageRepository:
|
||||||
|
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||||
|
self._sessions = sessions
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message:
|
||||||
|
return Message(
|
||||||
|
id=row.id,
|
||||||
|
chat_id=row.chat_id,
|
||||||
|
sender_id=row.sender_id,
|
||||||
|
text=row.text,
|
||||||
|
created_at=row.created_at,
|
||||||
|
attachment_ids=tuple(attachment_ids),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]:
|
||||||
|
result = await session.scalars(
|
||||||
|
select(MessageAttachmentRow.attachment_id)
|
||||||
|
.where(MessageAttachmentRow.message_id == message_id)
|
||||||
|
.order_by(MessageAttachmentRow.position)
|
||||||
|
)
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None:
|
||||||
|
return await session.scalar(
|
||||||
|
select(ChatRow.id)
|
||||||
|
.join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id)
|
||||||
|
.where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def create_for_member(
|
||||||
|
self,
|
||||||
|
message_id: UUID,
|
||||||
|
chat_id: UUID,
|
||||||
|
sender_id: UUID,
|
||||||
|
text: str,
|
||||||
|
attachment_ids: Sequence[UUID],
|
||||||
|
created_at: datetime,
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> Message | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id)
|
||||||
|
if accessible_chat_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if attachment_ids:
|
||||||
|
found = {
|
||||||
|
attachment_id: status
|
||||||
|
for attachment_id, status in (
|
||||||
|
await session.execute(
|
||||||
|
select(AttachmentRow.id, AttachmentRow.status).where(
|
||||||
|
AttachmentRow.id.in_(attachment_ids),
|
||||||
|
AttachmentRow.chat_id == accessible_chat_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
}
|
||||||
|
if set(found) != set(attachment_ids):
|
||||||
|
raise NotFound('вложение не найдено')
|
||||||
|
if any(status == 'failed' for status in found.values()):
|
||||||
|
raise Conflict('неуспешное вложение нельзя прикрепить')
|
||||||
|
|
||||||
|
statement = (
|
||||||
|
insert(MessageRow)
|
||||||
|
.values(
|
||||||
|
id=message_id,
|
||||||
|
chat_id=accessible_chat_id,
|
||||||
|
sender_id=sender_id,
|
||||||
|
text=text,
|
||||||
|
created_at=created_at,
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
)
|
||||||
|
.on_conflict_do_nothing(
|
||||||
|
index_elements=[
|
||||||
|
MessageRow.sender_id,
|
||||||
|
MessageRow.chat_id,
|
||||||
|
MessageRow.idempotency_key,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.returning(MessageRow.id)
|
||||||
|
)
|
||||||
|
created_id = await session.scalar(statement)
|
||||||
|
if created_id is not None:
|
||||||
|
row = await session.get(MessageRow, created_id)
|
||||||
|
session.add_all(
|
||||||
|
MessageAttachmentRow(
|
||||||
|
message_id=created_id,
|
||||||
|
attachment_id=attachment_id,
|
||||||
|
position=position,
|
||||||
|
)
|
||||||
|
for position, attachment_id in enumerate(attachment_ids)
|
||||||
|
)
|
||||||
|
await session.flush()
|
||||||
|
result_attachment_ids = tuple(attachment_ids)
|
||||||
|
else:
|
||||||
|
row = await session.scalar(
|
||||||
|
select(MessageRow).where(
|
||||||
|
MessageRow.sender_id == sender_id,
|
||||||
|
MessageRow.chat_id == accessible_chat_id,
|
||||||
|
MessageRow.idempotency_key == idempotency_key,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
raise RuntimeError('результат идемпотентной операции не найден')
|
||||||
|
result_attachment_ids = await self._attachment_ids(session, row.id)
|
||||||
|
if row.text != text or result_attachment_ids != tuple(attachment_ids):
|
||||||
|
raise Conflict('Idempotency-Key уже использован с другим сообщением')
|
||||||
|
return self._to_domain(row, result_attachment_ids)
|
||||||
|
|
||||||
|
async def page_for_member(
|
||||||
|
self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int
|
||||||
|
) -> list[Message] | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id)
|
||||||
|
if accessible_chat_id is None:
|
||||||
|
return None
|
||||||
|
query = (
|
||||||
|
select(MessageRow)
|
||||||
|
.where(MessageRow.chat_id == accessible_chat_id)
|
||||||
|
.order_by(MessageRow.created_at, MessageRow.id)
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
if after is not None:
|
||||||
|
query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id))
|
||||||
|
rows = list((await session.scalars(query)).all())
|
||||||
|
return [
|
||||||
|
self._to_domain(row, await self._attachment_ids(session, row.id))
|
||||||
|
for row in rows
|
||||||
|
]
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from ..domain.errors import Conflict
|
||||||
|
from ..domain.models import User
|
||||||
|
from ..models.user import UserRow
|
||||||
|
|
||||||
|
|
||||||
|
class UserRepository:
|
||||||
|
def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None:
|
||||||
|
self._sessions = sessions
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_domain(row: UserRow) -> User:
|
||||||
|
return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at)
|
||||||
|
|
||||||
|
async def register(
|
||||||
|
self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime
|
||||||
|
) -> User:
|
||||||
|
row = UserRow(
|
||||||
|
id=user_id,
|
||||||
|
username=username,
|
||||||
|
display_name=display_name,
|
||||||
|
password_hash=password_hash,
|
||||||
|
created_at=created_at,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
session.add(row)
|
||||||
|
await session.flush()
|
||||||
|
except IntegrityError as exc:
|
||||||
|
raise Conflict('username уже занят') from exc
|
||||||
|
return self._to_domain(row)
|
||||||
|
|
||||||
|
async def get_by_id(self, user_id: UUID) -> User | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
row = await session.get(UserRow, user_id)
|
||||||
|
return self._to_domain(row) if row is not None else None
|
||||||
|
|
||||||
|
async def get_credentials(self, username: str) -> UserRow | None:
|
||||||
|
async with self._sessions() as session:
|
||||||
|
return await session.scalar(select(UserRow).where(UserRow.username == username))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def to_domain(cls, row: UserRow) -> User:
|
||||||
|
return cls._to_domain(row)
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from tempfile import SpooledTemporaryFile
|
||||||
|
from typing import AsyncIterator, BinaryIO
|
||||||
|
import unicodedata
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
|
||||||
|
from ..domain.errors import (
|
||||||
|
Conflict,
|
||||||
|
Invalid,
|
||||||
|
NotFound,
|
||||||
|
PayloadTooLarge,
|
||||||
|
Unavailable,
|
||||||
|
UnsupportedMediaType,
|
||||||
|
)
|
||||||
|
from ..domain.models import Attachment, User
|
||||||
|
from ..observability.logging import safe_extra
|
||||||
|
from ..observability.propagation import inject_trace_context
|
||||||
|
from ..repositories import AttachmentRepository
|
||||||
|
from ..storage import ObjectStream, S3Storage
|
||||||
|
|
||||||
|
logger = logging.getLogger('app.attachments')
|
||||||
|
|
||||||
|
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_filename(filename: str) -> None:
|
||||||
|
if not 1 <= len(filename) <= 255:
|
||||||
|
raise Invalid('filename должен содержать от 1 до 255 символов')
|
||||||
|
if any(
|
||||||
|
character in {'/', '\\'} or unicodedata.category(character) == 'Cc'
|
||||||
|
for character in filename
|
||||||
|
):
|
||||||
|
raise Invalid('filename содержит запрещённые символы')
|
||||||
|
|
||||||
|
|
||||||
|
def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]:
|
||||||
|
try:
|
||||||
|
source.seek(0)
|
||||||
|
with Image.open(source) as image:
|
||||||
|
actual_type = MIME_BY_FORMAT.get(image.format or '')
|
||||||
|
if actual_type is None or actual_type != declared_type:
|
||||||
|
raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG')
|
||||||
|
width, height = image.size
|
||||||
|
if width <= 0 or height <= 0:
|
||||||
|
raise UnsupportedMediaType('изображение имеет неверные размеры')
|
||||||
|
if width * height > max_pixels:
|
||||||
|
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей')
|
||||||
|
image.verify()
|
||||||
|
except Image.DecompressionBombError as error:
|
||||||
|
raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error
|
||||||
|
except (UnidentifiedImageError, OSError, SyntaxError) as error:
|
||||||
|
raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error
|
||||||
|
finally:
|
||||||
|
source.seek(0)
|
||||||
|
return actual_type, width, height
|
||||||
|
|
||||||
|
|
||||||
|
class AttachmentService:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
repository: AttachmentRepository,
|
||||||
|
storage: S3Storage,
|
||||||
|
max_upload_bytes: int,
|
||||||
|
max_image_pixels: int,
|
||||||
|
job_max_attempts: int,
|
||||||
|
) -> None:
|
||||||
|
self._repository = repository
|
||||||
|
self._storage = storage
|
||||||
|
self._max_upload_bytes = max_upload_bytes
|
||||||
|
self._max_image_pixels = max_image_pixels
|
||||||
|
self._job_max_attempts = job_max_attempts
|
||||||
|
|
||||||
|
async def upload(
|
||||||
|
self,
|
||||||
|
actor: User,
|
||||||
|
chat_id: UUID,
|
||||||
|
filename: str,
|
||||||
|
declared_type: str,
|
||||||
|
chunks: AsyncIterator[bytes],
|
||||||
|
) -> Attachment:
|
||||||
|
_validate_filename(filename)
|
||||||
|
if declared_type not in MIME_BY_FORMAT.values():
|
||||||
|
raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg')
|
||||||
|
if not await self._repository.chat_exists_for_member(chat_id, actor.id):
|
||||||
|
raise NotFound('чат не найден')
|
||||||
|
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
size = 0
|
||||||
|
with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source:
|
||||||
|
async for chunk in chunks:
|
||||||
|
size += len(chunk)
|
||||||
|
if size > self._max_upload_bytes:
|
||||||
|
raise PayloadTooLarge('файл превышает лимит 10 MiB')
|
||||||
|
digest.update(chunk)
|
||||||
|
source.write(chunk)
|
||||||
|
if size == 0:
|
||||||
|
raise UnsupportedMediaType('пустой файл не является изображением')
|
||||||
|
|
||||||
|
actual_type, width, height = await asyncio.to_thread(
|
||||||
|
_inspect_image,
|
||||||
|
source,
|
||||||
|
declared_type,
|
||||||
|
self._max_image_pixels,
|
||||||
|
)
|
||||||
|
attachment_id = uuid4()
|
||||||
|
job_id = uuid4()
|
||||||
|
staging_key = f'staging/{attachment_id}/original'
|
||||||
|
await self._storage.put(staging_key, source, size, actual_type)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
trace_context = inject_trace_context()
|
||||||
|
try:
|
||||||
|
attachment = await self._repository.create_queued(
|
||||||
|
attachment_id=attachment_id,
|
||||||
|
job_id=job_id,
|
||||||
|
chat_id=chat_id,
|
||||||
|
owner_id=actor.id,
|
||||||
|
filename=filename,
|
||||||
|
staging_key=staging_key,
|
||||||
|
content_type=actual_type,
|
||||||
|
size_bytes=size,
|
||||||
|
sha256=digest.hexdigest(),
|
||||||
|
width=width,
|
||||||
|
height=height,
|
||||||
|
max_attempts=self._job_max_attempts,
|
||||||
|
now=now,
|
||||||
|
traceparent=trace_context.get('traceparent'),
|
||||||
|
tracestate=trace_context.get('tracestate'),
|
||||||
|
)
|
||||||
|
logger.info(
|
||||||
|
'attachment accepted for asynchronous processing',
|
||||||
|
extra=safe_extra(
|
||||||
|
'attachment.accepted', attachment_id=str(attachment_id),
|
||||||
|
job_id=str(job_id), pipeline_version=1,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return attachment
|
||||||
|
except Exception:
|
||||||
|
try:
|
||||||
|
await self._storage.remove(staging_key)
|
||||||
|
except Unavailable:
|
||||||
|
logger.warning(
|
||||||
|
'failed to compensate staging object',
|
||||||
|
extra=safe_extra(
|
||||||
|
'staging.cleanup_deferred', attachment_id=str(attachment_id)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def get(self, actor: User, attachment_id: UUID) -> Attachment:
|
||||||
|
attachment = await self._repository.get_for_member(attachment_id, actor.id)
|
||||||
|
if attachment is None:
|
||||||
|
raise NotFound('вложение не найдено')
|
||||||
|
return attachment
|
||||||
|
|
||||||
|
async def content(
|
||||||
|
self, actor: User, attachment_id: UUID, variant: str
|
||||||
|
) -> tuple[Attachment, ObjectStream, str, int]:
|
||||||
|
attachment = await self.get(actor, attachment_id)
|
||||||
|
if attachment.status != 'ready':
|
||||||
|
raise Conflict('вложение ещё не готово')
|
||||||
|
if variant == 'original':
|
||||||
|
key = attachment.original_key
|
||||||
|
content_type = attachment.content_type
|
||||||
|
size_bytes = attachment.size_bytes
|
||||||
|
else:
|
||||||
|
found = next((item for item in attachment.variants if item.name == variant), None)
|
||||||
|
if found is None:
|
||||||
|
raise NotFound('вариант вложения не найден')
|
||||||
|
key = found.object_key
|
||||||
|
content_type = found.content_type
|
||||||
|
size_bytes = found.size_bytes
|
||||||
|
if key is None:
|
||||||
|
raise Unavailable('объект вложения отсутствует в метаданных')
|
||||||
|
try:
|
||||||
|
stream = await self._storage.get(key)
|
||||||
|
except FileNotFoundError as error:
|
||||||
|
raise Unavailable('объект вложения отсутствует в хранилище') from error
|
||||||
|
return attachment, stream, content_type, size_bytes
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Optional, Sequence
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from ..domain.errors import Invalid, NotFound
|
||||||
|
from ..domain.models import Chat, Keyset, Page, User
|
||||||
|
from ..repositories import ChatRepository
|
||||||
|
|
||||||
|
|
||||||
|
class ChatService:
|
||||||
|
def __init__(self, repository: ChatRepository) -> None:
|
||||||
|
self._repository = repository
|
||||||
|
|
||||||
|
async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat:
|
||||||
|
if actor.id not in member_ids:
|
||||||
|
raise Invalid('вызывающий должен входить в member_ids')
|
||||||
|
return await self._repository.create(
|
||||||
|
chat_id=uuid4(),
|
||||||
|
title=title,
|
||||||
|
member_ids=member_ids,
|
||||||
|
created_at=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get(self, actor: User, chat_id: UUID) -> Chat:
|
||||||
|
chat = await self._repository.get_for_member(chat_id, actor.id)
|
||||||
|
if chat is None:
|
||||||
|
raise NotFound('чат не найден')
|
||||||
|
return chat
|
||||||
|
|
||||||
|
async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page:
|
||||||
|
rows = await self._repository.page_for_member(actor.id, after, limit + 1)
|
||||||
|
items = tuple(rows[:limit])
|
||||||
|
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Optional, Sequence
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from ..domain.errors import NotFound
|
||||||
|
from ..domain.models import Keyset, Message, Page, User
|
||||||
|
from ..repositories import MessageRepository
|
||||||
|
|
||||||
|
|
||||||
|
class MessageService:
|
||||||
|
def __init__(self, repository: MessageRepository) -> None:
|
||||||
|
self._repository = repository
|
||||||
|
|
||||||
|
async def send(
|
||||||
|
self,
|
||||||
|
actor: User,
|
||||||
|
chat_id: UUID,
|
||||||
|
text: str,
|
||||||
|
attachment_ids: Sequence[UUID],
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> Message:
|
||||||
|
message = await self._repository.create_for_member(
|
||||||
|
message_id=uuid4(),
|
||||||
|
chat_id=chat_id,
|
||||||
|
sender_id=actor.id,
|
||||||
|
text=text,
|
||||||
|
attachment_ids=attachment_ids,
|
||||||
|
created_at=datetime.now(timezone.utc),
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
)
|
||||||
|
if message is None:
|
||||||
|
raise NotFound('чат не найден')
|
||||||
|
return message
|
||||||
|
|
||||||
|
async def history(
|
||||||
|
self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int
|
||||||
|
) -> Page:
|
||||||
|
rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1)
|
||||||
|
if rows is None:
|
||||||
|
raise NotFound('чат не найден')
|
||||||
|
items = tuple(rows[:limit])
|
||||||
|
return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None)
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from ..auth.passwords import hash_password, password_matches
|
||||||
|
from ..domain.errors import Unauthorized
|
||||||
|
from ..domain.models import User
|
||||||
|
from ..repositories import UserRepository
|
||||||
|
|
||||||
|
|
||||||
|
class UserService:
|
||||||
|
def __init__(self, repository: UserRepository) -> None:
|
||||||
|
self._repository = repository
|
||||||
|
|
||||||
|
async def register(self, username: str, display_name: str, password: str) -> User:
|
||||||
|
return await self._repository.register(
|
||||||
|
user_id=uuid4(),
|
||||||
|
username=username,
|
||||||
|
display_name=display_name,
|
||||||
|
password_hash=hash_password(password),
|
||||||
|
created_at=datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
async def identify(self, user_id: UUID) -> User:
|
||||||
|
user = await self._repository.get_by_id(user_id)
|
||||||
|
if user is None:
|
||||||
|
raise Unauthorized('требуется действующая сессия')
|
||||||
|
return user
|
||||||
|
|
||||||
|
async def authenticate(self, username: str, password: str) -> User | None:
|
||||||
|
row = await self._repository.get_credentials(username)
|
||||||
|
if row is None or not password_matches(row.password_hash, password):
|
||||||
|
return None
|
||||||
|
return self._repository.to_domain(row)
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
from .s3 import ObjectStream, S3Storage, StoredObject
|
||||||
|
|
||||||
|
__all__ = ['ObjectStream', 'S3Storage', 'StoredObject']
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import certifi
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import BinaryIO, Iterator
|
||||||
|
from io import BytesIO
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from minio import Minio
|
||||||
|
from minio.error import InvalidResponseError, S3Error
|
||||||
|
from urllib3 import PoolManager, Retry, Timeout
|
||||||
|
from urllib3.exceptions import HTTPError
|
||||||
|
from opentelemetry import trace
|
||||||
|
from opentelemetry.trace import SpanKind
|
||||||
|
|
||||||
|
from ..domain.errors import Unavailable
|
||||||
|
|
||||||
|
tracer = trace.get_tracer('messenger.s3', '1.6.0')
|
||||||
|
|
||||||
|
|
||||||
|
def _unavailable(error: Exception) -> Unavailable:
|
||||||
|
return Unavailable('объектное хранилище недоступно')
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class ObjectStream:
|
||||||
|
response: object
|
||||||
|
|
||||||
|
def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]:
|
||||||
|
try:
|
||||||
|
while chunk := self.response.read(size):
|
||||||
|
yield chunk
|
||||||
|
finally:
|
||||||
|
self.response.close()
|
||||||
|
self.response.release_conn()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class StoredObject:
|
||||||
|
key: str
|
||||||
|
last_modified: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class S3Storage:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
endpoint: str,
|
||||||
|
access_key: str,
|
||||||
|
secret_key: str,
|
||||||
|
bucket: str,
|
||||||
|
region: str,
|
||||||
|
) -> None:
|
||||||
|
parsed = urlparse(endpoint)
|
||||||
|
if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}:
|
||||||
|
raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port')
|
||||||
|
self._client = Minio(
|
||||||
|
parsed.netloc,
|
||||||
|
access_key=access_key,
|
||||||
|
secret_key=secret_key,
|
||||||
|
secure=parsed.scheme == 'https',
|
||||||
|
region=region,
|
||||||
|
http_client=PoolManager(
|
||||||
|
maxsize=8,
|
||||||
|
timeout=Timeout(connect=2, read=5),
|
||||||
|
retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1),
|
||||||
|
cert_reqs='CERT_REQUIRED',
|
||||||
|
ca_certs=certifi.where(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.bucket = bucket
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _translate(error: Exception) -> None:
|
||||||
|
if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}:
|
||||||
|
raise FileNotFoundError from error
|
||||||
|
if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)):
|
||||||
|
raise _unavailable(error) from error
|
||||||
|
raise error
|
||||||
|
|
||||||
|
async def ping(self) -> None:
|
||||||
|
try:
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
's3 bucket exists', kind=SpanKind.CLIENT,
|
||||||
|
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||||
|
):
|
||||||
|
exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket)
|
||||||
|
except Exception as error:
|
||||||
|
self._translate(error)
|
||||||
|
if not exists:
|
||||||
|
raise Unavailable('bucket объектного хранилища не создан')
|
||||||
|
|
||||||
|
async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None:
|
||||||
|
source.seek(0)
|
||||||
|
try:
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
's3 put object', kind=SpanKind.CLIENT,
|
||||||
|
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||||
|
):
|
||||||
|
await asyncio.to_thread(
|
||||||
|
self._client.put_object,
|
||||||
|
self.bucket,
|
||||||
|
key,
|
||||||
|
source,
|
||||||
|
length,
|
||||||
|
content_type,
|
||||||
|
)
|
||||||
|
except Exception as error:
|
||||||
|
self._translate(error)
|
||||||
|
|
||||||
|
async def get(self, key: str) -> ObjectStream:
|
||||||
|
try:
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
's3 get object', kind=SpanKind.CLIENT,
|
||||||
|
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||||
|
):
|
||||||
|
response = await asyncio.to_thread(self._client.get_object, self.bucket, key)
|
||||||
|
except Exception as error:
|
||||||
|
self._translate(error)
|
||||||
|
return ObjectStream(response)
|
||||||
|
|
||||||
|
async def read(self, key: str, max_bytes: int) -> bytes:
|
||||||
|
stream = await self.get(key)
|
||||||
|
result = bytearray()
|
||||||
|
for chunk in stream.chunks():
|
||||||
|
result.extend(chunk)
|
||||||
|
if len(result) > max_bytes:
|
||||||
|
raise ValueError('stored object exceeds configured byte limit')
|
||||||
|
return bytes(result)
|
||||||
|
|
||||||
|
async def put_bytes(self, key: str, content: bytes, content_type: str) -> None:
|
||||||
|
await self.put(key, BytesIO(content), len(content), content_type)
|
||||||
|
|
||||||
|
async def exists(self, key: str) -> bool:
|
||||||
|
try:
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
's3 stat object', kind=SpanKind.CLIENT,
|
||||||
|
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||||
|
):
|
||||||
|
await asyncio.to_thread(self._client.stat_object, self.bucket, key)
|
||||||
|
return True
|
||||||
|
except Exception as error:
|
||||||
|
try:
|
||||||
|
self._translate(error)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return False
|
||||||
|
raise
|
||||||
|
|
||||||
|
async def remove(self, key: str) -> None:
|
||||||
|
try:
|
||||||
|
with tracer.start_as_current_span(
|
||||||
|
's3 remove object', kind=SpanKind.CLIENT,
|
||||||
|
attributes={'db.system': 's3', 's3.bucket.name': self.bucket},
|
||||||
|
):
|
||||||
|
await asyncio.to_thread(self._client.remove_object, self.bucket, key)
|
||||||
|
except Exception as error:
|
||||||
|
self._translate(error)
|
||||||
|
|
||||||
|
async def list(self, prefix: str = '') -> tuple[StoredObject, ...]:
|
||||||
|
def collect() -> tuple[StoredObject, ...]:
|
||||||
|
return tuple(
|
||||||
|
StoredObject(item.object_name, item.last_modified)
|
||||||
|
for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return await asyncio.to_thread(collect)
|
||||||
|
except Exception as error:
|
||||||
|
self._translate(error)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Операторские CLI-инструменты приложения."""
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..config import load_settings
|
||||||
|
from ..db import create_engine, create_session_factory
|
||||||
|
from ..models.attachment import AttachmentRow
|
||||||
|
from ..storage import S3Storage
|
||||||
|
|
||||||
|
|
||||||
|
async def cleanup(apply: bool, grace_seconds: int) -> int:
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
storage = S3Storage(
|
||||||
|
settings.s3_endpoint,
|
||||||
|
settings.s3_access_key_id,
|
||||||
|
settings.s3_secret_access_key,
|
||||||
|
settings.s3_bucket,
|
||||||
|
settings.s3_region,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
sessions = create_session_factory(engine)
|
||||||
|
async with sessions() as session:
|
||||||
|
known = set(
|
||||||
|
key
|
||||||
|
for key in await session.scalars(select(AttachmentRow.original_key))
|
||||||
|
if key is not None
|
||||||
|
)
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||||
|
removed = 0
|
||||||
|
for item in await storage.list('originals/'):
|
||||||
|
if item.key in known or item.last_modified > cutoff:
|
||||||
|
continue
|
||||||
|
action = 'DELETE' if apply else 'WOULD_DELETE'
|
||||||
|
print(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||||
|
if apply:
|
||||||
|
await storage.remove(item.key)
|
||||||
|
removed += 1
|
||||||
|
print(
|
||||||
|
f'summary mode={"apply" if apply else "dry-run"} '
|
||||||
|
f'known={len(known)} removed={removed} grace_seconds={grace_seconds}'
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata')
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument('--dry-run', action='store_true')
|
||||||
|
mode.add_argument('--apply', action='store_true')
|
||||||
|
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.grace_seconds < 60:
|
||||||
|
parser.error('--grace-seconds must be at least 60')
|
||||||
|
raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from ..config import load_settings
|
||||||
|
from ..db import create_engine, create_session_factory
|
||||||
|
from ..models.attachment import AttachmentRow
|
||||||
|
from ..storage import S3Storage
|
||||||
|
|
||||||
|
|
||||||
|
async def cleanup(apply: bool, retention_seconds: int) -> int:
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
sessions = create_session_factory(engine)
|
||||||
|
storage = S3Storage(
|
||||||
|
settings.s3_endpoint,
|
||||||
|
settings.s3_access_key_id,
|
||||||
|
settings.s3_secret_access_key,
|
||||||
|
settings.s3_bucket,
|
||||||
|
settings.s3_region,
|
||||||
|
)
|
||||||
|
cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds)
|
||||||
|
try:
|
||||||
|
async with sessions() as session:
|
||||||
|
active = set(
|
||||||
|
key
|
||||||
|
for key in await session.scalars(
|
||||||
|
select(AttachmentRow.staging_key).where(
|
||||||
|
AttachmentRow.staging_key.is_not(None),
|
||||||
|
AttachmentRow.status.in_(('queued', 'processing')),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if key is not None
|
||||||
|
)
|
||||||
|
removed = 0
|
||||||
|
for item in await storage.list('staging/'):
|
||||||
|
if item.key in active or item.last_modified > cutoff:
|
||||||
|
continue
|
||||||
|
print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}')
|
||||||
|
if apply:
|
||||||
|
await storage.remove(item.key)
|
||||||
|
removed += 1
|
||||||
|
print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}')
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description='Clean inactive durable staging objects')
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument('--dry-run', action='store_true')
|
||||||
|
mode.add_argument('--apply', action='store_true')
|
||||||
|
parser.add_argument('--retention-seconds', type=int)
|
||||||
|
args = parser.parse_args()
|
||||||
|
settings = load_settings()
|
||||||
|
retention = args.retention_seconds or settings.staging_retention_seconds
|
||||||
|
if retention < 60:
|
||||||
|
parser.error('retention must be at least 60 seconds')
|
||||||
|
raise SystemExit(asyncio.run(cleanup(args.apply, retention)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import tracemalloc
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Callable, Sequence
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
|
from ..config import load_settings
|
||||||
|
from ..db import create_engine, create_session_factory
|
||||||
|
from ..models.attachment import AttachmentRow
|
||||||
|
from ..storage import S3Storage, StoredObject
|
||||||
|
|
||||||
|
MISSING_ERROR_CODE = 'original_missing'
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class AttachmentSnapshot:
|
||||||
|
id: UUID
|
||||||
|
object_key: str
|
||||||
|
status: str
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ReconcilePlan:
|
||||||
|
orphan: tuple[StoredObject, ...]
|
||||||
|
fresh_orphan: tuple[StoredObject, ...]
|
||||||
|
missing: tuple[AttachmentSnapshot, ...]
|
||||||
|
fresh_missing: tuple[AttachmentSnapshot, ...]
|
||||||
|
correct: int
|
||||||
|
|
||||||
|
|
||||||
|
def build_plan(
|
||||||
|
attachments: Sequence[AttachmentSnapshot],
|
||||||
|
objects: Sequence[StoredObject],
|
||||||
|
cutoff: datetime,
|
||||||
|
) -> ReconcilePlan:
|
||||||
|
"""Compare immutable PostgreSQL and S3 snapshots without changing either side."""
|
||||||
|
known_keys = {attachment.object_key for attachment in attachments}
|
||||||
|
object_keys = {item.key for item in objects}
|
||||||
|
|
||||||
|
unknown_objects = [item for item in objects if item.key not in known_keys]
|
||||||
|
orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff)
|
||||||
|
fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff)
|
||||||
|
|
||||||
|
ready_without_object = [
|
||||||
|
attachment
|
||||||
|
for attachment in attachments
|
||||||
|
if attachment.status == 'ready' and attachment.object_key not in object_keys
|
||||||
|
]
|
||||||
|
missing = tuple(
|
||||||
|
attachment for attachment in ready_without_object if attachment.updated_at <= cutoff
|
||||||
|
)
|
||||||
|
fresh_missing = tuple(
|
||||||
|
attachment for attachment in ready_without_object if attachment.updated_at > cutoff
|
||||||
|
)
|
||||||
|
correct = sum(attachment.object_key in object_keys for attachment in attachments)
|
||||||
|
return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct)
|
||||||
|
|
||||||
|
|
||||||
|
async def _load_attachments(
|
||||||
|
sessions: async_sessionmaker[AsyncSession],
|
||||||
|
) -> tuple[AttachmentSnapshot, ...]:
|
||||||
|
async with sessions() as session:
|
||||||
|
rows = (
|
||||||
|
await session.execute(
|
||||||
|
select(
|
||||||
|
AttachmentRow.id,
|
||||||
|
AttachmentRow.original_key,
|
||||||
|
AttachmentRow.status,
|
||||||
|
AttachmentRow.updated_at,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return tuple(AttachmentSnapshot(*row) for row in rows)
|
||||||
|
|
||||||
|
|
||||||
|
async def _metadata_key_exists(
|
||||||
|
sessions: async_sessionmaker[AsyncSession], object_key: str
|
||||||
|
) -> bool:
|
||||||
|
async with sessions() as session:
|
||||||
|
attachment_id = await session.scalar(
|
||||||
|
select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1)
|
||||||
|
)
|
||||||
|
return attachment_id is not None
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile(
|
||||||
|
sessions: async_sessionmaker[AsyncSession],
|
||||||
|
storage: S3Storage,
|
||||||
|
*,
|
||||||
|
apply: bool,
|
||||||
|
grace_seconds: int,
|
||||||
|
now: datetime | None = None,
|
||||||
|
output: Callable[[str], None] = print,
|
||||||
|
) -> dict[str, int | float | str]:
|
||||||
|
started = time.perf_counter()
|
||||||
|
tracemalloc.start()
|
||||||
|
current_time = now or datetime.now(timezone.utc)
|
||||||
|
cutoff = current_time - timedelta(seconds=grace_seconds)
|
||||||
|
|
||||||
|
attachments = await _load_attachments(sessions)
|
||||||
|
objects = await storage.list('originals/')
|
||||||
|
plan = build_plan(attachments, objects, cutoff)
|
||||||
|
|
||||||
|
for item in plan.fresh_orphan:
|
||||||
|
output(
|
||||||
|
f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}'
|
||||||
|
)
|
||||||
|
for attachment in plan.fresh_missing:
|
||||||
|
output(
|
||||||
|
'SKIP_FRESH_MISSING '
|
||||||
|
f'attachment_id={attachment.id} key={attachment.object_key} '
|
||||||
|
f'updated={attachment.updated_at.isoformat()}'
|
||||||
|
)
|
||||||
|
|
||||||
|
removed = 0
|
||||||
|
race_protected = 0
|
||||||
|
for item in plan.orphan:
|
||||||
|
action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN'
|
||||||
|
if not apply:
|
||||||
|
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||||
|
continue
|
||||||
|
# PostgreSQL and S3 cannot share a transaction. Re-read the authoritative
|
||||||
|
# metadata immediately before a destructive action so a row created after
|
||||||
|
# the initial snapshot protects its object.
|
||||||
|
if await _metadata_key_exists(sessions, item.key):
|
||||||
|
output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}')
|
||||||
|
race_protected += 1
|
||||||
|
continue
|
||||||
|
output(f'{action} key={item.key} modified={item.last_modified.isoformat()}')
|
||||||
|
await storage.remove(item.key)
|
||||||
|
removed += 1
|
||||||
|
|
||||||
|
marked_failed = 0
|
||||||
|
for attachment in plan.missing:
|
||||||
|
if not apply:
|
||||||
|
output(
|
||||||
|
f'WOULD_MARK_MISSING attachment_id={attachment.id} '
|
||||||
|
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# The object may have been restored after list_objects(). Do not turn
|
||||||
|
# healthy metadata into failed based only on the earlier snapshot.
|
||||||
|
if await storage.exists(attachment.object_key):
|
||||||
|
output(
|
||||||
|
f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} '
|
||||||
|
f'key={attachment.object_key}'
|
||||||
|
)
|
||||||
|
race_protected += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
async with sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
result = await session.execute(
|
||||||
|
update(AttachmentRow)
|
||||||
|
.where(
|
||||||
|
AttachmentRow.id == attachment.id,
|
||||||
|
AttachmentRow.original_key == attachment.object_key,
|
||||||
|
AttachmentRow.status == 'ready',
|
||||||
|
AttachmentRow.updated_at == attachment.updated_at,
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
status='failed',
|
||||||
|
error_code=MISSING_ERROR_CODE,
|
||||||
|
updated_at=current_time,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
changed = result.rowcount or 0
|
||||||
|
if changed:
|
||||||
|
output(
|
||||||
|
f'MARK_MISSING attachment_id={attachment.id} '
|
||||||
|
f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}'
|
||||||
|
)
|
||||||
|
marked_failed += changed
|
||||||
|
else:
|
||||||
|
output(
|
||||||
|
f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} '
|
||||||
|
f'key={attachment.object_key}'
|
||||||
|
)
|
||||||
|
race_protected += 1
|
||||||
|
|
||||||
|
_, peak_bytes = tracemalloc.get_traced_memory()
|
||||||
|
tracemalloc.stop()
|
||||||
|
summary: dict[str, int | float | str] = {
|
||||||
|
'mode': 'apply' if apply else 'dry-run',
|
||||||
|
'metadata': len(attachments),
|
||||||
|
'objects': len(objects),
|
||||||
|
'correct': plan.correct,
|
||||||
|
'orphan': len(plan.orphan),
|
||||||
|
'missing': len(plan.missing),
|
||||||
|
'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing),
|
||||||
|
'removed': removed,
|
||||||
|
'marked_failed': marked_failed,
|
||||||
|
'race_protected': race_protected,
|
||||||
|
'grace_seconds': grace_seconds,
|
||||||
|
'elapsed_ms': round((time.perf_counter() - started) * 1000, 3),
|
||||||
|
'peak_memory_bytes': peak_bytes,
|
||||||
|
}
|
||||||
|
output(f'summary {json.dumps(summary, sort_keys=True)}')
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
async def run(apply: bool, grace_seconds: int) -> int:
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
storage = S3Storage(
|
||||||
|
settings.s3_endpoint,
|
||||||
|
settings.s3_access_key_id,
|
||||||
|
settings.s3_secret_access_key,
|
||||||
|
settings.s3_bucket,
|
||||||
|
settings.s3_region,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await reconcile(
|
||||||
|
create_session_factory(engine),
|
||||||
|
storage,
|
||||||
|
apply=apply,
|
||||||
|
grace_seconds=grace_seconds,
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Reconcile attachment metadata in PostgreSQL with original objects in S3'
|
||||||
|
)
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument('--dry-run', action='store_true')
|
||||||
|
mode.add_argument('--apply', action='store_true')
|
||||||
|
parser.add_argument('--grace-seconds', type=int, default=3600)
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.grace_seconds < 60:
|
||||||
|
parser.error('--grace-seconds must be at least 60')
|
||||||
|
raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from ..config import load_settings
|
||||||
|
from ..db import create_engine, create_session_factory
|
||||||
|
from ..repositories import JobRepository
|
||||||
|
from ..storage import S3Storage
|
||||||
|
|
||||||
|
|
||||||
|
async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int:
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
try:
|
||||||
|
repository = JobRepository(create_session_factory(engine))
|
||||||
|
if job_id is None:
|
||||||
|
assert attachment_id is not None
|
||||||
|
job_id = await repository.id_for_attachment(attachment_id)
|
||||||
|
if job_id is None:
|
||||||
|
print(f'REJECTED attachment_id={attachment_id} reason=job_not_found')
|
||||||
|
return 2
|
||||||
|
staging_key = await repository.failed_staging_key(job_id)
|
||||||
|
if staging_key is None:
|
||||||
|
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||||
|
return 2
|
||||||
|
storage = S3Storage(
|
||||||
|
settings.s3_endpoint,
|
||||||
|
settings.s3_access_key_id,
|
||||||
|
settings.s3_secret_access_key,
|
||||||
|
settings.s3_bucket,
|
||||||
|
settings.s3_region,
|
||||||
|
)
|
||||||
|
if not await storage.exists(staging_key):
|
||||||
|
print(f'REJECTED job_id={job_id} reason=staging_object_missing')
|
||||||
|
return 2
|
||||||
|
accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run)
|
||||||
|
if not accepted:
|
||||||
|
print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing')
|
||||||
|
return 2
|
||||||
|
print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}')
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description='Safely replay one failed attachment job')
|
||||||
|
target = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
target.add_argument('--job-id', type=UUID)
|
||||||
|
target.add_argument('--attachment-id', type=UUID)
|
||||||
|
parser.add_argument('--dry-run', action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from io import BytesIO
|
||||||
|
from uuid import NAMESPACE_URL, uuid5
|
||||||
|
|
||||||
|
from sqlalchemy import delete, insert
|
||||||
|
|
||||||
|
from ..config import load_settings
|
||||||
|
from ..db import create_engine, create_session_factory
|
||||||
|
from ..models.attachment import AttachmentRow
|
||||||
|
from ..models.chat import ChatMemberRow, ChatRow
|
||||||
|
from ..models.user import UserRow
|
||||||
|
from ..storage import S3Storage
|
||||||
|
|
||||||
|
USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user')
|
||||||
|
CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat')
|
||||||
|
OBJECT_PREFIX = 'originals/reconciler-benchmark/'
|
||||||
|
WORKERS = 16
|
||||||
|
|
||||||
|
|
||||||
|
async def _put_objects(
|
||||||
|
storage: S3Storage,
|
||||||
|
keys: list[str],
|
||||||
|
) -> None:
|
||||||
|
for offset in range(0, len(keys), WORKERS):
|
||||||
|
batch = keys[offset : offset + WORKERS]
|
||||||
|
await asyncio.gather(
|
||||||
|
*(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def prepare(size: int) -> None:
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
sessions = create_session_factory(engine)
|
||||||
|
storage = S3Storage(
|
||||||
|
settings.s3_endpoint,
|
||||||
|
settings.s3_access_key_id,
|
||||||
|
settings.s3_secret_access_key,
|
||||||
|
settings.s3_bucket,
|
||||||
|
settings.s3_region,
|
||||||
|
)
|
||||||
|
old = datetime.now(timezone.utc) - timedelta(hours=2)
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'),
|
||||||
|
'chat_id': CHAT_ID,
|
||||||
|
'owner_id': USER_ID,
|
||||||
|
'filename': f'benchmark-{index}.png',
|
||||||
|
'original_key': f'{OBJECT_PREFIX}{index}',
|
||||||
|
'staging_key': None,
|
||||||
|
'pipeline_version': 1,
|
||||||
|
'content_type': 'image/png',
|
||||||
|
'size_bytes': 1,
|
||||||
|
'sha256': '0' * 64,
|
||||||
|
'width': 1,
|
||||||
|
'height': 1,
|
||||||
|
'status': 'ready',
|
||||||
|
'error_code': None,
|
||||||
|
'created_at': old,
|
||||||
|
'updated_at': old,
|
||||||
|
}
|
||||||
|
for index in range(size)
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
async with sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
session.add(UserRow(
|
||||||
|
id=USER_ID,
|
||||||
|
username=f'bench-{USER_ID.hex[:16]}',
|
||||||
|
display_name='Reconciler benchmark',
|
||||||
|
password_hash='not-used',
|
||||||
|
created_at=old,
|
||||||
|
))
|
||||||
|
session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old))
|
||||||
|
await session.flush()
|
||||||
|
session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID))
|
||||||
|
await session.flush()
|
||||||
|
for offset in range(0, size, 500):
|
||||||
|
await session.execute(insert(AttachmentRow), rows[offset : offset + 500])
|
||||||
|
|
||||||
|
await _put_objects(storage, [row['original_key'] for row in rows])
|
||||||
|
print(f'prepared metadata={size} objects={size}')
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
async def cleanup() -> None:
|
||||||
|
settings = load_settings()
|
||||||
|
engine = create_engine(settings.database_url)
|
||||||
|
sessions = create_session_factory(engine)
|
||||||
|
storage = S3Storage(
|
||||||
|
settings.s3_endpoint,
|
||||||
|
settings.s3_access_key_id,
|
||||||
|
settings.s3_secret_access_key,
|
||||||
|
settings.s3_bucket,
|
||||||
|
settings.s3_region,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
objects = await storage.list(OBJECT_PREFIX)
|
||||||
|
keys = [item.key for item in objects]
|
||||||
|
for offset in range(0, len(keys), WORKERS):
|
||||||
|
await asyncio.gather(
|
||||||
|
*(storage.remove(key) for key in keys[offset : offset + WORKERS])
|
||||||
|
)
|
||||||
|
|
||||||
|
async with sessions() as session:
|
||||||
|
async with session.begin():
|
||||||
|
await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID))
|
||||||
|
await session.execute(delete(UserRow).where(UserRow.id == USER_ID))
|
||||||
|
print(f'cleaned objects={len(keys)}')
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks'
|
||||||
|
)
|
||||||
|
subparsers = parser.add_subparsers(dest='command', required=True)
|
||||||
|
prepare_parser = subparsers.add_parser('prepare')
|
||||||
|
prepare_parser.add_argument('--size', type=int, required=True)
|
||||||
|
subparsers.add_parser('cleanup')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.command == 'prepare':
|
||||||
|
if args.size < 1 or args.size > 10000:
|
||||||
|
parser.error('--size must be between 1 and 10000')
|
||||||
|
asyncio.run(prepare(args.size))
|
||||||
|
else:
|
||||||
|
asyncio.run(cleanup())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
services:
|
||||||
|
observability-test:
|
||||||
|
image: python:3.13.15-slim
|
||||||
|
working_dir: /tests
|
||||||
|
command: ["python", "observability_smoke.py", "--grafana-url", "http://grafana:3000"]
|
||||||
|
environment:
|
||||||
|
GRAFANA_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||||
|
GRAFANA_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana}
|
||||||
|
volumes:
|
||||||
|
- ./tests:/tests:ro
|
||||||
|
depends_on:
|
||||||
|
grafana:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:17.4-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: messenger_test
|
||||||
|
POSTGRES_PASSWORD: messenger_test
|
||||||
|
POSTGRES_DB: messenger_test
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7.4.2-alpine
|
||||||
|
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
broker:
|
||||||
|
image: rabbitmq:4.1.4-alpine
|
||||||
|
environment:
|
||||||
|
RABBITMQ_DEFAULT_USER: messenger_test
|
||||||
|
RABBITMQ_DEFAULT_PASS: messenger_test
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
s3:
|
||||||
|
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||||
|
user: "0"
|
||||||
|
command: ["server", "/data", "--console-address", ":9001"]
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: test-minio-admin
|
||||||
|
MINIO_ROOT_PASSWORD: test-minio-root-password
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
s3-bootstrap:
|
||||||
|
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
|
||||||
|
entrypoint: ["/bin/sh", "-c"]
|
||||||
|
command:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||||
|
mc mb --ignore-existing local/"$$S3_BUCKET"
|
||||||
|
mc anonymous set none local/"$$S3_BUCKET"
|
||||||
|
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||||
|
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
|
||||||
|
mc admin policy create local messenger-api /tmp/s3-api-policy.json
|
||||||
|
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: test-minio-admin
|
||||||
|
MINIO_ROOT_PASSWORD: test-minio-root-password
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||||
|
volumes:
|
||||||
|
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
build: .
|
||||||
|
command: ["alembic", "upgrade", "head"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: 3600
|
||||||
|
S3_ACCESS_KEY_ID: messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||||
|
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
api:
|
||||||
|
build: .
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:18080:8080"
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: 3600
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||||
|
MAX_UPLOAD_BYTES: 10485760
|
||||||
|
MAX_IMAGE_PIXELS: 40000000
|
||||||
|
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||||
|
JOB_MAX_ATTEMPTS: 5
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
publisher:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "app.jobs.publisher"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: 3600
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||||
|
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
broker:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 10
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "app.jobs.worker"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: 3600
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||||
|
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||||
|
WORKER_CONCURRENCY: 2
|
||||||
|
WORKER_PREFETCH: 2
|
||||||
|
JOB_MAX_ATTEMPTS: 5
|
||||||
|
JOB_TIMEOUT_SECONDS: 60
|
||||||
|
JOB_LEASE_SECONDS: 90
|
||||||
|
RETRY_BASE_SECONDS: 1
|
||||||
|
RETRY_MAX_SECONDS: 4
|
||||||
|
STAGING_RETENTION_SECONDS: 86400
|
||||||
|
THUMBNAIL_JPEG_QUALITY: 85
|
||||||
|
CROP_JPEG_QUALITY: 85
|
||||||
|
OPTIMIZED_WEBP_QUALITY: 82
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
broker:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 10
|
||||||
|
|
||||||
|
tests:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
dockerfile: Dockerfile.test
|
||||||
|
environment:
|
||||||
|
BASE_URL: http://api:8080
|
||||||
|
DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test
|
||||||
|
REDIS_URL: redis://redis:6379/0
|
||||||
|
SESSION_TTL_SECONDS: 3600
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID: messenger-api
|
||||||
|
S3_SECRET_ACCESS_KEY: test-minio-api-password
|
||||||
|
BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/
|
||||||
|
depends_on:
|
||||||
|
api:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
+346
@@ -0,0 +1,346 @@
|
|||||||
|
x-otel-environment: &otel-environment
|
||||||
|
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||||
|
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||||
|
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||||
|
OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-2}
|
||||||
|
OTEL_SERVICE_VERSION: ${OTEL_SERVICE_VERSION:-1.6.0}
|
||||||
|
OTEL_DEPLOYMENT_ENVIRONMENT: ${OTEL_DEPLOYMENT_ENVIRONMENT:-lab}
|
||||||
|
OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048}
|
||||||
|
OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048}
|
||||||
|
OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-5000}
|
||||||
|
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:17.4-alpine
|
||||||
|
cpus: "1.0"
|
||||||
|
mem_limit: 512m
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: ${POSTGRES_USER}
|
||||||
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||||
|
POSTGRES_DB: ${POSTGRES_DB}
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7.4.2-alpine
|
||||||
|
cpus: "0.25"
|
||||||
|
mem_limit: 128m
|
||||||
|
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||||
|
volumes:
|
||||||
|
- redis_data:/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
|
||||||
|
broker:
|
||||||
|
image: rabbitmq:4.1.4-alpine
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 384m
|
||||||
|
environment:
|
||||||
|
RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger}
|
||||||
|
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq}
|
||||||
|
volumes:
|
||||||
|
- broker_data:/var/lib/rabbitmq
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 20
|
||||||
|
|
||||||
|
s3:
|
||||||
|
image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597
|
||||||
|
user: "0"
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 256m
|
||||||
|
command: ["server", "/data", "--console-address", ":9001"]
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
volumes:
|
||||||
|
- s3_data:/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mc", "ready", "local"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
|
||||||
|
s3-bootstrap:
|
||||||
|
image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805
|
||||||
|
cpus: "0.25"
|
||||||
|
mem_limit: 128m
|
||||||
|
entrypoint: ["/bin/sh", "-c"]
|
||||||
|
command:
|
||||||
|
- |
|
||||||
|
set -eu
|
||||||
|
mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD"
|
||||||
|
mc mb --ignore-existing local/"$$S3_BUCKET"
|
||||||
|
mc anonymous set none local/"$$S3_BUCKET"
|
||||||
|
mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
|
||||||
|
sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json
|
||||||
|
mc admin policy create local messenger-api /tmp/s3-api-policy.json
|
||||||
|
mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin}
|
||||||
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||||
|
volumes:
|
||||||
|
- ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
migrate:
|
||||||
|
build: .
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 256m
|
||||||
|
command: ["alembic", "upgrade", "head"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: ${DATABASE_URL}
|
||||||
|
REDIS_URL: ${REDIS_URL}
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||||
|
BROKER_URL: ${BROKER_URL}
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
restart: "no"
|
||||||
|
|
||||||
|
api:
|
||||||
|
build: .
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 256m
|
||||||
|
environment:
|
||||||
|
<<: *otel-environment
|
||||||
|
OTEL_SERVICE_NAME: messenger-api
|
||||||
|
DATABASE_URL: ${DATABASE_URL}
|
||||||
|
REDIS_URL: ${REDIS_URL}
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||||
|
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||||
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||||
|
S3_REGION: ${S3_REGION:-us-east-1}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||||
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||||
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||||
|
BROKER_URL: ${BROKER_URL}
|
||||||
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||||
|
expose:
|
||||||
|
- "8080"
|
||||||
|
deploy:
|
||||||
|
replicas: 2
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 5
|
||||||
|
stop_grace_period: 10s
|
||||||
|
|
||||||
|
publisher:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "app.jobs.publisher"]
|
||||||
|
cpus: "0.25"
|
||||||
|
mem_limit: 192m
|
||||||
|
environment:
|
||||||
|
<<: *otel-environment
|
||||||
|
OTEL_SERVICE_NAME: messenger-publisher
|
||||||
|
DATABASE_URL: ${DATABASE_URL}
|
||||||
|
REDIS_URL: ${REDIS_URL}
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||||
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||||
|
S3_REGION: ${S3_REGION:-us-east-1}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||||
|
BROKER_URL: ${BROKER_URL}
|
||||||
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||||
|
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
broker:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 10s
|
||||||
|
|
||||||
|
worker:
|
||||||
|
build: .
|
||||||
|
command: ["python", "-m", "app.jobs.worker"]
|
||||||
|
cpus: "1.0"
|
||||||
|
mem_limit: 768m
|
||||||
|
environment:
|
||||||
|
<<: *otel-environment
|
||||||
|
OTEL_SERVICE_NAME: messenger-worker
|
||||||
|
DATABASE_URL: ${DATABASE_URL}
|
||||||
|
REDIS_URL: ${REDIS_URL}
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
|
||||||
|
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||||
|
S3_REGION: ${S3_REGION:-us-east-1}
|
||||||
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||||
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api}
|
||||||
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api}
|
||||||
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||||
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||||
|
BROKER_URL: ${BROKER_URL}
|
||||||
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||||
|
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||||
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||||
|
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||||
|
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90}
|
||||||
|
RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2}
|
||||||
|
RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60}
|
||||||
|
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||||
|
THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85}
|
||||||
|
CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85}
|
||||||
|
OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82}
|
||||||
|
depends_on:
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
broker:
|
||||||
|
condition: service_healthy
|
||||||
|
s3-bootstrap:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"]
|
||||||
|
interval: 10s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 3
|
||||||
|
restart: unless-stopped
|
||||||
|
stop_grace_period: 70s
|
||||||
|
|
||||||
|
lb:
|
||||||
|
image: haproxy:3.2.26-alpine
|
||||||
|
cpus: "0.25"
|
||||||
|
mem_limit: 128m
|
||||||
|
ports:
|
||||||
|
- "${APP_PORT:-8080}:8080"
|
||||||
|
volumes:
|
||||||
|
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||||
|
depends_on:
|
||||||
|
api:
|
||||||
|
condition: service_healthy
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 2s
|
||||||
|
retries: 5
|
||||||
|
stop_grace_period: 5s
|
||||||
|
|
||||||
|
otel-collector:
|
||||||
|
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 256m
|
||||||
|
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||||
|
depends_on:
|
||||||
|
loki:
|
||||||
|
condition: service_started
|
||||||
|
tempo:
|
||||||
|
condition: service_started
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
image: prom/prometheus:v3.2.1
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 384m
|
||||||
|
command:
|
||||||
|
- --config.file=/etc/prometheus/prometheus.yml
|
||||||
|
- --storage.tsdb.path=/prometheus
|
||||||
|
- --storage.tsdb.retention.time=7d
|
||||||
|
- --storage.tsdb.retention.size=1GB
|
||||||
|
volumes:
|
||||||
|
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||||
|
- prometheus_data:/prometheus
|
||||||
|
depends_on:
|
||||||
|
otel-collector:
|
||||||
|
condition: service_started
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:3.4.2
|
||||||
|
user: "0"
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 384m
|
||||||
|
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||||
|
- loki_data:/loki
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
tempo:
|
||||||
|
image: grafana/tempo:2.7.2
|
||||||
|
user: "0"
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 384m
|
||||||
|
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||||
|
volumes:
|
||||||
|
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||||
|
- tempo_data:/var/tempo
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:11.6.0
|
||||||
|
cpus: "0.5"
|
||||||
|
mem_limit: 384m
|
||||||
|
environment:
|
||||||
|
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||||
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana}
|
||||||
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||||
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||||
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||||
|
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||||
|
volumes:
|
||||||
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||||
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||||
|
- ./infra/grafana/runbooks:/var/lib/grafana/runbooks:ro
|
||||||
|
- grafana_data:/var/lib/grafana
|
||||||
|
depends_on:
|
||||||
|
prometheus:
|
||||||
|
condition: service_started
|
||||||
|
loki:
|
||||||
|
condition: service_started
|
||||||
|
tempo:
|
||||||
|
condition: service_started
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 20
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
|
redis_data:
|
||||||
|
s3_data:
|
||||||
|
broker_data:
|
||||||
|
prometheus_data:
|
||||||
|
loki_data:
|
||||||
|
tempo_data:
|
||||||
|
grafana_data:
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# Результаты повторной проверки 2026-11-20
|
||||||
|
|
||||||
|
Стенд: Apple M4 Pro, 24 GiB RAM; Docker Desktop VM 12 CPU / 9.70 GiB;
|
||||||
|
лимиты сервисов из `compose.yaml`, только synthetic data.
|
||||||
|
|
||||||
|
```text
|
||||||
|
make test
|
||||||
|
PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
|
||||||
|
make test-observability
|
||||||
|
PASS: dashboard, alerts, metrics, correlated logs and cross-process trace
|
||||||
|
|
||||||
|
make test-retry-trace
|
||||||
|
{"scenario":"retry_trace","attachment_id":"7ee0da1b-0c50-4d2e-9bf8-e2c68efb2ce4",
|
||||||
|
"trace_id":"e8e2aff13b7bdb632c9cec5007dcc6d4","producer_spans":2,
|
||||||
|
"consumer_spans":2,"result":"PASS"}
|
||||||
|
|
||||||
|
make test-alert-worker
|
||||||
|
{"scenario":"worker_outage","firing_state":"Alerting","detect_seconds":61.8,
|
||||||
|
"resolved_state":"Normal","recover_seconds":15.5,"result":"PASS"}
|
||||||
|
|
||||||
|
make test-alert-database
|
||||||
|
{"scenario":"database_outage","firing_state":"Alerting","detect_seconds":72.2,
|
||||||
|
"resolved_state":"Normal","recover_seconds":113.1,
|
||||||
|
"trace_id":"34f96d6fc934f7c54075c44a7b39c176","db_span":"connect",
|
||||||
|
"db_system":"postgresql","result":"PASS"}
|
||||||
|
|
||||||
|
make test-telemetry-outage
|
||||||
|
{"scenario":"telemetry_backend_outage","business_smoke":"PASS",
|
||||||
|
"collector":"running","duration_seconds":49.5,"rss_before_mib":194.6,
|
||||||
|
"rss_peak_mib":198.0,"memory_limit_mib":256,"failed_log_records":34,
|
||||||
|
"failed_spans":30,"result":"PASS"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Worker incident использовал attachment `a88a0cfd-bbda-4151-8584-de27154a8dcb`.
|
||||||
|
Это идентификатор только локальных синтетических данных; credentials/tokens/body в
|
||||||
|
доказательство не сохранялись.
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"lab": 6,
|
||||||
|
"implementation_commit": "9e6428d",
|
||||||
|
"host": "Apple M4 Pro, 24 GiB RAM",
|
||||||
|
"docker_vm": "12 CPU, 9.70 GiB RAM; per-service limits from compose.yaml",
|
||||||
|
"workload": "closed-loop POST messages; no automatic retries",
|
||||||
|
"requested_duration_seconds": 120.0,
|
||||||
|
"elapsed_seconds": 120.084,
|
||||||
|
"concurrency": 8,
|
||||||
|
"successful_requests": 25638,
|
||||||
|
"generated_messages": 25638,
|
||||||
|
"failed_requests": 0,
|
||||||
|
"error_ratio": 0.0,
|
||||||
|
"successful_rps": 213.501,
|
||||||
|
"successful_latency_ms": {
|
||||||
|
"p50": 17.158,
|
||||||
|
"p95": 99.774,
|
||||||
|
"p99": 171.24
|
||||||
|
},
|
||||||
|
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
{
|
||||||
|
"annotations": {"list": []},
|
||||||
|
"editable": false,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"description": "Business requests per second; health routes are excluded.",
|
||||||
|
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 0},
|
||||||
|
"id": 1,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [{"expr": "sum by (http_route, http_request_method) (rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[$__rate_interval]))", "legendFormat": "{{http_request_method}} {{http_route}}", "refId": "A"}],
|
||||||
|
"title": "API RPS",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"description": "5xx business requests divided by all business requests over 2 minutes.",
|
||||||
|
"fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 0},
|
||||||
|
"id": 2,
|
||||||
|
"options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}},
|
||||||
|
"targets": [{"expr": "(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\",http_response_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[2m])), 0.001)) or vector(0)", "legendFormat": "5xx ratio", "refId": "A"}],
|
||||||
|
"title": "API server error ratio",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"description": "Percentiles are computed from aggregate histogram buckets across API replicas.",
|
||||||
|
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 0},
|
||||||
|
"id": 3,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||||
|
{"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||||
|
{"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||||
|
],
|
||||||
|
"title": "API duration p50 / p95 / p99",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8},
|
||||||
|
"id": 4,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", "legendFormat": "{{outcome}}", "refId": "A"}],
|
||||||
|
"title": "Jobs by outcome",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 8},
|
||||||
|
"id": 5,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||||
|
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||||
|
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||||
|
],
|
||||||
|
"title": "Worker duration p50 / p95 / p99",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"description": "Time from durable acceptance to ready commit.",
|
||||||
|
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 8},
|
||||||
|
"id": 6,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"},
|
||||||
|
{"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"},
|
||||||
|
{"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"}
|
||||||
|
],
|
||||||
|
"title": "End-to-end time to ready",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"description": "Logical non-terminal jobs from PostgreSQL and age of the oldest one.",
|
||||||
|
"fieldConfig": {"defaults": {}, "overrides": [{"matcher": {"id": "byName", "options": "oldest age"}, "properties": [{"id": "unit", "value": "s"}]}]},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 16},
|
||||||
|
"id": 7,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "max(messenger_queue_depth{messaging_destination_name=\"attachments.process\"})", "legendFormat": "depth", "refId": "A"},
|
||||||
|
{"expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process\"})", "legendFormat": "oldest age", "refId": "B"}
|
||||||
|
],
|
||||||
|
"title": "Queue depth and oldest job",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 16},
|
||||||
|
"id": 8,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [
|
||||||
|
{"expr": "messenger_db_pool_used", "legendFormat": "used {{service_name}} {{service_instance_id}}", "refId": "A"},
|
||||||
|
{"expr": "messenger_db_pool_waiting", "legendFormat": "waiting {{service_name}} {{service_instance_id}}", "refId": "B"}
|
||||||
|
],
|
||||||
|
"title": "PostgreSQL pool used / waiting",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 16},
|
||||||
|
"id": 9,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "saturation", "refId": "A"}],
|
||||||
|
"title": "Worker saturation",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
|
||||||
|
"id": 10,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [{"expr": "rate(process_cpu_time_seconds_total[5m])", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}],
|
||||||
|
"title": "Process CPU",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||||
|
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
|
||||||
|
"id": 11,
|
||||||
|
"options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}},
|
||||||
|
"targets": [{"expr": "process_memory_rss_bytes", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}],
|
||||||
|
"title": "Process RSS",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "loki", "uid": "loki"},
|
||||||
|
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
|
||||||
|
"id": 12,
|
||||||
|
"options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true},
|
||||||
|
"targets": [{"expr": "{service_name=~\"$service\"} | json | http_route!~\"/health/.*\" | line_format \"{{.severity}} {{.service_name}} {{.event}}{{if .http_method}} {{.http_method}} {{.http_route}} status={{.http_status_code}} duration={{.duration_seconds}}{{end}} trace_id={{.trace_id}} {{.message}}\"", "refId": "A"}],
|
||||||
|
"title": "Correlated application logs",
|
||||||
|
"type": "logs"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "5s",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["mtusi", "lab6", "opentelemetry"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"current": {"selected": false, "text": ".*", "value": ".*"},
|
||||||
|
"datasource": {"type": "loki", "uid": "loki"},
|
||||||
|
"definition": "label_values(service_name)",
|
||||||
|
"hide": 0,
|
||||||
|
"includeAll": true,
|
||||||
|
"label": "Service",
|
||||||
|
"multi": true,
|
||||||
|
"name": "service",
|
||||||
|
"options": [],
|
||||||
|
"query": "label_values(service_name)",
|
||||||
|
"refresh": 1,
|
||||||
|
"regex": "",
|
||||||
|
"skipUrlSync": false,
|
||||||
|
"sort": 1,
|
||||||
|
"type": "query"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"time": {"from": "now-30m", "to": "now"},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "Messenger Observability",
|
||||||
|
"uid": "messenger-observability",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
groups:
|
||||||
|
- orgId: 1
|
||||||
|
name: Messenger SLO alerts
|
||||||
|
folder: MTUSI Lab 6
|
||||||
|
interval: 10s
|
||||||
|
rules:
|
||||||
|
- uid: messenger-api-high-error-rate
|
||||||
|
title: ApiHighServerErrorRate
|
||||||
|
condition: C
|
||||||
|
data:
|
||||||
|
- refId: A
|
||||||
|
relativeTimeRange:
|
||||||
|
from: 120
|
||||||
|
to: 0
|
||||||
|
datasourceUid: prometheus
|
||||||
|
model:
|
||||||
|
datasource:
|
||||||
|
type: prometheus
|
||||||
|
uid: prometheus
|
||||||
|
editorMode: code
|
||||||
|
expr: '(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*",http_response_status_class="5xx"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*"}[2m])), 0.001)) or vector(0)'
|
||||||
|
instant: true
|
||||||
|
intervalMs: 1000
|
||||||
|
maxDataPoints: 43200
|
||||||
|
range: false
|
||||||
|
refId: A
|
||||||
|
- refId: C
|
||||||
|
relativeTimeRange:
|
||||||
|
from: 0
|
||||||
|
to: 0
|
||||||
|
datasourceUid: __expr__
|
||||||
|
model:
|
||||||
|
conditions:
|
||||||
|
- evaluator:
|
||||||
|
params: [0.05]
|
||||||
|
type: gt
|
||||||
|
operator:
|
||||||
|
type: and
|
||||||
|
query:
|
||||||
|
params: [C]
|
||||||
|
reducer:
|
||||||
|
params: []
|
||||||
|
type: last
|
||||||
|
type: query
|
||||||
|
datasource:
|
||||||
|
type: __expr__
|
||||||
|
uid: __expr__
|
||||||
|
expression: A
|
||||||
|
refId: C
|
||||||
|
type: threshold
|
||||||
|
dashboardUid: messenger-observability
|
||||||
|
panelId: 2
|
||||||
|
noDataState: OK
|
||||||
|
execErrState: Error
|
||||||
|
for: 1m
|
||||||
|
annotations:
|
||||||
|
summary: Более 5% бизнес-запросов завершаются 5xx.
|
||||||
|
runbook_url: /var/lib/grafana/runbooks/api-high-error-rate.md
|
||||||
|
labels:
|
||||||
|
service: messenger-api
|
||||||
|
severity: warning
|
||||||
|
isPaused: false
|
||||||
|
- uid: messenger-queue-oldest-growing
|
||||||
|
title: QueueOldestJobGrowing
|
||||||
|
condition: C
|
||||||
|
data:
|
||||||
|
- refId: A
|
||||||
|
relativeTimeRange:
|
||||||
|
from: 60
|
||||||
|
to: 0
|
||||||
|
datasourceUid: prometheus
|
||||||
|
model:
|
||||||
|
datasource:
|
||||||
|
type: prometheus
|
||||||
|
uid: prometheus
|
||||||
|
editorMode: code
|
||||||
|
expr: '(max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process"}) > 20 and max(messenger_queue_depth{messaging_destination_name="attachments.process"}) > 0) or vector(0)'
|
||||||
|
instant: true
|
||||||
|
intervalMs: 1000
|
||||||
|
maxDataPoints: 43200
|
||||||
|
range: false
|
||||||
|
refId: A
|
||||||
|
- refId: C
|
||||||
|
relativeTimeRange:
|
||||||
|
from: 0
|
||||||
|
to: 0
|
||||||
|
datasourceUid: __expr__
|
||||||
|
model:
|
||||||
|
conditions:
|
||||||
|
- evaluator:
|
||||||
|
params: [0.5]
|
||||||
|
type: gt
|
||||||
|
operator:
|
||||||
|
type: and
|
||||||
|
query:
|
||||||
|
params: [C]
|
||||||
|
reducer:
|
||||||
|
params: []
|
||||||
|
type: last
|
||||||
|
type: query
|
||||||
|
datasource:
|
||||||
|
type: __expr__
|
||||||
|
uid: __expr__
|
||||||
|
expression: A
|
||||||
|
refId: C
|
||||||
|
type: threshold
|
||||||
|
dashboardUid: messenger-observability
|
||||||
|
panelId: 7
|
||||||
|
noDataState: OK
|
||||||
|
execErrState: Error
|
||||||
|
for: 15s
|
||||||
|
annotations:
|
||||||
|
summary: Возраст старейшей незавершённой job превысил 20 секунд.
|
||||||
|
runbook_url: /var/lib/grafana/runbooks/queue-oldest-job.md
|
||||||
|
labels:
|
||||||
|
service: messenger-worker
|
||||||
|
severity: warning
|
||||||
|
isPaused: false
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: Messenger
|
||||||
|
orgId: 1
|
||||||
|
folder: MTUSI Lab 6
|
||||||
|
type: file
|
||||||
|
disableDeletion: true
|
||||||
|
updateIntervalSeconds: 10
|
||||||
|
allowUiUpdates: false
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
|
||||||
|
deleteDatasources:
|
||||||
|
- name: Prometheus
|
||||||
|
orgId: 1
|
||||||
|
- name: Loki
|
||||||
|
orgId: 1
|
||||||
|
- name: Tempo
|
||||||
|
orgId: 1
|
||||||
|
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
uid: prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://prometheus:9090
|
||||||
|
isDefault: true
|
||||||
|
editable: false
|
||||||
|
- name: Loki
|
||||||
|
uid: loki
|
||||||
|
type: loki
|
||||||
|
access: proxy
|
||||||
|
url: http://loki:3100
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
derivedFields:
|
||||||
|
- name: TraceID
|
||||||
|
matcherRegex: 'trace[_]?id["=: ]+([0-9a-f]{32})'
|
||||||
|
datasourceUid: tempo
|
||||||
|
url: '$${__value.raw}'
|
||||||
|
- name: Tempo
|
||||||
|
uid: tempo
|
||||||
|
type: tempo
|
||||||
|
access: proxy
|
||||||
|
url: http://tempo:3200
|
||||||
|
editable: false
|
||||||
|
jsonData:
|
||||||
|
tracesToLogsV2:
|
||||||
|
datasourceUid: loki
|
||||||
|
spanStartTimeShift: -1m
|
||||||
|
spanEndTimeShift: 1m
|
||||||
|
filterByTraceID: true
|
||||||
|
filterBySpanID: false
|
||||||
|
serviceMap:
|
||||||
|
datasourceUid: prometheus
|
||||||
|
nodeGraph:
|
||||||
|
enabled: true
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# ApiHighServerErrorRate
|
||||||
|
|
||||||
|
Alert использует только 5xx бизнес-запросов и исключает `/health/*`. Клиентские 4xx
|
||||||
|
не считаются отказом сервера.
|
||||||
|
|
||||||
|
1. Открыть панели API error ratio и DB pool.
|
||||||
|
2. В Tempo найти медленные/error server spans за окно alert.
|
||||||
|
3. Сравнить длительность server span с дочерними PostgreSQL/S3 spans.
|
||||||
|
4. В Loki отфильтровать `{service_name="messenger-api"} | trace_id="..."`.
|
||||||
|
5. Проверить readiness зависимостей, не удаляя volumes и не перезапуская весь стенд.
|
||||||
|
|
||||||
|
После восстановления убедиться, что новые запросы успешны, error ratio падает и alert
|
||||||
|
переходит в Normal. Короткий единичный 5xx может не сработать из-за окна и `for: 1m`.
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# QueueOldestJobGrowing
|
||||||
|
|
||||||
|
Alert означает, что в PostgreSQL остаётся хотя бы одна `queued`/`processing` job старше
|
||||||
|
20 секунд. Это возраст backlog, а не duration одного consumer span.
|
||||||
|
|
||||||
|
1. Проверить queue depth, oldest age и worker busy/capacity.
|
||||||
|
2. В Loki открыть `{service_name="messenger-worker"}` и проверить retry/failure events.
|
||||||
|
3. По trace ID перейти в Tempo и проверить consumer, S3 и PostgreSQL spans.
|
||||||
|
4. Восстановить worker или отказавшую зависимость, не создавая новую attachment job.
|
||||||
|
5. Убедиться, что исходная job стала `ready`, backlog уменьшился и alert resolved.
|
||||||
|
|
||||||
|
Порог учебный и рассчитан на локальную демонстрацию; для production его связывают с
|
||||||
|
реальным SLO времени до ready.
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
global
|
||||||
|
log stdout format raw local0
|
||||||
|
|
||||||
|
defaults
|
||||||
|
log global
|
||||||
|
mode http
|
||||||
|
option httplog
|
||||||
|
timeout connect 2s
|
||||||
|
timeout client 30s
|
||||||
|
timeout server 30s
|
||||||
|
timeout check 2s
|
||||||
|
|
||||||
|
resolvers docker
|
||||||
|
nameserver docker_dns 127.0.0.11:53
|
||||||
|
resolve_retries 3
|
||||||
|
timeout resolve 1s
|
||||||
|
timeout retry 1s
|
||||||
|
hold valid 1s
|
||||||
|
hold obsolete 1s
|
||||||
|
|
||||||
|
frontend public
|
||||||
|
bind :8080
|
||||||
|
default_backend api_backend
|
||||||
|
|
||||||
|
backend api_backend
|
||||||
|
balance roundrobin
|
||||||
|
option httpchk
|
||||||
|
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||||
|
http-check expect status 200
|
||||||
|
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||||
|
server-template api 2 api:8080
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
|
||||||
|
common:
|
||||||
|
path_prefix: /loki
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2024-01-01
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
storage_config:
|
||||||
|
filesystem:
|
||||||
|
directory: /loki/chunks
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
allow_structured_metadata: true
|
||||||
|
retention_period: 168h
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
retention_enabled: true
|
||||||
|
delete_request_store: filesystem
|
||||||
|
|
||||||
|
analytics:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
extensions:
|
||||||
|
health_check:
|
||||||
|
endpoint: 0.0.0.0:13133
|
||||||
|
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
processors:
|
||||||
|
memory_limiter:
|
||||||
|
check_interval: 1s
|
||||||
|
limit_mib: 192
|
||||||
|
spike_limit_mib: 48
|
||||||
|
batch:
|
||||||
|
timeout: 1s
|
||||||
|
send_batch_size: 512
|
||||||
|
send_batch_max_size: 1024
|
||||||
|
|
||||||
|
exporters:
|
||||||
|
prometheus:
|
||||||
|
endpoint: 0.0.0.0:8889
|
||||||
|
resource_to_telemetry_conversion:
|
||||||
|
enabled: true
|
||||||
|
otlphttp/loki:
|
||||||
|
endpoint: http://loki:3100/otlp
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
num_consumers: 2
|
||||||
|
queue_size: 500
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 5s
|
||||||
|
max_elapsed_time: 30s
|
||||||
|
otlp/tempo:
|
||||||
|
endpoint: tempo:4317
|
||||||
|
tls:
|
||||||
|
insecure: true
|
||||||
|
sending_queue:
|
||||||
|
enabled: true
|
||||||
|
num_consumers: 2
|
||||||
|
queue_size: 500
|
||||||
|
retry_on_failure:
|
||||||
|
enabled: true
|
||||||
|
initial_interval: 1s
|
||||||
|
max_interval: 5s
|
||||||
|
max_elapsed_time: 30s
|
||||||
|
|
||||||
|
service:
|
||||||
|
extensions: [health_check]
|
||||||
|
telemetry:
|
||||||
|
metrics:
|
||||||
|
level: detailed
|
||||||
|
readers:
|
||||||
|
- pull:
|
||||||
|
exporter:
|
||||||
|
prometheus:
|
||||||
|
host: 0.0.0.0
|
||||||
|
port: 8888
|
||||||
|
pipelines:
|
||||||
|
metrics:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [prometheus]
|
||||||
|
logs:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlphttp/loki]
|
||||||
|
traces:
|
||||||
|
receivers: [otlp]
|
||||||
|
processors: [memory_limiter, batch]
|
||||||
|
exporters: [otlp/tempo]
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
global:
|
||||||
|
scrape_interval: 5s
|
||||||
|
evaluation_interval: 5s
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: otel-application-metrics
|
||||||
|
static_configs:
|
||||||
|
- targets: [otel-collector:8889]
|
||||||
|
- job_name: otel-collector
|
||||||
|
static_configs:
|
||||||
|
- targets: [otel-collector:8888]
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:ListBucket"],
|
||||||
|
"Resource": ["arn:aws:s3:::BUCKET_NAME"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::BUCKET_NAME/*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
server:
|
||||||
|
http_listen_port: 3200
|
||||||
|
|
||||||
|
distributor:
|
||||||
|
receivers:
|
||||||
|
otlp:
|
||||||
|
protocols:
|
||||||
|
grpc:
|
||||||
|
endpoint: 0.0.0.0:4317
|
||||||
|
http:
|
||||||
|
endpoint: 0.0.0.0:4318
|
||||||
|
|
||||||
|
ingester:
|
||||||
|
max_block_duration: 5m
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
compaction:
|
||||||
|
block_retention: 24h
|
||||||
|
|
||||||
|
storage:
|
||||||
|
trace:
|
||||||
|
backend: local
|
||||||
|
wal:
|
||||||
|
path: /var/tempo/wal
|
||||||
|
local:
|
||||||
|
path: /var/tempo/blocks
|
||||||
|
|
||||||
|
usage_report:
|
||||||
|
reporting_enabled: false
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
fastapi==0.115.12
|
||||||
|
uvicorn[standard]==0.34.2
|
||||||
|
SQLAlchemy[asyncio]==2.0.40
|
||||||
|
asyncpg==0.30.0
|
||||||
|
alembic==1.15.2
|
||||||
|
redis==5.2.1
|
||||||
|
argon2-cffi==25.1.0
|
||||||
|
minio==7.2.20
|
||||||
|
Pillow==13.0.0
|
||||||
|
aio-pika==9.5.5
|
||||||
|
opentelemetry-api==1.31.1
|
||||||
|
opentelemetry-sdk==1.31.1
|
||||||
|
opentelemetry-exporter-otlp-proto-http==1.31.1
|
||||||
|
opentelemetry-instrumentation-fastapi==0.52b1
|
||||||
|
opentelemetry-instrumentation-sqlalchemy==0.52b1
|
||||||
|
psutil==7.0.0
|
||||||
+28
-1
@@ -1,6 +1,11 @@
|
|||||||
# Публичные проверки
|
# Публичные проверки
|
||||||
|
|
||||||
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
|
Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; публичные
|
||||||
|
проверки не создают готовое приложение и сами не управляют Docker. Отказные проверки
|
||||||
|
lab6 ниже управляют только явно указанными Compose-сервисами. `make check` можно
|
||||||
|
выполнить на исходном skeleton, `make test` — только после реализации API. Полная
|
||||||
|
OpenAPI-валидация отдельно: `make validate` после установки
|
||||||
|
`requirements-dev.txt` в venv.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
make check
|
make check
|
||||||
@@ -13,6 +18,28 @@ python3 tests/smoke.py --base-url http://localhost:8080
|
|||||||
|
|
||||||
Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются.
|
Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются.
|
||||||
|
|
||||||
|
## Наблюдаемость и контролируемые отказы, lab6
|
||||||
|
|
||||||
|
При запущенном Compose-стенде доступны дополнительные проверки реализации:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make test-observability # dashboard, alerts, Prometheus, Loki и Tempo
|
||||||
|
make test-retry-trace # временная S3-ошибка и retry в одной трассе
|
||||||
|
make test-alert-worker # временно останавливает и восстанавливает worker
|
||||||
|
make test-alert-database # временно останавливает и восстанавливает PostgreSQL
|
||||||
|
make test-telemetry-outage # временно останавливает и восстанавливает Loki/Tempo
|
||||||
|
```
|
||||||
|
|
||||||
|
Отказные сценарии изменяют состояние только перечисленных контейнеров и всегда
|
||||||
|
выполняют восстановление в `finally`. Используются уникальные синтетические данные;
|
||||||
|
секреты и tokens не печатаются. Retry-тест временно отключает S3 и проверяет
|
||||||
|
две producer/consumer попытки в одной трассе. Worker-тест ждёт `QueueOldestJobGrowing` в
|
||||||
|
`Alerting`, затем `Normal`; DB-тест аналогично проверяет `ApiHighServerErrorRate`.
|
||||||
|
Тест telemetry outage держит backends выключенными более 30 с и фиксирует RSS
|
||||||
|
Collector и `send_failed` counters.
|
||||||
|
Окна alerts намеренно делают эти два теста не мгновенными. Запускайте их
|
||||||
|
последовательно, а не параллельно.
|
||||||
|
|
||||||
## Проверка сохранности, lab2+
|
## Проверка сохранности, lab2+
|
||||||
|
|
||||||
Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
|
Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
|
||||||
|
|||||||
@@ -0,0 +1,230 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Воспроизводит ограниченные отказы worker и PostgreSQL для проверки alerts."""
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
from client import Client
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
|
||||||
|
|
||||||
|
def compose(*args: str, capture: bool = False):
|
||||||
|
"""Запускает Docker Compose из корня проекта и при необходимости захватывает вывод."""
|
||||||
|
|
||||||
|
return subprocess.run(
|
||||||
|
["docker", "compose", *args],
|
||||||
|
cwd=ROOT,
|
||||||
|
check=True,
|
||||||
|
text=True,
|
||||||
|
capture_output=capture,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def grafana_rules() -> dict:
|
||||||
|
"""Получает текущие состояния provisioned alert rules через API Grafana."""
|
||||||
|
|
||||||
|
user = os.getenv("GRAFANA_ADMIN_USER", "admin")
|
||||||
|
password = os.getenv("GRAFANA_ADMIN_PASSWORD", "change-me-grafana")
|
||||||
|
token = base64.b64encode(f"{user}:{password}".encode()).decode()
|
||||||
|
result = compose(
|
||||||
|
"exec", "-T", "grafana", "wget", "-qO-",
|
||||||
|
"--header", f"Authorization: Basic {token}",
|
||||||
|
"http://127.0.0.1:3000/api/prometheus/grafana/api/v1/rules",
|
||||||
|
capture=True,
|
||||||
|
)
|
||||||
|
return json.loads(result.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def internal_json(url: str) -> dict:
|
||||||
|
"""Читает JSON из внутреннего observability-сервиса через Grafana-контейнер."""
|
||||||
|
|
||||||
|
result = compose(
|
||||||
|
"exec", "-T", "grafana", "wget", "-qO-", url, capture=True
|
||||||
|
)
|
||||||
|
return json.loads(result.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def database_failure_evidence(start_ns: int) -> dict[str, str]:
|
||||||
|
"""Находит связанный лог и ошибочный PostgreSQL CLIENT span."""
|
||||||
|
|
||||||
|
deadline = time.monotonic() + 30
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
query = urllib.parse.urlencode({
|
||||||
|
"query": (
|
||||||
|
'{service_name="messenger-api"} | json '
|
||||||
|
'| logger="app.error" | message=~".*(database|socket).*"'
|
||||||
|
),
|
||||||
|
"start": str(start_ns),
|
||||||
|
"end": str(time.time_ns()),
|
||||||
|
"direction": "backward",
|
||||||
|
"limit": "20",
|
||||||
|
})
|
||||||
|
loki = internal_json(
|
||||||
|
"http://loki:3100/loki/api/v1/query_range?" + query
|
||||||
|
)
|
||||||
|
streams = loki.get("data", {}).get("result", [])
|
||||||
|
if not streams:
|
||||||
|
time.sleep(2)
|
||||||
|
continue
|
||||||
|
entry = streams[0]["values"][0]
|
||||||
|
metadata = entry[2] if len(entry) > 2 else {}
|
||||||
|
body = json.loads(entry[1])
|
||||||
|
trace_id = metadata.get("trace_id") or body.get("trace_id")
|
||||||
|
if not trace_id:
|
||||||
|
time.sleep(2)
|
||||||
|
continue
|
||||||
|
trace = internal_json(f"http://tempo:3200/api/traces/{trace_id}")
|
||||||
|
for batch in trace.get("batches", []):
|
||||||
|
scopes = batch.get(
|
||||||
|
"scopeSpans", batch.get("instrumentationLibrarySpans", [])
|
||||||
|
)
|
||||||
|
for scope in scopes:
|
||||||
|
for span in scope.get("spans", []):
|
||||||
|
attributes = {
|
||||||
|
item["key"]: next(iter(item.get("value", {}).values()), "")
|
||||||
|
for item in span.get("attributes", [])
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
span.get("kind") == "SPAN_KIND_CLIENT"
|
||||||
|
and attributes.get("db.system") == "postgresql"
|
||||||
|
and span.get("status", {}).get("code") == "STATUS_CODE_ERROR"
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
"trace_id": trace_id,
|
||||||
|
"log_message": body.get("message", ""),
|
||||||
|
"db_span": span.get("name", ""),
|
||||||
|
"db_system": "postgresql",
|
||||||
|
}
|
||||||
|
time.sleep(2)
|
||||||
|
raise RuntimeError("linked PostgreSQL error log/trace was not found")
|
||||||
|
|
||||||
|
|
||||||
|
def rule_state(title: str) -> str:
|
||||||
|
"""Находит alert по заголовку и возвращает его текущее состояние."""
|
||||||
|
|
||||||
|
groups = grafana_rules().get("data", {}).get("groups", [])
|
||||||
|
for group in groups:
|
||||||
|
for rule in group.get("rules", []):
|
||||||
|
if rule.get("name") == title:
|
||||||
|
alerts = rule.get("alerts", [])
|
||||||
|
return alerts[0].get("state", rule.get("state", "unknown")) if alerts else rule.get("state", "unknown")
|
||||||
|
raise RuntimeError(f"Grafana rule not found: {title}")
|
||||||
|
|
||||||
|
|
||||||
|
def wait_state(title: str, expected: set[str], timeout: int) -> tuple[str, float]:
|
||||||
|
"""Ожидает одно из требуемых состояний alert и возвращает время ожидания."""
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
deadline = started + timeout
|
||||||
|
last = "unknown"
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
last = rule_state(title)
|
||||||
|
if last.lower() in expected:
|
||||||
|
return last, time.monotonic() - started
|
||||||
|
time.sleep(5)
|
||||||
|
raise RuntimeError(f"{title}: expected {sorted(expected)}, last state {last}")
|
||||||
|
|
||||||
|
|
||||||
|
def worker_scenario(base_url: str):
|
||||||
|
"""Останавливает worker и проверяет переход queue alert в Alerting и Normal."""
|
||||||
|
|
||||||
|
client = Client(base_url, timeout=10)
|
||||||
|
alice, bob, _, chat = client.bootstrap()
|
||||||
|
compose("stop", "worker")
|
||||||
|
try:
|
||||||
|
attachment, _ = client.attachment(alice, chat)
|
||||||
|
firing, detect_seconds = wait_state(
|
||||||
|
"QueueOldestJobGrowing", {"alerting", "firing"}, 100
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
compose("start", "worker")
|
||||||
|
ready = client.wait_ready(bob, attachment, wait_seconds=90)
|
||||||
|
normal, recover_seconds = wait_state(
|
||||||
|
"QueueOldestJobGrowing", {"normal", "inactive"}, 90
|
||||||
|
)
|
||||||
|
print(json.dumps({
|
||||||
|
"scenario": "worker_outage",
|
||||||
|
"attachment_id": ready["id"],
|
||||||
|
"firing_state": firing,
|
||||||
|
"detect_seconds": round(detect_seconds, 1),
|
||||||
|
"resolved_state": normal,
|
||||||
|
"recover_seconds": round(recover_seconds, 1),
|
||||||
|
"result": "PASS",
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
def database_scenario(base_url: str):
|
||||||
|
"""Останавливает PostgreSQL и проверяет firing/resolved для alert по 5xx."""
|
||||||
|
|
||||||
|
Client(base_url).call("GET", "/health/ready", schema="Health")
|
||||||
|
incident_started_ns = time.time_ns()
|
||||||
|
compose("stop", "postgres")
|
||||||
|
load = subprocess.Popen(
|
||||||
|
[
|
||||||
|
"docker", "compose", "-f", "compose.yaml", "-f",
|
||||||
|
"compose.observability-tests.yaml", "run", "--rm",
|
||||||
|
"observability-test", "python", "database_error_load.py",
|
||||||
|
"--base-url", "http://api:8080", "--duration", "150",
|
||||||
|
],
|
||||||
|
cwd=ROOT,
|
||||||
|
)
|
||||||
|
evidence = None
|
||||||
|
try:
|
||||||
|
firing, detect_seconds = wait_state(
|
||||||
|
"ApiHighServerErrorRate", {"alerting", "firing"}, 130
|
||||||
|
)
|
||||||
|
evidence = database_failure_evidence(incident_started_ns)
|
||||||
|
finally:
|
||||||
|
load.terminate()
|
||||||
|
try:
|
||||||
|
load.wait(timeout=10)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
load.kill()
|
||||||
|
compose("start", "postgres")
|
||||||
|
deadline = time.monotonic() + 90
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
Client(base_url).call("GET", "/health/ready", schema="Health")
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
time.sleep(3)
|
||||||
|
else:
|
||||||
|
raise RuntimeError("API readiness did not recover after PostgreSQL restart")
|
||||||
|
normal, recover_seconds = wait_state(
|
||||||
|
"ApiHighServerErrorRate", {"normal", "inactive"}, 180
|
||||||
|
)
|
||||||
|
print(json.dumps({
|
||||||
|
"scenario": "database_outage",
|
||||||
|
"firing_state": firing,
|
||||||
|
"detect_seconds": round(detect_seconds, 1),
|
||||||
|
"resolved_state": normal,
|
||||||
|
"recover_seconds": round(recover_seconds, 1),
|
||||||
|
"trace_id": evidence["trace_id"],
|
||||||
|
"log_message": evidence["log_message"],
|
||||||
|
"db_span": evidence["db_span"],
|
||||||
|
"db_system": evidence["db_system"],
|
||||||
|
"result": "PASS",
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Разбирает аргументы и запускает выбранный отказный сценарий."""
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("scenario", choices=("worker", "database"))
|
||||||
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.scenario == "worker":
|
||||||
|
worker_scenario(args.base_url)
|
||||||
|
else:
|
||||||
|
database_scenario(args.base_url)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Создаёт ограниченный поток DB-зависимых запросов для проверки alert по 5xx."""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Отправляет synthetic POST-запросы заданное время и считает ошибки."""
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--base-url", default="http://api:8080")
|
||||||
|
parser.add_argument("--duration", type=int, default=120)
|
||||||
|
args = parser.parse_args()
|
||||||
|
deadline = time.monotonic() + args.duration
|
||||||
|
attempts = errors = 0
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
suffix = uuid.uuid4().hex[:12]
|
||||||
|
body = json.dumps({
|
||||||
|
"username": "incident_" + suffix,
|
||||||
|
"display_name": "Synthetic incident",
|
||||||
|
"password": "Synthetic-test-only-" + suffix,
|
||||||
|
}).encode()
|
||||||
|
request = urllib.request.Request(
|
||||||
|
args.base_url.rstrip("/") + "/api/v1/users",
|
||||||
|
data=body,
|
||||||
|
method="POST",
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
attempts += 1
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=3) as response:
|
||||||
|
response.read()
|
||||||
|
errors += response.status >= 500
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
error.read()
|
||||||
|
errors += error.code >= 500
|
||||||
|
except (urllib.error.URLError, TimeoutError, OSError):
|
||||||
|
errors += 1
|
||||||
|
time.sleep(0.25)
|
||||||
|
print(f"database incident load: attempts={attempts} errors={errors}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user