diff --git a/.env.example b/.env.example index 2b32655..833199f 100644 --- a/.env.example +++ b/.env.example @@ -1,30 +1,53 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Local synthetic lab environment only. Copy to .env and replace every change-me value. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +LOG_LEVEL=info + +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-postgres +POSTGRES_DB=messenger +DATABASE_URL=postgresql+asyncpg://messenger:change-me-postgres@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 +SESSION_TTL_SECONDS=3600 IDEMPOTENCY_TTL_SECONDS=86400 + +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=change-me-minio-root S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-minio-api MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url + +BROKER_USER=messenger +BROKER_PASSWORD=change-me-rabbitmq +BROKER_URL=amqp://messenger:change-me-rabbitmq@broker:5672/ WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=90 +RETRY_BASE_SECONDS=2 +RETRY_MAX_SECONDS=60 STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. +THUMBNAIL_JPEG_QUALITY=85 +CROP_JPEG_QUALITY=85 +OPTIMIZED_WEBP_QUALITY=82 + OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +OTEL_SERVICE_VERSION=1.6.0 +OTEL_DEPLOYMENT_ENVIRONMENT=lab +OTEL_SDK_DISABLED=false +OTEL_EXPORTER_OTLP_TIMEOUT=2 +OTEL_BSP_MAX_QUEUE_SIZE=2048 +OTEL_BLRP_MAX_QUEUE_SIZE=2048 +OTEL_METRIC_EXPORT_INTERVAL=5000 + +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=change-me-grafana diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2997c10 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.13.15-slim + +WORKDIR /app +COPY requirements.txt . +RUN python -m pip install -r requirements.txt \ + && useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app +COPY app ./app +COPY alembic ./alembic +COPY alembic.ini . + +USER 10001:10001 +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"] + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"] diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 0000000..5780cf9 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,15 @@ +FROM python:3.13.15-slim + +WORKDIR /tests +COPY requirements.txt requirements-dev.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \ + && useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests + +COPY lab.json . +COPY app ./app +COPY contracts ./contracts +COPY tests ./tests +RUN chown -R tests:tests /tests + +USER 10001:10001 +CMD ["python", "-m", "pytest", "-q"] diff --git a/Makefile b/Makefile index 5d86d17..6682322 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,12 @@ BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) +OBSERVABILITY_COMPOSE = docker compose -f compose.yaml -f compose.observability-tests.yaml -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-observability test-retry-trace test-alert-worker test-alert-database test-telemetry-outage load test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI; test: public API; test-observability: Grafana/Prometheus/Loki/Tempo" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +19,21 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-observability: + $(OBSERVABILITY_COMPOSE) run --rm observability-test + +test-retry-trace: + $(PYTHON) tests/retry_trace.py --base-url "$(BASE_URL)" + +test-alert-worker: + $(PYTHON) tests/alert_scenarios.py worker --base-url "$(BASE_URL)" + +test-alert-database: + $(PYTHON) tests/alert_scenarios.py database --base-url "$(BASE_URL)" + +test-telemetry-outage: + $(PYTHON) tests/telemetry_outage.py --base-url "$(BASE_URL)" + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index b97ae42..183c630 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,207 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Сергей Возжин, БПИ2301. +- Целевая оценка: **4**. Трек оценки 5 не заявлен. +- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab6/pulls/61 +- Использованы Python 3.13, FastAPI, OpenTelemetry Python 1.31.1, + OTel Collector Contrib 0.123.0, Prometheus 3.2.1, Loki 3.4.2, + Tempo 2.7.2 и Grafana 11.6.0. Для планирования и реализации применялся ИИ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose v2, GNU Make и Python 3.9+ для host-side тестов. +Все образы и Python-зависимости закреплены в репозитории. + +```sh +cp .env.example .env +# Заменить все change-me значения согласованными локальными секретами. +# Пароли в DATABASE_URL/BROKER_URL должны совпадать с POSTGRES_*/BROKER_*. +docker compose config --quiet +docker compose up -d --build --wait +make check +make test BASE_URL=http://localhost:8080 +make test-observability +``` + +Grafana доступна только локально на `http://127.0.0.1:3000`. Dashboard, +datasources и alerts создаются provisioning-механизмом автоматически. Миграции +Alembic выполняет одноразовый сервис `migrate`; MinIO bucket и технического +пользователя создаёт `s3-bootstrap`. + +Безопасная остановка без удаления данных: + +```sh +docker compose down +``` + +Намеренное удаление всех **локальных учебных** данных выполняется отдельно и +необратимо: + +```sh +docker compose down --volumes +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +client -> HAProxy -> API x2 -> PostgreSQL / Redis / S3 + | + +-> transactional outbox -> publisher -> RabbitMQ -> worker -> S3 + PostgreSQL + +API / publisher / worker -> OTLP HTTP -> OTel Collector + |-> Prometheus exporter <- Prometheus + |-> OTLP logs -> Loki + `-> OTLP traces -> Tempo +Prometheus + Loki + Tempo <- provisioned datasources <- Grafana +``` + +Из ЛР 5 без изменения бизнес-API перенесены две API-реплики, HAProxy, +PostgreSQL, Redis sessions/idempotency, MinIO S3, RabbitMQ, transactional +outbox, publisher, worker и обработка изображений. PostgreSQL остаётся источником +истины для attachments/jobs/outbox. Durable acceptance атомарно создаёт attachment, +job и outbox row; публикация и обработка повторяемы. + +В outbox добавлены nullable `traceparent` и `tracestate`. API сохраняет W3C context +в той же транзакции, publisher восстанавливает его, создаёт PRODUCER span и передаёт +новый context в allowlisted AMQP headers. Worker извлекает headers и создаёт CONSUMER +span. В одной трассе видны `messenger-api`, `messenger-publisher`, +`messenger-worker`, SQL и ручные S3 CLIENT spans. Retry создаёт новую публикацию с +текущим context; повторная обработка остаётся наблюдаемой и идемпотентной. + +SDK использует bounded batch queues по 2048 элементов и timeout 2 с. Collector: +`memory_limiter` 192 MiB со spike 48 MiB, `batch` до 1024 элементов, exporter queues +по 500 элементов, два consumer, exponential retry не дольше 30 с. Поэтому отказ +Loki/Tempo приводит к ограниченной потере telemetry, а не блокировке API или +неограниченному росту RAM. +Collector self-metrics экспортируются через `readers.pull` на `:8888`; +Prometheus сохраняет RSS, размеры очередей и `send_failed` counters. + +Секреты и пользовательское содержимое не экспортируются. `safe_extra` отбрасывает +пароли, tokens, Authorization, secrets, message body, image bytes и signed URL. +Метрики содержат только низкокардинальные route templates, method/status class, +job type/outcome и queue name; UUID и filename labels отсутствуют. Технические IDs +допустимы только в logs/spans для синтетических данных. + +## Сигналы, dashboard и единицы + +| Контракт | Реальная серия | Единица / основные labels | +| --- | --- | --- | +| HTTP requests | `http_server_request_count_total` | requests; method, route template, status class | +| HTTP duration | `http_server_request_duration_seconds` | seconds, histogram | +| Jobs outcome | `messenger_jobs_processed_total` | jobs; job type, outcome | +| Job duration | `messenger_job_processing_duration_seconds` | seconds, histogram | +| Time to ready | `messenger_attachment_time_to_ready_seconds` | seconds, histogram | +| Queue | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | jobs / seconds | +| DB pool | `messenger_db_pool_used`, `messenger_db_pool_waiting` | connections / waiters | +| Worker | `messenger_worker_busy`, `messenger_worker_capacity` | slots | +| Process | `process_cpu_time_seconds_total`, `process_memory_rss_bytes` | CPU seconds / bytes | + +Dashboard UID `messenger-observability` содержит 12 panels: RPS, 5xx ratio, +HTTP p50/p95/p99, jobs по outcome, worker p50/p95/p99, end-to-end p50/p95/p99, +queue depth/oldest, DB pool, worker saturation, CPU, RSS и correlated logs. +Percentiles вычисляются из объединённых buckets всех реплик, например: + +```promql +histogram_quantile(0.99, + sum by (le) (rate(http_server_request_duration_seconds_bucket{ + service_name="messenger-api",http_route!~"/health/.*"}[5m]))) +``` + +Границы HTTP histogram: 5 ms, 10 ms, 25 ms, 50 ms, 100 ms, 250 ms, 500 ms, +1 s, 2.5 s, 5 s. Worker: 50 ms…60 s. Time-to-ready: 100 ms…120 s. +Средние значения и средние p99 отдельных реплик не используются. + +## SLI, SLO и alerts + +API availability SLI: доля бизнес-запросов со status `<500` среди всех +бизнес-запросов; `/health/live` и `/health/ready` исключены. Учебный SLO — не менее +99% good requests в rolling window 30 минут. 4xx входят в denominator, но не в +server-error numerator. + +Pipeline SLI: доля durably accepted attachments, достигших `ready` не позднее +60 секунд. Учебный SLO — не менее 95% за rolling 30 минут при доступном worker. + +Provisioned rules: + +- `ApiHighServerErrorRate`: 5xx / all business requests за 2 минуты > 5%, `for: 1m`; +- `QueueOldestJobGrowing`: oldest job > 20 s при depth > 0, `for: 15s`. + +Пороги и короткие окна выбраны для воспроизводимой учебной демонстрации, а не как +production defaults. У каждого правила есть dashboard/panel link и локальный runbook +в `infra/grafana/runbooks/`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Offline contract | `make check` | PASS | консоль и Git diff | +| Публичный API с telemetry | `make test` | `PASS: lab6 public API scenario` | `tests/smoke.py` | +| Dashboard/alerts/3 backends | `make test-observability` | PASS: dashboard, alerts, metrics, logs, cross-process trace | `tests/observability_smoke.py` | +| API → publisher → worker | тот же smoke после upload | один `trace_id`, services API/publisher/worker | Loki + Tempo API assertion | +| Retry в одной трассе | `make test-retry-trace` | 2 PRODUCER + 2 CONSUMER, SQL/S3, attachment ready | `tests/retry_trace.py` | +| Worker outage | `make test-alert-worker` | Alerting 61.8 s, Normal 15.5 s после запуска | `tests/alert_scenarios.py` | +| PostgreSQL outage | `make test-alert-database` | Alerting/Normal + linked log/failed PostgreSQL span | `tests/alert_scenarios.py` | +| Telemetry outage | `make test-telemetry-outage` | 49.5 s, smoke PASS, bounded RSS, loss counters | `tests/telemetry_outage.py` | +| Конфиги | `docker compose config --quiet`; JSON parse; Grafana logs | provisioning завершён без error | `infra/` и `compose.yaml` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Повторная проверка выполнена 2026-11-20: host Apple M4 Pro, 24 GiB RAM; +Docker Desktop VM — 12 CPU и 9.70 GiB RAM. Каждый Compose-сервис дополнительно +ограничен `cpus`/`mem_limit` из `compose.yaml`. Load evidence создал +25 638 синтетических сообщений за 120 с с concurrency 8. -## Эксперименты и отказы +## Разбор контролируемых инцидентов -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Временная ошибка S3 и retry -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Worker был остановлен до upload, затем S3 выключен только на первую +попытку. Worker сохранил `job.retry_scheduled`, после восстановления S3 +то же attachment дошло до `ready`. Трасса +`e8e2aff13b7bdb632c9cec5007dcc6d4` содержит два PRODUCER и два CONSUMER +span, а также SQL/S3 CLIENT spans. Это доказывает перенос context +через retry outbox event. + +### Worker недоступен + +Гипотеза: API продолжит durable acceptance, RabbitMQ и PostgreSQL сохранят работу, +но `queue_depth` и `queue_oldest_age_seconds` вырастут. Сценарий создал отдельные +синтетические users/chat, остановил только worker, загрузил PNG и опрашивал состояние +Grafana rule. Alert перешёл в `Alerting` за 61.8 с. После `docker compose start worker` +attachment стал ready, backlog уменьшился, rule стал `Normal` через 15.5 с. Потери +бизнес-данных: 0. Диагноз по runbook: HTTP остаётся доступным, jobs не завершаются, +worker saturation отсутствует, oldest age растёт — причина на consumer-side. + +### PostgreSQL недоступен + +Гипотеза: прямые business requests к API дадут 5xx, DB spans/логи укажут connection +failure, а queue-only alert не обязан срабатывать. PostgreSQL остановлен, отдельный +контейнер создавал synthetic `POST /api/v1/users` каждые 250 ms. Error-rate alert +стал `Alerting` за 72.2 с. PostgreSQL запущен в `finally`, readiness восстановилась, +rule вернулся в `Normal` через 113.1 с — старые 5xx должны были выйти из 2-минутного +окна. Потери подтверждённых данных: 0; неуспешные запросы клиент может повторить. +Связанный log и trace `34f96d6fc934f7c54075c44a7b39c176` содержат +socket connection error и `connect` CLIENT span с `db.system=postgresql` и +`STATUS_CODE_ERROR`. Это отделяет отказ БД от просто медленного HTTP handler. + +### Loki и Tempo недоступны + +Оба backend остановлены одновременно на 49.5 с, что дольше настроенного +30-секундного retry. Полный бизнес-smoke прошёл, Collector остался running. +RSS Collector изменился с 194.6 до пиковых 198.0 MiB при container limit +256 MiB. Internal counters зафиксировали потерю 34 log records и 30 spans. +Порог `memory_limiter=192 MiB` — мягкий порог отказа/сброса данных, а не +обещание RSS ниже 192 MiB; жёсткий предел задан Compose-лимитом 256 MiB. +Это ожидаемая bounded telemetry loss: наблюдаемость не является частью +business commit. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Стенд single-node: PostgreSQL, RabbitMQ, MinIO и telemetry backends не имеют HA. +- Sampling всегда 100%; исследование стоимости и tail sampling относится к оценке 5. +- Grafana notification delivery не настроена: по заданию достаточно локальных + firing/resolved states. +- Queue depth берётся из durable job table, поэтому отражает бизнес-backlog, а не + точное число RabbitMQ messages в каждую микросекунду. +- Collector Prometheus exporter хранит только текущее состояние; долговременную + историю сохраняет Prometheus volume. Loki и Tempo используют локальные volumes. +- Для production нужны TLS/mTLS OTLP, secrets manager, HA backends, remote storage, + более длинные SLO windows и проверенные notification routes. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..5c6d9f2 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,36 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = placeholder://configured-at-runtime + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..24524a9 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import async_engine_from_config + +from app.config import load_settings +from app.db.postgres import Base +from app import models # noqa: F401 - imports ORM models into Base.metadata + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _url() -> str: + return load_settings().database_url + + +def run_migrations_offline() -> None: + context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'}) + with context.begin_transaction(): + context.run_migrations() + + +def _run_migrations(connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def _run_migrations_async() -> None: + configuration = config.get_section(config.config_ini_section, {}) + configuration['sqlalchemy.url'] = _url() + connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(_run_migrations_async()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..36cd8ef --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/20261019_0001_initial_schema.py b/alembic/versions/20261019_0001_initial_schema.py new file mode 100644 index 0000000..e72e2f0 --- /dev/null +++ b/alembic/versions/20261019_0001_initial_schema.py @@ -0,0 +1,64 @@ +"""Initial PostgreSQL schema for lab 2. + +Revision ID: 20261019_0001 +Revises: +Create Date: 2026-10-19 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261019_0001' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'users', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('username', sa.String(length=32), nullable=False), + sa.Column('display_name', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('username'), + ) + op.create_table( + 'chats', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('title', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id'), + ) + op.create_table( + 'chat_members', + sa.Column('chat_id', sa.Uuid(), nullable=False), + sa.Column('user_id', sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('chat_id', 'user_id'), + ) + op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id']) + op.create_table( + 'messages', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('chat_id', sa.Uuid(), nullable=False), + sa.Column('sender_id', sa.Uuid(), nullable=False), + sa.Column('text', sa.String(length=4000), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sender_id'], ['users.id']), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id']) + + +def downgrade() -> None: + op.drop_index('messages_history_idx', table_name='messages') + op.drop_table('messages') + op.drop_index('chat_members_user_chat_idx', table_name='chat_members') + op.drop_table('chat_members') + op.drop_table('chats') + op.drop_table('users') diff --git a/alembic/versions/20261023_0002_message_idempotency.py b/alembic/versions/20261023_0002_message_idempotency.py new file mode 100644 index 0000000..60d5eeb --- /dev/null +++ b/alembic/versions/20261023_0002_message_idempotency.py @@ -0,0 +1,35 @@ +"""Add shared message idempotency for lab 3. + +Revision ID: 20261023_0002 +Revises: 20261019_0001 +Create Date: 2026-10-23 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261023_0002' +down_revision = '20261019_0001' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + 'messages', + sa.Column('idempotency_key', sa.String(length=128), nullable=True), + ) + op.create_unique_constraint( + 'messages_sender_chat_idempotency_key', + 'messages', + ['sender_id', 'chat_id', 'idempotency_key'], + ) + + +def downgrade() -> None: + op.drop_constraint( + 'messages_sender_chat_idempotency_key', + 'messages', + type_='unique', + ) + op.drop_column('messages', 'idempotency_key') diff --git a/alembic/versions/20261030_0003_attachments.py b/alembic/versions/20261030_0003_attachments.py new file mode 100644 index 0000000..da72459 --- /dev/null +++ b/alembic/versions/20261030_0003_attachments.py @@ -0,0 +1,69 @@ +"""Add S3 attachment metadata and message links for lab 4. + +Revision ID: 20261030_0003 +Revises: 20261023_0002 +Create Date: 2026-10-30 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261030_0003' +down_revision = '20261023_0002' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'attachments', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('chat_id', sa.Uuid(), nullable=False), + sa.Column('owner_id', sa.Uuid(), nullable=False), + sa.Column('filename', sa.String(length=255), nullable=False), + sa.Column('object_key', sa.String(length=255), nullable=False), + sa.Column('content_type', sa.String(length=32), nullable=False), + sa.Column('size_bytes', sa.BigInteger(), nullable=False), + sa.Column('sha256', sa.String(length=64), nullable=False), + sa.Column('width', sa.Integer(), nullable=False), + sa.Column('height', sa.Integer(), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('error_code', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['owner_id'], ['users.id']), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('object_key', name='attachments_object_key_key'), + ) + op.create_index( + 'attachments_chat_created_idx', + 'attachments', + ['chat_id', 'created_at', 'id'], + ) + op.create_table( + 'message_attachments', + sa.Column('message_id', sa.Uuid(), nullable=False), + sa.Column('attachment_id', sa.Uuid(), nullable=False), + sa.Column('position', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('message_id', 'attachment_id'), + sa.UniqueConstraint( + 'message_id', + 'position', + name='message_attachments_message_position_key', + ), + ) + op.create_index( + 'message_attachments_attachment_idx', + 'message_attachments', + ['attachment_id'], + ) + + +def downgrade() -> None: + op.drop_index('message_attachments_attachment_idx', table_name='message_attachments') + op.drop_table('message_attachments') + op.drop_index('attachments_chat_created_idx', table_name='attachments') + op.drop_table('attachments') diff --git a/alembic/versions/20261116_0004_async_attachments.py b/alembic/versions/20261116_0004_async_attachments.py new file mode 100644 index 0000000..0e37c9c --- /dev/null +++ b/alembic/versions/20261116_0004_async_attachments.py @@ -0,0 +1,126 @@ +"""Add asynchronous attachment jobs, variants and transactional outbox. + +Revision ID: 20261116_0004 +Revises: 20261030_0003 +Create Date: 2026-11-16 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261116_0004' +down_revision = '20261030_0003' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column('attachments', 'object_key', new_column_name='original_key') + op.alter_column('attachments', 'original_key', nullable=True) + op.add_column('attachments', sa.Column('staging_key', sa.String(length=255), nullable=True)) + op.add_column( + 'attachments', + sa.Column('pipeline_version', sa.Integer(), nullable=False, server_default='1'), + ) + op.create_unique_constraint('attachments_staging_key_key', 'attachments', ['staging_key']) + + op.create_table( + 'attachment_variants', + sa.Column('attachment_id', sa.Uuid(), nullable=False), + sa.Column('pipeline_version', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=16), nullable=False), + sa.Column('object_key', sa.String(length=255), nullable=False), + sa.Column('content_type', sa.String(length=32), nullable=False), + sa.Column('size_bytes', sa.BigInteger(), nullable=False), + sa.Column('width', sa.Integer(), nullable=False), + sa.Column('height', sa.Integer(), nullable=False), + sa.Column('sha256', sa.String(length=64), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name='variant_name_check'), + sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('attachment_id', 'pipeline_version', 'name'), + sa.UniqueConstraint('object_key', name='attachment_variants_object_key_key'), + ) + + op.create_table( + 'attachment_jobs', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('attachment_id', sa.Uuid(), nullable=False), + sa.Column('pipeline_version', sa.Integer(), nullable=False), + sa.Column('state', sa.String(length=16), nullable=False), + sa.Column('attempt', sa.Integer(), nullable=False), + sa.Column('max_attempts', sa.Integer(), nullable=False), + sa.Column('next_attempt_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('lease_owner', sa.String(length=128), nullable=True), + sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_error_code', sa.String(length=64), nullable=True), + sa.Column('replay_count', sa.Integer(), nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.CheckConstraint( + "state IN ('queued', 'processing', 'ready', 'failed')", + name='attachment_job_state_check', + ), + sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint( + 'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key' + ), + ) + op.create_index( + 'attachment_jobs_lease_idx', + 'attachment_jobs', + ['state', 'lease_until'], + ) + + op.create_table( + 'outbox_events', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('event_type', sa.String(length=64), nullable=False), + sa.Column('job_id', sa.Uuid(), nullable=False), + sa.Column('payload', sa.JSON(), nullable=False), + sa.Column('available_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('published_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('claim_owner', sa.String(length=128), nullable=True), + sa.Column('claim_until', sa.DateTime(timezone=True), nullable=True), + sa.Column('publish_attempts', sa.Integer(), nullable=False, server_default='0'), + sa.Column('last_error_code', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index( + 'outbox_events_pending_idx', + 'outbox_events', + ['published_at', 'available_at'], + ) + + op.create_table( + 'job_events', + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column('job_id', sa.Uuid(), nullable=False), + sa.Column('kind', sa.String(length=32), nullable=False), + sa.Column('attempt', sa.Integer(), nullable=False), + sa.Column('error_code', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['job_id'], ['attachment_jobs.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('job_events_job_created_idx', 'job_events', ['job_id', 'created_at']) + + +def downgrade() -> None: + op.drop_index('job_events_job_created_idx', table_name='job_events') + op.drop_table('job_events') + op.drop_index('outbox_events_pending_idx', table_name='outbox_events') + op.drop_table('outbox_events') + op.drop_index('attachment_jobs_lease_idx', table_name='attachment_jobs') + op.drop_table('attachment_jobs') + op.drop_table('attachment_variants') + op.drop_constraint('attachments_staging_key_key', 'attachments', type_='unique') + op.drop_column('attachments', 'pipeline_version') + op.drop_column('attachments', 'staging_key') + op.alter_column('attachments', 'original_key', nullable=False) + op.alter_column('attachments', 'original_key', new_column_name='object_key') diff --git a/alembic/versions/20261117_0005_outbox_trace_context.py b/alembic/versions/20261117_0005_outbox_trace_context.py new file mode 100644 index 0000000..6394d76 --- /dev/null +++ b/alembic/versions/20261117_0005_outbox_trace_context.py @@ -0,0 +1,24 @@ +"""Persist W3C trace context with outbox events. + +Revision ID: 20261117_0005 +Revises: 20261116_0004 +Create Date: 2026-11-17 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261117_0005' +down_revision = '20261116_0004' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column('outbox_events', sa.Column('traceparent', sa.String(length=55), nullable=True)) + op.add_column('outbox_events', sa.Column('tracestate', sa.String(length=512), nullable=True)) + + +def downgrade() -> None: + op.drop_column('outbox_events', 'tracestate') + op.drop_column('outbox_events', 'traceparent') diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..af04c35 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Мессенджер на PostgreSQL и Redis, лабораторная 2.""" diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..97e2e0c --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1 @@ +"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры.""" diff --git a/app/api/cursors.py b/app/api/cursors.py new file mode 100644 index 0000000..f0a9395 --- /dev/null +++ b/app/api/cursors.py @@ -0,0 +1,28 @@ +import base64 +import json +from uuid import UUID + +from ..domain.errors import Invalid +from ..domain.models import Keyset +from .timefmt import from_rfc3339, to_rfc3339 + +MAX_LENGTH = 512 + + +def encode(scope: str, position: Keyset) -> str: + payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)} + raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8') + return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=') + + +def decode(scope: str, value: str) -> Keyset: + if not value or len(value) > MAX_LENGTH: + raise Invalid('cursor недействителен для этого запроса') + try: + raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4)) + payload = json.loads(raw) + if not isinstance(payload, dict) or payload.get('s') != scope: + raise ValueError('scope mismatch') + return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i'])) + except Exception: + raise Invalid('cursor недействителен для этого запроса') from None diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 0000000..fa5eaa8 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from fastapi import Depends, HTTPException, Request, Security, status + +from ..db.redis import RedisSessionStore +from ..domain.errors import Unauthorized +from ..domain.models import User +from ..services.chats import ChatService +from ..services.attachments import AttachmentService +from ..services.messages import MessageService +from ..services.users import UserService +from .security import bearer_auth + +_BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +@dataclass(frozen=True, slots=True) +class Services: + attachments: AttachmentService + users: UserService + chats: ChatService + messages: MessageService + sessions: RedisSessionStore + + +def services(request: Request) -> Services: + return request.app.state.services + + +def user_service(container: Services = Depends(services)) -> UserService: + return container.users + + +def chat_service(container: Services = Depends(services)) -> ChatService: + return container.chats + + +def message_service(container: Services = Depends(services)) -> MessageService: + return container.messages + + +def attachment_service(container: Services = Depends(services)) -> AttachmentService: + return container.attachments + + +def session_store(container: Services = Depends(services)) -> RedisSessionStore: + return container.sessions + + +def _bearer_error() -> HTTPException: + return HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail='требуется действующая Bearer-сессия', + headers={'WWW-Authenticate': _BEARER_CHALLENGE}, + ) + + +async def bearer_token(authorization: str | None = Security(bearer_auth)) -> str: + if not authorization: + raise _bearer_error() + scheme, separator, token = authorization.partition(' ') + if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token: + raise _bearer_error() + return token + + +async def current_user( + token: str = Depends(bearer_token), + container: Services = Depends(services), +) -> User: + user_id = await container.sessions.get_user_id(token) + if user_id is None: + raise _bearer_error() + try: + return await container.users.identify(user_id) + except Unauthorized: + raise _bearer_error() from None diff --git a/app/api/errors.py b/app/api/errors.py new file mode 100644 index 0000000..8208ad1 --- /dev/null +++ b/app/api/errors.py @@ -0,0 +1,107 @@ +import logging + +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from sqlalchemy.exc import SQLAlchemyError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from ..domain.errors import ( + Conflict, + DomainError, + Invalid, + NotFound, + PayloadTooLarge, + Unauthorized, + Unavailable, + UnsupportedMediaType, +) + +logger = logging.getLogger('app.error') + +CODE_BY_STATUS = { + 400: 'invalid_request', + 401: 'unauthorized', + 404: 'not_found', + 409: 'conflict', + 413: 'payload_too_large', + 415: 'unsupported_media_type', + 429: 'rate_limited', + 500: 'internal_error', + 503: 'unavailable', +} + +STATUS_BY_DOMAIN = { + Invalid: 400, + Unauthorized: 401, + NotFound: 404, + Conflict: 409, + PayloadTooLarge: 413, + UnsupportedMediaType: 415, + Unavailable: 503, +} + + +def code_for(status: int) -> str: + if status in CODE_BY_STATUS: + return CODE_BY_STATUS[status] + return 'invalid_request' if status < 500 else 'internal_error' + + +def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse: + return JSONResponse( + status_code=status, + content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}}, + ) + + +def _domain_status(error: DomainError) -> int: + for kind, status in STATUS_BY_DOMAIN.items(): + if isinstance(error, kind): + return status + return 500 + + +def _describe(error: RequestValidationError) -> str: + """Краткое указание на пол, без stack trace и внутреннего устройства""" + for item in error.errors(): + if item.get('type') == 'json_invalid': + return 'тело запроса не является корректным JSON' + location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body') + return f'некорректное поле {location}' if location else 'некорректный запрос' + return 'некорректный запрос' + + +def request_id(request) -> str: + return getattr(request.state, 'request_id', 'unknown') + + +def install(app: FastAPI) -> None: + @app.exception_handler(DomainError) + async def _on_domain(request, error: DomainError) -> JSONResponse: + return error_response(_domain_status(error), error.message, request_id(request)) + + @app.exception_handler(RequestValidationError) + async def _on_validation(request, error: RequestValidationError) -> JSONResponse: + return error_response(400, _describe(error), request_id(request)) + + @app.exception_handler(SQLAlchemyError) + async def _on_database(request, error: SQLAlchemyError) -> JSONResponse: + logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__) + return error_response(503, 'база данных недоступна', request_id(request)) + + @app.exception_handler(OSError) + async def _on_storage_socket(request, error: OSError) -> JSONResponse: + logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__) + return error_response(503, 'обязательное хранилище недоступно', request_id(request)) + + @app.exception_handler(StarletteHTTPException) + async def _on_http(request, error: StarletteHTTPException) -> JSONResponse: + message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса' + response = error_response(error.status_code, message, request_id(request)) + for name, value in (error.headers or {}).items(): + response.headers[name] = value + return response + + +__all__ = ['code_for', 'error_response', 'install'] diff --git a/app/api/middleware.py b/app/api/middleware.py new file mode 100644 index 0000000..f2f5ca7 --- /dev/null +++ b/app/api/middleware.py @@ -0,0 +1,83 @@ +import logging +import os +import socket +import time +from uuid import uuid4 + +from opentelemetry import trace +from starlette.datastructures import MutableHeaders + +from ..observability import get_metrics +from ..observability.logging import safe_extra +from .errors import error_response + +logger = logging.getLogger('app') + + +class RequestContext: + def __init__(self, app) -> None: + self.app = app + self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname() + + async def __call__(self, scope, receive, send) -> None: + if scope['type'] != 'http': + await self.app(scope, receive, send) + return + + request_id = uuid4().hex + scope.setdefault('state', {})['request_id'] = request_id + is_health = scope.get('path') in {'/health/live', '/health/ready'} + started_at = time.perf_counter() + started = False + status_code = 500 + + async def send_wrapper(message) -> None: + nonlocal started, status_code + if message['type'] == 'http.response.start': + started = True + status_code = message['status'] + headers = MutableHeaders(scope=message) + headers['X-Request-Id'] = request_id + headers['X-Instance-Id'] = self.instance_id + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception: + logger.exception( + 'unhandled HTTP error', + extra=safe_extra('http.request.failed', request_id=request_id), + ) + if started: + raise + response = error_response(500, 'внутренняя ошибка', request_id) + await response(scope, receive, send_wrapper) + finally: + duration = time.perf_counter() - started_at + route = getattr(scope.get('route'), 'path', None) or 'unmatched' + method = scope.get('method', 'UNKNOWN') + status_class = f'{status_code // 100}xx' + span = trace.get_current_span() + if span.is_recording(): + span.set_attribute('request.id', request_id) + span.set_attribute('http.route', route) + if not is_health: + attributes = { + 'http.request.method': method, + 'http.route': route, + 'http.response.status_class': status_class, + } + instruments = get_metrics() + instruments.http_requests.add(1, attributes) + instruments.http_duration.record(duration, attributes) + logger.info( + 'HTTP request completed', + extra=safe_extra( + 'http.request.completed', + request_id=request_id, + http_method=method, + http_route=route, + http_status_code=status_code, + duration_seconds=round(duration, 6), + ), + ) diff --git a/app/api/routers/__init__.py b/app/api/routers/__init__.py new file mode 100644 index 0000000..dede845 --- /dev/null +++ b/app/api/routers/__init__.py @@ -0,0 +1,3 @@ +from . import auth, chats, health, messages, users + +__all__ = ['auth', 'chats', 'health', 'messages', 'users'] diff --git a/app/api/routers/attachments.py b/app/api/routers/attachments.py new file mode 100644 index 0000000..ab8f652 --- /dev/null +++ b/app/api/routers/attachments.py @@ -0,0 +1,79 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request +from fastapi.responses import StreamingResponse + +from ...domain.models import User +from ...services.attachments import AttachmentService +from ..deps import attachment_service, current_user +from ..schemas import AttachmentOut + +router = APIRouter(tags=['attachments']) + + +@router.post( + '/api/v1/chats/{chat_id}/attachments', + status_code=202, + response_model=AttachmentOut, + summary='Принять PNG/JPEG на асинхронную обработку', + openapi_extra={ + 'requestBody': { + 'required': True, + 'description': 'Сырые байты изображения, не multipart/form-data.', + 'content': { + 'image/png': { + 'schema': {'type': 'string', 'format': 'binary'}, + }, + 'image/jpeg': { + 'schema': {'type': 'string', 'format': 'binary'}, + }, + }, + }, + }, +) +async def upload( + chat_id: UUID, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + content_type: Annotated[str | None, Header(alias='Content-Type')] = None, + actor: User = Depends(current_user), + attachments: AttachmentService = Depends(attachment_service), +) -> AttachmentOut: + attachment = await attachments.upload( + actor, + chat_id, + filename, + content_type or '', + request.stream(), + ) + return AttachmentOut.from_domain(attachment) + + +@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut) +async def metadata( + attachment_id: UUID, + actor: User = Depends(current_user), + attachments: AttachmentService = Depends(attachment_service), +) -> AttachmentOut: + return AttachmentOut.from_domain(await attachments.get(actor, attachment_id)) + + +@router.get('/api/v1/attachments/{attachment_id}/content') +async def content( + attachment_id: UUID, + variant: str = Query('original'), + actor: User = Depends(current_user), + attachments: AttachmentService = Depends(attachment_service), +) -> StreamingResponse: + attachment, stream, content_type, size_bytes = await attachments.content( + actor, attachment_id, variant + ) + return StreamingResponse( + stream.chunks(), + media_type=content_type, + headers={ + 'Content-Length': str(size_bytes), + 'Content-Disposition': f'inline; filename="attachment-{attachment.id}"', + }, + ) diff --git a/app/api/routers/auth.py b/app/api/routers/auth.py new file mode 100644 index 0000000..aad330d --- /dev/null +++ b/app/api/routers/auth.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import base64 +from binascii import Error as Base64Error + +from fastapi import APIRouter, Depends, HTTPException, Security, status + +from ...auth.tokens import create_token +from ...db.redis import RedisSessionStore +from ...domain.models import User +from ...services.users import UserService +from ..deps import bearer_token, current_user, session_store, user_service +from ..security import basic_auth +from ..schemas import SessionOut, UserOut +from ..timefmt import to_rfc3339 + +router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth']) + +_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +def _unauthorized_basic() -> HTTPException: + return HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail='неверные учётные данные', + headers={'WWW-Authenticate': _BASIC_CHALLENGE}, + ) + + +def _parse_basic(authorization: str | None) -> tuple[str, str]: + if not authorization: + raise _unauthorized_basic() + scheme, separator, encoded = authorization.partition(' ') + if scheme.lower() != 'basic' or not separator or not encoded: + raise _unauthorized_basic() + try: + decoded = base64.b64decode(encoded, validate=True).decode('utf-8') + except (Base64Error, UnicodeDecodeError): + raise _unauthorized_basic() from None + username, separator, password = decoded.partition(':') + if not separator or not username: + raise _unauthorized_basic() + return username, password + + +@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut) +async def create_session( + authorization: str | None = Security(basic_auth), + users: UserService = Depends(user_service), + sessions: RedisSessionStore = Depends(session_store), +) -> SessionOut: + username, password = _parse_basic(authorization) + user = await users.authenticate(username, password) + if user is None: + raise _unauthorized_basic() + token = create_token() + expires_at = await sessions.create(user.id, token) + return SessionOut( + token=token, + token_type='Bearer', + expires_at=to_rfc3339(expires_at), + user=UserOut.from_domain(user), + ) + + +@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None) +async def revoke_session( + token: str = Depends(bearer_token), + _: User = Depends(current_user), + sessions: RedisSessionStore = Depends(session_store), +) -> None: + await sessions.revoke(token) diff --git a/app/api/routers/chats.py b/app/api/routers/chats.py new file mode 100644 index 0000000..1dd5126 --- /dev/null +++ b/app/api/routers/chats.py @@ -0,0 +1,49 @@ +from typing import Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Query + +from ...domain.models import User +from ...services.chats import ChatService +from .. import cursors +from ..deps import chat_service, current_user +from ..schemas import ( + LIMIT_DEFAULT, + LIMIT_MAX, + LIMIT_MIN, + ChatOut, + ChatPageOut, + CreateChat, +) + +router = APIRouter(prefix='/api/v1/chats', tags=['chats']) + + +@router.post('', status_code=201, response_model=ChatOut) +async def create( + body: CreateChat, + actor: User = Depends(current_user), + chats: ChatService = Depends(chat_service), +) -> ChatOut: + return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids)) + + +@router.get('', response_model=ChatPageOut) +async def page( + actor: User = Depends(current_user), + chats: ChatService = Depends(chat_service), + limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX), + cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH), +) -> ChatPageOut: + scope = f'chats:{actor.id}' + after = cursors.decode(scope, cursor) if cursor is not None else None + return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope) + + +@router.get('/{chat_id}', response_model=ChatOut) +async def get( + chat_id: UUID, + actor: User = Depends(current_user), + chats: ChatService = Depends(chat_service), +) -> ChatOut: + return ChatOut.from_domain(await chats.get(actor, chat_id)) diff --git a/app/api/routers/health.py b/app/api/routers/health.py new file mode 100644 index 0000000..853fa39 --- /dev/null +++ b/app/api/routers/health.py @@ -0,0 +1,29 @@ +from fastapi import APIRouter, HTTPException, Request +from redis.exceptions import RedisError +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError + +from ...domain.errors import Unavailable + +from ..schemas import HealthOut + +router = APIRouter(tags=['health']) + + +@router.get('/health/live', response_model=HealthOut) +async def live() -> HealthOut: + return HealthOut(status='ok') + + +@router.get('/health/ready', response_model=HealthOut) +async def ready(request: Request) -> HealthOut: + if not request.app.state.ready: + raise HTTPException(status_code=503, detail='приложение не готово принимать трафик') + try: + async with request.app.state.engine.connect() as connection: + await connection.execute(text('SELECT 1')) + await request.app.state.redis.ping() + await request.app.state.storage.ping() + except (SQLAlchemyError, RedisError, Unavailable, OSError): + raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None + return HealthOut(status='ok') diff --git a/app/api/routers/messages.py b/app/api/routers/messages.py new file mode 100644 index 0000000..7a86200 --- /dev/null +++ b/app/api/routers/messages.py @@ -0,0 +1,53 @@ +from typing import Annotated, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query + +from ...domain.models import User +from ...services.messages import MessageService +from .. import cursors +from ..deps import current_user, message_service +from ..schemas import ( + LIMIT_DEFAULT, + LIMIT_MAX, + LIMIT_MIN, + CreateMessage, + MessageOut, + MessagePageOut, +) + +router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages']) + + +@router.post('', status_code=201, response_model=MessageOut) +async def send( + chat_id: UUID, + body: CreateMessage, + idempotency_key: Annotated[ + str, + Header( + alias='Idempotency-Key', + min_length=1, + max_length=128, + pattern=r'^[A-Za-z0-9._:-]+$', + ), + ], + actor: User = Depends(current_user), + messages: MessageService = Depends(message_service), +) -> MessageOut: + return MessageOut.from_domain( + await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key) + ) + + +@router.get('', response_model=MessagePageOut) +async def page( + chat_id: UUID, + actor: User = Depends(current_user), + messages: MessageService = Depends(message_service), + limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX), + cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH), +) -> MessagePageOut: + scope = f'messages:{chat_id}' + after = cursors.decode(scope, cursor) if cursor is not None else None + return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope) diff --git a/app/api/routers/users.py b/app/api/routers/users.py new file mode 100644 index 0000000..7da40af --- /dev/null +++ b/app/api/routers/users.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter, Depends + +from ...domain.models import User +from ...services.users import UserService +from ..deps import current_user, user_service +from ..schemas import CreateUser, UserOut + +router = APIRouter(prefix='/api/v1', tags=['users']) + + +@router.post('/users', status_code=201, response_model=UserOut) +async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut: + return UserOut.from_domain(await users.register(body.username, body.display_name, body.password)) + + +@router.get('/users/me', response_model=UserOut) +async def me(actor: User = Depends(current_user)) -> UserOut: + return UserOut.from_domain(actor) diff --git a/app/api/schemas.py b/app/api/schemas.py new file mode 100644 index 0000000..9a649de --- /dev/null +++ b/app/api/schemas.py @@ -0,0 +1,209 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from ..domain.models import Attachment, Chat, Message, Page, User +from . import cursors +from .timefmt import to_rfc3339 + +NAME_MAX = 100 +TEXT_MAX = 4000 +MEMBERS_MAX = 100 +LIMIT_MIN = 1 +LIMIT_MAX = 100 +LIMIT_DEFAULT = 50 + + +def not_blank(value: str) -> str: + if not value.strip(): + raise ValueError('строка не может состоять только из пробельных символов') + return value + + +class Strict(BaseModel): + model_config = ConfigDict(extra='forbid') + + +class HealthOut(Strict): + status: str = 'ok' + + +class CreateUser(Strict): + username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=NAME_MAX) + password: str = Field(min_length=12, max_length=128) + + @field_validator('display_name') + @classmethod + def _display_name(cls, value: str) -> str: + return not_blank(value) + + +class UserOut(Strict): + id: UUID + username: str + display_name: str + created_at: str + + @classmethod + def from_domain(cls, user: User) -> 'UserOut': + return cls( + id=user.id, + username=user.username, + display_name=user.display_name, + created_at=to_rfc3339(user.created_at), + ) + + +class SessionOut(Strict): + token: str = Field(min_length=32) + token_type: str = 'Bearer' + expires_at: str + user: UserOut + + +class CreateChat(Strict): + title: str = Field(min_length=1, max_length=NAME_MAX) + member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX) + + @field_validator('title') + @classmethod + def _title(cls, value: str) -> str: + return not_blank(value) + + @field_validator('member_ids') + @classmethod + def _members_unique(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError('member_ids не должен содержать дубликаты') + return value + + +class ChatOut(Strict): + id: UUID + title: str + member_ids: list[UUID] + created_at: str + + @classmethod + def from_domain(cls, chat: Chat) -> 'ChatOut': + return cls( + id=chat.id, + title=chat.title, + member_ids=list(chat.member_ids), + created_at=to_rfc3339(chat.created_at), + ) + + +class CreateMessage(Strict): + text: str = Field(max_length=TEXT_MAX) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @field_validator('attachment_ids') + @classmethod + def _attachments_unique(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError('attachment_ids не должен содержать дубликаты') + return value + + @model_validator(mode='after') + def _content_present(self) -> 'CreateMessage': + if not self.text.strip() and not self.attachment_ids: + raise ValueError('сообщение должно содержать текст или вложение') + return self + + +class MessageOut(Strict): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: str + attachment_ids: list[UUID] + + @classmethod + def from_domain(cls, message: Message) -> 'MessageOut': + return cls( + id=message.id, + chat_id=message.chat_id, + sender_id=message.sender_id, + text=message.text, + created_at=to_rfc3339(message.created_at), + attachment_ids=list(message.attachment_ids), + ) + + +class VariantOut(Strict): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class AttachmentOut(Strict): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list[VariantOut] = Field(default_factory=list) + error_code: str | None + created_at: str + updated_at: str + + @classmethod + def from_domain(cls, attachment: Attachment) -> 'AttachmentOut': + return cls( + id=attachment.id, + chat_id=attachment.chat_id, + owner_id=attachment.owner_id, + filename=attachment.filename, + content_type=attachment.content_type, + size_bytes=attachment.size_bytes, + sha256=attachment.sha256, + status=attachment.status, + variants=[ + VariantOut( + name=item.name, + content_type=item.content_type, + size_bytes=item.size_bytes, + width=item.width, + height=item.height, + sha256=item.sha256, + ) + for item in attachment.variants + ], + error_code=attachment.error_code, + created_at=to_rfc3339(attachment.created_at), + updated_at=to_rfc3339(attachment.updated_at), + ) + + +class ChatPageOut(Strict): + items: list[ChatOut] + next_cursor: Optional[str] + + @classmethod + def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut': + return cls( + items=[ChatOut.from_domain(chat) for chat in page.items], + next_cursor=cursors.encode(scope, page.next) if page.next else None, + ) + + +class MessagePageOut(Strict): + items: list[MessageOut] + next_cursor: Optional[str] + + @classmethod + def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut': + return cls( + items=[MessageOut.from_domain(message) for message in page.items], + next_cursor=cursors.encode(scope, page.next) if page.next else None, + ) diff --git a/app/api/security.py b/app/api/security.py new file mode 100644 index 0000000..d525e1f --- /dev/null +++ b/app/api/security.py @@ -0,0 +1,29 @@ +"""OpenAPI security-схемы с сохранением собственного разбора Authorization.""" + +from fastapi import Request +from fastapi.openapi.models import HTTPBase as HTTPBaseModel +from fastapi.openapi.models import HTTPBearer as HTTPBearerModel +from fastapi.security.base import SecurityBase + + +class RawHTTPAuthorization(SecurityBase): + """Объявляет HTTP security scheme и возвращает исходный Authorization header.""" + + def __init__(self, model, scheme_name: str) -> None: + self.model = model + self.scheme_name = scheme_name + + async def __call__(self, request: Request) -> str | None: + """Возвращает заголовок без разбора, чтобы сохранить текущую валидацию API.""" + + return request.headers.get('Authorization') + + +basic_auth = RawHTTPAuthorization( + HTTPBaseModel(scheme='basic'), + scheme_name='basicAuth', +) +bearer_auth = RawHTTPAuthorization( + HTTPBearerModel(scheme='bearer', bearerFormat='opaque session token'), + scheme_name='bearerAuth', +) diff --git a/app/api/timefmt.py b/app/api/timefmt.py new file mode 100644 index 0000000..b9d7b10 --- /dev/null +++ b/app/api/timefmt.py @@ -0,0 +1,11 @@ +from datetime import datetime, timezone + + +def to_rfc3339(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def from_rfc3339(value: str) -> datetime: + if not isinstance(value, str) or not value.endswith('Z'): + raise ValueError('timestamp must end with Z') + return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc) diff --git a/app/auth/__init__.py b/app/auth/__init__.py new file mode 100644 index 0000000..9bd80fd --- /dev/null +++ b/app/auth/__init__.py @@ -0,0 +1,4 @@ +from .passwords import hash_password, password_matches +from .tokens import create_token + +__all__ = ['create_token', 'hash_password', 'password_matches'] diff --git a/app/auth/passwords.py b/app/auth/passwords.py new file mode 100644 index 0000000..70576fc --- /dev/null +++ b/app/auth/passwords.py @@ -0,0 +1,15 @@ +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, VerifyMismatchError + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def password_matches(stored_hash: str, password: str) -> bool: + try: + return _hasher.verify(stored_hash, password) + except (VerifyMismatchError, VerificationError): + return False diff --git a/app/auth/tokens.py b/app/auth/tokens.py new file mode 100644 index 0000000..c60beff --- /dev/null +++ b/app/auth/tokens.py @@ -0,0 +1,11 @@ +import hashlib +import secrets + + +def create_token() -> str: + return secrets.token_urlsafe(32) + + +def session_key(token: str) -> str: + digest = hashlib.sha256(token.encode('utf-8')).hexdigest() + return f'session:{digest}' diff --git a/app/config/__init__.py b/app/config/__init__.py new file mode 100644 index 0000000..e6b731e --- /dev/null +++ b/app/config/__init__.py @@ -0,0 +1,3 @@ +from .settings import Settings, load_settings + +__all__ = ['Settings', 'load_settings'] diff --git a/app/config/settings.py b/app/config/settings.py new file mode 100644 index 0000000..49e80ef --- /dev/null +++ b/app/config/settings.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _read_value(name: str, file_name: str) -> str: + value = os.getenv(name) + if value: + return value + path = os.getenv(file_name) + if path: + return Path(path).read_text(encoding='utf-8').strip() + raise RuntimeError(f'не задана настройка {name} или {file_name}') + + +@dataclass(frozen=True, slots=True) +class Settings: + database_url: str + redis_url: str + session_ttl_seconds: int + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + broker_url: str + worker_concurrency: int + worker_prefetch: int + job_max_attempts: int + job_timeout_seconds: int + job_lease_seconds: int + retry_base_seconds: int + retry_max_seconds: int + staging_retention_seconds: int + thumbnail_jpeg_quality: int + crop_jpeg_quality: int + optimized_webp_quality: int + + +def _positive_int(name: str, default: str) -> int: + try: + value = int(os.getenv(name, default)) + except ValueError as exc: + raise RuntimeError(f'{name} должен быть целым числом') from exc + if value <= 0: + raise RuntimeError(f'{name} должен быть больше нуля') + return value + + +def _quality(name: str, default: str) -> int: + value = _positive_int(name, default) + if value > 100: + raise RuntimeError(f'{name} должен быть не больше 100') + return value + + +def load_settings() -> Settings: + return Settings( + database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'), + redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'), + session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'), + s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + s3_region=os.getenv('S3_REGION', 'us-east-1'), + s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'), + s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'), + s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'), + max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'), + max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'), + broker_url=_read_value('BROKER_URL', 'BROKER_URL_FILE'), + worker_concurrency=_positive_int('WORKER_CONCURRENCY', '2'), + worker_prefetch=_positive_int('WORKER_PREFETCH', '2'), + job_max_attempts=_positive_int('JOB_MAX_ATTEMPTS', '5'), + job_timeout_seconds=_positive_int('JOB_TIMEOUT_SECONDS', '60'), + job_lease_seconds=_positive_int('JOB_LEASE_SECONDS', '90'), + retry_base_seconds=_positive_int('RETRY_BASE_SECONDS', '2'), + retry_max_seconds=_positive_int('RETRY_MAX_SECONDS', '60'), + staging_retention_seconds=_positive_int('STAGING_RETENTION_SECONDS', '86400'), + thumbnail_jpeg_quality=_quality('THUMBNAIL_JPEG_QUALITY', '85'), + crop_jpeg_quality=_quality('CROP_JPEG_QUALITY', '85'), + optimized_webp_quality=_quality('OPTIMIZED_WEBP_QUALITY', '82'), + ) diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 0000000..6e0b53d --- /dev/null +++ b/app/db/__init__.py @@ -0,0 +1,4 @@ +from .postgres import Base, create_engine, create_session_factory +from .redis import RedisSessionStore, create_redis + +__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory'] diff --git a/app/db/postgres.py b/app/db/postgres.py new file mode 100644 index 0000000..503ccd3 --- /dev/null +++ b/app/db/postgres.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase + +from ..observability import get_metrics + + +class Base(DeclarativeBase): + pass + + +def create_engine(url: str) -> AsyncEngine: + engine = create_async_engine( + url, + pool_size=5, + max_overflow=5, + pool_timeout=5, + pool_pre_ping=True, + connect_args={'server_settings': {'statement_timeout': '5000'}}, + ) + instruments = get_metrics() + attributes = {'pool.name': 'postgres'} + + def update_pool_metrics(*_args) -> None: + pool = engine.sync_engine.pool + instruments.db_pool_used.set(float(pool.checkedout()), attributes) + waiting = len(getattr(getattr(pool, '_pool', None), '_getters', ())) + instruments.db_pool_waiting.set(float(waiting), attributes) + + event.listen(engine.sync_engine, 'checkout', update_pool_metrics) + event.listen(engine.sync_engine, 'checkin', update_pool_metrics) + instruments.db_pool_used.set(0, attributes) + instruments.db_pool_waiting.set(0, attributes) + return engine + + +def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(engine, expire_on_commit=False) diff --git a/app/db/redis.py b/app/db/redis.py new file mode 100644 index 0000000..fc9e357 --- /dev/null +++ b/app/db/redis.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from ..auth.tokens import session_key +from ..domain.errors import Unavailable + + +def create_redis(url: str) -> Redis: + return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3) + + +class RedisSessionStore: + def __init__(self, client: Redis, ttl_seconds: int) -> None: + self._client = client + self._ttl_seconds = ttl_seconds + + async def create(self, user_id: UUID, token: str) -> datetime: + try: + await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds) + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc + return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds) + + async def get_user_id(self, token: str) -> UUID | None: + try: + value = await self._client.get(session_key(token)) + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc + if value is None: + return None + try: + return UUID(value) + except ValueError: + return None + + async def revoke(self, token: str) -> None: + try: + await self._client.delete(session_key(token)) + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc + + async def ping(self) -> None: + try: + await self._client.ping() + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc diff --git a/app/domain/__init__.py b/app/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/domain/errors.py b/app/domain/errors.py new file mode 100644 index 0000000..c6c4556 --- /dev/null +++ b/app/domain/errors.py @@ -0,0 +1,37 @@ +from __future__ import annotations + + +class DomainError(Exception): + """Сообщение попадает клиенту, поэтому не содержит внутренних деталей.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class Invalid(DomainError): + """Запрос нарушает правила предметной области.""" + + +class Unauthorized(DomainError): + """Нет действительной идентификации.""" + + +class NotFound(DomainError): + """Объекта нет либо он недоступен вызывающему: снаружи это неразличимо.""" + + +class Conflict(DomainError): + """Состояние не позволяет выполнить операцию.""" + + +class Unavailable(DomainError): + """Обязательная внешняя зависимость временно недоступна.""" + + +class PayloadTooLarge(DomainError): + """Тело запроса или декодированное содержимое превышает лимит.""" + + +class UnsupportedMediaType(DomainError): + """Формат содержимого не поддерживается или не соответствует MIME.""" diff --git a/app/domain/models.py b/app/domain/models.py new file mode 100644 index 0000000..1fcb3c0 --- /dev/null +++ b/app/domain/models.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Optional +from uuid import UUID + + +@dataclass(frozen=True, slots=True) +class User: + id: UUID + username: str + display_name: str + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class Chat: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + def has_member(self, user_id: UUID) -> bool: + return user_id in self.member_ids + + +@dataclass(frozen=True, slots=True) +class Message: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + +@dataclass(frozen=True, slots=True) +class Attachment: + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + original_key: str | None + staging_key: str | None + pipeline_version: int + content_type: str + size_bytes: int + sha256: str + width: int + height: int + status: str + error_code: str | None + created_at: datetime + updated_at: datetime + variants: tuple['AttachmentVariant', ...] = () + + +@dataclass(frozen=True, slots=True) +class AttachmentVariant: + name: str + object_key: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class AttachmentJob: + id: UUID + attachment_id: UUID + pipeline_version: int + state: str + attempt: int + max_attempts: int + next_attempt_at: datetime + lease_owner: str | None + lease_until: datetime | None + last_error_code: str | None + replay_count: int + created_at: datetime + updated_at: datetime + + +@dataclass(frozen=True, slots=True) +class OutboxEvent: + id: UUID + job_id: UUID + payload: dict + available_at: datetime + traceparent: str | None = None + tracestate: str | None = None + + +@dataclass(frozen=True, slots=True) +class Keyset: + created_at: datetime + id: UUID + + @property + def order(self) -> tuple[datetime, int]: + return (self.created_at, self.id.int) + + @classmethod + def from_item(cls, item) -> 'Keyset': + return cls(created_at=item.created_at, id=item.id) + + +def order_key(item) -> tuple[datetime, int]: + return (item.created_at, item.id.int) + + +@dataclass(frozen=True, slots=True) +class Page: + items: tuple + next: Optional[Keyset] diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py new file mode 100644 index 0000000..160c283 --- /dev/null +++ b/app/jobs/__init__.py @@ -0,0 +1 @@ +"""Durable attachment job publisher and worker.""" diff --git a/app/jobs/broker.py b/app/jobs/broker.py new file mode 100644 index 0000000..0c6a927 --- /dev/null +++ b/app/jobs/broker.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import asyncio +import logging + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType + +MAIN_EXCHANGE = 'attachments' +MAIN_QUEUE = 'attachments.process' +ROUTING_KEY = 'attachment.process.v1' +DEAD_EXCHANGE = 'attachments.dead' +DEAD_QUEUE = 'attachments.failed' +logger = logging.getLogger('app.broker') + + +async def connect(url: str) -> aio_pika.RobustConnection: + delay = 0.5 + while True: + try: + return await aio_pika.connect_robust(url, timeout=5) + except (OSError, aio_pika.exceptions.AMQPException): + logger.warning('broker is not ready; reconnecting in %.1fs', delay) + await asyncio.sleep(delay) + delay = min(delay * 2, 5.0) + + +async def topology(channel: aio_pika.abc.AbstractChannel): + dead_exchange = await channel.declare_exchange( + DEAD_EXCHANGE, ExchangeType.DIRECT, durable=True + ) + dead_queue = await channel.declare_queue(DEAD_QUEUE, durable=True) + await dead_queue.bind(dead_exchange, ROUTING_KEY) + exchange = await channel.declare_exchange( + MAIN_EXCHANGE, ExchangeType.DIRECT, durable=True + ) + queue = await channel.declare_queue( + MAIN_QUEUE, + durable=True, + arguments={ + 'x-dead-letter-exchange': DEAD_EXCHANGE, + 'x-dead-letter-routing-key': ROUTING_KEY, + }, + ) + await queue.bind(exchange, ROUTING_KEY) + return exchange, queue + + +def message(body: bytes, event_id: str, headers: dict[str, str] | None = None) -> aio_pika.Message: + return aio_pika.Message( + body=body, + content_type='application/json', + delivery_mode=DeliveryMode.PERSISTENT, + message_id=event_id, + type=ROUTING_KEY, + headers=headers or {}, + ) diff --git a/app/jobs/pipeline.py b/app/jobs/pipeline.py new file mode 100644 index 0000000..ed8bc5f --- /dev/null +++ b/app/jobs/pipeline.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +class PermanentImageError(Exception): + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +@dataclass(frozen=True, slots=True) +class GeneratedVariant: + name: str + content_type: str + width: int + height: int + sha256: str + content: bytes + + +def _jpeg_image(image: Image.Image) -> Image.Image: + if image.mode in {'RGBA', 'LA'} or 'transparency' in image.info: + rgba = image.convert('RGBA') + background = Image.new('RGBA', rgba.size, 'white') + background.alpha_composite(rgba) + return background.convert('RGB') + return image.convert('RGB') + + +def _encode(image: Image.Image, fmt: str, **options) -> bytes: + output = BytesIO() + image.save(output, format=fmt, **options) + return output.getvalue() + + +def _variant(name: str, content_type: str, image: Image.Image, content: bytes) -> GeneratedVariant: + return GeneratedVariant( + name=name, + content_type=content_type, + width=image.width, + height=image.height, + sha256=hashlib.sha256(content).hexdigest(), + content=content, + ) + + +def process_image( + source: bytes, + declared_type: str, + expected_sha256: str, + max_pixels: int, + thumbnail_quality: int, + crop_quality: int, + webp_quality: int, +) -> tuple[GeneratedVariant, ...]: + if hashlib.sha256(source).hexdigest() != expected_sha256: + raise PermanentImageError('checksum_mismatch') + try: + with Image.open(BytesIO(source)) as opened: + actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(opened.format or '') + if actual != declared_type: + raise PermanentImageError('unsupported_media_type') + if opened.width * opened.height > max_pixels: + raise PermanentImageError('image_too_large') + opened.load() + oriented = ImageOps.exif_transpose(opened) + base = oriented.copy() + except PermanentImageError: + raise + except (Image.DecompressionBombError, UnidentifiedImageError, OSError, SyntaxError) as error: + raise PermanentImageError('invalid_image') from error + + thumbnail = _jpeg_image(base.copy()) + thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS, reducing_gap=3.0) + thumbnail_bytes = _encode( + thumbnail, 'JPEG', quality=thumbnail_quality, optimize=True, progressive=True + ) + + crop_source = _jpeg_image(base.copy()) + crop = ImageOps.fit( + crop_source, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5) + ) + crop_bytes = _encode(crop, 'JPEG', quality=crop_quality, optimize=True, progressive=True) + + optimized = base.copy() + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS, reducing_gap=3.0) + optimized_bytes = _encode(optimized, 'WEBP', quality=webp_quality, method=6) + + return ( + _variant('thumbnail', 'image/jpeg', thumbnail, thumbnail_bytes), + _variant('crop', 'image/jpeg', crop, crop_bytes), + _variant('optimized', 'image/webp', optimized, optimized_bytes), + ) diff --git a/app/jobs/publisher.py b/app/jobs/publisher.py new file mode 100644 index 0000000..8568933 --- /dev/null +++ b/app/jobs/publisher.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import socket +from datetime import datetime, timezone + +from opentelemetry import trace +from opentelemetry.trace import SpanKind + +from ..config import load_settings +from ..observability import get_metrics +from ..observability.logging import safe_extra +from ..observability.propagation import extract_trace_context, inject_trace_context +from ..repositories import JobRepository, OutboxRepository +from .broker import ROUTING_KEY, connect, message, topology +from .runtime import configure_logging, dependencies + +logger = logging.getLogger('app.publisher') +tracer = trace.get_tracer('messenger.publisher', '1.6.0') + + +async def run(once: bool = False) -> None: + settings = load_settings() + engine, sessions, _storage = dependencies(settings) + outbox = OutboxRepository(sessions) + jobs = JobRepository(sessions) + instruments = get_metrics() + queue_attributes = {'messaging.destination.name': 'attachments.process'} + owner = f'{socket.gethostname()}:{os.getpid()}' + connection = await connect(settings.broker_url) + try: + channel = await connection.channel(publisher_confirms=True) + exchange, _queue = await topology(channel) + while True: + now = datetime.now(timezone.utc) + await jobs.recover_expired(now) + depth, oldest_age = await jobs.queue_stats(now) + instruments.queue_depth.set(depth, queue_attributes) + instruments.queue_oldest_age.set(oldest_age, queue_attributes) + events = await outbox.claim(owner, now, lease_seconds=30) + for event in events: + try: + parent = extract_trace_context({ + 'traceparent': event.traceparent, + 'tracestate': event.tracestate, + }) + with tracer.start_as_current_span( + 'attachments publish', + context=parent, + kind=SpanKind.PRODUCER, + attributes={ + 'messaging.system': 'rabbitmq', + 'messaging.destination.name': 'attachments.process', + 'messaging.operation.name': 'publish', + 'messaging.operation.type': 'send', + 'job.id': str(event.job_id), + }, + ): + body = json.dumps(event.payload, separators=(',', ':')).encode() + headers = inject_trace_context() + await exchange.publish( + message(body, str(event.id), headers), + routing_key=ROUTING_KEY, + mandatory=True, + ) + logger.info( + 'outbox event published', + extra=safe_extra( + 'outbox.published', event_id=str(event.id), + job_id=str(event.job_id), + ), + ) + if os.getenv('FAULT_PUBLISHER_AFTER_CONFIRM') == '1': + os._exit(91) + await outbox.published(event.id, owner, datetime.now(timezone.utc)) + except Exception: + logger.exception( + 'outbox publish failed', + extra=safe_extra( + 'outbox.publish_failed', event_id=str(event.id), + job_id=str(event.job_id), + ), + ) + await outbox.failed( + event.id, owner, 'broker_unavailable', datetime.now(timezone.utc) + ) + if once: + return + await asyncio.sleep(0.5 if events else 1.0) + finally: + await connection.close() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--once', action='store_true') + args = parser.parse_args() + telemetry = configure_logging('messenger-publisher') + try: + asyncio.run(run(args.once)) + finally: + telemetry.shutdown() + + +if __name__ == '__main__': + main() diff --git a/app/jobs/runtime.py b/app/jobs/runtime.py new file mode 100644 index 0000000..17082c9 --- /dev/null +++ b/app/jobs/runtime.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from ..config import Settings +from ..db import create_engine, create_session_factory +from ..observability import initialize_observability +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from ..storage import S3Storage + + +def configure_logging(service_name: str): + return initialize_observability(service_name) + + +def dependencies(settings: Settings): + engine = create_engine(settings.database_url) + SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine) + sessions = create_session_factory(engine) + storage = S3Storage( + endpoint=settings.s3_endpoint, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + bucket=settings.s3_bucket, + region=settings.s3_region, + ) + return engine, sessions, storage diff --git a/app/jobs/worker.py b/app/jobs/worker.py new file mode 100644 index 0000000..663ac19 --- /dev/null +++ b/app/jobs/worker.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +import asyncio +import json +import logging +import multiprocessing +import os +import random +import socket +import time +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import aio_pika +from opentelemetry import trace +from opentelemetry.trace import SpanKind + +from ..config import Settings, load_settings +from ..domain.errors import Unavailable +from ..domain.models import AttachmentVariant +from ..observability import get_metrics +from ..observability.logging import safe_extra +from ..observability.propagation import extract_trace_context +from ..repositories import JobRepository +from .broker import connect, topology +from .pipeline import GeneratedVariant, PermanentImageError, process_image +from .runtime import configure_logging, dependencies + +logger = logging.getLogger('app.worker') +tracer = trace.get_tracer('messenger.worker', '1.6.0') + + +class ProcessingTimeout(Exception): + pass + + +def _pipeline_child(connection, arguments: tuple, memory_bytes: int) -> None: + try: + if memory_bytes: + try: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes)) + except (ImportError, OSError, ValueError): + pass + connection.send(('ok', process_image(*arguments))) + except PermanentImageError as error: + connection.send(('permanent', error.code)) + except MemoryError: + connection.send(('temporary', 'processing_memory_limit')) + except BaseException: + connection.send(('temporary', 'processing_failed')) + finally: + connection.close() + + +def _run_isolated(arguments: tuple, timeout: int, memory_bytes: int) -> tuple[GeneratedVariant, ...]: + context = multiprocessing.get_context('spawn') + receiver, sender = context.Pipe(duplex=False) + process = context.Process(target=_pipeline_child, args=(sender, arguments, memory_bytes)) + process.start() + sender.close() + if not receiver.poll(timeout): + process.terminate() + process.join(5) + receiver.close() + raise ProcessingTimeout('processing_timeout') + try: + kind, result = receiver.recv() + except EOFError as error: + process.join(5) + receiver.close() + raise ProcessingTimeout('processing_failed') from error + receiver.close() + process.join(5) + if process.is_alive(): + process.terminate() + process.join(5) + raise ProcessingTimeout('processing_failed') + if kind == 'ok': + return result + if kind == 'permanent': + raise PermanentImageError(result) + raise ProcessingTimeout(result) + + +class Worker: + def __init__(self, settings: Settings, jobs: JobRepository, storage) -> None: + self.settings = settings + self.jobs = jobs + self.storage = storage + self.worker_id = f'{socket.gethostname()}:{os.getpid()}' + self.semaphore = asyncio.Semaphore(settings.worker_concurrency) + self.metrics = get_metrics() + self.metric_attributes = {'job.type': 'attachment.process.v1'} + self.metrics.worker_busy.set(0, self.metric_attributes) + self.metrics.worker_capacity.set(settings.worker_concurrency, self.metric_attributes) + + @asynccontextmanager + async def _busy_slot(self): + async with self.semaphore: + self.metrics.worker_busy.add(1, self.metric_attributes) + try: + yield + finally: + self.metrics.worker_busy.add(-1, self.metric_attributes) + + def _delay(self, attempt: int) -> float: + base = min( + self.settings.retry_max_seconds, + self.settings.retry_base_seconds * (2 ** max(0, attempt - 1)), + ) + return base + random.uniform(0, base) + + async def _handle_failure( + self, + incoming: aio_pika.IncomingMessage, + job_id: UUID, + attempt: int, + code: str, + permanent: bool, + ) -> None: + now = datetime.now(timezone.utc) + available_at = now + timedelta(seconds=self._delay(attempt)) + try: + state = await self.jobs.retry_or_fail( + job_id, + self.worker_id, + code, + permanent, + available_at, + now, + ) + except Exception: + logger.exception('failed to persist job failure job_id=%s', job_id) + await incoming.nack(requeue=True) + return + if state == 'failed': + logger.error( + 'job reached terminal failure', + extra=safe_extra( + 'job.failed', job_id=str(job_id), attempt=attempt, error_code=code + ), + ) + await incoming.reject(requeue=False) + else: + logger.warning( + 'job retry scheduled', + extra=safe_extra( + 'job.retry_scheduled', job_id=str(job_id), attempt=attempt, + error_code=code, + ), + ) + await incoming.ack() + + async def handle(self, incoming: aio_pika.IncomingMessage) -> None: + context = extract_trace_context(incoming.headers) + job_id: UUID | None = None + before = None + try: + preview = json.loads(incoming.body) + job_id = UUID(preview['job_id']) + before = await self.jobs.state(job_id) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + pass + started_at = time.perf_counter() + attributes = { + 'messaging.system': 'rabbitmq', + 'messaging.destination.name': 'attachments.process', + 'messaging.operation.name': 'process', + 'messaging.operation.type': 'process', + } + if job_id is not None: + attributes['job.id'] = str(job_id) + with tracer.start_as_current_span( + 'attachments process', context=context, kind=SpanKind.CONSUMER, + attributes=attributes, + ) as span: + await self._handle_message(incoming) + if before is None or before.state != 'queued' or job_id is None: + return + after = await self.jobs.state(job_id) + if after is None or after.state not in {'queued', 'ready', 'failed'}: + return + if after.state == 'queued' and after.attempt == before.attempt: + return + outcome = 'retry' if after.state == 'queued' else after.state + duration = time.perf_counter() - started_at + metric_attributes = {**self.metric_attributes, 'outcome': outcome} + self.metrics.jobs_processed.add(1, metric_attributes) + self.metrics.job_duration.record(duration, metric_attributes) + span.set_attribute('job.outcome', outcome) + span.set_attribute('job.attempt', before.attempt) + if outcome == 'ready': + ready_seconds = max( + 0.0, (datetime.now(timezone.utc) - after.created_at).total_seconds() + ) + self.metrics.time_to_ready.record( + ready_seconds, + {'pipeline.version': str(after.pipeline_version), 'outcome': 'ready'}, + ) + + async def _handle_message(self, incoming: aio_pika.IncomingMessage) -> None: + async with self._busy_slot(): + try: + payload = json.loads(incoming.body) + if payload.get('schema_version') != 1 or payload.get('type') != 'attachment.process.v1': + await incoming.reject(requeue=False) + return + job_id = UUID(payload['job_id']) + attachment_id = UUID(payload['attachment_id']) + attempt = int(payload['attempt']) + pipeline_version = int(payload['pipeline_version']) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + await incoming.reject(requeue=False) + return + + current = await self.jobs.state(job_id) + if current is None: + await incoming.reject(requeue=False) + return + if current.attachment_id != attachment_id or current.pipeline_version != pipeline_version: + await incoming.reject(requeue=False) + return + if current.state == 'ready': + if await self.jobs.ready_effect_complete(job_id): + await incoming.ack() + else: + await incoming.reject(requeue=False) + return + if current.state == 'failed': + await incoming.reject(requeue=False) + return + + now = datetime.now(timezone.utc) + claimed = await self.jobs.claim( + job_id, attempt, self.worker_id, now, self.settings.job_lease_seconds + ) + if claimed is None: + await incoming.ack() + return + job, attachment = claimed + if os.getenv('FAULT_WORKER_PERMANENT') == '1': + await self._handle_failure( + incoming, job.id, job.attempt, 'fault_injected', True + ) + os._exit(94) + if not attachment.staging_key: + await self._handle_failure( + incoming, job.id, job.attempt, 'staging_missing', True + ) + return + + try: + source = await self.storage.read( + attachment.staging_key, self.settings.max_upload_bytes + ) + arguments = ( + source, + attachment.content_type, + attachment.sha256, + self.settings.max_image_pixels, + self.settings.thumbnail_jpeg_quality, + self.settings.crop_jpeg_quality, + self.settings.optimized_webp_quality, + ) + generated = await asyncio.to_thread( + _run_isolated, + arguments, + self.settings.job_timeout_seconds, + 512 * 1024 * 1024, + ) + original_key = f'originals/{attachment.id}/v{job.pipeline_version}/original' + await self.storage.put_bytes(original_key, source, attachment.content_type) + variants: list[AttachmentVariant] = [] + suffixes = {'thumbnail': 'jpg', 'crop': 'jpg', 'optimized': 'webp'} + for item in generated: + key = ( + f'variants/{attachment.id}/v{job.pipeline_version}/' + f'{item.name}.{suffixes[item.name]}' + ) + await self.storage.put_bytes(key, item.content, item.content_type) + variants.append( + AttachmentVariant( + name=item.name, + object_key=key, + content_type=item.content_type, + size_bytes=len(item.content), + width=item.width, + height=item.height, + sha256=item.sha256, + ) + ) + if os.getenv('FAULT_WORKER_AFTER_OBJECTS') == '1': + os._exit(92) + completed = await self.jobs.complete( + job.id, + self.worker_id, + original_key, + tuple(variants), + datetime.now(timezone.utc), + ) + if not completed: + await incoming.nack(requeue=True) + return + if os.getenv('FAULT_WORKER_AFTER_COMMIT') == '1': + os._exit(93) + try: + await self.storage.remove(attachment.staging_key) + except Unavailable: + logger.warning( + 'staging cleanup deferred', + extra=safe_extra( + 'staging.cleanup_deferred', attachment_id=str(attachment.id) + ), + ) + logger.info( + 'job completed', + extra=safe_extra( + 'job.completed', job_id=str(job.id), + attachment_id=str(attachment.id), attempt=job.attempt, + ), + ) + await incoming.ack() + except PermanentImageError as error: + await self._handle_failure( + incoming, job.id, job.attempt, error.code, True + ) + except ProcessingTimeout as error: + await self._handle_failure( + incoming, job.id, job.attempt, str(error), False + ) + except (Unavailable, OSError, ValueError): + logger.exception( + 'temporary processing failure', + extra=safe_extra( + 'job.processing_failed', job_id=str(job.id), + error_code='storage_unavailable', + ), + ) + await self._handle_failure( + incoming, job.id, job.attempt, 'storage_unavailable', False + ) + except Exception: + logger.exception( + 'unexpected processing failure', + extra=safe_extra( + 'job.processing_failed', job_id=str(job.id), + error_code='processing_failed', + ), + ) + await self._handle_failure( + incoming, job.id, job.attempt, 'processing_failed', False + ) + + +async def run() -> None: + settings = load_settings() + engine, sessions, storage = dependencies(settings) + jobs = JobRepository(sessions) + worker = Worker(settings, jobs, storage) + connection = await connect(settings.broker_url) + try: + channel = await connection.channel() + await channel.set_qos(prefetch_count=settings.worker_prefetch) + _exchange, queue = await topology(channel) + await queue.consume(worker.handle, no_ack=False) + logger.info( + 'worker started id=%s concurrency=%d prefetch=%d', + worker.worker_id, + settings.worker_concurrency, + settings.worker_prefetch, + ) + await asyncio.Future() + finally: + await connection.close() + await engine.dispose() + + +def main() -> None: + telemetry = configure_logging('messenger-worker') + try: + asyncio.run(run()) + finally: + telemetry.shutdown() + + +if __name__ == '__main__': + main() diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..ef013f5 --- /dev/null +++ b/app/main.py @@ -0,0 +1,120 @@ +"""Сборка FastAPI приложения и подключение обязательных хранилищ.""" +from __future__ import annotations + +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor +from sqlalchemy import text + +from .api import errors +from .api.deps import Services +from .api.middleware import RequestContext +from .api.routers import attachments, auth, chats, health, messages, users +from .config import load_settings +from .db import RedisSessionStore, create_engine, create_redis, create_session_factory +from .observability import get_metrics, initialize_observability +from .observability.logging import safe_extra +from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository +from .services.attachments import AttachmentService +from .services.chats import ChatService +from .services.messages import MessageService +from .services.users import UserService +from .storage import S3Storage + +import logging + +logger = logging.getLogger('app') + +DESCRIPTION = 'PostgreSQL хранит задания, RabbitMQ доставляет их worker, S3 хранит изображения.' + + +def create_app() -> FastAPI: + telemetry = initialize_observability('messenger-api') + get_metrics() + + @asynccontextmanager + async def lifespan(application: FastAPI): + application.state.ready = False + settings = load_settings() + engine = create_engine(settings.database_url) + SQLAlchemyInstrumentor().instrument(engine=engine.sync_engine) + redis = create_redis(settings.redis_url) + storage = S3Storage( + endpoint=settings.s3_endpoint, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + bucket=settings.s3_bucket, + region=settings.s3_region, + ) + try: + async with engine.connect() as connection: + await connection.execute(text('SELECT 1')) + session_store = RedisSessionStore(redis, settings.session_ttl_seconds) + await session_store.ping() + await storage.ping() + session_factory = create_session_factory(engine) + user_repository = UserRepository(session_factory) + chat_repository = ChatRepository(session_factory) + message_repository = MessageRepository(session_factory) + attachment_repository = AttachmentRepository(session_factory) + application.state.services = Services( + attachments=AttachmentService( + attachment_repository, + storage, + settings.max_upload_bytes, + settings.max_image_pixels, + settings.job_max_attempts, + ), + users=UserService(user_repository), + chats=ChatService(chat_repository), + messages=MessageService(message_repository), + sessions=session_store, + ) + application.state.engine = engine + application.state.redis = redis + application.state.storage = storage + application.state.ready = True + logger.info( + 'application started', + extra=safe_extra( + 'api.startup', storage='postgres+s3', redis_sessions=True, + async_attachments=True, + ), + ) + yield + finally: + application.state.ready = False + await redis.aclose() + await engine.dispose() + logger.info('application stopped', extra=safe_extra('api.shutdown')) + telemetry.shutdown() + + api = FastAPI( + title='MTUSI Messenger. LR 6', + version='1.6.0', + description=DESCRIPTION, + lifespan=lifespan, + ) + api.router.redirect_slashes = False + api.state.ready = False + api.add_middleware(RequestContext) + errors.install(api) + for router in ( + health.router, + users.router, + auth.router, + chats.router, + messages.router, + attachments.router, + ): + api.include_router(router) + FastAPIInstrumentor.instrument_app( + api, + excluded_urls='/health/live,/health/ready', + ) + return api + + +app = create_app() diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..f26d693 --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,18 @@ +from .attachment import AttachmentRow, AttachmentVariantRow, MessageAttachmentRow +from .chat import ChatMemberRow, ChatRow +from .message import MessageRow +from .user import UserRow +from .job import AttachmentJobRow, JobEventRow, OutboxEventRow + +__all__ = [ + 'AttachmentRow', + 'AttachmentVariantRow', + 'AttachmentJobRow', + 'ChatMemberRow', + 'ChatRow', + 'MessageAttachmentRow', + 'MessageRow', + 'UserRow', + 'JobEventRow', + 'OutboxEventRow', +] diff --git a/app/models/attachment.py b/app/models/attachment.py new file mode 100644 index 0000000..c1444e3 --- /dev/null +++ b/app/models/attachment.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class AttachmentRow(Base): + __tablename__ = 'attachments' + __table_args__ = ( + Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'), + UniqueConstraint('original_key', name='attachments_object_key_key'), + UniqueConstraint('staging_key', name='attachments_staging_key_key'), + ) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False) + owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + original_key: Mapped[str | None] = mapped_column(String(255), nullable=True) + staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + width: Mapped[int] = mapped_column(Integer, nullable=False) + height: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False) + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class AttachmentVariantRow(Base): + __tablename__ = 'attachment_variants' + + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True + ) + pipeline_version: Mapped[int] = mapped_column(Integer, primary_key=True) + name: Mapped[str] = mapped_column(String(16), primary_key=True) + object_key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + width: Mapped[int] = mapped_column(Integer, nullable=False) + height: Mapped[int] = mapped_column(Integer, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MessageAttachmentRow(Base): + __tablename__ = 'message_attachments' + __table_args__ = ( + UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'), + Index('message_attachments_attachment_idx', 'attachment_id'), + ) + + message_id: Mapped[UUID] = mapped_column( + ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True + ) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True + ) + position: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/app/models/chat.py b/app/models/chat.py new file mode 100644 index 0000000..9c464ec --- /dev/null +++ b/app/models/chat.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import DateTime, ForeignKey, Index, String +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class ChatRow(Base): + __tablename__ = 'chats' + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + title: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class ChatMemberRow(Base): + __tablename__ = 'chat_members' + __table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),) + + chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True) diff --git a/app/models/job.py b/app/models/job.py new file mode 100644 index 0000000..5b888eb --- /dev/null +++ b/app/models/job.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class AttachmentJobRow(Base): + __tablename__ = 'attachment_jobs' + __table_args__ = ( + UniqueConstraint( + 'attachment_id', 'pipeline_version', name='attachment_job_attachment_version_key' + ), + Index('attachment_jobs_lease_idx', 'state', 'lease_until'), + ) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey('attachments.id', ondelete='CASCADE'), nullable=False + ) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False) + state: Mapped[str] = mapped_column(String(16), nullable=False) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False) + next_attempt_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + replay_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class OutboxEventRow(Base): + __tablename__ = 'outbox_events' + __table_args__ = (Index('outbox_events_pending_idx', 'published_at', 'available_at'),) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + event_type: Mapped[str] = mapped_column(String(64), nullable=False) + job_id: Mapped[UUID] = mapped_column( + ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False + ) + payload: Mapped[dict] = mapped_column(JSON, nullable=False) + traceparent: Mapped[str | None] = mapped_column(String(55), nullable=True) + tracestate: Mapped[str | None] = mapped_column(String(512), nullable=True) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + claim_owner: Mapped[str | None] = mapped_column(String(128), nullable=True) + claim_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class JobEventRow(Base): + __tablename__ = 'job_events' + __table_args__ = (Index('job_events_job_created_idx', 'job_id', 'created_at'),) + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + job_id: Mapped[UUID] = mapped_column( + ForeignKey('attachment_jobs.id', ondelete='CASCADE'), nullable=False + ) + kind: Mapped[str] = mapped_column(String(32), nullable=False) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) diff --git a/app/models/message.py b/app/models/message.py new file mode 100644 index 0000000..96a716f --- /dev/null +++ b/app/models/message.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class MessageRow(Base): + __tablename__ = 'messages' + __table_args__ = ( + Index('messages_history_idx', 'chat_id', 'created_at', 'id'), + UniqueConstraint( + 'sender_id', + 'chat_id', + 'idempotency_key', + name='messages_sender_chat_idempotency_key', + ), + ) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False) + sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False) + text: Mapped[str] = mapped_column(String(4000), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True) diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 0000000..f4aeefd --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import DateTime, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class UserRow(Base): + __tablename__ = 'users' + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False) + display_name: Mapped[str] = mapped_column(String(100), nullable=False) + password_hash: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) diff --git a/app/observability/__init__.py b/app/observability/__init__.py new file mode 100644 index 0000000..7f12e15 --- /dev/null +++ b/app/observability/__init__.py @@ -0,0 +1,9 @@ +from .metrics import Metrics, get_metrics +from .runtime import ObservabilityRuntime, initialize_observability + +__all__ = [ + 'Metrics', + 'ObservabilityRuntime', + 'get_metrics', + 'initialize_observability', +] diff --git a/app/observability/config.py b/app/observability/config.py new file mode 100644 index 0000000..fedefaa --- /dev/null +++ b/app/observability/config.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _positive_int(name: str, default: str) -> int: + try: + value = int(os.getenv(name, default)) + except ValueError as exc: + raise RuntimeError(f'{name} должен быть целым числом') from exc + if value <= 0: + raise RuntimeError(f'{name} должен быть больше нуля') + return value + + +@dataclass(frozen=True, slots=True) +class ObservabilitySettings: + enabled: bool + endpoint: str + service_name: str + service_version: str + environment: str + export_timeout_seconds: int + span_queue_size: int + log_queue_size: int + metric_export_interval_ms: int + + +def load_observability_settings(default_service_name: str) -> ObservabilitySettings: + disabled = os.getenv('OTEL_SDK_DISABLED', 'false').strip().lower() + if disabled not in {'true', 'false'}: + raise RuntimeError('OTEL_SDK_DISABLED должен быть true или false') + endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318') + return ObservabilitySettings( + enabled=disabled != 'true', + endpoint=endpoint.rstrip('/'), + service_name=os.getenv('OTEL_SERVICE_NAME', default_service_name), + service_version=os.getenv('OTEL_SERVICE_VERSION', '1.6.0'), + environment=os.getenv('OTEL_DEPLOYMENT_ENVIRONMENT', 'lab'), + export_timeout_seconds=_positive_int('OTEL_EXPORTER_OTLP_TIMEOUT', '2'), + span_queue_size=_positive_int('OTEL_BSP_MAX_QUEUE_SIZE', '2048'), + log_queue_size=_positive_int('OTEL_BLRP_MAX_QUEUE_SIZE', '2048'), + metric_export_interval_ms=_positive_int('OTEL_METRIC_EXPORT_INTERVAL', '5000'), + ) diff --git a/app/observability/logging.py b/app/observability/logging.py new file mode 100644 index 0000000..7d6222e --- /dev/null +++ b/app/observability/logging.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone + +from opentelemetry import trace + + +_STANDARD_FIELDS = { + 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 'funcName', + 'levelname', 'levelno', 'lineno', 'module', 'msecs', 'message', 'msg', 'name', + 'pathname', 'process', 'processName', 'relativeCreated', 'stack_info', 'thread', + 'threadName', 'taskName', +} + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, object] = { + 'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(), + 'severity': record.levelname, + 'logger': record.name, + 'message': record.getMessage(), + 'event': getattr(record, 'event', 'log.message'), + } + span_context = trace.get_current_span().get_span_context() + if span_context.is_valid: + payload['trace_id'] = format(span_context.trace_id, '032x') + payload['span_id'] = format(span_context.span_id, '016x') + for key, value in record.__dict__.items(): + if key in _STANDARD_FIELDS or key.startswith('_'): + continue + if isinstance(value, (str, int, float, bool)) or value is None: + payload[key] = value + if record.exc_info: + payload['exception.type'] = record.exc_info[0].__name__ + return json.dumps(payload, ensure_ascii=False, separators=(',', ':')) + + +def safe_extra(event: str, **attributes: object) -> dict[str, object]: + blocked = { + 'authorization', 'password', 'token', 'session_token', 'message_body', + 'image_bytes', 's3_secret_access_key', 'signed_url', + } + extra: dict[str, object] = {'event': event} + for key, value in attributes.items(): + normalized = key.lower() + if normalized in blocked or any(part in normalized for part in ('password', 'secret')): + continue + extra[key] = value + return extra diff --git a/app/observability/metrics.py b/app/observability/metrics.py new file mode 100644 index 0000000..52d604e --- /dev/null +++ b/app/observability/metrics.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import os +import threading +from dataclasses import dataclass +from typing import Mapping + +import psutil +from opentelemetry import metrics +from opentelemetry.metrics import Observation + + +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0) +JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0) +READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0) + + +class GaugeState: + def __init__(self) -> None: + self._lock = threading.Lock() + self._values: dict[tuple[tuple[str, str], ...], float] = {} + + def set(self, value: float, attributes: Mapping[str, str] | None = None) -> None: + key = tuple(sorted((attributes or {}).items())) + with self._lock: + self._values[key] = float(value) + + def add(self, amount: float, attributes: Mapping[str, str] | None = None) -> None: + key = tuple(sorted((attributes or {}).items())) + with self._lock: + self._values[key] = self._values.get(key, 0.0) + amount + + def observations(self, _options=None): + with self._lock: + snapshot = tuple(self._values.items()) + return [Observation(value, dict(attributes)) for attributes, value in snapshot] + + +@dataclass(slots=True) +class Metrics: + http_requests: object + http_duration: object + jobs_processed: object + job_duration: object + time_to_ready: object + queue_depth: GaugeState + queue_oldest_age: GaugeState + worker_busy: GaugeState + worker_capacity: GaugeState + db_pool_used: GaugeState + db_pool_waiting: GaugeState + + +_metrics: Metrics | None = None + + +def _process_rss(_options=None): + process = psutil.Process(os.getpid()) + return [Observation(process.memory_info().rss)] + + +def _process_cpu(_options=None): + process = psutil.Process(os.getpid()) + times = process.cpu_times() + return [Observation(times.user + times.system)] + + +def get_metrics() -> Metrics: + global _metrics + if _metrics is not None: + return _metrics + meter = metrics.get_meter('messenger.observability', '1.6.0') + queue_depth = GaugeState() + queue_oldest_age = GaugeState() + worker_busy = GaugeState() + worker_capacity = GaugeState() + db_pool_used = GaugeState() + db_pool_waiting = GaugeState() + meter.create_observable_gauge( + 'messenger.queue.depth', callbacks=[queue_depth.observations], unit='{job}' + ) + meter.create_observable_gauge( + 'messenger.queue.oldest.age', callbacks=[queue_oldest_age.observations], unit='s' + ) + meter.create_observable_gauge( + 'messenger.worker.busy', callbacks=[worker_busy.observations], unit='{worker}' + ) + meter.create_observable_gauge( + 'messenger.worker.capacity', callbacks=[worker_capacity.observations], unit='{worker}' + ) + meter.create_observable_gauge( + 'messenger.db.pool.used', callbacks=[db_pool_used.observations], unit='{connection}' + ) + meter.create_observable_gauge( + 'messenger.db.pool.waiting', callbacks=[db_pool_waiting.observations], unit='{request}' + ) + meter.create_observable_gauge('process.memory.rss', callbacks=[_process_rss], unit='By') + meter.create_observable_counter('process.cpu.time', callbacks=[_process_cpu], unit='s') + _metrics = Metrics( + http_requests=meter.create_counter( + 'http.server.request.count', unit='{request}', description='HTTP requests' + ), + http_duration=meter.create_histogram( + 'http.server.request.duration', unit='s', description='HTTP request duration' + ), + jobs_processed=meter.create_counter( + 'messenger.jobs.processed', unit='{job}', description='Finished job attempts' + ), + job_duration=meter.create_histogram( + 'messenger.job.processing.duration', unit='s', description='Job processing duration' + ), + time_to_ready=meter.create_histogram( + 'messenger.attachment.time_to_ready', unit='s', description='Durable accept to ready' + ), + queue_depth=queue_depth, + queue_oldest_age=queue_oldest_age, + worker_busy=worker_busy, + worker_capacity=worker_capacity, + db_pool_used=db_pool_used, + db_pool_waiting=db_pool_waiting, + ) + return _metrics diff --git a/app/observability/propagation.py b/app/observability/propagation.py new file mode 100644 index 0000000..b681a8e --- /dev/null +++ b/app/observability/propagation.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from collections.abc import Mapping + +from opentelemetry import propagate + + +_ALLOWED = {'traceparent', 'tracestate'} + + +def inject_trace_context(context=None) -> dict[str, str]: + carrier: dict[str, str] = {} + propagate.inject(carrier, context=context) + return {key: str(value) for key, value in carrier.items() if key in _ALLOWED} + + +def extract_trace_context(carrier: Mapping[str, object] | None): + normalized: dict[str, str] = {} + for key, value in (carrier or {}).items(): + lowered = str(key).lower() + if lowered not in _ALLOWED or value is None: + continue + if isinstance(value, bytes): + value = value.decode('ascii', errors='ignore') + normalized[lowered] = str(value) + return propagate.extract(normalized) diff --git a/app/observability/runtime.py b/app/observability/runtime.py new file mode 100644 index 0000000..f2442c7 --- /dev/null +++ b/app/observability/runtime.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import logging +import os +import socket +import sys +from dataclasses import dataclass + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +from .config import load_observability_settings +from .logging import JsonFormatter +from .metrics import HTTP_BUCKETS, JOB_BUCKETS, READY_BUCKETS + + +@dataclass(slots=True) +class ObservabilityRuntime: + tracer_provider: TracerProvider | None = None + meter_provider: MeterProvider | None = None + logger_provider: LoggerProvider | None = None + + def shutdown(self) -> None: + if self.logger_provider is not None: + self.logger_provider.shutdown() + if self.meter_provider is not None: + self.meter_provider.shutdown() + if self.tracer_provider is not None: + self.tracer_provider.shutdown() + + +_runtime: ObservabilityRuntime | None = None + + +def initialize_observability(default_service_name: str) -> ObservabilityRuntime: + global _runtime + if _runtime is not None: + return _runtime + settings = load_observability_settings(default_service_name) + console = logging.StreamHandler(sys.stdout) + console.setFormatter(JsonFormatter()) + handlers: list[logging.Handler] = [console] + runtime = ObservabilityRuntime() + if settings.enabled: + resource = Resource.create({ + 'service.name': settings.service_name, + 'service.version': settings.service_version, + 'service.instance.id': os.getenv( + 'INSTANCE_ID', f'{socket.gethostname()}:{os.getpid()}' + ), + 'deployment.environment.name': settings.environment, + }) + timeout = settings.export_timeout_seconds + tracer_provider = TracerProvider(resource=resource) + tracer_provider.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=f'{settings.endpoint}/v1/traces', timeout=timeout), + max_queue_size=settings.span_queue_size, + max_export_batch_size=min(512, settings.span_queue_size), + schedule_delay_millis=1000, + export_timeout_millis=timeout * 1000, + )) + trace.set_tracer_provider(tracer_provider) + + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f'{settings.endpoint}/v1/metrics', timeout=timeout), + export_interval_millis=settings.metric_export_interval_ms, + export_timeout_millis=timeout * 1000, + ) + meter_provider = MeterProvider( + resource=resource, + metric_readers=[metric_reader], + views=[ + View( + instrument_name='http.server.request.duration', + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS), + ), + View( + instrument_name='messenger.job.processing.duration', + aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS), + ), + View( + instrument_name='messenger.attachment.time_to_ready', + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS), + ), + ], + ) + metrics.set_meter_provider(meter_provider) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=f'{settings.endpoint}/v1/logs', timeout=timeout), + max_queue_size=settings.log_queue_size, + max_export_batch_size=min(512, settings.log_queue_size), + schedule_delay_millis=1000, + export_timeout_millis=timeout * 1000, + )) + set_logger_provider(logger_provider) + otlp_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + otlp_handler.setFormatter(JsonFormatter()) + handlers.append(otlp_handler) + runtime = ObservabilityRuntime(tracer_provider, meter_provider, logger_provider) + logging.basicConfig( + level=os.getenv('LOG_LEVEL', 'info').upper(), handlers=handlers, force=True + ) + _runtime = runtime + return runtime diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py new file mode 100644 index 0000000..1e0e7bf --- /dev/null +++ b/app/repositories/__init__.py @@ -0,0 +1,10 @@ +from .attachments import AttachmentRepository +from .chats import ChatRepository +from .messages import MessageRepository +from .users import UserRepository +from .jobs import JobRepository, OutboxRepository + +__all__ = [ + 'AttachmentRepository', 'ChatRepository', 'JobRepository', 'MessageRepository', + 'OutboxRepository', 'UserRepository', +] diff --git a/app/repositories/attachments.py b/app/repositories/attachments.py new file mode 100644 index 0000000..d42ee60 --- /dev/null +++ b/app/repositories/attachments.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID, uuid4 + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.models import Attachment, AttachmentVariant +from ..models.attachment import AttachmentRow, AttachmentVariantRow +from ..models.chat import ChatMemberRow, ChatRow +from ..models.job import AttachmentJobRow, OutboxEventRow + + +def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, now: datetime) -> dict: + return { + 'schema_version': 1, + 'job_id': str(job_id), + 'type': 'attachment.process.v1', + 'attachment_id': str(attachment_id), + 'pipeline_version': pipeline_version, + 'attempt': attempt, + 'created_at': now.isoformat().replace('+00:00', 'Z'), + } + + +class AttachmentRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _variant(row: AttachmentVariantRow) -> AttachmentVariant: + return AttachmentVariant( + name=row.name, + object_key=row.object_key, + content_type=row.content_type, + size_bytes=row.size_bytes, + width=row.width, + height=row.height, + sha256=row.sha256, + ) + + @classmethod + def _to_domain( + cls, row: AttachmentRow, variants: tuple[AttachmentVariantRow, ...] = () + ) -> Attachment: + return Attachment( + id=row.id, + chat_id=row.chat_id, + owner_id=row.owner_id, + filename=row.filename, + original_key=row.original_key, + staging_key=row.staging_key, + pipeline_version=row.pipeline_version, + content_type=row.content_type, + size_bytes=row.size_bytes, + sha256=row.sha256, + width=row.width, + height=row.height, + status=row.status, + error_code=row.error_code, + created_at=row.created_at, + updated_at=row.updated_at, + variants=tuple(cls._variant(item) for item in variants), + ) + + async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool: + async with self._sessions() as session: + found = await session.scalar( + select(ChatRow.id) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id) + ) + return found is not None + + async def create_queued( + self, + *, + attachment_id: UUID, + job_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + staging_key: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + max_attempts: int, + now: datetime, + traceparent: str | None = None, + tracestate: str | None = None, + ) -> Attachment: + row = AttachmentRow( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + original_key=None, + staging_key=staging_key, + pipeline_version=1, + content_type=content_type, + size_bytes=size_bytes, + sha256=sha256, + width=width, + height=height, + status='queued', + error_code=None, + created_at=now, + updated_at=now, + ) + job = AttachmentJobRow( + id=job_id, + attachment_id=attachment_id, + pipeline_version=1, + state='queued', + attempt=1, + max_attempts=max_attempts, + next_attempt_at=now, + lease_owner=None, + lease_until=None, + last_error_code=None, + replay_count=0, + created_at=now, + updated_at=now, + ) + outbox = OutboxEventRow( + id=uuid4(), + event_type='attachment.process.v1', + job_id=job_id, + payload=job_payload(job_id, attachment_id, 1, 1, now), + traceparent=traceparent, + tracestate=tracestate, + available_at=now, + published_at=None, + claim_owner=None, + claim_until=None, + publish_attempts=0, + last_error_code=None, + created_at=now, + updated_at=now, + ) + async with self._sessions() as session: + async with session.begin(): + session.add_all((row, job, outbox)) + await session.flush() + return self._to_domain(row) + + async def _variants( + self, session: AsyncSession, attachment_id: UUID, pipeline_version: int + ) -> tuple[AttachmentVariantRow, ...]: + rows = await session.scalars( + select(AttachmentVariantRow) + .where( + AttachmentVariantRow.attachment_id == attachment_id, + AttachmentVariantRow.pipeline_version == pipeline_version, + ) + .order_by(AttachmentVariantRow.name) + ) + return tuple(rows) + + async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None: + async with self._sessions() as session: + row = await session.scalar( + select(AttachmentRow) + .join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id) + .where( + AttachmentRow.id == attachment_id, + ChatMemberRow.user_id == user_id, + ) + ) + if row is None: + return None + return self._to_domain( + row, await self._variants(session, row.id, row.pipeline_version) + ) diff --git a/app/repositories/chats.py b/app/repositories/chats.py new file mode 100644 index 0000000..04d4261 --- /dev/null +++ b/app/repositories/chats.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Sequence +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.errors import Invalid +from ..domain.models import Chat, Keyset +from ..models.chat import ChatMemberRow, ChatRow +from ..models.user import UserRow + + +class ChatRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat: + return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at) + + @staticmethod + async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]: + result = await session.scalars( + select(ChatMemberRow.user_id) + .where(ChatMemberRow.chat_id == chat_id) + .order_by(ChatMemberRow.user_id) + ) + return tuple(result) + + async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat: + async with self._sessions() as session: + async with session.begin(): + found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids)))) + if len(found) != len(member_ids): + raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}') + row = ChatRow(id=chat_id, title=title, created_at=created_at) + session.add(row) + await session.flush() + session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids) + await session.flush() + return self._to_domain(row, member_ids) + + async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None: + async with self._sessions() as session: + row = await session.scalar( + select(ChatRow) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id) + ) + if row is None: + return None + return self._to_domain(row, await self._members(session, row.id)) + + async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]: + async with self._sessions() as session: + query = ( + select(ChatRow) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatMemberRow.user_id == user_id) + .order_by(ChatRow.created_at, ChatRow.id) + .limit(limit) + ) + if after is not None: + query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id)) + rows = list((await session.scalars(query)).all()) + return [self._to_domain(row, await self._members(session, row.id)) for row in rows] diff --git a/app/repositories/jobs.py b/app/repositories/jobs.py new file mode 100644 index 0000000..10dc8b9 --- /dev/null +++ b/app/repositories/jobs.py @@ -0,0 +1,388 @@ +from __future__ import annotations + +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +from sqlalchemy import func, or_, select, update +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.models import Attachment, AttachmentJob, AttachmentVariant, OutboxEvent +from ..models.attachment import AttachmentRow, AttachmentVariantRow +from ..models.job import AttachmentJobRow, JobEventRow, OutboxEventRow +from ..observability.propagation import inject_trace_context +from .attachments import AttachmentRepository, job_payload + + +class JobRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _job(row: AttachmentJobRow) -> AttachmentJob: + return AttachmentJob( + id=row.id, + attachment_id=row.attachment_id, + pipeline_version=row.pipeline_version, + state=row.state, + attempt=row.attempt, + max_attempts=row.max_attempts, + next_attempt_at=row.next_attempt_at, + lease_owner=row.lease_owner, + lease_until=row.lease_until, + last_error_code=row.last_error_code, + replay_count=row.replay_count, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + async def claim( + self, + job_id: UUID, + attempt: int, + worker_id: str, + now: datetime, + lease_seconds: int, + ) -> tuple[AttachmentJob, Attachment] | None: + async with self._sessions() as session: + async with session.begin(): + row = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if row is None or row.state != 'queued' or row.attempt != attempt: + return None + attachment_row = await session.get(AttachmentRow, row.attachment_id) + if attachment_row is None: + return None + row.state = 'processing' + row.lease_owner = worker_id + row.lease_until = now + timedelta(seconds=lease_seconds) + row.updated_at = now + attachment_row.status = 'processing' + attachment_row.updated_at = now + session.add( + JobEventRow( + job_id=row.id, + kind='claimed', + attempt=row.attempt, + error_code=None, + created_at=now, + ) + ) + await session.flush() + return self._job(row), AttachmentRepository._to_domain(attachment_row) + + async def state(self, job_id: UUID) -> AttachmentJob | None: + async with self._sessions() as session: + row = await session.get(AttachmentJobRow, job_id) + return self._job(row) if row is not None else None + + async def id_for_attachment(self, attachment_id: UUID) -> UUID | None: + async with self._sessions() as session: + return await session.scalar( + select(AttachmentJobRow.id).where( + AttachmentJobRow.attachment_id == attachment_id + ) + ) + + async def failed_staging_key(self, job_id: UUID) -> str | None: + async with self._sessions() as session: + return await session.scalar( + select(AttachmentRow.staging_key) + .join(AttachmentJobRow, AttachmentJobRow.attachment_id == AttachmentRow.id) + .where(AttachmentJobRow.id == job_id, AttachmentJobRow.state == 'failed') + ) + + async def ready_effect_complete(self, job_id: UUID) -> bool: + async with self._sessions() as session: + job = await session.get(AttachmentJobRow, job_id) + if job is None or job.state != 'ready': + return False + attachment = await session.get(AttachmentRow, job.attachment_id) + if attachment is None or not attachment.original_key: + return False + names = set( + await session.scalars( + select(AttachmentVariantRow.name).where( + AttachmentVariantRow.attachment_id == job.attachment_id, + AttachmentVariantRow.pipeline_version == job.pipeline_version, + ) + ) + ) + return names == {'thumbnail', 'crop', 'optimized'} + + async def complete( + self, + job_id: UUID, + worker_id: str, + original_key: str, + variants: tuple[AttachmentVariant, ...], + now: datetime, + ) -> bool: + async with self._sessions() as session: + async with session.begin(): + job = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if job is None or job.state == 'ready': + return job is not None + if job.state != 'processing' or job.lease_owner != worker_id: + return False + attachment = await session.get(AttachmentRow, job.attachment_id) + if attachment is None: + return False + for variant in variants: + statement = ( + insert(AttachmentVariantRow) + .values( + attachment_id=attachment.id, + pipeline_version=job.pipeline_version, + name=variant.name, + object_key=variant.object_key, + content_type=variant.content_type, + size_bytes=variant.size_bytes, + width=variant.width, + height=variant.height, + sha256=variant.sha256, + created_at=now, + updated_at=now, + ) + .on_conflict_do_update( + index_elements=[ + AttachmentVariantRow.attachment_id, + AttachmentVariantRow.pipeline_version, + AttachmentVariantRow.name, + ], + set_={ + 'object_key': variant.object_key, + 'content_type': variant.content_type, + 'size_bytes': variant.size_bytes, + 'width': variant.width, + 'height': variant.height, + 'sha256': variant.sha256, + 'updated_at': now, + }, + ) + ) + await session.execute(statement) + attachment.original_key = original_key + attachment.status = 'ready' + attachment.error_code = None + attachment.updated_at = now + job.state = 'ready' + job.lease_owner = None + job.lease_until = None + job.last_error_code = None + job.updated_at = now + session.add( + JobEventRow( + job_id=job.id, + kind='ready', + attempt=job.attempt, + error_code=None, + created_at=now, + ) + ) + return True + + async def retry_or_fail( + self, + job_id: UUID, + worker_id: str | None, + error_code: str, + permanent: bool, + available_at: datetime, + now: datetime, + ) -> str: + async with self._sessions() as session: + async with session.begin(): + job = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if job is None or job.state in {'ready', 'failed'}: + return job.state if job is not None else 'missing' + if job.state != 'processing': + return job.state + if worker_id is not None: + if job.lease_owner != worker_id: + return job.state + elif job.lease_until is None or job.lease_until >= now: + return job.state + attachment = await session.get(AttachmentRow, job.attachment_id) + exhausted = job.attempt >= job.max_attempts + if permanent or exhausted: + job.state = 'failed' + attachment.status = 'failed' + attachment.error_code = error_code if permanent else 'retry_exhausted' + kind = 'failed' + else: + job.attempt += 1 + job.state = 'queued' + job.next_attempt_at = available_at + attachment.status = 'queued' + attachment.error_code = None + trace_context = inject_trace_context() + session.add( + OutboxEventRow( + id=uuid4(), + event_type='attachment.process.v1', + job_id=job.id, + payload=job_payload( + job.id, + job.attachment_id, + job.pipeline_version, + job.attempt, + now, + ), + traceparent=trace_context.get('traceparent'), + tracestate=trace_context.get('tracestate'), + available_at=available_at, + published_at=None, + claim_owner=None, + claim_until=None, + publish_attempts=0, + last_error_code=None, + created_at=now, + updated_at=now, + ) + ) + kind = 'retry_scheduled' + job.last_error_code = error_code + job.lease_owner = None + job.lease_until = None + job.updated_at = now + attachment.updated_at = now + session.add( + JobEventRow( + job_id=job.id, + kind=kind, + attempt=job.attempt, + error_code=error_code, + created_at=now, + ) + ) + await session.flush() + return job.state + + async def queue_stats(self, now: datetime) -> tuple[int, float]: + async with self._sessions() as session: + depth, oldest = ( + await session.execute( + select(func.count(AttachmentJobRow.id), func.min(AttachmentJobRow.created_at)) + .where(AttachmentJobRow.state.in_(('queued', 'processing'))) + ) + ).one() + age = max(0.0, (now - oldest).total_seconds()) if oldest is not None else 0.0 + return int(depth or 0), age + + async def recover_expired(self, now: datetime) -> int: + async with self._sessions() as session: + ids = tuple( + await session.scalars( + select(AttachmentJobRow.id).where( + AttachmentJobRow.state == 'processing', + AttachmentJobRow.lease_until < now, + ) + ) + ) + recovered = 0 + for job_id in ids: + state = await self.retry_or_fail( + job_id, None, 'lease_expired', False, now, now + ) + recovered += state in {'queued', 'failed'} + return recovered + + async def replay(self, job_id: UUID, now: datetime, dry_run: bool = False) -> bool: + async with self._sessions() as session: + async with session.begin(): + job = await session.scalar( + select(AttachmentJobRow).where(AttachmentJobRow.id == job_id).with_for_update() + ) + if job is None or job.state != 'failed': + return False + attachment = await session.get(AttachmentRow, job.attachment_id) + if attachment is None or not attachment.staging_key: + return False + if dry_run: + return True + job.state = 'queued' + job.attempt += 1 + job.max_attempts = job.attempt + 4 + job.replay_count += 1 + job.last_error_code = None + job.next_attempt_at = now + job.updated_at = now + attachment.status = 'queued' + attachment.error_code = None + attachment.updated_at = now + session.add_all( + ( + OutboxEventRow( + id=uuid4(), event_type='attachment.process.v1', job_id=job.id, + payload=job_payload(job.id, job.attachment_id, job.pipeline_version, job.attempt, now), + traceparent=None, tracestate=None, + available_at=now, published_at=None, claim_owner=None, claim_until=None, + publish_attempts=0, last_error_code=None, created_at=now, updated_at=now, + ), + JobEventRow(job_id=job.id, kind='replayed', attempt=job.attempt, error_code=None, created_at=now), + ) + ) + return True + + +class OutboxRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def claim(self, owner: str, now: datetime, lease_seconds: int, limit: int = 20) -> tuple[OutboxEvent, ...]: + async with self._sessions() as session: + async with session.begin(): + rows = tuple( + await session.scalars( + select(OutboxEventRow) + .where( + OutboxEventRow.published_at.is_(None), + OutboxEventRow.available_at <= now, + or_(OutboxEventRow.claim_until.is_(None), OutboxEventRow.claim_until < now), + ) + .order_by(OutboxEventRow.available_at, OutboxEventRow.created_at) + .limit(limit) + .with_for_update(skip_locked=True) + ) + ) + until = now + timedelta(seconds=lease_seconds) + for row in rows: + row.claim_owner = owner + row.claim_until = until + row.publish_attempts += 1 + row.updated_at = now + return tuple( + OutboxEvent( + row.id, row.job_id, row.payload, row.available_at, + row.traceparent, row.tracestate, + ) + for row in rows + ) + + async def published(self, event_id: UUID, owner: str, now: datetime) -> None: + async with self._sessions() as session: + async with session.begin(): + await session.execute( + update(OutboxEventRow) + .where( + OutboxEventRow.id == event_id, + OutboxEventRow.claim_owner == owner, + OutboxEventRow.published_at.is_(None), + ) + .values(published_at=now, claim_owner=None, claim_until=None, last_error_code=None, updated_at=now) + ) + + async def failed(self, event_id: UUID, owner: str, code: str, now: datetime) -> None: + async with self._sessions() as session: + async with session.begin(): + await session.execute( + update(OutboxEventRow) + .where(OutboxEventRow.id == event_id, OutboxEventRow.claim_owner == owner) + .values(claim_owner=None, claim_until=None, last_error_code=code, updated_at=now) + ) diff --git a/app/repositories/messages.py b/app/repositories/messages.py new file mode 100644 index 0000000..2df72bf --- /dev/null +++ b/app/repositories/messages.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Sequence +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.errors import Conflict, NotFound +from ..domain.models import Keyset, Message +from ..models.attachment import AttachmentRow, MessageAttachmentRow +from ..models.chat import ChatMemberRow, ChatRow +from ..models.message import MessageRow + + +class MessageRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message: + return Message( + id=row.id, + chat_id=row.chat_id, + sender_id=row.sender_id, + text=row.text, + created_at=row.created_at, + attachment_ids=tuple(attachment_ids), + ) + + @staticmethod + async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]: + result = await session.scalars( + select(MessageAttachmentRow.attachment_id) + .where(MessageAttachmentRow.message_id == message_id) + .order_by(MessageAttachmentRow.position) + ) + return tuple(result) + + @staticmethod + async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None: + return await session.scalar( + select(ChatRow.id) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id) + ) + + async def create_for_member( + self, + message_id: UUID, + chat_id: UUID, + sender_id: UUID, + text: str, + attachment_ids: Sequence[UUID], + created_at: datetime, + idempotency_key: str, + ) -> Message | None: + async with self._sessions() as session: + async with session.begin(): + accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id) + if accessible_chat_id is None: + return None + + if attachment_ids: + found = { + attachment_id: status + for attachment_id, status in ( + await session.execute( + select(AttachmentRow.id, AttachmentRow.status).where( + AttachmentRow.id.in_(attachment_ids), + AttachmentRow.chat_id == accessible_chat_id, + ) + ) + ).all() + } + if set(found) != set(attachment_ids): + raise NotFound('вложение не найдено') + if any(status == 'failed' for status in found.values()): + raise Conflict('неуспешное вложение нельзя прикрепить') + + statement = ( + insert(MessageRow) + .values( + id=message_id, + chat_id=accessible_chat_id, + sender_id=sender_id, + text=text, + created_at=created_at, + idempotency_key=idempotency_key, + ) + .on_conflict_do_nothing( + index_elements=[ + MessageRow.sender_id, + MessageRow.chat_id, + MessageRow.idempotency_key, + ] + ) + .returning(MessageRow.id) + ) + created_id = await session.scalar(statement) + if created_id is not None: + row = await session.get(MessageRow, created_id) + session.add_all( + MessageAttachmentRow( + message_id=created_id, + attachment_id=attachment_id, + position=position, + ) + for position, attachment_id in enumerate(attachment_ids) + ) + await session.flush() + result_attachment_ids = tuple(attachment_ids) + else: + row = await session.scalar( + select(MessageRow).where( + MessageRow.sender_id == sender_id, + MessageRow.chat_id == accessible_chat_id, + MessageRow.idempotency_key == idempotency_key, + ) + ) + if row is None: + raise RuntimeError('результат идемпотентной операции не найден') + result_attachment_ids = await self._attachment_ids(session, row.id) + if row.text != text or result_attachment_ids != tuple(attachment_ids): + raise Conflict('Idempotency-Key уже использован с другим сообщением') + return self._to_domain(row, result_attachment_ids) + + async def page_for_member( + self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int + ) -> list[Message] | None: + async with self._sessions() as session: + accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id) + if accessible_chat_id is None: + return None + query = ( + select(MessageRow) + .where(MessageRow.chat_id == accessible_chat_id) + .order_by(MessageRow.created_at, MessageRow.id) + .limit(limit) + ) + if after is not None: + query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id)) + rows = list((await session.scalars(query)).all()) + return [ + self._to_domain(row, await self._attachment_ids(session, row.id)) + for row in rows + ] diff --git a/app/repositories/users.py b/app/repositories/users.py new file mode 100644 index 0000000..d65b165 --- /dev/null +++ b/app/repositories/users.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.errors import Conflict +from ..domain.models import User +from ..models.user import UserRow + + +class UserRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: UserRow) -> User: + return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at) + + async def register( + self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime + ) -> User: + row = UserRow( + id=user_id, + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=created_at, + ) + try: + async with self._sessions() as session: + async with session.begin(): + session.add(row) + await session.flush() + except IntegrityError as exc: + raise Conflict('username уже занят') from exc + return self._to_domain(row) + + async def get_by_id(self, user_id: UUID) -> User | None: + async with self._sessions() as session: + row = await session.get(UserRow, user_id) + return self._to_domain(row) if row is not None else None + + async def get_credentials(self, username: str) -> UserRow | None: + async with self._sessions() as session: + return await session.scalar(select(UserRow).where(UserRow.username == username)) + + @classmethod + def to_domain(cls, row: UserRow) -> User: + return cls._to_domain(row) diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/services/attachments.py b/app/services/attachments.py new file mode 100644 index 0000000..71e8905 --- /dev/null +++ b/app/services/attachments.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import asyncio +import hashlib +import logging +from datetime import datetime, timezone +from tempfile import SpooledTemporaryFile +from typing import AsyncIterator, BinaryIO +import unicodedata +from uuid import UUID, uuid4 + +from PIL import Image, UnidentifiedImageError + +from ..domain.errors import ( + Conflict, + Invalid, + NotFound, + PayloadTooLarge, + Unavailable, + UnsupportedMediaType, +) +from ..domain.models import Attachment, User +from ..observability.logging import safe_extra +from ..observability.propagation import inject_trace_context +from ..repositories import AttachmentRepository +from ..storage import ObjectStream, S3Storage + +logger = logging.getLogger('app.attachments') + +MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'} + + +def _validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255: + raise Invalid('filename должен содержать от 1 до 255 символов') + if any( + character in {'/', '\\'} or unicodedata.category(character) == 'Cc' + for character in filename + ): + raise Invalid('filename содержит запрещённые символы') + + +def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]: + try: + source.seek(0) + with Image.open(source) as image: + actual_type = MIME_BY_FORMAT.get(image.format or '') + if actual_type is None or actual_type != declared_type: + raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG') + width, height = image.size + if width <= 0 or height <= 0: + raise UnsupportedMediaType('изображение имеет неверные размеры') + if width * height > max_pixels: + raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') + image.verify() + except Image.DecompressionBombError as error: + raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error + except (UnidentifiedImageError, OSError, SyntaxError) as error: + raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error + finally: + source.seek(0) + return actual_type, width, height + + +class AttachmentService: + def __init__( + self, + repository: AttachmentRepository, + storage: S3Storage, + max_upload_bytes: int, + max_image_pixels: int, + job_max_attempts: int, + ) -> None: + self._repository = repository + self._storage = storage + self._max_upload_bytes = max_upload_bytes + self._max_image_pixels = max_image_pixels + self._job_max_attempts = job_max_attempts + + async def upload( + self, + actor: User, + chat_id: UUID, + filename: str, + declared_type: str, + chunks: AsyncIterator[bytes], + ) -> Attachment: + _validate_filename(filename) + if declared_type not in MIME_BY_FORMAT.values(): + raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg') + if not await self._repository.chat_exists_for_member(chat_id, actor.id): + raise NotFound('чат не найден') + + digest = hashlib.sha256() + size = 0 + with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source: + async for chunk in chunks: + size += len(chunk) + if size > self._max_upload_bytes: + raise PayloadTooLarge('файл превышает лимит 10 MiB') + digest.update(chunk) + source.write(chunk) + if size == 0: + raise UnsupportedMediaType('пустой файл не является изображением') + + actual_type, width, height = await asyncio.to_thread( + _inspect_image, + source, + declared_type, + self._max_image_pixels, + ) + attachment_id = uuid4() + job_id = uuid4() + staging_key = f'staging/{attachment_id}/original' + await self._storage.put(staging_key, source, size, actual_type) + now = datetime.now(timezone.utc) + trace_context = inject_trace_context() + try: + attachment = await self._repository.create_queued( + attachment_id=attachment_id, + job_id=job_id, + chat_id=chat_id, + owner_id=actor.id, + filename=filename, + staging_key=staging_key, + content_type=actual_type, + size_bytes=size, + sha256=digest.hexdigest(), + width=width, + height=height, + max_attempts=self._job_max_attempts, + now=now, + traceparent=trace_context.get('traceparent'), + tracestate=trace_context.get('tracestate'), + ) + logger.info( + 'attachment accepted for asynchronous processing', + extra=safe_extra( + 'attachment.accepted', attachment_id=str(attachment_id), + job_id=str(job_id), pipeline_version=1, + ), + ) + return attachment + except Exception: + try: + await self._storage.remove(staging_key) + except Unavailable: + logger.warning( + 'failed to compensate staging object', + extra=safe_extra( + 'staging.cleanup_deferred', attachment_id=str(attachment_id) + ), + ) + raise + + async def get(self, actor: User, attachment_id: UUID) -> Attachment: + attachment = await self._repository.get_for_member(attachment_id, actor.id) + if attachment is None: + raise NotFound('вложение не найдено') + return attachment + + async def content( + self, actor: User, attachment_id: UUID, variant: str + ) -> tuple[Attachment, ObjectStream, str, int]: + attachment = await self.get(actor, attachment_id) + if attachment.status != 'ready': + raise Conflict('вложение ещё не готово') + if variant == 'original': + key = attachment.original_key + content_type = attachment.content_type + size_bytes = attachment.size_bytes + else: + found = next((item for item in attachment.variants if item.name == variant), None) + if found is None: + raise NotFound('вариант вложения не найден') + key = found.object_key + content_type = found.content_type + size_bytes = found.size_bytes + if key is None: + raise Unavailable('объект вложения отсутствует в метаданных') + try: + stream = await self._storage.get(key) + except FileNotFoundError as error: + raise Unavailable('объект вложения отсутствует в хранилище') from error + return attachment, stream, content_type, size_bytes diff --git a/app/services/chats.py b/app/services/chats.py new file mode 100644 index 0000000..9b55f71 --- /dev/null +++ b/app/services/chats.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional, Sequence +from uuid import UUID, uuid4 + +from ..domain.errors import Invalid, NotFound +from ..domain.models import Chat, Keyset, Page, User +from ..repositories import ChatRepository + + +class ChatService: + def __init__(self, repository: ChatRepository) -> None: + self._repository = repository + + async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat: + if actor.id not in member_ids: + raise Invalid('вызывающий должен входить в member_ids') + return await self._repository.create( + chat_id=uuid4(), + title=title, + member_ids=member_ids, + created_at=datetime.now(timezone.utc), + ) + + async def get(self, actor: User, chat_id: UUID) -> Chat: + chat = await self._repository.get_for_member(chat_id, actor.id) + if chat is None: + raise NotFound('чат не найден') + return chat + + async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page: + rows = await self._repository.page_for_member(actor.id, after, limit + 1) + items = tuple(rows[:limit]) + return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None) diff --git a/app/services/messages.py b/app/services/messages.py new file mode 100644 index 0000000..509c437 --- /dev/null +++ b/app/services/messages.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional, Sequence +from uuid import UUID, uuid4 + +from ..domain.errors import NotFound +from ..domain.models import Keyset, Message, Page, User +from ..repositories import MessageRepository + + +class MessageService: + def __init__(self, repository: MessageRepository) -> None: + self._repository = repository + + async def send( + self, + actor: User, + chat_id: UUID, + text: str, + attachment_ids: Sequence[UUID], + idempotency_key: str, + ) -> Message: + message = await self._repository.create_for_member( + message_id=uuid4(), + chat_id=chat_id, + sender_id=actor.id, + text=text, + attachment_ids=attachment_ids, + created_at=datetime.now(timezone.utc), + idempotency_key=idempotency_key, + ) + if message is None: + raise NotFound('чат не найден') + return message + + async def history( + self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int + ) -> Page: + rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1) + if rows is None: + raise NotFound('чат не найден') + items = tuple(rows[:limit]) + return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None) diff --git a/app/services/users.py b/app/services/users.py new file mode 100644 index 0000000..9a516a8 --- /dev/null +++ b/app/services/users.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from ..auth.passwords import hash_password, password_matches +from ..domain.errors import Unauthorized +from ..domain.models import User +from ..repositories import UserRepository + + +class UserService: + def __init__(self, repository: UserRepository) -> None: + self._repository = repository + + async def register(self, username: str, display_name: str, password: str) -> User: + return await self._repository.register( + user_id=uuid4(), + username=username, + display_name=display_name, + password_hash=hash_password(password), + created_at=datetime.now(timezone.utc), + ) + + async def identify(self, user_id: UUID) -> User: + user = await self._repository.get_by_id(user_id) + if user is None: + raise Unauthorized('требуется действующая сессия') + return user + + async def authenticate(self, username: str, password: str) -> User | None: + row = await self._repository.get_credentials(username) + if row is None or not password_matches(row.password_hash, password): + return None + return self._repository.to_domain(row) diff --git a/app/storage/__init__.py b/app/storage/__init__.py new file mode 100644 index 0000000..17b3147 --- /dev/null +++ b/app/storage/__init__.py @@ -0,0 +1,3 @@ +from .s3 import ObjectStream, S3Storage, StoredObject + +__all__ = ['ObjectStream', 'S3Storage', 'StoredObject'] diff --git a/app/storage/s3.py b/app/storage/s3.py new file mode 100644 index 0000000..4c6f3dc --- /dev/null +++ b/app/storage/s3.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import asyncio +import certifi +from dataclasses import dataclass +from datetime import datetime +from typing import BinaryIO, Iterator +from io import BytesIO +from urllib.parse import urlparse + +from minio import Minio +from minio.error import InvalidResponseError, S3Error +from urllib3 import PoolManager, Retry, Timeout +from urllib3.exceptions import HTTPError +from opentelemetry import trace +from opentelemetry.trace import SpanKind + +from ..domain.errors import Unavailable + +tracer = trace.get_tracer('messenger.s3', '1.6.0') + + +def _unavailable(error: Exception) -> Unavailable: + return Unavailable('объектное хранилище недоступно') + + +@dataclass(slots=True) +class ObjectStream: + response: object + + def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]: + try: + while chunk := self.response.read(size): + yield chunk + finally: + self.response.close() + self.response.release_conn() + + +@dataclass(frozen=True, slots=True) +class StoredObject: + key: str + last_modified: datetime + + +class S3Storage: + def __init__( + self, + endpoint: str, + access_key: str, + secret_key: str, + bucket: str, + region: str, + ) -> None: + parsed = urlparse(endpoint) + if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}: + raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port') + self._client = Minio( + parsed.netloc, + access_key=access_key, + secret_key=secret_key, + secure=parsed.scheme == 'https', + region=region, + http_client=PoolManager( + maxsize=8, + timeout=Timeout(connect=2, read=5), + retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1), + cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where(), + ), + ) + self.bucket = bucket + + @staticmethod + def _translate(error: Exception) -> None: + if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}: + raise FileNotFoundError from error + if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)): + raise _unavailable(error) from error + raise error + + async def ping(self) -> None: + try: + with tracer.start_as_current_span( + 's3 bucket exists', kind=SpanKind.CLIENT, + attributes={'db.system': 's3', 's3.bucket.name': self.bucket}, + ): + exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket) + except Exception as error: + self._translate(error) + if not exists: + raise Unavailable('bucket объектного хранилища не создан') + + async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None: + source.seek(0) + try: + with tracer.start_as_current_span( + 's3 put object', kind=SpanKind.CLIENT, + attributes={'db.system': 's3', 's3.bucket.name': self.bucket}, + ): + await asyncio.to_thread( + self._client.put_object, + self.bucket, + key, + source, + length, + content_type, + ) + except Exception as error: + self._translate(error) + + async def get(self, key: str) -> ObjectStream: + try: + with tracer.start_as_current_span( + 's3 get object', kind=SpanKind.CLIENT, + attributes={'db.system': 's3', 's3.bucket.name': self.bucket}, + ): + response = await asyncio.to_thread(self._client.get_object, self.bucket, key) + except Exception as error: + self._translate(error) + return ObjectStream(response) + + async def read(self, key: str, max_bytes: int) -> bytes: + stream = await self.get(key) + result = bytearray() + for chunk in stream.chunks(): + result.extend(chunk) + if len(result) > max_bytes: + raise ValueError('stored object exceeds configured byte limit') + return bytes(result) + + async def put_bytes(self, key: str, content: bytes, content_type: str) -> None: + await self.put(key, BytesIO(content), len(content), content_type) + + async def exists(self, key: str) -> bool: + try: + with tracer.start_as_current_span( + 's3 stat object', kind=SpanKind.CLIENT, + attributes={'db.system': 's3', 's3.bucket.name': self.bucket}, + ): + await asyncio.to_thread(self._client.stat_object, self.bucket, key) + return True + except Exception as error: + try: + self._translate(error) + except FileNotFoundError: + return False + raise + + async def remove(self, key: str) -> None: + try: + with tracer.start_as_current_span( + 's3 remove object', kind=SpanKind.CLIENT, + attributes={'db.system': 's3', 's3.bucket.name': self.bucket}, + ): + await asyncio.to_thread(self._client.remove_object, self.bucket, key) + except Exception as error: + self._translate(error) + + async def list(self, prefix: str = '') -> tuple[StoredObject, ...]: + def collect() -> tuple[StoredObject, ...]: + return tuple( + StoredObject(item.object_name, item.last_modified) + for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True) + ) + + try: + return await asyncio.to_thread(collect) + except Exception as error: + self._translate(error) diff --git a/app/tools/__init__.py b/app/tools/__init__.py new file mode 100644 index 0000000..621302d --- /dev/null +++ b/app/tools/__init__.py @@ -0,0 +1 @@ +"""Операторские CLI-инструменты приложения.""" diff --git a/app/tools/cleanup_orphans.py b/app/tools/cleanup_orphans.py new file mode 100644 index 0000000..ae5eb14 --- /dev/null +++ b/app/tools/cleanup_orphans.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..storage import S3Storage + + +async def cleanup(apply: bool, grace_seconds: int) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + try: + sessions = create_session_factory(engine) + async with sessions() as session: + known = set( + key + for key in await session.scalars(select(AttachmentRow.original_key)) + if key is not None + ) + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + removed = 0 + for item in await storage.list('originals/'): + if item.key in known or item.last_modified > cutoff: + continue + action = 'DELETE' if apply else 'WOULD_DELETE' + print(f'{action} key={item.key} modified={item.last_modified.isoformat()}') + if apply: + await storage.remove(item.key) + removed += 1 + print( + f'summary mode={"apply" if apply else "dry-run"} ' + f'known={len(known)} removed={removed} grace_seconds={grace_seconds}' + ) + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata') + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--dry-run', action='store_true') + mode.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error('--grace-seconds must be at least 60') + raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/cleanup_staging.py b/app/tools/cleanup_staging.py new file mode 100644 index 0000000..301e6c5 --- /dev/null +++ b/app/tools/cleanup_staging.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..storage import S3Storage + + +async def cleanup(apply: bool, retention_seconds: int) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + cutoff = datetime.now(timezone.utc) - timedelta(seconds=retention_seconds) + try: + async with sessions() as session: + active = set( + key + for key in await session.scalars( + select(AttachmentRow.staging_key).where( + AttachmentRow.staging_key.is_not(None), + AttachmentRow.status.in_(('queued', 'processing')), + ) + ) + if key is not None + ) + removed = 0 + for item in await storage.list('staging/'): + if item.key in active or item.last_modified > cutoff: + continue + print(f'{"DELETE" if apply else "WOULD_DELETE"} key={item.key}') + if apply: + await storage.remove(item.key) + removed += 1 + print(f'summary active={len(active)} removed={removed} mode={"apply" if apply else "dry-run"}') + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description='Clean inactive durable staging objects') + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--dry-run', action='store_true') + mode.add_argument('--apply', action='store_true') + parser.add_argument('--retention-seconds', type=int) + args = parser.parse_args() + settings = load_settings() + retention = args.retention_seconds or settings.staging_retention_seconds + if retention < 60: + parser.error('retention must be at least 60 seconds') + raise SystemExit(asyncio.run(cleanup(args.apply, retention))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/reconcile_attachments.py b/app/tools/reconcile_attachments.py new file mode 100644 index 0000000..39e1ec3 --- /dev/null +++ b/app/tools/reconcile_attachments.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import time +import tracemalloc +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Callable, Sequence +from uuid import UUID + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..storage import S3Storage, StoredObject + +MISSING_ERROR_CODE = 'original_missing' + + +@dataclass(frozen=True, slots=True) +class AttachmentSnapshot: + id: UUID + object_key: str + status: str + updated_at: datetime + + +@dataclass(frozen=True, slots=True) +class ReconcilePlan: + orphan: tuple[StoredObject, ...] + fresh_orphan: tuple[StoredObject, ...] + missing: tuple[AttachmentSnapshot, ...] + fresh_missing: tuple[AttachmentSnapshot, ...] + correct: int + + +def build_plan( + attachments: Sequence[AttachmentSnapshot], + objects: Sequence[StoredObject], + cutoff: datetime, +) -> ReconcilePlan: + """Compare immutable PostgreSQL and S3 snapshots without changing either side.""" + known_keys = {attachment.object_key for attachment in attachments} + object_keys = {item.key for item in objects} + + unknown_objects = [item for item in objects if item.key not in known_keys] + orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff) + fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff) + + ready_without_object = [ + attachment + for attachment in attachments + if attachment.status == 'ready' and attachment.object_key not in object_keys + ] + missing = tuple( + attachment for attachment in ready_without_object if attachment.updated_at <= cutoff + ) + fresh_missing = tuple( + attachment for attachment in ready_without_object if attachment.updated_at > cutoff + ) + correct = sum(attachment.object_key in object_keys for attachment in attachments) + return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct) + + +async def _load_attachments( + sessions: async_sessionmaker[AsyncSession], +) -> tuple[AttachmentSnapshot, ...]: + async with sessions() as session: + rows = ( + await session.execute( + select( + AttachmentRow.id, + AttachmentRow.original_key, + AttachmentRow.status, + AttachmentRow.updated_at, + ) + ) + ).all() + return tuple(AttachmentSnapshot(*row) for row in rows) + + +async def _metadata_key_exists( + sessions: async_sessionmaker[AsyncSession], object_key: str +) -> bool: + async with sessions() as session: + attachment_id = await session.scalar( + select(AttachmentRow.id).where(AttachmentRow.original_key == object_key).limit(1) + ) + return attachment_id is not None + + +async def reconcile( + sessions: async_sessionmaker[AsyncSession], + storage: S3Storage, + *, + apply: bool, + grace_seconds: int, + now: datetime | None = None, + output: Callable[[str], None] = print, +) -> dict[str, int | float | str]: + started = time.perf_counter() + tracemalloc.start() + current_time = now or datetime.now(timezone.utc) + cutoff = current_time - timedelta(seconds=grace_seconds) + + attachments = await _load_attachments(sessions) + objects = await storage.list('originals/') + plan = build_plan(attachments, objects, cutoff) + + for item in plan.fresh_orphan: + output( + f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}' + ) + for attachment in plan.fresh_missing: + output( + 'SKIP_FRESH_MISSING ' + f'attachment_id={attachment.id} key={attachment.object_key} ' + f'updated={attachment.updated_at.isoformat()}' + ) + + removed = 0 + race_protected = 0 + for item in plan.orphan: + action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN' + if not apply: + output(f'{action} key={item.key} modified={item.last_modified.isoformat()}') + continue + # PostgreSQL and S3 cannot share a transaction. Re-read the authoritative + # metadata immediately before a destructive action so a row created after + # the initial snapshot protects its object. + if await _metadata_key_exists(sessions, item.key): + output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}') + race_protected += 1 + continue + output(f'{action} key={item.key} modified={item.last_modified.isoformat()}') + await storage.remove(item.key) + removed += 1 + + marked_failed = 0 + for attachment in plan.missing: + if not apply: + output( + f'WOULD_MARK_MISSING attachment_id={attachment.id} ' + f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}' + ) + continue + + # The object may have been restored after list_objects(). Do not turn + # healthy metadata into failed based only on the earlier snapshot. + if await storage.exists(attachment.object_key): + output( + f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} ' + f'key={attachment.object_key}' + ) + race_protected += 1 + continue + + async with sessions() as session: + async with session.begin(): + result = await session.execute( + update(AttachmentRow) + .where( + AttachmentRow.id == attachment.id, + AttachmentRow.original_key == attachment.object_key, + AttachmentRow.status == 'ready', + AttachmentRow.updated_at == attachment.updated_at, + ) + .values( + status='failed', + error_code=MISSING_ERROR_CODE, + updated_at=current_time, + ) + ) + changed = result.rowcount or 0 + if changed: + output( + f'MARK_MISSING attachment_id={attachment.id} ' + f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}' + ) + marked_failed += changed + else: + output( + f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} ' + f'key={attachment.object_key}' + ) + race_protected += 1 + + _, peak_bytes = tracemalloc.get_traced_memory() + tracemalloc.stop() + summary: dict[str, int | float | str] = { + 'mode': 'apply' if apply else 'dry-run', + 'metadata': len(attachments), + 'objects': len(objects), + 'correct': plan.correct, + 'orphan': len(plan.orphan), + 'missing': len(plan.missing), + 'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing), + 'removed': removed, + 'marked_failed': marked_failed, + 'race_protected': race_protected, + 'grace_seconds': grace_seconds, + 'elapsed_ms': round((time.perf_counter() - started) * 1000, 3), + 'peak_memory_bytes': peak_bytes, + } + output(f'summary {json.dumps(summary, sort_keys=True)}') + return summary + + +async def run(apply: bool, grace_seconds: int) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + try: + await reconcile( + create_session_factory(engine), + storage, + apply=apply, + grace_seconds=grace_seconds, + ) + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser( + description='Reconcile attachment metadata in PostgreSQL with original objects in S3' + ) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--dry-run', action='store_true') + mode.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error('--grace-seconds must be at least 60') + raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/replay_failed.py b/app/tools/replay_failed.py new file mode 100644 index 0000000..03fcde4 --- /dev/null +++ b/app/tools/replay_failed.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timezone +from uuid import UUID + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..repositories import JobRepository +from ..storage import S3Storage + + +async def replay(job_id: UUID | None, attachment_id: UUID | None, dry_run: bool) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + try: + repository = JobRepository(create_session_factory(engine)) + if job_id is None: + assert attachment_id is not None + job_id = await repository.id_for_attachment(attachment_id) + if job_id is None: + print(f'REJECTED attachment_id={attachment_id} reason=job_not_found') + return 2 + staging_key = await repository.failed_staging_key(job_id) + if staging_key is None: + print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing') + return 2 + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + if not await storage.exists(staging_key): + print(f'REJECTED job_id={job_id} reason=staging_object_missing') + return 2 + accepted = await repository.replay(job_id, datetime.now(timezone.utc), dry_run) + if not accepted: + print(f'REJECTED job_id={job_id} reason=not_failed_or_staging_missing') + return 2 + print(f'{"WOULD_REPLAY" if dry_run else "REPLAYED"} job_id={job_id}') + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description='Safely replay one failed attachment job') + target = parser.add_mutually_exclusive_group(required=True) + target.add_argument('--job-id', type=UUID) + target.add_argument('--attachment-id', type=UUID) + parser.add_argument('--dry-run', action='store_true') + args = parser.parse_args() + raise SystemExit(asyncio.run(replay(args.job_id, args.attachment_id, args.dry_run))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/seed_reconciler_benchmark.py b/app/tools/seed_reconciler_benchmark.py new file mode 100644 index 0000000..cfc1063 --- /dev/null +++ b/app/tools/seed_reconciler_benchmark.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +from io import BytesIO +from uuid import NAMESPACE_URL, uuid5 + +from sqlalchemy import delete, insert + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..models.chat import ChatMemberRow, ChatRow +from ..models.user import UserRow +from ..storage import S3Storage + +USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user') +CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat') +OBJECT_PREFIX = 'originals/reconciler-benchmark/' +WORKERS = 16 + + +async def _put_objects( + storage: S3Storage, + keys: list[str], +) -> None: + for offset in range(0, len(keys), WORKERS): + batch = keys[offset : offset + WORKERS] + await asyncio.gather( + *(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch) + ) + + +async def prepare(size: int) -> None: + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + old = datetime.now(timezone.utc) - timedelta(hours=2) + rows = [ + { + 'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'), + 'chat_id': CHAT_ID, + 'owner_id': USER_ID, + 'filename': f'benchmark-{index}.png', + 'original_key': f'{OBJECT_PREFIX}{index}', + 'staging_key': None, + 'pipeline_version': 1, + 'content_type': 'image/png', + 'size_bytes': 1, + 'sha256': '0' * 64, + 'width': 1, + 'height': 1, + 'status': 'ready', + 'error_code': None, + 'created_at': old, + 'updated_at': old, + } + for index in range(size) + ] + try: + async with sessions() as session: + async with session.begin(): + session.add(UserRow( + id=USER_ID, + username=f'bench-{USER_ID.hex[:16]}', + display_name='Reconciler benchmark', + password_hash='not-used', + created_at=old, + )) + session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old)) + await session.flush() + session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID)) + await session.flush() + for offset in range(0, size, 500): + await session.execute(insert(AttachmentRow), rows[offset : offset + 500]) + + await _put_objects(storage, [row['original_key'] for row in rows]) + print(f'prepared metadata={size} objects={size}') + finally: + await engine.dispose() + + +async def cleanup() -> None: + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + try: + objects = await storage.list(OBJECT_PREFIX) + keys = [item.key for item in objects] + for offset in range(0, len(keys), WORKERS): + await asyncio.gather( + *(storage.remove(key) for key in keys[offset : offset + WORKERS]) + ) + + async with sessions() as session: + async with session.begin(): + await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID)) + await session.execute(delete(UserRow).where(UserRow.id == USER_ID)) + print(f'cleaned objects={len(keys)}') + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser( + description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks' + ) + subparsers = parser.add_subparsers(dest='command', required=True) + prepare_parser = subparsers.add_parser('prepare') + prepare_parser.add_argument('--size', type=int, required=True) + subparsers.add_parser('cleanup') + args = parser.parse_args() + + if args.command == 'prepare': + if args.size < 1 or args.size > 10000: + parser.error('--size must be between 1 and 10000') + asyncio.run(prepare(args.size)) + else: + asyncio.run(cleanup()) + + +if __name__ == '__main__': + main() diff --git a/compose.observability-tests.yaml b/compose.observability-tests.yaml new file mode 100644 index 0000000..ce6b1ac --- /dev/null +++ b/compose.observability-tests.yaml @@ -0,0 +1,14 @@ +services: + observability-test: + image: python:3.13.15-slim + working_dir: /tests + command: ["python", "observability_smoke.py", "--grafana-url", "http://grafana:3000"] + environment: + GRAFANA_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GRAFANA_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana} + volumes: + - ./tests:/tests:ro + depends_on: + grafana: + condition: service_healthy + restart: "no" diff --git a/compose.tests.yaml b/compose.tests.yaml new file mode 100644 index 0000000..232faeb --- /dev/null +++ b/compose.tests.yaml @@ -0,0 +1,198 @@ +services: + postgres: + image: postgres:17.4-alpine + environment: + POSTGRES_USER: messenger_test + POSTGRES_PASSWORD: messenger_test + POSTGRES_DB: messenger_test + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"] + interval: 2s + timeout: 2s + retries: 30 + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + + broker: + image: rabbitmq:4.1.4-alpine + environment: + RABBITMQ_DEFAULT_USER: messenger_test + RABBITMQ_DEFAULT_PASS: messenger_test + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 2s + timeout: 5s + retries: 30 + + s3: + image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597 + user: "0" + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: test-minio-admin + MINIO_ROOT_PASSWORD: test-minio-root-password + S3_BUCKET: messenger-attachments + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 2s + timeout: 2s + retries: 30 + + s3-bootstrap: + image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805 + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" + mc mb --ignore-existing local/"$$S3_BUCKET" + mc anonymous set none local/"$$S3_BUCKET" + mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" + sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json + mc admin policy create local messenger-api /tmp/s3-api-policy.json + mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID" + environment: + MINIO_ROOT_USER: test-minio-admin + MINIO_ROOT_PASSWORD: test-minio-root-password + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + volumes: + - ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro + depends_on: + s3: + condition: service_healthy + restart: "no" + + migrate: + build: . + command: ["alembic", "upgrade", "head"] + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + ports: + - "127.0.0.1:18080:8080" + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + JOB_MAX_ATTEMPTS: 5 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"] + interval: 2s + timeout: 2s + retries: 30 + + publisher: + build: . + command: ["python", "-m", "app.jobs.publisher"] + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 10 + + worker: + build: . + command: ["python", "-m", "app.jobs.worker"] + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + WORKER_CONCURRENCY: 2 + WORKER_PREFETCH: 2 + JOB_MAX_ATTEMPTS: 5 + JOB_TIMEOUT_SECONDS: 60 + JOB_LEASE_SECONDS: 90 + RETRY_BASE_SECONDS: 1 + RETRY_MAX_SECONDS: 4 + STAGING_RETENTION_SECONDS: 86400 + THUMBNAIL_JPEG_QUALITY: 85 + CROP_JPEG_QUALITY: 85 + OPTIMIZED_WEBP_QUALITY: 82 + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 5s + timeout: 3s + retries: 10 + + tests: + build: + context: . + dockerfile: Dockerfile.test + environment: + BASE_URL: http://api:8080 + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + BROKER_URL: amqp://messenger_test:messenger_test@broker:5672/ + depends_on: + api: + condition: service_healthy + restart: "no" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..9b9965b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,346 @@ +x-otel-environment: &otel-environment + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} + OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} + OTEL_EXPORTER_OTLP_TIMEOUT: ${OTEL_EXPORTER_OTLP_TIMEOUT:-2} + OTEL_SERVICE_VERSION: ${OTEL_SERVICE_VERSION:-1.6.0} + OTEL_DEPLOYMENT_ENVIRONMENT: ${OTEL_DEPLOYMENT_ENVIRONMENT:-lab} + OTEL_BSP_MAX_QUEUE_SIZE: ${OTEL_BSP_MAX_QUEUE_SIZE:-2048} + OTEL_BLRP_MAX_QUEUE_SIZE: ${OTEL_BLRP_MAX_QUEUE_SIZE:-2048} + OTEL_METRIC_EXPORT_INTERVAL: ${OTEL_METRIC_EXPORT_INTERVAL:-5000} + +services: + postgres: + image: postgres:17.4-alpine + cpus: "1.0" + mem_limit: 512m + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + + redis: + image: redis:7.4.2-alpine + cpus: "0.25" + mem_limit: 128m + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + + broker: + image: rabbitmq:4.1.4-alpine + cpus: "0.5" + mem_limit: 384m + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-messenger} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:-change-me-rabbitmq} + volumes: + - broker_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + + s3: + image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597 + user: "0" + cpus: "0.5" + mem_limit: 256m + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - s3_data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 3s + retries: 20 + + s3-bootstrap: + image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805 + cpus: "0.25" + mem_limit: 128m + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" + mc mb --ignore-existing local/"$$S3_BUCKET" + mc anonymous set none local/"$$S3_BUCKET" + mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" + sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json + mc admin policy create local messenger-api /tmp/s3-api-policy.json + mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + volumes: + - ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro + depends_on: + s3: + condition: service_healthy + restart: "no" + + migrate: + build: . + cpus: "0.5" + mem_limit: 256m + command: ["alembic", "upgrade", "head"] + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + BROKER_URL: ${BROKER_URL} + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + cpus: "0.5" + mem_limit: 256m + environment: + <<: *otel-environment + OTEL_SERVICE_NAME: messenger-api + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + LOG_LEVEL: ${LOG_LEVEL:-info} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: ${BROKER_URL} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + expose: + - "8080" + deploy: + replicas: 2 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"] + interval: 2s + timeout: 2s + retries: 5 + stop_grace_period: 10s + + publisher: + build: . + command: ["python", "-m", "app.jobs.publisher"] + cpus: "0.25" + mem_limit: 192m + environment: + <<: *otel-environment + OTEL_SERVICE_NAME: messenger-publisher + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + BROKER_URL: ${BROKER_URL} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 10s + timeout: 3s + retries: 3 + restart: unless-stopped + stop_grace_period: 10s + + worker: + build: . + command: ["python", "-m", "app.jobs.worker"] + cpus: "1.0" + mem_limit: 768m + environment: + <<: *otel-environment + OTEL_SERVICE_NAME: messenger-worker + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: ${BROKER_URL} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + RETRY_BASE_SECONDS: ${RETRY_BASE_SECONDS:-2} + RETRY_MAX_SECONDS: ${RETRY_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + THUMBNAIL_JPEG_QUALITY: ${THUMBNAIL_JPEG_QUALITY:-85} + CROP_JPEG_QUALITY: ${CROP_JPEG_QUALITY:-85} + OPTIMIZED_WEBP_QUALITY: ${OPTIMIZED_WEBP_QUALITY:-82} + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import os; os.kill(1, 0)"] + interval: 10s + timeout: 3s + retries: 3 + restart: unless-stopped + stop_grace_period: 70s + + lb: + image: haproxy:3.2.26-alpine + cpus: "0.25" + mem_limit: 128m + ports: + - "${APP_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"] + interval: 2s + timeout: 2s + retries: 5 + stop_grace_period: 5s + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.123.0 + cpus: "0.5" + mem_limit: 256m + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + loki: + condition: service_started + tempo: + condition: service_started + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.2.1 + cpus: "0.5" + mem_limit: 384m + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --storage.tsdb.retention.size=1GB + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + depends_on: + otel-collector: + condition: service_started + restart: unless-stopped + + loki: + image: grafana/loki:3.4.2 + user: "0" + cpus: "0.5" + mem_limit: 384m + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki_data:/loki + restart: unless-stopped + + tempo: + image: grafana/tempo:2.7.2 + user: "0" + cpus: "0.5" + mem_limit: 384m + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo_data:/var/tempo + restart: unless-stopped + + grafana: + image: grafana/grafana:11.6.0 + cpus: "0.5" + mem_limit: 384m + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me-grafana} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_PLUGINS_PREINSTALL_DISABLED: "true" + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./infra/grafana/runbooks:/var/lib/grafana/runbooks:ro + - grafana_data:/var/lib/grafana + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + +volumes: + postgres_data: + redis_data: + s3_data: + broker_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/evidence/2026-11-17-results.md b/evidence/2026-11-17-results.md new file mode 100644 index 0000000..5509f5e --- /dev/null +++ b/evidence/2026-11-17-results.md @@ -0,0 +1,37 @@ +# Результаты повторной проверки 2026-11-20 + +Стенд: Apple M4 Pro, 24 GiB RAM; Docker Desktop VM 12 CPU / 9.70 GiB; +лимиты сервисов из `compose.yaml`, только synthetic data. + +```text +make test +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. + +make test-observability +PASS: dashboard, alerts, metrics, correlated logs and cross-process trace + +make test-retry-trace +{"scenario":"retry_trace","attachment_id":"7ee0da1b-0c50-4d2e-9bf8-e2c68efb2ce4", + "trace_id":"e8e2aff13b7bdb632c9cec5007dcc6d4","producer_spans":2, + "consumer_spans":2,"result":"PASS"} + +make test-alert-worker +{"scenario":"worker_outage","firing_state":"Alerting","detect_seconds":61.8, + "resolved_state":"Normal","recover_seconds":15.5,"result":"PASS"} + +make test-alert-database +{"scenario":"database_outage","firing_state":"Alerting","detect_seconds":72.2, + "resolved_state":"Normal","recover_seconds":113.1, + "trace_id":"34f96d6fc934f7c54075c44a7b39c176","db_span":"connect", + "db_system":"postgresql","result":"PASS"} + +make test-telemetry-outage +{"scenario":"telemetry_backend_outage","business_smoke":"PASS", + "collector":"running","duration_seconds":49.5,"rss_before_mib":194.6, + "rss_peak_mib":198.0,"memory_limit_mib":256,"failed_log_records":34, + "failed_spans":30,"result":"PASS"} +``` + +Worker incident использовал attachment `a88a0cfd-bbda-4151-8584-de27154a8dcb`. +Это идентификатор только локальных синтетических данных; credentials/tokens/body в +доказательство не сохранялись. diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..d5ce816 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,21 @@ +{ + "lab": 6, + "implementation_commit": "9e6428d", + "host": "Apple M4 Pro, 24 GiB RAM", + "docker_vm": "12 CPU, 9.70 GiB RAM; per-service limits from compose.yaml", + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 120.0, + "elapsed_seconds": 120.084, + "concurrency": 8, + "successful_requests": 25638, + "generated_messages": 25638, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 213.501, + "successful_latency_ms": { + "p50": 17.158, + "p95": 99.774, + "p99": 171.24 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/infra/grafana/dashboards/messenger-observability.json b/infra/grafana/dashboards/messenger-observability.json new file mode 100644 index 0000000..eab9ded --- /dev/null +++ b/infra/grafana/dashboards/messenger-observability.json @@ -0,0 +1,183 @@ +{ + "annotations": {"list": []}, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Business requests per second; health routes are excluded.", + "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 0}, + "id": 1, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum by (http_route, http_request_method) (rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[$__rate_interval]))", "legendFormat": "{{http_request_method}} {{http_route}}", "refId": "A"}], + "title": "API RPS", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "5xx business requests divided by all business requests over 2 minutes.", + "fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 0}, + "id": 2, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "single"}}, + "targets": [{"expr": "(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\",http_response_status_class=\"5xx\"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[2m])), 0.001)) or vector(0)", "legendFormat": "5xx ratio", "refId": "A"}], + "title": "API server error ratio", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Percentiles are computed from aggregate histogram buckets across API replicas.", + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 0}, + "id": 3, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=\"messenger-api\",http_route!~\"/health/.*\"}[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "API duration p50 / p95 / p99", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}, + "id": 4, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "sum by (outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))", "legendFormat": "{{outcome}}", "refId": "A"}], + "title": "Jobs by outcome", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}, + "id": 5, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_processing_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "Worker duration p50 / p95 / p99", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Time from durable acceptance to ready commit.", + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8}, + "id": 6, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "histogram_quantile(0.50, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A"}, + {"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B"}, + {"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C"} + ], + "title": "End-to-end time to ready", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Logical non-terminal jobs from PostgreSQL and age of the oldest one.", + "fieldConfig": {"defaults": {}, "overrides": [{"matcher": {"id": "byName", "options": "oldest age"}, "properties": [{"id": "unit", "value": "s"}]}]}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 16}, + "id": 7, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "max(messenger_queue_depth{messaging_destination_name=\"attachments.process\"})", "legendFormat": "depth", "refId": "A"}, + {"expr": "max(messenger_queue_oldest_age_seconds{messaging_destination_name=\"attachments.process\"})", "legendFormat": "oldest age", "refId": "B"} + ], + "title": "Queue depth and oldest job", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 16}, + "id": 8, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [ + {"expr": "messenger_db_pool_used", "legendFormat": "used {{service_name}} {{service_instance_id}}", "refId": "A"}, + {"expr": "messenger_db_pool_waiting", "legendFormat": "waiting {{service_name}} {{service_instance_id}}", "refId": "B"} + ], + "title": "PostgreSQL pool used / waiting", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"max": 1, "min": 0, "unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 16}, + "id": 9, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "max(messenger_worker_busy) / clamp_min(max(messenger_worker_capacity), 1)", "legendFormat": "saturation", "refId": "A"}], + "title": "Worker saturation", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "percentunit"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 24}, + "id": 10, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "rate(process_cpu_time_seconds_total[5m])", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}], + "title": "Process CPU", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 24}, + "id": 11, + "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}, + "targets": [{"expr": "process_memory_rss_bytes", "legendFormat": "{{service_name}} {{service_instance_id}}", "refId": "A"}], + "title": "Process RSS", + "type": "timeseries" + }, + { + "datasource": {"type": "loki", "uid": "loki"}, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 24}, + "id": 12, + "options": {"dedupStrategy": "none", "enableLogDetails": true, "prettifyLogMessage": false, "showCommonLabels": false, "showLabels": false, "showTime": true, "sortOrder": "Descending", "wrapLogMessage": true}, + "targets": [{"expr": "{service_name=~\"$service\"} | json | http_route!~\"/health/.*\" | line_format \"{{.severity}} {{.service_name}} {{.event}}{{if .http_method}} {{.http_method}} {{.http_route}} status={{.http_status_code}} duration={{.duration_seconds}}{{end}} trace_id={{.trace_id}} {{.message}}\"", "refId": "A"}], + "title": "Correlated application logs", + "type": "logs" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["mtusi", "lab6", "opentelemetry"], + "templating": { + "list": [ + { + "current": {"selected": false, "text": ".*", "value": ".*"}, + "datasource": {"type": "loki", "uid": "loki"}, + "definition": "label_values(service_name)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(service_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": {"from": "now-30m", "to": "now"}, + "timepicker": {}, + "timezone": "browser", + "title": "Messenger Observability", + "uid": "messenger-observability", + "version": 1, + "weekStart": "" +} diff --git a/infra/grafana/provisioning/alerting/rules.yaml b/infra/grafana/provisioning/alerting/rules.yaml new file mode 100644 index 0000000..05a011f --- /dev/null +++ b/infra/grafana/provisioning/alerting/rules.yaml @@ -0,0 +1,120 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: Messenger SLO alerts + folder: MTUSI Lab 6 + interval: 10s + rules: + - uid: messenger-api-high-error-rate + title: ApiHighServerErrorRate + condition: C + data: + - refId: A + relativeTimeRange: + from: 120 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: '(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*",http_response_status_class="5xx"}[2m])) / clamp_min(sum(rate(http_server_request_count_total{service_name="messenger-api",http_route!~"/health/.*"}[2m])), 0.001)) or vector(0)' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + range: false + refId: A + - refId: C + relativeTimeRange: + from: 0 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [0.05] + type: gt + operator: + type: and + query: + params: [C] + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + refId: C + type: threshold + dashboardUid: messenger-observability + panelId: 2 + noDataState: OK + execErrState: Error + for: 1m + annotations: + summary: Более 5% бизнес-запросов завершаются 5xx. + runbook_url: /var/lib/grafana/runbooks/api-high-error-rate.md + labels: + service: messenger-api + severity: warning + isPaused: false + - uid: messenger-queue-oldest-growing + title: QueueOldestJobGrowing + condition: C + data: + - refId: A + relativeTimeRange: + from: 60 + to: 0 + datasourceUid: prometheus + model: + datasource: + type: prometheus + uid: prometheus + editorMode: code + expr: '(max(messenger_queue_oldest_age_seconds{messaging_destination_name="attachments.process"}) > 20 and max(messenger_queue_depth{messaging_destination_name="attachments.process"}) > 0) or vector(0)' + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + range: false + refId: A + - refId: C + relativeTimeRange: + from: 0 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: [0.5] + type: gt + operator: + type: and + query: + params: [C] + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + refId: C + type: threshold + dashboardUid: messenger-observability + panelId: 7 + noDataState: OK + execErrState: Error + for: 15s + annotations: + summary: Возраст старейшей незавершённой job превысил 20 секунд. + runbook_url: /var/lib/grafana/runbooks/queue-oldest-job.md + labels: + service: messenger-worker + severity: warning + isPaused: false diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..e609107 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: Messenger + orgId: 1 + folder: MTUSI Lab 6 + type: file + disableDeletion: true + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..7683ea5 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,47 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + - name: Tempo + orgId: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: 'trace[_]?id["=: ]+([0-9a-f]{32})' + datasourceUid: tempo + url: '$${__value.raw}' + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -1m + spanEndTimeShift: 1m + filterByTraceID: true + filterBySpanID: false + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true diff --git a/infra/grafana/provisioning/plugins/.gitkeep b/infra/grafana/provisioning/plugins/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/infra/grafana/provisioning/plugins/.gitkeep @@ -0,0 +1 @@ + diff --git a/infra/grafana/runbooks/api-high-error-rate.md b/infra/grafana/runbooks/api-high-error-rate.md new file mode 100644 index 0000000..3f04fc0 --- /dev/null +++ b/infra/grafana/runbooks/api-high-error-rate.md @@ -0,0 +1,13 @@ +# ApiHighServerErrorRate + +Alert использует только 5xx бизнес-запросов и исключает `/health/*`. Клиентские 4xx +не считаются отказом сервера. + +1. Открыть панели API error ratio и DB pool. +2. В Tempo найти медленные/error server spans за окно alert. +3. Сравнить длительность server span с дочерними PostgreSQL/S3 spans. +4. В Loki отфильтровать `{service_name="messenger-api"} | trace_id="..."`. +5. Проверить readiness зависимостей, не удаляя volumes и не перезапуская весь стенд. + +После восстановления убедиться, что новые запросы успешны, error ratio падает и alert +переходит в Normal. Короткий единичный 5xx может не сработать из-за окна и `for: 1m`. diff --git a/infra/grafana/runbooks/queue-oldest-job.md b/infra/grafana/runbooks/queue-oldest-job.md new file mode 100644 index 0000000..4767905 --- /dev/null +++ b/infra/grafana/runbooks/queue-oldest-job.md @@ -0,0 +1,13 @@ +# QueueOldestJobGrowing + +Alert означает, что в PostgreSQL остаётся хотя бы одна `queued`/`processing` job старше +20 секунд. Это возраст backlog, а не duration одного consumer span. + +1. Проверить queue depth, oldest age и worker busy/capacity. +2. В Loki открыть `{service_name="messenger-worker"}` и проверить retry/failure events. +3. По trace ID перейти в Tempo и проверить consumer, S3 и PostgreSQL spans. +4. Восстановить worker или отказавшую зависимость, не создавая новую attachment job. +5. Убедиться, что исходная job стала `ready`, backlog уменьшился и alert resolved. + +Порог учебный и рассчитан на локальную демонстрацию; для production его связывают с +реальным SLO времени до ready. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..5bd0c14 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,31 @@ +global + log stdout format raw local0 + +defaults + log global + mode http + option httplog + timeout connect 2s + timeout client 30s + timeout server 30s + timeout check 2s + +resolvers docker + nameserver docker_dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend api_backend + +backend api_backend + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none + server-template api 2 api:8080 diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..d9898e5 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,41 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + +limits_config: + allow_structured_metadata: true + retention_period: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..ae749ea --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,76 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + timeout: 1s + send_batch_size: 512 + send_batch_max_size: 1024 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 500 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 500 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 30s + +service: + extensions: [health_check] + telemetry: + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..3c1286c --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: otel-application-metrics + static_configs: + - targets: [otel-collector:8889] + - job_name: otel-collector + static_configs: + - targets: [otel-collector:8888] diff --git a/infra/s3-api-policy.json b/infra/s3-api-policy.json new file mode 100644 index 0000000..a2fccff --- /dev/null +++ b/infra/s3-api-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::BUCKET_NAME"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::BUCKET_NAME/*"] + } + ] +} diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..bd94d08 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,29 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 24h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +usage_report: + reporting_enabled: false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1b6d764 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +SQLAlchemy[asyncio]==2.0.40 +asyncpg==0.30.0 +alembic==1.15.2 +redis==5.2.1 +argon2-cffi==25.1.0 +minio==7.2.20 +Pillow==13.0.0 +aio-pika==9.5.5 +opentelemetry-api==1.31.1 +opentelemetry-sdk==1.31.1 +opentelemetry-exporter-otlp-proto-http==1.31.1 +opentelemetry-instrumentation-fastapi==0.52b1 +opentelemetry-instrumentation-sqlalchemy==0.52b1 +psutil==7.0.0 diff --git a/tests/README.md b/tests/README.md index 0dfa3f8..3863395 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,6 +1,11 @@ # Публичные проверки -Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv. +Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; публичные +проверки не создают готовое приложение и сами не управляют Docker. Отказные проверки +lab6 ниже управляют только явно указанными Compose-сервисами. `make check` можно +выполнить на исходном skeleton, `make test` — только после реализации API. Полная +OpenAPI-валидация отдельно: `make validate` после установки +`requirements-dev.txt` в venv. ```sh make check @@ -13,6 +18,28 @@ python3 tests/smoke.py --base-url http://localhost:8080 Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются. +## Наблюдаемость и контролируемые отказы, lab6 + +При запущенном Compose-стенде доступны дополнительные проверки реализации: + +```sh +make test-observability # dashboard, alerts, Prometheus, Loki и Tempo +make test-retry-trace # временная S3-ошибка и retry в одной трассе +make test-alert-worker # временно останавливает и восстанавливает worker +make test-alert-database # временно останавливает и восстанавливает PostgreSQL +make test-telemetry-outage # временно останавливает и восстанавливает Loki/Tempo +``` + +Отказные сценарии изменяют состояние только перечисленных контейнеров и всегда +выполняют восстановление в `finally`. Используются уникальные синтетические данные; +секреты и tokens не печатаются. Retry-тест временно отключает S3 и проверяет +две producer/consumer попытки в одной трассе. Worker-тест ждёт `QueueOldestJobGrowing` в +`Alerting`, затем `Normal`; DB-тест аналогично проверяет `ApiHighServerErrorRate`. +Тест telemetry outage держит backends выключенными более 30 с и фиксирует RSS +Collector и `send_failed` counters. +Окна alerts намеренно делают эти два теста не мгновенными. Запускайте их +последовательно, а не параллельно. + ## Проверка сохранности, lab2+ Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные. diff --git a/tests/alert_scenarios.py b/tests/alert_scenarios.py new file mode 100644 index 0000000..fb6adc7 --- /dev/null +++ b/tests/alert_scenarios.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Воспроизводит ограниченные отказы worker и PostgreSQL для проверки alerts.""" +import argparse +import base64 +import json +import os +import subprocess +import time +import urllib.parse + +from client import Client + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def compose(*args: str, capture: bool = False): + """Запускает Docker Compose из корня проекта и при необходимости захватывает вывод.""" + + return subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + text=True, + capture_output=capture, + ) + + +def grafana_rules() -> dict: + """Получает текущие состояния provisioned alert rules через API Grafana.""" + + user = os.getenv("GRAFANA_ADMIN_USER", "admin") + password = os.getenv("GRAFANA_ADMIN_PASSWORD", "change-me-grafana") + token = base64.b64encode(f"{user}:{password}".encode()).decode() + result = compose( + "exec", "-T", "grafana", "wget", "-qO-", + "--header", f"Authorization: Basic {token}", + "http://127.0.0.1:3000/api/prometheus/grafana/api/v1/rules", + capture=True, + ) + return json.loads(result.stdout) + + +def internal_json(url: str) -> dict: + """Читает JSON из внутреннего observability-сервиса через Grafana-контейнер.""" + + result = compose( + "exec", "-T", "grafana", "wget", "-qO-", url, capture=True + ) + return json.loads(result.stdout) + + +def database_failure_evidence(start_ns: int) -> dict[str, str]: + """Находит связанный лог и ошибочный PostgreSQL CLIENT span.""" + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + query = urllib.parse.urlencode({ + "query": ( + '{service_name="messenger-api"} | json ' + '| logger="app.error" | message=~".*(database|socket).*"' + ), + "start": str(start_ns), + "end": str(time.time_ns()), + "direction": "backward", + "limit": "20", + }) + loki = internal_json( + "http://loki:3100/loki/api/v1/query_range?" + query + ) + streams = loki.get("data", {}).get("result", []) + if not streams: + time.sleep(2) + continue + entry = streams[0]["values"][0] + metadata = entry[2] if len(entry) > 2 else {} + body = json.loads(entry[1]) + trace_id = metadata.get("trace_id") or body.get("trace_id") + if not trace_id: + time.sleep(2) + continue + trace = internal_json(f"http://tempo:3200/api/traces/{trace_id}") + for batch in trace.get("batches", []): + scopes = batch.get( + "scopeSpans", batch.get("instrumentationLibrarySpans", []) + ) + for scope in scopes: + for span in scope.get("spans", []): + attributes = { + item["key"]: next(iter(item.get("value", {}).values()), "") + for item in span.get("attributes", []) + } + if ( + span.get("kind") == "SPAN_KIND_CLIENT" + and attributes.get("db.system") == "postgresql" + and span.get("status", {}).get("code") == "STATUS_CODE_ERROR" + ): + return { + "trace_id": trace_id, + "log_message": body.get("message", ""), + "db_span": span.get("name", ""), + "db_system": "postgresql", + } + time.sleep(2) + raise RuntimeError("linked PostgreSQL error log/trace was not found") + + +def rule_state(title: str) -> str: + """Находит alert по заголовку и возвращает его текущее состояние.""" + + groups = grafana_rules().get("data", {}).get("groups", []) + for group in groups: + for rule in group.get("rules", []): + if rule.get("name") == title: + alerts = rule.get("alerts", []) + return alerts[0].get("state", rule.get("state", "unknown")) if alerts else rule.get("state", "unknown") + raise RuntimeError(f"Grafana rule not found: {title}") + + +def wait_state(title: str, expected: set[str], timeout: int) -> tuple[str, float]: + """Ожидает одно из требуемых состояний alert и возвращает время ожидания.""" + + started = time.monotonic() + deadline = started + timeout + last = "unknown" + while time.monotonic() < deadline: + last = rule_state(title) + if last.lower() in expected: + return last, time.monotonic() - started + time.sleep(5) + raise RuntimeError(f"{title}: expected {sorted(expected)}, last state {last}") + + +def worker_scenario(base_url: str): + """Останавливает worker и проверяет переход queue alert в Alerting и Normal.""" + + client = Client(base_url, timeout=10) + alice, bob, _, chat = client.bootstrap() + compose("stop", "worker") + try: + attachment, _ = client.attachment(alice, chat) + firing, detect_seconds = wait_state( + "QueueOldestJobGrowing", {"alerting", "firing"}, 100 + ) + finally: + compose("start", "worker") + ready = client.wait_ready(bob, attachment, wait_seconds=90) + normal, recover_seconds = wait_state( + "QueueOldestJobGrowing", {"normal", "inactive"}, 90 + ) + print(json.dumps({ + "scenario": "worker_outage", + "attachment_id": ready["id"], + "firing_state": firing, + "detect_seconds": round(detect_seconds, 1), + "resolved_state": normal, + "recover_seconds": round(recover_seconds, 1), + "result": "PASS", + }, ensure_ascii=False)) + + +def database_scenario(base_url: str): + """Останавливает PostgreSQL и проверяет firing/resolved для alert по 5xx.""" + + Client(base_url).call("GET", "/health/ready", schema="Health") + incident_started_ns = time.time_ns() + compose("stop", "postgres") + load = subprocess.Popen( + [ + "docker", "compose", "-f", "compose.yaml", "-f", + "compose.observability-tests.yaml", "run", "--rm", + "observability-test", "python", "database_error_load.py", + "--base-url", "http://api:8080", "--duration", "150", + ], + cwd=ROOT, + ) + evidence = None + try: + firing, detect_seconds = wait_state( + "ApiHighServerErrorRate", {"alerting", "firing"}, 130 + ) + evidence = database_failure_evidence(incident_started_ns) + finally: + load.terminate() + try: + load.wait(timeout=10) + except subprocess.TimeoutExpired: + load.kill() + compose("start", "postgres") + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + Client(base_url).call("GET", "/health/ready", schema="Health") + break + except Exception: + time.sleep(3) + else: + raise RuntimeError("API readiness did not recover after PostgreSQL restart") + normal, recover_seconds = wait_state( + "ApiHighServerErrorRate", {"normal", "inactive"}, 180 + ) + print(json.dumps({ + "scenario": "database_outage", + "firing_state": firing, + "detect_seconds": round(detect_seconds, 1), + "resolved_state": normal, + "recover_seconds": round(recover_seconds, 1), + "trace_id": evidence["trace_id"], + "log_message": evidence["log_message"], + "db_span": evidence["db_span"], + "db_system": evidence["db_system"], + "result": "PASS", + }, ensure_ascii=False)) + + +def main(): + """Разбирает аргументы и запускает выбранный отказный сценарий.""" + + parser = argparse.ArgumentParser() + parser.add_argument("scenario", choices=("worker", "database")) + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + if args.scenario == "worker": + worker_scenario(args.base_url) + else: + database_scenario(args.base_url) + + +if __name__ == "__main__": + main() diff --git a/tests/database_error_load.py b/tests/database_error_load.py new file mode 100644 index 0000000..5f8271f --- /dev/null +++ b/tests/database_error_load.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Создаёт ограниченный поток DB-зависимых запросов для проверки alert по 5xx.""" +import argparse +import json +import time +import urllib.error +import urllib.request +import uuid + + +def main(): + """Отправляет synthetic POST-запросы заданное время и считает ошибки.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://api:8080") + parser.add_argument("--duration", type=int, default=120) + args = parser.parse_args() + deadline = time.monotonic() + args.duration + attempts = errors = 0 + while time.monotonic() < deadline: + suffix = uuid.uuid4().hex[:12] + body = json.dumps({ + "username": "incident_" + suffix, + "display_name": "Synthetic incident", + "password": "Synthetic-test-only-" + suffix, + }).encode() + request = urllib.request.Request( + args.base_url.rstrip("/") + "/api/v1/users", + data=body, + method="POST", + headers={"Content-Type": "application/json"}, + ) + attempts += 1 + try: + with urllib.request.urlopen(request, timeout=3) as response: + response.read() + errors += response.status >= 500 + except urllib.error.HTTPError as error: + error.read() + errors += error.code >= 500 + except (urllib.error.URLError, TimeoutError, OSError): + errors += 1 + time.sleep(0.25) + print(f"database incident load: attempts={attempts} errors={errors}") + + +if __name__ == "__main__": + main() diff --git a/tests/observability_smoke.py b/tests/observability_smoke.py new file mode 100644 index 0000000..24b5fa3 --- /dev/null +++ b/tests/observability_smoke.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Проверяет сквозной контур наблюдаемости ЛР6 через HTTP API Grafana.""" +import argparse +import base64 +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request + + +class Grafana: + """Минимальный HTTP-клиент Grafana с Basic-аутентификацией.""" + + def __init__(self, base_url: str, user: str, password: str): + """Сохраняет адрес Grafana и формирует заголовок авторизации.""" + + self.base_url = base_url.rstrip("/") + token = base64.b64encode(f"{user}:{password}".encode()).decode() + self.headers = {"Authorization": f"Basic {token}"} + + def get(self, path: str, params: dict[str, str] | None = None): + """Выполняет GET-запрос к Grafana и возвращает декодированный JSON.""" + + if params: + path += "?" + urllib.parse.urlencode(params) + request = urllib.request.Request(self.base_url + path, headers=self.headers) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return json.load(response) + except urllib.error.HTTPError as error: + body = error.read().decode(errors="replace") + raise RuntimeError(f"Grafana GET {path}: HTTP {error.code}: {body[:300]}") from error + + +def check(condition: bool, message: str): + """Завершает проверку понятной ошибкой, если условие не выполнено.""" + + if not condition: + raise RuntimeError(message) + + +def trace_services(payload) -> set[str]: + """Извлекает множество значений service.name из ответа Tempo с трассой.""" + + result: set[str] = set() + for batch in payload.get("batches", []): + resource = batch.get("resource", {}) + attrs = resource.get("attributes", []) + for attr in attrs: + if attr.get("key") == "service.name": + result.add(attr.get("value", {}).get("stringValue", "")) + return result + + +def main(): + """Проверяет dashboard, alerts, метрики, логи и cross-process trace.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--grafana-url", default="http://localhost:3000") + parser.add_argument("--user", default=os.getenv("GRAFANA_ADMIN_USER", "admin")) + parser.add_argument("--password", default=os.getenv("GRAFANA_ADMIN_PASSWORD"), required=os.getenv("GRAFANA_ADMIN_PASSWORD") is None) + args = parser.parse_args() + grafana = Grafana(args.grafana_url, args.user, args.password) + + health = grafana.get("/api/health") + check(health.get("database") == "ok", "Grafana database is not ready") + dashboard = grafana.get("/api/dashboards/uid/messenger-observability") + check(len(dashboard["dashboard"].get("panels", [])) >= 12, "dashboard panels are missing") + rules = grafana.get("/api/v1/provisioning/alert-rules") + titles = {rule.get("title") for rule in rules} + check({"ApiHighServerErrorRate", "QueueOldestJobGrowing"} <= titles, "alert rules are missing") + + prom = grafana.get( + "/api/datasources/proxy/uid/prometheus/api/v1/query", + {"query": 'sum(http_server_request_count_total{service_name="messenger-api"})'}, + ) + check(prom.get("status") == "success" and prom["data"]["result"], "API metric is absent") + + now_ns = time.time_ns() + loki = grafana.get( + "/api/datasources/proxy/uid/loki/loki/api/v1/query_range", + { + "query": ( + '{service_name="messenger-api"} | json ' + '| event="attachment.accepted" | line_format "{{.trace_id}}"' + ), + "start": str(now_ns - 3_600_000_000_000), + "end": str(now_ns), + "direction": "backward", + "limit": "20", + }, + ) + streams = loki.get("data", {}).get("result", []) + check(streams, "correlated attachment.accepted log is absent; run make test first") + entry = streams[0]["values"][0] + metadata = entry[2] if len(entry) > 2 else {} + trace_id = metadata.get("trace_id") + if not trace_id and len(entry[1]) == 32: + trace_id = entry[1] + if not trace_id: + try: + trace_id = json.loads(entry[1]).get("trace_id") + except json.JSONDecodeError: + trace_id = None + check(bool(trace_id), f"Loki log has no trace_id; metadata keys: {sorted(metadata)}") + + trace = grafana.get(f"/api/datasources/proxy/uid/tempo/api/traces/{trace_id}") + services = trace_services(trace) + expected = {"messenger-api", "messenger-publisher", "messenger-worker"} + check(expected <= services, f"cross-process trace is incomplete: {sorted(services)}") + print("PASS: dashboard, alerts, metrics, correlated logs and cross-process trace") + + +if __name__ == "__main__": + main() diff --git a/tests/retry_trace.py b/tests/retry_trace.py new file mode 100644 index 0000000..de052b4 --- /dev/null +++ b/tests/retry_trace.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Создаёт временную S3-ошибку и доказывает retry в одной Tempo-трассе.""" +import argparse +from datetime import datetime, timezone +import json +import os +import subprocess +import time +import urllib.parse + +from client import Client + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def compose(*args: str, capture: bool = False): + """Запускает Docker Compose и при необходимости возвращает его вывод.""" + + return subprocess.run( + ["docker", "compose", *args], cwd=ROOT, check=True, text=True, + capture_output=capture, + ) + + +def internal_json(url: str) -> dict: + """Читает Loki/Tempo JSON через внутреннюю сеть Compose.""" + + result = compose( + "exec", "-T", "grafana", "wget", "-qO-", url, capture=True + ) + return json.loads(result.stdout) + + +def wait_retry_log(since: str, timeout: int = 45) -> None: + """Ждёт запись worker о планировании повторной попытки.""" + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = compose("logs", "--since", since, "worker", capture=True) + if "job.retry_scheduled" in result.stdout + result.stderr: + return + time.sleep(1) + raise RuntimeError("worker did not schedule retry") + + +def trace_id_for_attachment(attachment_id: str) -> str: + """Находит trace_id лога приёма конкретного синтетического вложения.""" + + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + now_ns = time.time_ns() + query = urllib.parse.urlencode( + { + "query": ( + '{service_name="messenger-api"} | json ' + f'| event="attachment.accepted" | attachment_id="{attachment_id}"' + ), + "start": str(now_ns - 600_000_000_000), + "end": str(now_ns), + "direction": "backward", + "limit": "5", + } + ) + payload = internal_json( + "http://loki:3100/loki/api/v1/query_range?" + query + ) + streams = payload.get("data", {}).get("result", []) + if streams: + entry = streams[0]["values"][0] + metadata = entry[2] if len(entry) > 2 else {} + trace_id = metadata.get("trace_id") + if not trace_id: + trace_id = json.loads(entry[1]).get("trace_id") + if trace_id: + return trace_id + time.sleep(2) + raise RuntimeError("attachment trace_id is absent in Loki") + + +def trace_span_names(trace_id: str) -> list[str]: + """Ждёт полную трассу и возвращает имена всех её spans.""" + + deadline = time.monotonic() + 60 + last: list[str] = [] + while time.monotonic() < deadline: + payload = internal_json(f"http://tempo:3200/api/traces/{trace_id}") + names: list[str] = [] + for batch in payload.get("batches", []): + scopes = batch.get( + "scopeSpans", batch.get("instrumentationLibrarySpans", []) + ) + for scope in scopes: + names.extend(span.get("name", "") for span in scope.get("spans", [])) + last = names + if names.count("attachments publish") >= 2 and names.count("attachments process") >= 2: + return names + time.sleep(2) + raise RuntimeError(f"retry spans are incomplete: {last}") + + +def main(): + """Внедряет временный отказ S3, восстанавливает его и проверяет retry trace.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + client = Client(args.base_url, timeout=10) + alice, bob, _, chat = client.bootstrap() + compose("stop", "-t", "1", "worker") + attachment = None + retry_seen = False + try: + attachment, _ = client.attachment(alice, chat) + since = datetime.now(timezone.utc).isoformat() + compose("stop", "s3") + compose("up", "-d", "--no-deps", "worker") + wait_retry_log(since) + retry_seen = True + finally: + compose("start", "s3") + compose("up", "-d", "--no-deps", "worker") + if attachment is None or not retry_seen: + raise RuntimeError("retry scenario was not created") + time.sleep(5) + ready = client.wait_ready(bob, attachment, wait_seconds=120) + trace_id = trace_id_for_attachment(ready["id"]) + names = trace_span_names(trace_id) + if not any(name.startswith("s3 ") for name in names): + raise RuntimeError("S3 spans are absent from retry trace") + if not any(name.startswith(("SELECT ", "INSERT ", "UPDATE ", "connect")) for name in names): + raise RuntimeError("SQL spans are absent from retry trace") + print(json.dumps({ + "scenario": "retry_trace", + "attachment_id": ready["id"], + "trace_id": trace_id, + "producer_spans": names.count("attachments publish"), + "consumer_spans": names.count("attachments process"), + "result": "PASS", + })) + + +if __name__ == "__main__": + main() diff --git a/tests/telemetry_outage.py b/tests/telemetry_outage.py new file mode 100644 index 0000000..63a2944 --- /dev/null +++ b/tests/telemetry_outage.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Проверяет работу бизнес-контура при временной недоступности Loki и Tempo.""" +import argparse +import json +import os +import subprocess +import time +import urllib.parse + +from client import Client + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def compose(*args: str, capture: bool = False): + """Запускает Docker Compose из корня проекта и при необходимости захватывает вывод.""" + + return subprocess.run( + ["docker", "compose", *args], cwd=ROOT, check=True, text=True, + capture_output=capture, + ) + + +def prometheus_value(query: str) -> float: + """Читает одно числовое значение Prometheus через внутреннюю сеть Compose.""" + + url = "http://prometheus:9090/api/v1/query?" + urllib.parse.urlencode( + {"query": query} + ) + result = compose( + "exec", "-T", "grafana", "wget", "-qO-", url, capture=True + ) + payload = json.loads(result.stdout) + series = payload.get("data", {}).get("result", []) + if not series: + return 0.0 + return float(series[0]["value"][1]) + + +def main(): + """Держит backends недоступными >30 с и проверяет бизнес-контур и RSS.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + rss_query = 'max(otelcol_process_memory_rss_bytes{job="otel-collector"})' + failed_logs_query = "sum(otelcol_exporter_send_failed_log_records_total) or vector(0)" + failed_spans_query = "sum(otelcol_exporter_send_failed_spans_total) or vector(0)" + rss_before = prometheus_value(rss_query) + failed_logs_before = prometheus_value(failed_logs_query) + failed_spans_before = prometheus_value(failed_spans_query) + compose("stop", "loki", "tempo") + started = time.monotonic() + rss_samples = [rss_before] + try: + result = subprocess.run( + ["python3", "tests/smoke.py", "--base-url", args.base_url], + cwd=ROOT, check=True, text=True, capture_output=True, + ) + while time.monotonic() - started < 45: + time.sleep(5) + rss_samples.append(prometheus_value(rss_query)) + Client(args.base_url).call("GET", "/health/ready", schema="Health") + running = compose("ps", "--status", "running", "--services", capture=True).stdout.split() + if "otel-collector" not in running: + raise RuntimeError("OTel Collector stopped while exporters were retrying") + failed_logs = prometheus_value(failed_logs_query) - failed_logs_before + failed_spans = prometheus_value(failed_spans_query) - failed_spans_before + if failed_logs + failed_spans <= 0: + raise RuntimeError("Collector did not account for telemetry loss") + rss_peak = max(rss_samples) + memory_limit = 256 * 1024 * 1024 + if rss_peak >= memory_limit: + raise RuntimeError("Collector reached its container memory limit") + finally: + compose("start", "loki", "tempo") + print(json.dumps({ + "scenario": "telemetry_backend_outage", + "business_smoke": "PASS" if "PASS:" in result.stdout else "UNKNOWN", + "collector": "running", + "duration_seconds": round(time.monotonic() - started, 1), + "rss_before_mib": round(rss_before / 1024 / 1024, 1), + "rss_peak_mib": round(rss_peak / 1024 / 1024, 1), + "memory_limit_mib": 256, + "failed_log_records": int(failed_logs), + "failed_spans": int(failed_spans), + "result": "PASS", + })) + + +if __name__ == "__main__": + main()