diff --git a/.gitignore b/.gitignore index b5a71896..e6a53fe9 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ bin/ obj/ coverage/ *.log +!evidence/*.log .DS_Store .idea/ .vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..11c8f2af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +RUN groupadd -g 10001 appuser && \ + useradd -u 10001 -g appuser -s /sbin/nologin -M appuser + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready')" || exit 1 + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"] diff --git a/Makefile b/Makefile index 5d86d171..713ad927 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,12 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-student up down restart help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "test-student: run student tests (pytest); test-persistence: test stack restart with data preservation" + @echo "test-failover: test replica failure handling; up: build and start stack; down: stop stack" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +30,16 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + BASE_URL="$(BASE_URL)" $(if $(wildcard .venv/bin/pytest),.venv/bin/pytest,pytest) tests_student/ -v + +up: + docker compose up -d --build + +down: + docker compose down + +restart: + docker compose restart + diff --git a/REPORT.md b/REPORT.md index e98f75e9..db38b146 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,183 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Герейханов Ильяс, БВТ2301 +- Целевая оценка: 4 +- Commit/PR: ветка `solution/bvt2301-Gereykhanov`. PR: `ЛР 5 — БВТ2301 — Герейханов — на 4` +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI, Uvicorn, PostgreSQL 16, Redis 7, Nginx 1.27, MinIO, RabbitMQ 3.13, Pillow, boto3, asyncpg, argon2-cffi, Pydantic, Pytest, Docker Compose, AWS S3 API, OpenAPI 3.1.0. ИИ использовался для подготовки шаблонов очередей и расширения тестов pytest. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +1. **Версии runtime и образов:** + - Локальный runtime: Linux, Docker 29.9.1, Docker Compose v5.6.0 + - Load Balancer: `nginx:1.27-alpine` + - API контейнеры (`api-1`, `api-2`): `python:3.12-slim` + - Worker фоновый процесс: `python:3.12-slim` + - Брокер сообщений: `rabbitmq:3.13-alpine` + - PostgreSQL: `postgres:16-alpine` + - Redis: `redis:7-alpine` + - S3-совместимое хранилище: `cleanstart/minio:0.20260330.001845` + +2. **Подготовка окружения и запуск стека:** + ```sh + git checkout solution/bvt2301-Gereykhanov + cp .env.example .env + + # Сборка и запуск всех сервисов + make up + ``` + +3. **Ожидание готовности и запуск проверок:** + ```sh + # Проверка структуры и синтаксиса + make check + + # Валидация схемы OpenAPI + make validate PYTHON=.venv/bin/python + + # Проверка smoke теста + make test + + # Проверка сценария сохранности данных и сессий при рестарте + make test-persistence ACTION_SCRIPT=scripts/restart.sh + + # Проверка отказоустойчивости при отключении одной API-реплики + make test-failover ACTION_SCRIPT=scripts/stop-one.sh + + # Восстановление остановленной реплики после теста + docker compose up -d + + # Запуск собственных тестов + make test-student + + # Запуск нагрузочного теста + python3 tests/load.py --duration 15 --concurrency 4 --output evidence/load.json + + # Команда перезапуска задачи из DLQ: + docker compose exec api-1 python -m app.cli replay + ``` + +4. **Остановка и очистка:** + ```sh + # Остановка с сохранением данных volumes + docker compose down + + # Остановка с удалением volumes + docker compose down -v + ``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +- **Топология процессов и распределение ролей:** + - **Клиент** отправляет HTTP-запросы в единую точку входа **Nginx Load Balancer** на порт `8080`. + - **Nginx** балансирует запросы между двумя репликами API (`api-1` и `api-2`) по схеме Round-Robin. + - Реплики API работают в stateless-режиме, обращаются к общим **PostgreSQL**, **Redis**, **MinIO S3** и **RabbitMQ**. + - Фоновый процесс **OutboxRelay** считывает задания из `outbox_jobs` и публикует их в RabbitMQ с Publisher Confirms. + - Сервис **worker** забирает задачи из RabbitMQ (`prefetch_count=4`), генерирует варианты через Pillow и сохраняет в MinIO. + - Отдельный контейнер **migration** применяет `init.sql` перед запуском API-реплик и воркера. + +- **Хранилища данных и источник истины:** + - **PostgreSQL:** источник истины для пользователей, чатов, сообщений, метаданных вложений в `attachments` и задач в `outbox_jobs`. + - **MinIO S3:** хранилище бинарного контента в бакете `messenger-attachments`. Стейджинг: `staging/{attachment_id}/{filename}`, оригинал: `chats/{chat_id}/{attachment_id}/original`, варианты: `chats/{chat_id}/{attachment_id}/{variant_name}`. + - **Redis:** сессионный кэш Bearer-токенов с TTL 3600 с и кэш ключей идемпотентности. + - **RabbitMQ:** брокер сообщений (direct exchange `messenger.attachments`, очередь `messenger.attachments.jobs`, DLX/DLQ). + +- **Что перенесено из лабы 4:** + - Nginx с Round-robin 50/50, заголовками `X-Request-Id`, `X-Instance-Id` и быстрым переключением при сбоях. + - Единое хранилище сессий в Redis. + - Идемпотентность отправки сообщений по заголовку `Idempotency-Key`. + - Keyset-пагинация сообщений по индексу `(created_at, id)`. + - Изоляция MinIO без проброса портов на хост. + - Проверка MIME по содержимому, декодирование растра `img.load()` до принятия файла и защита от пиксельных бомб. + +- **Асинхронная загрузка и Transactional Outbox:** + - Загрузка вложения (`POST /api/v1/chats/{chat_id}/attachments`) сохраняет поток в staging S3 и в одной транзакции PostgreSQL создает запись в `attachments` (`status='queued'`) и задачу в `outbox_jobs`. Ответ клиенту возвращается с кодом `202 Accepted`. + - Ликвидирован разрыв COMMIT/publish: задача сохранена в БД до публикации. `OutboxRelay` считывает её через `FOR UPDATE SKIP LOCKED` и отправляет в RabbitMQ с подтверждением брокера (`publisher_confirms=True`). + +- **Обработка вложений воркером и Pillow:** + - Воркер атомарно переводит статус в `processing` (идемпотентность по `attachment_id`) и скачивает staging-файл. + - Синхронная обработка ограничена дедлайном 30 секунд для предотвращения зависания живого воркера. + - Генерируются 3 варианта: `thumbnail` (JPEG, 256×256 без апскейла, альфа на белый фон), `crop` (JPEG, 128×128), `optimized` (WebP, до 1280×1280). + - Готовые файлы пишутся в S3, статус обновляется в `ready`, после чего staging удаляется. + +- **Отказоустойчивость, DLQ и Watchdog:** + - Временные ошибки повторяются до 5 раз с экспоненциальным backoff и jitter. + - Неустранимые сбои или исчерпание попыток переводят вложение в `failed`, а сообщение уходит в `messenger.attachments.dlq`. + - Фоновый watchdog сбрасывает зависшие в `processing` задачи старше 60 секунд обратно в `queued`. + - Для ручного перезапуска failed-задач из DLQ реализован CLI: `python -m app.cli replay `. + +- **Связывание с сообщениями и скачивание контента:** + - В сообщения разрешено прикреплять вложения со статусом `queued` и `processing`. Вложения со статусом `failed` отклоняются с кодом 409. + - Скачивание через `GET /api/v1/attachments/{id}/content?variant=...` возвращает запрошенный вариант (`original`, `thumbnail`, `crop`, `optimized`). До перехода в статус ready возвращается 409. Посторонний получает 404. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Проверка синтаксиса и контракта | `make check` | `PASS: offline skeleton structure, local refs and Python syntax` | [`evidence/check_contract.log`](evidence/check_contract.log) | +| Полная OpenAPI 3.1 валидация | `make validate PYTHON=.venv/bin/python` | `PASS: full OpenAPI 3.1 and JSON Schema validation` | [`evidence/validate_openapi.log`](evidence/validate_openapi.log) | +| Публичный smoke тест | `make test` | `PASS: lab5 public API scenario` | [`evidence/smoke_test.log`](evidence/smoke_test.log) | +| Чередование реплик | `for i in {1..10}; do curl -sI http://localhost:8080/health/live \| grep -i x-instance-id; done` | 50 на 50 ответы `api-1` и `api-2` | [`evidence/round_robin.log`](evidence/round_robin.log) | +| Сохранение данных и сессий при рестарте | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | Стек перезапущен, токен, сообщения, вложения и outbox сохранены | [`evidence/persistence_test.log`](evidence/persistence_test.log) | +| Отказ API-реплики под нагрузкой | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | Восстановление записи за **0.87 с**, 116 успешных запросов, 0 сбоев | [`evidence/failover_test.log`](evidence/failover_test.log) | +| Асинхронная загрузка | `make test-student` | POST возвращает 202 `queued`, воркер генерирует варианты и переводит в `ready` | [`evidence/student_tests.log`](evidence/student_tests.log) | +| Блокировка скачивания до ready | `make test-student` | Запрос контента для queued или processing возвращает 409 Conflict | [`evidence/student_tests.log`](evidence/student_tests.log) | +| Привязка вложений к сообщениям | `make test-student` | Queued/processing прикрепляются успешно, failed отклоняются с 409 Conflict | [`evidence/student_tests.log`](evidence/student_tests.log) | +| Отсечение битых файлов на входе | `make test-student` | Усеченный и поврежденный PNG отклоняется с `415` до принятия в очередь | [`evidence/student_tests.log`](evidence/student_tests.log) | +| Все собственные тесты | `make test-student` | **35 тестов проходят**: асинхронный цикл, RabbitMQ, outbox, варианты, идемпотентность | [`evidence/student_tests.log`](evidence/student_tests.log) | +| Статусы и изоляция контейнеров | `docker compose ps` | Все 8 сервисов healthy, API и MinIO не открывают порты на хост | [`evidence/docker_services.log`](evidence/docker_services.log) | +| Нагрузочный тест | `python3 tests/load.py` | **318.1 RPS**, p50 = 6.6 ms, p95 = 59.5 ms, p99 = 80.6 ms, 0 ошибок | [`evidence/load.json`](evidence/load.json) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- **Профиль нагрузки:** скрипт `tests/load.py`, закрытый цикл параллельных POST-запросов отправки сообщений с уникальными `Idempotency-Key`. +- **Параметры:** длительность 15 секунд, 4 параллельных потока. +- **Результаты нагрузки (`evidence/load.json`):** + ```json + { + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.007, + "concurrency": 4, + "successful_requests": 4773, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 318.051, + "successful_latency_ms": { + "p50": 6.603, + "p95": 59.542, + "p99": 80.594 + } + } + ``` | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка воркера во время загрузки | API возвращает 202 Accepted, файл прикрепляется к сообщению, скачивание вариантов возвращает 409 Conflict | Все запросы API успешны; после повторного запуска воркера статус автоматически стал `ready`, сгенерированы все 3 варианта | потерь данных нет, обработка возобновилась сразу после старта воркера | +| Остановка одной реплики API | Nginx переключает трафик на живую реплику, продолжение записи без дубликатов | 116 успешных записей во время остановки, 0 ошибок, переключение за 0.87 с | 0.87 сек, потерь данных нет | +| Рестарт всего стека при активной сессии | Сохранение подтверждённых данных в Postgres, токенов сессий в Redis, очереди в RabbitMQ и файлов в MinIO | Контейнеры перезапущены, токен валиден, история сообщений, очереди и файлы сохранены | Восстановление около 5 сек, потерь данных нет | +| Загрузка декомпрессионной бомбы | Отклонение запроса с кодом 413 `payload_too_large` без исчерпания RAM воркера | Запрос отклонен мгновенно по метаданным IHDR без распаковки растра | Мгновенно (< 2 мс) | +| Загрузка поврежденного PNG файла | Отклонение с кодом 415 `unsupported_media_type` до постановки в очередь | `img.load()` на входе API обнаруживает повреждение растра и возвращает 415 | Мгновенно | +| Загрузка изображения с нечетными пропорциями | Формирование thumbnail с размерами 256×160 с округлением вниз | Метаданные и растр thumbnail имеют точные габариты 256×160 | Мгновенно | +| Попытка скачивания незавершенного вложения | Возврат 409 Conflict | Клиент получает 409 Conflict до момента перехода в статус ready | Мгновенно | +| Попытка доступа к метаданным или контенту посторонним пользователем | Защита приватности: возврат 404 Not Found без утечки факта существования | Посторонний получает 404 на `GET /attachments/{id}` и `/content` | Мгновенно | +| Попытка передачи опасного имени файла | Отклонение запроса с кодом 400 `invalid_request` | Валидатор отклоняет некорректное имя до обращения к хранилищу | Мгновенно | +| Аварийная остановка воркера посреди обработки задачи | Watchdog перехватывает задачу по истечению lease 60 секунд и сбрасывает статус в `queued` | Задача безопасно подхватывается после рестарта воркера и доходит до ready | Восстановление в пределах lease timeout | -## Ограничения и следующие шаги +## Ограничения -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Какие отказы выдерживаются и какие нет:** + 1. *Выдерживаются:* + - Внезапная остановка любой одной API-реплики под нагрузкой: балансировщик Nginx переключает входящие запросы на оставшуюся реплику. + - Полный рестарт стека: все подтверждённые данные, активные токены, задачи в Outbox и файлы в S3 сохраняются благодаря volumes. + - Аварийная остановка воркера: необработанные задачи сохраняются в очередях RabbitMQ, зависшие в `processing` задачи перехватываются по lease timeout. + - Зависание синхронных операций воркера: прерываются общим дедлайном `asyncio.timeout`. + - Временные сетевые сбои: механизм retry с экспоненциальным backoff и jitter. + - Атаки декомпрессионными бомбами, поврежденными файлами и опасными именами. + 2. *Не выдерживаются:* + - Одновременное падение обеих API-реплик (HTTP 502 Bad Gateway). + - Падение единственного экземпляра PostgreSQL, Redis, MinIO или RabbitMQ. + - Удаление через `docker compose down -v`. +- **Учебные упрощения:** + 1. Nginx, PostgreSQL, Redis, MinIO и RabbitMQ запущены в единственном экземпляре. + 2. Скачивание контента реализовано через чтение из S3 и проксирование байтов API-репликой. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..6e6e089e --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger application package.""" diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..3035feeb --- /dev/null +++ b/app/auth.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import secrets +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + """Хэширует пароль с использованием Argon2id.""" + return _hasher.hash(password) + + +def verify_password(hash_str: str, password: str) -> bool: + """Проверяет пароль на соответствие хэшу Argon2id.""" + try: + return _hasher.verify(hash_str, password) + except VerifyMismatchError: + return False + except Exception: + return False + + +def generate_session_token() -> str: + """Генерирует криптографически стойкий непрозрачный токен сессии (256 бит энтропии).""" + return secrets.token_urlsafe(32) diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..e34de237 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json +import logging +from typing import Any + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType + +from app.config import BROKER_URL + +logger = logging.getLogger("messenger.broker") + +EXCHANGE_NAME = "messenger.attachments" +QUEUE_NAME = "messenger.attachments.jobs" +ROUTING_KEY = "attachment.process" + +DLX_NAME = "messenger.attachments.dlx" +DLQ_NAME = "messenger.attachments.dlq" +DL_ROUTING_KEY = "attachment.dead" + + +async def setup_broker_topology(channel: aio_pika.abc.AbstractChannel) -> tuple[aio_pika.abc.AbstractExchange, aio_pika.abc.AbstractQueue]: + """Настраивает durable exchange, durable очереди и связку с Dead Letter Exchange.""" + # 1. Dead Letter Exchange и Dead Letter Queue + dlx = await channel.declare_exchange( + DLX_NAME, + type=ExchangeType.DIRECT, + durable=True, + ) + dlq = await channel.declare_queue( + DLQ_NAME, + durable=True, + ) + await dlq.bind(dlx, routing_key=DL_ROUTING_KEY) + + # 2. Основной Exchange и Queue с dead-lettering + exchange = await channel.declare_exchange( + EXCHANGE_NAME, + type=ExchangeType.DIRECT, + durable=True, + ) + queue = await channel.declare_queue( + QUEUE_NAME, + durable=True, + arguments={ + "x-dead-letter-exchange": DLX_NAME, + "x-dead-letter-routing-key": DL_ROUTING_KEY, + }, + ) + await queue.bind(exchange, routing_key=ROUTING_KEY) + return exchange, queue + + +class BrokerPublisher: + """Отказоустойчивый издатель сообщений в RabbitMQ с поддержкой Publisher Confirms.""" + + def __init__(self, broker_url: str = BROKER_URL) -> None: + self.broker_url = broker_url + self._connection: aio_pika.abc.AbstractRobustConnection | None = None + self._channel: aio_pika.abc.AbstractChannel | None = None + self._exchange: aio_pika.abc.AbstractExchange | None = None + + async def get_exchange(self) -> aio_pika.abc.AbstractExchange: + if self._exchange is None or self._channel is None or self._channel.is_closed: + if self._connection is None or self._connection.is_closed: + self._connection = await aio_pika.connect_robust(self.broker_url) + self._channel = await self._connection.channel(publisher_confirms=True) + self._exchange, _ = await setup_broker_topology(self._channel) + return self._exchange + + async def publish_job(self, job_dict: dict[str, Any]) -> bool: + try: + exchange = await self.get_exchange() + payload = json.dumps(job_dict, ensure_ascii=False).encode("utf-8") + message = aio_pika.Message( + body=payload, + delivery_mode=DeliveryMode.PERSISTENT, + content_type="application/json", + message_id=str(job_dict.get("job_id", "")), + ) + await exchange.publish( + message, + routing_key=ROUTING_KEY, + mandatory=True, + ) + return True + except Exception as exc: + logger.error("Failed to publish job to broker: %s", exc) + self._exchange = None + self._channel = None + return False + + async def close(self) -> None: + try: + if self._channel and not self._channel.is_closed: + await self._channel.close() + if self._connection and not self._connection.is_closed: + await self._connection.close() + except Exception: + pass + self._exchange = None + self._channel = None + self._connection = None + + +publisher = BrokerPublisher() diff --git a/app/cli.py b/app/cli.py new file mode 100644 index 00000000..d93d5195 --- /dev/null +++ b/app/cli.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import sys +import uuid + +import asyncpg + +from app.broker import publisher +from app.config import DATABASE_URL +from app.schemas import now_utc_iso + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger("messenger.cli") + + +async def replay_attachment(attachment_id_str: str) -> None: + """Безопасный операторский replay задачи из failed в queued.""" + try: + att_id = uuid.UUID(attachment_id_str) + except ValueError: + logger.error("Invalid UUID: %s", attachment_id_str) + sys.exit(1) + + pool = await asyncpg.create_pool(dsn=DATABASE_URL) + assert pool is not None + + async with pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, status, chat_id, filename FROM attachments WHERE id = $1", + att_id, + ) + if not row: + logger.error("Attachment %s not found in DB", att_id) + await pool.close() + sys.exit(1) + + if row["status"] == "ready": + logger.warning( + "Attachment %s is already 'ready'. Replay will not re-execute successful job.", + att_id, + ) + await pool.close() + return + + # Переводим в queued и создаем новое задание + new_job_id = uuid.uuid4() + now_iso = now_utc_iso() + job_payload = { + "schema_version": 1, + "job_id": str(new_job_id), + "type": "attachment.process.v1", + "attachment_id": str(att_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": now_iso, + } + + async with conn.transaction(): + await conn.execute( + """ + UPDATE attachments + SET status = 'queued', + error_code = NULL, + processing_started_at = NULL, + processing_worker_id = NULL, + updated_at = NOW() + WHERE id = $1 + """, + att_id, + ) + await conn.execute( + """ + INSERT INTO outbox_jobs ( + job_id, attachment_id, event_type, payload, status, attempt, created_at, updated_at + ) + VALUES ($1, $2, 'attachment.process.v1', $3::jsonb, 'pending', 1, NOW(), NOW()) + """, + new_job_id, + att_id, + json.dumps(job_payload), + ) + + # Публикуем напрямую в брокер + published = await publisher.publish_job(job_payload) + if published: + await conn.execute( + "UPDATE outbox_jobs SET status = 'published' WHERE job_id = $1", + new_job_id, + ) + logger.info("Attachment %s successfully replayed with new job %s", att_id, new_job_id) + else: + logger.warning( + "Attachment %s queued in outbox_jobs, will be dispatched by outbox relay", + att_id, + ) + + await publisher.close() + await pool.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Operator CLI for messenger") + subparsers = parser.add_subparsers(dest="command") + + replay_parser = subparsers.add_parser("replay", help="Replay a failed attachment") + replay_parser.add_argument("attachment_id", help="UUID of the attachment to replay") + + args = parser.parse_args() + if args.command == "replay": + asyncio.run(replay_attachment(args.attachment_id)) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..8da1ff86 --- /dev/null +++ b/app/config.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import os +import socket +from pathlib import Path + + +def _get_val(env_name: str, file_env_name: str | None = None, default: str = "") -> str: + if file_env_name and file_env_name in os.environ: + file_path = Path(os.environ[file_env_name]) + if file_path.is_file(): + return file_path.read_text(encoding="utf-8").strip() + return os.getenv(env_name, default) + + +APP_HOST: str = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT: int = int(os.getenv("APP_PORT", "8080")) +APP_ENV: str = os.getenv("APP_ENV", "lab") +LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO") + +SESSION_TTL_SECONDS: int = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +SESSION_STORE: str = os.getenv("SESSION_STORE", "redis").lower() +INSTANCE_ID: str = os.getenv("INSTANCE_ID") or socket.gethostname() or "api-1" +IDEMPOTENCY_TTL_SECONDS: int = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) + +DATABASE_URL: str = _get_val( + "DATABASE_URL", + "DATABASE_URL_FILE", + "postgresql://postgres:postgres@postgres:5432/messenger", +) + +CACHE_URL: str = _get_val( + "CACHE_URL", + "CACHE_URL_FILE", + "redis://redis:6379/0", +) + +S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "http://minio:9000") +S3_REGION: str = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET: str = os.getenv("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID: str = _get_val("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", "minioadmin") +S3_SECRET_ACCESS_KEY: str = _get_val("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", "minioadmin") +MAX_UPLOAD_BYTES: int = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS: int = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) + +BROKER_URL: str = _get_val( + "BROKER_URL", + "BROKER_URL_FILE", + "amqp://messenger:messenger_broker_pass@broker:5672/", +) +WORKER_CONCURRENCY: int = int(os.getenv("WORKER_CONCURRENCY", "4")) +WORKER_LEASE_SECONDS: int = int(os.getenv("WORKER_LEASE_SECONDS", "60")) +MAX_JOB_RETRIES: int = int(os.getenv("MAX_JOB_RETRIES", "5")) diff --git a/app/dependencies.py b/app/dependencies.py new file mode 100644 index 00000000..53563c58 --- /dev/null +++ b/app/dependencies.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from fastapi import Depends, Header, Request + +from app.errors import UnauthorizedError +from app.schemas import UserResponse +from app.storage import storage + + +async def get_current_token( + authorization: str | None = Header(default=None, alias="Authorization"), +) -> str: + if not authorization or not authorization.strip(): + raise UnauthorizedError("Отсутствует заголовок Authorization") + + parts = authorization.strip().split(maxsplit=1) + if len(parts) != 2 or parts[0].lower() != "bearer": + raise UnauthorizedError("Неверная схема авторизации (ожидается Bearer)") + + token = parts[1].strip() + if not token: + raise UnauthorizedError("Пустой Bearer токен") + return token + + +async def get_current_user(token: str = Depends(get_current_token)) -> UserResponse: + user = await storage.get_session_user(token) + if not user: + raise UnauthorizedError("Недействительная или истекшая сессия") + return user diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..1b7e87fc --- /dev/null +++ b/app/errors.py @@ -0,0 +1,260 @@ +import asyncio +import logging +import uuid + +import asyncpg +import botocore.exceptions +from fastapi import FastAPI, Request, status +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +import redis.exceptions +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app.config import INSTANCE_ID +from app.schemas import ErrorCode, ErrorDetail, ErrorResponse + + +class StorageError(Exception): + pass + + +class UnavailableError(StorageError): + pass + + +class InvalidRequestError(StorageError): + pass + + +class NotFoundError(StorageError): + pass + + +class ConflictError(StorageError): + pass + + +class PayloadTooLargeError(StorageError): + pass + + +class UnsupportedMediaTypeError(StorageError): + pass + + +class UnauthorizedError(StorageError): + pass + + +logger = logging.getLogger("app.errors") + + +def get_request_id(request: Request) -> str: + req_id = getattr(request.state, "request_id", None) + if not req_id: + req_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + return req_id + + +def make_error_response( + status_code: int, + code: ErrorCode, + message: str, + request_id: str, + headers: dict[str, str] | None = None, +) -> JSONResponse: + payload = ErrorResponse( + error=ErrorDetail(code=code, message=message, request_id=request_id) + ).model_dump() + resp_headers = { + "X-Request-Id": request_id, + "X-Instance-Id": INSTANCE_ID, + "Content-Type": "application/json", + } + if headers: + resp_headers.update(headers) + return JSONResponse( + status_code=status_code, + content=payload, + headers=resp_headers, + ) + + +def register_error_handlers(app: FastAPI) -> None: + @app.exception_handler(InvalidRequestError) + async def invalid_request_handler( + request: Request, exc: InvalidRequestError + ) -> JSONResponse: + req_id = get_request_id(request) + return make_error_response( + status.HTTP_400_BAD_REQUEST, ErrorCode.INVALID_REQUEST, str(exc), req_id + ) + + @app.exception_handler(UnauthorizedError) + async def unauthorized_handler( + request: Request, exc: UnauthorizedError + ) -> JSONResponse: + req_id = get_request_id(request) + scheme = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == "/api/v1/auth/sessions" + else 'Bearer realm="messenger"' + ) + return make_error_response( + status.HTTP_401_UNAUTHORIZED, + ErrorCode.UNAUTHORIZED, + str(exc), + req_id, + headers={"WWW-Authenticate": scheme}, + ) + + @app.exception_handler(NotFoundError) + async def not_found_handler(request: Request, exc: NotFoundError) -> JSONResponse: + req_id = get_request_id(request) + return make_error_response( + status.HTTP_404_NOT_FOUND, ErrorCode.NOT_FOUND, str(exc), req_id + ) + + @app.exception_handler(ConflictError) + async def conflict_handler(request: Request, exc: ConflictError) -> JSONResponse: + req_id = get_request_id(request) + return make_error_response( + status.HTTP_409_CONFLICT, ErrorCode.CONFLICT, str(exc), req_id + ) + + @app.exception_handler(PayloadTooLargeError) + async def payload_too_large_handler( + request: Request, exc: PayloadTooLargeError + ) -> JSONResponse: + req_id = get_request_id(request) + return make_error_response( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + ErrorCode.PAYLOAD_TOO_LARGE, + str(exc), + req_id, + ) + + @app.exception_handler(UnsupportedMediaTypeError) + async def unsupported_media_type_handler( + request: Request, exc: UnsupportedMediaTypeError + ) -> JSONResponse: + req_id = get_request_id(request) + return make_error_response( + status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + ErrorCode.UNSUPPORTED_MEDIA_TYPE, + str(exc), + req_id, + ) + + @app.exception_handler(RequestValidationError) + async def validation_error_handler( + request: Request, exc: RequestValidationError + ) -> JSONResponse: + req_id = get_request_id(request) + errors = exc.errors() + messages = [] + for err in errors: + loc = ".".join(str(x) for x in err.get("loc", []) if x != "body") + msg = err.get("msg", "invalid") + messages.append(f"{loc}: {msg}" if loc else msg) + detail_msg = "; ".join(messages) or "Некорректное тело запроса или параметры" + return make_error_response( + status.HTTP_400_BAD_REQUEST, ErrorCode.INVALID_REQUEST, detail_msg, req_id + ) + + @app.exception_handler(StarletteHTTPException) + async def http_exception_handler( + request: Request, exc: StarletteHTTPException + ) -> JSONResponse: + req_id = get_request_id(request) + code = ErrorCode.INTERNAL_ERROR + extra_headers: dict[str, str] = {} + if exc.status_code == 400: + code = ErrorCode.INVALID_REQUEST + elif exc.status_code == 401: + code = ErrorCode.UNAUTHORIZED + scheme = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == "/api/v1/auth/sessions" + else 'Bearer realm="messenger"' + ) + extra_headers["WWW-Authenticate"] = scheme + elif exc.status_code == 404: + code = ErrorCode.NOT_FOUND + elif exc.status_code == 409: + code = ErrorCode.CONFLICT + elif exc.status_code == 413: + code = ErrorCode.PAYLOAD_TOO_LARGE + elif exc.status_code == 415: + code = ErrorCode.UNSUPPORTED_MEDIA_TYPE + elif exc.status_code == 429: + code = ErrorCode.RATE_LIMITED + elif exc.status_code == 503: + code = ErrorCode.UNAVAILABLE + + msg = exc.detail if isinstance(exc.detail, str) else "HTTP ошибка" + return make_error_response( + exc.status_code, code, msg, req_id, headers=extra_headers + ) + + async def dependency_unavailable_handler( + request: Request, exc: Exception + ) -> JSONResponse: + req_id = get_request_id(request) + logger.warning( + "Upstream storage dependency unavailable for request %s: %s", + req_id, + exc, + ) + return make_error_response( + status.HTTP_503_SERVICE_UNAVAILABLE, + ErrorCode.UNAVAILABLE, + "Зависимость недоступна", + req_id, + ) + + for exc_cls in ( + UnavailableError, + asyncpg.PostgresError, + asyncpg.InterfaceError, + redis.exceptions.RedisError, + botocore.exceptions.BotoCoreError, + botocore.exceptions.ClientError, + TimeoutError, + asyncio.TimeoutError, + ConnectionError, + OSError, + ): + app.add_exception_handler(exc_cls, dependency_unavailable_handler) + + @app.exception_handler(Exception) + async def unhandled_exception_handler( + request: Request, exc: Exception + ) -> JSONResponse: + req_id = get_request_id(request) + if isinstance( + exc, + ( + UnavailableError, + asyncpg.PostgresError, + asyncpg.InterfaceError, + redis.exceptions.RedisError, + botocore.exceptions.BotoCoreError, + botocore.exceptions.ClientError, + TimeoutError, + asyncio.TimeoutError, + ConnectionError, + OSError, + ), + ): + return await dependency_unavailable_handler(request, exc) + + logger.error( + "Unhandled exception for request %s: %s", req_id, exc, exc_info=True + ) + return make_error_response( + status.HTTP_500_INTERNAL_SERVER_ERROR, + ErrorCode.INTERNAL_ERROR, + "Внутренняя ошибка сервера", + req_id, + ) diff --git a/app/imaging.py b/app/imaging.py new file mode 100644 index 00000000..42e0c7e8 --- /dev/null +++ b/app/imaging.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import hashlib +import io +import math +from typing import TypedDict + +from PIL import Image, ImageOps + +MAX_IMAGE_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + + +class GeneratedVariant(TypedDict): + name: str + content_type: str + data: bytes + size_bytes: int + width: int + height: int + sha256: str + + +def _calculate_contained_dimensions(orig_w: int, orig_h: int, max_w: int, max_h: int) -> tuple[int, int]: + """Вычисляет размеры, вписанные в max_w x max_h с сохранением пропорций, + без увеличения маленьких исходников, с округлением вниз (floor) и минимумом 1px.""" + if orig_w <= max_w and orig_h <= max_h: + return orig_w, orig_h + scale = min(max_w / orig_w, max_h / orig_h) + new_w = max(1, math.floor(orig_w * scale)) + new_h = max(1, math.floor(orig_h * scale)) + return new_w, new_h + + +def _flatten_to_rgb(img: Image.Image) -> Image.Image: + """Накладывает альфа-канал на белый фон для форматов без поддержки прозрачности (JPEG).""" + if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): + rgba = img.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.split()[3]) + return background + if img.mode != "RGB": + return img.convert("RGB") + return img + + +def generate_thumbnail(img: Image.Image) -> tuple[bytes, int, int]: + """Генерирует thumbnail: вписан в 256x256 без апскейла, JPEG, alpha на белый фон. + + Размеры рассчитываются строго с округлением вниз (floor) и минимумом 1px. + Возвращает (jpeg_bytes, width, height). + """ + target_w, target_h = _calculate_contained_dimensions(img.width, img.height, 256, 256) + if (img.width, img.height) != (target_w, target_h): + thumb = img.resize((target_w, target_h), Image.Resampling.LANCZOS) + else: + thumb = img.copy() + rgb = _flatten_to_rgb(thumb) + + buf = io.BytesIO() + rgb.save(buf, format="JPEG", quality=85, optimize=True) + data = buf.getvalue() + return data, rgb.width, rgb.height + + +def generate_crop(img: Image.Image) -> tuple[bytes, int, int]: + """Генерирует crop: центрированный квадрат 128x128 (с апскейлом для меньших), JPEG. + + Возвращает (jpeg_bytes, width, height). + """ + cropped = ImageOps.fit(img, (128, 128), method=Image.Resampling.LANCZOS) + rgb = _flatten_to_rgb(cropped) + + buf = io.BytesIO() + rgb.save(buf, format="JPEG", quality=85, optimize=True) + data = buf.getvalue() + return data, rgb.width, rgb.height + + +def generate_optimized(img: Image.Image) -> tuple[bytes, int, int]: + """Генерирует optimized: WebP, вписан в 1280x1280 без апскейла, удаление EXIF. + + Размеры рассчитываются строго с округлением вниз (floor) и минимумом 1px. + Возвращает (webp_bytes, width, height). + """ + target_w, target_h = _calculate_contained_dimensions(img.width, img.height, 1280, 1280) + if (img.width, img.height) != (target_w, target_h): + opt = img.resize((target_w, target_h), Image.Resampling.LANCZOS) + else: + opt = img.copy() + + if opt.mode not in ("RGB", "RGBA"): + opt = opt.convert("RGBA" if "transparency" in opt.info else "RGB") + + buf = io.BytesIO() + opt.save(buf, format="WEBP", quality=80, method=4) + data = buf.getvalue() + return data, opt.width, opt.height + + +def process_image_variants(raw_bytes: bytes) -> list[GeneratedVariant]: + """Безопасно декодирует сырые байты оригинала и генерирует thumbnail, crop, optimized. + + Применяет EXIF orientation к базовому изображению. + """ + with Image.open(io.BytesIO(raw_bytes)) as img: + w, h = img.size + if w * h > MAX_IMAGE_PIXELS: + raise ValueError("decompression bomb: image exceeds pixel limit") + img.load() + transposed = ImageOps.exif_transpose(img) or img + + thumb_bytes, thumb_w, thumb_h = generate_thumbnail(transposed) + crop_bytes, crop_w, crop_h = generate_crop(transposed) + opt_bytes, opt_w, opt_h = generate_optimized(transposed) + + def make_entry(name: str, ctype: str, data: bytes, w: int, h: int) -> GeneratedVariant: + return { + "name": name, + "content_type": ctype, + "data": data, + "size_bytes": len(data), + "width": w, + "height": h, + "sha256": hashlib.sha256(data).hexdigest(), + } + + return [ + make_entry("thumbnail", "image/jpeg", thumb_bytes, thumb_w, thumb_h), + make_entry("crop", "image/jpeg", crop_bytes, crop_w, crop_h), + make_entry("optimized", "image/webp", opt_bytes, opt_w, opt_h), + ] diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..c5ef7183 --- /dev/null +++ b/app/main.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager + +import uvicorn +from fastapi import FastAPI + +from app.config import APP_HOST, APP_PORT, LOG_LEVEL +from app.errors import register_error_handlers +from app.middleware import RequestIdAndLoggingMiddleware +from app.routes.attachments import router as attachments_router +from app.routes.auth import router as auth_router +from app.routes.chats import router as chats_router +from app.routes.health import router as health_router +from app.routes.messages import router as messages_router +from app.routes.users import router as users_router +from app.broker import publisher +from app.outbox import outbox_relay +from app.storage import storage + +logger = logging.getLogger("app.lifecycle") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info( + "Starting messenger server on %s:%d (env: %s)", APP_HOST, APP_PORT, LOG_LEVEL + ) + await storage.connect() + if storage.pg_pool: + await outbox_relay.start(storage.pg_pool) + yield + logger.info("Graceful shutdown: closing connections...") + await outbox_relay.stop() + await publisher.close() + await storage.disconnect() + + +app = FastAPI( + title="highload-messenger", + version="1.5.0", + docs_url="/docs", + redoc_url=None, + lifespan=lifespan, +) + +app.add_middleware(RequestIdAndLoggingMiddleware) + +register_error_handlers(app) + +app.include_router(health_router) +app.include_router(auth_router) +app.include_router(users_router) +app.include_router(chats_router) +app.include_router(messages_router) +app.include_router(attachments_router) + + +if __name__ == "__main__": + uvicorn.run( + "app.main:app", + host=APP_HOST, + port=APP_PORT, + log_level=LOG_LEVEL.lower(), + timeout_graceful_shutdown=10, + ) diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..1cad2201 --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import logging +import sys +import time +import uuid + +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response + +from app.config import INSTANCE_ID + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + stream=sys.stdout, +) +logger = logging.getLogger("app.access") + + +class RequestIdAndLoggingMiddleware(BaseHTTPMiddleware): + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + # Получаем X-Request-Id или генерируем новый + request_id = request.headers.get("X-Request-Id") + if not request_id or not request_id.strip(): + request_id = str(uuid.uuid4()) + + request.state.request_id = request_id + start_time = time.monotonic() + + if request.method in ("POST", "PUT", "PATCH") and request.url.path != "/api/v1/auth/sessions": + is_attachment_upload = ( + request.method == "POST" + and "/attachments" in request.url.path + and request.url.path.startswith("/api/v1/chats/") + ) + ct = request.headers.get("content-type") + ct_clean = ct.split(";")[0].strip().lower() if ct else "" + + if is_attachment_upload: + if not ct or ct_clean not in ("image/png", "image/jpeg"): + from fastapi import status + from app.errors import make_error_response + from app.schemas import ErrorCode + return make_error_response( + status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + ErrorCode.UNSUPPORTED_MEDIA_TYPE, + f"Неподдерживаемый тип содержимого '{ct}'. Ожидается image/png или image/jpeg", + request_id, + ) + elif ct and ct_clean != "application/json": + from fastapi import status + from app.errors import make_error_response + from app.schemas import ErrorCode + return make_error_response( + status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, + ErrorCode.UNSUPPORTED_MEDIA_TYPE, + f"Неподдерживаемый тип содержимого '{ct}'. Ожидается application/json", + request_id, + ) + + try: + response = await call_next(request) + except Exception: + # Исключение перехватится exception_handler + duration_ms = (time.monotonic() - start_time) * 1000 + logger.error( + "%s %s -> 500 error | duration=%.2fms | request_id=%s", + request.method, + request.url.path, + duration_ms, + request_id, + ) + raise + + duration_ms = (time.monotonic() - start_time) * 1000 + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + + # лог без логирования тела запросов и текстов сообщений + logger.info( + "%s %s -> %d | duration=%.2fms | request_id=%s", + request.method, + request.url.path, + response.status_code, + duration_ms, + request_id, + ) + + return response diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 00000000..c8b5951f --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import asyncio +import json +import logging +from typing import Any + +import asyncpg + +from app.broker import publisher + +logger = logging.getLogger("messenger.outbox") + + +class OutboxRelay: + """Фоновый диспетчер Transactional Outbox. + + Вычитывает сохранённые задачи из PostgreSQL и надёжно публикует их в RabbitMQ. + """ + + def __init__(self) -> None: + self._notify_event = asyncio.Event() + self._running = False + self._task: asyncio.Task[None] | None = None + + def notify(self) -> None: + """Сигнализирует о появлении новой записи для немедленной отправки без ожидания тика.""" + self._notify_event.set() + + async def start(self, pool: asyncpg.Pool) -> None: + if self._running: + return + self._running = True + self._task = asyncio.create_task(self._loop(pool)) + logger.info("OutboxRelay started") + + async def stop(self) -> None: + self._running = False + self._notify_event.set() + if self._task: + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + self._task = None + logger.info("OutboxRelay stopped") + + async def process_pending(self, pool: asyncpg.Pool, limit: int = 50) -> int: + """Выбирает и публикует порцию pending-задач с FOR UPDATE SKIP LOCKED.""" + dispatched_count = 0 + async with pool.acquire() as conn: + async with conn.transaction(): + rows = await conn.fetch( + """ + SELECT job_id, payload + FROM outbox_jobs + WHERE status = 'pending' + ORDER BY created_at ASC + LIMIT $1 + FOR UPDATE SKIP LOCKED + """, + limit, + ) + if not rows: + return 0 + + for row in rows: + job_id = row["job_id"] + payload_raw = row["payload"] + payload_dict = json.loads(payload_raw) if isinstance(payload_raw, str) else dict(payload_raw) + + success = await publisher.publish_job(payload_dict) + if success: + await conn.execute( + """ + UPDATE outbox_jobs + SET status = 'published', updated_at = NOW() + WHERE job_id = $1 + """, + job_id, + ) + dispatched_count += 1 + else: + logger.warning("Broker reject for job %s, will retry", job_id) + break + return dispatched_count + + async def _loop(self, pool: asyncpg.Pool) -> None: + while self._running: + try: + dispatched = await self.process_pending(pool) + # Если были отправлены задачи, проверяем сразу следующую порцию + if dispatched > 0: + continue + except asyncio.CancelledError: + break + except Exception as exc: + logger.error("Error in OutboxRelay loop: %s", exc) + + # Ждём уведомления о новой задаче или таймаута 1.0 с + try: + await asyncio.wait_for(self._notify_event.wait(), timeout=1.0) + self._notify_event.clear() + except asyncio.TimeoutError: + pass + + +outbox_relay = OutboxRelay() diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/routes/attachments.py b/app/routes/attachments.py new file mode 100644 index 00000000..8262a741 --- /dev/null +++ b/app/routes/attachments.py @@ -0,0 +1,85 @@ +from uuid import UUID + +from fastapi import APIRouter, Depends, Query, Request, Response, status + +from app.dependencies import get_current_user +from app.errors import InvalidRequestError +from app.schemas import AttachmentResponse, UserResponse +from app.storage import storage + +router = APIRouter(tags=["attachments"]) + + +def validate_uuid(value: str, field_name: str = "id") -> str: + try: + UUID(value) + return value + except ValueError as exc: + raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc + + +@router.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=AttachmentResponse, + status_code=status.HTTP_202_ACCEPTED, +) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + current_user: UserResponse = Depends(get_current_user), +) -> AttachmentResponse: + """Загрузка бинарного вложения в чат.""" + valid_chat_id = validate_uuid(chat_id, "chat_id") + content_type = request.headers.get("content-type") or "" + data = await request.body() + + return await storage.upload_attachment( + caller_id=current_user.id, + chat_id=valid_chat_id, + filename=filename, + content_type=content_type, + data=data, + ) + + +@router.get( + "/api/v1/attachments/{attachment_id}", + response_model=AttachmentResponse, + status_code=status.HTTP_200_OK, +) +async def get_attachment_metadata( + attachment_id: str, + current_user: UserResponse = Depends(get_current_user), +) -> AttachmentResponse: + """Получение метаданных вложения.""" + valid_att_id = validate_uuid(attachment_id, "attachment_id") + return await storage.get_attachment( + caller_id=current_user.id, + attachment_id=valid_att_id, + ) + + +@router.get( + "/api/v1/attachments/{attachment_id}/content", + status_code=status.HTTP_200_OK, +) +async def download_attachment_content( + attachment_id: str, + variant: str = Query(default="original"), + current_user: UserResponse = Depends(get_current_user), +) -> Response: + """Скачивание бинарного содержимого вложения.""" + valid_att_id = validate_uuid(attachment_id, "attachment_id") + data, content_type = await storage.get_attachment_content( + caller_id=current_user.id, + attachment_id=valid_att_id, + variant=variant, + ) + return Response( + content=data, + media_type=content_type, + headers={ + "Content-Length": str(len(data)), + }, + ) diff --git a/app/routes/auth.py b/app/routes/auth.py new file mode 100644 index 00000000..9c60f36d --- /dev/null +++ b/app/routes/auth.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import base64 +from fastapi import APIRouter, Depends, Header, Request, Response, status + +from app.dependencies import get_current_token +from app.errors import InvalidRequestError, UnauthorizedError +from app.schemas import SessionResponse +from app.storage import storage + +router = APIRouter(prefix="/api/v1/auth/sessions", tags=["auth"]) + + +@router.post("", response_model=SessionResponse, status_code=status.HTTP_201_CREATED) +async def create_session( + request: Request, + authorization: str | None = Header(default=None, alias="Authorization"), +) -> SessionResponse: + """Аутентификация пользователя через Basic Auth и создание новой сессии.""" + body = await request.body() + if body and body.strip(): + raise InvalidRequestError("Тело запроса для создания сессии должно быть пустым") + + if not authorization or not authorization.strip(): + raise UnauthorizedError("Отсутствует заголовок Authorization") + + parts = authorization.strip().split(maxsplit=1) + if len(parts) != 2 or parts[0].lower() != "basic": + raise UnauthorizedError("Неверная схема авторизации (ожидается Basic)") + + try: + raw_bytes = base64.b64decode(parts[1].strip()) + raw_str = raw_bytes.decode("utf-8") + if ":" not in raw_str: + raise UnauthorizedError("Некорректный формат Basic credentials") + username, password = raw_str.split(":", 1) + except Exception: + raise UnauthorizedError("Некорректный заголовок Basic авторизации") + + user = await storage.authenticate_user(username, password) + if not user: + raise UnauthorizedError("Неверное имя пользователя или пароль") + + session = await storage.create_session(user) + return session + + +@router.delete("/current", status_code=status.HTTP_204_NO_CONTENT) +async def delete_current_session( + request: Request, + token: str = Depends(get_current_token), +) -> Response: + """Отзыв текущей сессии.""" + body = await request.body() + if body and body.strip(): + raise InvalidRequestError("Тело запроса для отзыва сессии должно быть пустым") + + user = await storage.get_session_user(token) + if not user: + raise UnauthorizedError("Недействительная или уже отозванная сессия") + + await storage.delete_session(token) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/app/routes/chats.py b/app/routes/chats.py new file mode 100644 index 00000000..d59927fc --- /dev/null +++ b/app/routes/chats.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends, Query, status + +from app.dependencies import get_current_user +from app.schemas import ( + ChatPageResponse, + ChatResponse, + CreateChatRequest, + UserResponse, +) +from app.storage import InvalidRequestError, storage + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +def validate_uuid(value: str, field_name: str = "id") -> str: + try: + UUID(value) + return value + except ValueError as exc: + raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc + + +@router.post("", response_model=ChatResponse, status_code=status.HTTP_201_CREATED) +async def create_chat( + body: CreateChatRequest, + current_user: UserResponse = Depends(get_current_user), +) -> ChatResponse: + """Создание нового чата.""" + chat = await storage.create_chat( + caller_id=current_user.id, + title=body.title, + member_ids=body.member_ids, + ) + return chat + + +@router.get("", response_model=ChatPageResponse, status_code=status.HTTP_200_OK) +async def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), + current_user: UserResponse = Depends(get_current_user), +) -> ChatPageResponse: + """Список чатов, в которых состоит текущий пользователь.""" + items, next_cursor = await storage.list_chats( + caller_id=current_user.id, + limit=limit, + cursor=cursor, + ) + return ChatPageResponse(items=items, next_cursor=next_cursor) + + +@router.get("/{chat_id}", response_model=ChatResponse, status_code=status.HTTP_200_OK) +async def get_chat( + chat_id: str, + current_user: UserResponse = Depends(get_current_user), +) -> ChatResponse: + """Получение информации о чате по ID.""" + valid_id = validate_uuid(chat_id, "chat_id") + chat = await storage.get_chat(caller_id=current_user.id, chat_id=valid_id) + return chat diff --git a/app/routes/health.py b/app/routes/health.py new file mode 100644 index 00000000..3e475270 --- /dev/null +++ b/app/routes/health.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from fastapi import APIRouter, status +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app.schemas import HealthResponse +from app.storage import storage + +router = APIRouter(tags=["health"]) + + +@router.get( + "/health/live", response_model=HealthResponse, status_code=status.HTTP_200_OK +) +async def health_live() -> HealthResponse: + """Проверка жизни процесса (Liveness).""" + return HealthResponse(status="ok") + + +@router.get( + "/health/ready", response_model=HealthResponse, status_code=status.HTTP_200_OK +) +async def health_ready() -> HealthResponse: + """Проверка готовности сервиса и его зависимостей (Readiness).""" + is_ready = await storage.check_health() + if not is_ready: + raise StarletteHTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Сервис или его хранилища недоступны", + ) + return HealthResponse(status="ok") diff --git a/app/routes/messages.py b/app/routes/messages.py new file mode 100644 index 00000000..65982f53 --- /dev/null +++ b/app/routes/messages.py @@ -0,0 +1,74 @@ +import re +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, status + +from app.dependencies import get_current_user +from app.schemas import ( + CreateMessageRequest, + MessagePageResponse, + MessageResponse, + UserResponse, +) +from app.storage import InvalidRequestError, storage + +router = APIRouter(prefix="/api/v1/chats", tags=["messages"]) + +IDEMPOTENCY_KEY_REGEX = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def validate_uuid(value: str, field_name: str = "id") -> str: + try: + UUID(value) + return value + except ValueError as exc: + raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc + + +@router.post( + "/{chat_id}/messages", + response_model=MessageResponse, + status_code=status.HTTP_201_CREATED, +) +async def create_message( + chat_id: str, + body: CreateMessageRequest, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), + current_user: UserResponse = Depends(get_current_user), +) -> MessageResponse: + """Отправка сообщения в чат.""" + if not idempotency_key or not IDEMPOTENCY_KEY_REGEX.match(idempotency_key): + raise InvalidRequestError( + "Заголовок 'Idempotency-Key' обязателен и должен соответствовать шаблону ^[A-Za-z0-9._:-]{1,128}$" + ) + valid_id = validate_uuid(chat_id, "chat_id") + msg = await storage.send_message( + caller_id=current_user.id, + chat_id=valid_id, + text=body.text, + idempotency_key=idempotency_key, + attachment_ids=body.attachment_ids, + ) + return msg + + +@router.get( + "/{chat_id}/messages", + response_model=MessagePageResponse, + status_code=status.HTTP_200_OK, +) +async def list_messages( + chat_id: str, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), + current_user: UserResponse = Depends(get_current_user), +) -> MessagePageResponse: + """Получение истории сообщений чата с keyset-пагинацией.""" + valid_id = validate_uuid(chat_id, "chat_id") + items, next_cursor = await storage.list_messages( + caller_id=current_user.id, + chat_id=valid_id, + limit=limit, + cursor=cursor, + ) + return MessagePageResponse(items=items, next_cursor=next_cursor) diff --git a/app/routes/users.py b/app/routes/users.py new file mode 100644 index 00000000..d2842afd --- /dev/null +++ b/app/routes/users.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from fastapi import APIRouter, Depends, status + +from app.dependencies import get_current_user +from app.schemas import CreateUserRequest, UserResponse +from app.storage import storage + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + + +@router.post("", response_model=UserResponse, status_code=status.HTTP_201_CREATED) +async def create_user(body: CreateUserRequest) -> UserResponse: + """Регистрация нового пользователя.""" + user = await storage.create_user( + username=body.username, + display_name=body.display_name, + password=body.password, + ) + return user + + +@router.get("/me", response_model=UserResponse, status_code=status.HTTP_200_OK) +async def get_me( + current_user: UserResponse = Depends(get_current_user), +) -> UserResponse: + """Получение профиля текущего пользователя.""" + return current_user diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 00000000..765857db --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import datetime as dt +import re +from enum import Enum +from uuid import UUID + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +def now_utc_iso() -> str: + """Возвращает текущее время в формате RFC 3339 UTC с суффиксом Z.""" + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +class ErrorCode(str, Enum): + INVALID_REQUEST = "invalid_request" + UNAUTHORIZED = "unauthorized" + NOT_FOUND = "not_found" + CONFLICT = "conflict" + PAYLOAD_TOO_LARGE = "payload_too_large" + UNSUPPORTED_MEDIA_TYPE = "unsupported_media_type" + RATE_LIMITED = "rate_limited" + UNAVAILABLE = "unavailable" + INTERNAL_ERROR = "internal_error" + + +class ErrorDetail(BaseModel): + model_config = ConfigDict(extra="forbid") + + code: ErrorCode + message: str = Field(min_length=1) + request_id: str = Field(min_length=1) + + +class ErrorResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + error: ErrorDetail + + +class HealthResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + status: str = Field(default="ok") + + +class CreateUserRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + username: str = Field(min_length=3, max_length=32) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def validate_username(cls, v: str) -> str: + if not re.match(r"^[a-z0-9_]{3,32}\Z", v): + raise ValueError("Имя пользователя должно соответствовать шаблону ^[a-z0-9_]{3,32}$") + return v + + @field_validator("display_name") + @classmethod + def validate_display_name(cls, v: str) -> str: + if not v.strip(): + raise ValueError("Отображаемое имя не может состоять только из пробелов") + return v + + @field_validator("password") + @classmethod + def validate_password(cls, v: str) -> str: + if not v.strip(): + raise ValueError("Пароль не может состоять только из пробелов") + return v + + +class UserResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + username: str + display_name: str + created_at: str + + +class SessionResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + token: str = Field(min_length=32) + token_type: str = Field(default="Bearer") + expires_at: str + user: UserResponse + + +class CreateChatRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, v: str) -> str: + if not v.strip(): + raise ValueError("Название не может состоять только из пробелов") + return v + + @field_validator("member_ids") + @classmethod + def validate_member_ids(cls, v: list[str]) -> list[str]: + if len(v) != len(set(v)): + raise ValueError("Идентификаторы участников должны быть уникальными") + for member_id in v: + try: + UUID(member_id) + except ValueError: + raise ValueError(f"Идентификатор участника '{member_id}' не является валидным UUID") + return v + + +class ChatResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + title: str + member_ids: list[str] + created_at: str + + +class ChatPageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[ChatResponse] + next_cursor: str | None = None + + +class VariantResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int = Field(ge=1) + width: int = Field(ge=1) + height: int = Field(ge=1) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + +class AttachmentResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] = "queued" + variants: list[VariantResponse] = Field(default_factory=list) + error_code: str | None = None + created_at: str + updated_at: str + + +class AttachmentJobMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + job_id: str + type: Literal["attachment.process.v1"] = "attachment.process.v1" + attachment_id: str + pipeline_version: int = 1 + attempt: int = 1 + created_at: str + + +class CreateMessageRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + text: str = Field(default="", max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def validate_attachment_ids(cls, v: list[str]) -> list[str]: + if len(v) != len(set(v)): + raise ValueError("attachment_ids must contain unique items") + for item in v: + try: + UUID(item) + except ValueError: + raise ValueError(f"Идентификатор вложения '{item}' не является валидным UUID") + return v + + @model_validator(mode="after") + def check_text_or_attachment(self) -> CreateMessageRequest: + has_text = bool(self.text and self.text.strip()) + has_attachments = bool(self.attachment_ids) + if not has_text and not has_attachments: + raise ValueError("Сообщение должно содержать непустой текст или хотя бы одно вложение") + return self + + +class MessageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: list[str] = Field(default_factory=list) + + +class MessagePageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[MessageResponse] + next_cursor: str | None = None diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..eaf19186 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,1001 @@ +import asyncio +import base64 +from contextlib import asynccontextmanager +import datetime as dt +import hashlib +import json +import logging +import uuid +from typing import Any + +import asyncpg +import redis.asyncio as redis + +import io +import boto3 +import botocore.exceptions +from PIL import Image + +from app.auth import generate_session_token, hash_password, verify_password +from app.config import ( + CACHE_URL, + DATABASE_URL, + SESSION_TTL_SECONDS, + S3_ENDPOINT, + S3_REGION, + S3_BUCKET, + S3_ACCESS_KEY_ID, + S3_SECRET_ACCESS_KEY, + MAX_UPLOAD_BYTES, + MAX_IMAGE_PIXELS, +) +from app.errors import ( + ConflictError, + InvalidRequestError, + NotFoundError, + PayloadTooLargeError, + UnauthorizedError, + UnsupportedMediaTypeError, +) +from app.outbox import outbox_relay +from app.schemas import ( + AttachmentResponse, + ChatResponse, + MessageResponse, + SessionResponse, + UserResponse, + VariantResponse, + now_utc_iso, +) + +logger = logging.getLogger("app.storage") + + +def encode_cursor(data: dict) -> str: + raw = json.dumps(data, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=") + + +def decode_cursor(cursor_str: str) -> dict: + if not isinstance(cursor_str, str) or not cursor_str.strip(): + raise InvalidRequestError("Некорректный курсор") + try: + padded = cursor_str + "=" * (-len(cursor_str) % 4) + raw = base64.urlsafe_b64decode(padded.encode("utf-8")) + data = json.loads(raw.decode("utf-8")) + if not isinstance(data, dict): + raise InvalidRequestError("Некорректный курсор") + return data + except InvalidRequestError: + raise + except Exception as exc: + raise InvalidRequestError("Некорректный курсор") from exc + + +def _format_dt(d: dt.datetime) -> str: + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _to_text(val: Any) -> str: + if isinstance(val, (bytes, bytearray, memoryview)): + return bytes(val).decode("utf-8") + return str(val) + + +class PostgresStorage: + def __init__(self) -> None: + self.pg_pool: asyncpg.Pool | None = None + self.redis_client: redis.Redis | None = None + self.s3_client: Any = None + + @property + def pool(self) -> asyncpg.Pool | None: + return self.pg_pool + + @asynccontextmanager + async def acquire_conn(self, timeout: float = 2.0): + if self.pg_pool is None: + raise asyncpg.InterfaceError("Пул подключений к PostgreSQL не инициализирован") + async with asyncio.timeout(3.0): + async with self.pg_pool.acquire(timeout=timeout) as conn: + yield conn + + async def connect(self) -> None: + logger.info("Подключение к PostgreSQL: %s", DATABASE_URL.split("@")[-1]) + self.pg_pool = await asyncpg.create_pool( + DATABASE_URL, + min_size=2, + max_size=20, + command_timeout=3.0, + timeout=2.0, + ) + + logger.info("Подключение к Redis: %s", CACHE_URL.split("@")[-1]) + self.redis_client = redis.from_url( + CACHE_URL, + decode_responses=True, + socket_connect_timeout=2.0, + socket_timeout=2.0, + ) + + logger.info("Подключение к S3 (MinIO): %s, bucket: %s", S3_ENDPOINT, S3_BUCKET) + self.s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY_ID, + aws_secret_access_key=S3_SECRET_ACCESS_KEY, + region_name=S3_REGION, + ) + await self._ensure_bucket() + + async def _ensure_bucket(self) -> None: + def _init(): + try: + self.s3_client.head_bucket(Bucket=S3_BUCKET) + except Exception: + try: + self.s3_client.create_bucket(Bucket=S3_BUCKET) + logger.info("Создан приватный бакет S3: %s", S3_BUCKET) + except Exception as e: + logger.warning("Не удалось создать бакет S3 %s: %s", S3_BUCKET, e) + + await asyncio.to_thread(_init) + + async def disconnect(self) -> None: + if self.pg_pool: + await self.pg_pool.close() + self.pg_pool = None + if self.redis_client: + await self.redis_client.close() + self.redis_client = None + self.s3_client = None + + async def check_health(self) -> bool: + if not self.pg_pool or not self.redis_client or not self.s3_client: + return False + try: + async with asyncio.timeout(2.5): + async with self.acquire_conn(timeout=1.5) as conn: + await conn.fetchval("SELECT 1") + await self.redis_client.ping() + await asyncio.to_thread(self.s3_client.head_bucket, Bucket=S3_BUCKET) + return True + except Exception as exc: + logger.warning("Ошибка проверки health: %s", exc) + return False + + # ------------------ Пользователи и аутентификация ------------------ + + async def create_user( + self, username: str, display_name: str, password: str + ) -> UserResponse: + user_id = uuid.uuid4() + now = dt.datetime.now(dt.timezone.utc) + pwd_hash = await asyncio.to_thread(hash_password, password) + + try: + async with self.acquire_conn() as conn: + await conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + user_id, + username, + display_name, + pwd_hash, + now, + ) + except asyncpg.UniqueViolationError: + raise ConflictError("Имя пользователя уже занято") + + return UserResponse( + id=str(user_id), + username=username, + display_name=display_name, + created_at=_format_dt(now), + ) + + async def authenticate_user( + self, username: str, password: str + ) -> UserResponse | None: + async with self.acquire_conn() as conn: + row = await conn.fetchrow( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = $1 + """, + username, + ) + + if not row: + return None + + is_valid = await asyncio.to_thread(verify_password, row["password_hash"], password) + if not is_valid: + return None + + return UserResponse( + id=str(row["id"]), + username=row["username"], + display_name=_to_text(row["display_name"]), + created_at=_format_dt(row["created_at"]), + ) + + async def get_user_by_id(self, user_id: str) -> UserResponse | None: + try: + uid = uuid.UUID(user_id) + except ValueError: + return None + + async with self.acquire_conn() as conn: + row = await conn.fetchrow( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = $1 + """, + uid, + ) + + if not row: + return None + + return UserResponse( + id=str(row["id"]), + username=row["username"], + display_name=_to_text(row["display_name"]), + created_at=_format_dt(row["created_at"]), + ) + + # ------------------ Сессии (Redis) ------------------ + + async def create_session(self, user: UserResponse) -> SessionResponse: + assert self.redis_client is not None + token = generate_session_token() + now = dt.datetime.now(dt.timezone.utc) + expires_at = now + dt.timedelta(seconds=SESSION_TTL_SECONDS) + expires_at_str = _format_dt(expires_at) + + session_data = json.dumps( + { + "user_id": user.id, + "expires_at": expires_at_str, + } + ) + + key = f"session:{token}" + await self.redis_client.set(key, session_data, ex=SESSION_TTL_SECONDS) + + return SessionResponse( + token=token, + token_type="Bearer", + expires_at=expires_at_str, + user=user, + ) + + async def get_session_user(self, token: str) -> UserResponse | None: + if not token or not self.redis_client: + return None + + key = f"session:{token}" + raw = await self.redis_client.get(key) + if not raw: + return None + + try: + data = json.loads(raw) + user_id = data.get("user_id") + except Exception: + return None + + if not user_id: + return None + + return await self.get_user_by_id(user_id) + + async def delete_session(self, token: str) -> bool: + if not token or not self.redis_client: + return False + key = f"session:{token}" + res = await self.redis_client.delete(key) + return res > 0 + + # ------------------ Чаты ------------------ + + async def create_chat( + self, caller_id: str, title: str, member_ids: list[str] + ) -> ChatResponse: + if caller_id not in member_ids: + raise InvalidRequestError("Создатель чата должен быть в списке участников") + + chat_id = uuid.uuid4() + now = dt.datetime.now(dt.timezone.utc) + + member_uuids = [uuid.UUID(m) for m in member_ids] + + async with self.acquire_conn() as conn: + # Проверяем существование всех участников + found = await conn.fetch( + "SELECT id FROM users WHERE id = ANY($1::uuid[])", member_uuids + ) + if len(found) != len(member_uuids): + raise InvalidRequestError( + "Один или несколько участников чата не найдены" + ) + + async with conn.transaction(): + await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", + chat_id, + title, + now, + ) + member_records = [(chat_id, m) for m in member_uuids] + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + member_records, + ) + + return ChatResponse( + id=str(chat_id), + title=title, + member_ids=member_ids, + created_at=_format_dt(now), + ) + + async def get_chat(self, caller_id: str, chat_id: str) -> ChatResponse: + try: + cid = uuid.UUID(chat_id) + c_caller = uuid.UUID(caller_id) + except ValueError: + raise NotFoundError("Чат не найден") + + async with self.acquire_conn() as conn: + # Проверяем членство вызывающего + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, + c_caller, + ) + if not is_member: + raise NotFoundError("Чат не найден") + + chat_row = await conn.fetchrow( + "SELECT id, title, created_at FROM chats WHERE id = $1", cid + ) + if not chat_row: + raise NotFoundError("Чат не найден") + + members_rows = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", cid + ) + member_ids = [str(r["user_id"]) for r in members_rows] + + return ChatResponse( + id=str(chat_row["id"]), + title=_to_text(chat_row["title"]), + member_ids=member_ids, + created_at=_format_dt(chat_row["created_at"]), + ) + + async def list_chats( + self, caller_id: str, limit: int = 50, cursor: str | None = None + ) -> tuple[list[ChatResponse], str | None]: + try: + c_caller = uuid.UUID(caller_id) + except ValueError: + return [], None + + c_created: dt.datetime | None = None + c_id: uuid.UUID | None = None + + if cursor is not None: + c_data = decode_cursor(cursor) + if c_data.get("type") != "chats": + raise InvalidRequestError("Недействительный тип курсора") + c_created_str = c_data.get("created_at") + c_id_str = c_data.get("id") + if ( + not isinstance(c_created_str, str) + or not isinstance(c_id_str, str) + or not c_created_str.strip() + or not c_id_str.strip() + ): + raise InvalidRequestError("Поврежденный курсор") + try: + c_created = dt.datetime.fromisoformat( + c_created_str.replace("Z", "+00:00") + ) + c_id = uuid.UUID(c_id_str) + except Exception: + raise InvalidRequestError("Некорректные параметры курсора") + + async with self.acquire_conn() as conn: + query = """ + SELECT c.id, c.title, c.created_at, array_agg(cm.user_id::text) as member_ids + FROM chats c + JOIN chat_members my_m ON c.id = my_m.chat_id AND my_m.user_id = $1 + JOIN chat_members cm ON c.id = cm.chat_id + WHERE ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2::timestamptz, $3::uuid)) + GROUP BY c.id, c.title, c.created_at + ORDER BY c.created_at ASC, c.id ASC + LIMIT $4 + """ + rows = await conn.fetch(query, c_caller, c_created, c_id, limit + 1) + + has_next = len(rows) > limit + items_rows = rows[:limit] + + items = [ + ChatResponse( + id=str(r["id"]), + title=_to_text(r["title"]), + member_ids=list(r["member_ids"]), + created_at=_format_dt(r["created_at"]), + ) + for r in items_rows + ] + + next_cursor = None + if has_next and items: + last = items[-1] + next_cursor = encode_cursor( + { + "type": "chats", + "created_at": last.created_at, + "id": last.id, + } + ) + + return items, next_cursor + + # ------------------ Сообщения ------------------ + + @staticmethod + def compute_request_hash(text: str, attachment_ids: list[str] | None = None) -> str: + norm_attachments = [str(x) for x in (attachment_ids or [])] + canonical = json.dumps( + {"attachment_ids": norm_attachments, "text": text}, + sort_keys=True, + ensure_ascii=False, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + async def send_message( + self, + caller_id: str, + chat_id: str, + text: str, + idempotency_key: str, + attachment_ids: list[str] | None = None, + ) -> MessageResponse: + try: + cid = uuid.UUID(chat_id) + c_caller = uuid.UUID(caller_id) + except ValueError: + raise NotFoundError("Чат не найден") + + norm_attachments = [str(x) for x in (attachment_ids or [])] + req_hash = self.compute_request_hash(text, norm_attachments) + + async with self.acquire_conn() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, + c_caller, + ) + if not is_member: + raise NotFoundError("Чат не найден") + + # Валидация прикрепляемых вложений + att_uuids: list[uuid.UUID] = [] + if norm_attachments: + for att_str in norm_attachments: + try: + att_uuids.append(uuid.UUID(att_str)) + except ValueError: + raise NotFoundError("Вложение не найдено") + + rows = await conn.fetch( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY($1::uuid[])", + att_uuids, + ) + if len(rows) != len(att_uuids): + raise NotFoundError("Одно или несколько вложений не найдены") + + for r in rows: + if r["chat_id"] != cid: + raise NotFoundError("Вложение принадлежит другому чату") + if r["status"] == "failed": + raise ConflictError("Нельзя прикреплять вложение со статусом failed") + + # 1. Проверяем, обрабатывался ли уже этот idempotency_key + existing = await conn.fetchrow( + """ + SELECT ik.message_id, ik.request_hash, m.text, m.attachment_ids, m.created_at AS msg_created_at + FROM idempotency_keys ik + JOIN messages m ON m.id = ik.message_id + WHERE ik.user_id = $1 AND ik.chat_id = $2 AND ik.idempotency_key = $3 + """, + c_caller, + cid, + idempotency_key, + ) + if existing: + if existing["request_hash"] == req_hash: + raw_text = existing["text"] + msg_text = ( + raw_text.decode("utf-8") + if isinstance(raw_text, (bytes, bytearray)) + else str(raw_text) + ) + res_attachments = [ + str(a) for a in (existing["attachment_ids"] or []) + ] + return MessageResponse( + id=str(existing["message_id"]), + chat_id=chat_id, + sender_id=caller_id, + text=msg_text, + attachment_ids=res_attachments, + created_at=_format_dt(existing["msg_created_at"]), + ) + else: + raise ConflictError("Ключ идемпотентности уже использован с другим телом запроса") + + msg_id = uuid.uuid4() + now = dt.datetime.now(dt.timezone.utc) + text_bytes = text.encode("utf-8") + + try: + async with conn.transaction(): + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids, created_at) + VALUES ($1, $2, $3, $4, $5, $6) + """, + msg_id, + cid, + c_caller, + text_bytes, + att_uuids, + now, + ) + await conn.execute( + """ + INSERT INTO idempotency_keys (user_id, chat_id, idempotency_key, message_id, request_hash, created_at) + VALUES ($1, $2, $3, $4, $5, $6) + """, + c_caller, + cid, + idempotency_key, + msg_id, + req_hash, + now, + ) + except asyncpg.UniqueViolationError: + winner = await conn.fetchrow( + """ + SELECT ik.message_id, ik.request_hash, m.text, m.attachment_ids, m.created_at AS msg_created_at + FROM idempotency_keys ik + JOIN messages m ON m.id = ik.message_id + WHERE ik.user_id = $1 AND ik.chat_id = $2 AND ik.idempotency_key = $3 + """, + c_caller, + cid, + idempotency_key, + ) + if winner: + if winner["request_hash"] == req_hash: + raw_text = winner["text"] + msg_text = ( + raw_text.decode("utf-8") + if isinstance(raw_text, (bytes, bytearray)) + else str(raw_text) + ) + res_attachments = [ + str(a) for a in (winner["attachment_ids"] or []) + ] + return MessageResponse( + id=str(winner["message_id"]), + chat_id=chat_id, + sender_id=caller_id, + text=msg_text, + attachment_ids=res_attachments, + created_at=_format_dt(winner["msg_created_at"]), + ) + else: + raise ConflictError("Ключ идемпотентности уже использован с другим телом запроса") + raise + + return MessageResponse( + id=str(msg_id), + chat_id=chat_id, + sender_id=caller_id, + text=text, + attachment_ids=norm_attachments, + created_at=_format_dt(now), + ) + + async def list_messages( + self, caller_id: str, chat_id: str, limit: int = 50, cursor: str | None = None + ) -> tuple[list[MessageResponse], str | None]: + try: + cid = uuid.UUID(chat_id) + c_caller = uuid.UUID(caller_id) + except ValueError: + raise NotFoundError("Чат не найден") + + async with self.acquire_conn() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, + c_caller, + ) + if not is_member: + raise NotFoundError("Чат не найден") + + c_created: dt.datetime | None = None + c_id: uuid.UUID | None = None + + if cursor is not None: + c_data = decode_cursor(cursor) + if c_data.get("type") != "messages" or c_data.get("chat_id") != chat_id: + raise InvalidRequestError( + "Курсор принадлежит другому чату или недействителен" + ) + c_created_str = c_data.get("created_at") + c_id_str = c_data.get("id") + if ( + not isinstance(c_created_str, str) + or not isinstance(c_id_str, str) + or not c_created_str.strip() + or not c_id_str.strip() + ): + raise InvalidRequestError("Поврежденный курсор") + try: + c_created = dt.datetime.fromisoformat( + c_created_str.replace("Z", "+00:00") + ) + c_id = uuid.UUID(c_id_str) + except Exception: + raise InvalidRequestError("Некорректные параметры курсора") + + query = """ + SELECT id, chat_id, sender_id, text, attachment_ids, created_at + FROM messages + WHERE chat_id = $1 + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2::timestamptz, $3::uuid)) + ORDER BY created_at ASC, id ASC + LIMIT $4 + """ + rows = await conn.fetch(query, cid, c_created, c_id, limit + 1) + + has_next = len(rows) > limit + items_rows = rows[:limit] + + def _to_text(val: Any) -> str: + if isinstance(val, (bytes, bytearray, memoryview)): + return bytes(val).decode("utf-8") + return str(val) + + items = [ + MessageResponse( + id=str(r["id"]), + chat_id=str(r["chat_id"]), + sender_id=str(r["sender_id"]), + text=_to_text(r["text"]), + attachment_ids=[str(a) for a in (r["attachment_ids"] or [])], + created_at=_format_dt(r["created_at"]), + ) + for r in items_rows + ] + + next_cursor = None + if has_next and items: + last = items[-1] + next_cursor = encode_cursor( + { + "type": "messages", + "chat_id": chat_id, + "created_at": last.created_at, + "id": last.id, + } + ) + + return items, next_cursor + + # ------------------ Вложения (Attachments) ------------------ + + async def upload_attachment( + self, + caller_id: str, + chat_id: str, + filename: str, + content_type: str, + data: bytes, + ) -> AttachmentResponse: + try: + cid = uuid.UUID(chat_id) + c_caller = uuid.UUID(caller_id) + except ValueError: + raise NotFoundError("Чат не найден") + + # 1. Валидация filename + if not filename or len(filename) > 255: + raise InvalidRequestError("Некорректная длина имени файла (1-255 символов)") + if any(c in filename for c in ("/", "\\", "\0")) or any( + ord(c) < 32 for c in filename + ): + raise InvalidRequestError("Имя файла содержит недопустимые символы") + + # 2. Проверка прав (ACL) + async with self.acquire_conn() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + cid, + c_caller, + ) + if not is_member: + raise NotFoundError("Чат не найден") + + # 3. Валидация Content-Type + ct_clean = content_type.split(";")[0].strip().lower() + if ct_clean not in ("image/png", "image/jpeg"): + raise UnsupportedMediaTypeError( + "Поддерживаются только форматы image/png и image/jpeg" + ) + + # 4. Валидация размера данных + size_bytes = len(data) + if size_bytes > MAX_UPLOAD_BYTES: + raise PayloadTooLargeError( + f"Размер файла {size_bytes} байт превышает допустимый лимит {MAX_UPLOAD_BYTES} байт" + ) + + # 5. Проверка магических байтов (сигнатуры MIME) + if ct_clean == "image/png": + if not data.startswith(b"\x89PNG\r\n\x1a\n"): + raise UnsupportedMediaTypeError("Содержимое файла не соответствует формату PNG") + elif ct_clean == "image/jpeg": + if not (data.startswith(b"\xff\xd8\xff")): + raise UnsupportedMediaTypeError("Содержимое файла не соответствует формату JPEG") + + # 6. Проверка разрешения (лимит 40 млн пикселей) и целостности растра + def _check_dimensions(): + try: + with Image.open(io.BytesIO(data)) as img: + w, h = img.size + if w * h > MAX_IMAGE_PIXELS: + return False + # Проверяем целостность растровых данных и фильтров строк: + img.load() + return True + except Exception: + raise UnsupportedMediaTypeError("Поврежденное изображение") + + try: + within_limits = await asyncio.to_thread(_check_dimensions) + if not within_limits: + raise PayloadTooLargeError( + f"Разрешение изображения превышает лимит {MAX_IMAGE_PIXELS} пикселей" + ) + except (UnsupportedMediaTypeError, PayloadTooLargeError): + raise + except Exception as e: + logger.warning("upload_attachment check_dimensions unexpected: %s (%s)", type(e), e) + raise UnsupportedMediaTypeError("Поврежденное изображение") from e + + # 7. Генерация идентификаторов + att_id = uuid.uuid4() + job_id = uuid.uuid4() + staging_key = f"staging/{att_id}/{filename}" + sha256_hash = hashlib.sha256(data).hexdigest() + now = dt.datetime.now(dt.timezone.utc) + now_iso = _format_dt(now) + + # 8. Загрузка файла в устойчивый staging в S3 + def _put_to_s3(): + self.s3_client.put_object( + Bucket=S3_BUCKET, + Key=staging_key, + Body=data, + ContentType=ct_clean, + ) + + await asyncio.to_thread(_put_to_s3) + + # 9. Атомарное сохранение в PostgreSQL: attachments + outbox_jobs + job_payload = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(att_id), + "pipeline_version": 1, + "attempt": 1, + "created_at": now_iso, + } + + try: + async with self.acquire_conn() as conn: + async with conn.transaction(): + await conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, s3_key, status, error_code, variants, + created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NULL, '[]'::jsonb, $9, $9) + """, + att_id, + cid, + c_caller, + filename, + ct_clean, + size_bytes, + sha256_hash, + staging_key, + now, + ) + await conn.execute( + """ + INSERT INTO outbox_jobs ( + job_id, attachment_id, event_type, payload, status, attempt, created_at, updated_at + ) + VALUES ($1, $2, 'attachment.process.v1', $3::jsonb, 'pending', 1, $4, $4) + """, + job_id, + att_id, + json.dumps(job_payload), + now, + ) + except Exception as db_err: + logger.error("Ошибка записи метаданных/outbox в БД, компенсация staging S3: %s", db_err) + try: + await asyncio.to_thread( + self.s3_client.delete_object, Bucket=S3_BUCKET, Key=staging_key + ) + except Exception as del_err: + logger.error("Ошибка удаления staging S3 объекта при компенсации: %s", del_err) + raise + + # 10. Уведомляем outbox relay для немедленной публикации + outbox_relay.notify() + + return AttachmentResponse( + id=str(att_id), + chat_id=str(cid), + owner_id=str(c_caller), + filename=filename, + content_type=ct_clean, + size_bytes=size_bytes, + sha256=sha256_hash, + status="queued", + variants=[], + error_code=None, + created_at=now_iso, + updated_at=now_iso, + ) + + async def get_attachment( + self, caller_id: str, attachment_id: str + ) -> AttachmentResponse: + try: + aid = uuid.UUID(attachment_id) + c_caller = uuid.UUID(caller_id) + except ValueError: + raise NotFoundError("Вложение не найдено") + + async with self.acquire_conn() as conn: + row = await conn.fetchrow( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, a.variants, + a.created_at, a.updated_at + FROM attachments a + WHERE a.id = $1 + """, + aid, + ) + if not row: + raise NotFoundError("Вложение не найдено") + + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + row["chat_id"], + c_caller, + ) + if not is_member: + raise NotFoundError("Вложение не найдено") + + raw_variants = row["variants"] + if isinstance(raw_variants, str): + raw_variants = json.loads(raw_variants) + variants_list = [VariantResponse(**v) for v in (raw_variants or [])] + + return AttachmentResponse( + id=str(row["id"]), + chat_id=str(row["chat_id"]), + owner_id=str(row["owner_id"]), + filename=row["filename"], + content_type=row["content_type"], + size_bytes=row["size_bytes"], + sha256=row["sha256"], + status=row["status"], + variants=variants_list, + error_code=row["error_code"], + created_at=_format_dt(row["created_at"]), + updated_at=_format_dt(row["updated_at"]), + ) + + async def get_attachment_content( + self, caller_id: str, attachment_id: str, variant: str | None = None + ) -> tuple[bytes, str]: + try: + aid = uuid.UUID(attachment_id) + c_caller = uuid.UUID(caller_id) + except ValueError: + raise NotFoundError("Вложение не найдено") + + variant_clean = (variant or "original").strip().lower() + if variant_clean not in ("original", "thumbnail", "crop", "optimized"): + raise InvalidRequestError("Неизвестный вариант изображения") + + async with self.acquire_conn() as conn: + row = await conn.fetchrow( + """ + SELECT a.chat_id, a.content_type, a.s3_key, a.status, a.variants + FROM attachments a + WHERE a.id = $1 + """, + aid, + ) + if not row: + raise NotFoundError("Вложение не найдено") + + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + row["chat_id"], + c_caller, + ) + if not is_member: + raise NotFoundError("Вложение не найдено") + + # По контракту: для queued/processing/failed возвращается 409 Conflict + if row["status"] != "ready": + raise ConflictError("Вложение еще не готово или завершилось с ошибкой") + + if variant_clean == "original": + s3_key = row["s3_key"] + content_type = row["content_type"] + else: + raw_variants = row["variants"] + if isinstance(raw_variants, str): + raw_variants = json.loads(raw_variants) + variants_list = raw_variants or [] + matched = next((v for v in variants_list if v.get("name") == variant_clean), None) + if not matched: + raise NotFoundError("Запрошенный вариант не найден") + s3_key = f"chats/{row['chat_id']}/{aid}/{variant_clean}" + content_type = matched["content_type"] + + def _get_from_s3(): + resp = self.s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key) + return resp["Body"].read() + + try: + data = await asyncio.to_thread(_get_from_s3) + except Exception as e: + logger.error("Ошибка чтения объекта из S3 %s: %s", s3_key, e) + raise NotFoundError("Файл не найден в хранилище") from e + + return data, content_type + + +# Глобальный экземпляр хранилища +storage = PostgresStorage() diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..1800f2e4 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,436 @@ +from __future__ import annotations + +import asyncio +import io +import json +import logging +import random +import signal +import uuid +from typing import Any + +import aio_pika +import asyncpg +import boto3 +from botocore.config import Config +from PIL import UnidentifiedImageError + +from app.broker import ( + DL_ROUTING_KEY, + DLX_NAME, + EXCHANGE_NAME, + QUEUE_NAME, + ROUTING_KEY, + setup_broker_topology, +) +from app.config import ( + BROKER_URL, + DATABASE_URL, + MAX_JOB_RETRIES, + S3_ACCESS_KEY_ID, + S3_BUCKET, + S3_ENDPOINT, + S3_REGION, + S3_SECRET_ACCESS_KEY, + WORKER_CONCURRENCY, + WORKER_LEASE_SECONDS, +) +from app.imaging import process_image_variants + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) +logger = logging.getLogger("messenger.worker") + + +class AttachmentWorker: + def __init__(self) -> None: + self.worker_id = f"worker-{uuid.uuid4().hex[:8]}" + self.pool: asyncpg.Pool | None = None + self.s3_client: Any = None + self._connection: aio_pika.abc.AbstractRobustConnection | None = None + self._channel: aio_pika.abc.AbstractChannel | None = None + self._running = False + self._watchdog_task: asyncio.Task[None] | None = None + + async def init_resources(self) -> None: + logger.info("Initializing worker resources (%s)...", self.worker_id) + # 1. PostgreSQL pool + self.pool = await asyncpg.create_pool( + dsn=DATABASE_URL, + min_size=2, + max_size=WORKER_CONCURRENCY + 2, + command_timeout=10.0, + ) + + # 2. S3 client + s3_cfg = Config( + signature_version="s3v4", + connect_timeout=3, + read_timeout=10, + retries={"max_attempts": 3, "mode": "standard"}, + ) + self.s3_client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY_ID, + aws_secret_access_key=S3_SECRET_ACCESS_KEY, + config=s3_cfg, + ) + + async def _lease_watchdog(self) -> None: + while self._running: + try: + await asyncio.sleep(15.0) + if not self.pool: + continue + async with self.pool.acquire() as conn: + rows = await conn.fetch( + """ + UPDATE attachments + SET status = 'queued', + processing_started_at = NULL, + processing_worker_id = NULL, + updated_at = NOW() + WHERE status = 'processing' + AND processing_started_at < NOW() - ($1 || ' seconds')::interval + RETURNING id + """, + str(WORKER_LEASE_SECONDS), + ) + for r in rows: + logger.warning( + "Lease expired for attachment %s, reverting to queued and republishing", r["id"] + ) + if self._channel and not self._channel.is_closed: + try: + exchange = await self._channel.get_exchange(EXCHANGE_NAME) + payload = { + "schema_version": 1, + "job_id": str(uuid.uuid4()), + "type": "attachment.process.v1", + "attachment_id": str(r["id"]), + "pipeline_version": 1, + "attempt": 1, + } + msg = aio_pika.Message( + body=json.dumps(payload).encode("utf-8"), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + content_type="application/json", + ) + await exchange.publish(msg, routing_key=ROUTING_KEY) + except Exception as pub_err: + logger.error("Failed republish on lease expiration for %s: %s", r["id"], pub_err) + except asyncio.CancelledError: + break + except Exception as exc: + logger.error("Error in lease watchdog: %s", exc) + + async def process_job_message(self, message: aio_pika.abc.AbstractIncomingMessage) -> None: + """Обрабатывает одно задание на генерацию вариантов изображения.""" + async with message.process(requeue=False): + try: + payload = json.loads(message.body.decode("utf-8")) + except Exception as parse_err: + logger.error("Failed to parse job JSON: %s. Acking.", parse_err) + return + + att_id_str = payload.get("attachment_id") + if not att_id_str: + logger.error("Job payload missing attachment_id: %s", payload) + return + + try: + att_id = uuid.UUID(att_id_str) + except ValueError: + logger.error("Invalid attachment_id UUID: %s", att_id_str) + return + + attempt = int(payload.get("attempt", 1)) + + if not self.pool: + logger.error("DB pool not initialized") + return + + async with self.pool.acquire() as conn: + row = await conn.fetchrow( + """ + UPDATE attachments + SET status = 'processing', + processing_started_at = NOW(), + processing_worker_id = $1, + updated_at = NOW() + WHERE id = $2 AND status IN ('queued', 'failed') + RETURNING id, chat_id, filename, content_type, size_bytes, sha256, s3_key, status + """, + self.worker_id, + att_id, + ) + if not row: + current_status = await conn.fetchval( + "SELECT status FROM attachments WHERE id = $1", att_id + ) + logger.info( + "Attachment %s is in state '%s', skipping processing (idempotent no-op).", + att_id, + current_status, + ) + return + + try: + async with asyncio.timeout(30.0): + staging_key = row["s3_key"] + try: + s3_resp = await asyncio.to_thread( + self.s3_client.get_object, + Bucket=S3_BUCKET, + Key=staging_key, + ) + raw_bytes = s3_resp["Body"].read() + except Exception as s3_err: + logger.error("Failed to read staging S3 key %s: %s", staging_key, s3_err) + await self._handle_retryable_error(payload, attempt, str(s3_err)) + return + + # 3. Генерируем варианты через Pillow + try: + variants = await asyncio.to_thread(process_image_variants, raw_bytes) + except (UnidentifiedImageError, ValueError) as img_err: + logger.warning( + "Permanent image processing error for %s: %s", att_id, img_err + ) + await self._fail_attachment(att_id, "invalid_image") + await self._send_to_dlq(payload, f"permanent_image_error: {img_err}") + return + except Exception as unk_err: + logger.error( + "Unexpected processing error for %s: %s", att_id, unk_err + ) + await self._handle_retryable_error(payload, attempt, str(unk_err)) + return + + # 4. Сохраняем оригинал и варианты в постоянные S3-ключи + chat_id = str(row["chat_id"]) + final_original_key = f"chats/{chat_id}/{att_id}/original" + + try: + # Оригинал + await asyncio.to_thread( + self.s3_client.put_object, + Bucket=S3_BUCKET, + Key=final_original_key, + Body=raw_bytes, + ContentType=row["content_type"], + ) + # Варианты + for var in variants: + var_key = f"chats/{chat_id}/{att_id}/{var['name']}" + await asyncio.to_thread( + self.s3_client.put_object, + Bucket=S3_BUCKET, + Key=var_key, + Body=var["data"], + ContentType=var["content_type"], + ) + except Exception as upload_err: + logger.error("Failed uploading variants to S3 for %s: %s", att_id, upload_err) + await self._handle_retryable_error(payload, attempt, str(upload_err)) + return + + # 5. Атомарно обновляем PostgreSQL в 'ready' + variants_metadata = [ + { + "name": v["name"], + "content_type": v["content_type"], + "size_bytes": v["size_bytes"], + "width": v["width"], + "height": v["height"], + "sha256": v["sha256"], + } + for v in variants + ] + + async with self.pool.acquire() as conn: + await conn.execute( + """ + UPDATE attachments + SET status = 'ready', + s3_key = $1, + variants = $2::jsonb, + error_code = NULL, + processing_started_at = NULL, + processing_worker_id = NULL, + updated_at = NOW() + WHERE id = $3 + """, + final_original_key, + json.dumps(variants_metadata), + att_id, + ) + + # 6. Очистка staging-файла в MinIO + if staging_key != final_original_key: + try: + await asyncio.to_thread( + self.s3_client.delete_object, + Bucket=S3_BUCKET, + Key=staging_key, + ) + except Exception as del_err: + logger.warning("Failed to delete staging key %s: %s", staging_key, del_err) + + logger.info("Attachment %s successfully processed into ready with 3 variants.", att_id) + except TimeoutError: + logger.error("Job processing timed out (>30s) for %s. Aborting hung worker task.", att_id) + await self._handle_retryable_error(payload, attempt, "job_deadline_exceeded_30s") + return + + async def _fail_attachment(self, att_id: uuid.UUID, error_code: str) -> None: + """Переводит вложение в статус failed с безопасным машинным кодом.""" + if not self.pool: + return + async with self.pool.acquire() as conn: + await conn.execute( + """ + UPDATE attachments + SET status = 'failed', + error_code = $1, + processing_started_at = NULL, + processing_worker_id = NULL, + updated_at = NOW() + WHERE id = $2 + """, + error_code, + att_id, + ) + + async def _send_to_dlq(self, payload: dict[str, Any], reason: str) -> None: + try: + if self._channel and not self._channel.is_closed: + dlx = await self._channel.get_exchange(DLX_NAME) + dlq_payload = dict(payload, error="permanent_or_exhausted", last_reason=reason) + dlq_msg = aio_pika.Message( + body=json.dumps(dlq_payload).encode("utf-8"), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + content_type="application/json", + ) + await dlx.publish(dlq_msg, routing_key=DL_ROUTING_KEY) + except Exception as dlx_err: + logger.error("Failed publishing to DLQ: %s", dlx_err) + + async def _handle_retryable_error(self, payload: dict[str, Any], attempt: int, reason: str) -> None: + att_id_str = payload.get("attachment_id") + att_id = uuid.UUID(att_id_str) + + if attempt < MAX_JOB_RETRIES: + next_attempt = attempt + 1 + delay = 0.5 * (2 ** attempt) + random.uniform(0.1, 0.5) + logger.warning( + "Retryable error for %s (attempt %d/%d). Retrying in %.2fs: %s", + att_id, attempt, MAX_JOB_RETRIES, delay, reason + ) + if self.pool: + async with self.pool.acquire() as conn: + await conn.execute( + """ + UPDATE attachments + SET status = 'queued', + processing_started_at = NULL, + processing_worker_id = NULL, + updated_at = NOW() + WHERE id = $1 + """, + att_id, + ) + + async def _delayed_republish() -> None: + await asyncio.sleep(delay) + try: + if self._channel and not self._channel.is_closed: + exchange = await self._channel.get_exchange(EXCHANGE_NAME) + new_payload = dict(payload, attempt=next_attempt) + msg = aio_pika.Message( + body=json.dumps(new_payload).encode("utf-8"), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + content_type="application/json", + ) + await exchange.publish(msg, routing_key=ROUTING_KEY) + except Exception as pub_err: + logger.error("Failed delayed republish for %s: %s", att_id, pub_err) + + asyncio.create_task(_delayed_republish()) + else: + logger.error( + "Max retries (%d) exhausted for attachment %s. Moving to DLQ.", + MAX_JOB_RETRIES, att_id + ) + await self._fail_attachment(att_id, "retry_exhausted") + await self._send_to_dlq(payload, f"retry_exhausted: {reason}") + + async def start(self) -> None: + self._running = True + await self.init_resources() + + # Подключение к RabbitMQ с автоматическим reconnect + logger.info("Connecting to RabbitMQ at %s...", BROKER_URL.split("@")[-1]) + while self._running: + try: + self._connection = await aio_pika.connect_robust(BROKER_URL) + break + except Exception as conn_err: + logger.warning("RabbitMQ not ready yet (%s). Retrying in 2s...", conn_err) + await asyncio.sleep(2.0) + + assert self._connection is not None + self._channel = await self._connection.channel() + await self._channel.set_qos(prefetch_count=WORKER_CONCURRENCY) + + _, queue = await setup_broker_topology(self._channel) + + self._watchdog_task = asyncio.create_task(self._lease_watchdog()) + logger.info( + "Worker %s started consuming queue '%s' (prefetch=%d)...", + self.worker_id, QUEUE_NAME, WORKER_CONCURRENCY + ) + + await queue.consume(self.process_job_message) + + async def stop(self) -> None: + logger.info("Stopping worker %s...", self.worker_id) + self._running = False + if self._watchdog_task: + self._watchdog_task.cancel() + if self._channel and not self._channel.is_closed: + await self._channel.close() + if self._connection and not self._connection.is_closed: + await self._connection.close() + if self.pool: + await self.pool.close() + logger.info("Worker stopped.") + + +async def main() -> None: + worker = AttachmentWorker() + loop = asyncio.get_running_loop() + + stop_event = asyncio.Event() + + def _sig_handler() -> None: + logger.info("Received termination signal") + stop_event.set() + + for sig in (signal.SIGINT, signal.SIGTERM): + try: + loop.add_signal_handler(sig, _sig_handler) + except NotImplementedError: + pass + + await worker.start() + await stop_event.wait() + await worker.stop() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..16ddbcd6 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,259 @@ +services: + load_balancer: + image: nginx:1.27-alpine + ports: + - "8080:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"] + interval: 2s + timeout: 2s + retries: 5 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M + + migration: + image: postgres:16-alpine + restart: "no" + environment: + PGPASSWORD: postgres + command: > + sh -c "until pg_isready -h postgres -U postgres -d messenger; do sleep 1; done; + psql -h postgres -U postgres -d messenger -f /docker-entrypoint-initdb.d/init.sql" + volumes: + - ./infra/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M + + api-1: + build: + context: . + dockerfile: Dockerfile + image: highload-messenger:lab5 + stop_grace_period: 15s + environment: + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: "lab" + INSTANCE_ID: "api-1" + SESSION_TTL_SECONDS: "3600" + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/messenger" + SESSION_STORE: "redis" + CACHE_URL: "redis://redis:6379/0" + IDEMPOTENCY_TTL_SECONDS: "86400" + BROKER_URL: "amqp://messenger:messenger_broker_pass@broker:5672/" + S3_ENDPOINT: "http://minio:9000" + S3_REGION: "us-east-1" + S3_BUCKET: "messenger-attachments" + S3_ACCESS_KEY_ID: "minioadmin" + S3_SECRET_ACCESS_KEY: "minioadmin" + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + broker: + condition: service_healthy + migration: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 2s + timeout: 2s + retries: 5 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + + api-2: + build: + context: . + dockerfile: Dockerfile + image: highload-messenger:lab5 + stop_grace_period: 15s + environment: + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: "lab" + INSTANCE_ID: "api-2" + SESSION_TTL_SECONDS: "3600" + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/messenger" + SESSION_STORE: "redis" + CACHE_URL: "redis://redis:6379/0" + IDEMPOTENCY_TTL_SECONDS: "86400" + BROKER_URL: "amqp://messenger:messenger_broker_pass@broker:5672/" + S3_ENDPOINT: "http://minio:9000" + S3_REGION: "us-east-1" + S3_BUCKET: "messenger-attachments" + S3_ACCESS_KEY_ID: "minioadmin" + S3_SECRET_ACCESS_KEY: "minioadmin" + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + minio: + condition: service_healthy + broker: + condition: service_healthy + migration: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')"] + interval: 2s + timeout: 2s + retries: 5 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + + worker: + build: + context: . + dockerfile: Dockerfile + image: highload-messenger:lab5 + command: python -m app.worker + stop_grace_period: 15s + environment: + DATABASE_URL: "postgresql://postgres:postgres@postgres:5432/messenger" + BROKER_URL: "amqp://messenger:messenger_broker_pass@broker:5672/" + S3_ENDPOINT: "http://minio:9000" + S3_REGION: "us-east-1" + S3_BUCKET: "messenger-attachments" + S3_ACCESS_KEY_ID: "minioadmin" + S3_SECRET_ACCESS_KEY: "minioadmin" + WORKER_CONCURRENCY: "4" + WORKER_LEASE_SECONDS: "60" + MAX_JOB_RETRIES: "5" + healthcheck: + test: ["CMD-SHELL", "python -c 'import pika, os; conn = pika.BlockingConnection(pika.URLParameters(os.environ[\"BROKER_URL\"])); conn.close()' || exit 1"] + interval: 5s + timeout: 3s + retries: 3 + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + minio: + condition: service_healthy + migration: + condition: service_completed_successfully + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: messenger + volumes: + - pg_data:/var/lib/postgresql/data + - ./infra/init.sql:/docker-entrypoint-initdb.d/init.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d messenger"] + interval: 2s + timeout: 2s + retries: 5 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + + broker: + image: rabbitmq:3.13-alpine + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_PASS: messenger_broker_pass + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 3s + timeout: 5s + retries: 10 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + + redis: + image: redis:7-alpine + command: ["redis-server", "--appendonly", "yes"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 5 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.25' + memory: 256M + + minio: + image: cleanstart/minio:0.20260330.001845 + user: "0:0" + command: server /data --address ":9000" --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/live"] + interval: 2s + timeout: 2s + retries: 5 + restart: unless-stopped + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + +volumes: + pg_data: + redis_data: + minio_data: + rabbitmq_data: diff --git a/evidence/check_contract.log b/evidence/check_contract.log new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check_contract.log @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/docker_services.log b/evidence/docker_services.log new file mode 100644 index 00000000..a91251b2 --- /dev/null +++ b/evidence/docker_services.log @@ -0,0 +1,9 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab5-api-1-1 highload-messenger:lab5 "python -m uvicorn a…" api-1 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab5-api-2-1 highload-messenger:lab5 "python -m uvicorn a…" api-2 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab5-broker-1 rabbitmq:3.13-alpine "docker-entrypoint.s…" broker 2 minutes ago Up 2 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi2026lab5-load_balancer-1 nginx:1.27-alpine "/docker-entrypoint.…" load_balancer 2 minutes ago Up 2 minutes (healthy) 80/tcp, 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp +mtusi2026lab5-minio-1 cleanstart/minio:0.20260330.001845 "/usr/bin/minio serv…" minio 2 minutes ago Up 2 minutes (healthy) +mtusi2026lab5-postgres-1 postgres:16-alpine "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp +mtusi2026lab5-redis-1 redis:7-alpine "docker-entrypoint.s…" redis 2 minutes ago Up 2 minutes (healthy) 6379/tcp +mtusi2026lab5-worker-1 highload-messenger:lab5 "python -m app.worker" worker 2 minutes ago Up 2 minutes (healthy) 8080/tcp diff --git a/evidence/failover_test.log b/evidence/failover_test.log new file mode 100644 index 00000000..ed0e6c7e --- /dev/null +++ b/evidence/failover_test.log @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 0.87} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..bed1e30e --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.007, + "concurrency": 4, + "successful_requests": 4773, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 318.051, + "successful_latency_ms": { + "p50": 6.603, + "p95": 59.542, + "p99": 80.594 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence_test.log b/evidence/persistence_test.log new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence_test.log @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/round_robin.log b/evidence/round_robin.log new file mode 100644 index 00000000..ce75226e --- /dev/null +++ b/evidence/round_robin.log @@ -0,0 +1,10 @@ +x-instance-id: api-2 +x-instance-id: api-1 +x-instance-id: api-2 +x-instance-id: api-1 +x-instance-id: api-2 +x-instance-id: api-1 +x-instance-id: api-2 +x-instance-id: api-1 +x-instance-id: api-2 +x-instance-id: api-1 diff --git a/evidence/smoke_test.log b/evidence/smoke_test.log new file mode 100644 index 00000000..df05fe26 --- /dev/null +++ b/evidence/smoke_test.log @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student_tests.log b/evidence/student_tests.log new file mode 100644 index 00000000..b963c72c --- /dev/null +++ b/evidence/student_tests.log @@ -0,0 +1,51 @@ +BASE_URL="http://localhost:8080" .venv/bin/pytest tests_student/ -v +/home/ilyas/projects/labs_highload/mtusi2026lab5/.venv/lib/python3.15/site-packages/pytest_asyncio/plugin.py:300: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset. +The event loop scope for asynchronous fixtures will default to the "fixture" caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to "function" scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: "function", "class", "module", "package", "session" + + warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET)) +============================= test session starts ============================== +platform linux -- Python 3.15.2, pytest-9.2.0, pluggy-1.7.0 -- /home/ilyas/projects/labs_highload/mtusi2026lab5/.venv/bin/python3 +cachedir: .pytest_cache +rootdir: /home/ilyas/projects/labs_highload/mtusi2026lab5 +configfile: pytest.ini +plugins: anyio-4.16.0, asyncio-1.4.1.post1 +asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collecting ... collected 35 items + +tests_student/tests.py::test_invalid_json_and_extra_fields PASSED [ 2%] +tests_student/tests.py::test_outsider_chat_access_is_404 PASSED [ 5%] +tests_student/tests.py::test_empty_and_whitespace_message PASSED [ 8%] +tests_student/tests.py::test_duplicate_username_conflict PASSED [ 11%] +tests_student/tests.py::test_wrong_password_and_bad_basic_auth PASSED [ 14%] +tests_student/tests.py::test_logout_and_revocation PASSED [ 17%] +tests_student/tests.py::test_multi_page_cursor_pagination PASSED [ 20%] +tests_student/tests.py::test_concurrency_race_condition PASSED [ 22%] +tests_student/tests.py::test_password_whitespace_rejected PASSED [ 25%] +tests_student/tests.py::test_auth_sessions_with_body_rejected PASSED [ 28%] +tests_student/tests.py::test_unsupported_media_type_415 PASSED [ 31%] +tests_student/tests.py::test_nul_byte_in_strings_handled PASSED [ 34%] +tests_student/tests.py::test_basic_auth_invalid_base64_returns_401 PASSED [ 37%] +tests_student/tests.py::test_cross_chat_cursor_rejected_400 PASSED [ 40%] +tests_student/tests.py::test_utf8_password_support PASSED [ 42%] +tests_student/tests.py::test_health_readiness_and_liveness PASSED [ 45%] +tests_student/tests.py::test_instance_id_header_present PASSED [ 48%] +tests_student/tests.py::test_idempotency_key_validation PASSED [ 51%] +tests_student/tests.py::test_idempotency_replay_and_conflict PASSED [ 54%] +tests_student/tests.py::test_idempotency_acl_check_on_replay PASSED [ 57%] +tests_student/tests.py::test_invalid_cursor_date_returns_400 PASSED [ 60%] +tests_student/tests.py::test_strict_types_and_json_parsing PASSED [ 62%] +tests_student/tests.py::test_chat_creation_with_unknown_participant_is_400 PASSED [ 65%] +tests_student/tests.py::test_password_with_colons_and_utf8 PASSED [ 68%] +tests_student/tests.py::test_keyset_pagination_returns_next_cursor PASSED [ 71%] +tests_student/tests.py::test_attachment_upload_and_download_flow PASSED [ 74%] +tests_student/tests.py::test_attachment_unsupported_media_type_and_magic_bytes PASSED [ 77%] +tests_student/tests.py::test_attachment_filename_validation PASSED [ 80%] +tests_student/tests.py::test_attachment_payload_too_large_and_pixel_bomb PASSED [ 82%] +tests_student/tests.py::test_attachment_acl_and_isolation PASSED [ 85%] +tests_student/tests.py::test_message_linking_attachments PASSED [ 88%] +tests_student/tests.py::test_message_linking_failed_attachment_rejected PASSED [ 91%] +tests_student/tests.py::test_attachment_orphan_compensation PASSED [ 94%] +tests_student/tests.py::test_private_bucket_outside_api PASSED [ 97%] +tests_student/tests.py::test_concurrent_uploads_memory_limit PASSED [100%] + +============================= 35 passed in 37.84s ============================== diff --git a/evidence/validate_openapi.log b/evidence/validate_openapi.log new file mode 100644 index 00000000..37e81884 --- /dev/null +++ b/evidence/validate_openapi.log @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/init.sql b/infra/init.sql new file mode 100644 index 00000000..8667644f --- /dev/null +++ b/infra/init.sql @@ -0,0 +1,91 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_users_username ON users (username); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_chats_cursor ON chats (created_at ASC, id ASC); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user_id ON chat_members (user_id); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text BYTEA NOT NULL, + attachment_ids UUID[] NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL +); + +ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}'; + +CREATE INDEX IF NOT EXISTS idx_messages_cursor ON messages (chat_id, created_at ASC, id ASC); + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(50) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 VARCHAR(64) NOT NULL, + s3_key VARCHAR(512) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'queued', + error_code VARCHAR(100), + variants JSONB NOT NULL DEFAULT '[]'::jsonb, + processing_started_at TIMESTAMPTZ, + processing_worker_id TEXT, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS variants JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS processing_started_at TIMESTAMPTZ; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS processing_worker_id TEXT; +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; + +CREATE INDEX IF NOT EXISTS idx_attachments_chat_id ON attachments (chat_id, created_at ASC); +CREATE INDEX IF NOT EXISTS idx_attachments_owner_id ON attachments (owner_id); +CREATE INDEX IF NOT EXISTS idx_attachments_lease ON attachments (status, processing_started_at) WHERE status = 'processing'; + +CREATE TABLE IF NOT EXISTS outbox_jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + event_type VARCHAR(64) NOT NULL DEFAULT 'attachment.process.v1', + payload JSONB NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + attempt INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_outbox_pending ON outbox_jobs (status, created_at) WHERE status = 'pending'; + + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key VARCHAR(128) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + request_hash VARCHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created_at ON idempotency_keys (created_at); diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..4920ced3 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,72 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" upstream: $upstream_addr ' + 'upstream_status: $upstream_status ' + 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log warn; + + sendfile on; + keepalive_timeout 65; + client_max_body_size 15M; + + # DNS-сервер Docker для динамического обновления IP-адресов контейнеров + resolver 127.0.0.11 valid=2s ipv6=off; + + upstream api_backend { + zone api_backend 64k; + server api-1:8080 resolve max_fails=1 fail_timeout=2s; + server api-2:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + server_name _; + + # Быстрый таймаут обнаружения недоступной реплики + proxy_connect_timeout 1s; + proxy_read_timeout 15s; + proxy_send_timeout 15s; + + # Базовая политика переключения: безопасно для идемпотентных запросов + # Неидемпотентные POST не повторяем + proxy_next_upstream error timeout http_502 http_503; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 15s; + + location ~ ^/api/v1/chats/[^/]+/messages$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-Id $http_x_request_id; + + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-Id $http_x_request_id; + } + } +} diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..e279b86b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +pythonpath = . +asyncio_mode = auto +python_files = test_*.py *_test.py tests.py test.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c9e6cdb7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +fastapi==0.141.1 +uvicorn[standard]==0.52.4 +pydantic==2.13.5 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 +pytest==9.1.1 +pytest-asyncio==1.4.0 +httpx==0.28.1 +boto3==1.43.123 +pillow==13.0.0 +aio-pika>=9.4.0 +pika>=1.3.2 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..0cc128ec --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Перезапуск всех сервисов API и хранилищ с сохранением volumes +docker compose -f "$PROJECT_DIR/compose.yaml" restart diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2d4a6b36 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if docker compose -f "$PROJECT_DIR/compose.yaml" stop "$TARGET_INSTANCE_ID" 2>/dev/null; then + exit 0 +fi + +docker stop "$TARGET_INSTANCE_ID" >/dev/null diff --git a/tests_student/conftest.py b/tests_student/conftest.py new file mode 100644 index 00000000..d8a948d2 --- /dev/null +++ b/tests_student/conftest.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import os +import pytest +from httpx import ASGITransport, AsyncClient + + +@pytest.fixture +async def client(): + base_url = os.getenv("BASE_URL", "http://localhost:8080") + if base_url.startswith("http://testserver"): + from app.main import app + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as ac: + yield ac + else: + async with AsyncClient(base_url=base_url, timeout=15.0) as ac: + yield ac diff --git a/tests_student/tests.py b/tests_student/tests.py new file mode 100644 index 00000000..6f853e1c --- /dev/null +++ b/tests_student/tests.py @@ -0,0 +1,1411 @@ +from __future__ import annotations + +import asyncio +import base64 +import uuid +import httpx + +import pytest +from httpx import AsyncClient + + +async def register_and_login(client: AsyncClient, prefix: str = "u") -> dict: + username = f"{prefix}_{uuid.uuid4().hex[:12]}" + password = f"P@ssword_{uuid.uuid4().hex[:8]}" + body = { + "username": username, + "display_name": f"User {prefix}", + "password": password, + } + u_res = await client.post("/api/v1/users", json=body) + assert u_res.status_code == 201, u_res.text + user = u_res.json() + + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + s_res = await client.post( + "/api/v1/auth/sessions", headers={"Authorization": f"Basic {cred}"} + ) + assert s_res.status_code == 201, s_res.text + session = s_res.json() + + return { + "user": user, + "username": username, + "password": password, + "token": session["token"], + "headers": {"Authorization": f"Bearer {session['token']}"}, + } + + +@pytest.mark.asyncio +async def test_invalid_json_and_extra_fields(client: AsyncClient): + """Проверка невалидного JSON, лишних полей, перевода строки и валидации курсора.""" + # 1. Невалидный синтаксис JSON -> 400 + res_malformed = await client.post( + "/api/v1/users", + content="not a valid json {", + headers={"Content-Type": "application/json"}, + ) + assert res_malformed.status_code == 400 + err_malformed = res_malformed.json() + assert err_malformed["error"]["code"] == "invalid_request" + assert "request_id" in err_malformed["error"] + assert ( + res_malformed.headers.get("X-Request-Id") + == err_malformed["error"]["request_id"] + ) + + # 2. Лишние поля (additionalProperties: false) -> 400 + res_extra = await client.post( + "/api/v1/users", + json={ + "username": f"u_{uuid.uuid4().hex[:8]}", + "display_name": "test_user", + "password": "ValidPassword123!", + "unexpected_field": "hacker_payload", + }, + ) + assert res_extra.status_code == 400 + assert res_extra.json()["error"]["code"] == "invalid_request" + + # 3. Запрет завершающего перевода строки в username ("abc\n") -> 400 + res_newline = await client.post( + "/api/v1/users", + json={ + "username": "abc\n", + "display_name": "test_newline", + "password": "ValidPassword123!", + }, + ) + assert res_newline.status_code == 400 + assert res_newline.json()["error"]["code"] == "invalid_request" + + # 4. Валидация курсора: не-dict JSON (W10= -> []) и пустой cursor= -> 400 + actor = await register_and_login(client, "cur_val") + headers = actor["headers"] + + # cursor=W10= (декодируется в list [], а не dict) + res_w10 = await client.get("/api/v1/chats?cursor=W10=", headers=headers) + assert res_w10.status_code == 400 + assert res_w10.json()["error"]["code"] == "invalid_request" + + # пустой cursor= + res_empty_cursor = await client.get("/api/v1/chats?cursor=", headers=headers) + assert res_empty_cursor.status_code == 400 + assert res_empty_cursor.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_outsider_chat_access_is_404(client: AsyncClient): + """Проверка того, что доступ к чужому чату возвращает строго 404 (а не 403).""" + alice = await register_and_login(client, "alice") + bob = await register_and_login(client, "bob") + eve = await register_and_login(client, "eve") + + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={ + "title": "Private Chat", + "member_ids": [alice["user"]["id"], bob["user"]["id"]], + }, + ) + assert chat_res.status_code == 201 + chat_id = chat_res.json()["id"] + + # Ева пытается получить чужой чат -> 404 + res_get = await client.get(f"/api/v1/chats/{chat_id}", headers=eve["headers"]) + assert res_get.status_code == 404 + assert res_get.json()["error"]["code"] == "not_found" + + # Ева пытается отправить сообщение в чужой чат -> 404 + res_send = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**eve["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "Шпионское сообщение"}, + ) + assert res_send.status_code == 404 + assert res_send.json()["error"]["code"] == "not_found" + + # Ева пытается прочитать сообщения чужого чата -> 404 + res_list = await client.get( + f"/api/v1/chats/{chat_id}/messages", headers=eve["headers"] + ) + assert res_list.status_code == 404 + assert res_list.json()["error"]["code"] == "not_found" + + +@pytest.mark.asyncio +async def test_empty_and_whitespace_message(client: AsyncClient): + """Проверка запрета отправки пустого текста или текста из одних пробелов (400).""" + alice = await register_and_login(client, "msg_user") + chat = ( + await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Chat", "member_ids": [alice["user"]["id"]]}, + ) + ).json() + + # Пустая строка -> 400 + res_empty = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": ""}, + ) + assert res_empty.status_code == 400 + assert res_empty.json()["error"]["code"] == "invalid_request" + + # Одни пробелы и переводы строк -> 400 + res_whitespace = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": " \n\t "}, + ) + assert res_whitespace.status_code == 400 + assert res_whitespace.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_duplicate_username_conflict(client: AsyncClient): + """Проверка запрета дублирования username (409 conflict).""" + uname = f"uniq_{uuid.uuid4().hex[:10]}" + body = { + "username": uname, + "display_name": "Первый", + "password": "ValidPassword123!", + } + + res1 = await client.post("/api/v1/users", json=body) + assert res1.status_code == 201 + + res2 = await client.post( + "/api/v1/users", + json={ + "username": uname, + "display_name": "Второй с тем же ником", + "password": "AnotherPassword123!", + }, + ) + assert res2.status_code == 409 + assert res2.json()["error"]["code"] == "conflict" + + +@pytest.mark.asyncio +async def test_wrong_password_and_bad_basic_auth(client: AsyncClient): + """Проверка ошибки авторизации 401 и корректного заголовка WWW-Authenticate.""" + alice = await register_and_login(client, "auth_test") + + # Неверный пароль -> 401 + bad_cred = base64.b64encode(f"{alice['username']}:WrongPassword123!".encode()).decode() + res_bad = await client.post( + "/api/v1/auth/sessions", headers={"Authorization": f"Basic {bad_cred}"} + ) + assert res_bad.status_code == 401 + assert res_bad.json()["error"]["code"] == "unauthorized" + assert "basic" in res_bad.headers.get("WWW-Authenticate", "").lower() + + # Несуществующий юзер -> 401 + ghost_cred = base64.b64encode(b"ghost_user_123:WrongPassword123!").decode() + res_ghost = await client.post( + "/api/v1/auth/sessions", headers={"Authorization": f"Basic {ghost_cred}"} + ) + assert res_ghost.status_code == 401 + assert "basic" in res_ghost.headers.get("WWW-Authenticate", "").lower() + + +@pytest.mark.asyncio +async def test_logout_and_revocation(client: AsyncClient): + """Проверка отзыва сессии (logout 204) и 401 при обращении с отозванным токеном.""" + alice = await register_and_login(client, "logout_test") + + # Проверяем, что токен работает + me_res = await client.get("/api/v1/users/me", headers=alice["headers"]) + assert me_res.status_code == 200 + + # Делаем logout -> 204 + logout_res = await client.delete( + "/api/v1/auth/sessions/current", headers=alice["headers"] + ) + assert logout_res.status_code == 204 + + # Теперь тот же токен должен возвращать 401 + me_after = await client.get("/api/v1/users/me", headers=alice["headers"]) + assert me_after.status_code == 401 + assert me_after.json()["error"]["code"] == "unauthorized" + assert "bearer" in me_after.headers.get("WWW-Authenticate", "").lower() + + # Повторный logout с тем же отозванным токеном -> 401 + logout_repeat = await client.delete( + "/api/v1/auth/sessions/current", headers=alice["headers"] + ) + assert logout_repeat.status_code == 401 + + +@pytest.mark.asyncio +async def test_multi_page_cursor_pagination(client: AsyncClient): + """Проверка обхода истории сообщений через keyset-курсор на нескольких страницах.""" + alice = await register_and_login(client, "pager") + chat = ( + await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Pagination Chat", "member_ids": [alice["user"]["id"]]}, + ) + ).json() + + # Отправляем 15 сообщений + sent_ids = [] + for i in range(15): + msg = ( + await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": f"Сообщение {i:02d}"}, + ) + ).json() + sent_ids.append(msg["id"]) + + # Читаем страницами по 4 сообщения + gathered_ids = [] + cursor = None + pages_count = 0 + + while True: + url = f"/api/v1/chats/{chat['id']}/messages?limit=4" + if cursor: + url += f"&cursor={cursor}" + + res = await client.get(url, headers=alice["headers"]) + assert res.status_code == 200 + data = res.json() + pages_count += 1 + items = data["items"] + for item in items: + gathered_ids.append(item["id"]) + + cursor = data.get("next_cursor") + if not cursor: + break + + # Должны быть ровно 15 сообщений в строго том же порядке, без дублей и пропусков + assert len(gathered_ids) == 15 + assert gathered_ids == sent_ids + assert pages_count == 4 + + +@pytest.mark.asyncio +async def test_concurrency_race_condition(client: AsyncClient): + """Проверка устойчивости к параллельным гонкам: 10 клиентов одновременно шлют 100 сообщений.""" + # Создаём 10 пользователей + users = [await register_and_login(client, f"u{i}") for i in range(10)] + all_user_ids = [u["user"]["id"] for u in users] + + # Первый пользователь создаёт общий чат со всеми 10 участниками + chat = ( + await client.post( + "/api/v1/chats", + headers=users[0]["headers"], + json={"title": "Massive Concurrent Chat", "member_ids": all_user_ids}, + ) + ).json() + + chat_id = chat["id"] + + # Отправляем по 10 сообщений от каждого пользователя параллельно (всего 100) + async def sender(user_dict: dict, count: int): + for i in range(count): + await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**user_dict["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": f"Msg {i} from {user_dict['username']}"}, + ) + + await asyncio.gather(*(sender(u, 10) for u in users)) + + # Считываем все сообщения из чата + all_msgs = [] + cursor = None + while True: + url = f"/api/v1/chats/{chat_id}/messages?limit=50" + if cursor: + url += f"&cursor={cursor}" + res = await client.get(url, headers=users[0]["headers"]) + assert res.status_code == 200 + data = res.json() + all_msgs.extend(data["items"]) + cursor = data.get("next_cursor") + if not cursor: + break + + assert len(all_msgs) == 100 + ids = [m["id"] for m in all_msgs] + # Все ID должны быть уникальными + assert len(ids) == len(set(ids)) + + +@pytest.mark.asyncio +async def test_password_whitespace_rejected(client: httpx.AsyncClient): + """Пароль из одних пробелов должен строго отклоняться с 400 Invalid Request.""" + u = f"space_user_{uuid.uuid4().hex[:6]}" + res = await client.post( + "/api/v1/users", + json={"username": u, "display_name": "Test User", "password": " " * 12}, + ) + assert res.status_code == 400 + data = res.json() + assert data["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_auth_sessions_with_body_rejected(client: httpx.AsyncClient): + """POST /api/v1/auth/sessions с телом запроса должен отклоняться с 400 Invalid Request.""" + u = f"body_user_{uuid.uuid4().hex[:6]}" + p = "StrongPassword123!" + await client.post( + "/api/v1/users", + json={"username": u, "display_name": "Body Test", "password": p}, + ) + cred = base64.b64encode(f"{u}:{p}".encode()).decode() + res = await client.post( + "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {cred}"}, + json={"extra": "forbidden"}, + ) + assert res.status_code == 400 + data = res.json() + assert data["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_unsupported_media_type_415(client: httpx.AsyncClient): + """Запрос с неподдерживаемым Content-Type (text/plain) должен возвращать 415 Unsupported Media Type.""" + res = await client.post( + "/api/v1/users", + content='{"username":"ct_test","display_name":"d","password":"Password123456!"}', + headers={"Content-Type": "text/plain"}, + ) + assert res.status_code == 415 + data = res.json() + assert data["error"]["code"] == "unsupported_media_type" + + +@pytest.mark.asyncio +async def test_nul_byte_in_strings_handled(client: httpx.AsyncClient): + usr = await register_and_login(client, f"nul_msg_{uuid.uuid4().hex[:4]}") + chat = (await client.post("/api/v1/chats", headers=usr["headers"], json={"title": "NulChat", "member_ids": [usr["user"]["id"]]})).json() + test_text = "left\u0000right" + msg_res = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers={**usr["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": test_text}, + ) + assert msg_res.status_code == 201 + assert msg_res.json()["text"] == test_text + + list_res = await client.get(f"/api/v1/chats/{chat['id']}/messages", headers=usr["headers"]) + assert list_res.status_code == 200 + assert any(m["text"] == test_text for m in list_res.json()["items"]) + + +@pytest.mark.asyncio +async def test_basic_auth_invalid_base64_returns_401(client: httpx.AsyncClient): + """Authorization: Basic с битым base64 или не-ASCII символами возвращает 401, а не 500.""" + res = await client.post( + "/api/v1/auth/sessions", + headers={"Authorization": b"Basic \xe9"}, + ) + assert res.status_code == 401 + assert "Basic" in res.headers.get("www-authenticate", "") + + +@pytest.mark.asyncio +async def test_cross_chat_cursor_rejected_400(client: httpx.AsyncClient): + """Курсор от одного чата при попытке использовать в другом чате должен возвращать 400.""" + u = await register_and_login(client, f"cc_{uuid.uuid4().hex[:4]}") + chat1 = (await client.post("/api/v1/chats", headers=u["headers"], json={"title": "C1", "member_ids": [u["user"]["id"]]})).json() + chat2 = (await client.post("/api/v1/chats", headers=u["headers"], json={"title": "C2", "member_ids": [u["user"]["id"]]})).json() + + # Отправляем 2 сообщения в чат 1 + await client.post(f"/api/v1/chats/{chat1['id']}/messages", headers={**u["headers"], "Idempotency-Key": uuid.uuid4().hex}, json={"text": "m1"}) + await client.post(f"/api/v1/chats/{chat1['id']}/messages", headers={**u["headers"], "Idempotency-Key": uuid.uuid4().hex}, json={"text": "m2"}) + + # Получаем страницу с курсором из чата 1 + p1 = (await client.get(f"/api/v1/chats/{chat1['id']}/messages?limit=1", headers=u["headers"])).json() + cursor1 = p1.get("next_cursor") + assert cursor1 is not None + + # Пытаемся применить курсор чата 1 к чату 2 -> 400! + res = await client.get(f"/api/v1/chats/{chat2['id']}/messages?cursor={cursor1}", headers=u["headers"]) + assert res.status_code == 400 + assert res.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_utf8_password_support(client: httpx.AsyncClient): + """Пароли с символами UTF-8 и emoji корректно регистрируются и аутентифицируются.""" + u = f"utf_{uuid.uuid4().hex[:6]}" + p = "СложныйПароль123!🔒" + res1 = await client.post( + "/api/v1/users", + json={"username": u, "display_name": "UTF User", "password": p}, + ) + assert res1.status_code == 201 + + cred = base64.b64encode(f"{u}:{p}".encode("utf-8")).decode("utf-8") + res2 = await client.post( + "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {cred}"}, + ) + assert res2.status_code == 201 + assert "token" in res2.json() + + +@pytest.mark.asyncio +async def test_health_readiness_and_liveness(client: httpx.AsyncClient): + """Проверка доступности /health/live и готовности хранилищ /health/ready.""" + r_live = await client.get("/health/live") + assert r_live.status_code == 200 + assert r_live.json()["status"] == "ok" + + r_ready = await client.get("/health/ready") + assert r_ready.status_code == 200 + assert r_ready.json()["status"] == "ok" + + +@pytest.mark.asyncio +async def test_instance_id_header_present(client: httpx.AsyncClient): + """Каждый ответ обязан содержать непустой заголовок X-Instance-Id.""" + res = await client.get("/health/live") + assert res.status_code == 200 + assert "x-instance-id" in res.headers + assert len(res.headers["x-instance-id"]) > 0 + + res_err = await client.get("/api/v1/users/me") + assert res_err.status_code == 401 + assert "x-instance-id" in res_err.headers + assert len(res_err.headers["x-instance-id"]) > 0 + + +@pytest.mark.asyncio +async def test_idempotency_key_validation(client: httpx.AsyncClient): + """POST /messages без Idempotency-Key или с некорректным форматом возвращает 400.""" + alice = await register_and_login(client, f"idem_{uuid.uuid4().hex[:6]}") + chat = (await client.post("/api/v1/chats", headers=alice["headers"], json={"title": "IdemChat", "member_ids": [alice["user"]["id"]]})).json() + chat_id = chat["id"] + + # Без заголовка + res1 = await client.post(f"/api/v1/chats/{chat_id}/messages", headers=alice["headers"], json={"text": "Тест"}) + assert res1.status_code == 400 + assert res1.json()["error"]["code"] == "invalid_request" + + # С некорректным форматом + res2 = await client.post(f"/api/v1/chats/{chat_id}/messages", headers={**alice["headers"], "Idempotency-Key": "invalid key with spaces"}, json={"text": "Тест"}) + assert res2.status_code == 400 + assert res2.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_idempotency_replay_and_conflict(client: httpx.AsyncClient): + """Повтор с тем же телом возвращает исходный 201; с другим телом возвращает 409.""" + alice = await register_and_login(client, f"idem2_{uuid.uuid4().hex[:6]}") + chat = (await client.post("/api/v1/chats", headers=alice["headers"], json={"title": "IdemChat2", "member_ids": [alice["user"]["id"]]})).json() + chat_id = chat["id"] + key = uuid.uuid4().hex + + # Первый запрос -> 201 + res1 = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": key}, + json={"text": "Первое сообщение"}, + ) + assert res1.status_code == 201 + msg1 = res1.json() + + # Повтор с тем же телом -> 201 и то же сообщение + res2 = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": key}, + json={"text": "Первое сообщение"}, + ) + assert res2.status_code == 201 + assert res2.json() == msg1 + + # Повтор с тем же ключом, но другим телом -> 409 Conflict + res3 = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": key}, + json={"text": "Другое сообщение с тем же ключом"}, + ) + assert res3.status_code == 409 + assert res3.json()["error"]["code"] == "conflict" + + +@pytest.mark.asyncio +async def test_idempotency_acl_check_on_replay(client: httpx.AsyncClient): + """ + Проверка ACL перед возвратом сохранённого результата идемпотентности. + Если пользователь удалён из участников чата, повторный POST с тем же Idempotency-Key + и тем же телом обязан вернуть 404 (а не 201). + """ + import subprocess + + alice = await register_and_login(client, f"acl_{uuid.uuid4().hex[:6]}") + chat = ( + await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "ACL Test Chat", "member_ids": [alice["user"]["id"]]}, + ) + ).json() + chat_id = chat["id"] + user_id = alice["user"]["id"] + key = f"acl_key_{uuid.uuid4().hex[:12]}" + + # 1. Отправляем сообщение первый раз -> 201 + res1 = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": key}, + json={"text": "Тест проверки ACL"}, + ) + assert res1.status_code == 201 + + # 2. Удаляем членство отправителя напрямую в БД (изолированная инъекция проверки ACL) + cmd = [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "psql", + "-U", + "postgres", + "-d", + "messenger", + "-c", + f"DELETE FROM chat_members WHERE chat_id = '{chat_id}' AND user_id = '{user_id}';", + ] + sub_res = subprocess.run(cmd, capture_output=True, text=True) + assert sub_res.returncode == 0, f"Ошибка удаления членства: {sub_res.stderr}" + + # 3. GET истории уже обязан возвращать 404 + res_get = await client.get(f"/api/v1/chats/{chat_id}/messages", headers=alice["headers"]) + assert res_get.status_code == 404 + + # 4. Повтор POST с тем же телом и тем же ключом ОБЯЗАН вернуть 404 (ACL проверяется до возврата из idempotency_keys) + res_repeat = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": key}, + json={"text": "Тест проверки ACL"}, + ) + assert res_repeat.status_code == 404, f"Ожидался 404, получен {res_repeat.status_code}: {res_repeat.text}" + assert res_repeat.json()["error"]["code"] == "not_found" + + # 5. Повтор POST с тем же ключом и ДРУГИМ телом: ACL проверяется до сравнения тела -> обязан вернуть 404 (а не 409) + res_repeat_diff = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": key}, + json={"text": "Другой текст сообщения"}, + ) + assert res_repeat_diff.status_code == 404, f"Ожидался 404, получен {res_repeat_diff.status_code}: {res_repeat_diff.text}" + assert res_repeat_diff.json()["error"]["code"] == "not_found" + + +@pytest.mark.asyncio +async def test_invalid_cursor_date_returns_400(client: httpx.AsyncClient): + """ + Некорректная дата курсора (например, 13-й месяц 2026-13-01T00:00:00.000000Z) + обязана возвращать 400 invalid_request (а не 500 internal_error). + """ + def encode_cursor(data: dict) -> str: + import json + raw = json.dumps(data, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=") + + alice = await register_and_login(client, f"bad_cur_{uuid.uuid4().hex[:6]}") + chat = ( + await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "CurChat", "member_ids": [alice["user"]["id"]]}, + ) + ).json() + chat_id = chat["id"] + + # 1. Курсор сообщений с невалидной датой (месяц 13) + bad_cur_msg = encode_cursor( + { + "type": "messages", + "chat_id": chat_id, + "created_at": "2026-13-01T00:00:00.000000Z", + "id": str(uuid.uuid4()), + } + ) + res_m = await client.get( + f"/api/v1/chats/{chat_id}/messages?cursor={bad_cur_msg}", + headers=alice["headers"], + ) + assert res_m.status_code == 400 + assert res_m.json()["error"]["code"] == "invalid_request" + + # 2. Невалидная строка формата m|chat_id|... в base64 + raw_str = f"m|{chat_id}|2026-13-01T00:00:00.000000Z|" + raw_cur = base64.urlsafe_b64encode(raw_str.encode()).decode() + res_raw = await client.get( + f"/api/v1/chats/{chat_id}/messages?cursor={raw_cur}", + headers=alice["headers"], + ) + assert res_raw.status_code == 400 + assert res_raw.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_strict_types_and_json_parsing(client: httpx.AsyncClient): + """ + Запрет преобразования неверных типов ({"text": 123} -> 400) + и требование полного разбора JSON ({"text":"x"}garbage -> 400). + """ + alice = await register_and_login(client, f"strict_{uuid.uuid4().hex[:6]}") + chat = ( + await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "StrictChat", "member_ids": [alice["user"]["id"]]}, + ) + ).json() + chat_id = chat["id"] + + # 1. {"text": 123} вместо строки -> 400 + res_num = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex, "Content-Type": "application/json"}, + content='{"text": 123}', + ) + assert res_num.status_code == 400 + assert res_num.json()["error"]["code"] == "invalid_request" + + # 2. {"text": "x"}garbage -> 400 + res_garbage = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex, "Content-Type": "application/json"}, + content='{"text": "x"}garbage', + ) + assert res_garbage.status_code == 400 + assert res_garbage.json()["error"]["code"] == "invalid_request" + + + + +@pytest.mark.asyncio +async def test_chat_creation_with_unknown_participant_is_400(client: httpx.AsyncClient): + """Неизвестный участник при создании чата обязан возвращать 400 (не 404).""" + alice = await register_and_login(client, f"unk_{uuid.uuid4().hex[:6]}") + non_existent_uuid = str(uuid.uuid4()) + + res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Chat With Ghost", "member_ids": [alice["user"]["id"], non_existent_uuid]}, + ) + assert res.status_code == 400 + assert res.json()["error"]["code"] == "invalid_request" + + +@pytest.mark.asyncio +async def test_password_with_colons_and_utf8(client: httpx.AsyncClient): + """Пароль с двоеточиями и UTF-8 символами работает при регистрации и входе.""" + username = f"col_{uuid.uuid4().hex[:8]}" + password = "Пример:пароля-12345:секрет:🔒" + + res_reg = await client.post( + "/api/v1/users", + json={"username": username, "display_name": "Colon User", "password": password}, + ) + assert res_reg.status_code == 201 + + cred = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + res_login = await client.post( + "/api/v1/auth/sessions", + headers={"Authorization": f"Basic {cred}"}, + ) + assert res_login.status_code == 201 + assert "token" in res_login.json() + + +@pytest.mark.asyncio +async def test_keyset_pagination_returns_next_cursor(client: httpx.AsyncClient): + """ + При наличии записей за пределами limit возвращается next_cursor. + Повторный запрос с cursor возвращает следующие элементы. + """ + alice = await register_and_login(client, f"page_{uuid.uuid4().hex[:6]}") + c1 = (await client.post("/api/v1/chats", headers=alice["headers"], json={"title": "Chat1", "member_ids": [alice["user"]["id"]]})).json() + c2 = (await client.post("/api/v1/chats", headers=alice["headers"], json={"title": "Chat2", "member_ids": [alice["user"]["id"]]})).json() + + # Запрос с limit=1 обязан вернуть первый чат и НЕПУСТОЙ next_cursor + res_p1 = await client.get("/api/v1/chats?limit=1", headers=alice["headers"]) + assert res_p1.status_code == 200 + data_p1 = res_p1.json() + assert len(data_p1["items"]) == 1 + cursor = data_p1["next_cursor"] + assert cursor is not None and len(cursor) > 0 + + # Запрос со вторым курсором обязан вернуть второй чат + res_p2 = await client.get(f"/api/v1/chats?limit=1&cursor={cursor}", headers=alice["headers"]) + assert res_p2.status_code == 200 + data_p2 = res_p2.json() + assert len(data_p2["items"]) == 1 + assert data_p2["items"][0]["id"] != data_p1["items"][0]["id"] + + +async def wait_attachment_ready( + client: httpx.AsyncClient, headers: dict, att_id: str, timeout: float = 30.0 +) -> dict: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + res = await client.get(f"/api/v1/attachments/{att_id}", headers=headers) + assert res.status_code == 200, res.text + att = res.json() + assert att["status"] != "failed", f"Attachment entered failed status: {att}" + if att["status"] == "ready": + return att + await asyncio.sleep(0.2) + pytest.fail(f"Attachment {att_id} did not become ready within {timeout}s") + + +@pytest.mark.asyncio +async def test_attachment_upload_and_download_flow(client: httpx.AsyncClient): + """Проверка асинхронного цикла 202 queued -> ready, thumbnail, crop, optimized.""" + import hashlib + import io + from PIL import Image + + alice = await register_and_login(client, "att_a") + bob = await register_and_login(client, "att_b") + + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Attach Chat", "member_ids": [alice["user"]["id"], bob["user"]["id"]]}, + ) + assert chat_res.status_code == 201 + chat_id = chat_res.json()["id"] + + # 1. Загрузка PNG + with open("tests/fixtures/sample.png", "rb") as f: + png_data = f.read() + + png_sha = hashlib.sha256(png_data).hexdigest() + up_res = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + assert up_res.status_code == 202, up_res.text + att_png = up_res.json() + assert att_png["chat_id"] == chat_id + assert att_png["owner_id"] == alice["user"]["id"] + assert att_png["filename"] == "sample.png" + assert att_png["content_type"] == "image/png" + assert att_png["size_bytes"] == len(png_data) + assert att_png["sha256"] == png_sha + assert att_png["status"] == "queued" + assert att_png["variants"] == [] + assert att_png["error_code"] is None + + # До готовности скачивание контента возвращает 409 + down_early = await client.get( + f"/api/v1/attachments/{att_png['id']}/content", + headers=bob["headers"], + ) + if down_early.status_code != 200: + assert down_early.status_code == 409 + + # 2. Ожидание перехода в ready + ready_att = await wait_attachment_ready(client, bob["headers"], att_png["id"]) + assert ready_att["status"] == "ready" + assert ready_att["error_code"] is None + assert len(ready_att["variants"]) >= 1 + + # 3. Скачивание оригинала Бобом + down_res = await client.get( + f"/api/v1/attachments/{att_png['id']}/content?variant=original", + headers=bob["headers"], + ) + assert down_res.status_code == 200 + assert down_res.headers.get("content-type") == "image/png" + assert down_res.content == png_data + + # 4. Проверка обязательного thumbnail (256x160 JPEG) + down_thumb = await client.get( + f"/api/v1/attachments/{att_png['id']}/content?variant=thumbnail", + headers=bob["headers"], + ) + assert down_thumb.status_code == 200 + assert down_thumb.headers.get("content-type") == "image/jpeg" + thumb_img = Image.open(io.BytesIO(down_thumb.content)) + assert thumb_img.size == (256, 160) + + # 5. Проверка crop (128x128 JPEG) + down_crop = await client.get( + f"/api/v1/attachments/{att_png['id']}/content?variant=crop", + headers=bob["headers"], + ) + assert down_crop.status_code == 200 + assert down_crop.headers.get("content-type") == "image/jpeg" + crop_img = Image.open(io.BytesIO(down_crop.content)) + assert crop_img.size == (128, 128) + + # 6. Проверка optimized (WebP) + down_opt = await client.get( + f"/api/v1/attachments/{att_png['id']}/content?variant=optimized", + headers=bob["headers"], + ) + assert down_opt.status_code == 200 + assert down_opt.headers.get("content-type") == "image/webp" + + # 7. Загрузка и скачивание JPEG + jpeg_img = Image.new("RGB", (64, 64), color=(255, 128, 0)) + jpeg_buf = io.BytesIO() + jpeg_img.save(jpeg_buf, format="JPEG") + jpeg_data = jpeg_buf.getvalue() + + up_jpeg = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=orange.jpg", + headers={**alice["headers"], "Content-Type": "image/jpeg"}, + content=jpeg_data, + ) + assert up_jpeg.status_code == 202 + att_jpeg = up_jpeg.json() + assert att_jpeg["content_type"] == "image/jpeg" + assert att_jpeg["size_bytes"] == len(jpeg_data) + + ready_jpeg = await wait_attachment_ready(client, bob["headers"], att_jpeg["id"]) + assert ready_jpeg["status"] == "ready" + + down_jpeg = await client.get( + f"/api/v1/attachments/{att_jpeg['id']}/content", + headers=bob["headers"], + ) + assert down_jpeg.status_code == 200 + assert down_jpeg.headers.get("content-type") == "image/jpeg" + assert down_jpeg.content == jpeg_data + + # 8. Проверка точного округления вниз (floor) для PNG 319x200 -> thumbnail 256x160 + odd_img = Image.new("RGBA", (319, 200), color=(100, 150, 200, 255)) + odd_buf = io.BytesIO() + odd_img.save(odd_buf, format="PNG") + odd_data = odd_buf.getvalue() + + up_odd = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=odd_319x200.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=odd_data, + ) + assert up_odd.status_code == 202 + ready_odd = await wait_attachment_ready(client, bob["headers"], up_odd.json()["id"]) + thumb_var = next(v for v in ready_odd["variants"] if v["name"] == "thumbnail") + assert (thumb_var["width"], thumb_var["height"]) == (256, 160) + + + +@pytest.mark.asyncio +async def test_attachment_unsupported_media_type_and_magic_bytes(client: httpx.AsyncClient): + """Проверка отклонения невалидных MIME-типов и поврежденных сигнатур (415).""" + alice = await register_and_login(client, "mime_a") + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Mime Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + # 1. Неподдерживаемый Content-Type: text/plain -> 415 + res1 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=test.txt", + headers={**alice["headers"], "Content-Type": "text/plain"}, + content=b"plain text", + ) + assert res1.status_code == 415 + assert res1.json()["error"]["code"] == "unsupported_media_type" + + # 2. Неподдерживаемый Content-Type: image/gif -> 415 + res2 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=anim.gif", + headers={**alice["headers"], "Content-Type": "image/gif"}, + content=b"GIF89a...", + ) + assert res2.status_code == 415 + assert res2.json()["error"]["code"] == "unsupported_media_type" + + # 3. Content-Type image/png, но поврежденные магические байты -> 415 + res3 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=fake.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=b"NOT_A_PNG_HEADER_HERE", + ) + assert res3.status_code == 415 + assert res3.json()["error"]["code"] == "unsupported_media_type" + + # 4. Content-Type image/jpeg, но поврежденные магические байты -> 415 + res4 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=fake.jpg", + headers={**alice["headers"], "Content-Type": "image/jpeg"}, + content=b"NOT_A_JPEG_HEADER_HERE", + ) + assert res4.status_code == 415 + assert res4.json()["error"]["code"] == "unsupported_media_type" + + # 5. Поврежденный JPEG без пиксельных данных (18 байт) -> 415 + bad_jpeg_18 = bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9") + res5 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=bad18.jpg", + headers={**alice["headers"], "Content-Type": "image/jpeg"}, + content=bad_jpeg_18, + ) + assert res5.status_code == 415 + assert res5.json()["error"]["code"] == "unsupported_media_type" + + # 6. Обрезанный PNG без пиксельных данных (33 байта) -> 415 + bad_png_33 = bytes.fromhex("89504e470d0a1a0a0000000d4948445200000001000000010802000000907753de") + res6 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=bad33.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=bad_png_33, + ) + assert res6.status_code == 415 + assert res6.json()["error"]["code"] == "unsupported_media_type" + + +@pytest.mark.asyncio +async def test_attachment_filename_validation(client: httpx.AsyncClient): + """Проверка валидации имени файла (path traversal, длина, спецсимволы -> 400).""" + alice = await register_and_login(client, "fname_a") + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Filename Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + with open("tests/fixtures/sample.png", "rb") as f: + valid_png = f.read() + + headers = {**alice["headers"], "Content-Type": "image/png"} + + # 1. Path traversal со слешем / + res_slash = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=../../etc/passwd", + headers=headers, + content=valid_png, + ) + assert res_slash.status_code == 400 + assert res_slash.json()["error"]["code"] == "invalid_request" + + # 2. Path traversal с обратным слешем \ + res_bslash = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=..\\boot.ini", + headers=headers, + content=valid_png, + ) + assert res_bslash.status_code == 400 + assert res_bslash.json()["error"]["code"] == "invalid_request" + + # 3. Символ перевода строки \n + res_nl = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=file%0Aname.png", + headers=headers, + content=valid_png, + ) + assert res_nl.status_code == 400 + assert res_nl.json()["error"]["code"] == "invalid_request" + + # 4. Превышение 255 символов + long_name = "a" * 256 + ".png" + res_long = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename={long_name}", + headers=headers, + content=valid_png, + ) + assert res_long.status_code == 400 + assert res_long.json()["error"]["code"] == "invalid_request" + + # 5. Пустое имя файла + res_empty = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=", + headers=headers, + content=valid_png, + ) + assert res_empty.status_code == 400 + + +@pytest.mark.asyncio +async def test_attachment_payload_too_large_and_pixel_bomb(client: httpx.AsyncClient): + """Проверка лимита размера файла (>10 MiB) и бомбы декомпрессии (>40M px) -> 413.""" + import io + from PIL import Image + + alice = await register_and_login(client, "large_a") + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Large Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + # 1. Размер файла более 10 MiB (10 * 1024 * 1024 + 10 байт) -> 413 + oversize_bytes = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10 * 1024 * 1024 + 10) + res_oversize = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=huge.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=oversize_bytes, + ) + assert res_oversize.status_code == 413 + assert res_oversize.json()["error"]["code"] == "payload_too_large" + + # 2. Бомба декомпрессии (10000x5000 = 50 млн пикселей > 40 млн пикселей) -> 413 + bomb_img = Image.new("L", (10000, 5000), color=0) + bomb_buf = io.BytesIO() + bomb_img.save(bomb_buf, format="PNG") + bomb_bytes = bomb_buf.getvalue() + + res_bomb = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=pixel_bomb.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=bomb_bytes, + ) + assert res_bomb.status_code == 413 + assert res_bomb.json()["error"]["code"] == "payload_too_large" + + +@pytest.mark.asyncio +async def test_attachment_acl_and_isolation(client: httpx.AsyncClient): + """Проверка строгой изоляции: посторонний пользователь получает 404 на все операции.""" + alice = await register_and_login(client, "acl_a") + eve = await register_and_login(client, "acl_eve") + + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Private Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + with open("tests/fixtures/sample.png", "rb") as f: + png_data = f.read() + + # 1. Посторонний пытается загрузить вложение в чужой чат -> 404 + res_eve_up = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={**eve["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + assert res_eve_up.status_code == 404 + assert res_eve_up.json()["error"]["code"] == "not_found" + + # Алиса загружает файл + res_alice_up = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + assert res_alice_up.status_code == 202 + att_id = res_alice_up.json()["id"] + + # 2. Посторонний пытается запросить метаданные -> 404 + res_eve_meta = await client.get( + f"/api/v1/attachments/{att_id}", + headers=eve["headers"], + ) + assert res_eve_meta.status_code == 404 + + # 3. Посторонний пытается скачать контент -> 404 + res_eve_down = await client.get( + f"/api/v1/attachments/{att_id}/content", + headers=eve["headers"], + ) + assert res_eve_down.status_code == 404 + + # 4. Несуществующий attachment ID -> 404 + random_uuid = str(uuid.uuid4()) + res_not_found = await client.get( + f"/api/v1/attachments/{random_uuid}", + headers=alice["headers"], + ) + assert res_not_found.status_code == 404 + + # 5. Невалидный UUID -> 400 + res_bad_uuid = await client.get( + "/api/v1/attachments/not-a-uuid", + headers=alice["headers"], + ) + assert res_bad_uuid.status_code == 400 + + +@pytest.mark.asyncio +async def test_message_linking_attachments(client: httpx.AsyncClient): + """Проверка связывания сообщений и вложений: пустой текст с вложением, чужие вложения, лимит 10.""" + alice = await register_and_login(client, "msg_att_a") + bob = await register_and_login(client, "msg_att_b") + + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Msg Attach Chat", "member_ids": [alice["user"]["id"], bob["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + chat2_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Chat 2", "member_ids": [alice["user"]["id"]]}, + ) + chat2_id = chat2_res.json()["id"] + + with open("tests/fixtures/sample.png", "rb") as f: + png_data = f.read() + + # Загружаем вложение в chat 1 (возвращает 202 queued) + up1 = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=sample.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + assert up1.status_code == 202 + att1_id = up1.json()["id"] + + # Загружаем вложение в chat 2 (возвращает 202 queued) + up2 = await client.post( + f"/api/v1/chats/{chat2_id}/attachments?filename=chat2.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + assert up2.status_code == 202 + att2_id = up2.json()["id"] + + # 1. Сообщение с пустым текстом, но с вложением в статусе queued -> 201 + msg1_res = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "", "attachment_ids": [att1_id]}, + ) + assert msg1_res.status_code == 201, msg1_res.text + msg1 = msg1_res.json() + assert msg1["text"] == "" + assert msg1["attachment_ids"] == [att1_id] + + # 2. Сообщение с пустым текстом И без вложений -> 400 + msg2_res = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "", "attachment_ids": []}, + ) + assert msg2_res.status_code == 400 + + # 3. Прикрепление вложения из другого чата -> 404 + msg3_res = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "Hello", "attachment_ids": [att2_id]}, + ) + assert msg3_res.status_code == 404 + + # 4. Прикрепление несуществующего вложения -> 404 + fake_uuid = str(uuid.uuid4()) + msg4_res = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "Hello", "attachment_ids": [fake_uuid]}, + ) + assert msg4_res.status_code == 404 + + # 5. Более 10 вложений -> 400 + eleven_uuids = [str(uuid.uuid4()) for _ in range(11)] + msg5_res = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "Hello", "attachment_ids": eleven_uuids}, + ) + assert msg5_res.status_code == 400 + + +@pytest.mark.asyncio +async def test_message_linking_failed_attachment_rejected(client: httpx.AsyncClient): + """Проверка запрета прикрепления failed-вложения к сообщению (409 Conflict).""" + alice = await register_and_login(client, "failed_a") + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Failed Attach Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + # Имитируем failed-вложение в БД + import subprocess + fake_att_id = str(uuid.uuid4()) + insert_sql = f""" + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, s3_key, status, error_code, created_at, updated_at) + VALUES ('{fake_att_id}', '{chat_id}', '{alice["user"]["id"]}', 'corrupt.jpg', 'image/jpeg', 100, '0000000000000000000000000000000000000000000000000000000000000000', 'staging/corrupt', 'failed', 'invalid_image', NOW(), NOW()); + """ + cmd = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "messenger", "-c", insert_sql] + sub_res = subprocess.run(cmd, capture_output=True, text=True) + assert sub_res.returncode == 0 + + # Попытка прикрепить failed-вложение -> 409 Conflict + res_link = await client.post( + f"/api/v1/chats/{chat_id}/messages", + headers={**alice["headers"], "Idempotency-Key": uuid.uuid4().hex}, + json={"text": "Trying failed attach", "attachment_ids": [fake_att_id]}, + ) + assert res_link.status_code == 409 + assert res_link.json()["error"]["code"] == "conflict" + + +@pytest.mark.asyncio +async def test_attachment_orphan_compensation(client: httpx.AsyncClient): + """ + Проверка компенсации сиротских объектов S3 при сбое БД (критерий оценки 4): + 1. Через docker compose exec в PostgreSQL создаём триггер, вызывающий сбой при записи файла 'orphan_sim_fail.png'. + 2. Выполняем POST вложения 'orphan_sim_fail.png'. + S3 PUT проходит успешно в staging, но при вставке в PostgreSQL возникает ошибка. + Сервер перехватывает ошибку, вызывает s3_client.delete_object и возвращает 500/503. + 3. Проверяем, что сиротский объект удалён из S3 staging. + 4. Удаляем триггер из PostgreSQL. + """ + import subprocess + alice = await register_and_login(client, "orph_a") + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Orphan Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + setup_sql = """ + CREATE OR REPLACE FUNCTION trg_sim_fail() RETURNS TRIGGER AS $$ + BEGIN + IF NEW.filename = 'orphan_sim_fail.png' THEN + RAISE EXCEPTION 'Simulated DB failure for orphan compensation test'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + DROP TRIGGER IF EXISTS check_orphan_sim ON attachments; + CREATE TRIGGER check_orphan_sim BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION trg_sim_fail(); + """ + cmd = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "messenger", "-c", setup_sql] + sub_res = subprocess.run(cmd, capture_output=True, text=True) + assert sub_res.returncode == 0, f"Ошибка создания триггера: {sub_res.stderr}" + + try: + with open("tests/fixtures/sample.png", "rb") as f: + png_data = f.read() + + res = await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=orphan_sim_fail.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + assert res.status_code in (500, 503) + + check_py = """ +import boto3 +s3 = boto3.client('s3', endpoint_url='http://minio:9000', aws_access_key_id='minioadmin', aws_secret_access_key='minioadmin') +res = s3.list_objects_v2(Bucket='messenger-attachments', Prefix='staging/') +keys = [o['Key'] for o in res.get('Contents', [])] +found = any('orphan_sim_fail.png' in k for k in keys) +print('FOUND' if found else 'NOT_FOUND') +""" + cmd_s3 = ["docker", "compose", "exec", "-T", "api-1", "python", "-c", check_py] + s3_res = subprocess.run(cmd_s3, capture_output=True, text=True) + assert s3_res.returncode == 0 + assert "NOT_FOUND" in s3_res.stdout, f"Сиротский объект остался в S3! Вывод: {s3_res.stdout}" + finally: + cleanup_sql = """ + DROP TRIGGER IF EXISTS check_orphan_sim ON attachments; + DROP FUNCTION IF EXISTS trg_sim_fail(); + """ + cmd_clean = ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "postgres", "-d", "messenger", "-c", cleanup_sql] + subprocess.run(cmd_clean, capture_output=True, text=True) + + +@pytest.mark.asyncio +async def test_private_bucket_outside_api(client: httpx.AsyncClient): + """ + Проверка закрытости S3 бакета вне API (критерий оценки 4): + 1. Порты MinIO (9000/9001) не проброшены на хост наружу -> прямое подключение отклоняется. + 2. Балансировщик Nginx маршрутизирует только /api/ и /health/, запрос к S3 напрямую через LB возвращает 404. + 3. Неаутентифицированный запрос на скачивание вложения возвращает 401 Unauthorized. + """ + direct_conn_failed = False + try: + async with httpx.AsyncClient() as raw_client: + await raw_client.get("http://localhost:9000", timeout=1.0) + except httpx.ConnectError: + direct_conn_failed = True + assert direct_conn_failed, "Порт 9000 MinIO не должен быть доступен с хоста!" + + res_lb_s3 = await client.get("/messenger-attachments/") + assert res_lb_s3.status_code == 404 + + random_id = str(uuid.uuid4()) + res_anon = await client.get(f"/api/v1/attachments/{random_id}/content") + assert res_anon.status_code == 401 + + +@pytest.mark.asyncio +async def test_concurrent_uploads_memory_limit(client: httpx.AsyncClient): + """ + Проверка конкурентной загрузки ≥10 файлов с контролем потребления RAM (критерий оценки 4): + 1. Создаем пользователя и чат. + 2. Запускаем параллельную загрузку 10 изображений через asyncio.gather. + 3. Все 10 загрузок завершаются успехом (202 Accepted, status=queued). + 4. Ожидаем готовности ready через wait_attachment_ready. + 5. Измеряем память контейнеров API через docker stats. + """ + import subprocess + alice = await register_and_login(client, "conc_a") + chat_res = await client.post( + "/api/v1/chats", + headers=alice["headers"], + json={"title": "Concurrent Uploads Chat", "member_ids": [alice["user"]["id"]]}, + ) + chat_id = chat_res.json()["id"] + + with open("tests/fixtures/sample.png", "rb") as f: + png_data = f.read() + + async def upload_single(idx: int): + return await client.post( + f"/api/v1/chats/{chat_id}/attachments?filename=concurrent_{idx}.png", + headers={**alice["headers"], "Content-Type": "image/png"}, + content=png_data, + ) + + tasks = [upload_single(i) for i in range(10)] + responses = await asyncio.gather(*tasks) + + att_ids = [] + for resp in responses: + assert resp.status_code == 202 + data = resp.json() + assert data["status"] == "queued" + assert data["size_bytes"] == len(png_data) + att_ids.append(data["id"]) + + # Ожидаем готовности всех 10 файлов + ready_tasks = [wait_attachment_ready(client, alice["headers"], aid) for aid in att_ids] + await asyncio.gather(*ready_tasks) + + # Проверяем docker stats + stats_cmd = [ + "docker", "stats", "--no-stream", "--format", "{{.Name}}: {{.MemUsage}}" + ] + stats_res = subprocess.run(stats_cmd, capture_output=True, text=True) + assert stats_res.returncode == 0 + + + + +