diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..46c4106f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11-slim + +WORKDIR /app + +# libpq-dev для psycopg, gcc для сборки некоторых python-пакетов, +# libjpeg-dev и zlib1g-dev нужны для работы Pillow (обработка картинок) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + python3-dev \ + libjpeg-dev \ + zlib1g-dev \ + libpng-dev \ + libwebp-dev \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --no-cache-dir --index-url https://mirrors.aliyun.com/pypi/simple/ -r requirements.txt + +# Копируем код API, Воркера и скрипты (включая migrate.py) +COPY src/ ./src/ +COPY worker/ ./worker/ +COPY scripts/ ./scripts/ + +ENV PYTHONPATH=/app + +# По умолчанию запускаем API (воркер переопределит эту команду в compose) +CMD ["uvicorn", "src.app.main:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 00000000..1e5291b9 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq-dev gcc && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ +COPY worker/ ./worker/ + +ENV PYTHONPATH=/app +CMD ["python", "-m", "worker.main"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..7d6c1505 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,75 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Бахаева Лиана, БПИ2302 +- Целевая оценка: 4 - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: Материалы курса, документация RabbitMQ (reliability, confirm), FastAPI, MinIO, Redis. Использовалась помощь ИИ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Требования:** +- Docker Desktop (Compose v2) +- Python 3.9+ (для локального запуска тестов) +- Git Bash (или эквивалент) для Windows + +**Шаги:** +1. Клонировать репозиторий и перейти в корень проекта. +2. Убедиться, что файл `lab.json` содержит `{"number": 5}`. +3. Запустить инфраструктуру: `docker compose up -d --build` +4. Дождаться, пока контейнеры `postgres`, `minio`, `broker`, `redis` станут `Healthy` (команда `docker compose ps`). +5. Проверить готовность API (через Load Balancer): `curl -i http://localhost:8080/health/ready`. Ожидаемый ответ: HTTP/1.1 200 OK, `{"status":"ok"}`. +6. Запустить публичный smoke-тест: `PYTHONUTF8=1 PYTHONPATH=tests python tests/smoke.py`. Ожидаемый ответ: PASS: lab5 public API scenario... + +**Остановка и удаление данных:** +- Безопасная остановка (с сохранением данных): `docker compose stop` +- Полное удаление учебных данных: `docker compose down -v` (удаляет все volumes: pgdata, s3data, brokerdata, redisdata). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема процессов:** +- `lb` (Nginx): публичный вход на порт 8080. Распределяет трафик между `api1` и `api2` (least_conn). +- `api1`, `api2` (FastAPI): две реплики без локального состояния. Сессии и идемпотентность хранятся в Redis, данные — в Postgres. +- `postgres`: основное хранилище (пользователи, чаты, сообщения, вложения, outbox). +- `minio` (S3): приватный бакет `messenger-attachments`. +- `broker` (RabbitMQ): очередь `attachment_processing` с настройкой DLQ (`dead_letters`). +- `redis`: хранилище сессий и ключей идемпотентности (durable persistence через `appendonly yes`). +- `worker`: отдельный процесс. Читает `outbox_messages` из Postgres, отправляет задачи в RabbitMQ, скачивает файл из MinIO, генерирует варианты (thumbnail, crop, optimized), сохраняет их в MinIO и обновляет БД. + +**Источник истины и транзакционные границы:** +- Метаданные вложений и статусы — в Postgres. +- Байты изображений (оригинал и варианты) — в MinIO. +- Транзакционная граница (Transactional Outbox): при загрузке файла в одной транзакции Postgres создаются записи в `attachments` и `outbox_messages`. Это гарантирует, что если API успешно ответил 202, задача обязательно будет отправлена в RabbitMQ (даже если брокер временно недоступен). + +**Что перенесено из lab4 и что изменено:** +- Перенесена логика аутентификации (Argon2), работа с чатами и сообщениями, идемпотентность сообщений. +- Изменено: загрузка вложений теперь асинхронная (202 queued), добавлен Outbox и фоновый публикатор, добавлен Retry с backoff/jitter и DLQ, внедрено хранение сессий в Redis, формат optimized — WebP, crop — 128x128. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `PYTHONUTF8=1 PYTHONPATH=tests python tests/smoke.py` | PASS: lab5 public API scenario | tests/smoke.py (скриншот терминала) | +| Остановка worker -> upload -> restart worker | `docker compose stop worker`; загрузка файла через API; `docker compose start worker` | Файл переходит из queued в ready (Outbox спасает задачу) | Логи worker (обработка после рестарта) | +| Durable persistence Redis | Залогиниться, получить токен; `docker compose restart redis`; запрос `/users/me` с тем же токеном | 200 OK (сессия выжила) | Скриншот терминала | +| Ограниченный retry и DLQ | Отправить битый файл; 5 неудачных попыток воркера | Статус failed, задача уходит в dead_letters | Логи worker, `docker compose logs broker` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** Outbox защищает от потери задач при падении брокера. +**Условия:** Docker Desktop, 8 GB RAM, локальная машина. +**Нагрузка:** 10 параллельных загрузок PNG (тест `tests/load.py`). +**Результаты:** все 10 файлов успешно обработаны, задержка между загрузкой и ready составила ~2-3 секунды. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка RabbitMQ | API продолжает принимать файлы (202), задачи копятся в outbox | API отвечает 202, статус queued. После запуска брокера задачи обрабатываются | ~5 секунд (пока брокер поднимался), данные не потеряны | +| Остановка worker | Файлы загружаются (202), но не переходят в ready | API отвечает 202, статус queued. После запуска worker — ready | ~10 секунд, данные не потеряны | +| Рестарт Redis | Сессии сохраняются (durable persistence) | Старый токен работает после перезапуска | ~5 секунд, сессии не потеряны | + ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Что выдерживается:** отказ брокера (через Transactional Outbox), отказ worker (задачи копятся в очереди и обрабатываются после перезапуска), рестарт Redis (сессии и ключи идемпотентности сохраняются благодаря durable persistence). +- **Что не реализовано:** автоматическое масштабирование worker (сейчас ровно 1 экземпляр), сложные политики backoff с экспоненциальной задержкой через задержанные очереди (реализован упрощенный вариант с `asyncio.sleep` в воркере), сбор метрик (Prometheus/Grafana). +- **Учебное упрощение:** создание бакета MinIO выполняется вручную командой `docker compose exec minio ...` после первого запуска (или после `docker compose down -v`), так как автоматическое создание через `entrypoint` оказалось нестабильным из-за гонки при старте. В промышленной эксплуатации для этого используют отдельный init-контейнер или Infrastructure-as-Code. +- **Следующие шаги:** добавить отдельный init-контейнер для создания бакета (чтобы избежать ручных действий), внедрить задержанные очереди (delayed message exchange) для более чистого backoff без блокировки воркера, добавить метрики Prometheus для мониторинга размера очереди и времени обработки, а также горизонтальное масштабирование воркеров (уже сейчас можно запустить несколько реплик `worker`, так как задачи идемпотентны, но это не тестировалось под нагрузкой). \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..4cd8e3c9 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,143 @@ +x-api-common: &api-common + build: . + environment: + - DATABASE_URL=postgresql://messenger:messenger_password@postgres:5432/messenger + - S3_ENDPOINT=http://minio:9000 + - S3_ACCESS_KEY=minioadmin + - S3_SECRET_KEY=minioadmin + - S3_BUCKET=messenger-attachments + - BROKER_URL=amqp://guest:guest@broker:5672/ + - REDIS_URL=redis://redis:6379/0 + - APP_ENV=lab5 + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + broker: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + +services: + lb: + image: nginx:1.25-alpine + ports: + - "8080:8080" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - api1 + - api2 + + api1: + <<: *api-common + expose: + - "8080" + + api2: + <<: *api-common + expose: + - "8080" + + worker: + build: . + command: python -m worker.main + environment: + - DATABASE_URL=postgresql://messenger:messenger_password@postgres:5432/messenger + - S3_ENDPOINT=http://minio:9000 + - S3_ACCESS_KEY=minioadmin + - S3_SECRET_KEY=minioadmin + - S3_BUCKET=messenger-attachments + - BROKER_URL=amqp://guest:guest@broker:5672/ + - REDIS_URL=redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + broker: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + + migrate: + build: . + command: python scripts/migrate.py + environment: + - DATABASE_URL=postgresql://messenger:messenger_password@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + + postgres: + image: postgres:16-alpine + environment: + - POSTGRES_USER=messenger + - POSTGRES_PASSWORD=messenger_password + - POSTGRES_DB=messenger + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 5s + retries: 5 + + minio: + image: pgsty/minio:latest + command: server /data --console-address ":9001" + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + volumes: + - s3data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 5 + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://localhost:9000 minioadmin minioadmin; + /usr/bin/mc mb myminio/messenger-attachments --ignore-existing; + /usr/bin/mc anonymous set private myminio/messenger-attachments; + /usr/bin/minio server /data --console-address ':9001' + " + + broker: + image: rabbitmq:3.13-alpine + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest + volumes: + - brokerdata:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + command: redis-server --appendonly yes + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + +volumes: + pgdata: + s3data: + brokerdata: + redisdata: \ No newline at end of file diff --git a/infra/minio-policy.json b/infra/minio-policy.json new file mode 100644 index 00000000..e757ee44 --- /dev/null +++ b/infra/minio-policy.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments", + "arn:aws:s3:::messenger-attachments/*" + ] + } + ] +} \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..07ec076d --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,38 @@ +worker_processes auto; + +events { worker_connections 1024; } + +http { + upstream api_backend { + # least_conn лучше распределяет запросы, чем round-robin, + # особенно когда одна реплика чуть медленнее. + least_conn; + server api1:8080 max_fails=1 fail_timeout=2s; + server api2:8080 max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + + # Проверка, что LB жив (без проксирования в API) + location = /lb-health { + access_log off; + return 200 'ok'; + add_header Content-Type text/plain; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Важно для быстрого отключения упавшей реплики + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_connect_timeout 2s; + proxy_read_timeout 10s; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c31a3d2e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.29.0 +psycopg[binary,pool]>=3.1.18 +pydantic-settings>=2.2.1 +argon2-cffi>=23.1.0 +boto3>=1.34.0 +python-multipart>=0.0.9 +Pillow>=10.2.0 +aio-pika>=9.4.0 +redis[hiredis]>=5.0.1 \ No newline at end of file diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..e617f679 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,58 @@ +import os +import sys +import time +from pathlib import Path +import psycopg + +def get_dsn(): + file_path = os.environ.get("DATABASE_URL_FILE") + if file_path and os.path.exists(file_path): + return Path(file_path).read_text().strip() + return os.environ.get("DATABASE_URL") + +def main(): + dsn = get_dsn() + if not dsn: + print("ERROR: DATABASE_URL or DATABASE_URL_FILE not set", file=sys.stderr) + sys.exit(1) + + for i in range(10): + try: + with psycopg.connect(dsn, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + break + except psycopg.OperationalError: + print("Waiting for database...") + time.sleep(2) + else: + print("ERROR: Database is not ready", file=sys.stderr) + sys.exit(1) + + # ВАЖНО: путь к миграциям. Раз они лежат в src/migrations: + migrations_dir = Path(__file__).parent.parent / "src" / "migrations" + sql_files = sorted(migrations_dir.glob("*.sql")) + + with psycopg.connect(dsn, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz DEFAULT now() + ) + """) + + for sql_file in sql_files: + version = sql_file.name + cur.execute("SELECT 1 FROM schema_migrations WHERE version = %s", (version,)) + if cur.fetchone(): + continue + + print(f"Applying {version}...") + sql = sql_file.read_text(encoding="utf-8") + cur.execute(sql) + cur.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + print(f"Applied {version}.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..43cfb34a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +echo "Restarting API and Postgres (preserving volumes)..." +docker compose restart api1 api2 postgres +echo "Waiting for Postgres to be healthy..." +sleep 5 \ No newline at end of file diff --git a/scripts/stop_one.sh b/scripts/stop_one.sh new file mode 100644 index 00000000..14859318 --- /dev/null +++ b/scripts/stop_one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +if [ "$TARGET_INSTANCE_ID" != "api-1" ] && [ "$TARGET_INSTANCE_ID" != "api-2" ]; then + echo "Unknown instance: $TARGET_INSTANCE_ID" >&2 + exit 1 +fi +SERVICE_NAME="$TARGET_INSTANCE_ID" +if [ "$SERVICE_NAME" = "api-1" ]; then SERVICE_NAME="api1"; fi +if [ "$SERVICE_NAME" = "api-2" ]; then SERVICE_NAME="api2"; fi + +echo "Stopping container for $SERVICE_NAME..." +docker compose stop "$SERVICE_NAME" \ No newline at end of file diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 00000000..6c93b09b --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,60 @@ +from functools import lru_cache +from pathlib import Path +from pydantic_settings import BaseSettings, SettingsConfigDict + + +def _read_secret(path: str | None) -> str | None: + """Прочитать секрет из файла, если задан путь. Иначе None.""" + if not path: + return None + p = Path(path) + if not p.is_file(): + return None + return p.read_text(encoding="utf-8").strip() + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=None, extra="ignore") + + # Основные настройки приложения + app_host: str = "0.0.0.0" + app_port: int = 8080 + app_env: str = "lab" + instance_id: str = "" # пусто -> сгенерируется в instance.py + log_level: str = "info" + + # Сессии и идемпотентность + session_ttl_seconds: int = 3600 + idempotency_ttl_seconds: int = 86400 + + # Postgres: либо прямая строка, либо файл-секрет (приоритет у файла). + database_url: str = "" + database_url_file: str = "/run/secrets/database_url" + + # S3 / MinIO + s3_endpoint: str = "http://minio:9000" + s3_access_key: str = "minioadmin" + s3_secret_key: str = "minioadmin" + s3_bucket: str = "messenger-attachments" + s3_region: str = "us-east-1" + + # Брокер (RabbitMQ) + broker_url: str = "amqp://guest:guest@broker:5672/" + broker_queue: str = "attachment_processing" + + redis_url: str = "redis://redis:6379/0" + + @property + def db_dsn(self) -> str: + from_file = _read_secret(self.database_url_file) + dsn = from_file or self.database_url + if not dsn: + raise RuntimeError( + "DATABASE_URL не задан: укажи DATABASE_URL или смонтируй DATABASE_URL_FILE" + ) + return dsn + + +@lru_cache +def get_settings() -> Settings: + return Settings() \ No newline at end of file diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 00000000..870b6ee2 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,45 @@ +from contextlib import contextmanager +from typing import Iterator + +import psycopg +from psycopg_pool import ConnectionPool + +from .config import get_settings + +_pool: ConnectionPool | None = None + + +def init_pool() -> ConnectionPool: + global _pool + if _pool is None: + _pool = ConnectionPool( + conninfo=get_settings().db_dsn, + min_size=1, + max_size=10, + timeout=5, + open=True, + ) + return _pool + + +def close_pool() -> None: + global _pool + if _pool is not None: + _pool.close() + _pool = None + + +@contextmanager +def connection() -> Iterator[psycopg.Connection]: + pool = init_pool() + with pool.connection() as conn: + yield conn + + +@contextmanager +def transaction() -> Iterator[psycopg.Connection]: + """Явная транзакция: commit при успехе, rollback при исключении.""" + pool = init_pool() + with pool.connection() as conn: + with conn.transaction(): + yield conn \ No newline at end of file diff --git a/src/app/errors.py b/src/app/errors.py new file mode 100644 index 00000000..c7e071c0 --- /dev/null +++ b/src/app/errors.py @@ -0,0 +1,31 @@ +from typing import Any + +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + """Наше приложение бросает это; handler превращает в JSON-ответ.""" + + def __init__(self, status: int, code: str, message: str, headers: dict[str, str] | None = None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def error_response( + request: Request, + status: int, + code: str, + message: str, + extra_headers: dict[str, str] | None = None, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") or "unknown" + body: dict[str, Any] = { + "error": {"code": code, "message": message, "request_id": request_id} + } + headers = {"X-Request-Id": request_id} + if extra_headers: + headers.update(extra_headers) + return JSONResponse(status_code=status, content=body, headers=headers) \ No newline at end of file diff --git a/src/app/idempotency.py b/src/app/idempotency.py new file mode 100644 index 00000000..9855d870 --- /dev/null +++ b/src/app/idempotency.py @@ -0,0 +1,60 @@ +import hashlib +import json +import redis +from typing import Any, Callable +import psycopg + +from .config import get_settings +from .errors import ApiError +from .db import transaction + +redis_client = redis.Redis.from_url(get_settings().redis_url, decode_responses=True) + +def _body_hash(body: dict[str, Any]) -> str: + raw = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + +def run_idempotent( + *, + user_id: str, + scope: str, + key: str, + body: dict[str, Any], + action: Callable[[psycopg.Connection], tuple[int, dict[str, Any]]], +) -> tuple[int, dict[str, Any]]: + h = _body_hash(body) + redis_key = f"idem:{user_id}:{scope}:{key}" + ttl = get_settings().idempotency_ttl_seconds + + # Блокировка + locked = redis_client.set(redis_key, json.dumps({"status": 0, "hash": h}), nx=True, ex=ttl) + + if locked: + try: + with transaction() as conn: + status, payload = action(conn) + + redis_client.setex( + redis_key, + ttl, + json.dumps({"status": status, "hash": h, "payload": payload}) + ) + return status, payload + except Exception: + redis_client.delete(redis_key) + raise + + # Повторный запрос + existing_raw = redis_client.get(redis_key) + if not existing_raw: + raise ApiError(409, "conflict", "Конфликт идемпотентности, повторите") + + existing = json.loads(existing_raw) + + if existing.get("hash") != h: + raise ApiError(409, "conflict", "Тот же ключ с другим телом запроса") + + if existing.get("status") == 0: + raise ApiError(409, "conflict", "Повтор ещё не завершён") + + return existing["status"], existing["payload"] \ No newline at end of file diff --git a/src/app/image_check.py b/src/app/image_check.py new file mode 100644 index 00000000..e48b2307 --- /dev/null +++ b/src/app/image_check.py @@ -0,0 +1,13 @@ +def detect_image_type(data: bytes) -> str | None: + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + return None + + +def check_image(first_bytes: bytes, allowed_types: list[str]) -> tuple[str, int, int]: + content_type = detect_image_type(first_bytes) + if not content_type or content_type not in allowed_types: + raise ValueError("Invalid or unsupported image type") + return content_type, 0, 0 \ No newline at end of file diff --git a/src/app/instance.py b/src/app/instance.py new file mode 100644 index 00000000..f5419bff --- /dev/null +++ b/src/app/instance.py @@ -0,0 +1,15 @@ +import os +import secrets + +from .config import get_settings + + +def _build_instance_id() -> str: + s = get_settings() + if s.instance_id: + return s.instance_id + # Короткий случайный alias; при желании можно задать INSTANCE_ID в env. + return "api-" + secrets.token_hex(6) + + +INSTANCE_ID: str = _build_instance_id() \ No newline at end of file diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 00000000..d665be63 --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,56 @@ +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from . import db +from .errors import ApiError, error_response +from .instance import INSTANCE_ID +from .routes import health, users, sessions, chats, messages, attachments + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + db.init_pool() + except Exception: + pass + yield + db.close_pool() + + +app = FastAPI(title="MTUSI Messenger", version="1.4.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Неверный запрос") + + +@app.exception_handler(Exception) +async def unhandled_handler(request: Request, exc: Exception): + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +app.include_router(health.router) +app.include_router(users.router) +app.include_router(sessions.router) +app.include_router(chats.router) +app.include_router(messages.router) +app.include_router(attachments.router) \ No newline at end of file diff --git a/src/app/routes/__init__.py b/src/app/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/routes/attachments.py b/src/app/routes/attachments.py new file mode 100644 index 00000000..8aae82f6 --- /dev/null +++ b/src/app/routes/attachments.py @@ -0,0 +1,230 @@ +import hashlib +import io +import tempfile +import uuid +import json + +from fastapi import APIRouter, Depends, Request, Query +from fastapi.responses import StreamingResponse +from psycopg.rows import dict_row + +from ..db import transaction, connection +from ..errors import ApiError +from ..s3 import upload_fileobj, download_fileobj, delete_file +from ..image_check import check_image +from ..serializers import iso_z +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") +MAX_SIZE = 10 * 1024 * 1024 + +@router.post("/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., max_length=255), + actor: dict = Depends(require_actor), +): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + if "/" in filename or "\\" in filename or "\0" in filename: + raise ApiError(400, "invalid_request", "Опасное имя файла") + + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, actor["user_id"]), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Чат не найден") + + sha256 = hashlib.sha256() + size = 0 + first_chunk = b"" + + with tempfile.SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+b") as spooled: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_SIZE: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + if not first_chunk: + first_chunk = chunk[:16] + sha256.update(chunk) + spooled.write(chunk) + + if size == 0: + raise ApiError(400, "invalid_request", "Пустой файл") + + try: + content_type, _, _ = check_image(first_chunk, ["image/png", "image/jpeg"]) + except ValueError: + raise ApiError(415, "unsupported_media_type", "Неподдерживаемый формат") + + sha256_hex = sha256.hexdigest() + object_key = f"chats/{chat_id}/{uuid.uuid4().hex}" + + try: + spooled.seek(0) + upload_fileobj(spooled, object_key, content_type) + except Exception as e: + print(f"S3 error: {e}") + raise ApiError(503, "unavailable", "S3 недоступен") + + try: + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + # 1. Сохраняем вложение + cur.execute( + """ + INSERT INTO attachments (chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status) + VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued') + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, created_at, updated_at + """, + (chat_id, actor["user_id"], filename, content_type, size, sha256_hex, object_key), + ) + att = cur.fetchone() + + # 2. Сохраняем задачу в Outbox в ТОЙ ЖЕ транзакции + payload = {"attachment_id": str(att["id"]), "object_key": object_key} + cur.execute( + "INSERT INTO outbox_messages (attachment_id, payload) VALUES (%s, %s)", + (att["id"], json.dumps(payload)) + ) + except Exception as e: + print(f"DB failed, compensating S3: {e}") + try: + delete_file(object_key) + except Exception as s3_e: + print(f"Failed to compensate S3: {s3_e}") + raise ApiError(500, "internal_error", "Ошибка сохранения") + + return { + "id": str(att["id"]), + "chat_id": str(att["chat_id"]), + "owner_id": str(att["owner_id"]), + "filename": att["filename"], + "content_type": att["content_type"], + "size_bytes": att["size_bytes"], + "sha256": att["sha256"], + "status": att["status"], + "variants": [], + "error_code": None, + "created_at": iso_z(att["created_at"]), + "updated_at": iso_z(att["updated_at"]), + } + + +@router.get("/attachments/{attachment_id}") +def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): + try: + uuid.UUID(attachment_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный id") + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (actor["user_id"], attachment_id), + ) + att = cur.fetchone() + + if att: + cur.execute("SELECT * FROM attachment_variants WHERE attachment_id = %s", (attachment_id,)) + variants = cur.fetchall() + else: + variants = [] + + if att is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + return { + "id": str(att["id"]), + "chat_id": str(att["chat_id"]), + "owner_id": str(att["owner_id"]), + "filename": att["filename"], + "content_type": att["content_type"], + "size_bytes": att["size_bytes"], + "sha256": att["sha256"], + "status": att["status"], + "variants": [ + { + "name": v["name"], + "content_type": v["content_type"], + "size_bytes": v["size_bytes"], + "width": v["width"], + "height": v["height"], + "sha256": v["sha256"], + } for v in variants + ], + "error_code": att["error_code"], + "created_at": iso_z(att["created_at"]), + "updated_at": iso_z(att["updated_at"]), + } + + +@router.get("/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + actor: dict = Depends(require_actor) +): + try: + uuid.UUID(attachment_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный id") + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT a.object_key, a.content_type, a.filename, a.status + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (actor["user_id"], attachment_id), + ) + att = cur.fetchone() + + if att is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + if variant == "original": + if att["status"] == "failed": + raise ApiError(409, "conflict", "Обработка вложения завершилась ошибкой") + obj_key = att["object_key"] + content_type = att["content_type"] + else: + cur.execute( + "SELECT object_key, content_type FROM attachment_variants WHERE attachment_id = %s AND name = %s", + (attachment_id, variant) + ) + var = cur.fetchone() + if not var: + raise ApiError(404, "not_found", "Вариант еще не готов или недоступен") + obj_key = var["object_key"] + content_type = var["content_type"] + + try: + body = download_fileobj(obj_key) + except Exception: + raise ApiError(503, "unavailable", "S3 недоступен") + + def iter_body(): + yield from body.iter_chunks(1024 * 1024) + + return StreamingResponse( + iter_body(), + media_type=content_type, + headers={"Content-Disposition": f'attachment; filename="{att["filename"]}"'}, + ) \ No newline at end of file diff --git a/src/app/routes/chats.py b/src/app/routes/chats.py new file mode 100644 index 00000000..5c547465 --- /dev/null +++ b/src/app/routes/chats.py @@ -0,0 +1,169 @@ +import base64 +import uuid +from datetime import datetime +from typing import Optional + +import psycopg +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel, ConfigDict, Field, field_validator +from psycopg.rows import dict_row + +from ..db import connection, transaction +from ..errors import ApiError +from ..serializers import row_to_chat +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("title must not be blank") + return v + + @field_validator("member_ids") + @classmethod + def unique_members(cls, v: list[str]) -> list[str]: + if len(set(v)) != len(v): + raise ValueError("duplicate member_ids") + for s in v: + try: + uuid.UUID(s) + except ValueError: + raise ValueError("invalid uuid in member_ids") + return v + + +def _encode_cursor(created_at: datetime, id_) -> str: + raw = f"{created_at.isoformat()}|{id_}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def _decode_cursor(cursor: str): + pad = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + ts, id_ = raw.split("|", 1) + return datetime.fromisoformat(ts), id_ + + +@router.post("/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(require_actor)): + me = actor["user_id"] + if str(me) not in body.member_ids: + raise ApiError(400, "invalid_request", "Вызывающий должен входить в member_ids") + + try: + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id FROM users WHERE id = ANY(%s::uuid[])", + (body.member_ids,), + ) + found = {str(r["id"]) for r in cur.fetchall()} + missing = set(body.member_ids) - found + if missing: + raise ApiError(400, "invalid_request", "Неизвестные участники") + + cur.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (body.title,), + ) + chat = cur.fetchone() + + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat["id"], m) for m in body.member_ids], + ) + except ApiError: + raise + except psycopg.errors.ForeignKeyViolation: + raise ApiError(400, "invalid_request", "Неизвестные участники") + + chat["member_ids"] = body.member_ids + return row_to_chat(chat) + + +@router.get("/chats") +def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + actor: dict = Depends(require_actor), +): + params: list = [] + where = ["cm.user_id = %s"] + params.append(actor["user_id"]) + + if cursor: + try: + c_ts, c_id = _decode_cursor(cursor) + except Exception: + raise ApiError(400, "invalid_request", "Неверный cursor") + where.append("(c.created_at, c.id) > (%s, %s)") + params.extend([c_ts, c_id]) + + sql = f""" + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm2.user_id::text + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE {' AND '.join(where)} + ORDER BY c.created_at, c.id + LIMIT %s + """ + params.append(limit + 1) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute(sql, params) + rows = cur.fetchall() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = _encode_cursor(last["created_at"], str(last["id"])) + rows = rows[:limit] + + return {"items": [row_to_chat(r) for r in rows], "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}") +def get_chat(chat_id: str, actor: dict = Depends(require_actor)): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm2.user_id::text + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = %s AND cm.user_id = %s + """, + (chat_id, actor["user_id"]), + ) + row = cur.fetchone() + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return row_to_chat(row) \ No newline at end of file diff --git a/src/app/routes/health.py b/src/app/routes/health.py new file mode 100644 index 00000000..b0567a08 --- /dev/null +++ b/src/app/routes/health.py @@ -0,0 +1,42 @@ +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse +import socket +from urllib.parse import urlparse + +from ..config import get_settings +from ..db import connection +from ..errors import error_response +from ..s3 import get_s3_client + +router = APIRouter() + +@router.get("/health/live") +def live(): + return {"status": "ok"} + +@router.get("/health/ready") +def ready(request: Request): + try: + + with connection() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() + + + get_s3_client().head_bucket(Bucket=get_settings().s3_bucket) + + # 3. Broker (простая проверка TCP-порта) + settings = get_settings() + parsed = urlparse(settings.broker_url) + host = parsed.hostname or "broker" + port = parsed.port or 5672 + with socket.create_connection((host, port), timeout=2): + pass + + except Exception as e: + print(f"Readiness check failed: {e}") + return error_response( + request, 503, "unavailable", "Зависимость недоступна" + ) + return {"status": "ok"} \ No newline at end of file diff --git a/src/app/routes/messages.py b/src/app/routes/messages.py new file mode 100644 index 00000000..360e4e92 --- /dev/null +++ b/src/app/routes/messages.py @@ -0,0 +1,171 @@ +import re +import uuid +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Header, Query +from pydantic import BaseModel, ConfigDict, Field +from psycopg.rows import dict_row + +from ..db import connection +from ..errors import ApiError +from ..idempotency import run_idempotent +from ..serializers import row_to_message +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + +KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + +def _check_chat_member(conn, chat_id: str, user_id: str) -> None: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Чат не найден") + + +def _encode_cursor(created_at: datetime, id_) -> str: + import base64 + raw = f"{created_at.isoformat()}|{id_}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def _decode_cursor(cursor: str): + import base64 + pad = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + ts, id_ = raw.split("|", 1) + return datetime.fromisoformat(ts), id_ + + +@router.post("/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + body: CreateMessage, + idempotency_key: str = Header(..., alias="Idempotency-Key"), + actor: dict = Depends(require_actor), +): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + if not KEY_RE.fullmatch(idempotency_key or ""): + raise ApiError(400, "invalid_request", "Неверный Idempotency-Key") + + if not body.text.strip() and not body.attachment_ids: + raise ApiError(400, "invalid_request", "Нужен текст или вложение") + + scope = f"POST /api/v1/chats/{chat_id}/messages" + + def action(conn): + _check_chat_member(conn, chat_id, actor["user_id"]) + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + INSERT INTO messages (chat_id, sender_id, text) + VALUES (%s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, + (chat_id, actor["user_id"], body.text), + ) + row = cur.fetchone() + + if body.attachment_ids: + for att_id in body.attachment_ids: + cur.execute( + "SELECT 1 FROM attachments WHERE id = %s AND chat_id = %s", + (att_id, chat_id), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + cur.executemany( + "INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)", + [(row["id"], att_id) for att_id in body.attachment_ids] + ) + + row["attachment_ids"] = body.attachment_ids + + return 201, row_to_message(row) + + _, payload = run_idempotent( + user_id=actor["user_id"], + scope=scope, + key=idempotency_key, + body=body.model_dump(), + action=action, + ) + return payload + + +@router.get("/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + actor: dict = Depends(require_actor), +): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, actor["user_id"]), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Чат не найден") + + params: list = [chat_id] + where = ["m.chat_id = %s"] + if cursor: + try: + c_ts, c_id = _decode_cursor(cursor) + except Exception: + raise ApiError(400, "invalid_request", "Неверный cursor") + where.append("(m.created_at, m.id) > (%s, %s)") + params.extend([c_ts, c_id]) + + sql = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ), ARRAY[]::text[] + ) AS attachment_ids + FROM messages m + WHERE {' AND '.join(where)} + ORDER BY m.created_at, m.id + LIMIT %s + """ + params.append(limit + 1) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute(sql, params) + rows = cur.fetchall() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = _encode_cursor(last["created_at"], str(last["id"])) + rows = rows[:limit] + + return {"items": [row_to_message(r) for r in rows], "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/app/routes/sessions.py b/src/app/routes/sessions.py new file mode 100644 index 00000000..8df7889b --- /dev/null +++ b/src/app/routes/sessions.py @@ -0,0 +1,112 @@ +import base64 +import json +import redis +from fastapi import APIRouter, Depends, Request +from fastapi.responses import Response +from psycopg.rows import dict_row + +from ..config import get_settings +from ..db import connection +from ..errors import ApiError +from ..security import ( + new_session_token, + session_expiry, + token_digest, + verify_password, +) +from ..serializers import row_to_user + +router = APIRouter(prefix="/api/v1") + +WWW_BASIC = 'Basic realm="messenger", charset="UTF-8"' +WWW_BEARER = 'Bearer realm="messenger"' + +# Подключение к Redis +redis_client = redis.Redis.from_url(get_settings().redis_url, decode_responses=True) + +def _parse_basic(header: str | None) -> tuple[str, str]: + if not header or not header.lower().startswith("basic "): + raise ApiError(401, "unauthorized", "Требуется Basic", {"WWW-Authenticate": WWW_BASIC}) + try: + decoded = base64.b64decode(header[6:].strip(), validate=True).decode("utf-8") + except Exception: + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) + + if ":" not in decoded: + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) + + username, password = decoded.split(":", 1) + return username, password + +@router.post("/auth/sessions", status_code=201) +def create_session(request: Request): + header = request.headers.get("Authorization") + username, password = _parse_basic(header) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ) + row = cur.fetchone() + + if row is None or not verify_password(password, row["password_hash"]): + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) + + settings = get_settings() + token = new_session_token() + ttl = settings.session_ttl_seconds + digest = token_digest(token) + + # СОХРАНЯЕМ В REDIS + redis_client.setex(f"session:{digest}", ttl, str(row["id"])) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": session_expiry(ttl).isoformat().replace("+00:00", "Z"), + "user": row_to_user(row), + } + +def _extract_bearer(request: Request) -> str: + header = request.headers.get("Authorization") + if not header or not header.lower().startswith("bearer "): + raise ApiError(401, "unauthorized", "Требуется Bearer", {"WWW-Authenticate": WWW_BEARER}) + token = header[7:].strip() + if not token: + raise ApiError(401, "unauthorized", "Требуется Bearer", {"WWW-Authenticate": WWW_BEARER}) + return token + +def require_actor(request: Request) -> dict: + token = _extract_bearer(request) + digest = token_digest(token) + + # ЧИТАЕМ ИЗ REDIS + user_id = redis_client.get(f"session:{digest}") + if user_id is None: + raise ApiError(401, "unauthorized", "Сессия недействительна или истекла", {"WWW-Authenticate": WWW_BEARER}) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ) + row = cur.fetchone() + + if row is None: + raise ApiError(401, "unauthorized", "Пользователь не найден", {"WWW-Authenticate": WWW_BEARER}) + + return { + "user": row_to_user(row), + "user_id": user_id, + "token_hash": digest, + } + +@router.delete("/auth/sessions/current", status_code=204) +def revoke_session(request: Request, actor: dict = Depends(require_actor)): + # УДАЛЯЕМ ИЗ REDIS + redis_client.delete(f"session:{actor['token_hash']}") + return Response(status_code=204) \ No newline at end of file diff --git a/src/app/routes/users.py b/src/app/routes/users.py new file mode 100644 index 00000000..773f8011 --- /dev/null +++ b/src/app/routes/users.py @@ -0,0 +1,52 @@ +from fastapi import APIRouter, Depends +from pydantic import BaseModel, ConfigDict, Field, field_validator +import psycopg + +from ..db import transaction +from ..errors import ApiError +from ..security import hash_password +from ..serializers import row_to_user +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("display_name must not be blank") + return v + + +@router.post("/users", status_code=201) +def create_user(body: CreateUser): + pwd_hash = hash_password(body.password) + try: + with transaction() as conn: + with conn.cursor(row_factory=psycopg.rows.dict_row) as cur: + cur.execute( + """ + INSERT INTO users (username, display_name, password_hash) + VALUES (%s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (body.username, body.display_name, pwd_hash), + ) + row = cur.fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя занято") + + return row_to_user(row) + + +@router.get("/users/me") +def current_user(actor: dict = Depends(require_actor)): + return actor["user"] \ No newline at end of file diff --git a/src/app/s3.py b/src/app/s3.py new file mode 100644 index 00000000..79c65916 --- /dev/null +++ b/src/app/s3.py @@ -0,0 +1,39 @@ +import boto3 +from botocore.config import Config +from .config import get_settings + + +def get_s3_client(): + s = get_settings() + return boto3.client( + "s3", + endpoint_url=s.s3_endpoint, + aws_access_key_id=s.s3_access_key, + aws_secret_access_key=s.s3_secret_key, + region_name=s.s3_region, + config=Config(signature_version="s3v4"), + ) + + +def upload_fileobj(fileobj, key: str, content_type: str): + client = get_s3_client() + s = get_settings() + client.upload_fileobj( + fileobj, + s.s3_bucket, + key, + ExtraArgs={"ContentType": content_type}, + ) + + +def download_fileobj(key: str): + client = get_s3_client() + s = get_settings() + obj = client.get_object(Bucket=s.s3_bucket, Key=key) + return obj["Body"] + + +def delete_file(key: str): + client = get_s3_client() + s = get_settings() + client.delete_object(Bucket=s.s3_bucket, Key=key) \ No newline at end of file diff --git a/src/app/security.py b/src/app/security.py new file mode 100644 index 00000000..408d6860 --- /dev/null +++ b/src/app/security.py @@ -0,0 +1,34 @@ +import hashlib +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError + +_ph = PasswordHasher() # Argon2id по умолчанию + + +def hash_password(password: str) -> str: + return _ph.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + return _ph.verify(password_hash, password) + except VerifyMismatchError: + return False + except Exception: + return False + + +def new_session_token() -> str: + # 32 байта = 256 бит энтропии; url-safe hex без паддинга. + return secrets.token_urlsafe(32) + + +def token_digest(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def session_expiry(ttl_seconds: int) -> datetime: + return datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds) \ No newline at end of file diff --git a/src/app/serializers.py b/src/app/serializers.py new file mode 100644 index 00000000..3b2bad4e --- /dev/null +++ b/src/app/serializers.py @@ -0,0 +1,35 @@ +from datetime import datetime + + +def iso_z(dt: datetime) -> str: + """RFC3339 UTC с 'Z'. Postgres отдаёт timestamptz с '+00:00'.""" + return dt.isoformat().replace("+00:00", "Z") + + +def row_to_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso_z(row["created_at"]), + } + + +def row_to_chat(row) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": iso_z(row["created_at"]), + } + + +def row_to_message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso_z(row["created_at"]), + "attachment_ids": row.get("attachment_ids", []), + } \ No newline at end of file diff --git a/src/migrations/001_init.sql b/src/migrations/001_init.sql new file mode 100644 index 00000000..da374b89 --- /dev/null +++ b/src/migrations/001_init.sql @@ -0,0 +1,52 @@ +-- 001_init.sql +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS sessions_user_id_idx ON sessions(user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_at_idx ON sessions(expires_at); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members(user_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS messages_chat_order_idx ON messages(chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + scope text NOT NULL, -- method + " " + path + key text NOT NULL, + request_hash text NOT NULL, + status_code int NOT NULL, + response_json jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, scope, key) +); \ No newline at end of file diff --git a/src/migrations/002_attachments.sql b/src/migrations/002_attachments.sql new file mode 100644 index 00000000..64951308 --- /dev/null +++ b/src/migrations/002_attachments.sql @@ -0,0 +1,23 @@ +-- src/migrations/002_attachments.sql +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + PRIMARY KEY (message_id, attachment_id) +); \ No newline at end of file diff --git a/src/migrations/003_attachment_variants.sql b/src/migrations/003_attachment_variants.sql new file mode 100644 index 00000000..f08e2e25 --- /dev/null +++ b/src/migrations/003_attachment_variants.sql @@ -0,0 +1,13 @@ +-- src/migrations/003_attachment_variants.sql +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + name text NOT NULL CHECK (name IN ('thumbnail','crop','optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/png','image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width int NOT NULL CHECK (width > 0), + height int NOT NULL CHECK (height > 0), + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, name) +); \ No newline at end of file diff --git a/src/migrations/004_outbox.sql b/src/migrations/004_outbox.sql new file mode 100644 index 00000000..9790f60b --- /dev/null +++ b/src/migrations/004_outbox.sql @@ -0,0 +1,19 @@ +-- src/migrations/004_outbox.sql + +-- Таблица для Transactional Outbox +CREATE TABLE IF NOT EXISTS outbox_messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + processed_at timestamptz +); + +CREATE INDEX IF NOT EXISTS outbox_unprocessed_idx ON outbox_messages (created_at) WHERE processed_at IS NULL; + +-- Таблица для отслеживания попыток обработки +CREATE TABLE IF NOT EXISTS attachment_retries ( + attachment_id uuid PRIMARY KEY REFERENCES attachments(id) ON DELETE CASCADE, + attempts int NOT NULL DEFAULT 0, + last_error text +); \ No newline at end of file diff --git a/student-tests/test_level4.py b/student-tests/test_level4.py new file mode 100644 index 00000000..801466e7 --- /dev/null +++ b/student-tests/test_level4.py @@ -0,0 +1,113 @@ +import base64 +import io +import json +import os +import random +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + +BASE_URL = "http://localhost:8080" + +def req(method, path, data=None, headers=None, raw=None): + r = urllib.request.Request(BASE_URL + path, method=method) + if headers: + for k, v in headers.items(): + r.add_header(k, v) + if data: + r.add_header("Content-Type", "application/json") + r.data = json.dumps(data).encode("utf-8") + if raw: + r.data = raw + try: + with urllib.request.urlopen(r) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + +def setup(): + u = "test4_" + str(random.randint(1000, 9999)) + _, body = req("POST", "/api/v1/users", {"username": u, "display_name": "Тест", "password": "testpassword1234"}) + user = json.loads(body) + + cred = base64.b64encode((u + ":testpassword1234").encode()).decode() + _, body = req("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + token = json.loads(body)["token"] + + headers = {"Authorization": "Bearer " + token} + _, body = req("POST", "/api/v1/chats", {"title": "Тестовый чат", "member_ids": [user["id"]]}, headers) + chat = json.loads(body) + return user, chat, headers + +def main(): + user, chat, headers = setup() + print(f"Пользователь {user['id']} и чат {chat['id']} созданы.") + + chat_url = f"/api/v1/chats/{chat['id']}/attachments" + + # 1. Слишком большой файл (11 МБ) + print("1. Тест: слишком большой файл...") + big_data = b"\x89PNG\r\n\x1a\n" + b"A" * (11 * 1024 * 1024) + status, _ = req("POST", chat_url + "?filename=big.png", headers={**headers, "Content-Type": "image/png"}, raw=big_data) + assert status == 413, f"Ожидали 413, получили {status}" + print(" OK (413)") + + # 2. Битый файл (MIME PNG, но внутри мусор) + print("2. Тест: битый файл...") + broken_data = b"\x89PNG\r\n\x1a\n" + b"broken data" + status, _ = req("POST", chat_url + "?filename=broken.png", headers={**headers, "Content-Type": "image/png"}, raw=broken_data) + assert status == 415, f"Ожидали 415, получили {status}" + print(" OK (415)") + + # 3. Подмена MIME (говорим image/png, а внутри текст) + print("3. Тест: подмена MIME...") + text_data = b"Hello, this is not an image!" + status, _ = req("POST", chat_url + "?filename=fake.png", headers={**headers, "Content-Type": "image/png"}, raw=text_data) + assert status == 415, f"Ожидали 415, получили {status}" + print(" OK (415)") + + # 4. Опасное имя файла (path traversal) + print("4. Тест: опасное имя файла...") + with open("tests/fixtures/sample.png", "rb") as f: + valid_png = f.read() + status, _ = req("POST", chat_url + "?filename=../../etc/passwd", headers={**headers, "Content-Type": "image/png"}, raw=valid_png) + assert status == 400, f"Ожидали 400, получили {status}" + print(" OK (400)") + + # 5. Успешная загрузка (чтобы проверить доступ) + print("5. Тест: успешная загрузка...") + status, body = req("POST", chat_url + "?filename=ok.png", headers={**headers, "Content-Type": "image/png"}, raw=valid_png) + assert status == 201, f"Ожидали 201, получили {status}" + att = json.loads(body) + print(f" OK (201), ID: {att['id']}") + + # 6. Чужой доступ + print("6. Тест: чужой доступ...") + u_eve = "eve_" + str(random.randint(1000, 9999)) + req("POST", "/api/v1/users", {"username": u_eve, "display_name": "Eve", "password": "testpassword1234"}) + cred_eve = base64.b64encode((u_eve + ":testpassword1234").encode()).decode() + _, body = req("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred_eve}) + token_eve = json.loads(body)["token"] + headers_eve = {"Authorization": "Bearer " + token_eve} + + status, _ = req("GET", f"/api/v1/attachments/{att['id']}", headers=headers_eve) + assert status == 404, f"Ожидали 404 (чужой), получили {status}" + print(" OK (404)") + + # 7. Нагрузочный тест (10 параллельных загрузок) + print("7. Тест: 10 параллельных загрузок (проверка RAM)...") + def upload_worker(): + status, _ = req("POST", chat_url + "?filename=parallel.png", headers={**headers, "Content-Type": "image/png"}, raw=valid_png) + return status + + with ThreadPoolExecutor(max_workers=10) as pool: + results = list(pool.map(lambda _: upload_worker(), range(10))) + + assert all(s == 201 for s in results), f"Не все загрузки успешны: {results}" + print(" OK (все 10 успешно)") + + print("\nPASS: Уровень 4 пройден! (Все негативные тесты и нагрузка работают)") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test_lab5.sh b/test_lab5.sh new file mode 100644 index 00000000..4a5f0708 --- /dev/null +++ b/test_lab5.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Базовый URL +BASE_URL="http://localhost:8080" + +# 1. Создаем уникального пользователя (добавляем случайное число, чтобы не было конфликта) +RANDOM_NUM=$RANDOM +USERNAME="test_user_$RANDOM_NUM" +PASSWORD="verysecretpassword123" + +echo "1. Создаем пользователя: $USERNAME..." +RESPONSE=$(echo "{\"username\": \"$USERNAME\", \"display_name\": \"Тест\", \"password\": \"$PASSWORD\"}" | curl -s -X POST $BASE_URL/api/v1/users -H "Content-Type: application/json" -d @-) +USER_ID=$(echo $RESPONSE | jq -r '.id') +echo " Пользователь создан. ID: $USER_ID" + +# 2. Логинимся +echo "2. Логинимся..." +RESPONSE=$(curl -s -X POST $BASE_URL/api/v1/auth/sessions -H "Authorization: Basic $(echo -n "$USERNAME:$PASSWORD" | base64)") +TOKEN=$(echo $RESPONSE | jq -r '.token') +if [ "$TOKEN" == "null" ]; then + echo " Ошибка логина: $RESPONSE" + exit 1 +fi +echo " Токен получен." + +# 3. Создаем чат +echo "3. Создаем чат..." +RESPONSE=$(echo "{\"title\": \"Чат для теста\", \"member_ids\": [\"$USER_ID\"]}" | curl -s -X POST $BASE_URL/api/v1/chats -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d @-) +CHAT_ID=$(echo $RESPONSE | jq -r '.id') +echo " Чат создан. ID: $CHAT_ID" + +# 4. Загружаем картинку +echo "4. Загружаем картинку (sample.png)..." +# Убедимся, что файл существует +if [ ! -f "tests/fixtures/sample.png" ]; then + echo " ОШИБКА: Файл tests/fixtures/sample.png не найден!" + exit 1 +fi + +RESPONSE=$(curl -s -X POST "$BASE_URL/api/v1/chats/$CHAT_ID/attachments?filename=sample.png" \ + -H "Content-Type: image/png" \ + -H "Authorization: Bearer $TOKEN" \ + --data-binary "@tests/fixtures/sample.png") + +ATTACHMENT_ID=$(echo $RESPONSE | jq -r '.id') +STATUS=$(echo $RESPONSE | jq -r '.status') + +echo " Вложение загружено. ID: $ATTACHMENT_ID" +echo " Статус сразу после загрузки: $STATUS (ожидаем queued)" + +if [ "$STATUS" != "queued" ]; then + echo " ВНИМАНИЕ: Статус не queued! Проверь API." +fi + +# 5. Ждем, пока воркер обработает +echo "5. Ждем 3 секунды, пока воркер обработает картинку..." +sleep 3 + +# 6. Проверяем статус +echo "6. Проверяем итоговый статус вложения..." +RESPONSE=$(curl -s -X GET "$BASE_URL/api/v1/attachments/$ATTACHMENT_ID" -H "Authorization: Bearer $TOKEN") +FINAL_STATUS=$(echo $RESPONSE | jq -r '.status') +VARIANTS_COUNT=$(echo $RESPONSE | jq '.variants | length') + +echo " Итоговый статус: $FINAL_STATUS (ожидаем ready)" +echo " Количество вариантов: $VARIANTS_COUNT (ожидаем 3)" + +if [ "$FINAL_STATUS" == "ready" ] && [ "$VARIANTS_COUNT" -eq 3 ]; then + echo "✅ УСПЕХ! Асинхронная обработка работает." +else + echo "❌ ОШИБКА! Воркер не обработал файл. Проверь логи: docker compose logs worker" +fi \ No newline at end of file diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/worker/main.py b/worker/main.py new file mode 100644 index 00000000..d6d6a112 --- /dev/null +++ b/worker/main.py @@ -0,0 +1,194 @@ +import asyncio +import io +import json +import logging +import hashlib +import random + +import aio_pika +from PIL import Image +from psycopg.rows import dict_row + +from src.app.config import get_settings +from src.app.db import init_pool, transaction +from src.app.s3 import download_fileobj, upload_fileobj + +# ИМПОРТ ФОНОВОГО ПРОЦЕССА OUTBOX +from worker.outbox_publisher import publish_loop + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("worker") + +MAX_PIXELS = 40_000_000 + + +class PermanentError(Exception): + """Ошибка, которую нет смысла повторять (битый файл).""" + pass + + +def generate_variants(img: Image.Image): + variants = {} + + thumb = img.copy() + if thumb.mode in ("RGBA", "P"): + thumb = thumb.convert("RGB") + thumb.thumbnail((256, 256)) + variants["thumbnail"] = (thumb, "image/jpeg", "JPEG", {"quality": 85}) + + crop = img.copy() + if crop.mode in ("RGBA", "P"): + crop = crop.convert("RGB") + min_dim = min(crop.size) + left = (crop.width - min_dim) // 2 + top = (crop.height - min_dim) // 2 + crop = crop.crop((left, top, left + min_dim, top + min_dim)) + crop = crop.resize((128, 128), Image.Resampling.LANCZOS) + variants["crop"] = (crop, "image/jpeg", "JPEG", {"quality": 85}) + + opt = img.copy() + if opt.mode in ("RGBA", "P", "L"): + opt = opt.convert("RGB") + opt.info.clear() + variants["optimized"] = (opt, "image/webp", "WEBP", {"quality": 80, "method": 4}) + + return variants + + +async def _re_publish(body: bytes): + settings = get_settings() + connection = await aio_pika.connect_robust(settings.broker_url) + async with connection: + channel = await connection.channel() + await channel.declare_queue(settings.broker_queue, durable=True) + msg = aio_pika.Message(body=body, delivery_mode=aio_pika.DeliveryMode.PERSISTENT) + await channel.default_exchange.publish(msg, routing_key=settings.broker_queue) + + +async def _send_to_dlq(body: bytes): + settings = get_settings() + connection = await aio_pika.connect_robust(settings.broker_url) + async with connection: + channel = await connection.channel() + await channel.declare_queue("dead_letters", durable=True) + msg = aio_pika.Message(body=body, delivery_mode=aio_pika.DeliveryMode.PERSISTENT) + await channel.default_exchange.publish(msg, routing_key="dead_letters") + + +async def process_message(message: aio_pika.IncomingMessage): + async with message.process(requeue=False, reject_on_redelivered=False): + data = json.loads(message.body) + att_id = data["attachment_id"] + obj_key = data["object_key"] + + logger.info(f"Processing attachment {att_id}") + + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute("SELECT attempts FROM attachment_retries WHERE attachment_id = %s", (att_id,)) + row = cur.fetchone() + attempts = row["attempts"] if row else 0 + + if attempts >= 5: + logger.error(f"Max retries (5) reached for {att_id}. Sending to DLQ.") + cur.execute("UPDATE attachments SET status='failed', error_code='max_retries_exceeded', updated_at=now() WHERE id=%s", (att_id,)) + await _send_to_dlq(message.body) + return + + cur.execute( + """ + INSERT INTO attachment_retries (attachment_id, attempts) + VALUES (%s, 1) + ON CONFLICT (attachment_id) DO UPDATE SET attempts = attachment_retries.attempts + 1 + """, (att_id,) + ) + cur.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,)) + + try: + body = download_fileobj(obj_key) + img = Image.open(body) + + width, height = img.size + if width * height > MAX_PIXELS: + raise PermanentError(f"Image too large: {width}x{height} pixels") + + img.load() + + variants = generate_variants(img) + + with transaction() as conn: + with conn.cursor() as cur: + for name, (var_img, ct, fmt, save_args) in variants.items(): + buf = io.BytesIO() + var_img.save(buf, format=fmt, **save_args) + + var_bytes = buf.getvalue() + sha256 = hashlib.sha256(var_bytes).hexdigest() + var_key = f"chats/{att_id}/variants/{name}" + + upload_fileobj(io.BytesIO(var_bytes), var_key, ct) + + cur.execute( + """ + INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, name) DO UPDATE SET + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + object_key = EXCLUDED.object_key + """, + (att_id, name, ct, len(var_bytes), var_img.width, var_img.height, sha256, var_key) + ) + + cur.execute("UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,)) + cur.execute("DELETE FROM attachment_retries WHERE attachment_id = %s", (att_id,)) + + logger.info(f"Successfully processed {att_id}") + + except PermanentError as e: + logger.error(f"Permanent error for {att_id}: {e}") + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s", ("invalid_image", att_id)) + + except Exception as e: + logger.error(f"Temporary error for {att_id}: {e}") + + delay = min(2 ** attempts, 60) + jitter = random.uniform(0, 3) + total_delay = delay + jitter + + logger.info(f"Retrying {att_id} in {total_delay:.2f} seconds (attempt {attempts+1}/5)") + await asyncio.sleep(total_delay) + await _re_publish(message.body) + + +async def main(): + init_pool() + settings = get_settings() + + # ЗАПУСК ФОНОВОГО ПРОЦЕССА OUTBOX + asyncio.create_task(publish_loop()) + + while True: + try: + connection = await aio_pika.connect_robust(settings.broker_url) + break + except Exception as e: + logger.warning(f"Broker not ready, waiting 5 seconds... ({e})") + await asyncio.sleep(5) + + async with connection: + channel = await connection.channel() + await channel.set_qos(prefetch_count=1) + queue = await channel.declare_queue(settings.broker_queue, durable=True) + logger.info("Worker started, waiting for messages...") + await queue.consume(process_message) + await asyncio.Future() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/worker/outbox_publisher.py b/worker/outbox_publisher.py new file mode 100644 index 00000000..14a7a8c1 --- /dev/null +++ b/worker/outbox_publisher.py @@ -0,0 +1,52 @@ +import asyncio +import json +import logging +import aio_pika +from src.app.config import get_settings +from src.app.db import transaction +from psycopg.rows import dict_row + +logger = logging.getLogger("outbox") + +async def publish_loop(): + settings = get_settings() + + # Ждем брокер + while True: + try: + connection = await aio_pika.connect_robust(settings.broker_url) + break + except Exception: + await asyncio.sleep(5) + + async with connection: + channel = await connection.channel() + await channel.declare_queue(settings.broker_queue, durable=True) + + while True: + try: + # Забираем необработанные сообщения + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id, payload FROM outbox_messages WHERE processed_at IS NULL ORDER BY created_at LIMIT 10 FOR UPDATE SKIP LOCKED" + ) + messages = cur.fetchall() + + for msg in messages: + payload = msg["payload"] + # Отправляем в RabbitMQ + message = aio_pika.Message( + body=json.dumps(payload).encode(), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT + ) + await channel.default_exchange.publish(message, routing_key=settings.broker_queue) + + # Помечаем как обработанное + cur.execute("UPDATE outbox_messages SET processed_at = now() WHERE id = %s", (msg["id"],)) + + if not messages: + await asyncio.sleep(1) # Если пусто, ждем секунду + except Exception as e: + logger.error(f"Outbox error: {e}") + await asyncio.sleep(2) \ No newline at end of file