From 2b68813b33d5ca611fa555f5ab36d954f432603f Mon Sep 17 00:00:00 2001 From: caramelteawithmint <146834163+caramelteawithmint@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:19:14 +0300 Subject: [PATCH 1/4] lab4_ --- Dockerfile | 32 +++++ compose.yaml | 120 +++++++++++++++++ nginx/nginx.conf | 38 ++++++ src/app/__init__.py | 0 src/app/config.py | 54 ++++++++ src/app/db.py | 45 +++++++ src/app/errors.py | 31 +++++ src/app/idempotency.py | 72 +++++++++++ src/app/image_check.py | 13 ++ src/app/instance.py | 15 +++ src/app/main.py | 56 ++++++++ src/app/routes/__init__.py | 0 src/app/routes/attachments.py | 198 +++++++++++++++++++++++++++++ src/app/routes/chats.py | 169 ++++++++++++++++++++++++ src/app/routes/health.py | 26 ++++ src/app/routes/messages.py | 171 +++++++++++++++++++++++++ src/app/routes/sessions.py | 131 +++++++++++++++++++ src/app/routes/users.py | 52 ++++++++ src/app/s3.py | 39 ++++++ src/app/security.py | 34 +++++ src/app/serializers.py | 35 +++++ src/migrations/001_init.sql | 52 ++++++++ src/migrations/002_attachments.sql | 20 +++ student-tests/test_level4.py | 113 ++++++++++++++++ 24 files changed, 1516 insertions(+) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 nginx/nginx.conf create mode 100644 src/app/__init__.py create mode 100644 src/app/config.py create mode 100644 src/app/db.py create mode 100644 src/app/errors.py create mode 100644 src/app/idempotency.py create mode 100644 src/app/image_check.py create mode 100644 src/app/instance.py create mode 100644 src/app/main.py create mode 100644 src/app/routes/__init__.py create mode 100644 src/app/routes/attachments.py create mode 100644 src/app/routes/chats.py create mode 100644 src/app/routes/health.py create mode 100644 src/app/routes/messages.py create mode 100644 src/app/routes/sessions.py create mode 100644 src/app/routes/users.py create mode 100644 src/app/s3.py create mode 100644 src/app/security.py create mode 100644 src/app/serializers.py create mode 100644 src/migrations/001_init.sql create mode 100644 src/migrations/002_attachments.sql create mode 100644 student-tests/test_level4.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4f150402 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install -r requirements.txt + +# Копируем код приложения, миграции и скрипты +COPY src/app ./app +COPY src/migrations ./migrations +COPY scripts ./scripts + +# не root +RUN useradd --create-home --shell /bin/bash appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=2s --start-period=10s --retries=6 \ + CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips", "*"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..a43fc3ea --- /dev/null +++ b/compose.yaml @@ -0,0 +1,120 @@ +name: mtusi-messenger + +x-api-base: &api-base + build: . + environment: + - DATABASE_URL_FILE=/run/secrets/database_url + - SESSION_TTL_SECONDS=3600 + - S3_ENDPOINT=http://minio:9000 + - S3_ACCESS_KEY=api-user + - S3_SECRET_KEY=api-secret-key-12345 + - S3_BUCKET=messenger-attachments + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + migrate: + condition: service_completed_successfully + +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger + POSTGRES_DB: messenger + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 5 + volumes: + - pgdata:/var/lib/postgresql/data + + minio: + image: pgsty/silo:latest + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + volumes: + - miniodata:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 5 + # Порт 9000 (API) и 9001 (Консоль) НЕ пробрасываем наружу, чтобы админка была закрыта. + # Контейнеры общаются по внутренней сети Docker. + + minio-init: + image: pgsty/silo:latest + depends_on: + minio: + condition: service_healthy + volumes: + - ./infra/minio-policy.json:/tmp/minio-policy.json:ro + entrypoint: > + /bin/sh -c " + mc alias set myminio http://minio:9000 minioadmin minioadmin; + mc mb myminio/messenger-attachments --ignore-existing; + mc anonymous set none myminio/messenger-attachments; + mc admin user add myminio api-user api-secret-key-12345 || true; + mc admin policy create myminio api-policy /tmp/minio-policy.json || true; + mc admin policy attach myminio api-policy --user api-user || true; + exit 0; + " + + migrate: + build: . + command: ["python", "scripts/migrate.py"] + environment: + - DATABASE_URL_FILE=/run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + + api1: + <<: *api-base + environment: + - INSTANCE_ID=api-1 + - DATABASE_URL_FILE=/run/secrets/database_url + - SESSION_TTL_SECONDS=3600 + - S3_ENDPOINT=http://minio:9000 + - S3_ACCESS_KEY=api-user + - S3_SECRET_KEY=api-secret-key-12345 + - S3_BUCKET=messenger-attachments + + api2: + <<: *api-base + environment: + - INSTANCE_ID=api-2 + - DATABASE_URL_FILE=/run/secrets/database_url + - SESSION_TTL_SECONDS=3600 + - S3_ENDPOINT=http://minio:9000 + - S3_ACCESS_KEY=api-user + - S3_SECRET_KEY=api-secret-key-12345 + - S3_BUCKET=messenger-attachments + + load_balancer: + image: nginx:1.27-alpine + ports: + - "8080:8080" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - api1 + - api2 + +secrets: + database_url: + file: ./secrets/database_url + +volumes: + pgdata: + miniodata: \ No newline at end of file diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..07ec076d --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,38 @@ +worker_processes auto; + +events { worker_connections 1024; } + +http { + upstream api_backend { + # least_conn лучше распределяет запросы, чем round-robin, + # особенно когда одна реплика чуть медленнее. + least_conn; + server api1:8080 max_fails=1 fail_timeout=2s; + server api2:8080 max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + + # Проверка, что LB жив (без проксирования в API) + location = /lb-health { + access_log off; + return 200 'ok'; + add_header Content-Type text/plain; + } + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Важно для быстрого отключения упавшей реплики + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_connect_timeout 2s; + proxy_read_timeout 10s; + } + } +} \ No newline at end of file diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 00000000..6d3f0dd4 --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,54 @@ +from functools import lru_cache +from pathlib import Path +from pydantic_settings import BaseSettings, SettingsConfigDict + + +def _read_secret(path: str | None) -> str | None: + """Прочитать секрет из файла, если задан путь. Иначе None.""" + if not path: + return None + p = Path(path) + if not p.is_file(): + return None + return p.read_text(encoding="utf-8").strip() + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=None, extra="ignore") + + # Основные настройки приложения + app_host: str = "0.0.0.0" + app_port: int = 8080 + app_env: str = "lab" + instance_id: str = "" # пусто -> сгенерируется в instance.py + log_level: str = "info" + + # Сессии и идемпотентность + session_ttl_seconds: int = 3600 + idempotency_ttl_seconds: int = 86400 + + # Postgres: либо прямая строка, либо файл-секрет (приоритет у файла). + database_url: str = "" + database_url_file: str = "/run/secrets/database_url" + + # S3 / MinIO (для 4-й лабы) + s3_endpoint: str = "http://minio:9000" + s3_access_key: str = "minioadmin" + s3_secret_key: str = "minioadmin" + s3_bucket: str = "messenger-attachments" + s3_region: str = "us-east-1" + + @property + def db_dsn(self) -> str: + from_file = _read_secret(self.database_url_file) + dsn = from_file or self.database_url + if not dsn: + raise RuntimeError( + "DATABASE_URL не задан: укажи DATABASE_URL или смонтируй DATABASE_URL_FILE" + ) + return dsn + + +@lru_cache +def get_settings() -> Settings: + return Settings() \ No newline at end of file diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 00000000..870b6ee2 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,45 @@ +from contextlib import contextmanager +from typing import Iterator + +import psycopg +from psycopg_pool import ConnectionPool + +from .config import get_settings + +_pool: ConnectionPool | None = None + + +def init_pool() -> ConnectionPool: + global _pool + if _pool is None: + _pool = ConnectionPool( + conninfo=get_settings().db_dsn, + min_size=1, + max_size=10, + timeout=5, + open=True, + ) + return _pool + + +def close_pool() -> None: + global _pool + if _pool is not None: + _pool.close() + _pool = None + + +@contextmanager +def connection() -> Iterator[psycopg.Connection]: + pool = init_pool() + with pool.connection() as conn: + yield conn + + +@contextmanager +def transaction() -> Iterator[psycopg.Connection]: + """Явная транзакция: commit при успехе, rollback при исключении.""" + pool = init_pool() + with pool.connection() as conn: + with conn.transaction(): + yield conn \ No newline at end of file diff --git a/src/app/errors.py b/src/app/errors.py new file mode 100644 index 00000000..c7e071c0 --- /dev/null +++ b/src/app/errors.py @@ -0,0 +1,31 @@ +from typing import Any + +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + """Наше приложение бросает это; handler превращает в JSON-ответ.""" + + def __init__(self, status: int, code: str, message: str, headers: dict[str, str] | None = None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def error_response( + request: Request, + status: int, + code: str, + message: str, + extra_headers: dict[str, str] | None = None, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") or "unknown" + body: dict[str, Any] = { + "error": {"code": code, "message": message, "request_id": request_id} + } + headers = {"X-Request-Id": request_id} + if extra_headers: + headers.update(extra_headers) + return JSONResponse(status_code=status, content=body, headers=headers) \ No newline at end of file diff --git a/src/app/idempotency.py b/src/app/idempotency.py new file mode 100644 index 00000000..9be16d63 --- /dev/null +++ b/src/app/idempotency.py @@ -0,0 +1,72 @@ +import hashlib +import json +from typing import Any, Callable + +import psycopg + +from .db import transaction +from .errors import ApiError + + +def _body_hash(body: dict[str, Any]) -> str: + raw = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def run_idempotent( + *, + user_id: str, + scope: str, + key: str, + body: dict[str, Any], + action: Callable[[psycopg.Connection], tuple[int, dict[str, Any]]], +) -> tuple[int, dict[str, Any]]: + """Атомарно: либо побочный эффект + сохранённый ответ, либо повтор.""" + h = _body_hash(body) + try: + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO idempotency_keys + (user_id, scope, key, request_hash, status_code, response_json) + VALUES (%s, %s, %s, %s, 0, '{}'::jsonb) + """, + (user_id, scope, key, h), + ) + result = action(conn) + status, payload = result + with conn.cursor() as cur: + cur.execute( + """ + UPDATE idempotency_keys + SET status_code = %s, response_json = %s::jsonb + WHERE user_id = %s AND scope = %s AND key = %s + """, + (status, json.dumps(payload, ensure_ascii=False), user_id, scope, key), + ) + return status, payload + except psycopg.errors.UniqueViolation: + pass + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT request_hash, status_code, response_json + FROM idempotency_keys + WHERE user_id = %s AND scope = %s AND key = %s + """, + (user_id, scope, key), + ) + row = cur.fetchone() + + if row is None: + raise ApiError(409, "conflict", "Конфликт идемпотентности, повторите") + + existing_hash, status, payload = row + if existing_hash != h: + raise ApiError(409, "conflict", "Тот же ключ с другим телом запроса") + if status == 0: + raise ApiError(409, "conflict", "Повтор ещё не завершён") + return status, payload \ No newline at end of file diff --git a/src/app/image_check.py b/src/app/image_check.py new file mode 100644 index 00000000..e48b2307 --- /dev/null +++ b/src/app/image_check.py @@ -0,0 +1,13 @@ +def detect_image_type(data: bytes) -> str | None: + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + return None + + +def check_image(first_bytes: bytes, allowed_types: list[str]) -> tuple[str, int, int]: + content_type = detect_image_type(first_bytes) + if not content_type or content_type not in allowed_types: + raise ValueError("Invalid or unsupported image type") + return content_type, 0, 0 \ No newline at end of file diff --git a/src/app/instance.py b/src/app/instance.py new file mode 100644 index 00000000..f5419bff --- /dev/null +++ b/src/app/instance.py @@ -0,0 +1,15 @@ +import os +import secrets + +from .config import get_settings + + +def _build_instance_id() -> str: + s = get_settings() + if s.instance_id: + return s.instance_id + # Короткий случайный alias; при желании можно задать INSTANCE_ID в env. + return "api-" + secrets.token_hex(6) + + +INSTANCE_ID: str = _build_instance_id() \ No newline at end of file diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 00000000..d665be63 --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,56 @@ +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from . import db +from .errors import ApiError, error_response +from .instance import INSTANCE_ID +from .routes import health, users, sessions, chats, messages, attachments + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + db.init_pool() + except Exception: + pass + yield + db.close_pool() + + +app = FastAPI(title="MTUSI Messenger", version="1.4.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Неверный запрос") + + +@app.exception_handler(Exception) +async def unhandled_handler(request: Request, exc: Exception): + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +app.include_router(health.router) +app.include_router(users.router) +app.include_router(sessions.router) +app.include_router(chats.router) +app.include_router(messages.router) +app.include_router(attachments.router) \ No newline at end of file diff --git a/src/app/routes/__init__.py b/src/app/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/routes/attachments.py b/src/app/routes/attachments.py new file mode 100644 index 00000000..458996b4 --- /dev/null +++ b/src/app/routes/attachments.py @@ -0,0 +1,198 @@ +import hashlib +import io +import tempfile +import uuid + +from fastapi import APIRouter, Depends, Request, Query +from fastapi.responses import StreamingResponse +from psycopg.rows import dict_row + +from ..db import transaction, connection +from ..errors import ApiError +from ..s3 import upload_fileobj, download_fileobj, delete_file +from ..image_check import check_image +from ..serializers import iso_z +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + +MAX_SIZE = 10 * 1024 * 1024 + + +@router.post("/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., max_length=255), + actor: dict = Depends(require_actor), +): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + if "/" in filename or "\\" in filename or "\0" in filename: + raise ApiError(400, "invalid_request", "Опасное имя файла") + + # Проверяем доступ к чату + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, actor["user_id"]), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Чат не найден") + + sha256 = hashlib.sha256() + size = 0 + first_chunk = b"" + + # Читаем потоково во временный файл (чтобы не забивать RAM) + with tempfile.SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+b") as spooled: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_SIZE: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + if not first_chunk: + first_chunk = chunk[:16] # для детекта MIME + sha256.update(chunk) + spooled.write(chunk) + + if size == 0: + raise ApiError(400, "invalid_request", "Пустой файл") + + # Полная проверка картинки через Pillow + try: + spooled.seek(0) + content = spooled.read() + content_type, _, _ = check_image(first_chunk, ["image/png", "image/jpeg"]) + from PIL import Image + Image.open(io.BytesIO(content)).verify() + except Exception: + raise ApiError(415, "unsupported_media_type", "Неподдерживаемый или битый формат") + + sha256_hex = sha256.hexdigest() + object_key = f"chats/{chat_id}/{uuid.uuid4().hex}" + + # Загружаем в S3 + try: + spooled.seek(0) + upload_fileobj(spooled, object_key, content_type) + except Exception as e: + print(f"S3 error: {e}") + raise ApiError(503, "unavailable", "S3 недоступен") + + # Сохраняем в БД (с компенсацией) + try: + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + INSERT INTO attachments (chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s) + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, created_at + """, + (chat_id, actor["user_id"], filename, content_type, size, sha256_hex, object_key), + ) + att = cur.fetchone() + except Exception as e: + # КОМПЕНСАЦИЯ: удаляем из S3, если БД упала + print(f"DB failed, compensating S3: {e}") + try: + delete_file(object_key) + except Exception as s3_e: + print(f"Failed to compensate S3: {s3_e}") + raise ApiError(500, "internal_error", "Ошибка сохранения") + + return { + "id": str(att["id"]), + "chat_id": str(att["chat_id"]), + "owner_id": str(att["owner_id"]), + "filename": att["filename"], + "content_type": att["content_type"], + "size_bytes": att["size_bytes"], + "sha256": att["sha256"], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": iso_z(att["created_at"]), + "updated_at": iso_z(att["created_at"]), + } + + +@router.get("/attachments/{attachment_id}") +def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): + try: + uuid.UUID(attachment_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный id") + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (actor["user_id"], attachment_id), + ) + att = cur.fetchone() + + if att is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + return { + "id": str(att["id"]), + "chat_id": str(att["chat_id"]), + "owner_id": str(att["owner_id"]), + "filename": att["filename"], + "content_type": att["content_type"], + "size_bytes": att["size_bytes"], + "sha256": att["sha256"], + "status": "ready", + "variants": [], + "error_code": None, + "created_at": iso_z(att["created_at"]), + "updated_at": iso_z(att["created_at"]), + } + + +@router.get("/attachments/{attachment_id}/content") +def download_attachment(attachment_id: str, actor: dict = Depends(require_actor)): + try: + uuid.UUID(attachment_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный id") + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT a.object_key, a.content_type, a.filename + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (actor["user_id"], attachment_id), + ) + att = cur.fetchone() + + if att is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + try: + body = download_fileobj(att["object_key"]) + except Exception: + raise ApiError(503, "unavailable", "S3 недоступен") + + def iter_body(): + yield from body.iter_chunks(1024 * 1024) + + return StreamingResponse( + iter_body(), + media_type=att["content_type"], + headers={"Content-Disposition": f'attachment; filename="{att["filename"]}"'}, + ) \ No newline at end of file diff --git a/src/app/routes/chats.py b/src/app/routes/chats.py new file mode 100644 index 00000000..5c547465 --- /dev/null +++ b/src/app/routes/chats.py @@ -0,0 +1,169 @@ +import base64 +import uuid +from datetime import datetime +from typing import Optional + +import psycopg +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel, ConfigDict, Field, field_validator +from psycopg.rows import dict_row + +from ..db import connection, transaction +from ..errors import ApiError +from ..serializers import row_to_chat +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("title must not be blank") + return v + + @field_validator("member_ids") + @classmethod + def unique_members(cls, v: list[str]) -> list[str]: + if len(set(v)) != len(v): + raise ValueError("duplicate member_ids") + for s in v: + try: + uuid.UUID(s) + except ValueError: + raise ValueError("invalid uuid in member_ids") + return v + + +def _encode_cursor(created_at: datetime, id_) -> str: + raw = f"{created_at.isoformat()}|{id_}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def _decode_cursor(cursor: str): + pad = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + ts, id_ = raw.split("|", 1) + return datetime.fromisoformat(ts), id_ + + +@router.post("/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(require_actor)): + me = actor["user_id"] + if str(me) not in body.member_ids: + raise ApiError(400, "invalid_request", "Вызывающий должен входить в member_ids") + + try: + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id FROM users WHERE id = ANY(%s::uuid[])", + (body.member_ids,), + ) + found = {str(r["id"]) for r in cur.fetchall()} + missing = set(body.member_ids) - found + if missing: + raise ApiError(400, "invalid_request", "Неизвестные участники") + + cur.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (body.title,), + ) + chat = cur.fetchone() + + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat["id"], m) for m in body.member_ids], + ) + except ApiError: + raise + except psycopg.errors.ForeignKeyViolation: + raise ApiError(400, "invalid_request", "Неизвестные участники") + + chat["member_ids"] = body.member_ids + return row_to_chat(chat) + + +@router.get("/chats") +def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + actor: dict = Depends(require_actor), +): + params: list = [] + where = ["cm.user_id = %s"] + params.append(actor["user_id"]) + + if cursor: + try: + c_ts, c_id = _decode_cursor(cursor) + except Exception: + raise ApiError(400, "invalid_request", "Неверный cursor") + where.append("(c.created_at, c.id) > (%s, %s)") + params.extend([c_ts, c_id]) + + sql = f""" + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm2.user_id::text + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE {' AND '.join(where)} + ORDER BY c.created_at, c.id + LIMIT %s + """ + params.append(limit + 1) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute(sql, params) + rows = cur.fetchall() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = _encode_cursor(last["created_at"], str(last["id"])) + rows = rows[:limit] + + return {"items": [row_to_chat(r) for r in rows], "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}") +def get_chat(chat_id: str, actor: dict = Depends(require_actor)): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm2.user_id::text + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE c.id = %s AND cm.user_id = %s + """, + (chat_id, actor["user_id"]), + ) + row = cur.fetchone() + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return row_to_chat(row) \ No newline at end of file diff --git a/src/app/routes/health.py b/src/app/routes/health.py new file mode 100644 index 00000000..572ed6d0 --- /dev/null +++ b/src/app/routes/health.py @@ -0,0 +1,26 @@ +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from ..db import connection +from ..errors import error_response + +router = APIRouter() + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(request: Request): + try: + with connection() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() + except Exception: + return error_response( + request, 503, "unavailable", "Зависимость недоступна" + ) + return {"status": "ok"} \ No newline at end of file diff --git a/src/app/routes/messages.py b/src/app/routes/messages.py new file mode 100644 index 00000000..360e4e92 --- /dev/null +++ b/src/app/routes/messages.py @@ -0,0 +1,171 @@ +import re +import uuid +from datetime import datetime +from typing import Optional + +from fastapi import APIRouter, Depends, Header, Query +from pydantic import BaseModel, ConfigDict, Field +from psycopg.rows import dict_row + +from ..db import connection +from ..errors import ApiError +from ..idempotency import run_idempotent +from ..serializers import row_to_message +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + +KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + +def _check_chat_member(conn, chat_id: str, user_id: str) -> None: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Чат не найден") + + +def _encode_cursor(created_at: datetime, id_) -> str: + import base64 + raw = f"{created_at.isoformat()}|{id_}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii").rstrip("=") + + +def _decode_cursor(cursor: str): + import base64 + pad = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + ts, id_ = raw.split("|", 1) + return datetime.fromisoformat(ts), id_ + + +@router.post("/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + body: CreateMessage, + idempotency_key: str = Header(..., alias="Idempotency-Key"), + actor: dict = Depends(require_actor), +): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + if not KEY_RE.fullmatch(idempotency_key or ""): + raise ApiError(400, "invalid_request", "Неверный Idempotency-Key") + + if not body.text.strip() and not body.attachment_ids: + raise ApiError(400, "invalid_request", "Нужен текст или вложение") + + scope = f"POST /api/v1/chats/{chat_id}/messages" + + def action(conn): + _check_chat_member(conn, chat_id, actor["user_id"]) + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + INSERT INTO messages (chat_id, sender_id, text) + VALUES (%s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, + (chat_id, actor["user_id"], body.text), + ) + row = cur.fetchone() + + if body.attachment_ids: + for att_id in body.attachment_ids: + cur.execute( + "SELECT 1 FROM attachments WHERE id = %s AND chat_id = %s", + (att_id, chat_id), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + cur.executemany( + "INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)", + [(row["id"], att_id) for att_id in body.attachment_ids] + ) + + row["attachment_ids"] = body.attachment_ids + + return 201, row_to_message(row) + + _, payload = run_idempotent( + user_id=actor["user_id"], + scope=scope, + key=idempotency_key, + body=body.model_dump(), + action=action, + ) + return payload + + +@router.get("/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + actor: dict = Depends(require_actor), +): + try: + uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, "invalid_request", "Неверный chat_id") + + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, actor["user_id"]), + ) + if cur.fetchone() is None: + raise ApiError(404, "not_found", "Чат не найден") + + params: list = [chat_id] + where = ["m.chat_id = %s"] + if cursor: + try: + c_ts, c_id = _decode_cursor(cursor) + except Exception: + raise ApiError(400, "invalid_request", "Неверный cursor") + where.append("(m.created_at, m.id) > (%s, %s)") + params.extend([c_ts, c_id]) + + sql = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + ARRAY( + SELECT ma.attachment_id::text + FROM message_attachments ma + WHERE ma.message_id = m.id + ), ARRAY[]::text[] + ) AS attachment_ids + FROM messages m + WHERE {' AND '.join(where)} + ORDER BY m.created_at, m.id + LIMIT %s + """ + params.append(limit + 1) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute(sql, params) + rows = cur.fetchall() + + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = _encode_cursor(last["created_at"], str(last["id"])) + rows = rows[:limit] + + return {"items": [row_to_message(r) for r in rows], "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/app/routes/sessions.py b/src/app/routes/sessions.py new file mode 100644 index 00000000..77776ff5 --- /dev/null +++ b/src/app/routes/sessions.py @@ -0,0 +1,131 @@ +import base64 + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import Response +from psycopg.rows import dict_row + +from ..config import get_settings +from ..db import connection, transaction +from ..errors import ApiError +from ..security import ( + new_session_token, + session_expiry, + token_digest, + verify_password, +) +from ..serializers import row_to_user + +router = APIRouter(prefix="/api/v1") + +WWW_BASIC = 'Basic realm="messenger", charset="UTF-8"' +WWW_BEARER = 'Bearer realm="messenger"' + + +def _parse_basic(header: str | None) -> tuple[str, str]: + if not header or not header.lower().startswith("basic "): + raise ApiError(401, "unauthorized", "Требуется Basic", {"WWW-Authenticate": WWW_BASIC}) + try: + decoded = base64.b64decode(header[6:].strip(), validate=True).decode("utf-8") + except Exception: + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) + + if ":" not in decoded: + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) + + username, password = decoded.split(":", 1) + return username, password + + +@router.post("/auth/sessions", status_code=201) +def create_session(request: Request): + header = request.headers.get("Authorization") + username, password = _parse_basic(header) + + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ) + row = cur.fetchone() + + # Одинаковый 401 для «нет пользователя» и «неверный пароль». + if row is None or not verify_password(password, row["password_hash"]): + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) + + settings = get_settings() + token = new_session_token() + expires_at = session_expiry(settings.session_ttl_seconds) + + with transaction() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO sessions (user_id, token_hash, expires_at) + VALUES (%s, %s, %s) + """, + (row["id"], token_digest(token), expires_at), + ) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + "user": row_to_user(row), + } + + +def _extract_bearer(request: Request) -> str: + header = request.headers.get("Authorization") + if not header or not header.lower().startswith("bearer "): + raise ApiError(401, "unauthorized", "Требуется Bearer", {"WWW-Authenticate": WWW_BEARER}) + token = header[7:].strip() + if not token: + raise ApiError(401, "unauthorized", "Требуется Bearer", {"WWW-Authenticate": WWW_BEARER}) + return token + + +def require_actor(request: Request) -> dict: + """Зависимость: валидирует Bearer и возвращает actor с user и token_hash.""" + token = _extract_bearer(request) + digest = token_digest(token) + with connection() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + """ + SELECT s.user_id, s.expires_at, + u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + """, + (digest,), + ) + row = cur.fetchone() + if row is None: + raise ApiError(401, "unauthorized", "Сессия недействительна", {"WWW-Authenticate": WWW_BEARER}) + + # Проверка срока на стороне БД + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > now()", + (digest,), + ) + if cur.fetchone() is None: + raise ApiError(401, "unauthorized", "Сессия истекла", {"WWW-Authenticate": WWW_BEARER}) + + return { + "user": row_to_user(row), + "user_id": row["user_id"], + "token_hash": digest, + } + + +@router.delete("/auth/sessions/current", status_code=204) +def revoke_session(request: Request, actor: dict = Depends(require_actor)): + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM sessions WHERE token_hash = %s", (actor["token_hash"],)) + return Response(status_code=204) \ No newline at end of file diff --git a/src/app/routes/users.py b/src/app/routes/users.py new file mode 100644 index 00000000..773f8011 --- /dev/null +++ b/src/app/routes/users.py @@ -0,0 +1,52 @@ +from fastapi import APIRouter, Depends +from pydantic import BaseModel, ConfigDict, Field, field_validator +import psycopg + +from ..db import transaction +from ..errors import ApiError +from ..security import hash_password +from ..serializers import row_to_user +from .sessions import require_actor + +router = APIRouter(prefix="/api/v1") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def not_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("display_name must not be blank") + return v + + +@router.post("/users", status_code=201) +def create_user(body: CreateUser): + pwd_hash = hash_password(body.password) + try: + with transaction() as conn: + with conn.cursor(row_factory=psycopg.rows.dict_row) as cur: + cur.execute( + """ + INSERT INTO users (username, display_name, password_hash) + VALUES (%s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (body.username, body.display_name, pwd_hash), + ) + row = cur.fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя занято") + + return row_to_user(row) + + +@router.get("/users/me") +def current_user(actor: dict = Depends(require_actor)): + return actor["user"] \ No newline at end of file diff --git a/src/app/s3.py b/src/app/s3.py new file mode 100644 index 00000000..79c65916 --- /dev/null +++ b/src/app/s3.py @@ -0,0 +1,39 @@ +import boto3 +from botocore.config import Config +from .config import get_settings + + +def get_s3_client(): + s = get_settings() + return boto3.client( + "s3", + endpoint_url=s.s3_endpoint, + aws_access_key_id=s.s3_access_key, + aws_secret_access_key=s.s3_secret_key, + region_name=s.s3_region, + config=Config(signature_version="s3v4"), + ) + + +def upload_fileobj(fileobj, key: str, content_type: str): + client = get_s3_client() + s = get_settings() + client.upload_fileobj( + fileobj, + s.s3_bucket, + key, + ExtraArgs={"ContentType": content_type}, + ) + + +def download_fileobj(key: str): + client = get_s3_client() + s = get_settings() + obj = client.get_object(Bucket=s.s3_bucket, Key=key) + return obj["Body"] + + +def delete_file(key: str): + client = get_s3_client() + s = get_settings() + client.delete_object(Bucket=s.s3_bucket, Key=key) \ No newline at end of file diff --git a/src/app/security.py b/src/app/security.py new file mode 100644 index 00000000..408d6860 --- /dev/null +++ b/src/app/security.py @@ -0,0 +1,34 @@ +import hashlib +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError + +_ph = PasswordHasher() # Argon2id по умолчанию + + +def hash_password(password: str) -> str: + return _ph.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + return _ph.verify(password_hash, password) + except VerifyMismatchError: + return False + except Exception: + return False + + +def new_session_token() -> str: + # 32 байта = 256 бит энтропии; url-safe hex без паддинга. + return secrets.token_urlsafe(32) + + +def token_digest(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def session_expiry(ttl_seconds: int) -> datetime: + return datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds) \ No newline at end of file diff --git a/src/app/serializers.py b/src/app/serializers.py new file mode 100644 index 00000000..3b2bad4e --- /dev/null +++ b/src/app/serializers.py @@ -0,0 +1,35 @@ +from datetime import datetime + + +def iso_z(dt: datetime) -> str: + """RFC3339 UTC с 'Z'. Postgres отдаёт timestamptz с '+00:00'.""" + return dt.isoformat().replace("+00:00", "Z") + + +def row_to_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso_z(row["created_at"]), + } + + +def row_to_chat(row) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": iso_z(row["created_at"]), + } + + +def row_to_message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso_z(row["created_at"]), + "attachment_ids": row.get("attachment_ids", []), + } \ No newline at end of file diff --git a/src/migrations/001_init.sql b/src/migrations/001_init.sql new file mode 100644 index 00000000..da374b89 --- /dev/null +++ b/src/migrations/001_init.sql @@ -0,0 +1,52 @@ +-- 001_init.sql +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS sessions_user_id_idx ON sessions(user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_at_idx ON sessions(expires_at); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members(user_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS messages_chat_order_idx ON messages(chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + scope text NOT NULL, -- method + " " + path + key text NOT NULL, + request_hash text NOT NULL, + status_code int NOT NULL, + response_json jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, scope, key) +); \ No newline at end of file diff --git a/src/migrations/002_attachments.sql b/src/migrations/002_attachments.sql new file mode 100644 index 00000000..5055da0a --- /dev/null +++ b/src/migrations/002_attachments.sql @@ -0,0 +1,20 @@ +-- 002_attachments.sql +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + PRIMARY KEY (message_id, attachment_id) +); \ No newline at end of file diff --git a/student-tests/test_level4.py b/student-tests/test_level4.py new file mode 100644 index 00000000..801466e7 --- /dev/null +++ b/student-tests/test_level4.py @@ -0,0 +1,113 @@ +import base64 +import io +import json +import os +import random +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + +BASE_URL = "http://localhost:8080" + +def req(method, path, data=None, headers=None, raw=None): + r = urllib.request.Request(BASE_URL + path, method=method) + if headers: + for k, v in headers.items(): + r.add_header(k, v) + if data: + r.add_header("Content-Type", "application/json") + r.data = json.dumps(data).encode("utf-8") + if raw: + r.data = raw + try: + with urllib.request.urlopen(r) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + +def setup(): + u = "test4_" + str(random.randint(1000, 9999)) + _, body = req("POST", "/api/v1/users", {"username": u, "display_name": "Тест", "password": "testpassword1234"}) + user = json.loads(body) + + cred = base64.b64encode((u + ":testpassword1234").encode()).decode() + _, body = req("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + token = json.loads(body)["token"] + + headers = {"Authorization": "Bearer " + token} + _, body = req("POST", "/api/v1/chats", {"title": "Тестовый чат", "member_ids": [user["id"]]}, headers) + chat = json.loads(body) + return user, chat, headers + +def main(): + user, chat, headers = setup() + print(f"Пользователь {user['id']} и чат {chat['id']} созданы.") + + chat_url = f"/api/v1/chats/{chat['id']}/attachments" + + # 1. Слишком большой файл (11 МБ) + print("1. Тест: слишком большой файл...") + big_data = b"\x89PNG\r\n\x1a\n" + b"A" * (11 * 1024 * 1024) + status, _ = req("POST", chat_url + "?filename=big.png", headers={**headers, "Content-Type": "image/png"}, raw=big_data) + assert status == 413, f"Ожидали 413, получили {status}" + print(" OK (413)") + + # 2. Битый файл (MIME PNG, но внутри мусор) + print("2. Тест: битый файл...") + broken_data = b"\x89PNG\r\n\x1a\n" + b"broken data" + status, _ = req("POST", chat_url + "?filename=broken.png", headers={**headers, "Content-Type": "image/png"}, raw=broken_data) + assert status == 415, f"Ожидали 415, получили {status}" + print(" OK (415)") + + # 3. Подмена MIME (говорим image/png, а внутри текст) + print("3. Тест: подмена MIME...") + text_data = b"Hello, this is not an image!" + status, _ = req("POST", chat_url + "?filename=fake.png", headers={**headers, "Content-Type": "image/png"}, raw=text_data) + assert status == 415, f"Ожидали 415, получили {status}" + print(" OK (415)") + + # 4. Опасное имя файла (path traversal) + print("4. Тест: опасное имя файла...") + with open("tests/fixtures/sample.png", "rb") as f: + valid_png = f.read() + status, _ = req("POST", chat_url + "?filename=../../etc/passwd", headers={**headers, "Content-Type": "image/png"}, raw=valid_png) + assert status == 400, f"Ожидали 400, получили {status}" + print(" OK (400)") + + # 5. Успешная загрузка (чтобы проверить доступ) + print("5. Тест: успешная загрузка...") + status, body = req("POST", chat_url + "?filename=ok.png", headers={**headers, "Content-Type": "image/png"}, raw=valid_png) + assert status == 201, f"Ожидали 201, получили {status}" + att = json.loads(body) + print(f" OK (201), ID: {att['id']}") + + # 6. Чужой доступ + print("6. Тест: чужой доступ...") + u_eve = "eve_" + str(random.randint(1000, 9999)) + req("POST", "/api/v1/users", {"username": u_eve, "display_name": "Eve", "password": "testpassword1234"}) + cred_eve = base64.b64encode((u_eve + ":testpassword1234").encode()).decode() + _, body = req("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred_eve}) + token_eve = json.loads(body)["token"] + headers_eve = {"Authorization": "Bearer " + token_eve} + + status, _ = req("GET", f"/api/v1/attachments/{att['id']}", headers=headers_eve) + assert status == 404, f"Ожидали 404 (чужой), получили {status}" + print(" OK (404)") + + # 7. Нагрузочный тест (10 параллельных загрузок) + print("7. Тест: 10 параллельных загрузок (проверка RAM)...") + def upload_worker(): + status, _ = req("POST", chat_url + "?filename=parallel.png", headers={**headers, "Content-Type": "image/png"}, raw=valid_png) + return status + + with ThreadPoolExecutor(max_workers=10) as pool: + results = list(pool.map(lambda _: upload_worker(), range(10))) + + assert all(s == 201 for s in results), f"Не все загрузки успешны: {results}" + print(" OK (все 10 успешно)") + + print("\nPASS: Уровень 4 пройден! (Все негативные тесты и нагрузка работают)") + +if __name__ == "__main__": + main() \ No newline at end of file -- 2.54.0 From f856651e5742fc3925f50cbef3459e2dbdd30090 Mon Sep 17 00:00:00 2001 From: caramelteawithmint <146834163+caramelteawithmint@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:53:35 +0300 Subject: [PATCH 2/4] _1change --- Dockerfile | 37 ++-- Dockerfile.worker | 15 ++ compose.yaml | 202 +++++++++++---------- infra/minio-policy.json | 17 ++ requirements.txt | 9 + src/app/config.py | 6 +- src/app/routes/attachments.py | 111 +++++++---- src/app/routes/health.py | 22 ++- src/migrations/002_attachments.sql | 7 +- src/migrations/003_attachment_variants.sql | 13 ++ worker/__init__.py | 0 worker/main.py | 123 +++++++++++++ 12 files changed, 409 insertions(+), 153 deletions(-) create mode 100644 Dockerfile.worker create mode 100644 infra/minio-policy.json create mode 100644 requirements.txt create mode 100644 src/migrations/003_attachment_variants.sql create mode 100644 worker/__init__.py create mode 100644 worker/main.py diff --git a/Dockerfile b/Dockerfile index 4f150402..3c897133 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,21 @@ FROM python:3.11-slim -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 - -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl ca-certificates \ - && rm -rf /var/lib/apt/lists/* - WORKDIR /app +# libpq-dev для psycopg, gcc для сборки некоторых python-пакетов, +# libjpeg-dev и zlib1g-dev нужны для работы Pillow (обработка картинок) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq-dev gcc libjpeg-dev zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + COPY requirements.txt . -RUN pip install -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt -# Копируем код приложения, миграции и скрипты -COPY src/app ./app -COPY src/migrations ./migrations -COPY scripts ./scripts +# Копируем код API и Воркера +COPY src/ ./src/ +COPY worker/ ./worker/ -# не root -RUN useradd --create-home --shell /bin/bash appuser \ - && chown -R appuser:appuser /app -USER appuser +ENV PYTHONPATH=/app -EXPOSE 8080 - -HEALTHCHECK --interval=5s --timeout=2s --start-period=10s --retries=6 \ - CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1 - -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers", "--forwarded-allow-ips", "*"] \ No newline at end of file +# По умолчанию запускаем API (воркер переопределит эту команду в compose) +CMD ["uvicorn", "src.app.main:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 00000000..1e5291b9 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libpq-dev gcc && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ +COPY worker/ ./worker/ + +ENV PYTHONPATH=/app +CMD ["python", "-m", "worker.main"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index a43fc3ea..1de6c84f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,14 +1,13 @@ -name: mtusi-messenger - -x-api-base: &api-base +x-api-common: &api-common build: . environment: - DATABASE_URL_FILE=/run/secrets/database_url - - SESSION_TTL_SECONDS=3600 - S3_ENDPOINT=http://minio:9000 - - S3_ACCESS_KEY=api-user - - S3_SECRET_KEY=api-secret-key-12345 + - S3_ACCESS_KEY=minioadmin + - S3_SECRET_KEY=minioadmin - S3_BUCKET=messenger-attachments + - BROKER_URL=amqp://guest:guest@broker:5672/ + - APP_ENV=lab5 secrets: - database_url depends_on: @@ -16,93 +15,14 @@ x-api-base: &api-base condition: service_healthy minio: condition: service_healthy - migrate: + broker: + condition: service_healthy + minio-init: condition: service_completed_successfully services: - postgres: - image: postgres:16-alpine - environment: - POSTGRES_USER: messenger - POSTGRES_PASSWORD: messenger - POSTGRES_DB: messenger - healthcheck: - test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] - interval: 5s - timeout: 3s - retries: 5 - volumes: - - pgdata:/var/lib/postgresql/data - - minio: - image: pgsty/silo:latest - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - volumes: - - miniodata:/data - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 5s - timeout: 3s - retries: 5 - # Порт 9000 (API) и 9001 (Консоль) НЕ пробрасываем наружу, чтобы админка была закрыта. - # Контейнеры общаются по внутренней сети Docker. - - minio-init: - image: pgsty/silo:latest - depends_on: - minio: - condition: service_healthy - volumes: - - ./infra/minio-policy.json:/tmp/minio-policy.json:ro - entrypoint: > - /bin/sh -c " - mc alias set myminio http://minio:9000 minioadmin minioadmin; - mc mb myminio/messenger-attachments --ignore-existing; - mc anonymous set none myminio/messenger-attachments; - mc admin user add myminio api-user api-secret-key-12345 || true; - mc admin policy create myminio api-policy /tmp/minio-policy.json || true; - mc admin policy attach myminio api-policy --user api-user || true; - exit 0; - " - - migrate: - build: . - command: ["python", "scripts/migrate.py"] - environment: - - DATABASE_URL_FILE=/run/secrets/database_url - secrets: - - database_url - depends_on: - postgres: - condition: service_healthy - - api1: - <<: *api-base - environment: - - INSTANCE_ID=api-1 - - DATABASE_URL_FILE=/run/secrets/database_url - - SESSION_TTL_SECONDS=3600 - - S3_ENDPOINT=http://minio:9000 - - S3_ACCESS_KEY=api-user - - S3_SECRET_KEY=api-secret-key-12345 - - S3_BUCKET=messenger-attachments - - api2: - <<: *api-base - environment: - - INSTANCE_ID=api-2 - - DATABASE_URL_FILE=/run/secrets/database_url - - SESSION_TTL_SECONDS=3600 - - S3_ENDPOINT=http://minio:9000 - - S3_ACCESS_KEY=api-user - - S3_SECRET_KEY=api-secret-key-12345 - - S3_BUCKET=messenger-attachments - - load_balancer: - image: nginx:1.27-alpine + lb: + image: nginx:1.25-alpine ports: - "8080:8080" volumes: @@ -111,10 +31,110 @@ services: - api1 - api2 + api1: + <<: *api-common + expose: + - "8080" + + api2: + <<: *api-common + expose: + - "8080" + + worker: + build: . + command: python -m worker.main + environment: + - DATABASE_URL_FILE=/run/secrets/database_url + - S3_ENDPOINT=http://minio:9000 + - S3_ACCESS_KEY=minioadmin + - S3_SECRET_KEY=minioadmin + - S3_BUCKET=messenger-attachments + - BROKER_URL=amqp://guest:guest@broker:5672/ + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + broker: + condition: service_healthy + minio-init: + condition: service_completed_successfully + + migrate: + build: . + command: python scripts/migrate.py + environment: + - DATABASE_URL_FILE=/run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + + postgres: + image: postgres:16-alpine + environment: + - POSTGRES_USER=messenger + - POSTGRES_PASSWORD=messenger_password + - POSTGRES_DB=messenger + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 5s + retries: 5 + + minio: + image: quay.io/minio/minio:RELEASE.2024-02-24T06-42-55Z + command: server /data --console-address ":9001" + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + volumes: + - s3data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 5 + + minio-init: + image: quay.io/minio/mc:latest + depends_on: + minio: + condition: service_healthy + volumes: + - ./infra/minio-policy.json:/minio-policy.json + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc mb myminio/messenger-attachments --ignore-existing; + /usr/bin/mc anonymous set-json /minio-policy.json myminio/messenger-attachments; + exit 0; + " + + broker: + image: rabbitmq:3.13-alpine + environment: + - RABBITMQ_DEFAULT_USER=guest + - RABBITMQ_DEFAULT_PASS=guest + volumes: + - brokerdata:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 5 + secrets: database_url: file: ./secrets/database_url volumes: pgdata: - miniodata: \ No newline at end of file + s3data: + brokerdata: \ No newline at end of file diff --git a/infra/minio-policy.json b/infra/minio-policy.json new file mode 100644 index 00000000..e757ee44 --- /dev/null +++ b/infra/minio-policy.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments", + "arn:aws:s3:::messenger-attachments/*" + ] + } + ] +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..bf7f6e7e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.29.0 +psycopg[binary,pool]>=3.1.18 +pydantic-settings>=2.2.1 +argon2-cffi>=23.1.0 +boto3>=1.34.0 +python-multipart>=0.0.9 +Pillow>=10.2.0 +aio-pika>=9.4.0 \ No newline at end of file diff --git a/src/app/config.py b/src/app/config.py index 6d3f0dd4..45e446ae 100644 --- a/src/app/config.py +++ b/src/app/config.py @@ -31,13 +31,17 @@ class Settings(BaseSettings): database_url: str = "" database_url_file: str = "/run/secrets/database_url" - # S3 / MinIO (для 4-й лабы) + # S3 / MinIO s3_endpoint: str = "http://minio:9000" s3_access_key: str = "minioadmin" s3_secret_key: str = "minioadmin" s3_bucket: str = "messenger-attachments" s3_region: str = "us-east-1" + # Брокер (RabbitMQ) + broker_url: str = "amqp://guest:guest@broker:5672/" + broker_queue: str = "attachment_processing" + @property def db_dsn(self) -> str: from_file = _read_secret(self.database_url_file) diff --git a/src/app/routes/attachments.py b/src/app/routes/attachments.py index 458996b4..9564a9b6 100644 --- a/src/app/routes/attachments.py +++ b/src/app/routes/attachments.py @@ -2,11 +2,14 @@ import hashlib import io import tempfile import uuid +import json +import aio_pika from fastapi import APIRouter, Depends, Request, Query from fastapi.responses import StreamingResponse from psycopg.rows import dict_row +from ..config import get_settings from ..db import transaction, connection from ..errors import ApiError from ..s3 import upload_fileobj, download_fileobj, delete_file @@ -15,11 +18,27 @@ from ..serializers import iso_z from .sessions import require_actor router = APIRouter(prefix="/api/v1") - MAX_SIZE = 10 * 1024 * 1024 +async def _publish_task(attachment_id: str, object_key: str): + """Отправляет задачу в RabbitMQ.""" + settings = get_settings() + try: + connection = await aio_pika.connect_robust(settings.broker_url) + async with connection: + channel = await connection.channel() + await channel.declare_queue(settings.broker_queue, durable=True) + message = aio_pika.Message( + body=json.dumps({"attachment_id": attachment_id, "object_key": object_key}).encode(), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT + ) + await channel.default_exchange.publish(message, routing_key=settings.broker_queue) + except Exception as e: + print(f"Broker error: {e}") + # Если брокер упал, мы можем либо откатить загрузку, либо оставить статус queued + # Для лабы: логируем, воркер потом не найдет задачу, но health/ready покажет ошибку -@router.post("/chats/{chat_id}/attachments", status_code=201) +@router.post("/chats/{chat_id}/attachments", status_code=202) async def upload_attachment( chat_id: str, request: Request, @@ -34,7 +53,6 @@ async def upload_attachment( if "/" in filename or "\\" in filename or "\0" in filename: raise ApiError(400, "invalid_request", "Опасное имя файла") - # Проверяем доступ к чату with connection() as conn: with conn.cursor() as cur: cur.execute( @@ -48,34 +66,28 @@ async def upload_attachment( size = 0 first_chunk = b"" - # Читаем потоково во временный файл (чтобы не забивать RAM) with tempfile.SpooledTemporaryFile(max_size=MAX_SIZE, mode="w+b") as spooled: async for chunk in request.stream(): size += len(chunk) if size > MAX_SIZE: raise ApiError(413, "payload_too_large", "Файл слишком большой") if not first_chunk: - first_chunk = chunk[:16] # для детекта MIME + first_chunk = chunk[:16] sha256.update(chunk) spooled.write(chunk) if size == 0: raise ApiError(400, "invalid_request", "Пустой файл") - # Полная проверка картинки через Pillow + # Только быстрая проверка magic bytes, глубокий анализ оставляем воркеру try: - spooled.seek(0) - content = spooled.read() content_type, _, _ = check_image(first_chunk, ["image/png", "image/jpeg"]) - from PIL import Image - Image.open(io.BytesIO(content)).verify() - except Exception: - raise ApiError(415, "unsupported_media_type", "Неподдерживаемый или битый формат") + except ValueError: + raise ApiError(415, "unsupported_media_type", "Неподдерживаемый формат") sha256_hex = sha256.hexdigest() object_key = f"chats/{chat_id}/{uuid.uuid4().hex}" - # Загружаем в S3 try: spooled.seek(0) upload_fileobj(spooled, object_key, content_type) @@ -83,21 +95,19 @@ async def upload_attachment( print(f"S3 error: {e}") raise ApiError(503, "unavailable", "S3 недоступен") - # Сохраняем в БД (с компенсацией) try: with transaction() as conn: with conn.cursor(row_factory=dict_row) as cur: cur.execute( """ - INSERT INTO attachments (chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) - VALUES (%s, %s, %s, %s, %s, %s, %s) - RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, created_at + INSERT INTO attachments (chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status) + VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued') + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, created_at, updated_at """, (chat_id, actor["user_id"], filename, content_type, size, sha256_hex, object_key), ) att = cur.fetchone() except Exception as e: - # КОМПЕНСАЦИЯ: удаляем из S3, если БД упала print(f"DB failed, compensating S3: {e}") try: delete_file(object_key) @@ -105,6 +115,9 @@ async def upload_attachment( print(f"Failed to compensate S3: {s3_e}") raise ApiError(500, "internal_error", "Ошибка сохранения") + # Публикуем задачу + await _publish_task(str(att["id"]), object_key) + return { "id": str(att["id"]), "chat_id": str(att["chat_id"]), @@ -113,14 +126,13 @@ async def upload_attachment( "content_type": att["content_type"], "size_bytes": att["size_bytes"], "sha256": att["sha256"], - "status": "ready", + "status": att["status"], "variants": [], "error_code": None, "created_at": iso_z(att["created_at"]), - "updated_at": iso_z(att["created_at"]), + "updated_at": iso_z(att["updated_at"]), } - @router.get("/attachments/{attachment_id}") def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): try: @@ -140,6 +152,12 @@ def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): (actor["user_id"], attachment_id), ) att = cur.fetchone() + + if att: + cur.execute("SELECT * FROM attachment_variants WHERE attachment_id = %s", (attachment_id,)) + variants = cur.fetchall() + else: + variants = [] if att is None: raise ApiError(404, "not_found", "Вложение не найдено") @@ -152,16 +170,28 @@ def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): "content_type": att["content_type"], "size_bytes": att["size_bytes"], "sha256": att["sha256"], - "status": "ready", - "variants": [], - "error_code": None, + "status": att["status"], + "variants": [ + { + "name": v["name"], + "content_type": v["content_type"], + "size_bytes": v["size_bytes"], + "width": v["width"], + "height": v["height"], + "sha256": v["sha256"], + } for v in variants + ], + "error_code": att["error_code"], "created_at": iso_z(att["created_at"]), - "updated_at": iso_z(att["created_at"]), + "updated_at": iso_z(att["updated_at"]), } - @router.get("/attachments/{attachment_id}/content") -def download_attachment(attachment_id: str, actor: dict = Depends(require_actor)): +def download_attachment( + attachment_id: str, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + actor: dict = Depends(require_actor) +): try: uuid.UUID(attachment_id) except ValueError: @@ -171,7 +201,7 @@ def download_attachment(attachment_id: str, actor: dict = Depends(require_actor) with conn.cursor(row_factory=dict_row) as cur: cur.execute( """ - SELECT a.object_key, a.content_type, a.filename + SELECT a.object_key, a.content_type, a.filename, a.status FROM attachments a JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s WHERE a.id = %s @@ -180,11 +210,28 @@ def download_attachment(attachment_id: str, actor: dict = Depends(require_actor) ) att = cur.fetchone() - if att is None: - raise ApiError(404, "not_found", "Вложение не найдено") + if att is None: + raise ApiError(404, "not_found", "Вложение не найдено") + + if variant == "original": + if att["status"] == "failed": + raise ApiError(409, "conflict", "Обработка вложения завершилась ошибкой") + obj_key = att["object_key"] + content_type = att["content_type"] + else: + cur.execute( + "SELECT object_key, content_type FROM attachment_variants WHERE attachment_id = %s AND name = %s", + (attachment_id, variant) + ) + var = cur.fetchone() + if not var: + # Если вариант еще не готов, возвращаем 404 или 409 + raise ApiError(404, "not_found", "Вариант еще не готов или недоступен") + obj_key = var["object_key"] + content_type = var["content_type"] try: - body = download_fileobj(att["object_key"]) + body = download_fileobj(obj_key) except Exception: raise ApiError(503, "unavailable", "S3 недоступен") @@ -193,6 +240,6 @@ def download_attachment(attachment_id: str, actor: dict = Depends(require_actor) return StreamingResponse( iter_body(), - media_type=att["content_type"], + media_type=content_type, headers={"Content-Disposition": f'attachment; filename="{att["filename"]}"'}, ) \ No newline at end of file diff --git a/src/app/routes/health.py b/src/app/routes/health.py index 572ed6d0..b0567a08 100644 --- a/src/app/routes/health.py +++ b/src/app/routes/health.py @@ -1,25 +1,41 @@ from fastapi import APIRouter, Request from fastapi.responses import JSONResponse +import socket +from urllib.parse import urlparse +from ..config import get_settings from ..db import connection from ..errors import error_response +from ..s3 import get_s3_client router = APIRouter() - @router.get("/health/live") def live(): return {"status": "ok"} - @router.get("/health/ready") def ready(request: Request): try: + with connection() as conn: with conn.cursor() as cur: cur.execute("SELECT 1") cur.fetchone() - except Exception: + + + get_s3_client().head_bucket(Bucket=get_settings().s3_bucket) + + # 3. Broker (простая проверка TCP-порта) + settings = get_settings() + parsed = urlparse(settings.broker_url) + host = parsed.hostname or "broker" + port = parsed.port or 5672 + with socket.create_connection((host, port), timeout=2): + pass + + except Exception as e: + print(f"Readiness check failed: {e}") return error_response( request, 503, "unavailable", "Зависимость недоступна" ) diff --git a/src/migrations/002_attachments.sql b/src/migrations/002_attachments.sql index 5055da0a..64951308 100644 --- a/src/migrations/002_attachments.sql +++ b/src/migrations/002_attachments.sql @@ -1,4 +1,4 @@ --- 002_attachments.sql +-- src/migrations/002_attachments.sql CREATE TABLE IF NOT EXISTS attachments ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, @@ -8,7 +8,10 @@ CREATE TABLE IF NOT EXISTS attachments ( size_bytes bigint NOT NULL, sha256 text NOT NULL, object_key text NOT NULL, - created_at timestamptz NOT NULL DEFAULT now() + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() ); CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id); diff --git a/src/migrations/003_attachment_variants.sql b/src/migrations/003_attachment_variants.sql new file mode 100644 index 00000000..f08e2e25 --- /dev/null +++ b/src/migrations/003_attachment_variants.sql @@ -0,0 +1,13 @@ +-- src/migrations/003_attachment_variants.sql +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + name text NOT NULL CHECK (name IN ('thumbnail','crop','optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg','image/png','image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width int NOT NULL CHECK (width > 0), + height int NOT NULL CHECK (height > 0), + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, name) +); \ No newline at end of file diff --git a/worker/__init__.py b/worker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/worker/main.py b/worker/main.py new file mode 100644 index 00000000..21d41073 --- /dev/null +++ b/worker/main.py @@ -0,0 +1,123 @@ +import asyncio +import io +import json +import logging +from datetime import datetime, timezone + +import aio_pika +from PIL import Image +import psycopg +from psycopg.rows import dict_row + +from src.app.config import get_settings +from src.app.db import init_pool, transaction +from src.app.s3 import get_s3_client, download_fileobj, upload_fileobj + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("worker") + +def generate_variants(img: Image.Image): + variants = {} + + # 1. Thumbnail (макс 256x256, сохраняем пропорции) + thumb = img.copy() + thumb.thumbnail((256, 256)) + variants["thumbnail"] = thumb + + # 2. Crop (квадрат 512x512 по центру) + crop = img.copy() + min_dim = min(crop.size) + left = (crop.width - min_dim) // 2 + top = (crop.height - min_dim) // 2 + crop = crop.crop((left, top, left + min_dim, top + min_dim)) + crop = crop.resize((512, 512), Image.Resampling.LANCZOS) + variants["crop"] = crop + + # 3. Optimized (просто оригинал в JPEG с качеством 85) + opt = img.convert("RGB") + buf = io.BytesIO() + opt.save(buf, format="JPEG", quality=85) + variants["optimized"] = Image.open(buf) + + return variants + +async def process_message(message: aio_pika.IncomingMessage): + async with message.process(): + data = json.loads(message.body) + att_id = data["attachment_id"] + obj_key = data["object_key"] + + logger.info(f"Processing attachment {att_id}") + + try: + # 1. Обновляем статус на processing + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,)) + + # 2. Качаем из S3 + body = download_fileobj(obj_key) + img = Image.open(body) + img.load() # принудительная загрузка в память для обнаружения битых файлов + + # 3. Генерируем варианты + variants = generate_variants(img) + + # 4. Сохраняем в S3 и БД + with transaction() as conn: + with conn.cursor() as cur: + for name, var_img in variants.items(): + buf = io.BytesIO() + ct = "image/jpeg" + if name == "thumbnail": + var_img.save(buf, format="PNG") + ct = "image/png" + else: + var_img.save(buf, format="JPEG", quality=85) + + var_bytes = buf.getvalue() + sha256 = __import__("hashlib").sha256(var_bytes).hexdigest() + var_key = f"chats/{att_id}/variants/{name}" + + upload_fileobj(io.BytesIO(var_bytes), var_key, ct) + + cur.execute( + """ + INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, name) DO UPDATE SET + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256, + object_key = EXCLUDED.object_key + """, + (att_id, name, ct, len(var_bytes), var_img.width, var_img.height, sha256, var_key) + ) + + cur.execute("UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,)) + logger.info(f"Successfully processed {att_id}") + + except Exception as e: + logger.error(f"Error processing {att_id}: {e}") + with transaction() as conn: + with conn.cursor() as cur: + cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s", ("processing_error", att_id)) + raise e # Пробрасываем ошибку, чтобы RabbitMQ не удалил сообщение (at-least-once) + +async def main(): + init_pool() + settings = get_settings() + + connection = await aio_pika.connect_robust(settings.broker_url) + async with connection: + channel = await connection.channel() + await channel.set_qos(prefetch_count=1) + queue = await channel.declare_queue(settings.broker_queue, durable=True) + logger.info("Worker started, waiting for messages...") + await queue.consume(process_message) + await asyncio.Future() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file -- 2.54.0 From a306a68aa4a607a4f6b4f7964473d3af2aad56a9 Mon Sep 17 00:00:00 2001 From: caramelteawithmint <146834163+caramelteawithmint@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:05:30 +0300 Subject: [PATCH 3/4] 4_grade --- Dockerfile | 3 +- compose.yaml | 57 +++++++++++++++------------- requirements.txt | 3 +- scripts/migrate.py | 58 ++++++++++++++++++++++++++++ scripts/restart.sh | 6 +++ scripts/stop_one.sh | 13 +++++++ src/app/config.py | 2 + src/app/idempotency.py | 78 ++++++++++++++++---------------------- src/app/routes/sessions.py | 67 ++++++++++++-------------------- test_lab5.sh | 72 +++++++++++++++++++++++++++++++++++ worker/main.py | 41 ++++++++++++-------- 11 files changed, 268 insertions(+), 132 deletions(-) create mode 100644 scripts/migrate.py create mode 100644 scripts/restart.sh create mode 100644 scripts/stop_one.sh create mode 100644 test_lab5.sh diff --git a/Dockerfile b/Dockerfile index 3c897133..431c1d5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -# Копируем код API и Воркера +# Копируем код API, Воркера и скрипты (включая migrate.py) COPY src/ ./src/ COPY worker/ ./worker/ +COPY scripts/ ./scripts/ ENV PYTHONPATH=/app diff --git a/compose.yaml b/compose.yaml index 1de6c84f..4cd8e3c9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,15 +1,14 @@ x-api-common: &api-common build: . environment: - - DATABASE_URL_FILE=/run/secrets/database_url + - DATABASE_URL=postgresql://messenger:messenger_password@postgres:5432/messenger - S3_ENDPOINT=http://minio:9000 - S3_ACCESS_KEY=minioadmin - S3_SECRET_KEY=minioadmin - S3_BUCKET=messenger-attachments - BROKER_URL=amqp://guest:guest@broker:5672/ + - REDIS_URL=redis://redis:6379/0 - APP_ENV=lab5 - secrets: - - database_url depends_on: postgres: condition: service_healthy @@ -17,7 +16,9 @@ x-api-common: &api-common condition: service_healthy broker: condition: service_healthy - minio-init: + redis: + condition: service_healthy + migrate: condition: service_completed_successfully services: @@ -45,14 +46,13 @@ services: build: . command: python -m worker.main environment: - - DATABASE_URL_FILE=/run/secrets/database_url + - DATABASE_URL=postgresql://messenger:messenger_password@postgres:5432/messenger - S3_ENDPOINT=http://minio:9000 - S3_ACCESS_KEY=minioadmin - S3_SECRET_KEY=minioadmin - S3_BUCKET=messenger-attachments - BROKER_URL=amqp://guest:guest@broker:5672/ - secrets: - - database_url + - REDIS_URL=redis://redis:6379/0 depends_on: postgres: condition: service_healthy @@ -60,19 +60,22 @@ services: condition: service_healthy broker: condition: service_healthy - minio-init: + redis: + condition: service_healthy + migrate: condition: service_completed_successfully migrate: build: . command: python scripts/migrate.py environment: - - DATABASE_URL_FILE=/run/secrets/database_url - secrets: - - database_url + - DATABASE_URL=postgresql://messenger:messenger_password@postgres:5432/messenger + - REDIS_URL=redis://redis:6379/0 depends_on: postgres: condition: service_healthy + redis: + condition: service_healthy postgres: image: postgres:16-alpine @@ -89,7 +92,7 @@ services: retries: 5 minio: - image: quay.io/minio/minio:RELEASE.2024-02-24T06-42-55Z + image: pgsty/minio:latest command: server /data --console-address ":9001" environment: - MINIO_ROOT_USER=minioadmin @@ -101,20 +104,12 @@ services: interval: 5s timeout: 5s retries: 5 - - minio-init: - image: quay.io/minio/mc:latest - depends_on: - minio: - condition: service_healthy - volumes: - - ./infra/minio-policy.json:/minio-policy.json entrypoint: > /bin/sh -c " - /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin; + /usr/bin/mc alias set myminio http://localhost:9000 minioadmin minioadmin; /usr/bin/mc mb myminio/messenger-attachments --ignore-existing; - /usr/bin/mc anonymous set-json /minio-policy.json myminio/messenger-attachments; - exit 0; + /usr/bin/mc anonymous set private myminio/messenger-attachments; + /usr/bin/minio server /data --console-address ':9001' " broker: @@ -130,11 +125,19 @@ services: timeout: 5s retries: 5 -secrets: - database_url: - file: ./secrets/database_url + redis: + image: redis:7-alpine + command: redis-server --appendonly yes + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 volumes: pgdata: s3data: - brokerdata: \ No newline at end of file + brokerdata: + redisdata: \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bf7f6e7e..c31a3d2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ argon2-cffi>=23.1.0 boto3>=1.34.0 python-multipart>=0.0.9 Pillow>=10.2.0 -aio-pika>=9.4.0 \ No newline at end of file +aio-pika>=9.4.0 +redis[hiredis]>=5.0.1 \ No newline at end of file diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..e617f679 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,58 @@ +import os +import sys +import time +from pathlib import Path +import psycopg + +def get_dsn(): + file_path = os.environ.get("DATABASE_URL_FILE") + if file_path and os.path.exists(file_path): + return Path(file_path).read_text().strip() + return os.environ.get("DATABASE_URL") + +def main(): + dsn = get_dsn() + if not dsn: + print("ERROR: DATABASE_URL or DATABASE_URL_FILE not set", file=sys.stderr) + sys.exit(1) + + for i in range(10): + try: + with psycopg.connect(dsn, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + break + except psycopg.OperationalError: + print("Waiting for database...") + time.sleep(2) + else: + print("ERROR: Database is not ready", file=sys.stderr) + sys.exit(1) + + # ВАЖНО: путь к миграциям. Раз они лежат в src/migrations: + migrations_dir = Path(__file__).parent.parent / "src" / "migrations" + sql_files = sorted(migrations_dir.glob("*.sql")) + + with psycopg.connect(dsn, autocommit=True) as conn: + with conn.cursor() as cur: + cur.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz DEFAULT now() + ) + """) + + for sql_file in sql_files: + version = sql_file.name + cur.execute("SELECT 1 FROM schema_migrations WHERE version = %s", (version,)) + if cur.fetchone(): + continue + + print(f"Applying {version}...") + sql = sql_file.read_text(encoding="utf-8") + cur.execute(sql) + cur.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + print(f"Applied {version}.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..43cfb34a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +echo "Restarting API and Postgres (preserving volumes)..." +docker compose restart api1 api2 postgres +echo "Waiting for Postgres to be healthy..." +sleep 5 \ No newline at end of file diff --git a/scripts/stop_one.sh b/scripts/stop_one.sh new file mode 100644 index 00000000..14859318 --- /dev/null +++ b/scripts/stop_one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +if [ "$TARGET_INSTANCE_ID" != "api-1" ] && [ "$TARGET_INSTANCE_ID" != "api-2" ]; then + echo "Unknown instance: $TARGET_INSTANCE_ID" >&2 + exit 1 +fi +SERVICE_NAME="$TARGET_INSTANCE_ID" +if [ "$SERVICE_NAME" = "api-1" ]; then SERVICE_NAME="api1"; fi +if [ "$SERVICE_NAME" = "api-2" ]; then SERVICE_NAME="api2"; fi + +echo "Stopping container for $SERVICE_NAME..." +docker compose stop "$SERVICE_NAME" \ No newline at end of file diff --git a/src/app/config.py b/src/app/config.py index 45e446ae..6c93b09b 100644 --- a/src/app/config.py +++ b/src/app/config.py @@ -41,6 +41,8 @@ class Settings(BaseSettings): # Брокер (RabbitMQ) broker_url: str = "amqp://guest:guest@broker:5672/" broker_queue: str = "attachment_processing" + + redis_url: str = "redis://redis:6379/0" @property def db_dsn(self) -> str: diff --git a/src/app/idempotency.py b/src/app/idempotency.py index 9be16d63..9855d870 100644 --- a/src/app/idempotency.py +++ b/src/app/idempotency.py @@ -1,18 +1,19 @@ import hashlib import json +import redis from typing import Any, Callable - import psycopg -from .db import transaction +from .config import get_settings from .errors import ApiError +from .db import transaction +redis_client = redis.Redis.from_url(get_settings().redis_url, decode_responses=True) def _body_hash(body: dict[str, Any]) -> str: raw = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False) return hashlib.sha256(raw.encode("utf-8")).hexdigest() - def run_idempotent( *, user_id: str, @@ -21,52 +22,39 @@ def run_idempotent( body: dict[str, Any], action: Callable[[psycopg.Connection], tuple[int, dict[str, Any]]], ) -> tuple[int, dict[str, Any]]: - """Атомарно: либо побочный эффект + сохранённый ответ, либо повтор.""" h = _body_hash(body) - try: - with transaction() as conn: - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO idempotency_keys - (user_id, scope, key, request_hash, status_code, response_json) - VALUES (%s, %s, %s, %s, 0, '{}'::jsonb) - """, - (user_id, scope, key, h), - ) - result = action(conn) - status, payload = result - with conn.cursor() as cur: - cur.execute( - """ - UPDATE idempotency_keys - SET status_code = %s, response_json = %s::jsonb - WHERE user_id = %s AND scope = %s AND key = %s - """, - (status, json.dumps(payload, ensure_ascii=False), user_id, scope, key), - ) - return status, payload - except psycopg.errors.UniqueViolation: - pass + redis_key = f"idem:{user_id}:{scope}:{key}" + ttl = get_settings().idempotency_ttl_seconds - with transaction() as conn: - with conn.cursor() as cur: - cur.execute( - """ - SELECT request_hash, status_code, response_json - FROM idempotency_keys - WHERE user_id = %s AND scope = %s AND key = %s - """, - (user_id, scope, key), + # Блокировка + locked = redis_client.set(redis_key, json.dumps({"status": 0, "hash": h}), nx=True, ex=ttl) + + if locked: + try: + with transaction() as conn: + status, payload = action(conn) + + redis_client.setex( + redis_key, + ttl, + json.dumps({"status": status, "hash": h, "payload": payload}) ) - row = cur.fetchone() + return status, payload + except Exception: + redis_client.delete(redis_key) + raise - if row is None: + # Повторный запрос + existing_raw = redis_client.get(redis_key) + if not existing_raw: raise ApiError(409, "conflict", "Конфликт идемпотентности, повторите") - - existing_hash, status, payload = row - if existing_hash != h: + + existing = json.loads(existing_raw) + + if existing.get("hash") != h: raise ApiError(409, "conflict", "Тот же ключ с другим телом запроса") - if status == 0: + + if existing.get("status") == 0: raise ApiError(409, "conflict", "Повтор ещё не завершён") - return status, payload \ No newline at end of file + + return existing["status"], existing["payload"] \ No newline at end of file diff --git a/src/app/routes/sessions.py b/src/app/routes/sessions.py index 77776ff5..8df7889b 100644 --- a/src/app/routes/sessions.py +++ b/src/app/routes/sessions.py @@ -1,11 +1,12 @@ import base64 - +import json +import redis from fastapi import APIRouter, Depends, Request from fastapi.responses import Response from psycopg.rows import dict_row from ..config import get_settings -from ..db import connection, transaction +from ..db import connection from ..errors import ApiError from ..security import ( new_session_token, @@ -20,6 +21,8 @@ router = APIRouter(prefix="/api/v1") WWW_BASIC = 'Basic realm="messenger", charset="UTF-8"' WWW_BEARER = 'Bearer realm="messenger"' +# Подключение к Redis +redis_client = redis.Redis.from_url(get_settings().redis_url, decode_responses=True) def _parse_basic(header: str | None) -> tuple[str, str]: if not header or not header.lower().startswith("basic "): @@ -35,7 +38,6 @@ def _parse_basic(header: str | None) -> tuple[str, str]: username, password = decoded.split(":", 1) return username, password - @router.post("/auth/sessions", status_code=201) def create_session(request: Request): header = request.headers.get("Authorization") @@ -50,32 +52,24 @@ def create_session(request: Request): ) row = cur.fetchone() - # Одинаковый 401 для «нет пользователя» и «неверный пароль». if row is None or not verify_password(password, row["password_hash"]): raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": WWW_BASIC}) settings = get_settings() token = new_session_token() - expires_at = session_expiry(settings.session_ttl_seconds) + ttl = settings.session_ttl_seconds + digest = token_digest(token) - with transaction() as conn: - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO sessions (user_id, token_hash, expires_at) - VALUES (%s, %s, %s) - """, - (row["id"], token_digest(token), expires_at), - ) + # СОХРАНЯЕМ В REDIS + redis_client.setex(f"session:{digest}", ttl, str(row["id"])) return { "token": token, "token_type": "Bearer", - "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + "expires_at": session_expiry(ttl).isoformat().replace("+00:00", "Z"), "user": row_to_user(row), } - def _extract_bearer(request: Request) -> str: header = request.headers.get("Authorization") if not header or not header.lower().startswith("bearer "): @@ -85,47 +79,34 @@ def _extract_bearer(request: Request) -> str: raise ApiError(401, "unauthorized", "Требуется Bearer", {"WWW-Authenticate": WWW_BEARER}) return token - def require_actor(request: Request) -> dict: - """Зависимость: валидирует Bearer и возвращает actor с user и token_hash.""" token = _extract_bearer(request) digest = token_digest(token) + + # ЧИТАЕМ ИЗ REDIS + user_id = redis_client.get(f"session:{digest}") + if user_id is None: + raise ApiError(401, "unauthorized", "Сессия недействительна или истекла", {"WWW-Authenticate": WWW_BEARER}) + with connection() as conn: with conn.cursor(row_factory=dict_row) as cur: cur.execute( - """ - SELECT s.user_id, s.expires_at, - u.id, u.username, u.display_name, u.created_at - FROM sessions s - JOIN users u ON u.id = s.user_id - WHERE s.token_hash = %s - """, - (digest,), + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), ) row = cur.fetchone() + if row is None: - raise ApiError(401, "unauthorized", "Сессия недействительна", {"WWW-Authenticate": WWW_BEARER}) - - # Проверка срока на стороне БД - with connection() as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > now()", - (digest,), - ) - if cur.fetchone() is None: - raise ApiError(401, "unauthorized", "Сессия истекла", {"WWW-Authenticate": WWW_BEARER}) - + raise ApiError(401, "unauthorized", "Пользователь не найден", {"WWW-Authenticate": WWW_BEARER}) + return { "user": row_to_user(row), - "user_id": row["user_id"], + "user_id": user_id, "token_hash": digest, } - @router.delete("/auth/sessions/current", status_code=204) def revoke_session(request: Request, actor: dict = Depends(require_actor)): - with transaction() as conn: - with conn.cursor() as cur: - cur.execute("DELETE FROM sessions WHERE token_hash = %s", (actor["token_hash"],)) + # УДАЛЯЕМ ИЗ REDIS + redis_client.delete(f"session:{actor['token_hash']}") return Response(status_code=204) \ No newline at end of file diff --git a/test_lab5.sh b/test_lab5.sh new file mode 100644 index 00000000..4a5f0708 --- /dev/null +++ b/test_lab5.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Базовый URL +BASE_URL="http://localhost:8080" + +# 1. Создаем уникального пользователя (добавляем случайное число, чтобы не было конфликта) +RANDOM_NUM=$RANDOM +USERNAME="test_user_$RANDOM_NUM" +PASSWORD="verysecretpassword123" + +echo "1. Создаем пользователя: $USERNAME..." +RESPONSE=$(echo "{\"username\": \"$USERNAME\", \"display_name\": \"Тест\", \"password\": \"$PASSWORD\"}" | curl -s -X POST $BASE_URL/api/v1/users -H "Content-Type: application/json" -d @-) +USER_ID=$(echo $RESPONSE | jq -r '.id') +echo " Пользователь создан. ID: $USER_ID" + +# 2. Логинимся +echo "2. Логинимся..." +RESPONSE=$(curl -s -X POST $BASE_URL/api/v1/auth/sessions -H "Authorization: Basic $(echo -n "$USERNAME:$PASSWORD" | base64)") +TOKEN=$(echo $RESPONSE | jq -r '.token') +if [ "$TOKEN" == "null" ]; then + echo " Ошибка логина: $RESPONSE" + exit 1 +fi +echo " Токен получен." + +# 3. Создаем чат +echo "3. Создаем чат..." +RESPONSE=$(echo "{\"title\": \"Чат для теста\", \"member_ids\": [\"$USER_ID\"]}" | curl -s -X POST $BASE_URL/api/v1/chats -H "Content-Type: application/json" -H "Authorization: Bearer $TOKEN" -d @-) +CHAT_ID=$(echo $RESPONSE | jq -r '.id') +echo " Чат создан. ID: $CHAT_ID" + +# 4. Загружаем картинку +echo "4. Загружаем картинку (sample.png)..." +# Убедимся, что файл существует +if [ ! -f "tests/fixtures/sample.png" ]; then + echo " ОШИБКА: Файл tests/fixtures/sample.png не найден!" + exit 1 +fi + +RESPONSE=$(curl -s -X POST "$BASE_URL/api/v1/chats/$CHAT_ID/attachments?filename=sample.png" \ + -H "Content-Type: image/png" \ + -H "Authorization: Bearer $TOKEN" \ + --data-binary "@tests/fixtures/sample.png") + +ATTACHMENT_ID=$(echo $RESPONSE | jq -r '.id') +STATUS=$(echo $RESPONSE | jq -r '.status') + +echo " Вложение загружено. ID: $ATTACHMENT_ID" +echo " Статус сразу после загрузки: $STATUS (ожидаем queued)" + +if [ "$STATUS" != "queued" ]; then + echo " ВНИМАНИЕ: Статус не queued! Проверь API." +fi + +# 5. Ждем, пока воркер обработает +echo "5. Ждем 3 секунды, пока воркер обработает картинку..." +sleep 3 + +# 6. Проверяем статус +echo "6. Проверяем итоговый статус вложения..." +RESPONSE=$(curl -s -X GET "$BASE_URL/api/v1/attachments/$ATTACHMENT_ID" -H "Authorization: Bearer $TOKEN") +FINAL_STATUS=$(echo $RESPONSE | jq -r '.status') +VARIANTS_COUNT=$(echo $RESPONSE | jq '.variants | length') + +echo " Итоговый статус: $FINAL_STATUS (ожидаем ready)" +echo " Количество вариантов: $VARIANTS_COUNT (ожидаем 3)" + +if [ "$FINAL_STATUS" == "ready" ] && [ "$VARIANTS_COUNT" -eq 3 ]; then + echo "✅ УСПЕХ! Асинхронная обработка работает." +else + echo "❌ ОШИБКА! Воркер не обработал файл. Проверь логи: docker compose logs worker" +fi \ No newline at end of file diff --git a/worker/main.py b/worker/main.py index 21d41073..fccdf61e 100644 --- a/worker/main.py +++ b/worker/main.py @@ -2,30 +2,35 @@ import asyncio import io import json import logging -from datetime import datetime, timezone +import hashlib import aio_pika from PIL import Image -import psycopg from psycopg.rows import dict_row from src.app.config import get_settings from src.app.db import init_pool, transaction -from src.app.s3 import get_s3_client, download_fileobj, upload_fileobj +from src.app.s3 import download_fileobj, upload_fileobj logging.basicConfig(level=logging.INFO) logger = logging.getLogger("worker") + def generate_variants(img: Image.Image): variants = {} # 1. Thumbnail (макс 256x256, сохраняем пропорции) thumb = img.copy() + # Если картинка имеет прозрачность (RGBA), конвертируем в RGB для JPEG + if thumb.mode in ("RGBA", "P"): + thumb = thumb.convert("RGB") thumb.thumbnail((256, 256)) variants["thumbnail"] = thumb # 2. Crop (квадрат 512x512 по центру) crop = img.copy() + if crop.mode in ("RGBA", "P"): + crop = crop.convert("RGB") min_dim = min(crop.size) left = (crop.width - min_dim) // 2 top = (crop.height - min_dim) // 2 @@ -33,14 +38,13 @@ def generate_variants(img: Image.Image): crop = crop.resize((512, 512), Image.Resampling.LANCZOS) variants["crop"] = crop - # 3. Optimized (просто оригинал в JPEG с качеством 85) + # 3. Optimized (просто оригинал в JPEG) opt = img.convert("RGB") - buf = io.BytesIO() - opt.save(buf, format="JPEG", quality=85) - variants["optimized"] = Image.open(buf) + variants["optimized"] = opt return variants + async def process_message(message: aio_pika.IncomingMessage): async with message.process(): data = json.loads(message.body) @@ -68,15 +72,12 @@ async def process_message(message: aio_pika.IncomingMessage): with conn.cursor() as cur: for name, var_img in variants.items(): buf = io.BytesIO() + # Всегда сохраняем как JPEG (так требует контракт) ct = "image/jpeg" - if name == "thumbnail": - var_img.save(buf, format="PNG") - ct = "image/png" - else: - var_img.save(buf, format="JPEG", quality=85) + var_img.save(buf, format="JPEG", quality=85) var_bytes = buf.getvalue() - sha256 = __import__("hashlib").sha256(var_bytes).hexdigest() + sha256 = hashlib.sha256(var_bytes).hexdigest() var_key = f"chats/{att_id}/variants/{name}" upload_fileobj(io.BytesIO(var_bytes), var_key, ct) @@ -104,13 +105,22 @@ async def process_message(message: aio_pika.IncomingMessage): with transaction() as conn: with conn.cursor() as cur: cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s", ("processing_error", att_id)) - raise e # Пробрасываем ошибку, чтобы RabbitMQ не удалил сообщение (at-least-once) + raise e + async def main(): init_pool() settings = get_settings() - connection = await aio_pika.connect_robust(settings.broker_url) + # Ждем, пока брокер станет доступен + while True: + try: + connection = await aio_pika.connect_robust(settings.broker_url) + break # если подключились - выходим из цикла + except Exception as e: + logger.warning(f"Broker not ready, waiting 5 seconds... ({e})") + await asyncio.sleep(5) + async with connection: channel = await connection.channel() await channel.set_qos(prefetch_count=1) @@ -119,5 +129,6 @@ async def main(): await queue.consume(process_message) await asyncio.Future() + if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file -- 2.54.0 From a11a36754c140088e74b99c2a0d77bd6c8b2c6cd Mon Sep 17 00:00:00 2001 From: caramelteawithmint <146834163+caramelteawithmint@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:17:03 +0300 Subject: [PATCH 4/4] 4grade_and_report --- Dockerfile | 11 +++- REPORT.md | 66 +++++++++++++++---- src/app/routes/attachments.py | 35 +++------- src/migrations/004_outbox.sql | 19 ++++++ worker/main.py | 120 +++++++++++++++++++++++++--------- worker/outbox_publisher.py | 52 +++++++++++++++ 6 files changed, 231 insertions(+), 72 deletions(-) create mode 100644 src/migrations/004_outbox.sql create mode 100644 worker/outbox_publisher.py diff --git a/Dockerfile b/Dockerfile index 431c1d5c..46c4106f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,16 @@ WORKDIR /app # libpq-dev для psycopg, gcc для сборки некоторых python-пакетов, # libjpeg-dev и zlib1g-dev нужны для работы Pillow (обработка картинок) RUN apt-get update && apt-get install -y --no-install-recommends \ - libpq-dev gcc libjpeg-dev zlib1g-dev \ + build-essential \ + python3-dev \ + libjpeg-dev \ + zlib1g-dev \ + libpng-dev \ + libwebp-dev \ + libpq-dev \ && rm -rf /var/lib/apt/lists/* - COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --index-url https://mirrors.aliyun.com/pypi/simple/ -r requirements.txt # Копируем код API, Воркера и скрипты (включая migrate.py) COPY src/ ./src/ diff --git a/REPORT.md b/REPORT.md index e98f75e9..7d6c1505 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,75 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Бахаева Лиана, БПИ2302 +- Целевая оценка: 4 - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: Материалы курса, документация RabbitMQ (reliability, confirm), FastAPI, MinIO, Redis. Использовалась помощь ИИ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Требования:** +- Docker Desktop (Compose v2) +- Python 3.9+ (для локального запуска тестов) +- Git Bash (или эквивалент) для Windows + +**Шаги:** +1. Клонировать репозиторий и перейти в корень проекта. +2. Убедиться, что файл `lab.json` содержит `{"number": 5}`. +3. Запустить инфраструктуру: `docker compose up -d --build` +4. Дождаться, пока контейнеры `postgres`, `minio`, `broker`, `redis` станут `Healthy` (команда `docker compose ps`). +5. Проверить готовность API (через Load Balancer): `curl -i http://localhost:8080/health/ready`. Ожидаемый ответ: HTTP/1.1 200 OK, `{"status":"ok"}`. +6. Запустить публичный smoke-тест: `PYTHONUTF8=1 PYTHONPATH=tests python tests/smoke.py`. Ожидаемый ответ: PASS: lab5 public API scenario... + +**Остановка и удаление данных:** +- Безопасная остановка (с сохранением данных): `docker compose stop` +- Полное удаление учебных данных: `docker compose down -v` (удаляет все volumes: pgdata, s3data, brokerdata, redisdata). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема процессов:** +- `lb` (Nginx): публичный вход на порт 8080. Распределяет трафик между `api1` и `api2` (least_conn). +- `api1`, `api2` (FastAPI): две реплики без локального состояния. Сессии и идемпотентность хранятся в Redis, данные — в Postgres. +- `postgres`: основное хранилище (пользователи, чаты, сообщения, вложения, outbox). +- `minio` (S3): приватный бакет `messenger-attachments`. +- `broker` (RabbitMQ): очередь `attachment_processing` с настройкой DLQ (`dead_letters`). +- `redis`: хранилище сессий и ключей идемпотентности (durable persistence через `appendonly yes`). +- `worker`: отдельный процесс. Читает `outbox_messages` из Postgres, отправляет задачи в RabbitMQ, скачивает файл из MinIO, генерирует варианты (thumbnail, crop, optimized), сохраняет их в MinIO и обновляет БД. + +**Источник истины и транзакционные границы:** +- Метаданные вложений и статусы — в Postgres. +- Байты изображений (оригинал и варианты) — в MinIO. +- Транзакционная граница (Transactional Outbox): при загрузке файла в одной транзакции Postgres создаются записи в `attachments` и `outbox_messages`. Это гарантирует, что если API успешно ответил 202, задача обязательно будет отправлена в RabbitMQ (даже если брокер временно недоступен). + +**Что перенесено из lab4 и что изменено:** +- Перенесена логика аутентификации (Argon2), работа с чатами и сообщениями, идемпотентность сообщений. +- Изменено: загрузка вложений теперь асинхронная (202 queued), добавлен Outbox и фоновый публикатор, добавлен Retry с backoff/jitter и DLQ, внедрено хранение сессий в Redis, формат optimized — WebP, crop — 128x128. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `PYTHONUTF8=1 PYTHONPATH=tests python tests/smoke.py` | PASS: lab5 public API scenario | tests/smoke.py (скриншот терминала) | +| Остановка worker -> upload -> restart worker | `docker compose stop worker`; загрузка файла через API; `docker compose start worker` | Файл переходит из queued в ready (Outbox спасает задачу) | Логи worker (обработка после рестарта) | +| Durable persistence Redis | Залогиниться, получить токен; `docker compose restart redis`; запрос `/users/me` с тем же токеном | 200 OK (сессия выжила) | Скриншот терминала | +| Ограниченный retry и DLQ | Отправить битый файл; 5 неудачных попыток воркера | Статус failed, задача уходит в dead_letters | Логи worker, `docker compose logs broker` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** Outbox защищает от потери задач при падении брокера. +**Условия:** Docker Desktop, 8 GB RAM, локальная машина. +**Нагрузка:** 10 параллельных загрузок PNG (тест `tests/load.py`). +**Результаты:** все 10 файлов успешно обработаны, задержка между загрузкой и ready составила ~2-3 секунды. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка RabbitMQ | API продолжает принимать файлы (202), задачи копятся в outbox | API отвечает 202, статус queued. После запуска брокера задачи обрабатываются | ~5 секунд (пока брокер поднимался), данные не потеряны | +| Остановка worker | Файлы загружаются (202), но не переходят в ready | API отвечает 202, статус queued. После запуска worker — ready | ~10 секунд, данные не потеряны | +| Рестарт Redis | Сессии сохраняются (durable persistence) | Старый токен работает после перезапуска | ~5 секунд, сессии не потеряны | + ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Что выдерживается:** отказ брокера (через Transactional Outbox), отказ worker (задачи копятся в очереди и обрабатываются после перезапуска), рестарт Redis (сессии и ключи идемпотентности сохраняются благодаря durable persistence). +- **Что не реализовано:** автоматическое масштабирование worker (сейчас ровно 1 экземпляр), сложные политики backoff с экспоненциальной задержкой через задержанные очереди (реализован упрощенный вариант с `asyncio.sleep` в воркере), сбор метрик (Prometheus/Grafana). +- **Учебное упрощение:** создание бакета MinIO выполняется вручную командой `docker compose exec minio ...` после первого запуска (или после `docker compose down -v`), так как автоматическое создание через `entrypoint` оказалось нестабильным из-за гонки при старте. В промышленной эксплуатации для этого используют отдельный init-контейнер или Infrastructure-as-Code. +- **Следующие шаги:** добавить отдельный init-контейнер для создания бакета (чтобы избежать ручных действий), внедрить задержанные очереди (delayed message exchange) для более чистого backoff без блокировки воркера, добавить метрики Prometheus для мониторинга размера очереди и времени обработки, а также горизонтальное масштабирование воркеров (уже сейчас можно запустить несколько реплик `worker`, так как задачи идемпотентны, но это не тестировалось под нагрузкой). \ No newline at end of file diff --git a/src/app/routes/attachments.py b/src/app/routes/attachments.py index 9564a9b6..8aae82f6 100644 --- a/src/app/routes/attachments.py +++ b/src/app/routes/attachments.py @@ -3,13 +3,11 @@ import io import tempfile import uuid import json -import aio_pika from fastapi import APIRouter, Depends, Request, Query from fastapi.responses import StreamingResponse from psycopg.rows import dict_row -from ..config import get_settings from ..db import transaction, connection from ..errors import ApiError from ..s3 import upload_fileobj, download_fileobj, delete_file @@ -20,24 +18,6 @@ from .sessions import require_actor router = APIRouter(prefix="/api/v1") MAX_SIZE = 10 * 1024 * 1024 -async def _publish_task(attachment_id: str, object_key: str): - """Отправляет задачу в RabbitMQ.""" - settings = get_settings() - try: - connection = await aio_pika.connect_robust(settings.broker_url) - async with connection: - channel = await connection.channel() - await channel.declare_queue(settings.broker_queue, durable=True) - message = aio_pika.Message( - body=json.dumps({"attachment_id": attachment_id, "object_key": object_key}).encode(), - delivery_mode=aio_pika.DeliveryMode.PERSISTENT - ) - await channel.default_exchange.publish(message, routing_key=settings.broker_queue) - except Exception as e: - print(f"Broker error: {e}") - # Если брокер упал, мы можем либо откатить загрузку, либо оставить статус queued - # Для лабы: логируем, воркер потом не найдет задачу, но health/ready покажет ошибку - @router.post("/chats/{chat_id}/attachments", status_code=202) async def upload_attachment( chat_id: str, @@ -79,7 +59,6 @@ async def upload_attachment( if size == 0: raise ApiError(400, "invalid_request", "Пустой файл") - # Только быстрая проверка magic bytes, глубокий анализ оставляем воркеру try: content_type, _, _ = check_image(first_chunk, ["image/png", "image/jpeg"]) except ValueError: @@ -98,6 +77,7 @@ async def upload_attachment( try: with transaction() as conn: with conn.cursor(row_factory=dict_row) as cur: + # 1. Сохраняем вложение cur.execute( """ INSERT INTO attachments (chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status) @@ -107,6 +87,13 @@ async def upload_attachment( (chat_id, actor["user_id"], filename, content_type, size, sha256_hex, object_key), ) att = cur.fetchone() + + # 2. Сохраняем задачу в Outbox в ТОЙ ЖЕ транзакции + payload = {"attachment_id": str(att["id"]), "object_key": object_key} + cur.execute( + "INSERT INTO outbox_messages (attachment_id, payload) VALUES (%s, %s)", + (att["id"], json.dumps(payload)) + ) except Exception as e: print(f"DB failed, compensating S3: {e}") try: @@ -115,9 +102,6 @@ async def upload_attachment( print(f"Failed to compensate S3: {s3_e}") raise ApiError(500, "internal_error", "Ошибка сохранения") - # Публикуем задачу - await _publish_task(str(att["id"]), object_key) - return { "id": str(att["id"]), "chat_id": str(att["chat_id"]), @@ -133,6 +117,7 @@ async def upload_attachment( "updated_at": iso_z(att["updated_at"]), } + @router.get("/attachments/{attachment_id}") def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): try: @@ -186,6 +171,7 @@ def get_attachment(attachment_id: str, actor: dict = Depends(require_actor)): "updated_at": iso_z(att["updated_at"]), } + @router.get("/attachments/{attachment_id}/content") def download_attachment( attachment_id: str, @@ -225,7 +211,6 @@ def download_attachment( ) var = cur.fetchone() if not var: - # Если вариант еще не готов, возвращаем 404 или 409 raise ApiError(404, "not_found", "Вариант еще не готов или недоступен") obj_key = var["object_key"] content_type = var["content_type"] diff --git a/src/migrations/004_outbox.sql b/src/migrations/004_outbox.sql new file mode 100644 index 00000000..9790f60b --- /dev/null +++ b/src/migrations/004_outbox.sql @@ -0,0 +1,19 @@ +-- src/migrations/004_outbox.sql + +-- Таблица для Transactional Outbox +CREATE TABLE IF NOT EXISTS outbox_messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + processed_at timestamptz +); + +CREATE INDEX IF NOT EXISTS outbox_unprocessed_idx ON outbox_messages (created_at) WHERE processed_at IS NULL; + +-- Таблица для отслеживания попыток обработки +CREATE TABLE IF NOT EXISTS attachment_retries ( + attachment_id uuid PRIMARY KEY REFERENCES attachments(id) ON DELETE CASCADE, + attempts int NOT NULL DEFAULT 0, + last_error text +); \ No newline at end of file diff --git a/worker/main.py b/worker/main.py index fccdf61e..d6d6a112 100644 --- a/worker/main.py +++ b/worker/main.py @@ -3,6 +3,7 @@ import io import json import logging import hashlib +import random import aio_pika from PIL import Image @@ -12,22 +13,29 @@ from src.app.config import get_settings from src.app.db import init_pool, transaction from src.app.s3 import download_fileobj, upload_fileobj +# ИМПОРТ ФОНОВОГО ПРОЦЕССА OUTBOX +from worker.outbox_publisher import publish_loop + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("worker") +MAX_PIXELS = 40_000_000 + + +class PermanentError(Exception): + """Ошибка, которую нет смысла повторять (битый файл).""" + pass + def generate_variants(img: Image.Image): variants = {} - # 1. Thumbnail (макс 256x256, сохраняем пропорции) thumb = img.copy() - # Если картинка имеет прозрачность (RGBA), конвертируем в RGB для JPEG if thumb.mode in ("RGBA", "P"): thumb = thumb.convert("RGB") thumb.thumbnail((256, 256)) - variants["thumbnail"] = thumb + variants["thumbnail"] = (thumb, "image/jpeg", "JPEG", {"quality": 85}) - # 2. Crop (квадрат 512x512 по центру) crop = img.copy() if crop.mode in ("RGBA", "P"): crop = crop.convert("RGB") @@ -35,46 +43,84 @@ def generate_variants(img: Image.Image): left = (crop.width - min_dim) // 2 top = (crop.height - min_dim) // 2 crop = crop.crop((left, top, left + min_dim, top + min_dim)) - crop = crop.resize((512, 512), Image.Resampling.LANCZOS) - variants["crop"] = crop + crop = crop.resize((128, 128), Image.Resampling.LANCZOS) + variants["crop"] = (crop, "image/jpeg", "JPEG", {"quality": 85}) - # 3. Optimized (просто оригинал в JPEG) - opt = img.convert("RGB") - variants["optimized"] = opt + opt = img.copy() + if opt.mode in ("RGBA", "P", "L"): + opt = opt.convert("RGB") + opt.info.clear() + variants["optimized"] = (opt, "image/webp", "WEBP", {"quality": 80, "method": 4}) return variants +async def _re_publish(body: bytes): + settings = get_settings() + connection = await aio_pika.connect_robust(settings.broker_url) + async with connection: + channel = await connection.channel() + await channel.declare_queue(settings.broker_queue, durable=True) + msg = aio_pika.Message(body=body, delivery_mode=aio_pika.DeliveryMode.PERSISTENT) + await channel.default_exchange.publish(msg, routing_key=settings.broker_queue) + + +async def _send_to_dlq(body: bytes): + settings = get_settings() + connection = await aio_pika.connect_robust(settings.broker_url) + async with connection: + channel = await connection.channel() + await channel.declare_queue("dead_letters", durable=True) + msg = aio_pika.Message(body=body, delivery_mode=aio_pika.DeliveryMode.PERSISTENT) + await channel.default_exchange.publish(msg, routing_key="dead_letters") + + async def process_message(message: aio_pika.IncomingMessage): - async with message.process(): + async with message.process(requeue=False, reject_on_redelivered=False): data = json.loads(message.body) att_id = data["attachment_id"] obj_key = data["object_key"] logger.info(f"Processing attachment {att_id}") - try: - # 1. Обновляем статус на processing - with transaction() as conn: - with conn.cursor() as cur: - cur.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,)) + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute("SELECT attempts FROM attachment_retries WHERE attachment_id = %s", (att_id,)) + row = cur.fetchone() + attempts = row["attempts"] if row else 0 + + if attempts >= 5: + logger.error(f"Max retries (5) reached for {att_id}. Sending to DLQ.") + cur.execute("UPDATE attachments SET status='failed', error_code='max_retries_exceeded', updated_at=now() WHERE id=%s", (att_id,)) + await _send_to_dlq(message.body) + return - # 2. Качаем из S3 + cur.execute( + """ + INSERT INTO attachment_retries (attachment_id, attempts) + VALUES (%s, 1) + ON CONFLICT (attachment_id) DO UPDATE SET attempts = attachment_retries.attempts + 1 + """, (att_id,) + ) + cur.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,)) + + try: body = download_fileobj(obj_key) img = Image.open(body) - img.load() # принудительная загрузка в память для обнаружения битых файлов + + width, height = img.size + if width * height > MAX_PIXELS: + raise PermanentError(f"Image too large: {width}x{height} pixels") + + img.load() - # 3. Генерируем варианты variants = generate_variants(img) - # 4. Сохраняем в S3 и БД with transaction() as conn: with conn.cursor() as cur: - for name, var_img in variants.items(): + for name, (var_img, ct, fmt, save_args) in variants.items(): buf = io.BytesIO() - # Всегда сохраняем как JPEG (так требует контракт) - ct = "image/jpeg" - var_img.save(buf, format="JPEG", quality=85) + var_img.save(buf, format=fmt, **save_args) var_bytes = buf.getvalue() sha256 = hashlib.sha256(var_bytes).hexdigest() @@ -98,25 +144,39 @@ async def process_message(message: aio_pika.IncomingMessage): ) cur.execute("UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=%s", (att_id,)) + cur.execute("DELETE FROM attachment_retries WHERE attachment_id = %s", (att_id,)) + logger.info(f"Successfully processed {att_id}") - except Exception as e: - logger.error(f"Error processing {att_id}: {e}") + except PermanentError as e: + logger.error(f"Permanent error for {att_id}: {e}") with transaction() as conn: with conn.cursor() as cur: - cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s", ("processing_error", att_id)) - raise e + cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s", ("invalid_image", att_id)) + + except Exception as e: + logger.error(f"Temporary error for {att_id}: {e}") + + delay = min(2 ** attempts, 60) + jitter = random.uniform(0, 3) + total_delay = delay + jitter + + logger.info(f"Retrying {att_id} in {total_delay:.2f} seconds (attempt {attempts+1}/5)") + await asyncio.sleep(total_delay) + await _re_publish(message.body) async def main(): init_pool() settings = get_settings() - # Ждем, пока брокер станет доступен + # ЗАПУСК ФОНОВОГО ПРОЦЕССА OUTBOX + asyncio.create_task(publish_loop()) + while True: try: connection = await aio_pika.connect_robust(settings.broker_url) - break # если подключились - выходим из цикла + break except Exception as e: logger.warning(f"Broker not ready, waiting 5 seconds... ({e})") await asyncio.sleep(5) @@ -129,6 +189,6 @@ async def main(): await queue.consume(process_message) await asyncio.Future() - + if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file diff --git a/worker/outbox_publisher.py b/worker/outbox_publisher.py new file mode 100644 index 00000000..14a7a8c1 --- /dev/null +++ b/worker/outbox_publisher.py @@ -0,0 +1,52 @@ +import asyncio +import json +import logging +import aio_pika +from src.app.config import get_settings +from src.app.db import transaction +from psycopg.rows import dict_row + +logger = logging.getLogger("outbox") + +async def publish_loop(): + settings = get_settings() + + # Ждем брокер + while True: + try: + connection = await aio_pika.connect_robust(settings.broker_url) + break + except Exception: + await asyncio.sleep(5) + + async with connection: + channel = await connection.channel() + await channel.declare_queue(settings.broker_queue, durable=True) + + while True: + try: + # Забираем необработанные сообщения + with transaction() as conn: + with conn.cursor(row_factory=dict_row) as cur: + cur.execute( + "SELECT id, payload FROM outbox_messages WHERE processed_at IS NULL ORDER BY created_at LIMIT 10 FOR UPDATE SKIP LOCKED" + ) + messages = cur.fetchall() + + for msg in messages: + payload = msg["payload"] + # Отправляем в RabbitMQ + message = aio_pika.Message( + body=json.dumps(payload).encode(), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT + ) + await channel.default_exchange.publish(message, routing_key=settings.broker_queue) + + # Помечаем как обработанное + cur.execute("UPDATE outbox_messages SET processed_at = now() WHERE id = %s", (msg["id"],)) + + if not messages: + await asyncio.sleep(1) # Если пусто, ждем секунду + except Exception as e: + logger.error(f"Outbox error: {e}") + await asyncio.sleep(2) \ No newline at end of file -- 2.54.0