feat(lab5): async image processing with RabbitMQ, transactional outbox, and crash recovery

This commit is contained in:
2026-11-13 04:04:43 +03:00
parent 276dbdad8a
commit cfd54ef9aa
22 changed files with 1790 additions and 4 deletions
+23
View File
@@ -0,0 +1,23 @@
FROM python:3.12-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
gcc \
libpq-dev \
curl \
libmagic1 \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app/ ./app/
COPY alembic.ini .
COPY alembic/ ./alembic/
# Создаём непривилегированного пользователя для read-only контейнера
RUN useradd -m appuser && chown -R appuser:appuser /app
USER appuser
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
+8 -4
View File
@@ -1,17 +1,21 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Будилова Ярослава, БВТ2304
- Целевая оценка; 4
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI, SQLAlchemy, RabbitMQ (aio-pika), SeaweedFS (S3), PostgreSQL, Valkey.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
## Архитектура и решения
1. **Разделение ответственности:** API принимает файл, валидирует его (MIME через `python-magic`, размер до 10 MiB, разрешение до 40 Мпикселей), сохраняет во временный префикс S3 (`staging/`) и создаёт запись в БД (`Attachment`, `AttachmentJob`, `JobOutbox`). Возвращает `202 Accepted`.
2. **Transactional Outbox:** Фоновый процесс в API (`app/broker.py`) каждую секунду опрашивает таблицу `job_outbox` на наличие непубликованных задач и отправляет их в RabbitMQ с `DeliveryMode.PERSISTENT`. Это гарантирует, что даже если API упадёт сразу после COMMIT в БД, задача не будет потеряна.
3. **Worker:** Отдельный процесс, который забирает сообщения из очереди `attachment.process.v1`. Использует `SELECT ... FOR UPDATE` для блокировки задачи (lease на 30 секунд). Обрабатывает изображение (создаёт `thumbnail` 256x256, `crop` 128x128, `optimized` WebP 1280x1280), сохраняет в конечный префикс S3, обновляет статус в БД на `ready` и только затем отправляет `ack` брокеру.
4. **Идемпотентность:** Если Worker падает после записи в S3, но до `ack` или COMMIT, RabbitMQ повторно доставит сообщение. Worker проверяет статус задачи в БД: если она уже `ready` или `processing` с живым lease, он игнорирует дубль (`no-op`). Первичный ключ `(attachment_id, variant_name)` физически предотвращает создание дубликатов на уровне БД.
5. **Очистка:** После успешного COMMIT статус `ready`, Worker удаляет объект из `staging/`.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
## Доказательства критериев
+41
View File
@@ -0,0 +1,41 @@
[alembic]
script_location = alembic
prepend_sys_path = .
version_path_separator = os
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
+55
View File
@@ -0,0 +1,55 @@
import asyncio
import os
import sys
from logging.config import fileConfig
from sqlalchemy import pool
from sqlalchemy.engine import Connection
from sqlalchemy.ext.asyncio import async_engine_from_config
from alembic import context
# Добавляем корень проекта в sys.path, чтобы импортировать app
sys.path.append(os.path.dirname(os.path.dirname(__file__)))
from app.database import Base, DATABASE_URL
from app.models import User, UserSession, Chat, ChatMember, Message, IdempotencyKey, Attachment, MessageAttachment
config = context.config
if config.config_file_name is not None:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
context.configure(
url=DATABASE_URL,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def do_run_migrations(connection: Connection) -> None:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async_migrations() -> None:
connectable = async_engine_from_config(
{"sqlalchemy.url": DATABASE_URL},
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
async with connectable.connect() as connection:
await connection.run_sync(do_run_migrations)
await connectable.dispose()
def run_migrations_online() -> None:
asyncio.run(run_async_migrations())
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,146 @@
"""lab5 initial schema with async processing
Revision ID: 7fc277b41201
Revises:
Create Date: 2026-11-12 16:13:31.689026
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '7fc277b41201'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('chats',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('title', sa.String(length=100), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('job_outbox',
sa.Column('job_id', sa.UUID(), nullable=False),
sa.Column('payload', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column('published_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('job_id')
)
op.create_table('users',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('username', sa.String(length=32), nullable=False),
sa.Column('display_name', sa.String(length=100), nullable=False),
sa.Column('password_hash', sa.String(length=255), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_users_username'), 'users', ['username'], unique=True)
op.create_table('attachments',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('owner_id', sa.UUID(), nullable=False),
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.Column('staging_key', sa.String(length=512), nullable=True),
sa.Column('s3_key', sa.String(length=512), nullable=True),
sa.Column('original_filename', sa.String(length=255), nullable=False),
sa.Column('content_type', sa.String(length=100), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.Column('status', sa.String(length=20), nullable=False),
sa.Column('error_code', sa.String(length=50), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['owner_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('chat_members',
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.Column('user_id', sa.UUID(), nullable=False),
sa.Column('joined_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('chat_id', 'user_id')
)
op.create_table('idempotency_keys',
sa.Column('key', sa.String(length=255), nullable=False),
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.Column('response_data', sa.Text(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('key')
)
op.create_table('messages',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('chat_id', sa.UUID(), nullable=False),
sa.Column('sender_id', sa.UUID(), nullable=False),
sa.Column('text', sa.Text(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['sender_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_messages_chat_id'), 'messages', ['chat_id'], unique=False)
op.create_index(op.f('ix_messages_created_at'), 'messages', ['created_at'], unique=False)
op.create_table('sessions',
sa.Column('token', sa.String(length=255), nullable=False),
sa.Column('user_id', sa.UUID(), nullable=False),
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('token')
)
op.create_table('attachment_jobs',
sa.Column('job_id', sa.UUID(), nullable=False),
sa.Column('attachment_id', sa.UUID(), nullable=False),
sa.Column('attempt', sa.Integer(), nullable=False),
sa.Column('status', sa.String(length=20), nullable=False),
sa.Column('lease_until', sa.DateTime(timezone=True), nullable=True),
sa.Column('lease_owner', sa.String(length=100), nullable=True),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('job_id'),
sa.UniqueConstraint('attachment_id')
)
op.create_table('attachment_variants',
sa.Column('attachment_id', sa.UUID(), nullable=False),
sa.Column('variant_name', sa.String(length=20), nullable=False),
sa.Column('s3_key', sa.String(length=512), nullable=False),
sa.Column('content_type', sa.String(length=100), nullable=False),
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
sa.Column('width', sa.Integer(), nullable=False),
sa.Column('height', sa.Integer(), nullable=False),
sa.Column('sha256', sa.String(length=64), nullable=False),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('attachment_id', 'variant_name')
)
op.create_table('message_attachments',
sa.Column('message_id', sa.UUID(), nullable=False),
sa.Column('attachment_id', sa.UUID(), nullable=False),
sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'),
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('message_id', 'attachment_id')
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('message_attachments')
op.drop_table('attachment_variants')
op.drop_table('attachment_jobs')
op.drop_table('sessions')
op.drop_index(op.f('ix_messages_created_at'), table_name='messages')
op.drop_index(op.f('ix_messages_chat_id'), table_name='messages')
op.drop_table('messages')
op.drop_table('idempotency_keys')
op.drop_table('chat_members')
op.drop_table('attachments')
op.drop_index(op.f('ix_users_username'), table_name='users')
op.drop_table('users')
op.drop_table('job_outbox')
op.drop_table('chats')
# ### end Alembic commands ###
View File
+56
View File
@@ -0,0 +1,56 @@
import os
import json
import logging
import asyncio
from datetime import datetime, timezone
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
import aio_pika
from app.models import JobOutbox
logger = logging.getLogger(__name__)
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
_connection = None
_channel = None
async def get_channel():
global _connection, _channel
if _connection is None or _connection.is_closed:
_connection = await aio_pika.connect_robust(BROKER_URL)
_channel = await _connection.channel()
# Гарантируем, что очередь существует
await _channel.declare_queue("attachment.process.v1", durable=True)
return _channel
async def relay_outbox(db: AsyncSession):
"""Забирает непубликованные задачи из outbox и отправляет их в RabbitMQ"""
global _channel
if _channel is None:
await get_channel()
# Берём до 10 непубликованных записей
stmt = select(JobOutbox).where(JobOutbox.published_at.is_(None)).limit(10)
result = await db.execute(stmt)
outbox_records = result.scalars().all()
for record in outbox_records:
try:
payload = json.dumps(record.payload)
message = aio_pika.Message(
body=payload.encode(),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT
)
await _channel.default_exchange.publish(
message,
routing_key="attachment.process.v1"
)
# Помечаем как опубликованное
record.published_at = datetime.now(timezone.utc)
await db.commit()
logger.info(f"Relayed job {record.job_id} to broker")
except Exception as e:
logger.error(f"Failed to relay job {record.job_id}: {e}")
await db.rollback()
+13
View File
@@ -0,0 +1,13 @@
import os
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker, declarative_base
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db")
engine = create_async_engine(DATABASE_URL, echo=False)
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
Base = declarative_base()
async def get_db():
async with AsyncSessionLocal() as session:
yield session
+54
View File
@@ -0,0 +1,54 @@
import base64
import logging
from fastapi import Depends, HTTPException, status, Header
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models import User, UserSession
from app.redis_client import get_redis
logger = logging.getLogger(__name__)
security = HTTPBasic()
async def get_basic_auth(credentials: HTTPBasicCredentials = Depends(security)):
return credentials
async def get_current_user(
db: AsyncSession = Depends(get_db),
redis_client = Depends(get_redis),
authorization: str | None = Header(default=None),
x_user_id: str | None = Header(default=None, alias="X-User-Id")
):
if not authorization:
raise HTTPException(status_code=401, detail={"code": "unauthorized", "message": "Missing authorization header"}, headers={"WWW-Authenticate": "Bearer realm=\"messenger\""})
if not authorization.startswith("Bearer "):
raise HTTPException(status_code=401, detail={"code": "unauthorized", "message": "Invalid authorization scheme"}, headers={"WWW-Authenticate": "Bearer realm=\"messenger\""})
token = authorization[7:]
# Проверяем в Redis сначала
try:
user_id_str = await redis_client.get(f"session:{token}")
if user_id_str:
import uuid
result = await db.execute(select(User).where(User.id == uuid.UUID(user_id_str)))
user = result.scalar_one_or_none()
if user:
return user
except Exception as e:
logger.warning(f"Redis unavailable during auth: {e}")
# Fallback на PostgreSQL
result = await db.execute(select(UserSession).where(UserSession.token == token))
session = result.scalar_one_or_none()
if not session:
raise HTTPException(status_code=401, detail={"code": "unauthorized", "message": "Invalid or expired session"}, headers={"WWW-Authenticate": "Bearer realm=\"messenger\""})
result = await db.execute(select(User).where(User.id == session.user_id))
user = result.scalar_one_or_none()
if not user:
raise HTTPException(status_code=401, detail={"code": "unauthorized", "message": "User not found"})
return user
+93
View File
@@ -0,0 +1,93 @@
import os
import uuid
import logging
import traceback
import asyncio
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse
from fastapi.exceptions import RequestValidationError
from starlette.exceptions import HTTPException as StarletteHTTPException
from app.routers import auth, chats, attachments
from app.database import AsyncSessionLocal
from app.broker import relay_outbox
logger = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
# Запускаем фоновый relay при старте приложения
task = asyncio.create_task(run_relay())
yield
# Останавливаем при завершении
task.cancel()
async def run_relay():
while True:
try:
async with AsyncSessionLocal() as db:
await relay_outbox(db)
await asyncio.sleep(1) # Проверяем outbox каждую секунду
except Exception as e:
logger.error(f"Relay loop crashed: {e}")
await asyncio.sleep(5)
app = FastAPI(lifespan=lifespan)
app.include_router(auth.router)
app.include_router(chats.router)
app.include_router(attachments.router)
INSTANCE_ID = os.getenv("INSTANCE_ID", str(uuid.uuid4())[:8])
@app.middleware("http")
async def add_instance_and_request_id(request: Request, call_next):
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
request.state.request_id = request_id
try:
response = await call_next(request)
except Exception as e:
logger.error(f"Unhandled exception: {e}")
traceback.print_exc()
return JSONResponse(
status_code=500,
content={"error": {"code": "internal_error", "message": "Unexpected error", "request_id": request_id}},
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}
)
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.exception_handler(StarletteHTTPException)
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
detail = exc.detail
if isinstance(detail, dict):
code = detail.get("code", "internal_error")
message = detail.get("message", str(detail))
else:
code = "internal_error"
message = str(detail)
return JSONResponse(
status_code=exc.status_code,
content={"error": {"code": code, "message": message, "request_id": request.state.request_id}},
headers=exc.headers
)
@app.exception_handler(RequestValidationError)
async def validation_exception_handler(request: Request, exc: RequestValidationError):
return JSONResponse(
status_code=400,
content={"error": {"code": "invalid_request", "message": "Validation error", "request_id": request.state.request_id}},
headers={"X-Request-Id": request.state.request_id, "X-Instance-Id": INSTANCE_ID}
)
@app.get("/health/live")
async def health_live():
return {"status": "ok"}
@app.get("/health/ready")
async def health_ready():
return {"status": "ok"}
+122
View File
@@ -0,0 +1,122 @@
import uuid
from datetime import datetime, timezone
from sqlalchemy import Column, String, DateTime, ForeignKey, Text, Boolean, BigInteger, Integer
from sqlalchemy.orm import relationship
from sqlalchemy.dialects.postgresql import UUID, JSONB
from app.database import Base
class User(Base):
__tablename__ = "users"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
username = Column(String(32), unique=True, nullable=False, index=True)
display_name = Column(String(100), nullable=False)
password_hash = Column(String(255), nullable=False)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
sessions = relationship("UserSession", back_populates="user", cascade="all, delete-orphan")
chat_memberships = relationship("ChatMember", back_populates="user", cascade="all, delete-orphan")
class UserSession(Base):
__tablename__ = "sessions"
token = Column(String(255), primary_key=True)
user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
expires_at = Column(DateTime(timezone=True), nullable=False)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
user = relationship("User", back_populates="sessions")
class Chat(Base):
__tablename__ = "chats"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
title = Column(String(100), nullable=False)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
members = relationship("ChatMember", back_populates="chat", cascade="all, delete-orphan")
messages = relationship("Message", back_populates="chat", cascade="all, delete-orphan")
class ChatMember(Base):
__tablename__ = "chat_members"
chat_id = Column(UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True)
user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True)
joined_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
chat = relationship("Chat", back_populates="members")
user = relationship("User", back_populates="chat_memberships")
class Message(Base):
__tablename__ = "messages"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
chat_id = Column(UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), nullable=False, index=True)
sender_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
text = Column(Text, nullable=False)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True)
chat = relationship("Chat", back_populates="messages")
sender = relationship("User")
attachments = relationship("MessageAttachment", back_populates="message", cascade="all, delete-orphan")
class IdempotencyKey(Base):
__tablename__ = "idempotency_keys"
key = Column(String(255), primary_key=True)
chat_id = Column(UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), nullable=False)
response_data = Column(Text, nullable=False)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
# === НОВЫЕ МОДЕЛИ ДЛЯ LAB 5 ===
class Attachment(Base):
__tablename__ = "attachments"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
owner_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
chat_id = Column(UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), nullable=False)
# S3 ключи
staging_key = Column(String(512), nullable=True) # Временное хранилище до обработки
s3_key = Column(String(512), nullable=True) # Финальный ключ оригинала (заполняется worker)
original_filename = Column(String(255), nullable=False)
content_type = Column(String(100), nullable=False)
size_bytes = Column(BigInteger, nullable=False)
sha256 = Column(String(64), nullable=False)
# Статус обработки
status = Column(String(20), nullable=False, default="queued") # queued, processing, ready, failed
error_code = Column(String(50), nullable=True)
created_at = Column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc))
owner = relationship("User")
chat = relationship("Chat")
message_links = relationship("MessageAttachment", back_populates="attachment", cascade="all, delete-orphan")
variants = relationship("AttachmentVariant", back_populates="attachment", cascade="all, delete-orphan")
job = relationship("AttachmentJob", back_populates="attachment", uselist=False, cascade="all, delete-orphan")
class AttachmentVariant(Base):
__tablename__ = "attachment_variants"
attachment_id = Column(UUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True)
variant_name = Column(String(20), primary_key=True) # 'thumbnail', 'crop', 'optimized'
s3_key = Column(String(512), nullable=False)
content_type = Column(String(100), nullable=False)
size_bytes = Column(BigInteger, nullable=False)
width = Column(Integer, nullable=False)
height = Column(Integer, nullable=False)
sha256 = Column(String(64), nullable=False)
attachment = relationship("Attachment", back_populates="variants")
class AttachmentJob(Base):
__tablename__ = "attachment_jobs"
job_id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
attachment_id = Column(UUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), unique=True, nullable=False)
attempt = Column(Integer, nullable=False, default=1)
status = Column(String(20), nullable=False, default="queued") # queued, processing, ready, failed
lease_until = Column(DateTime(timezone=True), nullable=True)
lease_owner = Column(String(100), nullable=True)
attachment = relationship("Attachment", back_populates="job")
class JobOutbox(Base):
__tablename__ = "job_outbox"
job_id = Column(UUID(as_uuid=True), primary_key=True)
payload = Column(JSONB, nullable=False)
published_at = Column(DateTime(timezone=True), nullable=True)
class MessageAttachment(Base):
__tablename__ = "message_attachments"
message_id = Column(UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True)
attachment_id = Column(UUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True)
message = relationship("Message", back_populates="attachments")
attachment = relationship("Attachment", back_populates="message_links")
+9
View File
@@ -0,0 +1,9 @@
import os
import redis.asyncio as redis
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
redis_client = redis.from_url(REDIS_URL, decode_responses=True)
async def get_redis():
return redis_client
+245
View File
@@ -0,0 +1,245 @@
import os
import uuid
import hashlib
import magic
import tempfile
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, status, Query, Request
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models import Attachment, ChatMember, AttachmentJob, JobOutbox
from app.schemas import Attachment as AttachmentSchema, Variant, ErrorResponse
from app.dependencies import get_current_user
from app.s3_client import get_s3_client, upload_file_to_s3, delete_file_from_s3
from minio import Minio
from PIL import Image
import io
from fastapi.responses import Response
router = APIRouter()
@router.post("/api/v1/chats/{chat_id}/attachments", status_code=status.HTTP_202_ACCEPTED, responses={202: {"model": AttachmentSchema}, 400: {"model": ErrorResponse}, 401: {"model": ErrorResponse}, 404: {"model": ErrorResponse}, 413: {"model": ErrorResponse}, 415: {"model": ErrorResponse}, 503: {"model": ErrorResponse}})
async def upload_attachment(
chat_id: uuid.UUID,
filename: str = Query(..., min_length=1, max_length=255),
request: Request = None,
current_user = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
s3: Minio = Depends(get_s3_client)
):
# 1. Проверка доступа к чату
is_member = await db.execute(
select(ChatMember).where(ChatMember.chat_id == chat_id, ChatMember.user_id == current_user.id)
)
if not is_member.scalar_one_or_none():
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Chat not found or access denied"})
# 2. Чтение сырых байтов с ограничением (защита от DoS)
MAX_SIZE = 10 * 1024 * 1024 # 10 MiB
body = await request.body()
if len(body) > MAX_SIZE:
raise HTTPException(status_code=413, detail={"code": "payload_too_large", "message": "File size exceeds 10 MiB limit"})
if len(body) == 0:
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Empty file"})
# 3. Определение MIME-типа по содержимому
mime = magic.from_buffer(body, mime=True)
if mime not in ("image/png", "image/jpeg"):
raise HTTPException(status_code=415, detail={"code": "unsupported_media_type", "message": "Only image/png and image/jpeg are supported"})
# 4. Быстрая проверка размеров изображения (без полного декодирования)
try:
img = Image.open(io.BytesIO(body))
width, height = img.size
if width * height > 40_000_000:
raise HTTPException(status_code=413, detail={"code": "payload_too_large", "message": "Image resolution exceeds 40 million pixels"})
except Exception:
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Invalid or corrupted image file"})
# 5. Вычисление SHA256
sha256_hash = hashlib.sha256(body).hexdigest()
# 6. Генерация ключей
att_id = uuid.uuid4()
ext = "png" if mime == "image/png" else "jpeg"
staging_key = f"staging/{att_id}.{ext}"
# 7. Загрузка в S3 staging
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp:
tmp.write(body)
tmp_path = tmp.name
try:
await upload_file_to_s3(staging_key, tmp_path, mime)
except Exception:
raise HTTPException(status_code=503, detail={"code": "unavailable", "message": "Failed to upload to S3 staging"})
finally:
os.unlink(tmp_path)
# 8. Транзакция: Создание Attachment, Job и Outbox сообщения
job_id = uuid.uuid4()
now = datetime.now(timezone.utc)
new_attachment = Attachment(
id=att_id,
owner_id=current_user.id,
chat_id=chat_id,
staging_key=staging_key,
original_filename=filename,
content_type=mime,
size_bytes=len(body),
sha256=sha256_hash,
status="queued"
)
new_job = AttachmentJob(
job_id=job_id,
attachment_id=att_id,
attempt=1,
status="queued"
)
outbox_payload = {
"job_id": str(job_id),
"attachment_id": str(att_id),
"attempt": 1
}
new_outbox = JobOutbox(
job_id=job_id,
payload=outbox_payload,
published_at=None
)
db.add(new_attachment)
db.add(new_job)
db.add(new_outbox)
try:
await db.commit()
except Exception:
await db.rollback()
try:
await delete_file_from_s3(staging_key)
except Exception:
pass
raise HTTPException(status_code=503, detail={"code": "unavailable", "message": "Database error during job creation"})
now_str = now.isoformat().replace('+00:00', 'Z')
# Возвращаем ПОЛНЫЙ объект AttachmentSchema, как требует OpenAPI контракт
return AttachmentSchema(
id=str(att_id),
chat_id=str(chat_id),
owner_id=str(current_user.id),
filename=filename,
content_type=mime,
size_bytes=len(body),
sha256=sha256_hash,
status="queued",
variants=[],
error_code=None,
created_at=now_str,
updated_at=now_str
)
@router.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentSchema, responses={401: {"model": ErrorResponse}, 404: {"model": ErrorResponse}})
async def get_attachment_metadata(
attachment_id: uuid.UUID,
current_user = Depends(get_current_user),
db: AsyncSession = Depends(get_db)
):
stmt = select(Attachment).where(Attachment.id == attachment_id)
result = await db.execute(stmt)
attachment = result.scalar_one_or_none()
if not attachment:
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Attachment not found"})
is_member = await db.execute(
select(ChatMember).where(ChatMember.chat_id == attachment.chat_id, ChatMember.user_id == current_user.id)
)
if not is_member.scalar_one_or_none():
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Access denied"})
variants_list = []
if attachment.status == "ready":
from app.models import AttachmentVariant
var_stmt = select(AttachmentVariant).where(AttachmentVariant.attachment_id == attachment_id)
var_result = await db.execute(var_stmt)
for v in var_result.scalars().all():
variants_list.append(Variant(
name=v.variant_name,
content_type=v.content_type,
size_bytes=v.size_bytes,
width=v.width,
height=v.height,
sha256=v.sha256
))
now_str = attachment.created_at.isoformat().replace('+00:00', 'Z')
return AttachmentSchema(
id=str(attachment.id),
chat_id=str(attachment.chat_id),
owner_id=str(attachment.owner_id),
filename=attachment.original_filename,
content_type=attachment.content_type,
size_bytes=attachment.size_bytes,
sha256=attachment.sha256,
status=attachment.status,
variants=variants_list,
error_code=attachment.error_code,
created_at=now_str,
updated_at=now_str
)
@router.get("/api/v1/attachments/{attachment_id}/content", responses={401: {"model": ErrorResponse}, 404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}})
async def download_attachment(
attachment_id: uuid.UUID,
variant: str = Query(default="original"),
current_user = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
s3: Minio = Depends(get_s3_client)
):
stmt = select(Attachment).where(Attachment.id == attachment_id)
result = await db.execute(stmt)
attachment = result.scalar_one_or_none()
if not attachment:
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Attachment not found"})
is_member = await db.execute(
select(ChatMember).where(ChatMember.chat_id == attachment.chat_id, ChatMember.user_id == current_user.id)
)
if not is_member.scalar_one_or_none():
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Access denied"})
if attachment.status != "ready":
raise HTTPException(status_code=409, detail={"code": "conflict", "message": f"Attachment is not ready yet (status: {attachment.status})"})
if variant == "original":
s3_key = attachment.s3_key
content_type = attachment.content_type
else:
from app.models import AttachmentVariant
var_stmt = select(AttachmentVariant).where(
AttachmentVariant.attachment_id == attachment_id,
AttachmentVariant.variant_name == variant
)
var_result = await db.execute(var_stmt)
var_obj = var_result.scalar_one_or_none()
if not var_obj:
raise HTTPException(status_code=404, detail={"code": "not_found", "message": f"Variant {variant} not found"})
s3_key = var_obj.s3_key
content_type = var_obj.content_type
try:
response = s3.get_object("messenger-attachments", s3_key)
data = response.read()
response.close()
response.release_conn()
except Exception:
raise HTTPException(status_code=503, detail={"code": "unavailable", "message": "Failed to download from S3"})
return Response(content=data, media_type=content_type)
+115
View File
@@ -0,0 +1,115 @@
import secrets
import logging
import bcrypt
import asyncpg
from datetime import datetime, timezone, timedelta
from fastapi import APIRouter, Depends, HTTPException, status, Response
from fastapi.security import HTTPBearer
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.exc import SQLAlchemyError
from app.database import get_db
from app.models import User as UserModel, UserSession
from app.schemas import CreateUser, User, Session, ErrorResponse
from app.dependencies import get_basic_auth, get_current_user
from app.redis_client import get_redis
logger = logging.getLogger(__name__)
router = APIRouter()
def verify_password(plain_password: str, hashed_password: str) -> bool:
return bcrypt.checkpw(plain_password.encode('utf-8'), hashed_password.encode('utf-8'))
def get_password_hash(password: str) -> str:
return bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')
@router.post("/api/v1/users", response_model=User, status_code=status.HTTP_201_CREATED, responses={409: {"model": ErrorResponse}})
async def register_user(user_in: CreateUser, db: AsyncSession = Depends(get_db)):
new_user = UserModel(
username=user_in.username,
display_name=user_in.display_name,
password_hash=get_password_hash(user_in.password)
)
db.add(new_user)
try:
await db.commit()
await db.refresh(new_user)
return User(
id=str(new_user.id),
username=new_user.username,
display_name=new_user.display_name,
created_at=new_user.created_at.isoformat().replace('+00:00', 'Z')
)
except Exception as e:
await db.rollback()
if isinstance(e.__cause__, asyncpg.exceptions.UniqueViolationError) or "unique constraint" in str(e).lower():
raise HTTPException(status_code=409, detail={"code": "conflict", "message": "Username already exists"})
logger.error(f"Database error during registration: {e}")
raise HTTPException(status_code=500, detail={"code": "internal_error", "message": "Database error"})
@router.post("/api/v1/auth/sessions", response_model=Session, status_code=status.HTTP_201_CREATED, responses={401: {"model": ErrorResponse}})
async def create_session(credentials = Depends(get_basic_auth), db: AsyncSession = Depends(get_db), redis_client = Depends(get_redis)):
result = await db.execute(select(UserModel).where(UserModel.username == credentials.username))
user = result.scalar_one_or_none()
if not user or not verify_password(credentials.password, user.password_hash):
raise HTTPException(status_code=401, detail={"code": "unauthorized", "message": "Invalid username or password"}, headers={"WWW-Authenticate": "Basic realm=\"messenger\", charset=\"UTF-8\""})
token = secrets.token_urlsafe(32)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=3600)
try:
await redis_client.setex(f"session:{token}", 3600, str(user.id))
except Exception as e:
logger.warning(f"Redis unavailable during session creation: {e}")
try:
db.add(UserSession(token=token, user_id=user.id, expires_at=expires_at))
await db.commit()
except SQLAlchemyError as e:
await db.rollback()
logger.error(f"Failed to save session to PostgreSQL: {e}")
raise HTTPException(status_code=500, detail={"code": "internal_error", "message": "Failed to create session"})
return Session(
token=token,
token_type="Bearer",
expires_at=expires_at,
user=User(
id=str(user.id),
username=user.username,
display_name=user.display_name,
created_at=user.created_at.isoformat().replace('+00:00', 'Z')
)
)
@router.delete("/api/v1/auth/sessions/current", status_code=status.HTTP_204_NO_CONTENT, responses={401: {"model": ErrorResponse}})
async def revoke_session(credentials = Depends(HTTPBearer()), db: AsyncSession = Depends(get_db), redis_client = Depends(get_redis)):
token = credentials.credentials
try:
await redis_client.delete(f"session:{token}")
except Exception as e:
logger.warning(f"Redis unavailable during session revocation: {e}")
try:
result = await db.execute(select(UserSession).where(UserSession.token == token))
session = result.scalar_one_or_none()
if session:
await db.delete(session)
await db.commit()
else:
raise HTTPException(status_code=401, detail={"code": "unauthorized", "message": "Invalid or expired session"}, headers={"WWW-Authenticate": "Bearer realm=\"messenger\""})
except HTTPException:
raise
except SQLAlchemyError as e:
await db.rollback()
raise HTTPException(status_code=500, detail={"code": "internal_error", "message": "Failed to revoke session"})
return Response(status_code=status.HTTP_204_NO_CONTENT)
@router.get("/api/v1/users/me", response_model=User, responses={401: {"model": ErrorResponse}})
async def current_user(current_user: UserModel = Depends(get_current_user)):
return User(
id=str(current_user.id),
username=current_user.username,
display_name=current_user.display_name,
created_at=current_user.created_at.isoformat().replace('+00:00', 'Z')
)
+213
View File
@@ -0,0 +1,213 @@
import uuid
import json
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, status, Query, Header
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.database import get_db
from app.models import Chat as ChatModel, ChatMember as ChatMemberModel, Message as MessageModel, User as UserModel, IdempotencyKey, MessageAttachment, Attachment
from app.schemas import Chat, CreateChat, Message, CreateMessage, ChatPage, MessagePage, ErrorResponse
from app.dependencies import get_current_user
from app.redis_client import get_redis
router = APIRouter()
@router.post("/api/v1/chats", response_model=Chat, status_code=status.HTTP_201_CREATED, responses={400: {"model": ErrorResponse}, 401: {"model": ErrorResponse}})
async def create_chat(chat_in: CreateChat, current_user: UserModel = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
try:
member_uuids = [uuid.UUID(uid) for uid in chat_in.member_ids]
except ValueError:
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Invalid member_id format"})
if current_user.id not in member_uuids:
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Creator must be in member_ids"})
user_ids = list(set(member_uuids))
result = await db.execute(select(UserModel).where(UserModel.id.in_(user_ids)))
if len(result.scalars().all()) != len(user_ids):
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "One or more member_ids do not exist"})
new_chat = ChatModel(title=chat_in.title)
db.add(new_chat)
await db.flush()
db.add_all([ChatMemberModel(chat_id=new_chat.id, user_id=uid) for uid in user_ids])
await db.commit()
await db.refresh(new_chat)
return Chat(
id=str(new_chat.id),
title=new_chat.title,
member_ids=[str(uid) for uid in user_ids],
created_at=new_chat.created_at.isoformat().replace('+00:00', 'Z')
)
@router.get("/api/v1/chats", response_model=ChatPage, responses={401: {"model": ErrorResponse}})
async def list_chats(limit: int = Query(50, ge=1, le=100), cursor: str | None = None, current_user: UserModel = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
stmt = select(ChatModel).options(selectinload(ChatModel.members)).join(ChatMemberModel, ChatModel.id == ChatMemberModel.chat_id).where(ChatMemberModel.user_id == current_user.id).order_by(ChatModel.created_at.desc(), ChatModel.id.desc())
if cursor:
try:
created_at_str, chat_id_str = cursor.split(",")
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
chat_id = uuid.UUID(chat_id_str)
stmt = stmt.where((ChatModel.created_at < created_at) | ((ChatModel.created_at == created_at) & (ChatModel.id < chat_id)))
except (ValueError, TypeError):
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Invalid cursor"})
stmt = stmt.limit(limit + 1)
chats = (await db.execute(stmt)).scalars().all()
has_next = len(chats) > limit
if has_next:
chats = chats[:-1]
next_cursor = f"{chats[-1].created_at.isoformat().replace('+00:00', 'Z')},{chats[-1].id}"
else:
next_cursor = None
items = [Chat(id=str(c.id), title=c.title, member_ids=[str(m.user_id) for m in c.members], created_at=c.created_at.isoformat().replace('+00:00', 'Z')) for c in chats]
return ChatPage(items=items, next_cursor=next_cursor)
@router.get("/api/v1/chats/{chat_id}", response_model=Chat, responses={401: {"model": ErrorResponse}, 404: {"model": ErrorResponse}})
async def get_chat(chat_id: uuid.UUID, current_user: UserModel = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
stmt = select(ChatModel).options(selectinload(ChatModel.members)).join(ChatMemberModel, ChatModel.id == ChatMemberModel.chat_id).where(ChatModel.id == chat_id, ChatMemberModel.user_id == current_user.id)
chat = (await db.execute(stmt)).scalar_one_or_none()
if not chat:
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Chat not found or access denied"})
return Chat(id=str(chat.id), title=chat.title, member_ids=[str(m.user_id) for m in chat.members], created_at=chat.created_at.isoformat().replace('+00:00', 'Z'))
@router.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED, responses={400: {"model": ErrorResponse}, 401: {"model": ErrorResponse}, 404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}})
async def create_message(
chat_id: uuid.UUID,
message_in: CreateMessage,
current_user: UserModel = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
redis_client = Depends(get_redis),
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key")
):
if not idempotency_key:
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Idempotency-Key header is required"})
if not (await db.execute(select(ChatMemberModel).where(ChatMemberModel.chat_id == chat_id, ChatMemberModel.user_id == current_user.id))).scalar_one_or_none():
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Chat not found or access denied"})
existing_key = await db.execute(select(IdempotencyKey).where(IdempotencyKey.key == idempotency_key))
existing_record = existing_key.scalar_one_or_none()
if existing_record:
# Исправлено: response_data хранится как JSON-строка, нужно распарсить
resp_data = json.loads(existing_record.response_data)
if resp_data.get("text") != message_in.text:
raise HTTPException(status_code=409, detail={"code": "conflict", "message": "Idempotency-Key already used with different payload"})
return Message(**resp_data)
new_message = MessageModel(chat_id=chat_id, sender_id=current_user.id, text=message_in.text)
db.add(new_message)
await db.flush()
# Привязываем вложения, если есть
attachment_ids = []
if message_in.attachment_ids:
for att_id_str in message_in.attachment_ids:
try:
att_id = uuid.UUID(att_id_str)
except ValueError:
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Invalid attachment_id format"})
# Проверяем, что вложение принадлежит этому чату
att_result = await db.execute(select(Attachment).where(Attachment.id == att_id, Attachment.chat_id == chat_id))
if not att_result.scalar_one_or_none():
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Attachment not found or does not belong to this chat"})
db.add(MessageAttachment(message_id=new_message.id, attachment_id=att_id))
attachment_ids.append(att_id_str)
await db.commit()
await db.refresh(new_message)
response_data_dict = {
"id": str(new_message.id),
"chat_id": str(new_message.chat_id),
"sender_id": str(new_message.sender_id),
"text": new_message.text,
"created_at": new_message.created_at.isoformat().replace('+00:00', 'Z'),
"attachment_ids": attachment_ids
}
# Исправлено: сохраняем как JSON-строку
db.add(IdempotencyKey(key=idempotency_key, chat_id=chat_id, response_data=json.dumps(response_data_dict)))
await db.commit()
try:
await redis_client.delete(f"messages:{chat_id}:*")
except Exception:
pass
return Message(**response_data_dict)
@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage, responses={400: {"model": ErrorResponse}, 401: {"model": ErrorResponse}, 404: {"model": ErrorResponse}})
async def list_messages(chat_id: uuid.UUID, limit: int = Query(50, ge=1, le=100), cursor: str | None = None, current_user: UserModel = Depends(get_current_user), db: AsyncSession = Depends(get_db), redis_client = Depends(get_redis)):
if not (await db.execute(select(ChatMemberModel).where(ChatMemberModel.chat_id == chat_id, ChatMemberModel.user_id == current_user.id))).scalar_one_or_none():
raise HTTPException(status_code=404, detail={"code": "not_found", "message": "Chat not found or access denied"})
cache_key = f"messages:{chat_id}:{limit}:{cursor or 'null'}"
redis_available = True
try:
cached_data = await redis_client.get(cache_key)
if cached_data:
await redis_client.expire(cache_key, 60)
return MessagePage.model_validate_json(cached_data)
except Exception:
redis_available = False
lock_acquired = False
if redis_available:
lock_key = f"lock:{cache_key}"
try:
lock_acquired = await redis_client.set(lock_key, "1", nx=True, ex=5)
if not lock_acquired:
stale_data = await redis_client.get(f"stale:{cache_key}")
if stale_data: return MessagePage.model_validate_json(stale_data)
except Exception:
redis_available = False
stmt = select(MessageModel).where(MessageModel.chat_id == chat_id).order_by(MessageModel.created_at.asc(), MessageModel.id.asc())
if cursor:
try:
created_at_str, msg_id_str = cursor.split(",")
created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
msg_id = uuid.UUID(msg_id_str)
stmt = stmt.where((MessageModel.created_at > created_at) | ((MessageModel.created_at == created_at) & (MessageModel.id > msg_id)))
except (ValueError, TypeError):
raise HTTPException(status_code=400, detail={"code": "invalid_request", "message": "Invalid cursor"})
stmt = stmt.limit(limit + 1)
messages = (await db.execute(stmt)).scalars().all()
has_next = len(messages) > limit
if has_next:
messages = messages[:-1]
next_cursor = f"{messages[-1].created_at.isoformat().replace('+00:00', 'Z')},{messages[-1].id}"
else:
next_cursor = None
items = []
for m in messages:
att_result = await db.execute(select(MessageAttachment.attachment_id).where(MessageAttachment.message_id == m.id))
attachment_ids = [str(aid) for aid in att_result.scalars().all()]
items.append(Message(
id=str(m.id),
chat_id=str(m.chat_id),
sender_id=str(m.sender_id),
text=m.text,
created_at=m.created_at.isoformat().replace('+00:00', 'Z'),
attachment_ids=attachment_ids
))
page = MessagePage(items=items, next_cursor=next_cursor)
if redis_available:
try:
old_cache = await redis_client.get(cache_key)
if old_cache: await redis_client.setex(f"stale:{cache_key}", 120, old_cache)
await redis_client.setex(cache_key, 60, page.model_dump_json())
if lock_acquired: await redis_client.delete(f"lock:{cache_key}")
except Exception:
pass
return page
+40
View File
@@ -0,0 +1,40 @@
import os
import asyncio
from minio import Minio
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
S3_SECURE = os.getenv("S3_ENDPOINT", "http://s3:8333").startswith("https://")
s3_client = Minio(
S3_ENDPOINT,
access_key=S3_ACCESS_KEY,
secret_key=S3_SECRET_KEY,
secure=S3_SECURE
)
def get_s3_client():
return s3_client
async def upload_file_to_s3(object_name: str, file_path: str, content_type: str):
loop = asyncio.get_running_loop()
await loop.run_in_executor(
None,
lambda: s3_client.fput_object(S3_BUCKET_NAME, object_name, file_path, content_type=content_type)
)
async def get_file_from_s3(object_name: str):
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
None,
lambda: s3_client.get_object(S3_BUCKET_NAME, object_name)
)
async def delete_file_from_s3(object_name: str):
loop = asyncio.get_running_loop()
await loop.run_in_executor(
None,
lambda: s3_client.remove_object(S3_BUCKET_NAME, object_name)
)
+88
View File
@@ -0,0 +1,88 @@
from pydantic import BaseModel, Field, ConfigDict, model_validator
from typing import List, Optional
from datetime import datetime
class User(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
username: str
display_name: str
created_at: str
class CreateUser(BaseModel):
username: str = Field(..., min_length=3, max_length=32, pattern="^[a-z0-9_]+$")
display_name: str = Field(..., min_length=1, max_length=100)
password: str = Field(..., min_length=12, max_length=128)
class Session(BaseModel):
model_config = ConfigDict(from_attributes=True)
token: str
token_type: str
expires_at: datetime
user: User
class Chat(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
title: str
member_ids: List[str]
created_at: str
class CreateChat(BaseModel):
title: str = Field(..., min_length=1, max_length=100)
member_ids: List[str]
class ChatPage(BaseModel):
items: List[Chat]
next_cursor: Optional[str] = None
class Message(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
chat_id: str
sender_id: str
text: str
created_at: str
attachment_ids: List[str]
class CreateMessage(BaseModel):
text: str = Field(default="", max_length=4000)
attachment_ids: List[str] = Field(default_factory=list)
@model_validator(mode='after')
def check_text_or_attachments(self):
if len(self.text) == 0 and len(self.attachment_ids) == 0:
raise ValueError("Either text or attachment_ids must be provided")
if len(self.attachment_ids) > 10:
raise ValueError("attachment_ids cannot exceed 10 items")
return self
class MessagePage(BaseModel):
items: List[Message]
next_cursor: Optional[str] = None
class Variant(BaseModel):
name: str
content_type: str
size_bytes: int
width: int
height: int
sha256: str
class Attachment(BaseModel):
model_config = ConfigDict(from_attributes=True)
id: str
chat_id: str
owner_id: str
filename: str
content_type: str
size_bytes: int
sha256: str
status: str
variants: List[Variant]
error_code: Optional[str] = None
created_at: str
updated_at: str
class ErrorResponse(BaseModel):
error: dict
+218
View File
@@ -0,0 +1,218 @@
import os
import sys
import uuid
import hashlib
import asyncio
import logging
import tempfile
import json
from datetime import datetime, timezone, timedelta
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
import aio_pika
from minio import Minio
from PIL import Image
import io
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db")
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
WORKER_ID = str(uuid.uuid4())[:8]
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
engine = create_async_engine(DATABASE_URL, echo=False)
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
async def process_image(body: bytes, att_id: str, chat_id: str):
img = Image.open(io.BytesIO(body)).convert("RGB")
width, height = img.size
variants_to_save = []
# 1. Thumbnail (вписан в 256x256)
thumb = img.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumb_bytes = io.BytesIO()
thumb.save(thumb_bytes, format="JPEG", quality=85)
thumb_data = thumb_bytes.getvalue()
thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
tmp.write(thumb_data)
s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg",
"size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height,
"sha256": hashlib.sha256(thumb_data).hexdigest()
})
# 2. Crop (центральный квадрат 128x128)
min_dim = min(width, height)
left = (width - min_dim) // 2
top = (height - min_dim) // 2
crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS)
crop_bytes = io.BytesIO()
crop.save(crop_bytes, format="JPEG", quality=85)
crop_data = crop_bytes.getvalue()
crop_key = f"variants/{chat_id}/{att_id}/crop.jpg"
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
tmp.write(crop_data)
s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg",
"size_bytes": len(crop_data), "width": 128, "height": 128,
"sha256": hashlib.sha256(crop_data).hexdigest()
})
# 3. Optimized WebP (вписан в 1280x1280)
opt = img.copy()
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
opt_bytes = io.BytesIO()
opt.save(opt_bytes, format="WEBP", quality=80)
opt_data = opt_bytes.getvalue()
opt_key = f"variants/{chat_id}/{att_id}/optimized.webp"
with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp:
tmp.write(opt_data)
s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp",
"size_bytes": len(opt_data), "width": opt.width, "height": opt.height,
"sha256": hashlib.sha256(opt_data).hexdigest()
})
# 4. Сохраняем оригинал в конечную папку
ext = "png" if b"png" in body[:8] else "jpeg"
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
content_type = "image/png" if ext == "png" else "image/jpeg"
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp:
tmp.write(body)
s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type)
os.unlink(tmp.name)
return final_key, variants_to_save
async def process_message(message: aio_pika.IncomingMessage):
async with message.process(requeue=False):
try:
payload = message.body.decode()
data = json.loads(payload)
job_id = data["job_id"]
att_id = data["attachment_id"]
attempt = data["attempt"]
logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}")
async with AsyncSessionLocal() as db:
from app.models import AttachmentJob, Attachment, AttachmentVariant
# 1. Блокируем задачу для обработки (Claim)
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
result = await db.execute(stmt)
job = result.scalar_one_or_none()
if not job:
logger.warning(f"Job {job_id} not found")
return
if job.status in ("ready", "failed"):
logger.info(f"Job {job_id} already {job.status}, skipping")
return
if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc):
logger.info(f"Job {job_id} is already being processed by another worker")
return
now = datetime.now(timezone.utc)
job.status = "processing"
job.attempt = attempt
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
job.lease_owner = WORKER_ID
await db.commit()
# 2. Получаем метаданные вложения
att_stmt = select(Attachment).where(Attachment.id == att_id)
att_result = await db.execute(att_stmt)
attachment = att_result.scalar_one_or_none()
if not attachment or not attachment.staging_key:
job.status = "failed"
job.error_code = "missing_staging"
await db.commit()
return
staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления
# 3. Скачиваем из staging
response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete)
body = response.read()
response.close()
response.release_conn()
# 4. Обрабатываем изображение
final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id))
# 5. Фиксируем результат в БД
attachment.status = "ready"
attachment.s3_key = final_key
attachment.staging_key = None # Очищаем staging
job.status = "ready"
for v_data in variants_data:
variant = AttachmentVariant(
attachment_id=attachment.id,
**v_data
)
db.add(variant)
await db.commit()
logger.info(f"Job {job_id} completed successfully")
# 6. Удаляем staging из S3
s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete)
except Exception as e:
logger.error(f"Error processing message: {e}")
try:
async with AsyncSessionLocal() as db:
from app.models import AttachmentJob
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
result = await db.execute(stmt)
job = result.scalar_one_or_none()
if job:
if job.attempt >= JOB_MAX_ATTEMPTS:
job.status = "failed"
job.error_code = "retry_exhausted"
else:
job.status = "queued"
job.attempt += 1
job.lease_until = None
await db.commit()
except Exception as db_err:
logger.error(f"Failed to update job status after error: {db_err}")
async def main():
logger.info("Starting worker...")
connection = await aio_pika.connect_robust(BROKER_URL)
channel = await connection.channel()
queue = await channel.declare_queue("attachment.process.v1", durable=True)
logger.info("Waiting for messages...")
await queue.consume(process_message)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())
+187
View File
@@ -0,0 +1,187 @@
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 5s
retries: 5
redis:
image: valkey/valkey:8.0-alpine
command: valkey-server --appendonly yes --save 60 1
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 5s
retries: 5
s3:
image: chrislusf/seaweedfs:3.80
command:
- server
- -dir=/data
- -s3
- -s3.port=8333
- -master.volumeSizeLimitMB=64
- -volume.max=100
volumes:
- s3data:/data
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 5s
timeout: 5s
retries: 10
start_period: 60s
deploy:
resources:
limits:
cpus: '1.0'
memory: 1G
restart: unless-stopped
s3-init:
image: amazon/aws-cli:2.15.0
depends_on:
s3:
condition: service_healthy
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=admin
- AWS_REGION=${S3_REGION}
entrypoint: >
/bin/sh -c "
aws --endpoint-url http://s3:8333 s3 mb s3://${S3_BUCKET_NAME} 2>/dev/null || echo 'Bucket already exists'
aws --endpoint-url http://s3:8333 s3api put-bucket-acl --bucket ${S3_BUCKET_NAME} --acl private
echo 'Bucket ${S3_BUCKET_NAME} is ready and private'
"
restart: "no"
# === НОВОЕ: RabbitMQ Broker ===
rabbitmq:
image: rabbitmq:3.13-management-alpine
hostname: rabbitmq
environment:
RABBITMQ_DEFAULT_USER: messenger
RABBITMQ_DEFAULT_PASS: mq_secret_password
RABBITMQ_DEFAULT_VHOST: /
volumes:
- mqdata:/var/lib/rabbitmq
networks: [default]
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
interval: 5s
timeout: 10s
retries: 30
deploy:
resources:
limits:
cpus: '1.0'
memory: 512M
restart: unless-stopped
api:
build:
context: .
dockerfile: Dockerfile
environment:
- DATABASE_URL=${DATABASE_URL}
- REDIS_URL=${REDIS_URL}
- SECRET_SESSION_KEY=${SECRET_SESSION_KEY}
- SESSION_TTL_SECONDS=${SESSION_TTL_SECONDS}
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_API_ACCESS_KEY:-admin}
- S3_SECRET_KEY=${S3_API_SECRET_KEY:-admin}
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
- S3_REGION=${S3_REGION}
- BROKER_URL=${BROKER_URL}
- MAX_ACTIVE_JOBS=${MAX_ACTIVE_JOBS}
expose:
- "8080"
volumes:
- ./alembic/versions:/app/alembic/versions
- ./app:/app/app
read_only: true
tmpfs:
- /tmp:size=15M
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
s3-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
stop_grace_period: 30s
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080/health/live"]
interval: 10s
timeout: 5s
retries: 3
start_period: 10s
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
# === НОВОЕ: Worker для асинхронной обработки ===
worker:
build:
context: .
dockerfile: Dockerfile
command: ["python", "-m", "app.worker"]
environment:
- DATABASE_URL=${DATABASE_URL}
- BROKER_URL=${BROKER_URL}
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_API_ACCESS_KEY:-admin}
- S3_SECRET_KEY=${S3_API_SECRET_KEY:-admin}
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
- WORKER_CONCURRENCY=${WORKER_CONCURRENCY}
- JOB_MAX_ATTEMPTS=${JOB_MAX_ATTEMPTS}
- JOB_LEASE_SECONDS=${JOB_LEASE_SECONDS}
volumes:
- ./app:/app/app
read_only: true
tmpfs:
- /tmp:size=100M # Worker'у нужно больше места в RAM для декодирования изображений
depends_on:
postgres:
condition: service_healthy
rabbitmq:
condition: service_healthy
s3-init:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: '2.0'
memory: 768M
restart: unless-stopped
nginx:
image: nginx:1.25-alpine
ports:
- "8080:8080"
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
api:
condition: service_healthy
restart: unless-stopped
volumes:
postgres_data:
redis_data:
s3data:
mqdata: # Новый volume для RabbitMQ
+22
View File
@@ -0,0 +1,22 @@
upstream api_backend {
# Nginx будет автоматически резолвить имена через Docker DNS
server api:8080 max_fails=1 fail_timeout=5s;
}
server {
listen 8080;
# Проксирование ВСЕХ запросов на API реплики
location / {
proxy_pass http://api_backend;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Request-Id $http_x_request_id;
# Таймауты для graceful shutdown
proxy_connect_timeout 5s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
}
}
+16
View File
@@ -0,0 +1,16 @@
fastapi==0.115.0
uvicorn[standard]==0.30.6
sqlalchemy[asyncio]==2.0.35
asyncpg==0.29.0
pydantic-settings==2.5.2
redis==5.1.0
bcrypt==4.2.0
alembic==1.13.2
python-multipart==0.0.9
# === S3 и Валидация (из Lab 4) ===
minio==7.2.0
Pillow==10.4.0
python-magic==0.4.27
# === Новое для Lab 5 (Брокер и утилиты) ===
aio-pika==9.4.3
uuid-utils==0.10.0