This commit is contained in:
2026-10-30 20:08:17 +03:00
parent 4c7a796795
commit 959eee7f9d
25 changed files with 562 additions and 46 deletions
+82
View File
@@ -0,0 +1,82 @@
import os
import unicodedata
from uuid import uuid4
from PIL import Image, UnidentifiedImageError
from sqlalchemy import func, select
from .errors import APIError
from .models import Attachment, ChatMember
from .object_storage import BUCKET, PREFIX, s3_client
from .store import ensure_uuid, serialize, store
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
if MAX_BYTES < 1 or MAX_PIXELS < 1:
raise ValueError('Upload limits must be positive')
def validate_filename(filename):
if (not filename or not filename.strip() or len(filename) > 255
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
raise APIError(400, 'invalid_request', 'Неверное имя файла')
def validate_image(source, content_type):
try:
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
if picture.width * picture.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
if Image.MIME[picture.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
picture.verify()
source.seek(0)
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
except Image.DecompressionBombError as exc:
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
source.seek(0)
def lock_object(session, key):
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
def metadata(attachment):
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
'content_type', 'size_bytes', 'sha256', 'created_at')
return {**result, 'updated_at': result['created_at'], 'status': 'ready',
'variants': [], 'error_code': None}
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
validate_image(source, content_type)
attachment_id = str(uuid4())
key = PREFIX + attachment_id
with store.sessions.begin() as session:
store._chat(session, chat_id, actor_id)
lock_object(session, key)
s3_client().put_object(Bucket=BUCKET, Key=key, Body=source,
ContentLength=size, ContentType=content_type)
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
object_key=key, filename=filename, content_type=content_type,
size_bytes=size, sha256=digest)
session.add(attachment)
session.flush()
result = metadata(attachment)
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
return result
def get_attachment(attachment_id, actor_id):
with store.sessions() as session:
attachment = session.scalar(select(Attachment).join(
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
if attachment is None:
raise APIError(404, 'not_found', 'Объект не найден')
return attachment
+1 -1
View File
@@ -37,4 +37,4 @@ class HistoryCache:
return page
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')))
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
+61
View File
@@ -0,0 +1,61 @@
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
import argparse
import logging
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from .attachments import lock_object
from .database import create_db_engine
from .models import Attachment
from .object_storage import BUCKET, PREFIX, s3_client
GRACE = timedelta(hours=1)
def cleanup(sessions, client, now=None):
cutoff = (now or datetime.now(timezone.utc)) - GRACE
deleted = 0
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
for obj in page.get('Contents', []):
if obj['LastModified'] >= cutoff:
continue
key = obj['Key']
with sessions.begin() as session:
lock_object(session, key)
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
continue
# Recheck age after taking the same lock as the uploader.
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
return deleted
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
try:
while True:
try:
count = cleanup(sessionmaker(engine), s3_client())
logging.info('Cleanup completed: deleted=%d', count)
except Exception as exc:
logging.error('Cleanup deferred: %s', type(exc).__name__)
if args.once:
raise SystemExit(1) from None
if args.once:
break
time.sleep(3600)
finally:
engine.dispose()
if __name__ == '__main__':
main()
+4
View File
@@ -5,6 +5,7 @@ from dataclasses import dataclass
from starlette.requests import Request
from .errors import error_response
from .middleware import is_upload
@dataclass(frozen=True)
@@ -54,6 +55,9 @@ class ResourceLimitMiddleware:
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
self.active += 1
try:
# The upload endpoint authenticates and bounds its stream itself.
if is_upload(scope['method'], scope['path']):
return await self.app(scope, receive, send)
length = request.headers.get('content-length')
if length is not None:
try:
+4 -1
View File
@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
from sqlalchemy.orm import sessionmaker
from redis.exceptions import RedisError
from botocore.exceptions import BotoCoreError, ClientError
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
from .database import create_db_engine
@@ -16,6 +17,7 @@ from .routers.health import router as health_router
from .routers.users import router as users_router
from .routers.chats import router as chats_router
from .routers.auth import router as auth_router
from .routers.attachments import router as attachments_router
@asynccontextmanager
@@ -36,10 +38,11 @@ app.add_exception_handler(APIError, api_error_handler)
app.add_exception_handler(RequestValidationError, validation_error_handler)
app.add_exception_handler(HTTPException, http_error_handler)
app.add_exception_handler(Exception, internal_error_handler)
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError):
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
app.add_exception_handler(error, dependency_error_handler)
app.include_router(health_router)
app.include_router(users_router)
app.include_router(chats_router)
app.include_router(auth_router)
app.include_router(attachments_router)
+6
View File
@@ -3,6 +3,11 @@ from socket import gethostname
from starlette.responses import JSONResponse
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
import re
def is_upload(method, path):
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
INSTANCE_ID = gethostname()
@@ -14,6 +19,7 @@ class RequestIdMiddleware(BaseHTTPMiddleware):
if (request.method in {"POST", "PUT", "PATCH"}
and request.url.path.startswith("/api/")
and not is_upload(request.method, request.url.path)
and request.url.path != '/api/v1/auth/sessions'):
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type != "application/json":
+17 -1
View File
@@ -3,6 +3,7 @@ from uuid import uuid4
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.dialects.postgresql import ARRAY
class Base(DeclarativeBase):
@@ -44,13 +45,15 @@ class ChatMember(Base):
class Message(Base):
__tablename__ = 'messages'
__table_args__ = (
CheckConstraint('char_length(text) BETWEEN 1 AND 4000'),
CheckConstraint('char_length(text) <= 4000'),
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
)
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
text: Mapped[str] = mapped_column(Text)
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
@@ -60,6 +63,19 @@ class SchemaMigration(Base):
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
class Attachment(Base):
__tablename__ = 'attachments'
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
object_key: Mapped[str] = mapped_column(Text, unique=True)
filename: Mapped[str] = mapped_column(Text)
content_type: Mapped[str] = mapped_column(Text)
size_bytes: Mapped[int] = mapped_column(BigInteger)
sha256: Mapped[str] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
class IdempotencyRecord(Base):
__tablename__ = 'message_idempotency'
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
+23
View File
@@ -0,0 +1,23 @@
import os
from functools import lru_cache
import boto3
from botocore.config import Config
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
PREFIX = 'originals/'
@lru_cache(maxsize=1)
def s3_client():
return boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
config=Config(connect_timeout=2, read_timeout=5,
retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'},
request_checksum_calculation='when_required',
response_checksum_validation='when_required'),
)
+84
View File
@@ -0,0 +1,84 @@
import asyncio
import hashlib
from tempfile import TemporaryFile
from urllib.parse import quote
from fastapi import APIRouter, Depends, Query, Request
from starlette.background import BackgroundTask
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
from ..dependencies import current_user
from ..errors import APIError
from ..object_storage import BUCKET, s3_client
from ..store import ensure_uuid, store
router = APIRouter(prefix='/api/v1', tags=['attachments'])
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
decode_slot = asyncio.Semaphore(1)
@router.post('/chats/{chat_id}/attachments', status_code=201)
async def upload(chat_id: str, request: Request, filename: str = Query(...),
user: dict = Depends(current_user)):
chat_id = ensure_uuid(chat_id)
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
validate_filename(filename)
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
if content_type not in ('image/png', 'image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
try:
length = int(length)
if length < 0:
raise ValueError()
except ValueError as exc:
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
if length > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
size = 0
digest = hashlib.sha256()
with TemporaryFile() as source:
try:
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
digest.update(chunk)
await run_in_threadpool(source.write, chunk)
except (TimeoutError, ClientDisconnect) as exc:
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
if length is not None and size != length:
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
async with decode_slot:
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
filename, content_type, size, digest.hexdigest())
@router.get('/attachments/{attachment_id}')
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
return metadata(get_attachment(attachment_id, user['id']))
@router.get('/attachments/{attachment_id}/content')
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
attachment = get_attachment(attachment_id, user['id'])
if variant != 'original':
raise APIError(404, 'not_found', 'Вариант не найден')
body = s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body']
def chunks():
try:
yield from body.iter_chunks(chunk_size=64 * 1024)
finally:
body.close()
return StreamingResponse(chunks(), media_type=attachment.content_type,
headers={'Content-Length': str(attachment.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
'X-Content-Type-Options': 'nosniff'},
background=BackgroundTask(body.close))
+1 -1
View File
@@ -36,7 +36,7 @@ def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(cu
idempotency_key: Optional[str] = Header(default=None)):
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
return store.create_message(chat_id, user["id"], payload.text, idempotency_key)
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
@router.get("/{chat_id}/messages", response_model=MessagePage)
+2
View File
@@ -4,6 +4,7 @@ from sqlalchemy import select
from ..schemas.common import Health
from ..auth import sessions
from ..store import store
from ..object_storage import BUCKET, s3_client
router = APIRouter(prefix='/health')
@@ -18,4 +19,5 @@ def health_ready():
with store.sessions() as session:
session.execute(select(1))
sessions.client.ping()
s3_client().head_bucket(Bucket=BUCKET)
return {"status": "ok"}
+17 -7
View File
@@ -1,19 +1,28 @@
from typing import List, Optional
from pydantic import Field, StrictStr, field_validator
from pydantic import Field, StrictStr, field_validator, model_validator
from uuid import UUID
from .common import ContractModel
class CreateMessage(ContractModel):
text: StrictStr = Field(min_length=1, max_length=4000)
text: StrictStr = Field(max_length=4000)
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
@field_validator("text")
def validate_text(cls, value):
if not value.strip():
raise ValueError("blank text")
return value
@field_validator('attachment_ids')
def validate_ids(cls, value):
ids = [str(UUID(item)) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('duplicate attachments')
return ids
@model_validator(mode='after')
def validate_content(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('empty message')
return self
class Message(ContractModel):
@@ -21,6 +30,7 @@ class Message(ContractModel):
chat_id: str
sender_id: str
text: str
attachment_ids: List[str]
created_at: str
+11 -5
View File
@@ -10,7 +10,7 @@ from sqlalchemy.exc import IntegrityError
from .auth import password_hasher, verify_password, unauthorized
from .errors import APIError
from .cache import history_cache
from .models import User, Chat, ChatMember, Message, IdempotencyRecord
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
def ensure_uuid(value):
@@ -36,7 +36,7 @@ def serialize(model, *fields):
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
CHAT_FIELDS = ('id', 'title', 'created_at')
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at')
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
def after_condition(model, after):
@@ -130,10 +130,16 @@ class PostgresStore:
items = [self._members(session, chat) for chat in chats[:limit]]
return self._page(items, len(chats) > limit, 'chats', actor_id)
def create_message(self, chat_id, actor_id, text, idempotency_key=None):
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
chat_id = ensure_uuid(chat_id)
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
with self.sessions.begin() as session:
self._chat(session, chat_id, actor_id)
if attachment_ids:
found = list(session.scalars(select(Attachment.id).where(
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)))
if len(found) != len(attachment_ids):
raise APIError(404, 'not_found', 'Объект не найден')
if idempotency_key is not None:
# Serialize writers to this chat before looking up the key.
# The same row is updated below for history cache invalidation.
@@ -141,12 +147,12 @@ class PostgresStore:
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
if record is not None:
previous = session.get(Message, record.message_id)
if previous.text != text:
if previous.text != text or previous.attachment_ids != attachment_ids:
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
return serialize(previous, *MESSAGE_FIELDS)
session.execute(update(Chat).where(Chat.id == chat_id).values(
history_version=Chat.history_version + 1))
message = Message(chat_id=chat_id, sender_id=actor_id, text=text)
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
session.add(message)
session.flush()
if idempotency_key is not None: