forked from mtusi2026/mtusi2026lab4
feat: ЛР4 - S3 хранилище MinIO, загрузка вложений и защита от атак
This commit is contained in:
@@ -34,3 +34,11 @@ CACHE_URL: str = _get_val(
|
||||
"CACHE_URL_FILE",
|
||||
"redis://redis:6379/0",
|
||||
)
|
||||
|
||||
S3_ENDPOINT: str = os.getenv("S3_ENDPOINT", "http://minio:9000")
|
||||
S3_REGION: str = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET: str = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID: str = _get_val("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", "minioadmin")
|
||||
S3_SECRET_ACCESS_KEY: str = _get_val("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", "minioadmin")
|
||||
MAX_UPLOAD_BYTES: int = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS: int = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
import botocore.exceptions
|
||||
from fastapi import FastAPI, Request, status
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
@@ -33,6 +34,14 @@ class ConflictError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class PayloadTooLargeError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedMediaTypeError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class UnauthorizedError(StorageError):
|
||||
pass
|
||||
|
||||
@@ -113,6 +122,30 @@ def register_error_handlers(app: FastAPI) -> None:
|
||||
status.HTTP_409_CONFLICT, ErrorCode.CONFLICT, str(exc), req_id
|
||||
)
|
||||
|
||||
@app.exception_handler(PayloadTooLargeError)
|
||||
async def payload_too_large_handler(
|
||||
request: Request, exc: PayloadTooLargeError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
ErrorCode.PAYLOAD_TOO_LARGE,
|
||||
str(exc),
|
||||
req_id,
|
||||
)
|
||||
|
||||
@app.exception_handler(UnsupportedMediaTypeError)
|
||||
async def unsupported_media_type_handler(
|
||||
request: Request, exc: UnsupportedMediaTypeError
|
||||
) -> JSONResponse:
|
||||
req_id = get_request_id(request)
|
||||
return make_error_response(
|
||||
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
ErrorCode.UNSUPPORTED_MEDIA_TYPE,
|
||||
str(exc),
|
||||
req_id,
|
||||
)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(
|
||||
request: Request, exc: RequestValidationError
|
||||
@@ -185,6 +218,8 @@ def register_error_handlers(app: FastAPI) -> None:
|
||||
asyncpg.PostgresError,
|
||||
asyncpg.InterfaceError,
|
||||
redis.exceptions.RedisError,
|
||||
botocore.exceptions.BotoCoreError,
|
||||
botocore.exceptions.ClientError,
|
||||
TimeoutError,
|
||||
asyncio.TimeoutError,
|
||||
ConnectionError,
|
||||
@@ -204,6 +239,8 @@ def register_error_handlers(app: FastAPI) -> None:
|
||||
asyncpg.PostgresError,
|
||||
asyncpg.InterfaceError,
|
||||
redis.exceptions.RedisError,
|
||||
botocore.exceptions.BotoCoreError,
|
||||
botocore.exceptions.ClientError,
|
||||
TimeoutError,
|
||||
asyncio.TimeoutError,
|
||||
ConnectionError,
|
||||
|
||||
+3
-1
@@ -9,6 +9,7 @@ from fastapi import FastAPI
|
||||
from app.config import APP_HOST, APP_PORT, LOG_LEVEL
|
||||
from app.errors import register_error_handlers
|
||||
from app.middleware import RequestIdAndLoggingMiddleware
|
||||
from app.routes.attachments import router as attachments_router
|
||||
from app.routes.auth import router as auth_router
|
||||
from app.routes.chats import router as chats_router
|
||||
from app.routes.health import router as health_router
|
||||
@@ -32,7 +33,7 @@ async def lifespan(app: FastAPI):
|
||||
|
||||
app = FastAPI(
|
||||
title="highload-messenger",
|
||||
version="1.3.0",
|
||||
version="1.4.0",
|
||||
docs_url="/docs",
|
||||
redoc_url=None,
|
||||
lifespan=lifespan,
|
||||
@@ -47,6 +48,7 @@ app.include_router(auth_router)
|
||||
app.include_router(users_router)
|
||||
app.include_router(chats_router)
|
||||
app.include_router(messages_router)
|
||||
app.include_router(attachments_router)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+20
-4
@@ -32,19 +32,35 @@ class RequestIdAndLoggingMiddleware(BaseHTTPMiddleware):
|
||||
start_time = time.monotonic()
|
||||
|
||||
if request.method in ("POST", "PUT", "PATCH") and request.url.path != "/api/v1/auth/sessions":
|
||||
is_attachment_upload = (
|
||||
request.method == "POST"
|
||||
and "/attachments" in request.url.path
|
||||
and request.url.path.startswith("/api/v1/chats/")
|
||||
)
|
||||
ct = request.headers.get("content-type")
|
||||
if ct:
|
||||
ct_clean = ct.split(";")[0].strip().lower()
|
||||
if ct_clean != "application/json":
|
||||
ct_clean = ct.split(";")[0].strip().lower() if ct else ""
|
||||
|
||||
if is_attachment_upload:
|
||||
if not ct or ct_clean not in ("image/png", "image/jpeg"):
|
||||
from fastapi import status
|
||||
from app.errors import make_error_response
|
||||
from app.schemas import ErrorCode
|
||||
return make_error_response(
|
||||
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
ErrorCode.UNSUPPORTED_MEDIA_TYPE,
|
||||
f"Неподдерживаемый тип содержимого '{ct}'. Ожидается application/json",
|
||||
f"Неподдерживаемый тип содержимого '{ct}'. Ожидается image/png или image/jpeg",
|
||||
request_id,
|
||||
)
|
||||
elif ct and ct_clean != "application/json":
|
||||
from fastapi import status
|
||||
from app.errors import make_error_response
|
||||
from app.schemas import ErrorCode
|
||||
return make_error_response(
|
||||
status.HTTP_415_UNSUPPORTED_MEDIA_TYPE,
|
||||
ErrorCode.UNSUPPORTED_MEDIA_TYPE,
|
||||
f"Неподдерживаемый тип содержимого '{ct}'. Ожидается application/json",
|
||||
request_id,
|
||||
)
|
||||
|
||||
try:
|
||||
response = await call_next(request)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request, Response, status
|
||||
|
||||
from app.dependencies import get_current_user
|
||||
from app.errors import InvalidRequestError
|
||||
from app.schemas import AttachmentResponse, UserResponse
|
||||
from app.storage import storage
|
||||
|
||||
router = APIRouter(tags=["attachments"])
|
||||
|
||||
|
||||
def validate_uuid(value: str, field_name: str = "id") -> str:
|
||||
try:
|
||||
UUID(value)
|
||||
return value
|
||||
except ValueError as exc:
|
||||
raise InvalidRequestError(f"Поле '{field_name}' не является валидным UUID") from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
response_model=AttachmentResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def upload_attachment(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> AttachmentResponse:
|
||||
"""Загрузка бинарного вложения в чат."""
|
||||
valid_chat_id = validate_uuid(chat_id, "chat_id")
|
||||
content_type = request.headers.get("content-type") or ""
|
||||
data = await request.body()
|
||||
|
||||
return await storage.upload_attachment(
|
||||
caller_id=current_user.id,
|
||||
chat_id=valid_chat_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
response_model=AttachmentResponse,
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
async def get_attachment_metadata(
|
||||
attachment_id: str,
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> AttachmentResponse:
|
||||
"""Получение метаданных вложения."""
|
||||
valid_att_id = validate_uuid(attachment_id, "attachment_id")
|
||||
return await storage.get_attachment(
|
||||
caller_id=current_user.id,
|
||||
attachment_id=valid_att_id,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/attachments/{attachment_id}/content",
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
async def download_attachment_content(
|
||||
attachment_id: str,
|
||||
variant: str | None = Query(default=None),
|
||||
current_user: UserResponse = Depends(get_current_user),
|
||||
) -> Response:
|
||||
"""Скачивание бинарного содержимого вложения."""
|
||||
valid_att_id = validate_uuid(attachment_id, "attachment_id")
|
||||
data, content_type = await storage.get_attachment_content(
|
||||
caller_id=current_user.id,
|
||||
attachment_id=valid_att_id,
|
||||
variant=variant,
|
||||
)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=content_type,
|
||||
headers={
|
||||
"Content-Length": str(len(data)),
|
||||
},
|
||||
)
|
||||
@@ -47,6 +47,7 @@ async def create_message(
|
||||
chat_id=valid_id,
|
||||
text=body.text,
|
||||
idempotency_key=idempotency_key,
|
||||
attachment_ids=body.attachment_ids,
|
||||
)
|
||||
return msg
|
||||
|
||||
|
||||
+49
-6
@@ -5,7 +5,7 @@ import re
|
||||
from enum import Enum
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
def now_utc_iso() -> str:
|
||||
@@ -134,18 +134,60 @@ class ChatPageResponse(BaseModel):
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class VariantResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class AttachmentResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str = "ready"
|
||||
variants: list[VariantResponse] = Field(default_factory=list)
|
||||
error_code: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class CreateMessageRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("text")
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def validate_text(cls, v: str) -> str:
|
||||
if not v.strip():
|
||||
raise ValueError("Текст сообщения не может состоять только из пробелов")
|
||||
def validate_attachment_ids(cls, v: list[str]) -> list[str]:
|
||||
if len(v) != len(set(v)):
|
||||
raise ValueError("attachment_ids must contain unique items")
|
||||
for item in v:
|
||||
try:
|
||||
UUID(item)
|
||||
except ValueError:
|
||||
raise ValueError(f"Идентификатор вложения '{item}' не является валидным UUID")
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_text_or_attachment(self) -> CreateMessageRequest:
|
||||
has_text = bool(self.text and self.text.strip())
|
||||
has_attachments = bool(self.attachment_ids)
|
||||
if not has_text and not has_attachments:
|
||||
raise ValueError("Сообщение должно содержать непустой текст или хотя бы одно вложение")
|
||||
return self
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
@@ -155,6 +197,7 @@ class MessageResponse(BaseModel):
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
attachment_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class MessagePageResponse(BaseModel):
|
||||
|
||||
+347
-21
@@ -11,23 +11,39 @@ from typing import Any
|
||||
import asyncpg
|
||||
import redis.asyncio as redis
|
||||
|
||||
import io
|
||||
import boto3
|
||||
import botocore.exceptions
|
||||
from PIL import Image
|
||||
|
||||
from app.auth import generate_session_token, hash_password, verify_password
|
||||
from app.config import (
|
||||
CACHE_URL,
|
||||
DATABASE_URL,
|
||||
SESSION_TTL_SECONDS,
|
||||
S3_ENDPOINT,
|
||||
S3_REGION,
|
||||
S3_BUCKET,
|
||||
S3_ACCESS_KEY_ID,
|
||||
S3_SECRET_ACCESS_KEY,
|
||||
MAX_UPLOAD_BYTES,
|
||||
MAX_IMAGE_PIXELS,
|
||||
)
|
||||
from app.errors import (
|
||||
ConflictError,
|
||||
InvalidRequestError,
|
||||
NotFoundError,
|
||||
PayloadTooLargeError,
|
||||
UnauthorizedError,
|
||||
UnsupportedMediaTypeError,
|
||||
)
|
||||
from app.schemas import (
|
||||
AttachmentResponse,
|
||||
ChatResponse,
|
||||
MessageResponse,
|
||||
SessionResponse,
|
||||
UserResponse,
|
||||
VariantResponse,
|
||||
now_utc_iso,
|
||||
)
|
||||
|
||||
@@ -71,12 +87,13 @@ class PostgresStorage:
|
||||
def __init__(self) -> None:
|
||||
self.pg_pool: asyncpg.Pool | None = None
|
||||
self.redis_client: redis.Redis | None = None
|
||||
self.s3_client: Any = None
|
||||
|
||||
@asynccontextmanager
|
||||
async def acquire_conn(self, timeout: float = 5.0):
|
||||
async def acquire_conn(self, timeout: float = 2.0):
|
||||
if self.pg_pool is None:
|
||||
raise asyncpg.InterfaceError("Пул подключений к PostgreSQL не инициализирован")
|
||||
async with asyncio.timeout(10.0):
|
||||
async with asyncio.timeout(3.0):
|
||||
async with self.pg_pool.acquire(timeout=timeout) as conn:
|
||||
yield conn
|
||||
|
||||
@@ -86,18 +103,41 @@ class PostgresStorage:
|
||||
DATABASE_URL,
|
||||
min_size=2,
|
||||
max_size=20,
|
||||
command_timeout=5.0,
|
||||
timeout=5.0,
|
||||
command_timeout=3.0,
|
||||
timeout=2.0,
|
||||
)
|
||||
|
||||
logger.info("Подключение к Redis: %s", CACHE_URL.split("@")[-1])
|
||||
self.redis_client = redis.from_url(
|
||||
CACHE_URL,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=3.0,
|
||||
socket_timeout=3.0,
|
||||
socket_connect_timeout=2.0,
|
||||
socket_timeout=2.0,
|
||||
)
|
||||
|
||||
logger.info("Подключение к S3 (MinIO): %s, bucket: %s", S3_ENDPOINT, S3_BUCKET)
|
||||
self.s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
region_name=S3_REGION,
|
||||
)
|
||||
await self._ensure_bucket()
|
||||
|
||||
async def _ensure_bucket(self) -> None:
|
||||
def _init():
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=S3_BUCKET)
|
||||
except Exception:
|
||||
try:
|
||||
self.s3_client.create_bucket(Bucket=S3_BUCKET)
|
||||
logger.info("Создан приватный бакет S3: %s", S3_BUCKET)
|
||||
except Exception as e:
|
||||
logger.warning("Не удалось создать бакет S3 %s: %s", S3_BUCKET, e)
|
||||
|
||||
await asyncio.to_thread(_init)
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.pg_pool:
|
||||
await self.pg_pool.close()
|
||||
@@ -105,15 +145,18 @@ class PostgresStorage:
|
||||
if self.redis_client:
|
||||
await self.redis_client.close()
|
||||
self.redis_client = None
|
||||
self.s3_client = None
|
||||
|
||||
async def check_health(self) -> bool:
|
||||
if not self.pg_pool or not self.redis_client:
|
||||
if not self.pg_pool or not self.redis_client or not self.s3_client:
|
||||
return False
|
||||
try:
|
||||
async with self.acquire_conn() as conn:
|
||||
await conn.fetchval("SELECT 1")
|
||||
await self.redis_client.ping()
|
||||
return True
|
||||
async with asyncio.timeout(2.5):
|
||||
async with self.acquire_conn(timeout=1.5) as conn:
|
||||
await conn.fetchval("SELECT 1")
|
||||
await self.redis_client.ping()
|
||||
await asyncio.to_thread(self.s3_client.head_bucket, Bucket=S3_BUCKET)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Ошибка проверки health: %s", exc)
|
||||
return False
|
||||
@@ -408,12 +451,22 @@ class PostgresStorage:
|
||||
# ------------------ Сообщения ------------------
|
||||
|
||||
@staticmethod
|
||||
def compute_request_hash(text: str) -> str:
|
||||
canonical = json.dumps({"text": text}, sort_keys=True, ensure_ascii=False)
|
||||
def compute_request_hash(text: str, attachment_ids: list[str] | None = None) -> str:
|
||||
norm_attachments = [str(x) for x in (attachment_ids or [])]
|
||||
canonical = json.dumps(
|
||||
{"attachment_ids": norm_attachments, "text": text},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
async def send_message(
|
||||
self, caller_id: str, chat_id: str, text: str, idempotency_key: str
|
||||
self,
|
||||
caller_id: str,
|
||||
chat_id: str,
|
||||
text: str,
|
||||
idempotency_key: str,
|
||||
attachment_ids: list[str] | None = None,
|
||||
) -> MessageResponse:
|
||||
try:
|
||||
cid = uuid.UUID(chat_id)
|
||||
@@ -421,7 +474,8 @@ class PostgresStorage:
|
||||
except ValueError:
|
||||
raise NotFoundError("Чат не найден")
|
||||
|
||||
req_hash = self.compute_request_hash(text)
|
||||
norm_attachments = [str(x) for x in (attachment_ids or [])]
|
||||
req_hash = self.compute_request_hash(text, norm_attachments)
|
||||
|
||||
async with self.acquire_conn() as conn:
|
||||
is_member = await conn.fetchval(
|
||||
@@ -432,10 +486,32 @@ class PostgresStorage:
|
||||
if not is_member:
|
||||
raise NotFoundError("Чат не найден")
|
||||
|
||||
# Валидация прикрепляемых вложений
|
||||
att_uuids: list[uuid.UUID] = []
|
||||
if norm_attachments:
|
||||
for att_str in norm_attachments:
|
||||
try:
|
||||
att_uuids.append(uuid.UUID(att_str))
|
||||
except ValueError:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
rows = await conn.fetch(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY($1::uuid[])",
|
||||
att_uuids,
|
||||
)
|
||||
if len(rows) != len(att_uuids):
|
||||
raise NotFoundError("Одно или несколько вложений не найдены")
|
||||
|
||||
for r in rows:
|
||||
if r["chat_id"] != cid:
|
||||
raise NotFoundError("Вложение принадлежит другому чату")
|
||||
if r["status"] != "ready":
|
||||
raise ConflictError("Вложение еще не готово или повреждено")
|
||||
|
||||
# 1. Проверяем, обрабатывался ли уже этот idempotency_key
|
||||
existing = await conn.fetchrow(
|
||||
"""
|
||||
SELECT ik.message_id, ik.request_hash, m.text, m.created_at AS msg_created_at
|
||||
SELECT ik.message_id, ik.request_hash, m.text, m.attachment_ids, m.created_at AS msg_created_at
|
||||
FROM idempotency_keys ik
|
||||
JOIN messages m ON m.id = ik.message_id
|
||||
WHERE ik.user_id = $1 AND ik.chat_id = $2 AND ik.idempotency_key = $3
|
||||
@@ -452,11 +528,15 @@ class PostgresStorage:
|
||||
if isinstance(raw_text, (bytes, bytearray))
|
||||
else str(raw_text)
|
||||
)
|
||||
res_attachments = [
|
||||
str(a) for a in (existing["attachment_ids"] or [])
|
||||
]
|
||||
return MessageResponse(
|
||||
id=str(existing["message_id"]),
|
||||
chat_id=chat_id,
|
||||
sender_id=caller_id,
|
||||
text=msg_text,
|
||||
attachment_ids=res_attachments,
|
||||
created_at=_format_dt(existing["msg_created_at"]),
|
||||
)
|
||||
else:
|
||||
@@ -470,13 +550,14 @@ class PostgresStorage:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
""",
|
||||
msg_id,
|
||||
cid,
|
||||
c_caller,
|
||||
text_bytes,
|
||||
att_uuids,
|
||||
now,
|
||||
)
|
||||
await conn.execute(
|
||||
@@ -492,10 +573,9 @@ class PostgresStorage:
|
||||
now,
|
||||
)
|
||||
except asyncpg.UniqueViolationError:
|
||||
# Конкурентная вставка из другой реплики выиграла гонку
|
||||
winner = await conn.fetchrow(
|
||||
"""
|
||||
SELECT ik.message_id, ik.request_hash, m.text, m.created_at AS msg_created_at
|
||||
SELECT ik.message_id, ik.request_hash, m.text, m.attachment_ids, m.created_at AS msg_created_at
|
||||
FROM idempotency_keys ik
|
||||
JOIN messages m ON m.id = ik.message_id
|
||||
WHERE ik.user_id = $1 AND ik.chat_id = $2 AND ik.idempotency_key = $3
|
||||
@@ -512,11 +592,15 @@ class PostgresStorage:
|
||||
if isinstance(raw_text, (bytes, bytearray))
|
||||
else str(raw_text)
|
||||
)
|
||||
res_attachments = [
|
||||
str(a) for a in (winner["attachment_ids"] or [])
|
||||
]
|
||||
return MessageResponse(
|
||||
id=str(winner["message_id"]),
|
||||
chat_id=chat_id,
|
||||
sender_id=caller_id,
|
||||
text=msg_text,
|
||||
attachment_ids=res_attachments,
|
||||
created_at=_format_dt(winner["msg_created_at"]),
|
||||
)
|
||||
else:
|
||||
@@ -528,6 +612,7 @@ class PostgresStorage:
|
||||
chat_id=chat_id,
|
||||
sender_id=caller_id,
|
||||
text=text,
|
||||
attachment_ids=norm_attachments,
|
||||
created_at=_format_dt(now),
|
||||
)
|
||||
|
||||
@@ -576,7 +661,7 @@ class PostgresStorage:
|
||||
raise InvalidRequestError("Некорректные параметры курсора")
|
||||
|
||||
query = """
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
SELECT id, chat_id, sender_id, text, attachment_ids, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (created_at, id) > ($2::timestamptz, $3::uuid))
|
||||
@@ -599,6 +684,7 @@ class PostgresStorage:
|
||||
chat_id=str(r["chat_id"]),
|
||||
sender_id=str(r["sender_id"]),
|
||||
text=_to_text(r["text"]),
|
||||
attachment_ids=[str(a) for a in (r["attachment_ids"] or [])],
|
||||
created_at=_format_dt(r["created_at"]),
|
||||
)
|
||||
for r in items_rows
|
||||
@@ -618,6 +704,246 @@ class PostgresStorage:
|
||||
|
||||
return items, next_cursor
|
||||
|
||||
# ------------------ Вложения (Attachments) ------------------
|
||||
|
||||
async def upload_attachment(
|
||||
self,
|
||||
caller_id: str,
|
||||
chat_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
data: bytes,
|
||||
) -> AttachmentResponse:
|
||||
try:
|
||||
cid = uuid.UUID(chat_id)
|
||||
c_caller = uuid.UUID(caller_id)
|
||||
except ValueError:
|
||||
raise NotFoundError("Чат не найден")
|
||||
|
||||
# 1. Валидация filename
|
||||
if not filename or len(filename) > 255:
|
||||
raise InvalidRequestError("Некорректная длина имени файла (1-255 символов)")
|
||||
if any(c in filename for c in ("/", "\\", "\0")) or any(
|
||||
ord(c) < 32 for c in filename
|
||||
):
|
||||
raise InvalidRequestError("Имя файла содержит недопустимые символы")
|
||||
|
||||
# 2. Проверка прав (ACL)
|
||||
async with self.acquire_conn() as conn:
|
||||
is_member = await conn.fetchval(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
cid,
|
||||
c_caller,
|
||||
)
|
||||
if not is_member:
|
||||
raise NotFoundError("Чат не найден")
|
||||
|
||||
# 3. Валидация Content-Type
|
||||
ct_clean = content_type.split(";")[0].strip().lower()
|
||||
if ct_clean not in ("image/png", "image/jpeg"):
|
||||
raise UnsupportedMediaTypeError(
|
||||
"Поддерживаются только форматы image/png и image/jpeg"
|
||||
)
|
||||
|
||||
# 4. Валидация размера данных
|
||||
size_bytes = len(data)
|
||||
if size_bytes > MAX_UPLOAD_BYTES:
|
||||
raise PayloadTooLargeError(
|
||||
f"Размер файла {size_bytes} байт превышает допустимый лимит {MAX_UPLOAD_BYTES} байт"
|
||||
)
|
||||
|
||||
# 5. Проверка магических байтов (сигнатуры MIME)
|
||||
if ct_clean == "image/png":
|
||||
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
raise UnsupportedMediaTypeError("Содержимое файла не соответствует формату PNG")
|
||||
elif ct_clean == "image/jpeg":
|
||||
if not (data.startswith(b"\xff\xd8\xff")):
|
||||
raise UnsupportedMediaTypeError("Содержимое файла не соответствует формату JPEG")
|
||||
|
||||
# 6. Проверка разрешения (лимит 40 млн пикселей) и целостности растра
|
||||
def _check_dimensions():
|
||||
try:
|
||||
with Image.open(io.BytesIO(data)) as img:
|
||||
w, h = img.size
|
||||
if w * h > MAX_IMAGE_PIXELS:
|
||||
return False
|
||||
# Проверяем целостность растровых данных и фильтров строк:
|
||||
img.load()
|
||||
return True
|
||||
except Exception:
|
||||
raise UnsupportedMediaTypeError("Поврежденное изображение")
|
||||
|
||||
try:
|
||||
within_limits = await asyncio.to_thread(_check_dimensions)
|
||||
if not within_limits:
|
||||
raise PayloadTooLargeError(
|
||||
f"Разрешение изображения превышает лимит {MAX_IMAGE_PIXELS} пикселей"
|
||||
)
|
||||
except (UnsupportedMediaTypeError, PayloadTooLargeError):
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.warning("upload_attachment check_dimensions unexpected: %s (%s)", type(e), e)
|
||||
raise UnsupportedMediaTypeError("Поврежденное изображение") from e
|
||||
|
||||
# 7. Генерация идентификаторов
|
||||
att_id = uuid.uuid4()
|
||||
s3_key = f"chats/{chat_id}/{att_id}/{filename}"
|
||||
sha256_hash = hashlib.sha256(data).hexdigest()
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
# 8. Загрузка файла в S3
|
||||
def _put_to_s3():
|
||||
self.s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=s3_key,
|
||||
Body=data,
|
||||
ContentType=ct_clean,
|
||||
)
|
||||
|
||||
await asyncio.to_thread(_put_to_s3)
|
||||
|
||||
# 9. Сохранение метаданных в PostgreSQL (с компенсацией удаления из S3 при ошибке)
|
||||
try:
|
||||
async with self.acquire_conn() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type,
|
||||
size_bytes, sha256, s3_key, status, error_code,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ready', NULL, $9, $9)
|
||||
""",
|
||||
att_id,
|
||||
cid,
|
||||
c_caller,
|
||||
filename,
|
||||
ct_clean,
|
||||
size_bytes,
|
||||
sha256_hash,
|
||||
s3_key,
|
||||
now,
|
||||
)
|
||||
except Exception as db_err:
|
||||
logger.error("Ошибка записи метаданных в БД, компенсация S3: %s", db_err)
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
self.s3_client.delete_object, Bucket=S3_BUCKET, Key=s3_key
|
||||
)
|
||||
except Exception as del_err:
|
||||
logger.error("Ошибка удаления S3 объекта при компенсации: %s", del_err)
|
||||
raise
|
||||
|
||||
return AttachmentResponse(
|
||||
id=str(att_id),
|
||||
chat_id=str(cid),
|
||||
owner_id=str(c_caller),
|
||||
filename=filename,
|
||||
content_type=ct_clean,
|
||||
size_bytes=size_bytes,
|
||||
sha256=sha256_hash,
|
||||
status="ready",
|
||||
variants=[],
|
||||
error_code=None,
|
||||
created_at=_format_dt(now),
|
||||
updated_at=_format_dt(now),
|
||||
)
|
||||
|
||||
async def get_attachment(
|
||||
self, caller_id: str, attachment_id: str
|
||||
) -> AttachmentResponse:
|
||||
try:
|
||||
aid = uuid.UUID(attachment_id)
|
||||
c_caller = uuid.UUID(caller_id)
|
||||
except ValueError:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
async with self.acquire_conn() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
|
||||
a.size_bytes, a.sha256, a.status, a.error_code, a.created_at, a.updated_at
|
||||
FROM attachments a
|
||||
WHERE a.id = $1
|
||||
""",
|
||||
aid,
|
||||
)
|
||||
if not row:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
is_member = await conn.fetchval(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
row["chat_id"],
|
||||
c_caller,
|
||||
)
|
||||
if not is_member:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
return AttachmentResponse(
|
||||
id=str(row["id"]),
|
||||
chat_id=str(row["chat_id"]),
|
||||
owner_id=str(row["owner_id"]),
|
||||
filename=row["filename"],
|
||||
content_type=row["content_type"],
|
||||
size_bytes=row["size_bytes"],
|
||||
sha256=row["sha256"],
|
||||
status=row["status"],
|
||||
variants=[],
|
||||
error_code=row["error_code"],
|
||||
created_at=_format_dt(row["created_at"]),
|
||||
updated_at=_format_dt(row["updated_at"]),
|
||||
)
|
||||
|
||||
async def get_attachment_content(
|
||||
self, caller_id: str, attachment_id: str, variant: str | None = None
|
||||
) -> tuple[bytes, str]:
|
||||
try:
|
||||
aid = uuid.UUID(attachment_id)
|
||||
c_caller = uuid.UUID(caller_id)
|
||||
except ValueError:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
if variant not in (None, "", "original"):
|
||||
raise NotFoundError("Вариант не найден")
|
||||
|
||||
async with self.acquire_conn() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT a.chat_id, a.content_type, a.s3_key, a.status
|
||||
FROM attachments a
|
||||
WHERE a.id = $1
|
||||
""",
|
||||
aid,
|
||||
)
|
||||
if not row:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
is_member = await conn.fetchval(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
row["chat_id"],
|
||||
c_caller,
|
||||
)
|
||||
if not is_member:
|
||||
raise NotFoundError("Вложение не найдено")
|
||||
|
||||
if row["status"] != "ready":
|
||||
raise ConflictError("Вложение повреждено или еще не готово")
|
||||
|
||||
s3_key = row["s3_key"]
|
||||
content_type = row["content_type"]
|
||||
|
||||
def _get_from_s3():
|
||||
resp = self.s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key)
|
||||
return resp["Body"].read()
|
||||
|
||||
try:
|
||||
data = await asyncio.to_thread(_get_from_s3)
|
||||
except Exception as e:
|
||||
logger.error("Ошибка чтения объекта из S3 %s: %s", s3_key, e)
|
||||
raise NotFoundError("Файл не найден в хранилище") from e
|
||||
|
||||
return data, content_type
|
||||
|
||||
|
||||
# Глобальный экземпляр хранилища
|
||||
storage = PostgresStorage()
|
||||
|
||||
Reference in New Issue
Block a user