add level 3
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
import asyncio
|
||||
from hashlib import sha256
|
||||
from tempfile import TemporaryFile
|
||||
import unicodedata
|
||||
from uuid import UUID, uuid4
|
||||
import warnings
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request, Response
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .errors import ApiError
|
||||
from .models import Attachment
|
||||
from .routes import get_current_user, get_store
|
||||
from .storage.postgres import PostgresStore
|
||||
|
||||
router = APIRouter()
|
||||
Image.MAX_IMAGE_PIXELS = 40000000
|
||||
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def inspect_image(file, content_type, max_pixels):
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file, formats=("PNG", "JPEG")) as image:
|
||||
if image.width * image.height > max_pixels:
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
||||
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format]
|
||||
if actual != content_type or getattr(image, "n_frames", 1) != 1:
|
||||
raise ApiError(415, "unsupported_media_type", "Нужен обычный PNG или JPEG")
|
||||
image.verify()
|
||||
# verify checks PNG CRC; load also catches truncated pixel streams/JPEGs.
|
||||
file.seek(0)
|
||||
with Image.open(file, formats=("PNG", "JPEG")) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error:
|
||||
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from error
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error:
|
||||
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error
|
||||
file.seek(0)
|
||||
|
||||
|
||||
@router.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=201)
|
||||
async def upload(chat_id: UUID, request: Request, filename: str = Query(...),
|
||||
user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)):
|
||||
await store.get_chat_for_member(chat_id, user["id"])
|
||||
if (not 1 <= len(filename) <= 255 or not filename.strip() or
|
||||
any(c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"} for c in filename)):
|
||||
raise ApiError(400, "invalid_request", "Некорректное имя файла")
|
||||
content_type = request.headers.get("content-type", "").lower()
|
||||
if content_type not in {"image/png", "image/jpeg"}:
|
||||
raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG")
|
||||
s3 = request.app.state.s3
|
||||
limit = s3.settings.max_bytes
|
||||
if "content-length" in request.headers:
|
||||
try:
|
||||
length = int(request.headers["content-length"])
|
||||
if length < 0:
|
||||
raise ValueError()
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Некорректная длина") from None
|
||||
if length > limit:
|
||||
raise ApiError(413, "payload_too_large", "Файл больше лимита")
|
||||
try:
|
||||
async with asyncio.timeout(30):
|
||||
async with request.app.state.upload_slot:
|
||||
# Temporary scratch space only; deleted on every exit, never used for downloads.
|
||||
with TemporaryFile() as file:
|
||||
size = 0
|
||||
digest = sha256()
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
raise ApiError(413, "payload_too_large", "Файл больше лимита")
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(file.write, chunk)
|
||||
await run_in_threadpool(inspect_image, file, content_type, s3.settings.max_pixels)
|
||||
attachment_id = uuid4()
|
||||
key = f"originals/{attachment_id}"
|
||||
await s3.put(key, file, size, content_type)
|
||||
return await store.create_attachment(
|
||||
attachment_id, chat_id, user["id"], filename, content_type,
|
||||
size, digest.hexdigest(), key,
|
||||
)
|
||||
except TimeoutError:
|
||||
raise ApiError(503, "unavailable", "Время загрузки истекло") from None
|
||||
|
||||
|
||||
@router.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
|
||||
async def metadata(attachment_id: UUID, user: dict = Depends(get_current_user),
|
||||
store: PostgresStore = Depends(get_store)):
|
||||
return await store.attachment_for_member(attachment_id, user["id"])
|
||||
|
||||
|
||||
@router.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download(attachment_id: UUID, request: Request,
|
||||
variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"),
|
||||
user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)):
|
||||
attachment = await store.attachment_for_member(attachment_id, user["id"])
|
||||
if variant != "original":
|
||||
raise ApiError(404, "not_found", "Вариант не найден")
|
||||
data = await request.app.state.s3.download(attachment)
|
||||
return Response(data, media_type=attachment["content_type"],
|
||||
headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"})
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Idempotent private bucket bootstrap; executed with separate admin credentials."""
|
||||
import time
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from .storage.s3 import S3Settings, S3Store
|
||||
|
||||
|
||||
def main():
|
||||
store = S3Store(S3Settings.load())
|
||||
try:
|
||||
for attempt in range(60):
|
||||
try:
|
||||
store.client.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError):
|
||||
if attempt == 59:
|
||||
raise RuntimeError("S3 startup timed out") from None
|
||||
time.sleep(1)
|
||||
bucket = store.settings.bucket
|
||||
try:
|
||||
store.client.head_bucket(Bucket=bucket)
|
||||
except ClientError as error:
|
||||
if str(error.response["Error"]["Code"]) not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
store.client.create_bucket(Bucket=bucket)
|
||||
# No anonymous identity; revoke any previously configured public grants/policy.
|
||||
store.client.put_bucket_acl(Bucket=bucket, ACL="private")
|
||||
try:
|
||||
store.client.delete_bucket_policy(Bucket=bucket)
|
||||
except ClientError as error:
|
||||
if error.response["Error"]["Code"] != "NoSuchBucketPolicy":
|
||||
raise
|
||||
print("PASS: private S3 bucket ready")
|
||||
finally:
|
||||
store.client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -15,6 +15,10 @@ _CONFIGURATION_KEYS = {
|
||||
"INSTANCE_ID",
|
||||
"CURSOR_SECRET",
|
||||
"CURSOR_SECRET_FILE",
|
||||
"S3_ENDPOINT", "S3_REGION", "S3_BUCKET",
|
||||
"S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE",
|
||||
"S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE",
|
||||
"MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+8
-1
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
@@ -15,6 +16,8 @@ from .cursor import configure_secret
|
||||
from .models import ErrorDetails, ErrorResponse
|
||||
from .config import load_settings
|
||||
from .routes import router
|
||||
from .attachments import router as attachment_router
|
||||
from .storage.s3 import S3Settings, S3Store
|
||||
from .storage.postgres import PostgresStore
|
||||
|
||||
|
||||
@@ -41,17 +44,20 @@ async def lifespan(application: FastAPI):
|
||||
await store.connect()
|
||||
application.state.store = store
|
||||
application.state.settings = settings
|
||||
application.state.s3 = S3Store(S3Settings.load())
|
||||
application.state.upload_slot = asyncio.Semaphore(1)
|
||||
application.state.accepting_requests = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
application.state.accepting_requests = False
|
||||
application.state.s3.client.close()
|
||||
await store.close()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUSI Messenger",
|
||||
version="1.3.0",
|
||||
version="1.4.0",
|
||||
redoc_url=None,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
@@ -144,6 +150,7 @@ async def handle_unexpected_error(request: Request, error: Exception) -> JSONRes
|
||||
|
||||
|
||||
app.include_router(router)
|
||||
app.include_router(attachment_router)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
+32
-8
@@ -1,7 +1,7 @@
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
@@ -46,16 +46,22 @@ class ChatPageResponse(BaseModel):
|
||||
class CreateMessageRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("text")
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def text_must_not_be_whitespace(cls, value: str) -> str:
|
||||
"""Не допускает сообщение, состоящее только из пробельных символов."""
|
||||
def valid_ids(cls, values: list[str]) -> list[str]:
|
||||
normalized = [str(UUID(value)) for value in values]
|
||||
if len(normalized) != len(set(normalized)):
|
||||
raise ValueError("Повторяющиеся attachment_ids")
|
||||
return normalized
|
||||
|
||||
if not value.strip():
|
||||
raise ValueError("text не может состоять только из пробелов")
|
||||
return value
|
||||
@model_validator(mode="after")
|
||||
def has_content(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError("Сообщение без содержимого")
|
||||
return self
|
||||
|
||||
|
||||
class MessagePageResponse(BaseModel):
|
||||
@@ -104,6 +110,7 @@ class Message(BaseModel):
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
attachment_ids: list[UUID] = Field(default_factory=list)
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@@ -114,3 +121,20 @@ class Session(BaseModel):
|
||||
token_type: str = "Bearer"
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
|
||||
class Attachment(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str = "ready"
|
||||
variants: list = Field(default_factory=list)
|
||||
error_code: str | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@@ -83,6 +83,7 @@ async def readiness(request: Request, store: PostgresStore = Depends(get_store))
|
||||
if not request.app.state.accepting_requests:
|
||||
raise ApiError(503, "unavailable", "API завершает работу")
|
||||
await store.ready()
|
||||
await request.app.state.s3.ready()
|
||||
return {"status": "ok"}
|
||||
|
||||
@router.post("/api/v1/users", response_model=User, status_code=201)
|
||||
@@ -164,6 +165,7 @@ async def create_message(
|
||||
payload.text,
|
||||
idempotency_key,
|
||||
request.app.state.settings.idempotency_ttl_seconds,
|
||||
[UUID(value) for value in payload.attachment_ids],
|
||||
)
|
||||
|
||||
@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePageResponse)
|
||||
|
||||
+45
-7
@@ -83,6 +83,7 @@ class PostgresStore:
|
||||
"chat_id": row["chat_id"],
|
||||
"sender_id": row["sender_id"],
|
||||
"text": row["text"],
|
||||
"attachment_ids": row["attachment_ids"],
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
|
||||
@@ -216,12 +217,17 @@ class PostgresStore:
|
||||
return [self._chat(row) for row in rows]
|
||||
|
||||
async def create_message_idempotent(
|
||||
self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int
|
||||
self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int, attachment_ids: list[UUID] | None = None
|
||||
) -> dict:
|
||||
"""Создаёт сообщение и фиксирует результат retry в одной транзакции."""
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
attachment_ids = attachment_ids or []
|
||||
# Preserve hashes of text-only requests already accepted by lab3.
|
||||
canonical = {"text": text}
|
||||
if attachment_ids:
|
||||
canonical["attachment_ids"] = [str(value) for value in attachment_ids]
|
||||
request_hash = sha256(
|
||||
json.dumps({"text": text}, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||
json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||
).hexdigest()
|
||||
try:
|
||||
async with self.pool.acquire() as connection, connection.transaction():
|
||||
@@ -233,6 +239,13 @@ class PostgresStore:
|
||||
await connection.execute(
|
||||
"DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds
|
||||
)
|
||||
if attachment_ids:
|
||||
count = await connection.fetchval(
|
||||
"SELECT count(*) FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[])",
|
||||
chat_id, attachment_ids,
|
||||
)
|
||||
if count != len(attachment_ids):
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
claimed = await connection.fetchrow(
|
||||
"INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) "
|
||||
"VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id",
|
||||
@@ -251,15 +264,15 @@ class PostgresStore:
|
||||
if recorded["message_id"] is None:
|
||||
raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса")
|
||||
row = await connection.fetchrow(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = $1", recorded["message_id"]
|
||||
"SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE id = $1", recorded["message_id"]
|
||||
)
|
||||
if row is None:
|
||||
raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение")
|
||||
return self._message(row)
|
||||
row = await connection.fetchrow(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) "
|
||||
"RETURNING id, chat_id, sender_id, text, created_at",
|
||||
uuid4(), chat_id, sender_id, text,
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) VALUES ($1, $2, $3, $4, $5) "
|
||||
"RETURNING id, chat_id, sender_id, text, attachment_ids, created_at",
|
||||
uuid4(), chat_id, sender_id, text, attachment_ids,
|
||||
)
|
||||
await connection.execute(
|
||||
"UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' "
|
||||
@@ -274,7 +287,7 @@ class PostgresStore:
|
||||
self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int
|
||||
) -> list[dict]:
|
||||
await self.get_chat_for_member(chat_id, user_id)
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1"
|
||||
query = "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE chat_id = $1"
|
||||
args: list = [chat_id]
|
||||
if after is not None:
|
||||
query += " AND (created_at, id) > ($2, $3)"
|
||||
@@ -286,3 +299,28 @@ class PostgresStore:
|
||||
except DB_ERRORS as error:
|
||||
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
||||
return [self._message(row) for row in rows]
|
||||
|
||||
|
||||
async def create_attachment(self, attachment_id, chat_id, owner_id, filename,
|
||||
content_type, size_bytes, digest, object_key) -> dict:
|
||||
try:
|
||||
row = await self.pool.fetchrow(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) "
|
||||
"VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *",
|
||||
attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, object_key,
|
||||
)
|
||||
except DB_ERRORS as error:
|
||||
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
||||
return dict(row)
|
||||
|
||||
async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict:
|
||||
try:
|
||||
row = await self.pool.fetchrow(
|
||||
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = $1 AND m.user_id = $2", attachment_id, user_id,
|
||||
)
|
||||
except DB_ERRORS as error:
|
||||
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
||||
if row is None:
|
||||
raise ApiError(404, "not_found", "Вложение не найдено")
|
||||
return dict(row)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Bounded S3 operations; synchronous SDK calls run outside the event loop."""
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from ..errors import ApiError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class S3Settings:
|
||||
endpoint: str
|
||||
region: str
|
||||
bucket: str
|
||||
access_key: str
|
||||
secret_key: str
|
||||
max_bytes: int
|
||||
max_pixels: int
|
||||
|
||||
@classmethod
|
||||
def load(cls):
|
||||
def secret(name):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
result = Path(path).read_text().strip() if path else os.environ.get(name, "")
|
||||
if not result:
|
||||
raise ValueError(f"Missing {name}")
|
||||
return result
|
||||
|
||||
max_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
|
||||
max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
if not 1 <= max_bytes <= 10485760 or not 1 <= max_pixels <= 40000000:
|
||||
raise ValueError("Image limits exceed contract")
|
||||
return cls(os.environ.get("S3_ENDPOINT", "http://s3:8333"),
|
||||
os.environ.get("S3_REGION", "us-east-1"),
|
||||
os.environ.get("S3_BUCKET", "messenger-attachments"),
|
||||
secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"),
|
||||
max_bytes, max_pixels)
|
||||
|
||||
|
||||
class S3Store:
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.client = boto3.client(
|
||||
"s3", endpoint_url=settings.endpoint, region_name=settings.region,
|
||||
aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key,
|
||||
config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5,
|
||||
retries={"total_max_attempts": 1},
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required"),
|
||||
)
|
||||
|
||||
async def call(self, method, **kwargs):
|
||||
try:
|
||||
return await run_in_threadpool(getattr(self.client, method), **kwargs)
|
||||
except (BotoCoreError, ClientError, OSError) as error:
|
||||
raise ApiError(503, "unavailable", "S3 недоступно") from error
|
||||
|
||||
async def ready(self):
|
||||
await self.call("head_bucket", Bucket=self.settings.bucket)
|
||||
|
||||
async def put(self, key, file, size, content_type):
|
||||
file.seek(0)
|
||||
await self.call("put_object", Bucket=self.settings.bucket, Key=key,
|
||||
Body=file, ContentLength=size, ContentType=content_type)
|
||||
|
||||
async def download(self, attachment):
|
||||
response = await self.call("get_object", Bucket=self.settings.bucket,
|
||||
Key=attachment["object_key"])
|
||||
body = response["Body"]
|
||||
try:
|
||||
data = await run_in_threadpool(body.read, self.settings.max_bytes + 1)
|
||||
except (BotoCoreError, OSError) as error:
|
||||
raise ApiError(503, "unavailable", "S3 недоступно") from error
|
||||
finally:
|
||||
body.close()
|
||||
if (len(data) != attachment["size_bytes"] or
|
||||
sha256(data).hexdigest() != attachment["sha256"]):
|
||||
raise ApiError(503, "unavailable", "Оригинал недоступен")
|
||||
return data
|
||||
Reference in New Issue
Block a user