add level 3

This commit is contained in:
Матвей
2026-10-20 17:24:32 +03:00
parent 3b52b91fb7
commit e9ef559c34
33 changed files with 1138 additions and 96 deletions
+104
View File
@@ -0,0 +1,104 @@
import asyncio
from hashlib import sha256
from tempfile import TemporaryFile
import unicodedata
from uuid import UUID, uuid4
import warnings
from fastapi import APIRouter, Depends, Query, Request, Response
from PIL import Image, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from .errors import ApiError
from .models import Attachment
from .routes import get_current_user, get_store
from .storage.postgres import PostgresStore
router = APIRouter()
Image.MAX_IMAGE_PIXELS = 40000000
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
def inspect_image(file, content_type, max_pixels):
try:
file.seek(0)
with Image.open(file, formats=("PNG", "JPEG")) as image:
if image.width * image.height > max_pixels:
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format]
if actual != content_type or getattr(image, "n_frames", 1) != 1:
raise ApiError(415, "unsupported_media_type", "Нужен обычный PNG или JPEG")
image.verify()
# verify checks PNG CRC; load also catches truncated pixel streams/JPEGs.
file.seek(0)
with Image.open(file, formats=("PNG", "JPEG")) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error:
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from error
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error:
raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error
file.seek(0)
@router.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=201)
async def upload(chat_id: UUID, request: Request, filename: str = Query(...),
user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)):
await store.get_chat_for_member(chat_id, user["id"])
if (not 1 <= len(filename) <= 255 or not filename.strip() or
any(c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"} for c in filename)):
raise ApiError(400, "invalid_request", "Некорректное имя файла")
content_type = request.headers.get("content-type", "").lower()
if content_type not in {"image/png", "image/jpeg"}:
raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG")
s3 = request.app.state.s3
limit = s3.settings.max_bytes
if "content-length" in request.headers:
try:
length = int(request.headers["content-length"])
if length < 0:
raise ValueError()
except ValueError:
raise ApiError(400, "invalid_request", "Некорректная длина") from None
if length > limit:
raise ApiError(413, "payload_too_large", "Файл больше лимита")
try:
async with asyncio.timeout(30):
async with request.app.state.upload_slot:
# Temporary scratch space only; deleted on every exit, never used for downloads.
with TemporaryFile() as file:
size = 0
digest = sha256()
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise ApiError(413, "payload_too_large", "Файл больше лимита")
digest.update(chunk)
await run_in_threadpool(file.write, chunk)
await run_in_threadpool(inspect_image, file, content_type, s3.settings.max_pixels)
attachment_id = uuid4()
key = f"originals/{attachment_id}"
await s3.put(key, file, size, content_type)
return await store.create_attachment(
attachment_id, chat_id, user["id"], filename, content_type,
size, digest.hexdigest(), key,
)
except TimeoutError:
raise ApiError(503, "unavailable", "Время загрузки истекло") from None
@router.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
async def metadata(attachment_id: UUID, user: dict = Depends(get_current_user),
store: PostgresStore = Depends(get_store)):
return await store.attachment_for_member(attachment_id, user["id"])
@router.get("/api/v1/attachments/{attachment_id}/content")
async def download(attachment_id: UUID, request: Request,
variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"),
user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)):
attachment = await store.attachment_for_member(attachment_id, user["id"])
if variant != "original":
raise ApiError(404, "not_found", "Вариант не найден")
data = await request.app.state.s3.download(attachment)
return Response(data, media_type=attachment["content_type"],
headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"})
+38
View File
@@ -0,0 +1,38 @@
"""Idempotent private bucket bootstrap; executed with separate admin credentials."""
import time
from botocore.exceptions import BotoCoreError, ClientError
from .storage.s3 import S3Settings, S3Store
def main():
store = S3Store(S3Settings.load())
try:
for attempt in range(60):
try:
store.client.list_buckets()
break
except (BotoCoreError, ClientError):
if attempt == 59:
raise RuntimeError("S3 startup timed out") from None
time.sleep(1)
bucket = store.settings.bucket
try:
store.client.head_bucket(Bucket=bucket)
except ClientError as error:
if str(error.response["Error"]["Code"]) not in {"404", "NoSuchBucket", "NotFound"}:
raise
store.client.create_bucket(Bucket=bucket)
# No anonymous identity; revoke any previously configured public grants/policy.
store.client.put_bucket_acl(Bucket=bucket, ACL="private")
try:
store.client.delete_bucket_policy(Bucket=bucket)
except ClientError as error:
if error.response["Error"]["Code"] != "NoSuchBucketPolicy":
raise
print("PASS: private S3 bucket ready")
finally:
store.client.close()
if __name__ == "__main__":
main()
+4
View File
@@ -15,6 +15,10 @@ _CONFIGURATION_KEYS = {
"INSTANCE_ID",
"CURSOR_SECRET",
"CURSOR_SECRET_FILE",
"S3_ENDPOINT", "S3_REGION", "S3_BUCKET",
"S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE",
"S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE",
"MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS",
}
+8 -1
View File
@@ -1,3 +1,4 @@
import asyncio
import logging
import sys
import time
@@ -15,6 +16,8 @@ from .cursor import configure_secret
from .models import ErrorDetails, ErrorResponse
from .config import load_settings
from .routes import router
from .attachments import router as attachment_router
from .storage.s3 import S3Settings, S3Store
from .storage.postgres import PostgresStore
@@ -41,17 +44,20 @@ async def lifespan(application: FastAPI):
await store.connect()
application.state.store = store
application.state.settings = settings
application.state.s3 = S3Store(S3Settings.load())
application.state.upload_slot = asyncio.Semaphore(1)
application.state.accepting_requests = True
try:
yield
finally:
application.state.accepting_requests = False
application.state.s3.client.close()
await store.close()
app = FastAPI(
title="MTUSI Messenger",
version="1.3.0",
version="1.4.0",
redoc_url=None,
lifespan=lifespan,
)
@@ -144,6 +150,7 @@ async def handle_unexpected_error(request: Request, error: Exception) -> JSONRes
app.include_router(router)
app.include_router(attachment_router)
def main() -> None:
+32 -8
View File
@@ -1,7 +1,7 @@
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
class CreateUserRequest(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
@@ -46,16 +46,22 @@ class ChatPageResponse(BaseModel):
class CreateMessageRequest(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
text: str = Field(min_length=1, max_length=4000)
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list, max_length=10)
@field_validator("text")
@field_validator("attachment_ids")
@classmethod
def text_must_not_be_whitespace(cls, value: str) -> str:
"""Не допускает сообщение, состоящее только из пробельных символов."""
def valid_ids(cls, values: list[str]) -> list[str]:
normalized = [str(UUID(value)) for value in values]
if len(normalized) != len(set(normalized)):
raise ValueError("Повторяющиеся attachment_ids")
return normalized
if not value.strip():
raise ValueError("text не может состоять только из пробелов")
return value
@model_validator(mode="after")
def has_content(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError("Сообщение без содержимого")
return self
class MessagePageResponse(BaseModel):
@@ -104,6 +110,7 @@ class Message(BaseModel):
chat_id: UUID
sender_id: UUID
text: str
attachment_ids: list[UUID] = Field(default_factory=list)
created_at: datetime
@@ -114,3 +121,20 @@ class Session(BaseModel):
token_type: str = "Bearer"
expires_at: datetime
user: User
class Attachment(BaseModel):
model_config = ConfigDict(extra="ignore")
id: UUID
chat_id: UUID
owner_id: UUID
filename: str
content_type: str
size_bytes: int
sha256: str
status: str = "ready"
variants: list = Field(default_factory=list)
error_code: str | None = None
created_at: datetime
updated_at: datetime
+2
View File
@@ -83,6 +83,7 @@ async def readiness(request: Request, store: PostgresStore = Depends(get_store))
if not request.app.state.accepting_requests:
raise ApiError(503, "unavailable", "API завершает работу")
await store.ready()
await request.app.state.s3.ready()
return {"status": "ok"}
@router.post("/api/v1/users", response_model=User, status_code=201)
@@ -164,6 +165,7 @@ async def create_message(
payload.text,
idempotency_key,
request.app.state.settings.idempotency_ttl_seconds,
[UUID(value) for value in payload.attachment_ids],
)
@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePageResponse)
+45 -7
View File
@@ -83,6 +83,7 @@ class PostgresStore:
"chat_id": row["chat_id"],
"sender_id": row["sender_id"],
"text": row["text"],
"attachment_ids": row["attachment_ids"],
"created_at": row["created_at"],
}
@@ -216,12 +217,17 @@ class PostgresStore:
return [self._chat(row) for row in rows]
async def create_message_idempotent(
self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int
self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int, attachment_ids: list[UUID] | None = None
) -> dict:
"""Создаёт сообщение и фиксирует результат retry в одной транзакции."""
path = f"/api/v1/chats/{chat_id}/messages"
attachment_ids = attachment_ids or []
# Preserve hashes of text-only requests already accepted by lab3.
canonical = {"text": text}
if attachment_ids:
canonical["attachment_ids"] = [str(value) for value in attachment_ids]
request_hash = sha256(
json.dumps({"text": text}, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
).hexdigest()
try:
async with self.pool.acquire() as connection, connection.transaction():
@@ -233,6 +239,13 @@ class PostgresStore:
await connection.execute(
"DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds
)
if attachment_ids:
count = await connection.fetchval(
"SELECT count(*) FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[])",
chat_id, attachment_ids,
)
if count != len(attachment_ids):
raise ApiError(404, "not_found", "Вложение не найдено")
claimed = await connection.fetchrow(
"INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) "
"VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id",
@@ -251,15 +264,15 @@ class PostgresStore:
if recorded["message_id"] is None:
raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса")
row = await connection.fetchrow(
"SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = $1", recorded["message_id"]
"SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE id = $1", recorded["message_id"]
)
if row is None:
raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение")
return self._message(row)
row = await connection.fetchrow(
"INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) "
"RETURNING id, chat_id, sender_id, text, created_at",
uuid4(), chat_id, sender_id, text,
"INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) VALUES ($1, $2, $3, $4, $5) "
"RETURNING id, chat_id, sender_id, text, attachment_ids, created_at",
uuid4(), chat_id, sender_id, text, attachment_ids,
)
await connection.execute(
"UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' "
@@ -274,7 +287,7 @@ class PostgresStore:
self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int
) -> list[dict]:
await self.get_chat_for_member(chat_id, user_id)
query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1"
query = "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE chat_id = $1"
args: list = [chat_id]
if after is not None:
query += " AND (created_at, id) > ($2, $3)"
@@ -286,3 +299,28 @@ class PostgresStore:
except DB_ERRORS as error:
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
return [self._message(row) for row in rows]
async def create_attachment(self, attachment_id, chat_id, owner_id, filename,
content_type, size_bytes, digest, object_key) -> dict:
try:
row = await self.pool.fetchrow(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) "
"VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *",
attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, object_key,
)
except DB_ERRORS as error:
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
return dict(row)
async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict:
try:
row = await self.pool.fetchrow(
"SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
"WHERE a.id = $1 AND m.user_id = $2", attachment_id, user_id,
)
except DB_ERRORS as error:
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
if row is None:
raise ApiError(404, "not_found", "Вложение не найдено")
return dict(row)
+85
View File
@@ -0,0 +1,85 @@
"""Bounded S3 operations; synchronous SDK calls run outside the event loop."""
from dataclasses import dataclass
from hashlib import sha256
import os
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from starlette.concurrency import run_in_threadpool
from ..errors import ApiError
@dataclass(frozen=True)
class S3Settings:
endpoint: str
region: str
bucket: str
access_key: str
secret_key: str
max_bytes: int
max_pixels: int
@classmethod
def load(cls):
def secret(name):
path = os.environ.get(name + "_FILE")
result = Path(path).read_text().strip() if path else os.environ.get(name, "")
if not result:
raise ValueError(f"Missing {name}")
return result
max_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
if not 1 <= max_bytes <= 10485760 or not 1 <= max_pixels <= 40000000:
raise ValueError("Image limits exceed contract")
return cls(os.environ.get("S3_ENDPOINT", "http://s3:8333"),
os.environ.get("S3_REGION", "us-east-1"),
os.environ.get("S3_BUCKET", "messenger-attachments"),
secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"),
max_bytes, max_pixels)
class S3Store:
def __init__(self, settings):
self.settings = settings
self.client = boto3.client(
"s3", endpoint_url=settings.endpoint, region_name=settings.region,
aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key,
config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5,
retries={"total_max_attempts": 1},
s3={"addressing_style": "path"},
request_checksum_calculation="when_required",
response_checksum_validation="when_required"),
)
async def call(self, method, **kwargs):
try:
return await run_in_threadpool(getattr(self.client, method), **kwargs)
except (BotoCoreError, ClientError, OSError) as error:
raise ApiError(503, "unavailable", "S3 недоступно") from error
async def ready(self):
await self.call("head_bucket", Bucket=self.settings.bucket)
async def put(self, key, file, size, content_type):
file.seek(0)
await self.call("put_object", Bucket=self.settings.bucket, Key=key,
Body=file, ContentLength=size, ContentType=content_type)
async def download(self, attachment):
response = await self.call("get_object", Bucket=self.settings.bucket,
Key=attachment["object_key"])
body = response["Body"]
try:
data = await run_in_threadpool(body.read, self.settings.max_bytes + 1)
except (BotoCoreError, OSError) as error:
raise ApiError(503, "unavailable", "S3 недоступно") from error
finally:
body.close()
if (len(data) != attachment["size_bytes"] or
sha256(data).hexdigest() != attachment["sha256"]):
raise ApiError(503, "unavailable", "Оригинал недоступен")
return data