148 lines
7.4 KiB
Python
148 lines
7.4 KiB
Python
"""Синхронная загрузка: ограниченный поток -> проверка -> журнал -> S3 -> COMMIT."""
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
from tempfile import TemporaryFile
|
|
import unicodedata
|
|
from uuid import UUID, uuid4
|
|
import warnings
|
|
|
|
from PIL import Image, UnidentifiedImageError
|
|
from starlette.concurrency import run_in_threadpool
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
from src import db, object_store, storage
|
|
from src.errors import ApiError
|
|
|
|
|
|
MAX_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760"))
|
|
MAX_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
|
if not 1 <= MAX_BYTES <= 10485760 or not 1 <= MAX_PIXELS <= 40000000:
|
|
raise ValueError("Лимиты загрузки должны быть положительными и не превышать контракт")
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
warnings.filterwarnings("error", category=Image.DecompressionBombWarning)
|
|
# Максимум одно полное декодирование на реплику, независимо от числа клиентов.
|
|
upload_slot = asyncio.Semaphore(1)
|
|
|
|
|
|
def public_attachment(row):
|
|
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]),
|
|
"filename": row["filename"], "content_type": row["content_type"],
|
|
"size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": "ready",
|
|
"variants": [], "error_code": None, "created_at": storage.utc(row["created_at"]),
|
|
"updated_at": storage.utc(row["created_at"])}
|
|
|
|
|
|
def validate_filename(filename):
|
|
if not 1 <= len(filename) <= 255 or not filename.strip() or any(
|
|
char in "/\\" or unicodedata.category(char) == "Cc" for char in filename
|
|
):
|
|
raise ApiError(400, "invalid_request", "Недопустимое имя файла")
|
|
try:
|
|
filename.encode("utf-8")
|
|
except UnicodeError:
|
|
raise ApiError(400, "invalid_request", "Недопустимое имя файла") from None
|
|
|
|
|
|
def validate_image(file, declared_type):
|
|
try:
|
|
file.seek(0)
|
|
with Image.open(file, formats=["PNG", "JPEG"]) as image:
|
|
true_type = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format]
|
|
if image.width * image.height > MAX_PIXELS:
|
|
raise ApiError(413, "payload_too_large", "Слишком много пикселей")
|
|
if true_type != declared_type:
|
|
raise ApiError(415, "unsupported_media_type", "MIME не совпадает с содержимым")
|
|
image.verify()
|
|
# verify проверяет структуру/CRC, load дополнительно обнаруживает обрыв pixel data.
|
|
file.seek(0)
|
|
with Image.open(file, formats=["PNG", "JPEG"]) as image:
|
|
image.load()
|
|
return true_type
|
|
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
|
raise ApiError(413, "payload_too_large", "Слишком много пикселей") from None
|
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
|
raise ApiError(415, "unsupported_media_type", "Повреждённый PNG/JPEG или другой формат") from None
|
|
|
|
|
|
def lock_upload(conn, key, blocking=True):
|
|
function = "pg_advisory_xact_lock" if blocking else "pg_try_advisory_xact_lock"
|
|
return conn.execute(f"SELECT {function}(hashtextextended(%s, 4)) AS locked", (key,)).fetchone()["locked"]
|
|
|
|
|
|
def save(user_id, chat_id, filename, content_type, file, size, sha256):
|
|
attachment_id = uuid4()
|
|
key = "attachments/" + attachment_id.hex
|
|
# Даже при kill -9 после PUT остаётся долговечное намерение для cleaner.
|
|
with db.connection() as conn:
|
|
storage.require_member(conn, user_id, chat_id)
|
|
conn.execute("INSERT INTO attachment_uploads(id, object_key) VALUES (%s, %s)", (attachment_id, key))
|
|
with db.connection() as conn:
|
|
conn.execute("SET LOCAL idle_in_transaction_session_timeout='30s'")
|
|
lock_upload(conn, key)
|
|
if not conn.execute("SELECT 1 FROM attachment_uploads WHERE id=%s", (attachment_id,)).fetchone():
|
|
raise ApiError(503, "unavailable", "Загрузка больше не активна")
|
|
object_store.put(key, file, size, content_type)
|
|
row = conn.execute("""INSERT INTO attachments(id, chat_id, owner_id, object_key, filename,
|
|
content_type, size_bytes, sha256) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) RETURNING *""",
|
|
(attachment_id, UUID(chat_id), UUID(user_id), key, filename, content_type, size, sha256)).fetchone()
|
|
conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,))
|
|
result = public_attachment(row)
|
|
# Ответ только после успешного COMMIT; при неопределённом исходе ничего не удаляем.
|
|
return result
|
|
|
|
|
|
async def upload(request, chat_id, filename):
|
|
user_id = request.state.user["id"]
|
|
await run_in_threadpool(storage.get_chat, user_id, chat_id)
|
|
validate_filename(filename)
|
|
content_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
|
if content_type not in {"image/png", "image/jpeg"}:
|
|
raise ApiError(415, "unsupported_media_type", "Ожидаются сырые bytes PNG/JPEG")
|
|
length = request.headers.get("Content-Length")
|
|
if length is not None:
|
|
if not length.isascii() or not length.isdecimal():
|
|
raise ApiError(400, "invalid_request", "Неверный Content-Length")
|
|
if len(length) > 10 or int(length) > MAX_BYTES:
|
|
raise ApiError(413, "payload_too_large", "Файл больше допустимого размера")
|
|
try:
|
|
await asyncio.wait_for(upload_slot.acquire(), timeout=20)
|
|
except TimeoutError:
|
|
raise ApiError(503, "unavailable", "Загрузки заняты; повторите позднее") from None
|
|
try:
|
|
with TemporaryFile() as file:
|
|
size = 0
|
|
digest = hashlib.sha256()
|
|
try:
|
|
async with asyncio.timeout(30):
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise ApiError(413, "payload_too_large", "Файл больше допустимого размера")
|
|
digest.update(chunk)
|
|
await run_in_threadpool(file.write, chunk)
|
|
except (TimeoutError, ClientDisconnect):
|
|
raise ApiError(400, "invalid_request", "Загрузка прервана или истёк таймаут") from None
|
|
true_type = await run_in_threadpool(validate_image, file, content_type)
|
|
return await run_in_threadpool(save, user_id, chat_id, filename, true_type, file, size, digest.hexdigest())
|
|
finally:
|
|
upload_slot.release()
|
|
|
|
|
|
def find(user_id, attachment_id):
|
|
with db.connection() as conn:
|
|
row = conn.execute("""SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id
|
|
WHERE a.id=%s AND m.user_id=%s""", (UUID(attachment_id), UUID(user_id))).fetchone()
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "Вложение не найдено")
|
|
return row
|
|
|
|
|
|
def chunks(response):
|
|
try:
|
|
while chunk := response.read(65536):
|
|
yield chunk
|
|
finally:
|
|
response.close()
|
|
response.release_conn()
|