forked from mtusi2026/mtusi2026lab6
468 lines
16 KiB
Python
468 lines
16 KiB
Python
"""Attachments: durable upload to staging, queued metadata + outbox (lab5).
|
|
|
|
Upload, in this order (all checks before the bytes are stored):
|
|
|
|
1. filename (400) -> chat membership (404) -> Content-Type header (415)
|
|
-> Content-Length (413 without reading the body);
|
|
2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429;
|
|
3. the body is read in chunks into a SpooledTemporaryFile (only
|
|
UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more
|
|
than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400;
|
|
4. content validation in a thread (app.images): format by magic bytes,
|
|
pixel limit from the header (413), integrity (415);
|
|
5. upload intent COMMIT -> staging S3 PUT -> one transaction "delete intent +
|
|
insert queued attachment + outbox" -> 202. Worker produces all variants.
|
|
|
|
Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs
|
|
in the background):
|
|
- PUT failed or timed out: the outcome is unknown - S3 may still finish
|
|
writing the object AFTER our DELETE. So the object is deleted best
|
|
effort, but the intent is KEPT; the janitor deletes the object again
|
|
after UPLOAD_GRACE_SECONDS and only then the intent;
|
|
- COMMIT failed: the outcome is unknown too - COMMIT may have succeeded
|
|
and only the answer was lost. Under the intent's row lock: no intent ->
|
|
the row is committed, keep the object; else delete object, then intent;
|
|
- the process died after PUT: the intent stays; the janitor deletes the
|
|
object and the intent after UPLOAD_GRACE_SECONDS.
|
|
So an object without a row exists only while its intent exists, and an
|
|
object of a committed row is never deleted.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import tempfile
|
|
import unicodedata
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
from uuid import UUID, uuid4
|
|
|
|
from fastapi import Request
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
from app import repositories
|
|
from app.config import settings
|
|
from app.db import SessionLocal
|
|
from app.errors import (
|
|
ApiError,
|
|
Conflict,
|
|
InvalidRequest,
|
|
NotFound,
|
|
PayloadTooLarge,
|
|
RateLimited,
|
|
Unavailable,
|
|
UnsupportedMediaType,
|
|
)
|
|
from app.images import InvalidImage, TooManyPixels, validate_image
|
|
from app.storage import ObjectMissing, StorageUnavailable, storage
|
|
|
|
|
|
logger = logging.getLogger("messenger.attachments")
|
|
|
|
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
|
# Characters that can turn a display name into a path or hide its real
|
|
# extension: separators, NUL/control characters, bidi overrides etc.
|
|
FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
|
|
|
|
upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
|
|
# Full integrity checks are CPU work too. In particular, progressive JPEG
|
|
# retains decoder coefficients even with draft(); never run eight at once.
|
|
validation_slot = asyncio.Semaphore(1)
|
|
# Strong references to background compensation tasks (asyncio keeps only
|
|
# weak ones); if the process dies first, the intent stays for the janitor.
|
|
_compensations: set[asyncio.Task] = set()
|
|
|
|
|
|
def _to_attachment(row: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"id": row["id"],
|
|
"chat_id": row["chat_id"],
|
|
"owner_id": row["owner_id"],
|
|
"filename": row["filename"],
|
|
"content_type": row["content_type"],
|
|
"size_bytes": row["size_bytes"],
|
|
"sha256": row["sha256"],
|
|
"status": row["status"],
|
|
"variants": [
|
|
{key: value for key, value in variant.items() if key != "object_key"}
|
|
for variant in _variants(row)
|
|
],
|
|
"error_code": row["error_code"],
|
|
"created_at": row["created_at"],
|
|
"updated_at": row["updated_at"],
|
|
}
|
|
|
|
|
|
def _variants(row: dict[str, Any]) -> list[dict[str, Any]]:
|
|
value = row.get("variants") or []
|
|
return json.loads(value) if isinstance(value, str) else value
|
|
|
|
|
|
def validate_filename(filename: str) -> str:
|
|
if not 1 <= len(filename) <= 255 or not filename.strip():
|
|
raise InvalidRequest("filename must be 1-255 characters, not blank.")
|
|
|
|
if "/" in filename or "\\" in filename:
|
|
raise InvalidRequest("filename must not contain path separators.")
|
|
|
|
if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename):
|
|
raise InvalidRequest("filename must not contain control characters.")
|
|
|
|
if filename.strip(". ") == "":
|
|
raise InvalidRequest("filename must not be a relative path.")
|
|
|
|
return filename
|
|
|
|
|
|
def declared_content_type(request: Request) -> str:
|
|
raw = request.headers.get("Content-Type", "")
|
|
media_type = raw.split(";", 1)[0].strip().lower()
|
|
|
|
if media_type not in ALLOWED_CONTENT_TYPES:
|
|
raise UnsupportedMediaType(
|
|
"Content-Type must be image/png or image/jpeg (raw bytes, not multipart)."
|
|
)
|
|
|
|
return media_type
|
|
|
|
|
|
def _too_large(message: str) -> PayloadTooLarge:
|
|
error = PayloadTooLarge(message)
|
|
# The rest of the body is not read: the connection cannot be reused.
|
|
error.headers["Connection"] = "close"
|
|
return error
|
|
|
|
|
|
def check_declared_length(request: Request) -> None:
|
|
raw = request.headers.get("Content-Length")
|
|
if raw is None:
|
|
return # chunked: the limit is enforced while reading
|
|
|
|
if not raw.isdigit():
|
|
raise InvalidRequest("Invalid Content-Length.")
|
|
|
|
if int(raw) > settings.max_upload_bytes:
|
|
raise _too_large("File is larger than 10 MiB.")
|
|
|
|
|
|
async def _receive_body(request: Request, spool) -> tuple[int, str]:
|
|
"""Stream the body into the spool; return (size, sha256)."""
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
|
|
try:
|
|
async with asyncio.timeout(settings.upload_timeout_seconds):
|
|
async for chunk in request.stream():
|
|
if not chunk:
|
|
continue
|
|
size += len(chunk)
|
|
if size > settings.max_upload_bytes:
|
|
raise _too_large("File is larger than 10 MiB.")
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
except TimeoutError:
|
|
raise ApiError(
|
|
400,
|
|
"invalid_request",
|
|
"Upload was not completed in time.",
|
|
{"Connection": "close"},
|
|
) from None
|
|
except ClientDisconnect:
|
|
# Nobody will read this answer; nothing was stored anywhere.
|
|
logger.info("upload interrupted by the client after %d bytes", size)
|
|
raise InvalidRequest("Upload was interrupted.") from None
|
|
|
|
return size, digest.hexdigest()
|
|
|
|
|
|
async def _acquire_slot() -> None:
|
|
try:
|
|
async with asyncio.timeout(settings.upload_slot_wait_seconds):
|
|
await upload_slots.acquire()
|
|
except TimeoutError:
|
|
raise RateLimited("Too many concurrent uploads, retry later.", 2) from None
|
|
|
|
|
|
async def _discard_after_failed_put(object_key: str) -> None:
|
|
"""Best effort only; the intent stays for the janitor (late PUT)."""
|
|
try:
|
|
await storage.delete(object_key)
|
|
except StorageUnavailable:
|
|
logger.warning("compensation: S3 delete failed, janitor will retry")
|
|
|
|
|
|
async def _compensate_failed_commit(object_key: str) -> None:
|
|
"""Delete the object only if its upload really did not commit."""
|
|
try:
|
|
async with SessionLocal() as db:
|
|
row = await repositories.lock_upload_intent(db, object_key)
|
|
if row is None:
|
|
# COMMIT succeeded after all (or the janitor finished it).
|
|
await db.rollback()
|
|
return
|
|
if not row["committed"]:
|
|
try:
|
|
await storage.delete(object_key)
|
|
except StorageUnavailable:
|
|
await db.rollback()
|
|
logger.warning("compensation: S3 delete failed, janitor will retry")
|
|
return
|
|
await repositories.delete_upload_intent(db, object_key)
|
|
except Exception as exc:
|
|
logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__)
|
|
|
|
|
|
async def upload_attachment(
|
|
request: Request,
|
|
db: AsyncSession,
|
|
*,
|
|
chat_id: UUID,
|
|
filename: str,
|
|
current_user: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
validate_filename(filename)
|
|
|
|
chat = await repositories.get_accessible_chat(
|
|
db, chat_id=chat_id, user_id=current_user["id"]
|
|
)
|
|
if chat is None:
|
|
raise NotFound()
|
|
# Do not keep a pooled connection idle in a transaction while the
|
|
# client uploads up to 10 MiB.
|
|
await db.rollback()
|
|
|
|
content_type = declared_content_type(request)
|
|
check_declared_length(request)
|
|
|
|
await _acquire_slot()
|
|
try:
|
|
with tempfile.SpooledTemporaryFile(
|
|
max_size=settings.upload_spool_memory_bytes
|
|
) as spool:
|
|
size, sha256 = await _receive_body(request, spool)
|
|
if size == 0:
|
|
raise InvalidRequest("Request body with image bytes is required.")
|
|
|
|
try:
|
|
async with validation_slot:
|
|
validation = asyncio.create_task(asyncio.to_thread(
|
|
validate_image, spool, content_type, settings.max_image_pixels
|
|
))
|
|
try:
|
|
info = await asyncio.shield(validation)
|
|
except asyncio.CancelledError:
|
|
# A thread cannot be killed: keep its spool and the
|
|
# validation slot alive until it has really stopped.
|
|
await asyncio.gather(validation, return_exceptions=True)
|
|
raise
|
|
except TooManyPixels:
|
|
raise PayloadTooLarge("Image has more than 40 million pixels.") from None
|
|
except InvalidImage as exc:
|
|
raise UnsupportedMediaType(str(exc)) from None
|
|
|
|
return await _store(
|
|
db,
|
|
spool=spool,
|
|
chat_id=chat_id,
|
|
owner_id=current_user["id"],
|
|
filename=filename,
|
|
content_type=info.content_type,
|
|
size=size,
|
|
sha256=sha256,
|
|
width=info.width,
|
|
height=info.height,
|
|
)
|
|
finally:
|
|
upload_slots.release()
|
|
|
|
|
|
async def _store(
|
|
db: AsyncSession,
|
|
*,
|
|
spool,
|
|
chat_id: UUID,
|
|
owner_id: UUID,
|
|
filename: str,
|
|
content_type: str,
|
|
size: int,
|
|
sha256: str,
|
|
width: int,
|
|
height: int,
|
|
) -> dict[str, Any]:
|
|
attachment_id = uuid4()
|
|
# Server-generated random key, unrelated to the filename and to the
|
|
# public attachment ID.
|
|
object_key = f"staging/{uuid4().hex}"
|
|
|
|
await repositories.create_upload_intent(
|
|
db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id
|
|
)
|
|
|
|
try:
|
|
await storage.put(object_key, spool, size, content_type)
|
|
except StorageUnavailable as exc:
|
|
logger.warning("S3 PUT failed: %s", exc)
|
|
_in_background(_discard_after_failed_put(object_key))
|
|
raise Unavailable() from None
|
|
|
|
if settings.fault_injection == "crash_after_s3_put":
|
|
logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT")
|
|
os._exit(1)
|
|
|
|
try:
|
|
if settings.fault_injection == "db_error_after_s3_put":
|
|
raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT")
|
|
|
|
row = await repositories.finalize_attachment(
|
|
db,
|
|
attachment_id=attachment_id,
|
|
chat_id=chat_id,
|
|
owner_id=owner_id,
|
|
filename=filename,
|
|
content_type=content_type,
|
|
size_bytes=size,
|
|
sha256=sha256,
|
|
width=width,
|
|
height=height,
|
|
object_key=object_key,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__)
|
|
_in_background(_compensate_failed_commit(object_key))
|
|
raise Unavailable() from None
|
|
|
|
if row is None:
|
|
# The janitor already claimed this intent (upload slower than the
|
|
# grace period): the object is (being) deleted, report a failure.
|
|
logger.warning("upload intent expired before COMMIT")
|
|
raise Unavailable()
|
|
|
|
return _to_attachment(row)
|
|
|
|
|
|
async def get_attachment(
|
|
db: AsyncSession,
|
|
*,
|
|
attachment_id: UUID,
|
|
current_user: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
row = await repositories.get_accessible_attachment(
|
|
db, attachment_id=attachment_id, user_id=current_user["id"]
|
|
)
|
|
if row is None:
|
|
raise NotFound()
|
|
return _to_attachment(row)
|
|
|
|
|
|
async def open_attachment_content(
|
|
db: AsyncSession,
|
|
*,
|
|
attachment_id: UUID,
|
|
variant: str,
|
|
current_user: dict[str, Any],
|
|
) -> tuple[AsyncIterator[bytes], dict[str, str]]:
|
|
"""Authorize in PostgreSQL, then stream the bytes from S3 (no redirect)."""
|
|
row = await repositories.get_accessible_attachment(
|
|
db, attachment_id=attachment_id, user_id=current_user["id"]
|
|
)
|
|
if row is None:
|
|
raise NotFound()
|
|
await db.rollback()
|
|
|
|
if row["status"] != "ready":
|
|
raise Conflict("Attachment content is not ready.")
|
|
|
|
content = row
|
|
if variant != "original":
|
|
content = next((item for item in _variants(row) if item["name"] == variant), None)
|
|
if content is None:
|
|
raise NotFound()
|
|
|
|
try:
|
|
chunks, length = await storage.open_download(content["object_key"])
|
|
except StorageUnavailable as exc:
|
|
logger.warning("S3 GET failed: %s", exc)
|
|
raise Unavailable() from None
|
|
except ObjectMissing:
|
|
# Metadata says "ready" but the bytes are gone: an inconsistency,
|
|
# not a user error (logged as ERROR).
|
|
logger.error("object of attachment %s is missing in S3", attachment_id)
|
|
raise
|
|
|
|
if length != content["size_bytes"]:
|
|
logger.error("object of attachment %s has a wrong size", attachment_id)
|
|
raise ObjectMissing(content["object_key"])
|
|
|
|
headers = {
|
|
"Content-Type": content["content_type"],
|
|
"Content-Length": str(length),
|
|
"Content-Disposition": "inline; filename*=UTF-8''"
|
|
+ quote(row["filename"], safe=""),
|
|
"ETag": f'"{content["sha256"]}"',
|
|
"Cache-Control": "private, no-store",
|
|
"X-Content-Type-Options": "nosniff",
|
|
}
|
|
return chunks, headers
|
|
|
|
|
|
def _in_background(compensation) -> None:
|
|
"""Do not keep the client waiting for compensation: with S3 down each
|
|
attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503
|
|
anyway."""
|
|
task = asyncio.create_task(compensation)
|
|
_compensations.add(task)
|
|
task.add_done_callback(_compensations.discard)
|
|
|
|
|
|
# --- Janitor ----------------------------------------------------------------
|
|
|
|
async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int:
|
|
"""Delete objects of upload intents older than the grace period.
|
|
|
|
One intent per transaction: lock it (SKIP LOCKED), delete the object in
|
|
S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and
|
|
stop: the intent stays and the next run retries.
|
|
"""
|
|
cleaned = 0
|
|
for _ in range(limit):
|
|
async with SessionLocal() as db:
|
|
row = await repositories.lock_stale_intent(db, grace_seconds)
|
|
if row is None:
|
|
await db.rollback()
|
|
break
|
|
|
|
if not row["committed"]:
|
|
try:
|
|
await storage.delete(row["object_key"])
|
|
except StorageUnavailable:
|
|
await db.rollback()
|
|
logger.warning("janitor: S3 unavailable, will retry")
|
|
break
|
|
|
|
await repositories.delete_upload_intent(db, row["object_key"])
|
|
cleaned += 1
|
|
logger.warning(
|
|
"janitor: removed abandoned upload %s (object %s)",
|
|
row["object_key"],
|
|
"kept, row committed" if row["committed"] else "deleted",
|
|
)
|
|
return cleaned
|
|
|
|
|
|
async def janitor_loop() -> None:
|
|
while True:
|
|
# Jitter: replicas do not all wake up at the same moment.
|
|
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
|
|
try:
|
|
await cleanup_stale_uploads(settings.upload_grace_seconds)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning("janitor run failed: %s", type(exc).__name__)
|