Files
mtusi2026lab6/app/attachments.py
T

468 lines
16 KiB
Python

"""Attachments: durable upload to staging, queued metadata + outbox (lab5).
Upload, in this order (all checks before the bytes are stored):
1. filename (400) -> chat membership (404) -> Content-Type header (415)
-> Content-Length (413 without reading the body);
2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429;
3. the body is read in chunks into a SpooledTemporaryFile (only
UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more
than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400;
4. content validation in a thread (app.images): format by magic bytes,
pixel limit from the header (413), integrity (415);
5. upload intent COMMIT -> staging S3 PUT -> one transaction "delete intent +
insert queued attachment + outbox" -> 202. Worker produces all variants.
Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs
in the background):
- PUT failed or timed out: the outcome is unknown - S3 may still finish
writing the object AFTER our DELETE. So the object is deleted best
effort, but the intent is KEPT; the janitor deletes the object again
after UPLOAD_GRACE_SECONDS and only then the intent;
- COMMIT failed: the outcome is unknown too - COMMIT may have succeeded
and only the answer was lost. Under the intent's row lock: no intent ->
the row is committed, keep the object; else delete object, then intent;
- the process died after PUT: the intent stays; the janitor deletes the
object and the intent after UPLOAD_GRACE_SECONDS.
So an object without a row exists only while its intent exists, and an
object of a committed row is never deleted.
"""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import random
import tempfile
import unicodedata
from collections.abc import AsyncIterator
from typing import Any
from urllib.parse import quote
from uuid import UUID, uuid4
from fastapi import Request
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import ClientDisconnect
from app import repositories
from app.config import settings
from app.db import SessionLocal
from app.errors import (
ApiError,
Conflict,
InvalidRequest,
NotFound,
PayloadTooLarge,
RateLimited,
Unavailable,
UnsupportedMediaType,
)
from app.images import InvalidImage, TooManyPixels, validate_image
from app.storage import ObjectMissing, StorageUnavailable, storage
logger = logging.getLogger("messenger.attachments")
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
# Characters that can turn a display name into a path or hide its real
# extension: separators, NUL/control characters, bidi overrides etc.
FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
# Full integrity checks are CPU work too. In particular, progressive JPEG
# retains decoder coefficients even with draft(); never run eight at once.
validation_slot = asyncio.Semaphore(1)
# Strong references to background compensation tasks (asyncio keeps only
# weak ones); if the process dies first, the intent stays for the janitor.
_compensations: set[asyncio.Task] = set()
def _to_attachment(row: dict[str, Any]) -> dict[str, Any]:
return {
"id": row["id"],
"chat_id": row["chat_id"],
"owner_id": row["owner_id"],
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [
{key: value for key, value in variant.items() if key != "object_key"}
for variant in _variants(row)
],
"error_code": row["error_code"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
def _variants(row: dict[str, Any]) -> list[dict[str, Any]]:
value = row.get("variants") or []
return json.loads(value) if isinstance(value, str) else value
def validate_filename(filename: str) -> str:
if not 1 <= len(filename) <= 255 or not filename.strip():
raise InvalidRequest("filename must be 1-255 characters, not blank.")
if "/" in filename or "\\" in filename:
raise InvalidRequest("filename must not contain path separators.")
if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename):
raise InvalidRequest("filename must not contain control characters.")
if filename.strip(". ") == "":
raise InvalidRequest("filename must not be a relative path.")
return filename
def declared_content_type(request: Request) -> str:
raw = request.headers.get("Content-Type", "")
media_type = raw.split(";", 1)[0].strip().lower()
if media_type not in ALLOWED_CONTENT_TYPES:
raise UnsupportedMediaType(
"Content-Type must be image/png or image/jpeg (raw bytes, not multipart)."
)
return media_type
def _too_large(message: str) -> PayloadTooLarge:
error = PayloadTooLarge(message)
# The rest of the body is not read: the connection cannot be reused.
error.headers["Connection"] = "close"
return error
def check_declared_length(request: Request) -> None:
raw = request.headers.get("Content-Length")
if raw is None:
return # chunked: the limit is enforced while reading
if not raw.isdigit():
raise InvalidRequest("Invalid Content-Length.")
if int(raw) > settings.max_upload_bytes:
raise _too_large("File is larger than 10 MiB.")
async def _receive_body(request: Request, spool) -> tuple[int, str]:
"""Stream the body into the spool; return (size, sha256)."""
digest = hashlib.sha256()
size = 0
try:
async with asyncio.timeout(settings.upload_timeout_seconds):
async for chunk in request.stream():
if not chunk:
continue
size += len(chunk)
if size > settings.max_upload_bytes:
raise _too_large("File is larger than 10 MiB.")
digest.update(chunk)
spool.write(chunk)
except TimeoutError:
raise ApiError(
400,
"invalid_request",
"Upload was not completed in time.",
{"Connection": "close"},
) from None
except ClientDisconnect:
# Nobody will read this answer; nothing was stored anywhere.
logger.info("upload interrupted by the client after %d bytes", size)
raise InvalidRequest("Upload was interrupted.") from None
return size, digest.hexdigest()
async def _acquire_slot() -> None:
try:
async with asyncio.timeout(settings.upload_slot_wait_seconds):
await upload_slots.acquire()
except TimeoutError:
raise RateLimited("Too many concurrent uploads, retry later.", 2) from None
async def _discard_after_failed_put(object_key: str) -> None:
"""Best effort only; the intent stays for the janitor (late PUT)."""
try:
await storage.delete(object_key)
except StorageUnavailable:
logger.warning("compensation: S3 delete failed, janitor will retry")
async def _compensate_failed_commit(object_key: str) -> None:
"""Delete the object only if its upload really did not commit."""
try:
async with SessionLocal() as db:
row = await repositories.lock_upload_intent(db, object_key)
if row is None:
# COMMIT succeeded after all (or the janitor finished it).
await db.rollback()
return
if not row["committed"]:
try:
await storage.delete(object_key)
except StorageUnavailable:
await db.rollback()
logger.warning("compensation: S3 delete failed, janitor will retry")
return
await repositories.delete_upload_intent(db, object_key)
except Exception as exc:
logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__)
async def upload_attachment(
request: Request,
db: AsyncSession,
*,
chat_id: UUID,
filename: str,
current_user: dict[str, Any],
) -> dict[str, Any]:
validate_filename(filename)
chat = await repositories.get_accessible_chat(
db, chat_id=chat_id, user_id=current_user["id"]
)
if chat is None:
raise NotFound()
# Do not keep a pooled connection idle in a transaction while the
# client uploads up to 10 MiB.
await db.rollback()
content_type = declared_content_type(request)
check_declared_length(request)
await _acquire_slot()
try:
with tempfile.SpooledTemporaryFile(
max_size=settings.upload_spool_memory_bytes
) as spool:
size, sha256 = await _receive_body(request, spool)
if size == 0:
raise InvalidRequest("Request body with image bytes is required.")
try:
async with validation_slot:
validation = asyncio.create_task(asyncio.to_thread(
validate_image, spool, content_type, settings.max_image_pixels
))
try:
info = await asyncio.shield(validation)
except asyncio.CancelledError:
# A thread cannot be killed: keep its spool and the
# validation slot alive until it has really stopped.
await asyncio.gather(validation, return_exceptions=True)
raise
except TooManyPixels:
raise PayloadTooLarge("Image has more than 40 million pixels.") from None
except InvalidImage as exc:
raise UnsupportedMediaType(str(exc)) from None
return await _store(
db,
spool=spool,
chat_id=chat_id,
owner_id=current_user["id"],
filename=filename,
content_type=info.content_type,
size=size,
sha256=sha256,
width=info.width,
height=info.height,
)
finally:
upload_slots.release()
async def _store(
db: AsyncSession,
*,
spool,
chat_id: UUID,
owner_id: UUID,
filename: str,
content_type: str,
size: int,
sha256: str,
width: int,
height: int,
) -> dict[str, Any]:
attachment_id = uuid4()
# Server-generated random key, unrelated to the filename and to the
# public attachment ID.
object_key = f"staging/{uuid4().hex}"
await repositories.create_upload_intent(
db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id
)
try:
await storage.put(object_key, spool, size, content_type)
except StorageUnavailable as exc:
logger.warning("S3 PUT failed: %s", exc)
_in_background(_discard_after_failed_put(object_key))
raise Unavailable() from None
if settings.fault_injection == "crash_after_s3_put":
logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT")
os._exit(1)
try:
if settings.fault_injection == "db_error_after_s3_put":
raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT")
row = await repositories.finalize_attachment(
db,
attachment_id=attachment_id,
chat_id=chat_id,
owner_id=owner_id,
filename=filename,
content_type=content_type,
size_bytes=size,
sha256=sha256,
width=width,
height=height,
object_key=object_key,
)
except Exception as exc:
logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__)
_in_background(_compensate_failed_commit(object_key))
raise Unavailable() from None
if row is None:
# The janitor already claimed this intent (upload slower than the
# grace period): the object is (being) deleted, report a failure.
logger.warning("upload intent expired before COMMIT")
raise Unavailable()
return _to_attachment(row)
async def get_attachment(
db: AsyncSession,
*,
attachment_id: UUID,
current_user: dict[str, Any],
) -> dict[str, Any]:
row = await repositories.get_accessible_attachment(
db, attachment_id=attachment_id, user_id=current_user["id"]
)
if row is None:
raise NotFound()
return _to_attachment(row)
async def open_attachment_content(
db: AsyncSession,
*,
attachment_id: UUID,
variant: str,
current_user: dict[str, Any],
) -> tuple[AsyncIterator[bytes], dict[str, str]]:
"""Authorize in PostgreSQL, then stream the bytes from S3 (no redirect)."""
row = await repositories.get_accessible_attachment(
db, attachment_id=attachment_id, user_id=current_user["id"]
)
if row is None:
raise NotFound()
await db.rollback()
if row["status"] != "ready":
raise Conflict("Attachment content is not ready.")
content = row
if variant != "original":
content = next((item for item in _variants(row) if item["name"] == variant), None)
if content is None:
raise NotFound()
try:
chunks, length = await storage.open_download(content["object_key"])
except StorageUnavailable as exc:
logger.warning("S3 GET failed: %s", exc)
raise Unavailable() from None
except ObjectMissing:
# Metadata says "ready" but the bytes are gone: an inconsistency,
# not a user error (logged as ERROR).
logger.error("object of attachment %s is missing in S3", attachment_id)
raise
if length != content["size_bytes"]:
logger.error("object of attachment %s has a wrong size", attachment_id)
raise ObjectMissing(content["object_key"])
headers = {
"Content-Type": content["content_type"],
"Content-Length": str(length),
"Content-Disposition": "inline; filename*=UTF-8''"
+ quote(row["filename"], safe=""),
"ETag": f'"{content["sha256"]}"',
"Cache-Control": "private, no-store",
"X-Content-Type-Options": "nosniff",
}
return chunks, headers
def _in_background(compensation) -> None:
"""Do not keep the client waiting for compensation: with S3 down each
attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503
anyway."""
task = asyncio.create_task(compensation)
_compensations.add(task)
task.add_done_callback(_compensations.discard)
# --- Janitor ----------------------------------------------------------------
async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int:
"""Delete objects of upload intents older than the grace period.
One intent per transaction: lock it (SKIP LOCKED), delete the object in
S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and
stop: the intent stays and the next run retries.
"""
cleaned = 0
for _ in range(limit):
async with SessionLocal() as db:
row = await repositories.lock_stale_intent(db, grace_seconds)
if row is None:
await db.rollback()
break
if not row["committed"]:
try:
await storage.delete(row["object_key"])
except StorageUnavailable:
await db.rollback()
logger.warning("janitor: S3 unavailable, will retry")
break
await repositories.delete_upload_intent(db, row["object_key"])
cleaned += 1
logger.warning(
"janitor: removed abandoned upload %s (object %s)",
row["object_key"],
"kept, row committed" if row["committed"] else "deleted",
)
return cleaned
async def janitor_loop() -> None:
while True:
# Jitter: replicas do not all wake up at the same moment.
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
try:
await cleanup_stale_uploads(settings.upload_grace_seconds)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning("janitor run failed: %s", type(exc).__name__)