"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket. Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order: auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413) -> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After) -> body in chunks: size limit, read deadline, format + pixel count from the first bytes -> full integrity check (images.verify) -> SHA-256 / size known -> INSERT row status='pending' with a fresh random object_key, COMMIT (intent) -> PUT object to S3 (Content-MD5) -> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready. Why 201 is honest: it is sent only after BOTH the object and the ready row are durable. What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md): * before the intent COMMIT: nothing (no object was written); * after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the object - idempotent, a missing key is fine - and then the row, once it is older than PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of COMMIT we do not know whether the row is ready, and deleting the object could destroy a ready attachment. Only the janitor decides, under a row lock, from the committed state; * after the ready COMMIT (answer lost): a correct ready attachment the client does not know. """ from __future__ import annotations import asyncio import datetime as dt import hashlib import logging import secrets import uuid from dataclasses import dataclass import psycopg from psycopg import errors as pg_errors from starlette.requests import Request from . import images from .config import Settings from .db import Database from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType from .storage import OBJECT_PREFIX, Storage from .timefmt import rfc3339 log = logging.getLogger("messenger.attachments") JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas @dataclass(frozen=True, slots=True) class Attachment: id: str chat_id: str owner_id: str filename: str content_type: str size_bytes: int sha256: str object_key: str status: str created_at: dt.datetime updated_at: dt.datetime lost: bool = False def to_json(self) -> dict: """Public representation: no object key, no bucket, no internal state.""" return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename, "content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256, "status": "ready", "variants": [], "error_code": None, "created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)} COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, " "a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL") def _attachment(row) -> Attachment: return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7], row[8], row[9], row[10], row[11]) def new_object_key() -> str: """Server-generated, 128 random bits, unrelated to the attachment id and the filename.""" return OBJECT_PREFIX + secrets.token_hex(16) # ------------------------------------------------------------------ SQL async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str, info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None: async def work(conn: psycopg.AsyncConnection) -> None: await conn.execute( "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " "width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')", (attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height, object_key)) try: await db.run(work, transaction=False) except pg_errors.ForeignKeyViolation: raise NotFound("Чат не найден") from None async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None: """pending -> ready. None if the row is no longer pending (the janitor removed a stale intent).""" async def work(conn: psycopg.AsyncConnection) -> Attachment | None: cur = await conn.execute( "UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() " f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,)) row = await cur.fetchone() return _attachment(row) if row else None return await db.run(work, transaction=False) async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment: """A READY attachment of a chat the user is a member of; anything else is 404 (unknown, foreign chat, unfinished upload) - existence of foreign objects is never revealed.""" async def work(conn: psycopg.AsyncConnection) -> Attachment: cur = await conn.execute( f"SELECT {COLUMNS} FROM attachments a " "JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s " "WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id)) row = await cur.fetchone() if row is None: raise NotFound("Вложение не найдено") return _attachment(row) return await db.run(work, transaction=False) async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int: """Janitor round: remove intents of uploads that never became ready. Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object. Order: object first (idempotent), row second; a crash in between leaves the pending row and the next round repeats. Only status='pending' rows older than the grace period are touched: fresh uploads and ready/used attachments are never selected. """ async def work(conn: psycopg.AsyncConnection) -> int: cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,)) if not (await cur.fetchone())[0]: return 0 cur = await conn.execute( "SELECT id, object_key FROM attachments WHERE status = 'pending' " "AND created_at < clock_timestamp() - make_interval(secs => %s) " "ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch)) rows = await cur.fetchall() if not rows: return 0 await asyncio.gather(*(storage.delete(key) for _, key in rows)) await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'", ([r[0] for r in rows],)) return len(rows) return await db.run(work, deadline=storage.timeout + 5) # ------------------------------------------------------------------ request body class Busy(Unavailable): """All upload slots of this replica are taken: retry shortly (no body was read).""" @property def headers(self) -> dict[str, str]: return {"Retry-After": "1"} @dataclass class Upload: data: bytearray info: images.ImageInfo sha256: bytes class BodyReader: """Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it is consumed and can discard the rest after an early rejection.""" def __init__(self, request: Request): self.receive = request.receive self.done = False async def chunk(self, timeout: float) -> bytes | None: if self.done: return None message = await asyncio.wait_for(self.receive(), timeout) if message["type"] == "http.disconnect": self.done = True raise InvalidRequest("Клиент прервал загрузку") if not message.get("more_body", False): self.done = True return message.get("body", b"") async def discard(self, limit: int, timeout: float) -> None: """After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that writes the whole body before reading the answer still gets our JSON error instead of a connection reset. Nothing is kept in memory; beyond the limit the connection is dropped.""" loop = asyncio.get_running_loop() deadline = loop.time() + timeout seen = 0 try: while not self.done and seen <= limit: remaining = deadline - loop.time() if remaining <= 0: return seen += len(await self.chunk(remaining) or b"") except (ApiError, asyncio.TimeoutError): return def declared_type(request: Request) -> str: """Content-Type of the upload must be image/png or image/jpeg (parameters ignored).""" media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower() if media_type not in (images.PNG, images.JPEG): raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg") return media_type def declared_length(request: Request, limit: int) -> int | None: raw = request.headers.get("content-length") if raw is None: return None # chunked transfer: the limit is enforced while reading if not raw.isdigit(): raise InvalidRequest("Неверный Content-Length") length = int(raw) if length > limit: raise PayloadTooLarge(f"Файл больше {limit} байт") if length == 0: raise InvalidRequest("Тело запроса пустое: нужны байты изображения") return length async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore, length: int | None) -> Upload: """Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly Content-Length bytes allocated once when the length is declared (no growth, no final copy); it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without reading/decoding the rest.""" loop = asyncio.get_running_loop() deadline = loop.time() + settings.upload_read_timeout_seconds limit = settings.max_upload_bytes buffer = bytearray(length) if length else bytearray() view = memoryview(buffer) if length else None filled = 0 digest = hashlib.sha256() info: images.ImageInfo | None = None while True: remaining = deadline - loop.time() if remaining <= 0: raise InvalidRequest("Тело запроса не получено вовремя") try: chunk = await reader.chunk(remaining) except asyncio.TimeoutError: raise InvalidRequest("Тело запроса не получено вовремя") from None if chunk is None: break if filled + len(chunk) > (length or limit): raise PayloadTooLarge(f"Файл больше {limit} байт") if view is not None: view[filled:filled + len(chunk)] = chunk else: buffer += chunk filled += len(chunk) digest.update(chunk) if info is None and filled: info = _sniff(view[:filled] if view is not None else buffer, settings, declared) if view is not None: view.release() if not filled: raise InvalidRequest("Тело запроса пустое: нужны байты изображения") if length and filled != length: raise InvalidRequest("Тело запроса короче Content-Length") if info is None: raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG") try: async with verify_slots: await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds, jpeg_max_memory_mb=settings.jpeg_max_memory_mb) except images.ImageError as exc: raise _image_error(exc) from None except asyncio.TimeoutError: raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None except (images.VerifierUnavailable, MemoryError): log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}}) raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None return Upload(buffer, info, digest.digest()) def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None: try: info = images.sniff(buffer, settings.max_image_pixels) except images.ImageError as exc: raise _image_error(exc) from None if info is not None and info.content_type != declared: # "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa). raise UnsupportedMediaType("Содержимое не совпадает с Content-Type") return info def _image_error(exc: images.ImageError) -> ApiError: return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)