forked from mtusi2026/mtusi2026lab5
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались. Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker добавляются в следующих коммитах). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
301 lines
14 KiB
Python
301 lines
14 KiB
Python
"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket.
|
|
|
|
Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order:
|
|
|
|
auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413)
|
|
-> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After)
|
|
-> body in chunks: size limit, read deadline, format + pixel count from the first bytes
|
|
-> full integrity check (images.verify) -> SHA-256 / size known
|
|
-> INSERT row status='pending' with a fresh random object_key, COMMIT (intent)
|
|
-> PUT object to S3 (Content-MD5)
|
|
-> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready.
|
|
|
|
Why 201 is honest: it is sent only after BOTH the object and the ready row are durable.
|
|
What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md):
|
|
|
|
* before the intent COMMIT: nothing (no object was written);
|
|
* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the
|
|
object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the
|
|
object - idempotent, a missing key is fine - and then the row, once it is older than
|
|
PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of
|
|
COMMIT we do not know whether the row is ready, and deleting the object could destroy a
|
|
ready attachment. Only the janitor decides, under a row lock, from the committed state;
|
|
* after the ready COMMIT (answer lost): a correct ready attachment the client does not know.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import datetime as dt
|
|
import hashlib
|
|
import logging
|
|
import secrets
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
|
|
import psycopg
|
|
from psycopg import errors as pg_errors
|
|
from starlette.requests import Request
|
|
|
|
from . import images
|
|
from .config import Settings
|
|
from .db import Database
|
|
from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
|
|
from .storage import OBJECT_PREFIX, Storage
|
|
from .timefmt import rfc3339
|
|
|
|
log = logging.getLogger("messenger.attachments")
|
|
|
|
JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Attachment:
|
|
id: str
|
|
chat_id: str
|
|
owner_id: str
|
|
filename: str
|
|
content_type: str
|
|
size_bytes: int
|
|
sha256: str
|
|
object_key: str
|
|
status: str
|
|
created_at: dt.datetime
|
|
updated_at: dt.datetime
|
|
lost: bool = False
|
|
|
|
def to_json(self) -> dict:
|
|
"""Public representation: no object key, no bucket, no internal state."""
|
|
return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename,
|
|
"content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256,
|
|
"status": "ready", "variants": [], "error_code": None,
|
|
"created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)}
|
|
|
|
|
|
COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, "
|
|
"a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL")
|
|
|
|
|
|
def _attachment(row) -> Attachment:
|
|
return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7],
|
|
row[8], row[9], row[10], row[11])
|
|
|
|
|
|
def new_object_key() -> str:
|
|
"""Server-generated, 128 random bits, unrelated to the attachment id and the filename."""
|
|
return OBJECT_PREFIX + secrets.token_hex(16)
|
|
|
|
|
|
# ------------------------------------------------------------------ SQL
|
|
async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str,
|
|
info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None:
|
|
async def work(conn: psycopg.AsyncConnection) -> None:
|
|
await conn.execute(
|
|
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
|
"width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')",
|
|
(attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height,
|
|
object_key))
|
|
try:
|
|
await db.run(work, transaction=False)
|
|
except pg_errors.ForeignKeyViolation:
|
|
raise NotFound("Чат не найден") from None
|
|
|
|
|
|
async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None:
|
|
"""pending -> ready. None if the row is no longer pending (the janitor removed a stale intent)."""
|
|
async def work(conn: psycopg.AsyncConnection) -> Attachment | None:
|
|
cur = await conn.execute(
|
|
"UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() "
|
|
f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,))
|
|
row = await cur.fetchone()
|
|
return _attachment(row) if row else None
|
|
return await db.run(work, transaction=False)
|
|
|
|
|
|
async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment:
|
|
"""A READY attachment of a chat the user is a member of; anything else is 404 (unknown,
|
|
foreign chat, unfinished upload) - existence of foreign objects is never revealed."""
|
|
async def work(conn: psycopg.AsyncConnection) -> Attachment:
|
|
cur = await conn.execute(
|
|
f"SELECT {COLUMNS} FROM attachments a "
|
|
"JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s "
|
|
"WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id))
|
|
row = await cur.fetchone()
|
|
if row is None:
|
|
raise NotFound("Вложение не найдено")
|
|
return _attachment(row)
|
|
return await db.run(work, transaction=False)
|
|
|
|
|
|
async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int:
|
|
"""Janitor round: remove intents of uploads that never became ready.
|
|
|
|
Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in
|
|
mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object.
|
|
Order: object first (idempotent), row second; a crash in between leaves the pending row and
|
|
the next round repeats. Only status='pending' rows older than the grace period are touched:
|
|
fresh uploads and ready/used attachments are never selected.
|
|
"""
|
|
async def work(conn: psycopg.AsyncConnection) -> int:
|
|
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,))
|
|
if not (await cur.fetchone())[0]:
|
|
return 0
|
|
cur = await conn.execute(
|
|
"SELECT id, object_key FROM attachments WHERE status = 'pending' "
|
|
"AND created_at < clock_timestamp() - make_interval(secs => %s) "
|
|
"ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch))
|
|
rows = await cur.fetchall()
|
|
if not rows:
|
|
return 0
|
|
await asyncio.gather(*(storage.delete(key) for _, key in rows))
|
|
await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'",
|
|
([r[0] for r in rows],))
|
|
return len(rows)
|
|
return await db.run(work, deadline=storage.timeout + 5)
|
|
|
|
|
|
# ------------------------------------------------------------------ request body
|
|
class Busy(Unavailable):
|
|
"""All upload slots of this replica are taken: retry shortly (no body was read)."""
|
|
|
|
@property
|
|
def headers(self) -> dict[str, str]:
|
|
return {"Retry-After": "1"}
|
|
|
|
|
|
@dataclass
|
|
class Upload:
|
|
data: bytearray
|
|
info: images.ImageInfo
|
|
sha256: bytes
|
|
|
|
|
|
class BodyReader:
|
|
"""Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it
|
|
is consumed and can discard the rest after an early rejection."""
|
|
|
|
def __init__(self, request: Request):
|
|
self.receive = request.receive
|
|
self.done = False
|
|
|
|
async def chunk(self, timeout: float) -> bytes | None:
|
|
if self.done:
|
|
return None
|
|
message = await asyncio.wait_for(self.receive(), timeout)
|
|
if message["type"] == "http.disconnect":
|
|
self.done = True
|
|
raise InvalidRequest("Клиент прервал загрузку")
|
|
if not message.get("more_body", False):
|
|
self.done = True
|
|
return message.get("body", b"")
|
|
|
|
async def discard(self, limit: int, timeout: float) -> None:
|
|
"""After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that
|
|
writes the whole body before reading the answer still gets our JSON error instead of a
|
|
connection reset. Nothing is kept in memory; beyond the limit the connection is dropped."""
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + timeout
|
|
seen = 0
|
|
try:
|
|
while not self.done and seen <= limit:
|
|
remaining = deadline - loop.time()
|
|
if remaining <= 0:
|
|
return
|
|
seen += len(await self.chunk(remaining) or b"")
|
|
except (ApiError, asyncio.TimeoutError):
|
|
return
|
|
|
|
|
|
def declared_type(request: Request) -> str:
|
|
"""Content-Type of the upload must be image/png or image/jpeg (parameters ignored)."""
|
|
media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower()
|
|
if media_type not in (images.PNG, images.JPEG):
|
|
raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg")
|
|
return media_type
|
|
|
|
|
|
def declared_length(request: Request, limit: int) -> int | None:
|
|
raw = request.headers.get("content-length")
|
|
if raw is None:
|
|
return None # chunked transfer: the limit is enforced while reading
|
|
if not raw.isdigit():
|
|
raise InvalidRequest("Неверный Content-Length")
|
|
length = int(raw)
|
|
if length > limit:
|
|
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
|
if length == 0:
|
|
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
|
return length
|
|
|
|
|
|
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
|
|
length: int | None) -> Upload:
|
|
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
|
|
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
|
|
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
|
|
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
|
|
reading/decoding the rest."""
|
|
loop = asyncio.get_running_loop()
|
|
deadline = loop.time() + settings.upload_read_timeout_seconds
|
|
limit = settings.max_upload_bytes
|
|
buffer = bytearray(length) if length else bytearray()
|
|
view = memoryview(buffer) if length else None
|
|
filled = 0
|
|
digest = hashlib.sha256()
|
|
info: images.ImageInfo | None = None
|
|
while True:
|
|
remaining = deadline - loop.time()
|
|
if remaining <= 0:
|
|
raise InvalidRequest("Тело запроса не получено вовремя")
|
|
try:
|
|
chunk = await reader.chunk(remaining)
|
|
except asyncio.TimeoutError:
|
|
raise InvalidRequest("Тело запроса не получено вовремя") from None
|
|
if chunk is None:
|
|
break
|
|
if filled + len(chunk) > (length or limit):
|
|
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
|
if view is not None:
|
|
view[filled:filled + len(chunk)] = chunk
|
|
else:
|
|
buffer += chunk
|
|
filled += len(chunk)
|
|
digest.update(chunk)
|
|
if info is None and filled:
|
|
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
|
|
if view is not None:
|
|
view.release()
|
|
if not filled:
|
|
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
|
if length and filled != length:
|
|
raise InvalidRequest("Тело запроса короче Content-Length")
|
|
if info is None:
|
|
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
|
|
try:
|
|
async with verify_slots:
|
|
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
|
|
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
|
|
except images.ImageError as exc:
|
|
raise _image_error(exc) from None
|
|
except asyncio.TimeoutError:
|
|
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
|
|
except (images.VerifierUnavailable, MemoryError):
|
|
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
|
|
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
|
|
return Upload(buffer, info, digest.digest())
|
|
|
|
|
|
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
|
|
try:
|
|
info = images.sniff(buffer, settings.max_image_pixels)
|
|
except images.ImageError as exc:
|
|
raise _image_error(exc) from None
|
|
if info is not None and info.content_type != declared:
|
|
# "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa).
|
|
raise UnsupportedMediaType("Содержимое не совпадает с Content-Type")
|
|
return info
|
|
|
|
|
|
def _image_error(exc: images.ImageError) -> ApiError:
|
|
return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)
|
|
|