diff --git a/src/app/api/attachments.py b/src/app/api/attachments.py index cef7f9d3..088e66a1 100644 --- a/src/app/api/attachments.py +++ b/src/app/api/attachments.py @@ -1,4 +1,4 @@ -"""Вложения (lab4): синхронный binary upload PNG/JPEG, авторизованный download.""" +"""Вложения (lab5): upload → 202 queued, async worker, download вариантов.""" import hashlib import io import os @@ -11,6 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import Response from PIL import Image, UnidentifiedImageError from starlette.concurrency import run_in_threadpool +from botocore.exceptions import ClientError from ..models.schemas import Attachment, User from ..storage import s3 @@ -22,6 +23,7 @@ router = APIRouter(prefix="/api/v1", tags=["attachments"]) MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +PIPELINE_VERSION = 1 def _require_member(chat_id: UUID, user_id: UUID): @@ -52,11 +54,7 @@ def _detect_content_type(data: bytes) -> Optional[str]: def _validate_image(data: bytes) -> str: - """MIME по содержимому + лимит пикселей + полная проверка декодирования. - - Полное декодирование (load) выполняется через Pillow, чтобы отвергнуть - повреждённые/обрезанные файлы, которые open() принимает по заголовку. - """ + """MIME по содержимому + лимит пикселей + полная проверка декодирования.""" ctype = _detect_content_type(data) if ctype is None: raise HTTPException(status_code=415, detail="unsupported_media_type") @@ -67,14 +65,10 @@ def _validate_image(data: bytes) -> str: raise HTTPException(status_code=415, detail="unsupported_media_type") if width * height > MAX_IMAGE_PIXELS: raise HTTPException(status_code=413, detail="payload_too_large") - # Полное декодирование всех данных: битый IDAT / обрезанный файл - # поднимут OSError/ValueError и будут отвергнуты как 415. img.load() except HTTPException: raise except Image.DecompressionBombError: - # Pillow защищается от «бомб» по собственному порогу: для нас - # это превышение лимита пикселей → 413. raise HTTPException(status_code=413, detail="payload_too_large") except Exception: raise HTTPException(status_code=415, detail="unsupported_media_type") @@ -83,9 +77,9 @@ def _validate_image(data: bytes) -> str: @router.post( path="/chats/{chat_id}/attachments", - summary="Принять один PNG/JPEG, до 10 MiB", + summary="Принять один PNG/JPEG, до 10 MiB (lab5: 202 queued)", response_model=Attachment, - status_code=201, + status_code=202, ) async def upload_attachment( chat_id: UUID, @@ -96,7 +90,6 @@ async def upload_attachment( _require_member(chat_id, user.id) safe_name = _validate_filename(filename) - # Content-Length — быстрая отсечка; не доверяем полностью. cl = request.headers.get("content-length") if cl is not None: try: @@ -105,7 +98,6 @@ async def upload_attachment( except ValueError: raise HTTPException(status_code=400, detail="invalid_request") - # Потоковое чтение с жёстким лимитом. chunks: list[bytes] = [] total = 0 async for chunk in request.stream(): @@ -119,16 +111,15 @@ async def upload_attachment( if not body: raise HTTPException(status_code=400, detail="invalid_request") - # Pillow — в threadpool, чтобы не блокировать event loop. content_type = await run_in_threadpool(_validate_image, body) sha256 = hashlib.sha256(body).hexdigest() attachment_id = uuid4() - object_key = s3.original_key(chat_id, attachment_id) + staging_key = s3.staging_key(chat_id, attachment_id) - # 1) Сначала сохраняем в S3. Если следующий шаг упадёт — компенсируем. + # 1) Durable staging в S3 (приватный prefix). try: - await run_in_threadpool(s3.put_bytes, object_key, body, content_type) + await run_in_threadpool(s3.put_bytes, staging_key, body, content_type) except Exception as exc: # noqa: BLE001 raise HTTPException(status_code=503, detail="unavailable") from exc @@ -141,19 +132,31 @@ async def upload_attachment( content_type=content_type, size_bytes=len(body), sha256=sha256, - status="ready", + status="queued", variants=[], error_code=None, created_at=now, updated_at=now, ) - # 2) INSERT в БД. При ЛЮБОМ исключении удаляем S3-объект (компенсация). + job_id = uuid4() + outbox_payload = { + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": PIPELINE_VERSION, + "created_at": now.isoformat().replace("+00:00", "Z"), + } + + # 2) INSERT attachment + outbox в ОДНОЙ транзакции. + # При сбое — компенсация: удаляем staging. try: - await run_in_threadpool(storage.create_attachment, att, object_key) + await run_in_threadpool( + storage.create_attachment_queued, att, staging_key, outbox_payload + ) except Exception: try: - await run_in_threadpool(s3.delete_object, object_key) + await run_in_threadpool(s3.delete_object, staging_key) except Exception: pass raise HTTPException(status_code=503, detail="unavailable") @@ -180,15 +183,16 @@ def get_attachment( @router.get( path="/attachments/{attachment_id}/content", - summary="Скачать байты через авторизованный API", + summary="Скачать байты (original или variant) через авторизованный API", ) def download_attachment( attachment_id: UUID, user: User = Depends(current_user), variant: str = Query("original"), ): - if variant != "original": - raise HTTPException(status_code=404, detail="not_found") + if variant not in ("original", "thumbnail", "crop", "optimized"): + raise HTTPException(status_code=400, detail="invalid_request") + pair = storage.get_attachment_with_key(attachment_id) if pair is None: raise HTTPException(status_code=404, detail="not_found") @@ -197,16 +201,28 @@ def download_attachment( if att.status != "ready": raise HTTPException(status_code=409, detail="conflict") - try: - data, ctype = s3.get_object(object_key) - except Exception as exc: # noqa: BLE001 - # Объект отсутствует при готовых метаданных: 404. - if exc.__class__.__name__ == "NoSuchKey": + if variant == "original": + key = object_key + ctype_expected = att.content_type + else: + vs = [v for v in att.variants if v.name == variant] + if not vs: raise HTTPException(status_code=404, detail="not_found") + v = vs[0] + key = s3.variant_key(att.chat_id, att.id, variant) + ctype_expected = v.content_type + + try: + data, ctype = s3.get_object(key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in ("404", "NoSuchKey", "NotFound"): + raise HTTPException(status_code=404, detail="not_found") + raise HTTPException(status_code=503, detail="unavailable") from exc + except Exception as exc: # noqa: BLE001 raise HTTPException(status_code=503, detail="unavailable") from exc return Response( content=data, - media_type=ctype, - headers={"Content-Length": str(len(data))}, + media_type=ctype or ctype_expected, ) \ No newline at end of file diff --git a/src/app/storage/postgres.py b/src/app/storage/postgres.py index a7b0ce22..2bc9519f 100644 --- a/src/app/storage/postgres.py +++ b/src/app/storage/postgres.py @@ -1,8 +1,9 @@ import hashlib import hmac +import json as _json import secrets import threading -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import List, Optional from uuid import UUID, uuid4 @@ -70,13 +71,14 @@ def _row_to_message(row, attachment_ids: List[UUID]) -> Message: ) -def _row_to_attachment(row) -> Attachment: +def _row_to_attachment(row, variants: Optional[list] = None) -> Attachment: # row: id, chat_id, owner_id, filename, content_type, size_bytes, sha256, # status, error_code, created_at, updated_at return Attachment( id=row[0], chat_id=row[1], owner_id=row[2], filename=row[3], content_type=row[4], size_bytes=row[5], - sha256=row[6], status=row[7], variants=[], error_code=row[8], + sha256=row[6], status=row[7], variants=variants or [], + error_code=row[8], created_at=row[9], updated_at=row[10], ) @@ -91,8 +93,6 @@ class PostgresStorage: def create_user(self, username: str, display_name: str, password: str) -> User: with self._lock, connection() as conn: with conn.cursor() as cur: - # SELECT-проверка + INSERT в одной транзакции; уникальный индекс - # users.username отсекает гонку на уровне БД (UniqueViolation). cur.execute("SELECT 1 FROM users WHERE username = %s", (username,)) if cur.fetchone() is not None: raise ValueError("username_taken") @@ -108,8 +108,6 @@ class PostgresStorage: _hash_password(password), user.created_at), ) except Exception as exc: - # Параллельная регистрация того же username: уникальный индекс - # выбросил UniqueViolation — превращаем в доменную ошибку. if exc.__class__.__name__ == "UniqueViolation": raise ValueError("username_taken") raise @@ -148,9 +146,7 @@ class PostgresStorage: return None return _row_to_user(row[:4]) - # ---------- Сессии (Redis или PostgreSQL — см. storage/sessions.py) ---------- - # Тонкие делегаты: роутеры продолжают вызывать storage.create_session(...), - # storage.get_session_user(...), storage.revoke_session(...) без изменений. + # ---------- Сессии ---------- def create_session(self, user_id: UUID, ttl_seconds: int) -> tuple: return _sess_create(user_id, ttl_seconds) @@ -221,10 +217,7 @@ class PostgresStorage: return chats # ---------- Сообщения ---------- - def _load_message_attachments( - self, cur, message_ids: List[UUID] - ) -> dict[UUID, List[UUID]]: - """Одним запросом достаём attachment_ids для набора сообщений.""" + def _load_message_attachments(self, cur, message_ids: List[UUID]) -> dict[UUID, List[UUID]]: if not message_ids: return {} cur.execute( @@ -238,17 +231,8 @@ class PostgresStorage: return result def list_messages_page( - self, - chat_id: UUID, - limit: int, - after: Optional[tuple] = None, + self, chat_id: UUID, limit: int, after: Optional[tuple] = None, ) -> List[Message]: - """Keyset-пагинация в SQL - - after = (created_at, id) последней записи предыдущей страницы или None. - Запрос использует индекс idx_messages_chat_created (chat_id, created_at, id) - и не делает полный scan всей истории на каждую страницу. - """ with connection() as conn, conn.cursor() as cur: if after is None: cur.execute( @@ -282,25 +266,8 @@ class PostgresStorage: key: str, body_hash: str, ) -> tuple[str, Optional[Message], Optional[dict]]: - """Атомарная отправка сообщения с идемпотентностью. - - Возвращает одно из: - ('created', message, None) — мы выиграли гонку и создали сообщение; - ('replay', None, info) — ключ уже занят; info = {body_hash, - response_status, response_body} - ('retry', None, None) — конфликт при INSERT, но строки нет - (крайне редко); вызывающий может - повторить запрос. - - ВСЁ делается в ОДНОЙ транзакции: либо строка idempotency_keys - и строка messages фиксируются вместе, либо не фиксируется ничего. - Это устраняет блокер, когда сообщение сохранено, а ключ остался - с response_status=0 и повтор зависал. - """ - import json as _json with connection() as conn: with conn.cursor() as cur: - # 1) Пытаемся занять ключ. ON CONFLICT DO NOTHING — атомарно. cur.execute( "INSERT INTO idempotency_keys" " (user_id, method, path, key, body_hash, response_status, response_body)" @@ -312,7 +279,6 @@ class PostgresStorage: won = cur.fetchone() is not None if won: - # 2) Мы выиграли: создаём сообщение в ТОЙ ЖЕ транзакции. message = Message( id=uuid4(), chat_id=chat_id, sender_id=sender_id, text=text, created_at=now_utc(), @@ -332,7 +298,6 @@ class PostgresStorage: [(message.id, aid, idx) for idx, aid in enumerate(attachment_ids)], ) - # 3) Пишем финальный ответ в ту же строку ключа. body_json = _json.loads(message.model_dump_json()) cur.execute( "UPDATE idempotency_keys" @@ -341,11 +306,9 @@ class PostgresStorage: (201, _json.dumps(body_json), user_id, method, path, key), ) - # Один commit — либо всё, либо ничего. conn.commit() return ("created", message, None) - # Ключ уже занят. Читаем строку в этой же транзакции. cur.execute( "SELECT body_hash, response_status, response_body" " FROM idempotency_keys" @@ -356,7 +319,6 @@ class PostgresStorage: conn.commit() if row is None: - # Только что был конфликт, а строки нет — редкий случай; повтор. return ("retry", None, None) return ( "replay", @@ -369,11 +331,8 @@ class PostgresStorage: ) # ---------- Вложения ---------- - def create_attachment( - self, - attachment: Attachment, - object_key: str, - ) -> Attachment: + def create_attachment(self, attachment: Attachment, object_key: str) -> Attachment: + """lab4 fallback (синхронный ready). Не используется в lab5.""" with connection() as conn, conn.cursor() as cur: cur.execute( "INSERT INTO attachments" @@ -391,27 +350,112 @@ class PostgresStorage: conn.commit() return attachment + def create_attachment_queued( + self, + attachment: Attachment, + staging_key: str, + outbox_payload: dict, + ) -> Attachment: + """lab5: attachment в queued + строка outbox в ОДНОЙ транзакции.""" + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO attachments" + " (id, chat_id, owner_id, filename, content_type, size_bytes," + " sha256, object_key, status, error_code, processing_until," + " created_at, updated_at)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + ( + attachment.id, attachment.chat_id, attachment.owner_id, + attachment.filename, attachment.content_type, + attachment.size_bytes, attachment.sha256, staging_key, + "queued", None, None, + attachment.created_at, attachment.updated_at, + ), + ) + cur.execute( + "INSERT INTO outbox (job_id, payload) VALUES (%s, %s)", + (outbox_payload["job_id"], _json.dumps(outbox_payload)), + ) + conn.commit() + return attachment + + def claim_processing(self, attachment_id: UUID, lease_seconds: int = 60) -> bool: + with connection() as conn: + with conn.cursor() as cur: + now = now_utc() + cur.execute( + "UPDATE attachments SET status='processing'," + " processing_until=%s, updated_at=%s" + " WHERE id=%s AND status IN ('queued','processing')" + " RETURNING id", + (now + timedelta(seconds=lease_seconds), now, attachment_id), + ) + won = cur.fetchone() is not None + conn.commit() + return won + + def recover_stuck_processing(self) -> int: + with connection() as conn: + with conn.cursor() as cur: + now = now_utc() + cur.execute( + "UPDATE attachments SET status='queued', processing_until=NULL," + " updated_at=%s" + " WHERE status='processing' AND processing_until IS NOT NULL" + " AND processing_until < %s", + (now, now), + ) + count = cur.rowcount + conn.commit() + return count + + def finalize_ready( + self, + attachment_id: UUID, + pipeline_version: int, + variants: list[dict], + ) -> None: + with connection() as conn: + with conn.cursor() as cur: + original = next(v for v in variants if v["name"] == "original") + cur.execute( + "UPDATE attachments SET status='ready', error_code=NULL," + " object_key=%s, processing_until=NULL, updated_at=%s" + " WHERE id=%s", + (original["object_key"], now_utc(), attachment_id), + ) + for v in variants: + if v["name"] == "original": + continue + cur.execute( + "INSERT INTO attachment_variants" + " (attachment_id, pipeline_version, variant, object_key," + " content_type, size_bytes, width, height, sha256)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)" + " ON CONFLICT (attachment_id, pipeline_version, variant)" + " DO NOTHING", + ( + attachment_id, pipeline_version, v["name"], + v["object_key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"], + ), + ) + conn.commit() + + def finalize_failed(self, attachment_id: UUID, error_code: str) -> None: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE attachments SET status='failed', error_code=%s," + " processing_until=NULL, updated_at=%s WHERE id=%s", + (error_code, now_utc(), attachment_id), + ) + conn.commit() + def get_attachment(self, attachment_id: UUID) -> Optional[Attachment]: - with connection() as conn, conn.cursor() as cur: - cur.execute( - "SELECT id, chat_id, owner_id, filename, content_type, size_bytes," - " sha256, status, error_code, created_at, updated_at" - " FROM attachments WHERE id = %s", - (attachment_id,), - ) - row = cur.fetchone() - if row is None: - return None - cur.execute( - "SELECT object_key FROM attachments WHERE id = %s", (attachment_id,), - ) - obj_row = cur.fetchone() - att = _row_to_attachment(row) - att_variants: List[Variant] = [] # lab4: вариантов нет - att.variants = att_variants - # object_key нужен вызывающему для чтения из S3; отдаём через - # отдельный метод get_attachment_with_key. - return att + pair = self.get_attachment_with_key(attachment_id) + return pair[0] if pair else None def get_attachment_with_key( self, attachment_id: UUID @@ -426,13 +470,12 @@ class PostgresStorage: row = cur.fetchone() if row is None: return None - att = _row_to_attachment(row[:11]) + att = _row_to_attachment(row[:11], self.get_attachment_variants(attachment_id)) return att, row[11] def list_attachments_by_ids( self, ids: List[UUID], chat_id: UUID ) -> dict[UUID, tuple[Attachment, str]]: - """Достаём вложения этого чата. Возвращает {id: (Attachment, object_key)}.""" if not ids: return {} with connection() as conn, conn.cursor() as cur: @@ -445,7 +488,6 @@ class PostgresStorage: result: dict[UUID, tuple[Attachment, str]] = {} for row in cur.fetchall(): att = _row_to_attachment(row[:11]) - att.variants = [] result[row[0]] = (att, row[11]) return result @@ -454,5 +496,69 @@ class PostgresStorage: cur.execute("DELETE FROM attachments WHERE id = %s", (attachment_id,)) conn.commit() + def get_attachment_variants(self, attachment_id: UUID) -> list[dict]: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT variant, content_type, size_bytes, width, height, sha256" + " FROM attachment_variants WHERE attachment_id=%s" + " ORDER BY variant", + (attachment_id,), + ) + return [ + { + "name": r[0], "content_type": r[1], "size_bytes": r[2], + "width": r[3], "height": r[4], "sha256": r[5], + } + for r in cur.fetchall() + ] + + def get_staging_key(self, attachment_id: UUID) -> Optional[str]: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT object_key FROM attachments WHERE id=%s", + (attachment_id,), + ) + row = cur.fetchone() + return row[0] if row else None + + # ---------- Outbox (lab5 level 4) ---------- + def list_unpublished_outbox(self, limit: int = 100) -> list[tuple[int, dict]]: + """Возвращает [(outbox_id, payload_dict), ...]. + + Всегда dict; если payload в БД пришёл строкой (например, из старой + версии), распаковываем через json.loads. + """ + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT id, payload FROM outbox WHERE published_at IS NULL" + " ORDER BY id LIMIT %s", + (limit,), + ) + rows = cur.fetchall() + result: list[tuple[int, dict]] = [] + for r in rows: + raw = r[1] + if isinstance(raw, dict): + payload = raw + else: + payload = _json.loads(raw) + if not isinstance(payload, dict): + # защита от мусора — пропустить + continue + result.append((int(r[0]), payload)) + return result + + def mark_outbox_published(self, outbox_id: int) -> None: + with connection() as conn: + with conn.cursor() as cur: + cur.execute( + "UPDATE outbox SET published_at=now() WHERE id=%s", + (outbox_id,), + ) + conn.commit() + storage = PostgresStorage() # единственный экземпляр на процесс \ No newline at end of file