diff --git a/openspec/changes/add-lab4-upload-resilience/design.md b/openspec/changes/add-lab4-upload-resilience/design.md index 3215cf65..2f46c716 100644 --- a/openspec/changes/add-lab4-upload-resilience/design.md +++ b/openspec/changes/add-lab4-upload-resilience/design.md @@ -78,6 +78,10 @@ Grace period `UPLOAD_GRACE_SECONDS=900`, не меньше 60 (валидаци `scripts/s3-check.sh` получает проверку консоли из отдельного контейнера сети стенда (`wget http://s3:9001/` — отказ соединения); остальное уже покрывает строку 4.4: два повторных bootstrap без изменения снимка IAM, разрешённые и запрещённые операции ключа API, анонимный доступ, прямой HTTP GET. +### D7. Платформа: события команды из нескольких транзакций + +Компенсация фиксирует удаление уже после неудачного `COMMIT`, а проход уборки — по транзакции на строку и завершается откатом пустого поиска. Базовая `PostgresUnitOfWork` при любом откате очищала все накопленные события команды, а `CommandBus` не публиковал события, если обработчик бросил исключение, — журнал терял зафиксированные удаления. Теперь откат убирает только события, записанные внутри этой транзакции, а шина публикует собранное и при исключении обработчика. Это механизм платформы (`shared/`), доменных понятий он не добавляет; события по-прежнему появляются только из зафиксированных транзакций. Проверяется тестами компенсации и параллельной уборки по событиям `AttachmentDiscarded`. + ## Risks / Trade-offs - [Grace меньше реального времени жизни загрузки] → минимум 60 с в настройках, по умолчанию 900 с; даже при ошибке D1 гарантирует `503` и удаление объекта вместо `ready` без строки. diff --git a/openspec/changes/add-lab4-upload-resilience/tasks.md b/openspec/changes/add-lab4-upload-resilience/tasks.md index 358b19c8..87c77bdd 100644 --- a/openspec/changes/add-lab4-upload-resilience/tasks.md +++ b/openspec/changes/add-lab4-upload-resilience/tasks.md @@ -1,17 +1,18 @@ ## 1. Зона `attachments`: домен и приложение -- [ ] 1.1 Домен: событие `AttachmentDiscarded(reason)`, `Attachment.discard(reason)`, ошибка `UploadAborted` -- [ ] 1.2 Порты: репозиторий `lock`, `next_abandoned`, `remove`; `UploadSlots`; команда `SweepAbandonedUploads` -- [ ] 1.3 Обработчики: запись под слотом, компенсация `discard` после `PUT`, уборка по одной строке -- [ ] 1.4 Проверка группы: unit-тест `discard`, `make lint` +- [x] 1.1 Домен: событие `AttachmentDiscarded(reason)`, `Attachment.discard(reason)`, ошибка `UploadAborted` +- [x] 1.2 Порты: репозиторий `lock`, `next_abandoned`, `remove`; `UploadSlots`; команда `SweepAbandonedUploads` +- [x] 1.3 Обработчики: запись под слотом, компенсация `discard` после `PUT`, уборка по одной строке +- [x] 1.4 Проверка группы: unit-тест `discard`, `make lint` ## 2. Зона `attachments`: инфраструктура, настройки и точки входа -- [ ] 2.1 Адаптер PostgreSQL: `FOR UPDATE`, `FOR UPDATE SKIP LOCKED` по частичному индексу, `DELETE … AND status='pending'`, `UPDATE 0` → `Unavailable` -- [ ] 2.2 Настройки `UPLOAD_CONCURRENCY`, `UPLOAD_GRACE_SECONDS` (≥ 60), `UPLOAD_SWEEP_INTERVAL_SECONDS`, `UPLOAD_SWEEP_BATCH`; провайдер dishka -- [ ] 2.3 HTTP: `ClientDisconnect` → `UploadAborted` (`400`); журнал `audit.attachment_discarded` -- [ ] 2.4 `UploadSweeper` и его запуск/остановка в lifespan `main.py` -- [ ] 2.5 Проверка группы: API-тесты против одноразового PostgreSQL — отказ `COMMIT` после `PUT` (компенсация), снятое намерение (`503`, объект удалён), сбой `PUT` и уборка после grace, свежее и готовое переживают проход, два прохода одновременно, фоновая уборка, ожидание слота, обрыв (`400`), зависшее тело (`503`, слот свободен) +- [x] 2.1 Адаптер PostgreSQL: `FOR UPDATE`, `FOR UPDATE SKIP LOCKED` по частичному индексу, `DELETE … AND status='pending'`, `UPDATE 0` → `Unavailable` +- [x] 2.2 Настройки `UPLOAD_CONCURRENCY`, `UPLOAD_GRACE_SECONDS` (≥ 60), `UPLOAD_SWEEP_INTERVAL_SECONDS`, `UPLOAD_SWEEP_BATCH`; провайдер dishka +- [x] 2.3 HTTP: `ClientDisconnect` → `UploadAborted` (`400`); журнал `audit.attachment_discarded` +- [x] 2.4 `UploadSweeper` и его запуск/остановка в lifespan `main.py` +- [x] 2.5 Платформа: откат транзакции не стирает события прежних транзакций команды, шина публикует их и при ошибке обработчика +- [x] 2.6 Проверка группы: API-тесты против одноразового PostgreSQL — отказ `COMMIT` после `PUT` (компенсация), снятое намерение (`503`, объект удалён), сбой `PUT` и уборка после grace, свежее и готовое переживают проход, два прохода одновременно, фоновая уборка, ожидание слота, обрыв (`400`), зависшее тело (`503`, слот свободен) ## 3. Стенд diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py index df9c2b17..b42141fa 100644 --- a/src/messenger/attachments/application/commands.py +++ b/src/messenger/attachments/application/commands.py @@ -1,5 +1,6 @@ from collections.abc import AsyncIterator from dataclasses import dataclass +from datetime import timedelta from uuid import UUID from messenger.shared.application.commands import Command @@ -12,3 +13,9 @@ class UploadAttachment(Command): filename: str | None content_type: str | None body: AsyncIterator[bytes] + + +@dataclass(frozen=True, slots=True) +class SweepAbandonedUploads(Command): + grace: timedelta + limit: int diff --git a/src/messenger/attachments/application/handlers.py b/src/messenger/attachments/application/handlers.py index 4abb0307..36e767b5 100644 --- a/src/messenger/attachments/application/handlers.py +++ b/src/messenger/attachments/application/handlers.py @@ -1,7 +1,8 @@ +import asyncio from collections.abc import AsyncIterator from hashlib import sha256 -from messenger.attachments.application.commands import UploadAttachment +from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment from messenger.attachments.application.ports import ( AttachmentsUnitOfWork, ChatMembership, @@ -9,9 +10,13 @@ from messenger.attachments.application.ports import ( ObjectStorage, ) from messenger.attachments.domain.errors import ChatNotFound, PayloadTooLarge -from messenger.attachments.domain.models import Attachment +from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageKind -from messenger.shared.application.ports import Clock, IdGenerator +from messenger.shared.application.ports import Clock, IdGenerator, Logger + + +class UploadSlots(asyncio.Semaphore): + pass async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, str]: @@ -27,8 +32,27 @@ async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, s return b"".join(chunks), digest.hexdigest() +async def drop( + uow: AttachmentsUnitOfWork, storage: ObjectStorage, attachment: Attachment, reason: str +) -> None: + await storage.delete(attachment.object_key) + attachment.discard(reason) + await uow.attachments.remove(attachment) + await uow.commit() + + class UploadAttachmentHandler: - __slots__ = ("_clock", "_ids", "_images", "_limit", "_membership", "_storage", "_uow") + __slots__ = ( + "_clock", + "_ids", + "_images", + "_limit", + "_logger", + "_membership", + "_slots", + "_storage", + "_uow", + ) def __init__( self, @@ -40,6 +64,8 @@ class UploadAttachmentHandler: clock: Clock, ids: IdGenerator, limit: int, + slots: UploadSlots, + logger: Logger, ) -> None: self._uow = uow self._membership = membership @@ -48,12 +74,29 @@ class UploadAttachmentHandler: self._clock = clock self._ids = ids self._limit = limit + self._slots = slots + self._logger = logger async def __call__(self, command: UploadAttachment) -> Attachment: if not await self._membership.is_member(command.chat_id, command.owner_id): raise ChatNotFound("чат не найден") filename = Filename(command.filename or "") declared = ImageKind.declared(command.content_type) + async with self._slots: + attachment = await self._write(command, filename, declared) + attachment.store(self._clock.now()) + try: + async with self._uow.transaction(): + await self._uow.attachments.save(attachment) + await self._uow.commit() + except Exception: + await self._compensate(attachment) + raise + return attachment + + async def _write( + self, command: UploadAttachment, filename: Filename, declared: ImageKind + ) -> Attachment: data, digest = await read_limited(command.body, self._limit) image = await self._images.inspect(data, declared) attachment = Attachment.stage( @@ -70,8 +113,38 @@ class UploadAttachmentHandler: await self._uow.attachments.add(attachment) await self._uow.commit() await self._storage.put(attachment.object_key, data, attachment.content_type) - attachment.store(self._clock.now()) - async with self._uow.transaction(): - await self._uow.attachments.save(attachment) - await self._uow.commit() return attachment + + async def _compensate(self, staged: Attachment) -> None: + try: + async with self._uow.transaction(): + current = await self._uow.attachments.lock(staged.id) + if current is None: + await self._storage.delete(staged.object_key) + elif current.status is AttachmentStatus.PENDING: + await drop(self._uow, self._storage, current, "compensation") + except Exception as error: + self._logger.warning( + "attachment.compensation_failed", + attachment_id=str(staged.id), + error=type(error).__name__, + ) + + +class SweepAbandonedUploadsHandler: + __slots__ = ("_clock", "_storage", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage, clock: Clock) -> None: + self._uow = uow + self._storage = storage + self._clock = clock + + async def __call__(self, command: SweepAbandonedUploads) -> int: + before = self._clock.now() - command.grace + for swept in range(command.limit): + async with self._uow.transaction(): + abandoned = await self._uow.attachments.next_abandoned(before) + if abandoned is None: + return swept + await drop(self._uow, self._storage, abandoned, "abandoned") + return command.limit diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index d2c9b07e..b1339546 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -1,5 +1,6 @@ from collections.abc import AsyncIterator from dataclasses import dataclass +from datetime import datetime from typing import Protocol, runtime_checkable from uuid import UUID @@ -39,6 +40,12 @@ class AttachmentRepository(Protocol): async def save(self, attachment: Attachment) -> None: ... + async def lock(self, attachment_id: UUID) -> Attachment | None: ... + + async def next_abandoned(self, before: datetime) -> Attachment | None: ... + + async def remove(self, attachment: Attachment) -> None: ... + @runtime_checkable class AttachmentsUnitOfWork(UnitOfWork, Protocol): diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py index 3ba87298..956e4d95 100644 --- a/src/messenger/attachments/config.py +++ b/src/messenger/attachments/config.py @@ -18,3 +18,7 @@ class AttachmentsSettings(Settings): max_upload_bytes: int = Field(default=UPLOAD_BYTES_MAX, ge=1, le=UPLOAD_BYTES_MAX) max_image_pixels: int = Field(default=IMAGE_PIXELS_MAX, ge=1, le=IMAGE_PIXELS_MAX) image_check_concurrency: int = Field(default=2, ge=1, le=16) + upload_concurrency: int = Field(default=3, ge=1, le=64) + upload_grace_seconds: int = Field(default=900, ge=60) + upload_sweep_interval_seconds: float = Field(default=30.0, gt=0) + upload_sweep_batch: int = Field(default=100, ge=1, le=10_000) diff --git a/src/messenger/attachments/domain/errors.py b/src/messenger/attachments/domain/errors.py index 09439002..bec7ffdf 100644 --- a/src/messenger/attachments/domain/errors.py +++ b/src/messenger/attachments/domain/errors.py @@ -23,3 +23,7 @@ class ChatNotFound(AttachmentError): class AttachmentNotFound(AttachmentError): pass + + +class UploadAborted(AttachmentError): + pass diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py index 02796c8d..f12bbe97 100644 --- a/src/messenger/attachments/domain/events.py +++ b/src/messenger/attachments/domain/events.py @@ -11,3 +11,11 @@ class AttachmentStored(DomainEvent): owner_id: UUID size_bytes: int name: str = field(init=False, default="attachment.stored") + + +@dataclass(frozen=True, slots=True) +class AttachmentDiscarded(DomainEvent): + attachment_id: UUID + chat_id: UUID + reason: str + name: str = field(init=False, default="attachment.discarded") diff --git a/src/messenger/attachments/domain/models.py b/src/messenger/attachments/domain/models.py index bb09e532..fef585eb 100644 --- a/src/messenger/attachments/domain/models.py +++ b/src/messenger/attachments/domain/models.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import StrEnum from uuid import UUID -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.domain.values import Filename, ImageInfo from messenger.shared.domain.events import DomainEvent @@ -73,6 +73,11 @@ class Attachment: ) ) + def discard(self, reason: str) -> None: + self._events.append( + AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason) + ) + def collect_events(self) -> list[DomainEvent]: events, self._events = self._events, [] return events diff --git a/src/messenger/attachments/entrypoints/errors.py b/src/messenger/attachments/entrypoints/errors.py index 1e6748d6..56e3e8e0 100644 --- a/src/messenger/attachments/entrypoints/errors.py +++ b/src/messenger/attachments/entrypoints/errors.py @@ -4,11 +4,13 @@ from messenger.attachments.domain.errors import ( InvalidFilename, PayloadTooLarge, UnsupportedImage, + UploadAborted, ) from messenger.shared.domain.errors import DomainError ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { InvalidFilename: ("invalid_request", 400), + UploadAborted: ("invalid_request", 400), ChatNotFound: ("not_found", 404), AttachmentNotFound: ("not_found", 404), PayloadTooLarge: ("payload_too_large", 413), diff --git a/src/messenger/attachments/entrypoints/http.py b/src/messenger/attachments/entrypoints/http.py index 14ec3623..ac31decd 100644 --- a/src/messenger/attachments/entrypoints/http.py +++ b/src/messenger/attachments/entrypoints/http.py @@ -1,14 +1,17 @@ import re +from collections.abc import AsyncIterator from typing import Annotated, Literal from uuid import UUID from dishka.integrations.fastapi import FromDishka, inject from fastapi import APIRouter, Query, Request, status from fastapi.responses import StreamingResponse +from starlette.requests import ClientDisconnect from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import UploadAttachment from messenger.attachments.application.queries import GetAttachment, OpenAttachment +from messenger.attachments.domain.errors import UploadAborted from messenger.attachments.entrypoints.schemas import AttachmentOut from messenger.shared.entrypoints.actor import Actor @@ -19,6 +22,14 @@ uploads_router = APIRouter(prefix="/api/v1/chats/{chat_id}/attachments", tags=[" attachments_router = APIRouter(prefix="/api/v1/attachments", tags=["attachments"]) +async def body_of(request: Request) -> AsyncIterator[bytes]: + try: + async for chunk in request.stream(): + yield chunk + except ClientDisconnect as error: + raise UploadAborted("клиент оборвал загрузку до конца тела") from error + + @uploads_router.post("", status_code=status.HTTP_201_CREATED) @inject async def upload_attachment( @@ -34,7 +45,7 @@ async def upload_attachment( chat_id=chat_id, filename=filename, content_type=request.headers.get("content-type"), - body=request.stream(), + body=body_of(request), ) ) return AttachmentOut.of(attachment) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index 09875073..a7ca5464 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -3,8 +3,12 @@ from collections.abc import AsyncIterator from dishka import Provider, Scope, provide from messenger.attachments.application.bus import AttachmentsCommandBus -from messenger.attachments.application.commands import UploadAttachment -from messenger.attachments.application.handlers import UploadAttachmentHandler +from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment +from messenger.attachments.application.handlers import ( + SweepAbandonedUploadsHandler, + UploadAttachmentHandler, + UploadSlots, +) from messenger.attachments.application.ports import ( AttachmentDirectory, AttachmentsUnitOfWork, @@ -52,6 +56,10 @@ class AttachmentsProvider(Provider): def images(self, settings: AttachmentsSettings) -> ImageInspector: return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency) + @provide(scope=Scope.APP) + def slots(self, settings: AttachmentsSettings) -> UploadSlots: + return UploadSlots(settings.upload_concurrency) + @provide(scope=Scope.APP) def directory(self, database: Database) -> AttachmentDirectory: return PostgresAttachmentDirectory(database) @@ -83,9 +91,22 @@ class AttachmentsProvider(Provider): clock: Clock, ids: IdGenerator, settings: AttachmentsSettings, + slots: UploadSlots, + logger: Logger, ) -> AttachmentsCommandBus: - handler = UploadAttachmentHandler( - uow, membership, images, storage, clock=clock, ids=ids, limit=settings.max_upload_bytes + upload = UploadAttachmentHandler( + uow, + membership, + images, + storage, + clock=clock, + ids=ids, + limit=settings.max_upload_bytes, + slots=slots, + logger=logger, ) - handlers: dict[type[Command], CommandHandler] = {UploadAttachment: handler} + handlers: dict[type[Command], CommandHandler] = { + UploadAttachment: upload, + SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock), + } return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py index b0bf792d..55ba5e5a 100644 --- a/src/messenger/attachments/entrypoints/journal.py +++ b/src/messenger/attachments/entrypoints/journal.py @@ -1,7 +1,7 @@ -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.shared.application.ports import Logger -LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes"}) +LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes", "count"}) class AttachmentsJournal: @@ -18,3 +18,11 @@ class AttachmentsJournal: actor_id=str(event.owner_id), size_bytes=event.size_bytes, ) + + async def on_attachment_discarded(self, event: AttachmentDiscarded) -> None: + self._logger.info( + "audit.attachment_discarded", + attachment_id=str(event.attachment_id), + chat_id=str(event.chat_id), + reason=event.reason, + ) diff --git a/src/messenger/attachments/entrypoints/sweeper.py b/src/messenger/attachments/entrypoints/sweeper.py new file mode 100644 index 00000000..63b08da4 --- /dev/null +++ b/src/messenger/attachments/entrypoints/sweeper.py @@ -0,0 +1,53 @@ +import asyncio +from contextlib import suppress +from datetime import timedelta + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import SweepAbandonedUploads +from messenger.attachments.config import AttachmentsSettings +from messenger.shared.application.ports import Logger + + +class UploadSweeper: + __slots__ = ("_command", "_container", "_interval", "_logger", "_task") + + def __init__( + self, container: AsyncContainer, settings: AttachmentsSettings, logger: Logger + ) -> None: + self._container = container + self._logger = logger + self._interval = settings.upload_sweep_interval_seconds + self._command = SweepAbandonedUploads( + grace=timedelta(seconds=settings.upload_grace_seconds), + limit=settings.upload_sweep_batch, + ) + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def sweep(self) -> int: + async with self._container() as scope: + bus = await scope.get(AttachmentsCommandBus) + return await bus.execute(self._command) + + async def _run(self) -> None: + while True: + await asyncio.sleep(self._interval) + try: + swept = await self.sweep() + except Exception as error: + self._logger.warning("attachments.sweep_failed", error=type(error).__name__) + continue + if swept: + self._logger.info("attachments.swept", count=swept) diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index c023234c..c47d0f18 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -1,3 +1,4 @@ +from datetime import datetime from uuid import UUID from asyncpg import ForeignKeyViolationError, Record @@ -5,6 +6,7 @@ from asyncpg import ForeignKeyViolationError, Record from messenger.attachments.domain.errors import ChatNotFound from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind +from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.postgres import ( Connection, Database, @@ -18,12 +20,19 @@ INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_byt VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) """ MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'" -READY = """ +COLUMNS = """ SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at FROM attachments -WHERE id = $1 AND status = 'ready' """ +READY = COLUMNS + "WHERE id = $1 AND status = 'ready'" +LOCK = COLUMNS + "WHERE id = $1 FOR UPDATE" +ABANDONED = ( + COLUMNS + + """WHERE status = 'pending' AND created_at < $1 +ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" +) +REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" def attachment_of(row: Record) -> Attachment: @@ -72,9 +81,23 @@ class PostgresAttachmentRepository: self._seen.append(attachment) async def save(self, attachment: Attachment) -> None: - await self._connection.execute( + done = await self._connection.execute( MARK, attachment.id, attachment.status.value, attachment.updated_at ) + if done == "UPDATE 0": + raise Unavailable("намерение загрузки уже снято уборкой") + self._seen.append(attachment) + + async def lock(self, attachment_id: UUID) -> Attachment | None: + row = await self._connection.fetchrow(LOCK, attachment_id) + return None if row is None else attachment_of(row) + + async def next_abandoned(self, before: datetime) -> Attachment | None: + row = await self._connection.fetchrow(ABANDONED, before) + return None if row is None else attachment_of(row) + + async def remove(self, attachment: Attachment) -> None: + await self._connection.execute(REMOVE, attachment.id) self._seen.append(attachment) diff --git a/src/messenger/main.py b/src/messenger/main.py index 41cfb09e..8bebfc4f 100644 --- a/src/messenger/main.py +++ b/src/messenger/main.py @@ -8,12 +8,13 @@ from fastapi import FastAPI from messenger.attachments.application.ports import ChatMembership from messenger.attachments.config import AttachmentsSettings -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.entrypoints.errors import ERROR_CODES as ATTACHMENT_ERRORS from messenger.attachments.entrypoints.http import UPLOAD_PATH, attachments_router, uploads_router from messenger.attachments.entrypoints.ioc import AttachmentsProvider from messenger.attachments.entrypoints.journal import LOG_FIELDS as ATTACHMENT_FIELDS from messenger.attachments.entrypoints.journal import AttachmentsJournal +from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.identity.config import IdentitySettings from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered from messenger.identity.entrypoints.authentication import challenge_for, guard @@ -76,6 +77,7 @@ def wire_events(bus: EventBus, logger: Logger) -> None: attachments = AttachmentsJournal(logger) bus.subscribe(AttachmentStored, attachments.on_attachment_stored) + bus.subscribe(AttachmentDiscarded, attachments.on_attachment_discarded) def create_app( @@ -102,16 +104,19 @@ def create_app( FastapiProvider(), *overrides, ) + sweeper = UploadSweeper(container, attachments, logger) @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: bus = await container.get(EventBus) wire_events(bus, logger) bus.start() + sweeper.start() logger.info("service.started") try: yield finally: + await sweeper.stop() await bus.stop() await container.close() logger.info("service.stopped") @@ -142,4 +147,5 @@ def create_app( app.include_router(uploads_router) app.include_router(attachments_router) setup_dishka(container, app) + app.state.sweeper = sweeper return app diff --git a/student-tests/api/test_upload_consistency.py b/student-tests/api/test_upload_consistency.py new file mode 100644 index 00000000..30bc91cf --- /dev/null +++ b/student-tests/api/test_upload_consistency.py @@ -0,0 +1,244 @@ +import asyncio +from collections.abc import AsyncIterator +from typing import Any +from uuid import uuid4 + +import asyncpg +from httpx import AsyncClient +from starlette.types import Message +from test_attachments import storage, trio, upload + +from conftest import APPS, DATABASE, actor, keyed, resolve, running_app, scalar +from messenger.attachments.domain.events import AttachmentDiscarded +from messenger.attachments.entrypoints.sweeper import UploadSweeper +from messenger.shared.application.messagebus import EventBus +from pictures import SAMPLE + +AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1" + + +async def execute(sql: str, *args: Any) -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + await connection.execute(sql, *args) + finally: + await connection.close() + + +def sweeper(http: AsyncClient) -> UploadSweeper: + return APPS[id(http)].state.sweeper + + +async def discarded(http: AsyncClient) -> list[AttachmentDiscarded]: + seen: list[AttachmentDiscarded] = [] + + async def keep(event: AttachmentDiscarded) -> None: + seen.append(event) + + (await resolve(http, EventBus)).project(AttachmentDiscarded, keep) + return seen + + +async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: + store = await storage(http) + store.failing = True + name = f"intent-{uuid4().hex}.png" + try: + assert (await upload(http, user, chat, filename=name)).status_code == 503 + finally: + store.failing = False + found = await scalar( + "SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name + ) + assert found is not None + store.objects[f"attachments/{found}"] = (SAMPLE, "image/png") + return found + + +async def exists(attachment_id: str) -> bool: + return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1 + + +async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + store = await storage(client) + name = f"fail-{uuid4().hex}.png" + trigger = f"lab4_fail_{uuid4().hex[:8]}" + await execute( + f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$; + CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW + WHEN (NEW.filename = '{name}' AND NEW.status = 'ready') EXECUTE FUNCTION {trigger}()""" + ) + before, seen = set(store.objects), await discarded(client) + try: + response = await upload(client, alice, chat, filename=name) + finally: + await execute(f"DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}()") + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert set(store.objects) == before + assert [event.reason for event in seen] == ["compensation"] + + +async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + store = await storage(client) + + async def swept_meanwhile(key: str) -> None: + await execute("DELETE FROM attachments WHERE object_key = $1", key) + + store.after_put = swept_meanwhile + before = set(store.objects) + try: + response = await upload(client, alice, chat, filename="late.png") + finally: + store.after_put = None + assert response.status_code == 503 + assert set(store.objects) == before + + +async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + pending = await intent(client, alice, chat) + await sweeper(client).sweep() + assert await exists(pending) + await execute(AGED, pending) + assert await sweeper(client).sweep() >= 1 + assert not await exists(pending) + assert f"attachments/{pending}" not in (await storage(client)).objects + + +async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None: + alice, bob, _, chat = await trio(client) + ready = (await upload(client, alice, chat)).json() + sent = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [ready["id"]]}, + headers=keyed(alice), + ) + assert sent.status_code == 201 + old, fresh = await intent(client, alice, chat), await intent(client, alice, chat) + await execute(AGED, old) + await execute(AGED, ready["id"]) + assert await sweeper(client).sweep() == 1 + objects = (await storage(client)).objects + assert not await exists(old) + assert f"attachments/{old}" not in objects + assert await exists(fresh) + assert f"attachments/{fresh}" in objects + download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob)) + assert download.status_code == 200 + assert download.content == SAMPLE + assert await sweeper(client).sweep() == 0 + + +async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + pending = [await intent(client, alice, chat) for _ in range(5)] + for attachment_id in pending: + await execute(AGED, attachment_id) + seen = await discarded(client) + first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep()) + assert first + second == 5 + assert sorted(str(event.attachment_id) for event in seen) == sorted(pending) + assert {event.reason for event in seen} == {"abandoned"} + assert await sweeper(client).sweep() == 0 + objects = (await storage(client)).objects + assert not any(f"attachments/{attachment_id}" in objects for attachment_id in pending) + + +async def test_background_sweeper_removes_abandoned_intents() -> None: + async with running_app(upload_sweep_interval_seconds=0.05) as http: + alice, _, _, chat = await trio(http) + pending = await intent(http, alice, chat) + await execute(AGED, pending) + for _ in range(100): + if not await exists(pending): + break + await asyncio.sleep(0.05) + assert not await exists(pending) + + +async def test_upload_waits_for_a_free_slot() -> None: + async with running_app(upload_concurrency=1) as http: + alice, _, _, chat = await trio(http) + gate, holding, read = asyncio.Event(), asyncio.Event(), [] + + async def held() -> AsyncIterator[bytes]: + read.append("first") + holding.set() + yield SAMPLE[:1000] + await gate.wait() + yield SAMPLE[1000:] + + async def queued() -> AsyncIterator[bytes]: + read.append("second") + yield SAMPLE + + first = asyncio.create_task(upload(http, alice, chat, held())) + await holding.wait() + second = asyncio.create_task(upload(http, alice, chat, queued())) + await asyncio.sleep(0.3) + assert read == ["first"] + gate.set() + assert [(await first).status_code, (await second).status_code] == [201, 201] + assert read == ["first", "second"] + + +async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + name = f"abort-{uuid4().hex}.png" + headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()] + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": f"/api/v1/chats/{chat['id']}/attachments", + "raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(), + "query_string": f"filename={name}".encode(), + "root_path": "", + "headers": [ + *headers, + (b"content-type", b"image/png"), + (b"content-length", b"1000000"), + ], + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + } + messages = iter( + [ + {"type": "http.request", "body": SAMPLE[:1000], "more_body": True}, + {"type": "http.disconnect"}, + ] + ) + sent: list[Message] = [] + + async def receive() -> dict[str, Any]: + return next(messages, {"type": "http.disconnect"}) + + async def send(message: Message) -> None: + sent.append(message) + + await APPS[id(client)](scope, receive, send) + assert sent[0]["status"] == 400 + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + + +async def test_stalled_upload_times_out_and_frees_the_slot() -> None: + async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http: + alice, _, _, chat = await trio(http) + name = f"stalled-{uuid4().hex}.png" + + async def stalled() -> AsyncIterator[bytes]: + yield SAMPLE[:1000] + await asyncio.Event().wait() + + response = await upload(http, alice, chat, stalled(), filename=name) + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert (await upload(http, alice, chat)).status_code == 201 diff --git a/student-tests/fakes.py b/student-tests/fakes.py index fc59c177..e0d3633e 100644 --- a/student-tests/fakes.py +++ b/student-tests/fakes.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable, Callable from datetime import datetime, timedelta from typing import Any @@ -36,16 +36,19 @@ async def pieces(data: bytes, size: int = 65_536) -> AsyncIterator[bytes]: class MemoryObjectStorage: - __slots__ = ("failing", "objects") + __slots__ = ("after_put", "failing", "objects") def __init__(self) -> None: self.objects: dict[str, tuple[bytes, str]] = {} self.failing = False + self.after_put: Callable[[str], Awaitable[None]] | None = None async def put(self, key: str, data: bytes, content_type: str) -> None: if self.failing: raise Unavailable("S3 недоступно") self.objects[key] = (bytes(data), content_type) + if self.after_put is not None: + await self.after_put(key) async def open(self, key: str) -> ObjectContent: if self.failing: diff --git a/student-tests/unit/test_attachments_domain.py b/student-tests/unit/test_attachments_domain.py index 3063a278..e26d3101 100644 --- a/student-tests/unit/test_attachments_domain.py +++ b/student-tests/unit/test_attachments_domain.py @@ -4,7 +4,7 @@ from uuid import uuid4 import pytest from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind @@ -67,3 +67,24 @@ def test_attachment_is_pending_until_stored() -> None: events = attachment.collect_events() assert [type(event) for event in events] == [AttachmentStored] assert attachment.collect_events() == [] + + +def test_discarded_intent_records_its_reason() -> None: + attachment = Attachment.stage( + attachment_id=uuid4(), + chat_id=uuid4(), + owner_id=uuid4(), + filename=Filename("a.png"), + image=ImageInfo(ImageKind.PNG, 1, 1), + size_bytes=1, + sha256="0" * 64, + now=MOMENT, + ) + attachment.discard("abandoned") + (event,) = attachment.collect_events() + assert isinstance(event, AttachmentDiscarded) + assert (event.attachment_id, event.chat_id, event.reason) == ( + attachment.id, + attachment.chat_id, + "abandoned", + )