From 0e4eb1738c14dd6ebf7f88de02b50ec62f2dd87c Mon Sep 17 00:00:00 2001 From: Artyom Date: Wed, 28 Oct 2026 22:39:51 +0300 Subject: [PATCH] Compensate failed uploads and sweep abandoned intents When the ready commit fails after the object was written, the upload handler now locks the row, keeps it if it turned out ready, and otherwise deletes the object first and the pending row second before answering 503. A ready update that touches no row means the intent was already swept, so it fails as unavailable and the object goes too; a 201 without a row cannot happen. Every worker runs an UploadSweeper that every 30 s removes pending intents older than UPLOAD_GRACE_SECONDS (900 s, at least 60 s) one row at a time under FOR UPDATE SKIP LOCKED over the partial pending index, object first, and logs an audit event for each discard. Bodies are buffered only inside one of UPLOAD_CONCURRENCY slots per worker and released before the ready commit, and a client that disconnects in the middle of a body becomes a 400 instead of an unhandled 500. The tests run against a throwaway PostgreSQL: a trigger rejects the ready commit and the compensation leaves neither row nor object, a row removed during the write gives 503, a failed write stays until the grace ends, fresh intents and an aged ready attachment in a message survive a sweep, two concurrent sweeps share five intents exactly, the background loop runs by itself, a second upload waits for the slot without reading its body, and aborted and stalled bodies leave nothing. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27 --- .../add-lab4-upload-resilience/design.md | 4 + .../add-lab4-upload-resilience/tasks.md | 19 +- .../attachments/application/commands.py | 7 + .../attachments/application/handlers.py | 89 ++++++- .../attachments/application/ports.py | 7 + src/messenger/attachments/config.py | 4 + src/messenger/attachments/domain/errors.py | 4 + src/messenger/attachments/domain/events.py | 8 + src/messenger/attachments/domain/models.py | 7 +- .../attachments/entrypoints/errors.py | 2 + src/messenger/attachments/entrypoints/http.py | 13 +- src/messenger/attachments/entrypoints/ioc.py | 31 ++- .../attachments/entrypoints/journal.py | 12 +- .../attachments/entrypoints/sweeper.py | 53 ++++ .../attachments/infrastructure/postgres.py | 29 ++- src/messenger/main.py | 8 +- student-tests/api/test_upload_consistency.py | 244 ++++++++++++++++++ student-tests/fakes.py | 7 +- student-tests/unit/test_attachments_domain.py | 23 +- 19 files changed, 538 insertions(+), 33 deletions(-) create mode 100644 src/messenger/attachments/entrypoints/sweeper.py create mode 100644 student-tests/api/test_upload_consistency.py diff --git a/openspec/changes/add-lab4-upload-resilience/design.md b/openspec/changes/add-lab4-upload-resilience/design.md index 3215cf65..2f46c716 100644 --- a/openspec/changes/add-lab4-upload-resilience/design.md +++ b/openspec/changes/add-lab4-upload-resilience/design.md @@ -78,6 +78,10 @@ Grace period `UPLOAD_GRACE_SECONDS=900`, не меньше 60 (валидаци `scripts/s3-check.sh` получает проверку консоли из отдельного контейнера сети стенда (`wget http://s3:9001/` — отказ соединения); остальное уже покрывает строку 4.4: два повторных bootstrap без изменения снимка IAM, разрешённые и запрещённые операции ключа API, анонимный доступ, прямой HTTP GET. +### D7. Платформа: события команды из нескольких транзакций + +Компенсация фиксирует удаление уже после неудачного `COMMIT`, а проход уборки — по транзакции на строку и завершается откатом пустого поиска. Базовая `PostgresUnitOfWork` при любом откате очищала все накопленные события команды, а `CommandBus` не публиковал события, если обработчик бросил исключение, — журнал терял зафиксированные удаления. Теперь откат убирает только события, записанные внутри этой транзакции, а шина публикует собранное и при исключении обработчика. Это механизм платформы (`shared/`), доменных понятий он не добавляет; события по-прежнему появляются только из зафиксированных транзакций. Проверяется тестами компенсации и параллельной уборки по событиям `AttachmentDiscarded`. + ## Risks / Trade-offs - [Grace меньше реального времени жизни загрузки] → минимум 60 с в настройках, по умолчанию 900 с; даже при ошибке D1 гарантирует `503` и удаление объекта вместо `ready` без строки. diff --git a/openspec/changes/add-lab4-upload-resilience/tasks.md b/openspec/changes/add-lab4-upload-resilience/tasks.md index 358b19c8..87c77bdd 100644 --- a/openspec/changes/add-lab4-upload-resilience/tasks.md +++ b/openspec/changes/add-lab4-upload-resilience/tasks.md @@ -1,17 +1,18 @@ ## 1. Зона `attachments`: домен и приложение -- [ ] 1.1 Домен: событие `AttachmentDiscarded(reason)`, `Attachment.discard(reason)`, ошибка `UploadAborted` -- [ ] 1.2 Порты: репозиторий `lock`, `next_abandoned`, `remove`; `UploadSlots`; команда `SweepAbandonedUploads` -- [ ] 1.3 Обработчики: запись под слотом, компенсация `discard` после `PUT`, уборка по одной строке -- [ ] 1.4 Проверка группы: unit-тест `discard`, `make lint` +- [x] 1.1 Домен: событие `AttachmentDiscarded(reason)`, `Attachment.discard(reason)`, ошибка `UploadAborted` +- [x] 1.2 Порты: репозиторий `lock`, `next_abandoned`, `remove`; `UploadSlots`; команда `SweepAbandonedUploads` +- [x] 1.3 Обработчики: запись под слотом, компенсация `discard` после `PUT`, уборка по одной строке +- [x] 1.4 Проверка группы: unit-тест `discard`, `make lint` ## 2. Зона `attachments`: инфраструктура, настройки и точки входа -- [ ] 2.1 Адаптер PostgreSQL: `FOR UPDATE`, `FOR UPDATE SKIP LOCKED` по частичному индексу, `DELETE … AND status='pending'`, `UPDATE 0` → `Unavailable` -- [ ] 2.2 Настройки `UPLOAD_CONCURRENCY`, `UPLOAD_GRACE_SECONDS` (≥ 60), `UPLOAD_SWEEP_INTERVAL_SECONDS`, `UPLOAD_SWEEP_BATCH`; провайдер dishka -- [ ] 2.3 HTTP: `ClientDisconnect` → `UploadAborted` (`400`); журнал `audit.attachment_discarded` -- [ ] 2.4 `UploadSweeper` и его запуск/остановка в lifespan `main.py` -- [ ] 2.5 Проверка группы: API-тесты против одноразового PostgreSQL — отказ `COMMIT` после `PUT` (компенсация), снятое намерение (`503`, объект удалён), сбой `PUT` и уборка после grace, свежее и готовое переживают проход, два прохода одновременно, фоновая уборка, ожидание слота, обрыв (`400`), зависшее тело (`503`, слот свободен) +- [x] 2.1 Адаптер PostgreSQL: `FOR UPDATE`, `FOR UPDATE SKIP LOCKED` по частичному индексу, `DELETE … AND status='pending'`, `UPDATE 0` → `Unavailable` +- [x] 2.2 Настройки `UPLOAD_CONCURRENCY`, `UPLOAD_GRACE_SECONDS` (≥ 60), `UPLOAD_SWEEP_INTERVAL_SECONDS`, `UPLOAD_SWEEP_BATCH`; провайдер dishka +- [x] 2.3 HTTP: `ClientDisconnect` → `UploadAborted` (`400`); журнал `audit.attachment_discarded` +- [x] 2.4 `UploadSweeper` и его запуск/остановка в lifespan `main.py` +- [x] 2.5 Платформа: откат транзакции не стирает события прежних транзакций команды, шина публикует их и при ошибке обработчика +- [x] 2.6 Проверка группы: API-тесты против одноразового PostgreSQL — отказ `COMMIT` после `PUT` (компенсация), снятое намерение (`503`, объект удалён), сбой `PUT` и уборка после grace, свежее и готовое переживают проход, два прохода одновременно, фоновая уборка, ожидание слота, обрыв (`400`), зависшее тело (`503`, слот свободен) ## 3. Стенд diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py index df9c2b17..b42141fa 100644 --- a/src/messenger/attachments/application/commands.py +++ b/src/messenger/attachments/application/commands.py @@ -1,5 +1,6 @@ from collections.abc import AsyncIterator from dataclasses import dataclass +from datetime import timedelta from uuid import UUID from messenger.shared.application.commands import Command @@ -12,3 +13,9 @@ class UploadAttachment(Command): filename: str | None content_type: str | None body: AsyncIterator[bytes] + + +@dataclass(frozen=True, slots=True) +class SweepAbandonedUploads(Command): + grace: timedelta + limit: int diff --git a/src/messenger/attachments/application/handlers.py b/src/messenger/attachments/application/handlers.py index 4abb0307..36e767b5 100644 --- a/src/messenger/attachments/application/handlers.py +++ b/src/messenger/attachments/application/handlers.py @@ -1,7 +1,8 @@ +import asyncio from collections.abc import AsyncIterator from hashlib import sha256 -from messenger.attachments.application.commands import UploadAttachment +from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment from messenger.attachments.application.ports import ( AttachmentsUnitOfWork, ChatMembership, @@ -9,9 +10,13 @@ from messenger.attachments.application.ports import ( ObjectStorage, ) from messenger.attachments.domain.errors import ChatNotFound, PayloadTooLarge -from messenger.attachments.domain.models import Attachment +from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageKind -from messenger.shared.application.ports import Clock, IdGenerator +from messenger.shared.application.ports import Clock, IdGenerator, Logger + + +class UploadSlots(asyncio.Semaphore): + pass async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, str]: @@ -27,8 +32,27 @@ async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, s return b"".join(chunks), digest.hexdigest() +async def drop( + uow: AttachmentsUnitOfWork, storage: ObjectStorage, attachment: Attachment, reason: str +) -> None: + await storage.delete(attachment.object_key) + attachment.discard(reason) + await uow.attachments.remove(attachment) + await uow.commit() + + class UploadAttachmentHandler: - __slots__ = ("_clock", "_ids", "_images", "_limit", "_membership", "_storage", "_uow") + __slots__ = ( + "_clock", + "_ids", + "_images", + "_limit", + "_logger", + "_membership", + "_slots", + "_storage", + "_uow", + ) def __init__( self, @@ -40,6 +64,8 @@ class UploadAttachmentHandler: clock: Clock, ids: IdGenerator, limit: int, + slots: UploadSlots, + logger: Logger, ) -> None: self._uow = uow self._membership = membership @@ -48,12 +74,29 @@ class UploadAttachmentHandler: self._clock = clock self._ids = ids self._limit = limit + self._slots = slots + self._logger = logger async def __call__(self, command: UploadAttachment) -> Attachment: if not await self._membership.is_member(command.chat_id, command.owner_id): raise ChatNotFound("чат не найден") filename = Filename(command.filename or "") declared = ImageKind.declared(command.content_type) + async with self._slots: + attachment = await self._write(command, filename, declared) + attachment.store(self._clock.now()) + try: + async with self._uow.transaction(): + await self._uow.attachments.save(attachment) + await self._uow.commit() + except Exception: + await self._compensate(attachment) + raise + return attachment + + async def _write( + self, command: UploadAttachment, filename: Filename, declared: ImageKind + ) -> Attachment: data, digest = await read_limited(command.body, self._limit) image = await self._images.inspect(data, declared) attachment = Attachment.stage( @@ -70,8 +113,38 @@ class UploadAttachmentHandler: await self._uow.attachments.add(attachment) await self._uow.commit() await self._storage.put(attachment.object_key, data, attachment.content_type) - attachment.store(self._clock.now()) - async with self._uow.transaction(): - await self._uow.attachments.save(attachment) - await self._uow.commit() return attachment + + async def _compensate(self, staged: Attachment) -> None: + try: + async with self._uow.transaction(): + current = await self._uow.attachments.lock(staged.id) + if current is None: + await self._storage.delete(staged.object_key) + elif current.status is AttachmentStatus.PENDING: + await drop(self._uow, self._storage, current, "compensation") + except Exception as error: + self._logger.warning( + "attachment.compensation_failed", + attachment_id=str(staged.id), + error=type(error).__name__, + ) + + +class SweepAbandonedUploadsHandler: + __slots__ = ("_clock", "_storage", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage, clock: Clock) -> None: + self._uow = uow + self._storage = storage + self._clock = clock + + async def __call__(self, command: SweepAbandonedUploads) -> int: + before = self._clock.now() - command.grace + for swept in range(command.limit): + async with self._uow.transaction(): + abandoned = await self._uow.attachments.next_abandoned(before) + if abandoned is None: + return swept + await drop(self._uow, self._storage, abandoned, "abandoned") + return command.limit diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index d2c9b07e..b1339546 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -1,5 +1,6 @@ from collections.abc import AsyncIterator from dataclasses import dataclass +from datetime import datetime from typing import Protocol, runtime_checkable from uuid import UUID @@ -39,6 +40,12 @@ class AttachmentRepository(Protocol): async def save(self, attachment: Attachment) -> None: ... + async def lock(self, attachment_id: UUID) -> Attachment | None: ... + + async def next_abandoned(self, before: datetime) -> Attachment | None: ... + + async def remove(self, attachment: Attachment) -> None: ... + @runtime_checkable class AttachmentsUnitOfWork(UnitOfWork, Protocol): diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py index 3ba87298..956e4d95 100644 --- a/src/messenger/attachments/config.py +++ b/src/messenger/attachments/config.py @@ -18,3 +18,7 @@ class AttachmentsSettings(Settings): max_upload_bytes: int = Field(default=UPLOAD_BYTES_MAX, ge=1, le=UPLOAD_BYTES_MAX) max_image_pixels: int = Field(default=IMAGE_PIXELS_MAX, ge=1, le=IMAGE_PIXELS_MAX) image_check_concurrency: int = Field(default=2, ge=1, le=16) + upload_concurrency: int = Field(default=3, ge=1, le=64) + upload_grace_seconds: int = Field(default=900, ge=60) + upload_sweep_interval_seconds: float = Field(default=30.0, gt=0) + upload_sweep_batch: int = Field(default=100, ge=1, le=10_000) diff --git a/src/messenger/attachments/domain/errors.py b/src/messenger/attachments/domain/errors.py index 09439002..bec7ffdf 100644 --- a/src/messenger/attachments/domain/errors.py +++ b/src/messenger/attachments/domain/errors.py @@ -23,3 +23,7 @@ class ChatNotFound(AttachmentError): class AttachmentNotFound(AttachmentError): pass + + +class UploadAborted(AttachmentError): + pass diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py index 02796c8d..f12bbe97 100644 --- a/src/messenger/attachments/domain/events.py +++ b/src/messenger/attachments/domain/events.py @@ -11,3 +11,11 @@ class AttachmentStored(DomainEvent): owner_id: UUID size_bytes: int name: str = field(init=False, default="attachment.stored") + + +@dataclass(frozen=True, slots=True) +class AttachmentDiscarded(DomainEvent): + attachment_id: UUID + chat_id: UUID + reason: str + name: str = field(init=False, default="attachment.discarded") diff --git a/src/messenger/attachments/domain/models.py b/src/messenger/attachments/domain/models.py index bb09e532..fef585eb 100644 --- a/src/messenger/attachments/domain/models.py +++ b/src/messenger/attachments/domain/models.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import StrEnum from uuid import UUID -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.domain.values import Filename, ImageInfo from messenger.shared.domain.events import DomainEvent @@ -73,6 +73,11 @@ class Attachment: ) ) + def discard(self, reason: str) -> None: + self._events.append( + AttachmentDiscarded(attachment_id=self.id, chat_id=self.chat_id, reason=reason) + ) + def collect_events(self) -> list[DomainEvent]: events, self._events = self._events, [] return events diff --git a/src/messenger/attachments/entrypoints/errors.py b/src/messenger/attachments/entrypoints/errors.py index 1e6748d6..56e3e8e0 100644 --- a/src/messenger/attachments/entrypoints/errors.py +++ b/src/messenger/attachments/entrypoints/errors.py @@ -4,11 +4,13 @@ from messenger.attachments.domain.errors import ( InvalidFilename, PayloadTooLarge, UnsupportedImage, + UploadAborted, ) from messenger.shared.domain.errors import DomainError ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { InvalidFilename: ("invalid_request", 400), + UploadAborted: ("invalid_request", 400), ChatNotFound: ("not_found", 404), AttachmentNotFound: ("not_found", 404), PayloadTooLarge: ("payload_too_large", 413), diff --git a/src/messenger/attachments/entrypoints/http.py b/src/messenger/attachments/entrypoints/http.py index 14ec3623..ac31decd 100644 --- a/src/messenger/attachments/entrypoints/http.py +++ b/src/messenger/attachments/entrypoints/http.py @@ -1,14 +1,17 @@ import re +from collections.abc import AsyncIterator from typing import Annotated, Literal from uuid import UUID from dishka.integrations.fastapi import FromDishka, inject from fastapi import APIRouter, Query, Request, status from fastapi.responses import StreamingResponse +from starlette.requests import ClientDisconnect from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import UploadAttachment from messenger.attachments.application.queries import GetAttachment, OpenAttachment +from messenger.attachments.domain.errors import UploadAborted from messenger.attachments.entrypoints.schemas import AttachmentOut from messenger.shared.entrypoints.actor import Actor @@ -19,6 +22,14 @@ uploads_router = APIRouter(prefix="/api/v1/chats/{chat_id}/attachments", tags=[" attachments_router = APIRouter(prefix="/api/v1/attachments", tags=["attachments"]) +async def body_of(request: Request) -> AsyncIterator[bytes]: + try: + async for chunk in request.stream(): + yield chunk + except ClientDisconnect as error: + raise UploadAborted("клиент оборвал загрузку до конца тела") from error + + @uploads_router.post("", status_code=status.HTTP_201_CREATED) @inject async def upload_attachment( @@ -34,7 +45,7 @@ async def upload_attachment( chat_id=chat_id, filename=filename, content_type=request.headers.get("content-type"), - body=request.stream(), + body=body_of(request), ) ) return AttachmentOut.of(attachment) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index 09875073..a7ca5464 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -3,8 +3,12 @@ from collections.abc import AsyncIterator from dishka import Provider, Scope, provide from messenger.attachments.application.bus import AttachmentsCommandBus -from messenger.attachments.application.commands import UploadAttachment -from messenger.attachments.application.handlers import UploadAttachmentHandler +from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment +from messenger.attachments.application.handlers import ( + SweepAbandonedUploadsHandler, + UploadAttachmentHandler, + UploadSlots, +) from messenger.attachments.application.ports import ( AttachmentDirectory, AttachmentsUnitOfWork, @@ -52,6 +56,10 @@ class AttachmentsProvider(Provider): def images(self, settings: AttachmentsSettings) -> ImageInspector: return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency) + @provide(scope=Scope.APP) + def slots(self, settings: AttachmentsSettings) -> UploadSlots: + return UploadSlots(settings.upload_concurrency) + @provide(scope=Scope.APP) def directory(self, database: Database) -> AttachmentDirectory: return PostgresAttachmentDirectory(database) @@ -83,9 +91,22 @@ class AttachmentsProvider(Provider): clock: Clock, ids: IdGenerator, settings: AttachmentsSettings, + slots: UploadSlots, + logger: Logger, ) -> AttachmentsCommandBus: - handler = UploadAttachmentHandler( - uow, membership, images, storage, clock=clock, ids=ids, limit=settings.max_upload_bytes + upload = UploadAttachmentHandler( + uow, + membership, + images, + storage, + clock=clock, + ids=ids, + limit=settings.max_upload_bytes, + slots=slots, + logger=logger, ) - handlers: dict[type[Command], CommandHandler] = {UploadAttachment: handler} + handlers: dict[type[Command], CommandHandler] = { + UploadAttachment: upload, + SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock), + } return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py index b0bf792d..55ba5e5a 100644 --- a/src/messenger/attachments/entrypoints/journal.py +++ b/src/messenger/attachments/entrypoints/journal.py @@ -1,7 +1,7 @@ -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.shared.application.ports import Logger -LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes"}) +LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes", "count"}) class AttachmentsJournal: @@ -18,3 +18,11 @@ class AttachmentsJournal: actor_id=str(event.owner_id), size_bytes=event.size_bytes, ) + + async def on_attachment_discarded(self, event: AttachmentDiscarded) -> None: + self._logger.info( + "audit.attachment_discarded", + attachment_id=str(event.attachment_id), + chat_id=str(event.chat_id), + reason=event.reason, + ) diff --git a/src/messenger/attachments/entrypoints/sweeper.py b/src/messenger/attachments/entrypoints/sweeper.py new file mode 100644 index 00000000..63b08da4 --- /dev/null +++ b/src/messenger/attachments/entrypoints/sweeper.py @@ -0,0 +1,53 @@ +import asyncio +from contextlib import suppress +from datetime import timedelta + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import SweepAbandonedUploads +from messenger.attachments.config import AttachmentsSettings +from messenger.shared.application.ports import Logger + + +class UploadSweeper: + __slots__ = ("_command", "_container", "_interval", "_logger", "_task") + + def __init__( + self, container: AsyncContainer, settings: AttachmentsSettings, logger: Logger + ) -> None: + self._container = container + self._logger = logger + self._interval = settings.upload_sweep_interval_seconds + self._command = SweepAbandonedUploads( + grace=timedelta(seconds=settings.upload_grace_seconds), + limit=settings.upload_sweep_batch, + ) + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def sweep(self) -> int: + async with self._container() as scope: + bus = await scope.get(AttachmentsCommandBus) + return await bus.execute(self._command) + + async def _run(self) -> None: + while True: + await asyncio.sleep(self._interval) + try: + swept = await self.sweep() + except Exception as error: + self._logger.warning("attachments.sweep_failed", error=type(error).__name__) + continue + if swept: + self._logger.info("attachments.swept", count=swept) diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index c023234c..c47d0f18 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -1,3 +1,4 @@ +from datetime import datetime from uuid import UUID from asyncpg import ForeignKeyViolationError, Record @@ -5,6 +6,7 @@ from asyncpg import ForeignKeyViolationError, Record from messenger.attachments.domain.errors import ChatNotFound from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind +from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.postgres import ( Connection, Database, @@ -18,12 +20,19 @@ INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_byt VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) """ MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'" -READY = """ +COLUMNS = """ SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at FROM attachments -WHERE id = $1 AND status = 'ready' """ +READY = COLUMNS + "WHERE id = $1 AND status = 'ready'" +LOCK = COLUMNS + "WHERE id = $1 FOR UPDATE" +ABANDONED = ( + COLUMNS + + """WHERE status = 'pending' AND created_at < $1 +ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" +) +REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" def attachment_of(row: Record) -> Attachment: @@ -72,9 +81,23 @@ class PostgresAttachmentRepository: self._seen.append(attachment) async def save(self, attachment: Attachment) -> None: - await self._connection.execute( + done = await self._connection.execute( MARK, attachment.id, attachment.status.value, attachment.updated_at ) + if done == "UPDATE 0": + raise Unavailable("намерение загрузки уже снято уборкой") + self._seen.append(attachment) + + async def lock(self, attachment_id: UUID) -> Attachment | None: + row = await self._connection.fetchrow(LOCK, attachment_id) + return None if row is None else attachment_of(row) + + async def next_abandoned(self, before: datetime) -> Attachment | None: + row = await self._connection.fetchrow(ABANDONED, before) + return None if row is None else attachment_of(row) + + async def remove(self, attachment: Attachment) -> None: + await self._connection.execute(REMOVE, attachment.id) self._seen.append(attachment) diff --git a/src/messenger/main.py b/src/messenger/main.py index 41cfb09e..8bebfc4f 100644 --- a/src/messenger/main.py +++ b/src/messenger/main.py @@ -8,12 +8,13 @@ from fastapi import FastAPI from messenger.attachments.application.ports import ChatMembership from messenger.attachments.config import AttachmentsSettings -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.entrypoints.errors import ERROR_CODES as ATTACHMENT_ERRORS from messenger.attachments.entrypoints.http import UPLOAD_PATH, attachments_router, uploads_router from messenger.attachments.entrypoints.ioc import AttachmentsProvider from messenger.attachments.entrypoints.journal import LOG_FIELDS as ATTACHMENT_FIELDS from messenger.attachments.entrypoints.journal import AttachmentsJournal +from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.identity.config import IdentitySettings from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered from messenger.identity.entrypoints.authentication import challenge_for, guard @@ -76,6 +77,7 @@ def wire_events(bus: EventBus, logger: Logger) -> None: attachments = AttachmentsJournal(logger) bus.subscribe(AttachmentStored, attachments.on_attachment_stored) + bus.subscribe(AttachmentDiscarded, attachments.on_attachment_discarded) def create_app( @@ -102,16 +104,19 @@ def create_app( FastapiProvider(), *overrides, ) + sweeper = UploadSweeper(container, attachments, logger) @asynccontextmanager async def lifespan(_app: FastAPI) -> AsyncIterator[None]: bus = await container.get(EventBus) wire_events(bus, logger) bus.start() + sweeper.start() logger.info("service.started") try: yield finally: + await sweeper.stop() await bus.stop() await container.close() logger.info("service.stopped") @@ -142,4 +147,5 @@ def create_app( app.include_router(uploads_router) app.include_router(attachments_router) setup_dishka(container, app) + app.state.sweeper = sweeper return app diff --git a/student-tests/api/test_upload_consistency.py b/student-tests/api/test_upload_consistency.py new file mode 100644 index 00000000..30bc91cf --- /dev/null +++ b/student-tests/api/test_upload_consistency.py @@ -0,0 +1,244 @@ +import asyncio +from collections.abc import AsyncIterator +from typing import Any +from uuid import uuid4 + +import asyncpg +from httpx import AsyncClient +from starlette.types import Message +from test_attachments import storage, trio, upload + +from conftest import APPS, DATABASE, actor, keyed, resolve, running_app, scalar +from messenger.attachments.domain.events import AttachmentDiscarded +from messenger.attachments.entrypoints.sweeper import UploadSweeper +from messenger.shared.application.messagebus import EventBus +from pictures import SAMPLE + +AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1" + + +async def execute(sql: str, *args: Any) -> None: + connection = await asyncpg.connect(DATABASE["url"]) + try: + await connection.execute(sql, *args) + finally: + await connection.close() + + +def sweeper(http: AsyncClient) -> UploadSweeper: + return APPS[id(http)].state.sweeper + + +async def discarded(http: AsyncClient) -> list[AttachmentDiscarded]: + seen: list[AttachmentDiscarded] = [] + + async def keep(event: AttachmentDiscarded) -> None: + seen.append(event) + + (await resolve(http, EventBus)).project(AttachmentDiscarded, keep) + return seen + + +async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) -> str: + store = await storage(http) + store.failing = True + name = f"intent-{uuid4().hex}.png" + try: + assert (await upload(http, user, chat, filename=name)).status_code == 503 + finally: + store.failing = False + found = await scalar( + "SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name + ) + assert found is not None + store.objects[f"attachments/{found}"] = (SAMPLE, "image/png") + return found + + +async def exists(attachment_id: str) -> bool: + return await scalar("SELECT count(*) FROM attachments WHERE id = $1", attachment_id) == 1 + + +async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + store = await storage(client) + name = f"fail-{uuid4().hex}.png" + trigger = f"lab4_fail_{uuid4().hex[:8]}" + await execute( + f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$; + CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW + WHEN (NEW.filename = '{name}' AND NEW.status = 'ready') EXECUTE FUNCTION {trigger}()""" + ) + before, seen = set(store.objects), await discarded(client) + try: + response = await upload(client, alice, chat, filename=name) + finally: + await execute(f"DROP TRIGGER {trigger} ON attachments; DROP FUNCTION {trigger}()") + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert set(store.objects) == before + assert [event.reason for event in seen] == ["compensation"] + + +async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + store = await storage(client) + + async def swept_meanwhile(key: str) -> None: + await execute("DELETE FROM attachments WHERE object_key = $1", key) + + store.after_put = swept_meanwhile + before = set(store.objects) + try: + response = await upload(client, alice, chat, filename="late.png") + finally: + store.after_put = None + assert response.status_code == 503 + assert set(store.objects) == before + + +async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + pending = await intent(client, alice, chat) + await sweeper(client).sweep() + assert await exists(pending) + await execute(AGED, pending) + assert await sweeper(client).sweep() >= 1 + assert not await exists(pending) + assert f"attachments/{pending}" not in (await storage(client)).objects + + +async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None: + alice, bob, _, chat = await trio(client) + ready = (await upload(client, alice, chat)).json() + sent = await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "", "attachment_ids": [ready["id"]]}, + headers=keyed(alice), + ) + assert sent.status_code == 201 + old, fresh = await intent(client, alice, chat), await intent(client, alice, chat) + await execute(AGED, old) + await execute(AGED, ready["id"]) + assert await sweeper(client).sweep() == 1 + objects = (await storage(client)).objects + assert not await exists(old) + assert f"attachments/{old}" not in objects + assert await exists(fresh) + assert f"attachments/{fresh}" in objects + download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob)) + assert download.status_code == 200 + assert download.content == SAMPLE + assert await sweeper(client).sweep() == 0 + + +async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + pending = [await intent(client, alice, chat) for _ in range(5)] + for attachment_id in pending: + await execute(AGED, attachment_id) + seen = await discarded(client) + first, second = await asyncio.gather(sweeper(client).sweep(), sweeper(client).sweep()) + assert first + second == 5 + assert sorted(str(event.attachment_id) for event in seen) == sorted(pending) + assert {event.reason for event in seen} == {"abandoned"} + assert await sweeper(client).sweep() == 0 + objects = (await storage(client)).objects + assert not any(f"attachments/{attachment_id}" in objects for attachment_id in pending) + + +async def test_background_sweeper_removes_abandoned_intents() -> None: + async with running_app(upload_sweep_interval_seconds=0.05) as http: + alice, _, _, chat = await trio(http) + pending = await intent(http, alice, chat) + await execute(AGED, pending) + for _ in range(100): + if not await exists(pending): + break + await asyncio.sleep(0.05) + assert not await exists(pending) + + +async def test_upload_waits_for_a_free_slot() -> None: + async with running_app(upload_concurrency=1) as http: + alice, _, _, chat = await trio(http) + gate, holding, read = asyncio.Event(), asyncio.Event(), [] + + async def held() -> AsyncIterator[bytes]: + read.append("first") + holding.set() + yield SAMPLE[:1000] + await gate.wait() + yield SAMPLE[1000:] + + async def queued() -> AsyncIterator[bytes]: + read.append("second") + yield SAMPLE + + first = asyncio.create_task(upload(http, alice, chat, held())) + await holding.wait() + second = asyncio.create_task(upload(http, alice, chat, queued())) + await asyncio.sleep(0.3) + assert read == ["first"] + gate.set() + assert [(await first).status_code, (await second).status_code] == [201, 201] + assert read == ["first", "second"] + + +async def test_aborted_upload_is_a_client_error(client: AsyncClient) -> None: + alice, _, _, chat = await trio(client) + name = f"abort-{uuid4().hex}.png" + headers = [(k.lower().encode(), v.encode()) for k, v in actor(alice).items()] + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": f"/api/v1/chats/{chat['id']}/attachments", + "raw_path": f"/api/v1/chats/{chat['id']}/attachments".encode(), + "query_string": f"filename={name}".encode(), + "root_path": "", + "headers": [ + *headers, + (b"content-type", b"image/png"), + (b"content-length", b"1000000"), + ], + "client": ("127.0.0.1", 50000), + "server": ("testserver", 80), + } + messages = iter( + [ + {"type": "http.request", "body": SAMPLE[:1000], "more_body": True}, + {"type": "http.disconnect"}, + ] + ) + sent: list[Message] = [] + + async def receive() -> dict[str, Any]: + return next(messages, {"type": "http.disconnect"}) + + async def send(message: Message) -> None: + sent.append(message) + + await APPS[id(client)](scope, receive, send) + assert sent[0]["status"] == 400 + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + + +async def test_stalled_upload_times_out_and_frees_the_slot() -> None: + async with running_app(upload_concurrency=1, request_timeout_seconds=0.5) as http: + alice, _, _, chat = await trio(http) + name = f"stalled-{uuid4().hex}.png" + + async def stalled() -> AsyncIterator[bytes]: + yield SAMPLE[:1000] + await asyncio.Event().wait() + + response = await upload(http, alice, chat, stalled(), filename=name) + assert response.status_code == 503 + assert response.json()["error"]["code"] == "unavailable" + assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert (await upload(http, alice, chat)).status_code == 201 diff --git a/student-tests/fakes.py b/student-tests/fakes.py index fc59c177..e0d3633e 100644 --- a/student-tests/fakes.py +++ b/student-tests/fakes.py @@ -1,4 +1,4 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable, Callable from datetime import datetime, timedelta from typing import Any @@ -36,16 +36,19 @@ async def pieces(data: bytes, size: int = 65_536) -> AsyncIterator[bytes]: class MemoryObjectStorage: - __slots__ = ("failing", "objects") + __slots__ = ("after_put", "failing", "objects") def __init__(self) -> None: self.objects: dict[str, tuple[bytes, str]] = {} self.failing = False + self.after_put: Callable[[str], Awaitable[None]] | None = None async def put(self, key: str, data: bytes, content_type: str) -> None: if self.failing: raise Unavailable("S3 недоступно") self.objects[key] = (bytes(data), content_type) + if self.after_put is not None: + await self.after_put(key) async def open(self, key: str) -> ObjectContent: if self.failing: diff --git a/student-tests/unit/test_attachments_domain.py b/student-tests/unit/test_attachments_domain.py index 3063a278..e26d3101 100644 --- a/student-tests/unit/test_attachments_domain.py +++ b/student-tests/unit/test_attachments_domain.py @@ -4,7 +4,7 @@ from uuid import uuid4 import pytest from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage -from messenger.attachments.domain.events import AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind @@ -67,3 +67,24 @@ def test_attachment_is_pending_until_stored() -> None: events = attachment.collect_events() assert [type(event) for event in events] == [AttachmentStored] assert attachment.collect_events() == [] + + +def test_discarded_intent_records_its_reason() -> None: + attachment = Attachment.stage( + attachment_id=uuid4(), + chat_id=uuid4(), + owner_id=uuid4(), + filename=Filename("a.png"), + image=ImageInfo(ImageKind.PNG, 1, 1), + size_bytes=1, + sha256="0" * 64, + now=MOMENT, + ) + attachment.discard("abandoned") + (event,) = attachment.collect_events() + assert isinstance(event, AttachmentDiscarded) + assert (event.attachment_id, event.chat_id, event.reason) == ( + attachment.id, + attachment.chat_id, + "abandoned", + )