diff --git a/openspec/changes/add-lab5-async-attachments/tasks.md b/openspec/changes/add-lab5-async-attachments/tasks.md index 9d2849ec..4a57c854 100644 --- a/openspec/changes/add-lab5-async-attachments/tasks.md +++ b/openspec/changes/add-lab5-async-attachments/tasks.md @@ -5,25 +5,25 @@ ## 2. Зона `attachments`: домен и приложение -- [ ] 2.1 Домен: статусы `queued/processing/failed`, `Attachment.enqueue`, `Variant`, геометрия `fit`, `Job` и его сообщение по `jobs.schema.json`, `Lease`, `RetryPolicy`, `FailureCode`, `Outcome`, события `AttachmentQueued`, `AttachmentReady`, `AttachmentFailed`, `JobRescheduled`, ошибка `AttachmentNotReady` -- [ ] 2.2 Порты: `ObjectStorage.read`/`copy`, `VariantRenderer`, `JobPublisher`, `Faults`, репозитории `jobs` и `outbox` в `AttachmentsUnitOfWork`, `AttachmentDirectory.visible` -- [ ] 2.3 Upload: `queued` вместо `ready`, задание и outbox в той же транзакции; запросы `GetAttachment`/`OpenAttachment` с `409` до `ready` и вариантами опубликованной версии -- [ ] 2.4 Обработчики `ProcessJob` (захват, staging → оригинал, рендер, запись вариантов, фиксация с fencing, повтор/финальная ошибка), `PublishOutbox`, `RecoverExpiredLeases`, `CleanStaging` -- [ ] 2.5 Проверка группы: unit-тесты `fit`, `RetryPolicy`, сообщения задания против `contracts/jobs.schema.json` +- [x] 2.1 Домен: статусы `queued/processing/failed`, `Attachment.enqueue`, `Variant`, геометрия `fit`, `Job` и его сообщение по `jobs.schema.json`, `Lease`, `RetryPolicy`, `FailureCode`, `Outcome`, события `AttachmentQueued`, `AttachmentReady`, `AttachmentFailed`, `JobRescheduled`, ошибка `AttachmentNotReady` +- [x] 2.2 Порты: `ObjectStorage.read`/`copy`, `VariantRenderer`, `JobPublisher`, `Faults`, репозитории `jobs` и `outbox` в `AttachmentsUnitOfWork`, `AttachmentDirectory.visible` +- [x] 2.3 Upload: `queued` вместо `ready`, задание и outbox в той же транзакции; запросы `GetAttachment`/`OpenAttachment` с `409` до `ready` и вариантами опубликованной версии +- [x] 2.4 Обработчики `ProcessJob` (захват, staging → оригинал, рендер, запись вариантов, фиксация с fencing, повтор/финальная ошибка), `PublishOutbox`, `RecoverExpiredLeases`, `CleanStaging` +- [x] 2.5 Проверка группы: unit-тесты `fit`, `RetryPolicy`, сообщения задания против `contracts/jobs.schema.json` ## 3. Зона `attachments`: инфраструктура -- [ ] 3.1 PostgreSQL: вложение с ключами и вариантами, задания, outbox, события, захват аренды, фиксация с fencing и `published_version < v`, повтор, финальная ошибка, истёкшие аренды, staging к уборке -- [ ] 3.2 S3: `read` и `copy`; Pillow: `PillowVariantRenderer` (EXIF orientation, 16 бит, alpha на белом, `fit`, JPEG без метаданных, классификация `invalid_image`) -- [ ] 3.3 FastStream: адаптер `JobPublisher` (exchange без объявления, `persist`, `mandatory`, confirm, `message_id`), `FAULT_POINT` с точкой `transient_error`; зависимости `faststream[rabbit]`, `dishka-faststream` в `uv.lock` -- [ ] 3.4 Проверка группы: тест рендера (fixture 320×200 → 256×160, RGBA на белом, EXIF 6, 17×1000 → 4×256, битые данные → `invalid_image`) +- [x] 3.1 PostgreSQL: вложение с ключами и вариантами, задания, outbox, события, захват аренды, фиксация с fencing и `published_version < v`, повтор, финальная ошибка, истёкшие аренды, staging к уборке +- [x] 3.2 S3: `read` и `copy`; Pillow: `PillowVariantRenderer` (EXIF orientation, 16 бит, alpha на белом, `fit`, JPEG без метаданных, классификация `invalid_image`) +- [x] 3.3 FastStream: адаптер `JobPublisher` (exchange без объявления, `persist`, `mandatory`, confirm, `message_id`), `FAULT_POINT` с точкой `transient_error`; зависимости `faststream[rabbit]`, `dishka-faststream` в `uv.lock` +- [x] 3.4 Проверка группы: тест рендера (fixture 320×200 → 256×160, RGBA на белом, EXIF 6, 17×1000 → 4×256, битые данные → `invalid_image`) ## 4. Точки входа и сборка процессов -- [ ] 4.1 HTTP: `202` upload, `variant` из четырёх значений, `409`, схема `AttachmentOut` со статусом и вариантами, реестр ошибок, журнал -- [ ] 4.2 `PipelineSettings`, `PipelineProvider`, точка входа worker (FastStream-подписчик, проверка сообщения, `ack`/`reject`/`nack`), циклы relay, heartbeat -- [ ] 4.3 Сборки `relay.py` и `worker.py`, тест графа импортов: сборки процессов знают одну зону, `shared/` не импортирует FastStream -- [ ] 4.4 Проверка группы: тесты pipeline против одноразового PostgreSQL с памятью вместо S3 — `ready`, дубль после `ready` без новых строк, параллельные копии, временная ошибка → `queued` и новая строка outbox, исчерпание → `retry_exhausted`, `invalid_image` без повторов, fencing чужой аренды, возврат истёкшей аренды, уборка staging, публикация outbox +- [x] 4.1 HTTP: `202` upload, `variant` из четырёх значений, `409`, схема `AttachmentOut` со статусом и вариантами, реестр ошибок, журнал +- [x] 4.2 `PipelineSettings`, `PipelineProvider`, точка входа worker (FastStream-подписчик, проверка сообщения, `ack`/`reject`/`nack`), циклы relay, heartbeat +- [x] 4.3 Сборки `relay.py` и `worker.py`, тест графа импортов: сборки процессов знают одну зону, `shared/` не импортирует FastStream +- [x] 4.4 Проверка группы: тесты pipeline против одноразового PostgreSQL с памятью вместо S3 — `ready`, дубль после `ready` без новых строк, параллельные копии, временная ошибка → `queued` и новая строка outbox, исчерпание → `retry_exhausted`, `invalid_image` без повторов, fencing чужой аренды, возврат истёкшей аренды, уборка staging, публикация outbox ## 5. Связь зон diff --git a/pyproject.toml b/pyproject.toml index 1b2e3f49..d1b68b6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ dependencies = [ "redis>=8.1.0", "aiobotocore>=3.10.0", "pillow>=12.0.0", + "faststream[rabbit]>=0.7.2,<0.8", + "dishka-faststream>=0.1", ] [dependency-groups] @@ -88,6 +90,8 @@ ignore = [ "src/messenger/shared/config.py" = ["ARG002", "ARG003"] "src/messenger/main.py" = ["TID251"] "src/messenger/migrate.py" = ["TID251"] +"src/messenger/relay.py" = ["TID251"] +"src/messenger/worker.py" = ["TID251"] "src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"] [tool.ruff.lint.flake8-tidy-imports.banned-api] diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py index b42141fa..b7543f3d 100644 --- a/src/messenger/attachments/application/commands.py +++ b/src/messenger/attachments/application/commands.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from datetime import timedelta from uuid import UUID +from messenger.attachments.domain.jobs import Job from messenger.shared.application.commands import Command @@ -19,3 +20,23 @@ class UploadAttachment(Command): class SweepAbandonedUploads(Command): grace: timedelta limit: int + + +@dataclass(frozen=True, slots=True) +class ProcessJob(Command): + job: Job + + +@dataclass(frozen=True, slots=True) +class PublishOutbox(Command): + limit: int + + +@dataclass(frozen=True, slots=True) +class RecoverExpiredLeases(Command): + limit: int + + +@dataclass(frozen=True, slots=True) +class CleanStaging(Command): + limit: int diff --git a/src/messenger/attachments/application/handlers.py b/src/messenger/attachments/application/handlers.py index 36e767b5..0ff449aa 100644 --- a/src/messenger/attachments/application/handlers.py +++ b/src/messenger/attachments/application/handlers.py @@ -10,6 +10,7 @@ from messenger.attachments.application.ports import ( ObjectStorage, ) from messenger.attachments.domain.errors import ChatNotFound, PayloadTooLarge +from messenger.attachments.domain.jobs import Job from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageKind from messenger.shared.application.ports import Clock, IdGenerator, Logger @@ -35,7 +36,8 @@ async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, s async def drop( uow: AttachmentsUnitOfWork, storage: ObjectStorage, attachment: Attachment, reason: str ) -> None: - await storage.delete(attachment.object_key) + if attachment.staging_key is not None: + await storage.delete(attachment.staging_key) attachment.discard(reason) await uow.attachments.remove(attachment) await uow.commit() @@ -49,6 +51,7 @@ class UploadAttachmentHandler: "_limit", "_logger", "_membership", + "_pipeline_version", "_slots", "_storage", "_uow", @@ -66,6 +69,7 @@ class UploadAttachmentHandler: limit: int, slots: UploadSlots, logger: Logger, + pipeline_version: int = 1, ) -> None: self._uow = uow self._membership = membership @@ -76,6 +80,7 @@ class UploadAttachmentHandler: self._limit = limit self._slots = slots self._logger = logger + self._pipeline_version = pipeline_version async def __call__(self, command: UploadAttachment) -> Attachment: if not await self._membership.is_member(command.chat_id, command.owner_id): @@ -84,10 +89,13 @@ class UploadAttachmentHandler: declared = ImageKind.declared(command.content_type) async with self._slots: attachment = await self._write(command, filename, declared) - attachment.store(self._clock.now()) + now = self._clock.now() + attachment.enqueue(now) + job = Job(self._ids.new_id(), attachment.id, self._pipeline_version, 1, now) try: async with self._uow.transaction(): await self._uow.attachments.save(attachment) + await self._uow.jobs.add(job) await self._uow.commit() except Exception: await self._compensate(attachment) @@ -112,7 +120,7 @@ class UploadAttachmentHandler: async with self._uow.transaction(): await self._uow.attachments.add(attachment) await self._uow.commit() - await self._storage.put(attachment.object_key, data, attachment.content_type) + await self._storage.put(str(attachment.staging_key), data, attachment.content_type) return attachment async def _compensate(self, staged: Attachment) -> None: @@ -120,7 +128,7 @@ class UploadAttachmentHandler: async with self._uow.transaction(): current = await self._uow.attachments.lock(staged.id) if current is None: - await self._storage.delete(staged.object_key) + await self._storage.delete(str(staged.staging_key)) elif current.status is AttachmentStatus.PENDING: await drop(self._uow, self._storage, current, "compensation") except Exception as error: diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py new file mode 100644 index 00000000..803c0961 --- /dev/null +++ b/src/messenger/attachments/application/pipeline.py @@ -0,0 +1,219 @@ +import asyncio +from datetime import timedelta +from hashlib import sha256 + +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.application.ports import ( + AttachmentsUnitOfWork, + Faults, + JobPublisher, + ObjectStorage, + VariantRenderer, +) +from messenger.attachments.domain.errors import UnreadableImage +from messenger.attachments.domain.events import JobSettled +from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy +from messenger.attachments.domain.models import original_key +from messenger.attachments.domain.variants import Variant, variant_key +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Clock, IdGenerator + +TRANSIENT_FAULT = "transient_error" + + +def settled( + job: Job, + outcome: Outcome, + *, + error_code: str | None = None, + delay_seconds: float | None = None, +) -> JobSettled: + return JobSettled( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt, + outcome=outcome.value, + error_code=error_code, + delay_seconds=delay_seconds, + ) + + +class ProcessJobHandler: + __slots__ = ("_clock", "_faults", "_ids", "_lease", "_policy", "_renderer", "_storage", "_uow") + + def __init__( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + renderer: VariantRenderer, + faults: Faults, + *, + clock: Clock, + ids: IdGenerator, + policy: RetryPolicy, + lease: timedelta, + ) -> None: + self._uow = uow + self._storage = storage + self._renderer = renderer + self._faults = faults + self._clock = clock + self._ids = ids + self._policy = policy + self._lease = lease + + async def __call__(self, command: ProcessJob) -> Outcome: + async with self._uow.transaction(): + claim = await self._uow.jobs.claim(command.job, self._ids.new_id(), self._lease) + if claim is None: + await self._uow.jobs.note(command.job, Outcome.DUPLICATE) + self._uow.record(settled(command.job, Outcome.DUPLICATE)) + await self._uow.commit() + if claim is None: + return Outcome.DUPLICATE + try: + variants = await self._produce(claim) + except UnreadableImage as error: + return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error) + except Unavailable as error: + return await self._retry(claim.lease, error) + except Exception as error: + return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error) + return await self._complete(claim.lease, variants) + + async def _produce(self, claim: Claim) -> list[Variant]: + job = claim.lease.job + self._faults.reach(TRANSIENT_FAULT) + data = await self._storage.read(claim.source_key) + renditions = await self._renderer.render(data, job.pipeline_version) + if claim.staged: + await self._storage.copy(claim.source_key, original_key(job.attachment_id)) + variants: list[Variant] = [] + for rendition in renditions: + digest = sha256(rendition.data).hexdigest() + key = variant_key( + job.attachment_id, job.pipeline_version, rendition.name, digest, rendition.extension + ) + await self._storage.put(key, rendition.data, rendition.content_type) + variants.append( + Variant( + name=rendition.name, + content_type=rendition.content_type, + size_bytes=len(rendition.data), + width=rendition.width, + height=rendition.height, + sha256=digest, + object_key=key, + ) + ) + return variants + + async def _complete(self, lease: Lease, variants: list[Variant]) -> Outcome: + async with self._uow.transaction(): + outcome = await self._uow.jobs.complete( + lease, variants, original_key(lease.job.attachment_id) + ) + self._uow.record(settled(lease.job, outcome)) + await self._uow.commit() + return outcome + + async def _retry(self, lease: Lease, error: Exception) -> Outcome: + delay = self._policy.delay(lease.job.attempt) + if delay is None: + return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error) + async with self._uow.transaction(): + at = self._clock.now() + delay + job = await self._uow.jobs.reschedule( + lease, at, Outcome.RESCHEDULED, type(error).__name__ + ) + outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED + if job is None: + await self._uow.jobs.note(lease.job, outcome) + self._uow.record(settled(lease.job, outcome, delay_seconds=delay.total_seconds())) + await self._uow.commit() + return outcome + + async def _fail(self, lease: Lease, code: FailureCode, error: Exception) -> Outcome: + async with self._uow.transaction(): + done = await self._uow.jobs.fail(lease, code, type(error).__name__) + outcome = Outcome.FAILED if done else Outcome.FENCED + if not done: + await self._uow.jobs.note(lease.job, outcome) + self._uow.record(settled(lease.job, outcome, error_code=code.value)) + await self._uow.commit() + return outcome + + +class PublishOutboxHandler: + __slots__ = ("_publisher", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher) -> None: + self._uow = uow + self._publisher = publisher + + async def __call__(self, command: PublishOutbox) -> int: + failure: Unavailable | None = None + published: list[int] = [] + async with self._uow.transaction(): + for entry in await self._uow.outbox.pending(command.limit): + try: + await self._publisher.publish(entry.job_id, entry.body) + except Unavailable as error: + failure = error + break + published.append(entry.id) + if published: + await self._uow.outbox.published(published) + await self._uow.commit() + if failure is not None: + raise failure + return len(published) + + +class RecoverExpiredLeasesHandler: + __slots__ = ("_clock", "_policy", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, clock: Clock, policy: RetryPolicy) -> None: + self._uow = uow + self._clock = clock + self._policy = policy + + async def __call__(self, command: RecoverExpiredLeases) -> int: + async with self._uow.transaction(): + leases = await self._uow.jobs.expired(command.limit) + for lease in leases: + if self._policy.delay(lease.job.attempt) is None: + await self._uow.jobs.fail( + lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value + ) + outcome = Outcome.FAILED + else: + await self._uow.jobs.reschedule( + lease, self._clock.now(), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value + ) + outcome = Outcome.LEASE_EXPIRED + self._uow.record(settled(lease.job, outcome)) + await self._uow.commit() + return len(leases) + + +class CleanStagingHandler: + __slots__ = ("_storage", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage) -> None: + self._uow = uow + self._storage = storage + + async def __call__(self, command: CleanStaging) -> int: + async with self._uow.transaction(): + staged = await self._uow.attachments.staged(command.limit) + await asyncio.gather(*(self._storage.delete(item.key) for item in staged)) + await self._uow.attachments.unstage([item.attachment_id for item in staged]) + await self._uow.commit() + return len(staged) diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index b1339546..dd2760a9 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -1,11 +1,13 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from typing import Protocol, runtime_checkable from uuid import UUID +from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome from messenger.attachments.domain.models import Attachment from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.domain.variants import Variant from messenger.shared.application.ports import UnitOfWork @@ -15,12 +17,39 @@ class ObjectContent: chunks: AsyncIterator[bytes] +@dataclass(frozen=True, slots=True) +class Rendition: + name: str + content_type: str + extension: str + data: bytes + width: int + height: int + + +@dataclass(frozen=True, slots=True) +class OutboxEntry: + id: int + job_id: UUID + body: bytes + + +@dataclass(frozen=True, slots=True) +class StagedObject: + attachment_id: UUID + key: str + + @runtime_checkable class ObjectStorage(Protocol): async def put(self, key: str, data: bytes, content_type: str) -> None: ... async def open(self, key: str) -> ObjectContent: ... + async def read(self, key: str) -> bytes: ... + + async def copy(self, source: str, target: str) -> None: ... + async def delete(self, key: str) -> None: ... @@ -29,6 +58,21 @@ class ImageInspector(Protocol): async def inspect(self, data: bytes, declared: ImageKind) -> ImageInfo: ... +@runtime_checkable +class VariantRenderer(Protocol): + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: ... + + +@runtime_checkable +class JobPublisher(Protocol): + async def publish(self, job_id: UUID, body: bytes) -> None: ... + + +@runtime_checkable +class Faults(Protocol): + def reach(self, point: str) -> None: ... + + @runtime_checkable class ChatMembership(Protocol): async def is_member(self, chat_id: UUID, user_id: UUID) -> bool: ... @@ -46,13 +90,53 @@ class AttachmentRepository(Protocol): async def remove(self, attachment: Attachment) -> None: ... + async def staged(self, limit: int) -> list[StagedObject]: ... + + async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ... + + +@runtime_checkable +class JobRepository(Protocol): + async def add(self, job: Job) -> None: ... + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: ... + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: ... + + async def reschedule( + self, lease: Lease, at: datetime, kind: Outcome, error: str + ) -> Job | None: ... + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: ... + + async def note(self, job: Job, kind: Outcome) -> None: ... + + async def expired(self, limit: int) -> list[Lease]: ... + + +@runtime_checkable +class OutboxRepository(Protocol): + async def pending(self, limit: int) -> list[OutboxEntry]: ... + + async def published(self, entry_ids: Sequence[int]) -> None: ... + @runtime_checkable class AttachmentsUnitOfWork(UnitOfWork, Protocol): @property def attachments(self) -> AttachmentRepository: ... + @property + def jobs(self) -> JobRepository: ... + + @property + def outbox(self) -> OutboxRepository: ... + @runtime_checkable class AttachmentDirectory(Protocol): - async def ready(self, attachment_id: UUID) -> Attachment | None: ... + async def visible(self, attachment_id: UUID) -> Attachment | None: ... + + async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... diff --git a/src/messenger/attachments/application/queries.py b/src/messenger/attachments/application/queries.py index d88c6740..973d759d 100644 --- a/src/messenger/attachments/application/queries.py +++ b/src/messenger/attachments/application/queries.py @@ -7,10 +7,12 @@ from messenger.attachments.application.ports import ( ObjectContent, ObjectStorage, ) -from messenger.attachments.domain.errors import AttachmentNotFound +from messenger.attachments.domain.errors import AttachmentNotFound, AttachmentNotReady from messenger.attachments.domain.models import Attachment from messenger.shared.application.ports import Logger +ORIGINAL = "original" + class GetAttachment: __slots__ = ("_attachments", "_membership") @@ -20,7 +22,7 @@ class GetAttachment: self._membership = membership async def __call__(self, attachment_id: UUID, user_id: UUID) -> Attachment: - attachment = await self._attachments.ready(attachment_id) + attachment = await self._attachments.visible(attachment_id) if attachment is None or not await self._membership.is_member(attachment.chat_id, user_id): raise AttachmentNotFound("вложение не найдено") return attachment @@ -35,12 +37,21 @@ class OpenAttachment: self._logger = logger async def __call__( - self, attachment_id: UUID, user_id: UUID - ) -> tuple[Attachment, ObjectContent]: + self, attachment_id: UUID, user_id: UUID, variant: str = ORIGINAL + ) -> tuple[str, ObjectContent]: attachment = await self._get(attachment_id, user_id) + if not attachment.ready: + raise AttachmentNotReady("вложение ещё не обработано или обработка не удалась") + if variant == ORIGINAL: + key, content_type = attachment.original_key, attachment.content_type + else: + found = attachment.variant(variant) + if found is None: + raise AttachmentNotFound("у вложения нет такого варианта") + key, content_type = found.object_key, found.content_type try: - content = await self._storage.open(attachment.object_key) + content = await self._storage.open(str(key)) except ObjectMissingError as error: self._logger.warning("attachment.object_missing", attachment_id=str(attachment.id)) raise AttachmentNotFound("вложение не найдено") from error - return attachment, content + return content_type, content diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py index 956e4d95..458408e4 100644 --- a/src/messenger/attachments/config.py +++ b/src/messenger/attachments/config.py @@ -1,3 +1,5 @@ +from pathlib import Path + from pydantic import Field, SecretStr from messenger.shared.config import Settings @@ -22,3 +24,22 @@ class AttachmentsSettings(Settings): upload_grace_seconds: int = Field(default=900, ge=60) upload_sweep_interval_seconds: float = Field(default=30.0, gt=0) upload_sweep_batch: int = Field(default=100, ge=1, le=10_000) + + +class PipelineSettings(AttachmentsSettings): + broker_url: SecretStr = Field(min_length=8) + worker_concurrency: int = Field(default=2, ge=1, le=32) + job_max_attempts: int = Field(default=5, ge=1, le=20) + job_timeout_seconds: float = Field(default=60.0, ge=1, le=290) + job_backoff_base_seconds: float = Field(default=2.0, gt=0, le=60) + job_backoff_cap_seconds: float = Field(default=60.0, gt=0, le=3600) + staging_retention_seconds: int = Field(default=86_400, ge=60) + outbox_poll_seconds: float = Field(default=1.0, gt=0, le=10) + outbox_batch: int = Field(default=50, ge=1, le=1000) + sweep_interval_seconds: float = Field(default=5.0, gt=0, le=300) + sweep_batch: int = Field(default=100, ge=1, le=10_000) + publish_timeout_seconds: float = Field(default=5.0, gt=0, le=60) + shutdown_grace_seconds: float = Field(default=5.0, gt=0, le=9) + heartbeat_file: Path | None = None + heartbeat_interval_seconds: float = Field(default=5.0, gt=0, le=10) + fault_point: str = "" diff --git a/src/messenger/attachments/domain/errors.py b/src/messenger/attachments/domain/errors.py index bec7ffdf..f69bf779 100644 --- a/src/messenger/attachments/domain/errors.py +++ b/src/messenger/attachments/domain/errors.py @@ -25,5 +25,13 @@ class AttachmentNotFound(AttachmentError): pass +class AttachmentNotReady(AttachmentError): + pass + + class UploadAborted(AttachmentError): pass + + +class UnreadableImage(AttachmentError): + pass diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py index f12bbe97..58a7c53b 100644 --- a/src/messenger/attachments/domain/events.py +++ b/src/messenger/attachments/domain/events.py @@ -5,12 +5,12 @@ from messenger.shared.domain.events import DomainEvent @dataclass(frozen=True, slots=True) -class AttachmentStored(DomainEvent): +class AttachmentQueued(DomainEvent): attachment_id: UUID chat_id: UUID owner_id: UUID size_bytes: int - name: str = field(init=False, default="attachment.stored") + name: str = field(init=False, default="attachment.queued") @dataclass(frozen=True, slots=True) @@ -19,3 +19,15 @@ class AttachmentDiscarded(DomainEvent): chat_id: UUID reason: str name: str = field(init=False, default="attachment.discarded") + + +@dataclass(frozen=True, slots=True) +class JobSettled(DomainEvent): + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + outcome: str + error_code: str | None = None + delay_seconds: float | None = None + name: str = field(init=False, default="job.settled") diff --git a/src/messenger/attachments/domain/jobs.py b/src/messenger/attachments/domain/jobs.py new file mode 100644 index 00000000..e53d618d --- /dev/null +++ b/src/messenger/attachments/domain/jobs.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass, replace +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from typing import Any +from uuid import UUID + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 + + +class FailureCode(StrEnum): + INVALID_IMAGE = "invalid_image" + PROCESSING_FAILED = "processing_failed" + RETRY_EXHAUSTED = "retry_exhausted" + + +class Outcome(StrEnum): + DONE = "done" + DUPLICATE = "duplicate_noop" + FENCED = "fenced" + SUPERSEDED = "superseded" + RESCHEDULED = "retry_scheduled" + LEASE_EXPIRED = "lease_expired" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class Job: + id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + created_at: datetime + + def payload(self) -> dict[str, Any]: + stamp = self.created_at.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return { + "schema_version": SCHEMA_VERSION, + "job_id": str(self.id), + "type": JOB_TYPE, + "attachment_id": str(self.attachment_id), + "pipeline_version": self.pipeline_version, + "attempt": self.attempt, + "created_at": stamp, + } + + def retried(self, attempt: int) -> Job: + return replace(self, attempt=attempt) + + +@dataclass(frozen=True, slots=True) +class Lease: + job: Job + token: UUID + + +@dataclass(frozen=True, slots=True) +class Claim: + lease: Lease + source_key: str + staged: bool + + +@dataclass(frozen=True, slots=True) +class RetryPolicy: + max_attempts: int + base_seconds: float + cap_seconds: float + + def delay(self, attempt: int) -> timedelta | None: + if attempt >= self.max_attempts: + return None + return timedelta(seconds=min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1))) diff --git a/src/messenger/attachments/domain/models.py b/src/messenger/attachments/domain/models.py index fef585eb..cc917040 100644 --- a/src/messenger/attachments/domain/models.py +++ b/src/messenger/attachments/domain/models.py @@ -3,16 +3,25 @@ from datetime import datetime from enum import StrEnum from uuid import UUID -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued from messenger.attachments.domain.values import Filename, ImageInfo +from messenger.attachments.domain.variants import Variant from messenger.shared.domain.events import DomainEvent -OBJECT_PREFIX = "attachments/" +STAGING_PREFIX = "staging/" +ORIGINALS_PREFIX = "originals/" class AttachmentStatus(StrEnum): PENDING = "pending" + QUEUED = "queued" + PROCESSING = "processing" READY = "ready" + FAILED = "failed" + + +def original_key(attachment_id: UUID) -> str: + return f"{ORIGINALS_PREFIX}{attachment_id}" @dataclass(slots=True) @@ -24,10 +33,14 @@ class Attachment: image: ImageInfo size_bytes: int sha256: str - object_key: str status: AttachmentStatus created_at: datetime updated_at: datetime + staging_key: str | None = None + original_key: str | None = None + published_version: int = 0 + error_code: str | None = None + variants: tuple[Variant, ...] = () _events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False) @classmethod @@ -51,21 +64,28 @@ class Attachment: image=image, size_bytes=size_bytes, sha256=sha256, - object_key=f"{OBJECT_PREFIX}{attachment_id}", status=AttachmentStatus.PENDING, created_at=now, updated_at=now, + staging_key=f"{STAGING_PREFIX}{attachment_id}", ) @property def content_type(self) -> str: return self.image.kind.value - def store(self, now: datetime) -> None: - self.status = AttachmentStatus.READY + @property + def ready(self) -> bool: + return self.status is AttachmentStatus.READY + + def variant(self, name: str) -> Variant | None: + return next((variant for variant in self.variants if variant.name == name), None) + + def enqueue(self, now: datetime) -> None: + self.status = AttachmentStatus.QUEUED self.updated_at = max(now, self.created_at) self._events.append( - AttachmentStored( + AttachmentQueued( attachment_id=self.id, chat_id=self.chat_id, owner_id=self.owner_id, diff --git a/src/messenger/attachments/domain/variants.py b/src/messenger/attachments/domain/variants.py new file mode 100644 index 00000000..b435f9b4 --- /dev/null +++ b/src/messenger/attachments/domain/variants.py @@ -0,0 +1,31 @@ +from dataclasses import dataclass +from uuid import UUID + +THUMBNAIL = "thumbnail" +THUMBNAIL_BOX = 256 +VARIANTS_PREFIX = "variants/" + + +@dataclass(frozen=True, slots=True) +class Variant: + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + object_key: str + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + if max(width, height) <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def variant_key( + attachment_id: UUID, pipeline_version: int, name: str, sha256: str, extension: str +) -> str: + return f"{VARIANTS_PREFIX}{attachment_id}/{pipeline_version}/{name}-{sha256[:16]}.{extension}" diff --git a/src/messenger/attachments/entrypoints/errors.py b/src/messenger/attachments/entrypoints/errors.py index 56e3e8e0..374038cb 100644 --- a/src/messenger/attachments/entrypoints/errors.py +++ b/src/messenger/attachments/entrypoints/errors.py @@ -1,5 +1,6 @@ from messenger.attachments.domain.errors import ( AttachmentNotFound, + AttachmentNotReady, ChatNotFound, InvalidFilename, PayloadTooLarge, @@ -13,6 +14,7 @@ ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { UploadAborted: ("invalid_request", 400), ChatNotFound: ("not_found", 404), AttachmentNotFound: ("not_found", 404), + AttachmentNotReady: ("conflict", 409), PayloadTooLarge: ("payload_too_large", 413), UnsupportedImage: ("unsupported_media_type", 415), } diff --git a/src/messenger/attachments/entrypoints/heartbeat.py b/src/messenger/attachments/entrypoints/heartbeat.py new file mode 100644 index 00000000..707ce11d --- /dev/null +++ b/src/messenger/attachments/entrypoints/heartbeat.py @@ -0,0 +1,44 @@ +import asyncio +from collections.abc import Awaitable, Callable +from contextlib import suppress +from pathlib import Path + +from messenger.shared.application.ports import Logger + + +class Heartbeat: + __slots__ = ("_interval", "_logger", "_path", "_probe", "_task") + + def __init__( + self, + path: Path | None, + probe: Callable[[], Awaitable[bool]], + interval: float, + logger: Logger, + ) -> None: + self._path = path + self._probe = probe + self._interval = interval + self._logger = logger + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + if self._path is not None: + self._task = asyncio.create_task(self._run(self._path)) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _run(self, path: Path) -> None: + while True: + try: + if await self._probe(): + await asyncio.to_thread(path.touch) + except Exception as error: + self._logger.warning("heartbeat.failed", error=type(error).__name__) + await asyncio.sleep(self._interval) diff --git a/src/messenger/attachments/entrypoints/http.py b/src/messenger/attachments/entrypoints/http.py index ac31decd..9899fecd 100644 --- a/src/messenger/attachments/entrypoints/http.py +++ b/src/messenger/attachments/entrypoints/http.py @@ -1,6 +1,6 @@ import re from collections.abc import AsyncIterator -from typing import Annotated, Literal +from typing import Annotated from uuid import UUID from dishka.integrations.fastapi import FromDishka, inject @@ -12,7 +12,7 @@ from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import UploadAttachment from messenger.attachments.application.queries import GetAttachment, OpenAttachment from messenger.attachments.domain.errors import UploadAborted -from messenger.attachments.entrypoints.schemas import AttachmentOut +from messenger.attachments.entrypoints.schemas import AttachmentOut, VariantName from messenger.shared.entrypoints.actor import Actor UPLOAD_PATH = re.compile(r"/api/v1/chats/[^/]+/attachments/?") @@ -30,7 +30,7 @@ async def body_of(request: Request) -> AsyncIterator[bytes]: raise UploadAborted("клиент оборвал загрузку до конца тела") from error -@uploads_router.post("", status_code=status.HTTP_201_CREATED) +@uploads_router.post("", status_code=status.HTTP_202_ACCEPTED) @inject async def upload_attachment( chat_id: UUID, @@ -67,11 +67,11 @@ async def download_attachment( attachment_id: UUID, actor: Actor, query: FromDishka[OpenAttachment], - _variant: Annotated[Literal["original"], Query(alias="variant")] = "original", + variant: Annotated[VariantName, Query()] = "original", ) -> StreamingResponse: - attachment, content = await query(attachment_id, actor) + content_type, content = await query(attachment_id, actor, variant) return StreamingResponse( content.chunks, - media_type=attachment.content_type, + media_type=content_type, headers={**CONTENT_HEADERS, "Content-Length": str(content.size)}, ) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index a7ca5464..cd97fb07 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -1,28 +1,50 @@ from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import timedelta from dishka import Provider, Scope, provide +from faststream.rabbit import RabbitBroker from messenger.attachments.application.bus import AttachmentsCommandBus -from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, + SweepAbandonedUploads, + UploadAttachment, +) from messenger.attachments.application.handlers import ( SweepAbandonedUploadsHandler, UploadAttachmentHandler, UploadSlots, ) +from messenger.attachments.application.pipeline import ( + CleanStagingHandler, + ProcessJobHandler, + PublishOutboxHandler, + RecoverExpiredLeasesHandler, +) from messenger.attachments.application.ports import ( AttachmentDirectory, AttachmentsUnitOfWork, ChatMembership, + Faults, ImageInspector, + JobPublisher, ObjectStorage, + VariantRenderer, ) from messenger.attachments.application.queries import GetAttachment, OpenAttachment -from messenger.attachments.config import AttachmentsSettings -from messenger.attachments.infrastructure.images import PillowImageInspector +from messenger.attachments.config import AttachmentsSettings, PipelineSettings +from messenger.attachments.domain.jobs import RetryPolicy +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.attachments.infrastructure.images import PillowImageInspector, PillowVariantRenderer from messenger.attachments.infrastructure.postgres import ( PostgresAttachmentDirectory, PostgresAttachmentsUnitOfWork, ) +from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client from messenger.shared.application.commands import Command from messenger.shared.application.messagebus import CommandHandler @@ -30,6 +52,20 @@ from messenger.shared.application.ports import Clock, EventPublisher, IdGenerato from messenger.shared.infrastructure.postgres import Database +@asynccontextmanager +async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]: + async with s3_client( + endpoint=settings.s3_endpoint, + region=settings.s3_region, + access_key_id=settings.s3_access_key_id.get_secret_value(), + secret_access_key=settings.s3_secret_access_key.get_secret_value(), + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + max_attempts=settings.s3_max_attempts, + ) as client: + yield S3ObjectStorage(client, settings.s3_bucket) + + class AttachmentsProvider(Provider): def __init__(self, settings: AttachmentsSettings) -> None: super().__init__() @@ -41,16 +77,8 @@ class AttachmentsProvider(Provider): @provide(scope=Scope.APP) async def storage(self, settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]: - async with s3_client( - endpoint=settings.s3_endpoint, - region=settings.s3_region, - access_key_id=settings.s3_access_key_id.get_secret_value(), - secret_access_key=settings.s3_secret_access_key.get_secret_value(), - connect_timeout=settings.s3_connect_timeout_seconds, - read_timeout=settings.s3_read_timeout_seconds, - max_attempts=settings.s3_max_attempts, - ) as client: - yield S3ObjectStorage(client, settings.s3_bucket) + async with object_storage(settings) as storage: + yield storage @provide(scope=Scope.APP) def images(self, settings: AttachmentsSettings) -> ImageInspector: @@ -110,3 +138,92 @@ class AttachmentsProvider(Provider): SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock), } return AttachmentsCommandBus(handlers, uow, publisher) + + +class PipelineProvider(Provider): + def __init__(self, settings: PipelineSettings) -> None: + super().__init__() + self._settings = settings + + @provide(scope=Scope.APP) + def settings(self) -> PipelineSettings: + return self._settings + + @provide(scope=Scope.APP) + async def storage(self, settings: PipelineSettings) -> AsyncIterator[ObjectStorage]: + async with object_storage(settings) as storage: + yield storage + + @provide(scope=Scope.APP) + def policy(self, settings: PipelineSettings) -> RetryPolicy: + return RetryPolicy( + settings.job_max_attempts, + settings.job_backoff_base_seconds, + settings.job_backoff_cap_seconds, + ) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(database) + + +class WorkerProvider(Provider): + @provide(scope=Scope.APP) + def renderer(self, settings: PipelineSettings) -> VariantRenderer: + return PillowVariantRenderer(settings.max_image_pixels, settings.worker_concurrency) + + @provide(scope=Scope.APP) + def faults(self, settings: PipelineSettings, logger: Logger) -> Faults: + return EnvironmentFaults(settings.fault_point, logger) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + renderer: VariantRenderer, + faults: Faults, + publisher: EventPublisher, + clock: Clock, + ids: IdGenerator, + policy: RetryPolicy, + settings: PipelineSettings, + ) -> AttachmentsCommandBus: + process = ProcessJobHandler( + uow, + storage, + renderer, + faults, + clock=clock, + ids=ids, + policy=policy, + lease=timedelta(seconds=settings.job_timeout_seconds), + ) + return AttachmentsCommandBus({ProcessJob: process}, uow, publisher) + + +class RelayProvider(Provider): + def __init__(self, broker: RabbitBroker) -> None: + super().__init__() + self._broker = broker + + @provide(scope=Scope.APP) + def publisher(self, settings: PipelineSettings) -> JobPublisher: + return RabbitJobPublisher(self._broker, settings.publish_timeout_seconds) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + jobs: JobPublisher, + publisher: EventPublisher, + clock: Clock, + policy: RetryPolicy, + ) -> AttachmentsCommandBus: + handlers: dict[type[Command], CommandHandler] = { + PublishOutbox: PublishOutboxHandler(uow, jobs), + RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, clock, policy), + CleanStaging: CleanStagingHandler(uow, storage), + } + return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py index 55ba5e5a..9b771e8a 100644 --- a/src/messenger/attachments/entrypoints/journal.py +++ b/src/messenger/attachments/entrypoints/journal.py @@ -1,7 +1,22 @@ -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued, JobSettled +from messenger.shared.application.messagebus import EventBus from messenger.shared.application.ports import Logger -LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes", "count"}) +LOG_FIELDS = frozenset( + { + "attachment_id", + "chat_id", + "actor_id", + "size_bytes", + "count", + "job_id", + "attempt", + "pipeline_version", + "outcome", + "error_code", + "delay_seconds", + } +) class AttachmentsJournal: @@ -10,9 +25,9 @@ class AttachmentsJournal: def __init__(self, logger: Logger) -> None: self._logger = logger - async def on_attachment_stored(self, event: AttachmentStored) -> None: + async def on_attachment_queued(self, event: AttachmentQueued) -> None: self._logger.info( - "audit.attachment_stored", + "audit.attachment_queued", attachment_id=str(event.attachment_id), chat_id=str(event.chat_id), actor_id=str(event.owner_id), @@ -26,3 +41,22 @@ class AttachmentsJournal: chat_id=str(event.chat_id), reason=event.reason, ) + + async def on_job_settled(self, event: JobSettled) -> None: + self._logger.info( + "audit.job_settled", + job_id=str(event.job_id), + attachment_id=str(event.attachment_id), + pipeline_version=event.pipeline_version, + attempt=event.attempt, + outcome=event.outcome, + error_code=event.error_code, + delay_seconds=event.delay_seconds, + ) + + +def subscribe_journal(bus: EventBus, logger: Logger) -> None: + journal = AttachmentsJournal(logger) + bus.subscribe(AttachmentQueued, journal.on_attachment_queued) + bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded) + bus.subscribe(JobSettled, journal.on_job_settled) diff --git a/src/messenger/attachments/entrypoints/relay.py b/src/messenger/attachments/entrypoints/relay.py new file mode 100644 index 00000000..fad85f7f --- /dev/null +++ b/src/messenger/attachments/entrypoints/relay.py @@ -0,0 +1,67 @@ +import asyncio +from collections.abc import Sequence +from contextlib import suppress + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ( + CleanStaging, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.config import PipelineSettings +from messenger.shared.application.commands import Command +from messenger.shared.application.ports import Logger + + +class Relay: + __slots__ = ("_container", "_logger", "_settings", "_tasks") + + def __init__( + self, container: AsyncContainer, settings: PipelineSettings, logger: Logger + ) -> None: + self._container = container + self._settings = settings + self._logger = logger + self._tasks: list[asyncio.Task[None]] = [] + + def start(self) -> None: + settings = self._settings + loops: tuple[tuple[float, Sequence[Command]], ...] = ( + (settings.outbox_poll_seconds, (PublishOutbox(settings.outbox_batch),)), + ( + settings.sweep_interval_seconds, + (RecoverExpiredLeases(settings.sweep_batch), CleanStaging(settings.sweep_batch)), + ), + ) + self._tasks = [asyncio.create_task(self._every(*loop)) for loop in loops] + + async def stop(self) -> None: + for task in self._tasks: + task.cancel() + for task in self._tasks: + with suppress(asyncio.CancelledError): + await task + self._tasks = [] + + async def run(self, command: Command) -> int: + async with self._container() as scope: + bus = await scope.get(AttachmentsCommandBus) + return await bus.execute(command) + + async def _every(self, interval: float, commands: Sequence[Command]) -> None: + while True: + for command in commands: + try: + done = await self.run(command) + except Exception as error: + self._logger.warning( + "relay.pass_failed", + reason=type(command).__name__, + error=type(error).__name__, + ) + continue + if done: + self._logger.info("relay.pass", reason=type(command).__name__, count=done) + await asyncio.sleep(interval) diff --git a/src/messenger/attachments/entrypoints/schemas.py b/src/messenger/attachments/entrypoints/schemas.py index 86daa6cd..f482d4a5 100644 --- a/src/messenger/attachments/entrypoints/schemas.py +++ b/src/messenger/attachments/entrypoints/schemas.py @@ -1,10 +1,34 @@ from datetime import datetime -from typing import Literal +from typing import Annotated, Literal from uuid import UUID -from pydantic import BaseModel +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from messenger.attachments.domain.jobs import Job from messenger.attachments.domain.models import Attachment +from messenger.attachments.domain.variants import Variant + +type VariantName = Literal["original", "thumbnail", "crop", "optimized"] + + +class VariantOut(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + @classmethod + def of(cls, variant: Variant) -> VariantOut: + return cls( + name=variant.name, + content_type=variant.content_type, + size_bytes=variant.size_bytes, + width=variant.width, + height=variant.height, + sha256=variant.sha256, + ) class AttachmentOut(BaseModel): @@ -15,9 +39,9 @@ class AttachmentOut(BaseModel): content_type: str size_bytes: int sha256: str - status: Literal["ready"] = "ready" - variants: tuple[()] = () - error_code: None = None + status: str + variants: list[VariantOut] + error_code: str | None created_at: datetime updated_at: datetime @@ -31,6 +55,30 @@ class AttachmentOut(BaseModel): content_type=attachment.content_type, size_bytes=attachment.size_bytes, sha256=attachment.sha256, + status=attachment.status.value, + variants=[VariantOut.of(variant) for variant in attachment.variants], + error_code=attachment.error_code, created_at=attachment.created_at, updated_at=attachment.updated_at, ) + + +class JobMessage(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal[1] + job_id: UUID + type: Literal["attachment.process.v1"] + attachment_id: UUID + pipeline_version: Annotated[int, Field(ge=1)] + attempt: Annotated[int, Field(ge=1)] + created_at: AwareDatetime + traceparent: Annotated[ + str | None, Field(pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + ] = None + tracestate: Annotated[str | None, Field(max_length=512)] = None + + def job(self) -> Job: + return Job( + self.job_id, self.attachment_id, self.pipeline_version, self.attempt, self.created_at + ) diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py new file mode 100644 index 00000000..8de721bd --- /dev/null +++ b/src/messenger/attachments/entrypoints/worker.py @@ -0,0 +1,50 @@ +import asyncio + +from dishka import FromDishka +from faststream import AckPolicy +from faststream.rabbit import Channel, QueueType, RabbitBroker, RabbitQueue +from faststream.rabbit.annotations import RabbitMessage +from pydantic import ValidationError + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.rabbitmq import QUEUE +from messenger.shared.application.ports import Logger + +REQUEUE_DELAY_SECONDS = 1.0 +JOBS = RabbitQueue(QUEUE, queue_type=QueueType.QUORUM, declare=False) + + +async def raw(message: RabbitMessage) -> bytes: + return message.body + + +async def handle_job( + message: RabbitMessage, + bus: FromDishka[AttachmentsCommandBus], + logger: FromDishka[Logger], +) -> None: + try: + job = JobMessage.model_validate_json(message.body).job() + except ValidationError: + logger.warning("job.rejected", reason="message does not match jobs.schema.json") + await message.reject(requeue=False) + return + try: + await bus.execute(ProcessJob(job)) + except Exception as error: + logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__) + await asyncio.sleep(REQUEUE_DELAY_SECONDS) + await message.nack(requeue=True) + return + await message.ack() + + +def subscribe(broker: RabbitBroker, concurrency: int) -> None: + broker.subscriber( + JOBS, + channel=Channel(prefetch_count=concurrency), + ack_policy=AckPolicy.MANUAL, + decoder=raw, + )(handle_job) diff --git a/src/messenger/attachments/infrastructure/faults.py b/src/messenger/attachments/infrastructure/faults.py new file mode 100644 index 00000000..814d3ce1 --- /dev/null +++ b/src/messenger/attachments/infrastructure/faults.py @@ -0,0 +1,26 @@ +import os +import sys + +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +TRANSIENT = frozenset({"transient_error"}) +CRASH_EXIT_CODE = 137 + + +class EnvironmentFaults: + __slots__ = ("_logger", "_point") + + def __init__(self, point: str, logger: Logger) -> None: + self._point = point + self._logger = logger + + def reach(self, point: str) -> None: + if not self._point or point != self._point: + return + self._logger.warning("fault.injected", reason=point) + if point in TRANSIENT: + raise Unavailable("внедрённая временная ошибка") + sys.stdout.flush() + sys.stderr.flush() + os._exit(CRASH_EXIT_CODE) diff --git a/src/messenger/attachments/infrastructure/images.py b/src/messenger/attachments/infrastructure/images.py index 6d63f670..e7cf1213 100644 --- a/src/messenger/attachments/infrastructure/images.py +++ b/src/messenger/attachments/infrastructure/images.py @@ -2,14 +2,19 @@ import asyncio import warnings from io import BytesIO -from PIL import Image +from PIL import Image, ImageOps -from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import PayloadTooLarge, UnreadableImage, UnsupportedImage from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.domain.variants import THUMBNAIL, THUMBNAIL_BOX, fit KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG} BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning) JPEG_SCALE = 8 +WIDE_MODES = frozenset({"I", "I;16", "I;16B", "I;16L", "I;16N"}) +ALPHA_MODES = frozenset({"LA", "La", "PA", "RGBa"}) +JPEG_QUALITY = 85 class PillowImageInspector: @@ -47,3 +52,51 @@ class PillowImageInspector: except Exception as error: raise UnsupportedImage("изображение повреждено или обрезано") from error return ImageInfo(kind, width, height) + + +def flatten(image: Image.Image) -> Image.Image: + if image.mode in WIDE_MODES: + image = image.convert("I").point(lambda value: value / 256, "L") + if image.mode == "P": + image = image.convert("RGBA" if "transparency" in image.info else "RGB") + if image.mode in ALPHA_MODES: + image = image.convert("RGBA") + if image.mode == "RGBA": + background = Image.new("RGB", image.size, "white") + background.paste(image, mask=image.getchannel("A")) + return background + return image if image.mode == "RGB" else image.convert("RGB") + + +def jpeg(name: str, image: Image.Image, box: int) -> Rendition: + size = fit(image.width, image.height, box) + scaled = image if size == image.size else image.resize(size, Image.Resampling.LANCZOS) + buffer = BytesIO() + scaled.save(buffer, "JPEG", quality=JPEG_QUALITY, optimize=True) + return Rendition(name, ImageKind.JPEG.value, "jpg", buffer.getvalue(), *size) + + +def render_variants(data: bytes, max_pixels: int) -> list[Rendition]: + try: + with Image.open(BytesIO(data), formats=tuple(KINDS)) as image: + if image.width * image.height > max_pixels: + raise UnreadableImage("изображение больше допустимого числа пикселей") + image.load() + upright = flatten(ImageOps.exif_transpose(image)) + except UnreadableImage: + raise + except Exception as error: + raise UnreadableImage("изображение не декодируется") from error + return [jpeg(THUMBNAIL, upright, THUMBNAIL_BOX)] + + +class PillowVariantRenderer: + __slots__ = ("_max_pixels", "_slots") + + def __init__(self, max_pixels: int, concurrency: int) -> None: + self._max_pixels = max_pixels + self._slots = asyncio.Semaphore(concurrency) + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + async with self._slots: + return await asyncio.to_thread(render_variants, data, self._max_pixels) diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index c47d0f18..20a9ad59 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -1,11 +1,16 @@ -from datetime import datetime +import json +from collections.abc import Sequence +from datetime import datetime, timedelta from uuid import UUID from asyncpg import ForeignKeyViolationError, Record +from messenger.attachments.application.ports import OutboxEntry, StagedObject from messenger.attachments.domain.errors import ChatNotFound +from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind +from messenger.attachments.domain.variants import Variant from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.postgres import ( Connection, @@ -16,26 +21,151 @@ from messenger.shared.infrastructure.postgres import ( INSERT = """ INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - width, height, object_key, status, created_at, updated_at) + width, height, staging_key, status, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) """ MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'" COLUMNS = """ -SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, - object_key, status, created_at, updated_at -FROM attachments +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, + a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, + a.created_at, a.updated_at +FROM attachments a """ -READY = COLUMNS + "WHERE id = $1 AND status = 'ready'" -LOCK = COLUMNS + "WHERE id = $1 FOR UPDATE" +VISIBLE = """ +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, + a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, + a.created_at, a.updated_at, + (SELECT coalesce(jsonb_agg(jsonb_build_array(v.name, v.content_type, v.size_bytes, v.width, + v.height, v.sha256, v.object_key) + ORDER BY v.name), '[]') + FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.published_version) AS variants +FROM attachments a +WHERE a.id = $1 AND a.status <> 'pending' +""" +FAILED_AMONG = """ +SELECT EXISTS (SELECT 1 FROM attachments + WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'failed') +""" +LOCK = COLUMNS + "WHERE a.id = $1 FOR UPDATE" ABANDONED = ( COLUMNS - + """WHERE status = 'pending' AND created_at < $1 -ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" + + """WHERE a.status = 'pending' AND a.created_at < $1 +ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" ) REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" +STAGED = """ +SELECT id, staging_key FROM attachments +WHERE staging_key IS NOT NULL AND status = 'ready' +ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED +""" +UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])" + +ADD_JOB = """ +WITH job AS ( + INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, attempt, + created_at, updated_at) + VALUES ($1, $2, $3, 'queued', $4, $5, $5) + RETURNING job_id +), box AS ( + INSERT INTO outbox (job_id, payload, available_at) + SELECT job_id, $6::jsonb, now() FROM job +) +INSERT INTO job_events (job_id, attachment_id, kind, attempt) +SELECT job_id, $2, 'enqueued', $4 FROM job +""" +CLAIM = """ +WITH claimed AS ( + UPDATE jobs + SET status = 'processing', lease_token = $4, claims = claims + 1, updated_at = now(), + lease_until = now() + make_interval(secs => $5) + WHERE job_id = $1 AND attachment_id = $2 AND pipeline_version = $3 + AND ((status = 'queued' AND coalesce(next_attempt_at, '-infinity') <= now()) + OR (status = 'processing' AND lease_until < now())) + RETURNING job_id, attachment_id, attempt, created_at +), moved AS ( + UPDATE attachments a SET status = 'processing', updated_at = greatest(now(), a.created_at) + FROM claimed c WHERE a.id = c.attachment_id AND a.status = 'queued' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt) + SELECT job_id, attachment_id, 'claimed', attempt FROM claimed +) +SELECT c.attempt, c.created_at, a.staging_key, a.original_key +FROM claimed c JOIN attachments a ON a.id = c.attachment_id +""" +FINISH = """ +UPDATE jobs SET status = 'done', lease_token = NULL, lease_until = NULL, next_attempt_at = NULL, + last_error = NULL, updated_at = now() +WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 +""" +ADD_VARIANTS = """ +INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, + size_bytes, width, height, sha256, created_at) +SELECT $1, $2, v.name, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256, + now() +FROM unnest($3::text[], $4::text[], $5::text[], $6::bigint[], $7::int[], $8::int[], $9::text[]) + AS v (name, object_key, content_type, size_bytes, width, height, sha256) +ON CONFLICT DO NOTHING +""" +PUBLISH = """ +UPDATE attachments +SET status = 'ready', published_version = $2, error_code = NULL, original_key = $3, + updated_at = greatest(now(), created_at) +WHERE id = $1 AND published_version < $2 +""" +SUPERSEDE = "UPDATE jobs SET status = 'superseded', updated_at = now() WHERE job_id = $1" +RESCHEDULE = """ +WITH job AS ( + UPDATE jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL, + next_attempt_at = $3, last_error = $4, updated_at = now() + WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 + RETURNING job_id, attachment_id, attempt +), moved AS ( + UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status = 'processing' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, $5, attempt, + jsonb_build_object('error', $4::text, 'next_attempt_at', $3::timestamptz) + FROM job +) +SELECT attempt FROM job +""" +ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)" +FAIL = """ +WITH job AS ( + UPDATE jobs SET status = 'failed', lease_token = NULL, lease_until = NULL, last_error = $4, + updated_at = now() + WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 + RETURNING job_id, attachment_id, attempt +), moved AS ( + UPDATE attachments a SET status = 'failed', error_code = $3, + updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status IN ('queued', 'processing') +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, 'failed', attempt, + jsonb_build_object('error_code', $3::text, 'error', $4::text) + FROM job +) +SELECT attempt FROM job +""" +NOTE = "INSERT INTO job_events (job_id, attachment_id, kind, attempt) VALUES ($1, $2, $3, $4)" +EXPIRED = """ +SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token FROM jobs +WHERE status = 'processing' AND lease_until < now() +ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED +""" +PENDING = """ +SELECT id, job_id, payload::text AS body FROM outbox +WHERE published_at IS NULL AND available_at <= now() +ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED +""" +PUBLISHED = "UPDATE outbox SET published_at = now() WHERE id = ANY($1::bigint[])" -def attachment_of(row: Record) -> Attachment: +def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment: return Attachment( id=row["id"], chat_id=row["chat_id"], @@ -44,13 +174,21 @@ def attachment_of(row: Record) -> Attachment: image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]), size_bytes=row["size_bytes"], sha256=row["sha256"], - object_key=row["object_key"], status=AttachmentStatus(row["status"]), created_at=row["created_at"], updated_at=row["updated_at"], + staging_key=row["staging_key"], + original_key=row["original_key"], + published_version=row["published_version"], + error_code=row["error_code"], + variants=variants, ) +def payload(job: Job) -> str: + return json.dumps(job.payload(), separators=(",", ":")) + + class PostgresAttachmentRepository: __slots__ = ("_connection", "_seen") @@ -71,7 +209,7 @@ class PostgresAttachmentRepository: attachment.sha256, attachment.image.width, attachment.image.height, - attachment.object_key, + attachment.staging_key, attachment.status.value, attachment.created_at, attachment.updated_at, @@ -100,6 +238,127 @@ class PostgresAttachmentRepository: await self._connection.execute(REMOVE, attachment.id) self._seen.append(attachment) + async def staged(self, limit: int) -> list[StagedObject]: + rows = await self._connection.fetch(STAGED, limit) + return [StagedObject(row["id"], row["staging_key"]) for row in rows] + + async def unstage(self, attachment_ids: Sequence[UUID]) -> None: + if attachment_ids: + await self._connection.execute(UNSTAGE, list(attachment_ids)) + + +class PostgresJobRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def add(self, job: Job) -> None: + await self._connection.execute( + ADD_JOB, + job.id, + job.attachment_id, + job.pipeline_version, + job.attempt, + job.created_at, + payload(job), + ) + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: + row = await self._connection.fetchrow( + CLAIM, + job.id, + job.attachment_id, + job.pipeline_version, + token, + lease.total_seconds(), + ) + if row is None: + return None + claimed = Job( + job.id, job.attachment_id, job.pipeline_version, row["attempt"], row["created_at"] + ) + staged = row["staging_key"] is not None + source = row["staging_key"] if staged else row["original_key"] + return Claim(Lease(claimed, token), source, staged) + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: + job = lease.job + if await self._connection.execute(FINISH, job.id, lease.token) == "UPDATE 0": + await self.note(job, Outcome.FENCED) + return Outcome.FENCED + await self._connection.execute( + ADD_VARIANTS, + job.attachment_id, + job.pipeline_version, + [v.name for v in variants], + [v.object_key for v in variants], + [v.content_type for v in variants], + [v.size_bytes for v in variants], + [v.width for v in variants], + [v.height for v in variants], + [v.sha256 for v in variants], + ) + published = await self._connection.execute( + PUBLISH, job.attachment_id, job.pipeline_version, original_key + ) + outcome = Outcome.SUPERSEDED if published == "UPDATE 0" else Outcome.DONE + if outcome is Outcome.SUPERSEDED: + await self._connection.execute(SUPERSEDE, job.id) + await self.note(job, outcome) + return outcome + + async def reschedule(self, lease: Lease, at: datetime, kind: Outcome, error: str) -> Job | None: + attempt = await self._connection.fetchval( + RESCHEDULE, lease.job.id, lease.token, at, error, kind.value + ) + if attempt is None: + return None + job = lease.job.retried(attempt) + await self._connection.execute(ENQUEUE, job.id, payload(job), at) + return job + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: + attempt = await self._connection.fetchval( + FAIL, lease.job.id, lease.token, code.value, error + ) + return attempt is not None + + async def note(self, job: Job, kind: Outcome) -> None: + await self._connection.execute(NOTE, job.id, job.attachment_id, kind.value, job.attempt) + + async def expired(self, limit: int) -> list[Lease]: + rows = await self._connection.fetch(EXPIRED, limit) + return [ + Lease( + Job( + row["job_id"], + row["attachment_id"], + row["pipeline_version"], + row["attempt"], + row["created_at"], + ), + row["lease_token"], + ) + for row in rows + ] + + +class PostgresOutboxRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def pending(self, limit: int) -> list[OutboxEntry]: + rows = await self._connection.fetch(PENDING, limit) + return [OutboxEntry(row["id"], row["job_id"], row["body"].encode()) for row in rows] + + async def published(self, entry_ids: Sequence[int]) -> None: + await self._connection.execute(PUBLISHED, list(entry_ids)) + class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): __slots__ = () @@ -108,6 +367,18 @@ class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): def attachments(self) -> PostgresAttachmentRepository: return PostgresAttachmentRepository(self.connection, self.seen) + @property + def jobs(self) -> PostgresJobRepository: + return PostgresJobRepository(self.connection) + + @property + def outbox(self) -> PostgresOutboxRepository: + return PostgresOutboxRepository(self.connection) + + +def variants_of(raw: str) -> tuple[Variant, ...]: + return tuple(Variant(*item) for item in json.loads(raw)) + class PostgresAttachmentDirectory: __slots__ = ("_database",) @@ -115,7 +386,11 @@ class PostgresAttachmentDirectory: def __init__(self, database: Database) -> None: self._database = database - async def ready(self, attachment_id: UUID) -> Attachment | None: + async def visible(self, attachment_id: UUID) -> Attachment | None: async with self._database.connection() as connection: - row = await connection.fetchrow(READY, attachment_id) - return None if row is None else attachment_of(row) + row = await connection.fetchrow(VISIBLE, attachment_id) + return None if row is None else attachment_of(row, variants_of(row["variants"])) + + async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: + async with self._database.connection() as connection: + return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id)) diff --git a/src/messenger/attachments/infrastructure/rabbitmq.py b/src/messenger/attachments/infrastructure/rabbitmq.py new file mode 100644 index 00000000..778c1af9 --- /dev/null +++ b/src/messenger/attachments/infrastructure/rabbitmq.py @@ -0,0 +1,37 @@ +from uuid import UUID + +from faststream.rabbit import RabbitBroker, RabbitExchange +from pamqp.commands import Basic + +from messenger.attachments.domain.jobs import JOB_TYPE +from messenger.shared.application.errors import Unavailable + +EXCHANGE = RabbitExchange("attachments", declare=False) +QUEUE = "attachments.process" +DEAD_LETTERS = "attachments.process.dlq" + + +class RabbitJobPublisher: + __slots__ = ("_broker", "_timeout") + + def __init__(self, broker: RabbitBroker, timeout: float) -> None: + self._broker = broker + self._timeout = timeout + + async def publish(self, job_id: UUID, body: bytes) -> None: + try: + confirmation = await self._broker.publish( + body, + exchange=EXCHANGE, + routing_key=JOB_TYPE, + mandatory=True, + persist=True, + message_id=str(job_id), + message_type=JOB_TYPE, + content_type="application/json", + timeout=self._timeout, + ) + except Exception as error: + raise Unavailable("брокер не принял задание") from error + if not isinstance(confirmation, Basic.Ack): + raise Unavailable("брокер не подтвердил задание") diff --git a/src/messenger/attachments/infrastructure/s3.py b/src/messenger/attachments/infrastructure/s3.py index 5d47ecf0..3acea375 100644 --- a/src/messenger/attachments/infrastructure/s3.py +++ b/src/messenger/attachments/infrastructure/s3.py @@ -77,6 +77,21 @@ class S3ObjectStorage: raise Unavailable("S3 недоступно") from error return ObjectContent(size=response["ContentLength"], chunks=self._chunks(response["Body"])) + async def read(self, key: str) -> bytes: + content = await self.open(key) + try: + return b"".join([chunk async for chunk in content.chunks]) + except Exception as error: + raise Unavailable("S3 оборвал чтение объекта") from error + + async def copy(self, source: str, target: str) -> None: + try: + await self._client.copy_object( + Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source} + ) + except FAILURES as error: + raise Unavailable("S3 недоступно") from error + async def delete(self, key: str) -> None: try: await self._client.delete_object(Bucket=self._bucket, Key=key) diff --git a/src/messenger/main.py b/src/messenger/main.py index ddaf3c6b..cf02aa5e 100644 --- a/src/messenger/main.py +++ b/src/messenger/main.py @@ -8,12 +8,11 @@ from fastapi import FastAPI from messenger.attachments.application.ports import ChatMembership from messenger.attachments.config import AttachmentsSettings -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.entrypoints.errors import ERROR_CODES as ATTACHMENT_ERRORS from messenger.attachments.entrypoints.http import UPLOAD_PATH, attachments_router, uploads_router from messenger.attachments.entrypoints.ioc import AttachmentsProvider from messenger.attachments.entrypoints.journal import LOG_FIELDS as ATTACHMENT_FIELDS -from messenger.attachments.entrypoints.journal import AttachmentsJournal +from messenger.attachments.entrypoints.journal import subscribe_journal from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.identity.config import IdentitySettings from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered @@ -75,9 +74,7 @@ def wire_events(bus: EventBus, logger: Logger) -> None: bus.subscribe(ChatCreated, messaging.on_chat_created) bus.subscribe(MessagePosted, messaging.on_message_posted) - attachments = AttachmentsJournal(logger) - bus.subscribe(AttachmentStored, attachments.on_attachment_stored) - bus.subscribe(AttachmentDiscarded, attachments.on_attachment_discarded) + subscribe_journal(bus, logger) def create_app( diff --git a/src/messenger/relay.py b/src/messenger/relay.py new file mode 100644 index 00000000..7db49563 --- /dev/null +++ b/src/messenger/relay.py @@ -0,0 +1,66 @@ +import asyncio +import signal + +from dishka import make_async_container +from faststream.rabbit import Channel, RabbitBroker + +from messenger.attachments.config import PipelineSettings +from messenger.attachments.entrypoints.heartbeat import Heartbeat +from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.attachments.entrypoints.relay import Relay +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +PING_TIMEOUT_SECONDS = 2.0 + + +async def serve() -> None: + platform = PlatformSettings() + pipeline = PipelineSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + broker = RabbitBroker( + pipeline.broker_url.get_secret_value(), + default_channel=Channel(publisher_confirms=True, on_return_raises=True), + logger=None, + ) + container = make_async_container( + PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker) + ) + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + await broker.connect() + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + relay = Relay(container, pipeline, logger) + heartbeat = Heartbeat( + pipeline.heartbeat_file, + lambda: broker.ping(PING_TIMEOUT_SECONDS), + pipeline.heartbeat_interval_seconds, + logger, + ) + relay.start() + heartbeat.start() + logger.info("relay.started") + try: + await stop.wait() + finally: + await heartbeat.stop() + await relay.stop() + await bus.stop() + await broker.stop() + await container.close() + logger.info("relay.stopped") + + +def main() -> None: + asyncio.run(serve()) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 00000000..a18902c1 --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,78 @@ +import asyncio +from collections.abc import Sequence + +from dishka import Provider, make_async_container +from dishka_faststream import setup_dishka +from faststream import FastStream +from faststream.rabbit import RabbitBroker + +from messenger.attachments.config import PipelineSettings +from messenger.attachments.entrypoints.heartbeat import Heartbeat +from messenger.attachments.entrypoints.ioc import PipelineProvider, WorkerProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.attachments.entrypoints.worker import subscribe +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +PING_TIMEOUT_SECONDS = 2.0 + + +def create_worker( + *, + platform: PlatformSettings | None = None, + pipeline: PipelineSettings | None = None, + overrides: Sequence[Provider] = (), +) -> FastStream: + platform = platform or PlatformSettings() + pipeline = pipeline or PipelineSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + broker = RabbitBroker( + pipeline.broker_url.get_secret_value(), + graceful_timeout=pipeline.shutdown_grace_seconds, + logger=None, + ) + subscribe(broker, pipeline.worker_concurrency) + container = make_async_container( + PlatformProvider(platform, logger), + PipelineProvider(pipeline), + WorkerProvider(), + *overrides, + ) + heartbeat = Heartbeat( + pipeline.heartbeat_file, + lambda: broker.ping(PING_TIMEOUT_SECONDS), + pipeline.heartbeat_interval_seconds, + logger, + ) + app = FastStream(broker, logger=None) + + @app.on_startup + async def start_bus() -> None: + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + + @app.after_startup + async def started() -> None: + heartbeat.start() + logger.info("worker.started", limit=pipeline.worker_concurrency) + + @app.after_shutdown + async def stopped() -> None: + await heartbeat.stop() + await (await container.get(EventBus)).stop() + await container.close() + logger.info("worker.stopped") + + setup_dishka(container, app, auto_inject=True) + return app + + +def main() -> None: + asyncio.run(create_worker().run()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/adapters/test_rabbitmq.py b/student-tests/adapters/test_rabbitmq.py new file mode 100644 index 00000000..6f800555 --- /dev/null +++ b/student-tests/adapters/test_rabbitmq.py @@ -0,0 +1,259 @@ +import asyncio +import base64 +import hashlib +import json +import os +import re +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +import pytest +from dishka import Provider, Scope, make_async_container, provide +from dishka_faststream import setup_dishka +from faststream.rabbit import Channel, RabbitBroker + +from conftest import ROOT, STARTUP_SECONDS, container, image, published +from fakes import NullLogger +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.domain.jobs import Job, Outcome +from messenger.attachments.entrypoints.worker import subscribe +from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +RABBITMQ_IMAGE = re.compile(r"image: (rabbitmq:\S+)") +PASSWORDS = {role: os.urandom(12).hex() for role in ("api", "worker", "ops")} +GRANTS = { + "api": ("^$", "^attachments$", "^$"), + "worker": ("^$", "^$", "^attachments\\.process$"), + "ops": (".*", ".*", ".*"), +} + + +def user(role: str) -> dict[str, Any]: + salt = os.urandom(4) + digest = hashlib.sha256(salt + PASSWORDS[role].encode()).digest() + return { + "name": f"messenger-{role}", + "password_hash": base64.b64encode(salt + digest).decode(), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["administrator"] if role == "ops" else [], + } + + +def definitions() -> dict[str, Any]: + return { + "users": [user(role) for role in PASSWORDS], + "permissions": [ + { + "user": f"messenger-{role}", + "vhost": "messenger", + **dict(zip(("configure", "write", "read"), grant, strict=True)), + } + for role, grant in GRANTS.items() + ], + } + + +def ready(name: str) -> bool: + probe = ["docker", "exec", name, "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + return subprocess.run(probe, capture_output=True, check=False).returncode == 0 + + +@pytest.fixture(scope="session") +def rabbitmq() -> Iterator[dict[str, Any]]: + name = f"mtusi-lab5-test-rabbitmq-{uuid4().hex[:8]}" + folder = ROOT / "build" / name + folder.mkdir(parents=True) + users = folder / "20-users.json" + users.write_text(json.dumps(definitions())) + users.chmod(0o644) + infra = ROOT / "infra" / "rabbitmq" + mounts = [ + f"{infra / 'rabbitmq.conf'}:/etc/rabbitmq/conf.d/20-messenger.conf:ro", + f"{infra / 'enabled_plugins'}:/etc/rabbitmq/enabled_plugins:ro", + f"{infra / 'definitions.json'}:/etc/rabbitmq/definitions.d/10-topology.json:ro", + f"{users}:/etc/rabbitmq/definitions.d/20-users.json:ro", + ] + volumes = [part for mount in mounts for part in ("-v", mount)] + container( + name, + "--hostname", + "broker", + "--user", + "100:101", + *volumes, + "-p", + "127.0.0.1::5672", + image(RABBITMQ_IMAGE), + ) + try: + deadline = time.monotonic() + STARTUP_SECONDS * 2 + while not ready(name): + assert time.monotonic() < deadline, "RabbitMQ не стал готов" + time.sleep(1) + port = published(name, 5672) + urls = { + role: f"amqp://messenger-{role}:{secret}@127.0.0.1:{port}/messenger" + for role, secret in PASSWORDS.items() + } + yield {"name": name, "urls": urls} + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + users.unlink() + folder.rmdir() + + +def depth(broker: dict[str, Any], queue: str) -> int: + listing = subprocess.run( + [ + "docker", + "exec", + broker["name"], + "rabbitmqctl", + "-q", + "list_queues", + "-p", + "messenger", + "name", + "messages", + "--formatter", + "json", + ], + capture_output=True, + check=True, + text=True, + ).stdout + return {row["name"]: row["messages"] for row in json.loads(listing)}[queue] + + +async def wait_for(probe: Any, seconds: float = 20) -> None: + deadline = time.monotonic() + seconds + while not probe(): + assert time.monotonic() < deadline + await asyncio.sleep(0.1) + + +class NoEvents: + @asynccontextmanager + async def transaction(self) -> AsyncIterator[None]: + yield + + async def commit(self) -> None: ... + + def record(self, event: object) -> None: ... + + def collect_new_events(self) -> list[Any]: + return [] + + +class Recorder: + def __init__(self) -> None: + self.jobs: list[Job] = [] + self.failures = 0 + + async def __call__(self, command: ProcessJob) -> Outcome: + if self.failures: + self.failures -= 1 + raise Unavailable("PostgreSQL недоступен") + self.jobs.append(command.job) + return Outcome.DONE + + +class FakeBus(Provider): + def __init__(self, recorder: Recorder) -> None: + super().__init__() + self._recorder = recorder + + @provide(scope=Scope.APP) + def logger(self) -> Logger: + return NullLogger() + + @provide(scope=Scope.REQUEST) + def bus(self) -> AttachmentsCommandBus: + return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher()) + + +class NoPublisher: + async def publish(self, events: Any) -> None: ... + + +@asynccontextmanager +async def worker(rabbitmq: dict[str, Any], recorder: Recorder) -> AsyncIterator[None]: + broker = RabbitBroker(rabbitmq["urls"]["worker"], logger=None, graceful_timeout=2) + subscribe(broker, 2) + setup_dishka(make_async_container(FakeBus(recorder)), broker=broker, auto_inject=True) + await broker.start() + try: + yield + finally: + await broker.stop() + + +@asynccontextmanager +async def publisher( + rabbitmq: dict[str, Any], role: str = "api" +) -> AsyncIterator[RabbitJobPublisher]: + broker = RabbitBroker( + rabbitmq["urls"][role], + default_channel=Channel(publisher_confirms=True, on_return_raises=True), + logger=None, + ) + await broker.connect() + try: + yield RabbitJobPublisher(broker, 5) + finally: + await broker.stop() + + +def job() -> Job: + return Job(uuid4(), uuid4(), 1, 1, datetime.now(UTC)) + + +def body(payload: dict[str, Any]) -> bytes: + return json.dumps(payload).encode() + + +async def test_confirmed_job_is_consumed_and_acked(rabbitmq: dict[str, Any]) -> None: + recorder, sent = Recorder(), job() + async with publisher(rabbitmq) as jobs: + await jobs.publish(sent.id, body(sent.payload())) + await wait_for(lambda: depth(rabbitmq, "attachments.process") == 1) + async with worker(rabbitmq, recorder): + await wait_for(lambda: recorder.jobs) + await wait_for(lambda: depth(rabbitmq, "attachments.process") == 0) + assert recorder.jobs == [sent] + + +async def test_message_off_the_schema_goes_to_the_dead_letter_queue( + rabbitmq: dict[str, Any], +) -> None: + before = depth(rabbitmq, "attachments.process.dlq") + async with publisher(rabbitmq) as jobs: + await jobs.publish(uuid4(), body({"schema_version": 2, "job_id": str(uuid4())})) + async with worker(rabbitmq, Recorder()): + await wait_for(lambda: depth(rabbitmq, "attachments.process.dlq") == before + 1) + assert depth(rabbitmq, "attachments.process") == 0 + + +async def test_failed_processing_is_requeued_and_retried(rabbitmq: dict[str, Any]) -> None: + recorder, sent = Recorder(), job() + recorder.failures = 1 + async with publisher(rabbitmq) as jobs: + await jobs.publish(sent.id, body(sent.payload())) + async with worker(rabbitmq, recorder): + await wait_for(lambda: recorder.jobs) + assert recorder.jobs == [sent] + + +async def test_worker_account_cannot_publish(rabbitmq: dict[str, Any]) -> None: + sent = job() + with pytest.raises(Unavailable): + async with publisher(rabbitmq, "worker") as jobs: + await jobs.publish(sent.id, body(sent.payload())) diff --git a/student-tests/api/test_attachments.py b/student-tests/api/test_attachments.py index 9f4ad41c..30f667bc 100644 --- a/student-tests/api/test_attachments.py +++ b/student-tests/api/test_attachments.py @@ -11,6 +11,7 @@ from conftest import actor, make_chat, register, resolve, scalar from fakes import MemoryObjectStorage from messenger.attachments.application.ports import ObjectStorage from pictures import GIF, JPEG, SAMPLE, WEBP, png +from pipeline import settle LIMIT = 10_485_760 PNG = {"Content-Type": "image/png"} @@ -47,7 +48,7 @@ async def trio(http: AsyncClient) -> tuple[dict[str, Any], ...]: async def test_member_uploads_and_another_member_downloads(client: AsyncClient) -> None: alice, bob, eve, chat = await trio(client) response = await upload(client, alice, chat) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text attachment = response.json() assert set(attachment) == { "id", @@ -70,13 +71,21 @@ async def test_member_uploads_and_another_member_downloads(client: AsyncClient) assert attachment["size_bytes"] == len(SAMPLE) == 111817 assert attachment["sha256"] == sha256(SAMPLE).hexdigest() assert (attachment["status"], attachment["variants"], attachment["error_code"]) == ( - "ready", + "queued", [], None, ) assert attachment["created_at"].endswith("Z") path = f"/api/v1/attachments/{attachment['id']}" assert (await client.get(path, headers=actor(bob))).json() == attachment + await settle(client) + ready = (await client.get(path, headers=actor(bob))).json() + assert (ready["status"], ready["error_code"], ready["sha256"]) == ( + "ready", + None, + attachment["sha256"], + ) + assert [variant["name"] for variant in ready["variants"]] == ["thumbnail"] content = await client.get( f"{path}/content", headers={**actor(bob), "Accept": "application/json"} ) @@ -93,19 +102,24 @@ async def test_member_uploads_and_another_member_downloads(client: AsyncClient) assert stranger.json()["error"]["code"] == "not_found" -async def test_object_key_is_assigned_by_server(client: AsyncClient) -> None: +async def test_object_keys_are_assigned_by_server(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) attachment = (await upload(client, alice, chat, filename="a.png")).json() - key = await scalar("SELECT object_key FROM attachments WHERE id = $1", attachment["id"]) - assert key == f"attachments/{attachment['id']}" - assert set((await storage(client)).objects) >= {key} + staging = await scalar("SELECT staging_key FROM attachments WHERE id = $1", attachment["id"]) + assert staging == f"staging/{attachment['id']}" + assert set((await storage(client)).objects) >= {staging} + await settle(client) + original = await scalar("SELECT original_key FROM attachments WHERE id = $1", attachment["id"]) + assert original == f"originals/{attachment['id']}" + assert (await storage(client)).objects[original][0] == SAMPLE async def test_jpeg_is_accepted_by_content(client: AsyncClient) -> None: alice, bob, _, chat = await trio(client) response = await upload(client, alice, chat, JPEG, headers={"Content-Type": "IMAGE/JPEG"}) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()["content_type"] == "image/jpeg" + await settle(client) download = await client.get( f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(bob) ) @@ -127,7 +141,7 @@ async def test_access_is_checked_before_content(client: AsyncClient) -> None: async def test_exact_limit_is_accepted(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) response = await upload(client, alice, chat, png(16, 16, size=LIMIT)) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()["size_bytes"] == LIMIT @@ -154,7 +168,7 @@ async def test_one_byte_over_chunked(client: AsyncClient) -> None: @pytest.mark.parametrize( ("width", "height", "status"), - [(8000, 5000, 201), (8001, 5000, 413), (100_000, 100_000, 413)], + [(8000, 5000, 202), (8001, 5000, 413), (100_000, 100_000, 413)], ) async def test_pixel_limit_is_read_from_header( client: AsyncClient, width: int, height: int, status: int @@ -224,16 +238,33 @@ async def test_dangerous_filename_is_rejected(client: AsyncClient, filename: str async def test_display_name_is_kept(client: AsyncClient, filename: str) -> None: alice, _, _, chat = await trio(client) response = await upload(client, alice, chat, filename=filename) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()["filename"] == filename -async def test_only_original_variant_exists(client: AsyncClient) -> None: - alice, _, _, chat = await trio(client) +async def test_content_waits_for_ready_and_serves_level_three_variants( + client: AsyncClient, +) -> None: + alice, bob, _, chat = await trio(client) attachment = (await upload(client, alice, chat)).json() path = f"/api/v1/attachments/{attachment['id']}/content" + for variant in ("original", "thumbnail", "crop"): + early = await client.get(f"{path}?variant={variant}", headers=actor(bob)) + assert early.status_code == 409, early.text + assert early.json()["error"]["code"] == "conflict" + await settle(client) assert (await client.get(f"{path}?variant=original", headers=actor(alice))).status_code == 200 - assert (await client.get(f"{path}?variant=thumbnail", headers=actor(alice))).status_code == 400 + thumbnail = await client.get(f"{path}?variant=thumbnail", headers=actor(bob)) + meta = (await client.get(path.removesuffix("/content"), headers=actor(bob))).json() + (variant,) = meta["variants"] + assert thumbnail.headers["content-type"] == "image/jpeg" + assert len(thumbnail.content) == variant["size_bytes"] + assert sha256(thumbnail.content).hexdigest() == variant["sha256"] + assert (variant["width"], variant["height"]) == (256, 160) + for absent in ("crop", "optimized"): + missing = await client.get(f"{path}?variant={absent}", headers=actor(alice)) + assert missing.status_code == 404 + assert (await client.get(f"{path}?variant=preview", headers=actor(alice))).status_code == 400 assert ( await client.get("/api/v1/attachments/x/content", headers=actor(alice)) ).status_code == 400 @@ -244,6 +275,7 @@ async def test_only_original_variant_exists(client: AsyncClient) -> None: async def test_storage_failure_leaves_nothing_visible(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) ready = await upload(client, alice, chat) + await settle(client) store = await storage(client) store.failing = True try: @@ -271,6 +303,7 @@ async def test_storage_failure_leaves_nothing_visible(client: AsyncClient) -> No async def test_lost_object_is_not_found(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) attachment = (await upload(client, alice, chat)).json() + await settle(client) (await storage(client)).objects.clear() response = await client.get( f"/api/v1/attachments/{attachment['id']}/content", headers=actor(alice) diff --git a/student-tests/api/test_bounded_contexts.py b/student-tests/api/test_bounded_contexts.py index 6ae98776..9299c528 100644 --- a/student-tests/api/test_bounded_contexts.py +++ b/student-tests/api/test_bounded_contexts.py @@ -154,22 +154,46 @@ def test_every_zone_owns_its_http_layer() -> None: assert (SOURCE / zone / "entrypoints" / part).is_file(), f"{zone}: нет {part}" +PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,)} + + def test_only_main_knows_several_zones() -> None: offenders: dict[str, list[str]] = {} for path in SOURCE.rglob("*.py"): relative = path.relative_to(SOURCE) if path.name in {"__init__.py", "main.py"} or relative.parts[0] in ZONES: continue + allowed = PROCESSES.get(str(relative), ()) borrowed = [ module for module in imports_of(path) if module.startswith((IDENTITY, MESSAGING, ATTACHMENTS)) + and not module.startswith(allowed or ("-",)) ] if borrowed: offenders[str(relative)] = borrowed assert not offenders +def test_broker_framework_stays_out_of_shared_and_core_layers() -> None: + offenders: dict[str, list[str]] = {} + for path in [ + *modules_under("shared"), + *( + p + for zone in ZONES + for layer in ("domain", "application") + for p in modules_under(zone, layer) + ), + ]: + borrowed = [ + m for m in imports_of(path) if m.split(".")[0] in {"faststream", "aio_pika", "aiormq"} + ] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + def test_application_has_no_separate_entrypoints_root() -> None: assert not (SOURCE / "entrypoints").exists() assert (SOURCE / "main.py").is_file() @@ -233,7 +257,7 @@ def test_zone_errors_do_not_leak_transport_codes() -> None: IDENTITY_TABLES = ("users", "sessions") MESSAGING_TABLES = ("chats", "chat_members", "messages", "idempotency_keys", "message_attachments") -ATTACHMENT_TABLES = ("attachments",) +ATTACHMENT_TABLES = ("attachments", "attachment_variants", "jobs", "outbox", "job_events") def sql_tables(zone: str, tables: tuple[str, ...]) -> set[str]: diff --git a/student-tests/api/test_message_attachments.py b/student-tests/api/test_message_attachments.py index dc95b5aa..494a2fb5 100644 --- a/student-tests/api/test_message_attachments.py +++ b/student-tests/api/test_message_attachments.py @@ -14,7 +14,7 @@ async def attach(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) content=SAMPLE, headers={**actor(user), "Content-Type": "image/png"}, ) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text return response.json()["id"] diff --git a/student-tests/api/test_pipeline.py b/student-tests/api/test_pipeline.py new file mode 100644 index 00000000..91f98761 --- /dev/null +++ b/student-tests/api/test_pipeline.py @@ -0,0 +1,244 @@ +import asyncio +import json +from typing import Any +from uuid import uuid4 + +import asyncpg +import pytest +from httpx import AsyncClient +from test_attachments import storage, trio, upload + +from conftest import DATABASE, actor +from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.shared.application.errors import Unavailable +from pictures import SAMPLE +from pipeline import pipeline_of + + +async def rows(sql: str, *args: Any) -> list[dict[str, Any]]: + connection = await asyncpg.connect(DATABASE["url"]) + try: + return [dict(row) for row in await connection.fetch(sql, *args)] + finally: + await connection.close() + + +async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]: + alice, bob, _, chat = await trio(http) + response = await upload(http, alice, chat) + assert response.status_code == 202, response.text + return alice, bob, response.json()["id"] + + +async def state(attachment_id: str) -> dict[str, Any]: + (row,) = await rows( + "SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, " + "j.status AS job_status, j.attempt, j.claims FROM attachments a " + "JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1", + attachment_id, + ) + return row + + +async def variants(attachment_id: str) -> list[dict[str, Any]]: + return await rows( + "SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants " + "WHERE attachment_id = $1 ORDER BY name", + attachment_id, + ) + + +async def events(attachment_id: str) -> list[str]: + found = await rows( + "SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id + ) + return [row["kind"] for row in found] + + +async def job_of(attachment_id: str) -> Job: + (row,) = await rows( + "SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) " + "WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1", + attachment_id, + ) + return JobMessage.model_validate_json(row["body"]).job() + + +async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None: + _, bob, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + outcomes = await pipeline.drain() + assert Outcome.DONE in outcomes + done = await state(attachment_id) + assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1) + assert done["original_key"] == f"originals/{attachment_id}" + assert done["staging_key"] == f"staging/{attachment_id}" + (thumbnail,) = await variants(attachment_id) + assert thumbnail["object_key"].startswith(f"variants/{attachment_id}/1/thumbnail-") + assert await pipeline.clean() >= 1 + assert (await state(attachment_id))["staging_key"] is None + objects = (await storage(client)).objects + assert f"staging/{attachment_id}" not in objects + assert objects[f"originals/{attachment_id}"][0] == SAMPLE + content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob)) + assert content.content == SAMPLE + assert await events(attachment_id) == ["enqueued", "claimed", "done"] + + +async def test_message_is_a_contract_job(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + mine = [ + json.loads(body) + for body in pipeline.publisher.bodies + if json.loads(body)["attachment_id"] == attachment_id + ] + assert len(mine) == 1 + assert set(mine[0]) == { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + } + assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1) + assert await pipeline.publish() == [] + + +async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.drain() + before, job = await variants(attachment_id), await job_of(attachment_id) + assert await pipeline.process(job) is Outcome.DUPLICATE + assert await variants(attachment_id) == before + assert (await state(attachment_id))["status"] == "ready" + assert (await events(attachment_id))[-1] == "duplicate_noop" + + +async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job)) + assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE]) + assert len(await variants(attachment_id)) == 1 + assert (await state(attachment_id))["claims"] == 1 + + +async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + store = await storage(client) + store.failing = True + try: + assert await pipeline.process(job) is Outcome.RESCHEDULED + finally: + store.failing = False + queued = await state(attachment_id) + assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2) + (retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id] + assert (retry.id, retry.attempt) == (job.id, 2) + assert await pipeline.process(retry) is Outcome.DONE + assert (await state(attachment_id))["status"] == "ready" + + +async def test_attempts_are_bounded(client: AsyncClient) -> None: + alice, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.faults = FailingFaults() + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["job_status"]) == ( + "failed", + "retry_exhausted", + "failed", + ) + assert failed["attempt"] == pipeline.policy.max_attempts + assert (await events(attachment_id)).count("retry_scheduled") == 4 + content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice)) + assert content.status_code == 409 + + +async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + (await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png") + pipeline = await pipeline_of(client) + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["attempt"]) == ( + "failed", + "invalid_image", + 1, + ) + + +async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + uow = pipeline.uow() + async with uow.transaction(): + claim = await uow.jobs.claim(job, uuid4(), pipeline.lease) + await uow.commit() + assert claim is not None + assert (await state(attachment_id))["status"] == "processing" + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1", + attachment_id, + ) + assert await pipeline.recover() >= 1 + assert (await state(attachment_id))["status"] == "queued" + late = pipeline.uow() + async with late.transaction(): + outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}") + await late.commit() + assert outcome is Outcome.FENCED + assert await variants(attachment_id) == [] + await pipeline.drain() + recovered = await state(attachment_id) + assert (recovered["status"], recovered["attempt"]) == ("ready", 2) + assert {"lease_expired", "fenced"} <= set(await events(attachment_id)) + + +async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.publisher.failing = True + with pytest.raises(Unavailable): + await pipeline.publish() + pending = await rows( + "SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1", + attachment_id, + ) + assert pending == [{"published_at": None}] + pipeline.publisher.failing = False + assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish()) + + +async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60)) + pipeline.faults = FailingFaults() + await pipeline.drain() + queued = await state(attachment_id) + assert (queued["status"], queued["attempt"]) == ("queued", 2) + (row,) = await rows( + "SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs " + "WHERE attachment_id = $1", + attachment_id, + ) + assert 25 < row["wait"] <= 30 + + +class FailingFaults: + def reach(self, point: str) -> None: + raise Unavailable(point) diff --git a/student-tests/api/test_upload_consistency.py b/student-tests/api/test_upload_consistency.py index 30bc91cf..b65bb084 100644 --- a/student-tests/api/test_upload_consistency.py +++ b/student-tests/api/test_upload_consistency.py @@ -13,6 +13,7 @@ from messenger.attachments.domain.events import AttachmentDiscarded from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.shared.application.messagebus import EventBus from pictures import SAMPLE +from pipeline import settle AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1" @@ -51,7 +52,7 @@ async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) "SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name ) assert found is not None - store.objects[f"attachments/{found}"] = (SAMPLE, "image/png") + store.objects[f"staging/{found}"] = (SAMPLE, "image/png") return found @@ -68,7 +69,7 @@ async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> N f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$; CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW - WHEN (NEW.filename = '{name}' AND NEW.status = 'ready') EXECUTE FUNCTION {trigger}()""" + WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()""" ) before, seen = set(store.objects), await discarded(client) try: @@ -78,6 +79,14 @@ async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> N assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert ( + await scalar( + "SELECT count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id " + "WHERE a.filename = $1", + name, + ) + == 0 + ) assert set(store.objects) == before assert [event.reason for event in seen] == ["compensation"] @@ -87,7 +96,7 @@ async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: store = await storage(client) async def swept_meanwhile(key: str) -> None: - await execute("DELETE FROM attachments WHERE object_key = $1", key) + await execute("DELETE FROM attachments WHERE staging_key = $1", key) store.after_put = swept_meanwhile before = set(store.objects) @@ -107,12 +116,13 @@ async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) await execute(AGED, pending) assert await sweeper(client).sweep() >= 1 assert not await exists(pending) - assert f"attachments/{pending}" not in (await storage(client)).objects + assert f"staging/{pending}" not in (await storage(client)).objects async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None: alice, bob, _, chat = await trio(client) ready = (await upload(client, alice, chat)).json() + await settle(client) sent = await client.post( f"/api/v1/chats/{chat['id']}/messages", json={"text": "", "attachment_ids": [ready["id"]]}, @@ -125,9 +135,9 @@ async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClie assert await sweeper(client).sweep() == 1 objects = (await storage(client)).objects assert not await exists(old) - assert f"attachments/{old}" not in objects + assert f"staging/{old}" not in objects assert await exists(fresh) - assert f"attachments/{fresh}" in objects + assert f"staging/{fresh}" in objects download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob)) assert download.status_code == 200 assert download.content == SAMPLE @@ -146,7 +156,7 @@ async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: assert {event.reason for event in seen} == {"abandoned"} assert await sweeper(client).sweep() == 0 objects = (await storage(client)).objects - assert not any(f"attachments/{attachment_id}" in objects for attachment_id in pending) + assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending) async def test_background_sweeper_removes_abandoned_intents() -> None: @@ -183,7 +193,7 @@ async def test_upload_waits_for_a_free_slot() -> None: await asyncio.sleep(0.3) assert read == ["first"] gate.set() - assert [(await first).status_code, (await second).status_code] == [201, 201] + assert [(await first).status_code, (await second).status_code] == [202, 202] assert read == ["first", "second"] @@ -241,4 +251,4 @@ async def test_stalled_upload_times_out_and_frees_the_slot() -> None: assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 - assert (await upload(http, alice, chat)).status_code == 201 + assert (await upload(http, alice, chat)).status_code == 202 diff --git a/student-tests/fakes.py b/student-tests/fakes.py index e0d3633e..2b0e23f8 100644 --- a/student-tests/fakes.py +++ b/student-tests/fakes.py @@ -58,7 +58,20 @@ class MemoryObjectStorage: data = self.objects[key][0] return ObjectContent(size=len(data), chunks=pieces(data)) + async def read(self, key: str) -> bytes: + content = await self.open(key) + return b"".join([chunk async for chunk in content.chunks]) + + async def copy(self, source: str, target: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") + if source not in self.objects: + raise ObjectMissingError(source) + self.objects[target] = self.objects[source] + async def delete(self, key: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") self.objects.pop(key, None) diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..f3bfcae5 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,117 @@ +from datetime import timedelta +from uuid import UUID + +from httpx import AsyncClient + +from conftest import resolve +from fakes import NullLogger +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.application.pipeline import ( + CleanStagingHandler, + ProcessJobHandler, + PublishOutboxHandler, + RecoverExpiredLeasesHandler, +) +from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer +from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.attachments.infrastructure.images import PillowVariantRenderer +from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork +from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.clock import MonotonicUtcClock +from messenger.shared.infrastructure.ids import Uuid7Generator +from messenger.shared.infrastructure.postgres import Database + +FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001) + + +class CapturingPublisher: + __slots__ = ("bodies", "failing", "job_ids") + + def __init__(self) -> None: + self.bodies: list[bytes] = [] + self.job_ids: list[UUID] = [] + self.failing = False + + async def publish(self, job_id: UUID, body: bytes) -> None: + if self.failing: + raise Unavailable("брокер недоступен") + self.job_ids.append(job_id) + self.bodies.append(body) + + +class InlinePipeline: + __slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage") + + def __init__( + self, + database: Database, + storage: ObjectStorage, + *, + policy: RetryPolicy = FAST, + lease: timedelta = timedelta(seconds=60), + ) -> None: + self._database = database + self.storage = storage + self.policy = policy + self.lease = lease + self.publisher = CapturingPublisher() + self.renderer: VariantRenderer = PillowVariantRenderer(40_000_000, 2) + self.faults: Faults = EnvironmentFaults("", NullLogger()) + + def uow(self) -> PostgresAttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(self._database) + + async def publish(self) -> list[Job]: + start = len(self.publisher.bodies) + await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100)) + return [ + JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:] + ] + + async def process(self, job: Job) -> Outcome: + handler = ProcessJobHandler( + self.uow(), + self.storage, + self.renderer, + self.faults, + clock=MonotonicUtcClock(), + ids=Uuid7Generator(), + policy=self.policy, + lease=self.lease, + ) + return await handler(ProcessJob(job)) + + async def drain(self, rounds: int = 20) -> list[Outcome]: + outcomes: list[Outcome] = [] + for _ in range(rounds): + jobs = await self.publish() + if not jobs: + return outcomes + outcomes.extend([await self.process(job) for job in jobs]) + return outcomes + + async def recover(self) -> int: + handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy) + return await handler(RecoverExpiredLeases(100)) + + async def clean(self) -> int: + return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100)) + + +async def pipeline_of( + http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60) +) -> InlinePipeline: + database = await resolve(http, Database) + storage = await resolve(http, ObjectStorage) + return InlinePipeline(database, storage, policy=policy, lease=lease) + + +async def settle(http: AsyncClient) -> list[Outcome]: + return await (await pipeline_of(http)).drain() diff --git a/student-tests/unit/test_attachments_domain.py b/student-tests/unit/test_attachments_domain.py index e26d3101..db91c483 100644 --- a/student-tests/unit/test_attachments_domain.py +++ b/student-tests/unit/test_attachments_domain.py @@ -4,7 +4,7 @@ from uuid import uuid4 import pytest from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind @@ -46,7 +46,7 @@ def test_declared_type_accepts_only_png_and_jpeg(header: str | None) -> None: ImageKind.declared(header) -def test_attachment_is_pending_until_stored() -> None: +def test_attachment_is_pending_until_queued() -> None: attachment_id = uuid4() attachment = Attachment.stage( attachment_id=attachment_id, @@ -59,13 +59,19 @@ def test_attachment_is_pending_until_stored() -> None: now=MOMENT, ) assert attachment.status is AttachmentStatus.PENDING - assert attachment.object_key == f"attachments/{attachment_id}" + assert attachment.staging_key == f"staging/{attachment_id}" + assert (attachment.original_key, attachment.published_version, attachment.variants) == ( + None, + 0, + (), + ) assert attachment.collect_events() == [] - attachment.store(MOMENT - timedelta(seconds=1)) - assert attachment.status is AttachmentStatus.READY + attachment.enqueue(MOMENT - timedelta(seconds=1)) + assert attachment.status is AttachmentStatus.QUEUED + assert not attachment.ready assert attachment.updated_at == MOMENT events = attachment.collect_events() - assert [type(event) for event in events] == [AttachmentStored] + assert [type(event) for event in events] == [AttachmentQueued] assert attachment.collect_events() == [] diff --git a/student-tests/unit/test_pipeline_domain.py b/student-tests/unit/test_pipeline_domain.py new file mode 100644 index 00000000..c801899f --- /dev/null +++ b/student-tests/unit/test_pipeline_domain.py @@ -0,0 +1,86 @@ +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import uuid4 + +import pytest + +from messenger.attachments.domain.jobs import Job, RetryPolicy +from messenger.attachments.domain.variants import fit, variant_key +from messenger.attachments.entrypoints.schemas import JobMessage + +SCHEMA = json.loads( + (Path(__file__).resolve().parents[2] / "contracts" / "jobs.schema.json").read_text() +) + + +@pytest.mark.parametrize( + ("size", "fitted"), + [ + ((320, 200), (256, 160)), + ((300, 200), (256, 170)), + ((801, 601), (256, 192)), + ((17, 1000), (4, 256)), + ((3000, 2), (256, 1)), + ((2, 3000), (1, 256)), + ((64, 64), (64, 64)), + ((256, 100), (256, 100)), + ((257, 257), (256, 256)), + ], +) +def test_thumbnail_fits_the_box_rounding_down( + size: tuple[int, int], fitted: tuple[int, int] +) -> None: + assert fit(*size, 256) == fitted + + +def test_retry_policy_doubles_up_to_the_cap_and_stops_at_the_limit() -> None: + policy = RetryPolicy(max_attempts=5, base_seconds=2, cap_seconds=10) + delays = [policy.delay(attempt) for attempt in range(1, 6)] + assert delays == [ + timedelta(seconds=2), + timedelta(seconds=4), + timedelta(seconds=8), + timedelta(seconds=10), + None, + ] + + +def test_job_message_follows_the_contract_schema() -> None: + job = Job(uuid4(), uuid4(), 1, 3, datetime(2026, 9, 1, 12, 0, tzinfo=UTC)) + payload = job.payload() + assert set(SCHEMA["required"]) <= set(payload) <= set(SCHEMA["properties"]) + assert payload["schema_version"] == SCHEMA["properties"]["schema_version"]["const"] + assert payload["type"] == SCHEMA["properties"]["type"]["const"] + assert payload["created_at"] == "2026-09-01T12:00:00.000000Z" + assert JobMessage.model_validate_json(json.dumps(payload)).job() == job + assert job.retried(4).payload()["job_id"] == payload["job_id"] + + +@pytest.mark.parametrize("example", SCHEMA["examples"]) +def test_contract_examples_are_accepted(example: dict[str, object]) -> None: + assert JobMessage.model_validate_json(json.dumps(example)).attempt == example["attempt"] + + +@pytest.mark.parametrize( + "change", + [ + {"schema_version": 2}, + {"type": "attachment.process.v2"}, + {"attempt": 0}, + {"pipeline_version": 0}, + {"filename": "a.png"}, + {"traceparent": "01-xyz"}, + {"created_at": "2026-09-01T12:00:00"}, + ], +) +def test_messages_off_the_schema_are_rejected(change: dict[str, object]) -> None: + message = {**SCHEMA["examples"][0], **change} + with pytest.raises(ValueError, match="validation error"): + JobMessage.model_validate_json(json.dumps(message)) + + +def test_variant_keys_carry_version_and_content_hash() -> None: + attachment = uuid4() + key = variant_key(attachment, 2, "thumbnail", "ab" * 32, "jpg") + assert key == f"variants/{attachment}/2/thumbnail-{'ab' * 8}.jpg" diff --git a/student-tests/unit/test_renderer.py b/student-tests/unit/test_renderer.py new file mode 100644 index 00000000..471fccdd --- /dev/null +++ b/student-tests/unit/test_renderer.py @@ -0,0 +1,89 @@ +from io import BytesIO + +import pytest +from PIL import Image + +from messenger.attachments.domain.errors import UnreadableImage +from messenger.attachments.infrastructure.images import render_variants +from pictures import SAMPLE + +LIMIT = 40_000_000 + + +def encoded(image: Image.Image, kind: str = "PNG", **options: object) -> bytes: + buffer = BytesIO() + image.save(buffer, kind, **options) + return buffer.getvalue() + + +def thumbnail(data: bytes) -> Image.Image: + (rendition,) = render_variants(data, LIMIT) + assert (rendition.name, rendition.content_type, rendition.extension) == ( + "thumbnail", + "image/jpeg", + "jpg", + ) + image = Image.open(BytesIO(rendition.data)) + assert image.format == "JPEG" + assert image.size == (rendition.width, rendition.height) + assert not image.getexif() + return image + + +def test_fixture_gives_256_by_160() -> None: + assert thumbnail(SAMPLE).size == (256, 160) + + +@pytest.mark.parametrize( + ("mode", "size", "fitted"), + [ + ("L", (17, 1000), (4, 256)), + ("RGB", (64, 64), (64, 64)), + ("RGB", (3000, 2), (256, 1)), + ("I;16", (300, 200), (256, 170)), + ("CMYK", (801, 601), (256, 192)), + ], +) +def test_sizes_round_down_and_never_upscale( + mode: str, size: tuple[int, int], fitted: tuple[int, int] +) -> None: + kind = "JPEG" if mode == "CMYK" else "PNG" + assert thumbnail(encoded(Image.new(mode, size), kind)).size == fitted + + +@pytest.mark.parametrize("mode", ["RGBA", "LA"]) +def test_transparency_is_composed_on_white(mode: str) -> None: + transparent = Image.new(mode, (300, 300), (0,) * len(mode)) + assert thumbnail(encoded(transparent)).convert("RGB").getpixel((5, 5)) == (255, 255, 255) + + +def test_palette_transparency_is_composed_on_white() -> None: + palette = Image.new("P", (200, 100)) + image = thumbnail(encoded(palette, transparency=0)) + assert image.size == (200, 100) + assert image.convert("RGB").getpixel((5, 5)) == (255, 255, 255) + + +@pytest.mark.parametrize(("orientation", "fitted"), [(6, (128, 256)), (3, (256, 128))]) +def test_exif_orientation_is_applied_before_sizing( + orientation: int, fitted: tuple[int, int] +) -> None: + exif = Image.Exif() + exif[0x0112] = orientation + data = encoded(Image.new("RGB", (400, 200), (10, 200, 10)), "JPEG", exif=exif) + assert thumbnail(data).size == fitted + + +@pytest.mark.parametrize( + "data", + [b"", b"garbage", SAMPLE[: len(SAMPLE) // 2], b"GIF89a" + bytes(20)], + ids=["empty", "garbage", "truncated", "gif"], +) +def test_undecodable_data_is_a_permanent_error(data: bytes) -> None: + with pytest.raises(UnreadableImage): + render_variants(data, LIMIT) + + +def test_pixel_limit_is_checked_before_decoding() -> None: + with pytest.raises(UnreadableImage): + render_variants(SAMPLE, 320 * 200 - 1) diff --git a/uv.lock b/uv.lock index e2f4b436..5c98efc5 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,19 @@ version = 1 revision = 5 requires-python = "==3.14.*" +[[package]] +name = "aio-pika" +version = "10.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiormq" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/31/70f0d7a9fe1affc3241d86d24c24714a4a2ac648961efd3d835ef734a6a1/aio_pika-10.2.1.tar.gz", hash = "sha256:e4f77afd9179a3cca5a904c3b0a1190256a36d9f90650db908b1b2245eb27487", size = 101040, upload-time = "2026-10-25T11:47:24.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/ae/f69eb62f240f743d5d3aff659a6ed76077f73b722a0d8f9f9559ab825836/aio_pika-10.2.1-py3-none-any.whl", hash = "sha256:66c85d311e30f1dd3e28109d686c9bf1d9dd1a74582004c7cb7bed284d8fe3b1", size = 61568, upload-time = "2026-10-25T11:47:23.119Z" }, +] + [[package]] name = "aiobotocore" version = "3.10.0" @@ -97,6 +110,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, ] +[[package]] +name = "aiormq" +version = "7.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pamqp" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/11/8b93eac7630c5087c34df3d34ce0bdf50d6ea2fb83061f17e8c73877141d/aiormq-7.2.4.tar.gz", hash = "sha256:5fd20bb9583d73de85572198496b65cf028f27527c1733c9840d458e0b40e8fa", size = 81569, upload-time = "2026-10-24T12:36:22.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/a7/346b74a5f19d0acbd9d428f5f7b070537a8a079e45a74fba58c7831e058d/aiormq-7.2.4-py3-none-any.whl", hash = "sha256:2b9d6dade171318185ab8f507717b630f20b8b2303a075f9293d3f6b0d166f1d", size = 40694, upload-time = "2026-10-24T12:36:21.054Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -304,6 +330,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/08/dd3259a319ba45c48d975caf237c830c7873a203288cb4ab186cc5bef585/dishka-1.10.1-py3-none-any.whl", hash = "sha256:84f40c1115eb391ce3fb558fd7946c5bcaf6361c35b77b7a185b7697a5720ec7", size = 120062, upload-time = "2026-04-25T12:07:59.153Z" }, ] +[[package]] +name = "dishka-faststream" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dishka" }, + { name = "faststream" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/32/c8df3791072a5ea11abaa8e142536fe8cd7745f3fe25e4be4fdf3ac112f5/dishka_faststream-0.7.0.tar.gz", hash = "sha256:b8c6e95bf30f32fc67b3289697cee51c3047778afe195aa9a1062d21e64bfb7b", size = 118092, upload-time = "2026-06-04T17:34:30.856Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/9f/b1918e3ee6814f1b94489ff367b9178e2730a3d23370c24f2f2bce5886e5/dishka_faststream-0.7.0-py3-none-any.whl", hash = "sha256:6f68ce1520b9f15478c8e8407e0d36cdcf1261d237a81e0090fb34ab4775e67e", size = 10551, upload-time = "2026-06-04T17:34:29.953Z" }, +] + +[[package]] +name = "fast-depends" +version = "3.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/76/1e67c34a18a75aff77f25de2d56de8ad3d38c05915ae513ba35293fd920b/fast_depends-3.0.10.tar.gz", hash = "sha256:234c1dcfcdbc1cdb0ebe543fbf8e8234c069ba39518fbdf1f71dad6e3e6111ad", size = 19807, upload-time = "2026-10-28T07:36:21.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/af/af933c80bd2c60070dec7ea324350ce698d579bada6ab6511800f749ab57/fast_depends-3.0.10-py3-none-any.whl", hash = "sha256:e49306f54b691a59076e6fe1fdae9418bff4af80d1c9d9fb82a6d517b4d4c04c", size = 25535, upload-time = "2026-10-28T07:36:20.374Z" }, +] + +[package.optional-dependencies] +pydantic = [ + { name = "pydantic" }, +] + [[package]] name = "fastapi" version = "0.141.1" @@ -320,6 +377,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] +[[package]] +name = "faststream" +version = "0.7.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "fast-depends", extra = ["pydantic"] }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/b6/d15b1c88f1adc3bf11e4e23f8a057e489e2e60a47f7d4b821a3ae264f4c2/faststream-0.7.8.tar.gz", hash = "sha256:9e367d9ba8b59eeafad19f46641b11f20d23a7ed75ea27b2503856a8ae0c075a", size = 392145, upload-time = "2026-10-22T20:50:29.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/79/daae6f842665756f5fa45683d4762333348c4ad4dba2e9d9f37644717d74/faststream-0.7.8-py3-none-any.whl", hash = "sha256:ffe412278cf116216078dcf29dcec93274994a98d036be2bc388071dff7e259e", size = 618852, upload-time = "2026-10-22T20:50:27.388Z" }, +] + +[package.optional-dependencies] +rabbit = [ + { name = "aio-pika" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -465,7 +541,9 @@ dependencies = [ { name = "argon2-cffi" }, { name = "asyncpg" }, { name = "dishka" }, + { name = "dishka-faststream" }, { name = "fastapi" }, + { name = "faststream", extra = ["rabbit"] }, { name = "gunicorn" }, { name = "orjson" }, { name = "pillow" }, @@ -492,7 +570,9 @@ requires-dist = [ { name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "asyncpg", specifier = ">=0.32.0" }, { name = "dishka", specifier = ">=1.7.0" }, + { name = "dishka-faststream", specifier = ">=0.1" }, { name = "fastapi", specifier = ">=0.118.0" }, + { name = "faststream", extras = ["rabbit"], specifier = ">=0.7.2,<0.8" }, { name = "gunicorn", specifier = ">=26.2.0" }, { name = "orjson", specifier = ">=3.12.0" }, { name = "pillow", specifier = ">=12.0.0" }, @@ -593,6 +673,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pamqp" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/4c/33a0ddaaac7bc42f9a542dbaaee8b580ceca3f89bf5da7c498d1fa97ff9a/pamqp-4.0.1.tar.gz", hash = "sha256:9dd13b828e346622793981f14a5df817fce5de998c746209d6c0154eb8403970", size = 137192, upload-time = "2026-07-06T16:37:51.732Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/14/1dfc08b743ba995a38dee0ea09beb46a05c7fe8ac53d729095905f7bf11d/pamqp-4.0.1-py3-none-any.whl", hash = "sha256:a547f45128b06e42ce8d7a739b0cfcc40f2c724770622eaaff4a3f587b1cf7d0", size = 32773, upload-time = "2026-07-06T16:37:50.623Z" }, +] + [[package]] name = "pillow" version = "13.0.0"