From b152b6e07e365167c9660dcf8dfde24681aa0284 Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 29 Oct 2026 01:08:06 +0300 Subject: [PATCH] Process attachment jobs through an outbox, a relay and a worker Upload now stages the checked original under staging/, and one transaction moves the intent to queued, inserts the job (pipeline version 1, attempt 1), its outbox row and an audit event; the 202 is the in-memory snapshot, never a reread. Content answers 409 until the attachment is ready, the thumbnail variant is served from the published version and absent variants are 404. The relay publishes outbox rows through FastStream with publisher confirms, mandatory routing and persistent delivery, marks a row only after the broker acked it, returns jobs whose lease expired and deletes the staging object of ready attachments. The worker consumes the quorum queue through a FastStream subscriber that declares nothing, takes prefetch from WORKER_CONCURRENCY and acks manually after the result commit, rejects messages off jobs.schema.json to the dead letter exchange and requeues after an infrastructure error. A job is claimed under a lease token; the original is copied to originals/ and a 256 px JPEG thumbnail is written under a content-addressed key; the commit is fenced by the token, inserts variants with ON CONFLICT DO NOTHING and publishes only a newer pipeline version. A transient error reschedules the same job with an exponential delay and a new outbox row, the last attempt fails with retry_exhausted, an undecodable image fails at once with invalid_image, and a redelivered finished job is a recorded no-op. The worker and the relay are assembled in worker.py and relay.py with the dishka FastStream integration and the platform providers of the API; the shared core imports neither FastStream nor aio-pika. Tests cover the thumbnail geometry and rendering, the job message against the contract, the pipeline against PostgreSQL (duplicates, parallel copies, retries, the attempt bound, leases and fencing) and the broker adapters against a throwaway RabbitMQ with the stand's topology and permissions. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../add-lab5-async-attachments/tasks.md | 26 +- pyproject.toml | 4 + .../attachments/application/commands.py | 21 ++ .../attachments/application/handlers.py | 16 +- .../attachments/application/pipeline.py | 219 +++++++++++++ .../attachments/application/ports.py | 90 +++++- .../attachments/application/queries.py | 23 +- src/messenger/attachments/config.py | 21 ++ src/messenger/attachments/domain/errors.py | 8 + src/messenger/attachments/domain/events.py | 16 +- src/messenger/attachments/domain/jobs.py | 73 +++++ src/messenger/attachments/domain/models.py | 34 +- src/messenger/attachments/domain/variants.py | 31 ++ .../attachments/entrypoints/errors.py | 2 + .../attachments/entrypoints/heartbeat.py | 44 +++ src/messenger/attachments/entrypoints/http.py | 12 +- src/messenger/attachments/entrypoints/ioc.py | 143 +++++++- .../attachments/entrypoints/journal.py | 42 ++- .../attachments/entrypoints/relay.py | 67 ++++ .../attachments/entrypoints/schemas.py | 58 +++- .../attachments/entrypoints/worker.py | 50 +++ .../attachments/infrastructure/faults.py | 26 ++ .../attachments/infrastructure/images.py | 57 +++- .../attachments/infrastructure/postgres.py | 305 +++++++++++++++++- .../attachments/infrastructure/rabbitmq.py | 37 +++ .../attachments/infrastructure/s3.py | 15 + src/messenger/main.py | 7 +- src/messenger/relay.py | 66 ++++ src/messenger/worker.py | 78 +++++ student-tests/adapters/test_rabbitmq.py | 259 +++++++++++++++ student-tests/api/test_attachments.py | 59 +++- student-tests/api/test_bounded_contexts.py | 26 +- student-tests/api/test_message_attachments.py | 2 +- student-tests/api/test_pipeline.py | 244 ++++++++++++++ student-tests/api/test_upload_consistency.py | 28 +- student-tests/fakes.py | 13 + student-tests/pipeline.py | 117 +++++++ student-tests/unit/test_attachments_domain.py | 18 +- student-tests/unit/test_pipeline_domain.py | 86 +++++ student-tests/unit/test_renderer.py | 89 +++++ uv.lock | 89 +++++ 41 files changed, 2506 insertions(+), 115 deletions(-) create mode 100644 src/messenger/attachments/application/pipeline.py create mode 100644 src/messenger/attachments/domain/jobs.py create mode 100644 src/messenger/attachments/domain/variants.py create mode 100644 src/messenger/attachments/entrypoints/heartbeat.py create mode 100644 src/messenger/attachments/entrypoints/relay.py create mode 100644 src/messenger/attachments/entrypoints/worker.py create mode 100644 src/messenger/attachments/infrastructure/faults.py create mode 100644 src/messenger/attachments/infrastructure/rabbitmq.py create mode 100644 src/messenger/relay.py create mode 100644 src/messenger/worker.py create mode 100644 student-tests/adapters/test_rabbitmq.py create mode 100644 student-tests/api/test_pipeline.py create mode 100644 student-tests/pipeline.py create mode 100644 student-tests/unit/test_pipeline_domain.py create mode 100644 student-tests/unit/test_renderer.py diff --git a/openspec/changes/add-lab5-async-attachments/tasks.md b/openspec/changes/add-lab5-async-attachments/tasks.md index 9d2849ec..4a57c854 100644 --- a/openspec/changes/add-lab5-async-attachments/tasks.md +++ b/openspec/changes/add-lab5-async-attachments/tasks.md @@ -5,25 +5,25 @@ ## 2. Зона `attachments`: домен и приложение -- [ ] 2.1 Домен: статусы `queued/processing/failed`, `Attachment.enqueue`, `Variant`, геометрия `fit`, `Job` и его сообщение по `jobs.schema.json`, `Lease`, `RetryPolicy`, `FailureCode`, `Outcome`, события `AttachmentQueued`, `AttachmentReady`, `AttachmentFailed`, `JobRescheduled`, ошибка `AttachmentNotReady` -- [ ] 2.2 Порты: `ObjectStorage.read`/`copy`, `VariantRenderer`, `JobPublisher`, `Faults`, репозитории `jobs` и `outbox` в `AttachmentsUnitOfWork`, `AttachmentDirectory.visible` -- [ ] 2.3 Upload: `queued` вместо `ready`, задание и outbox в той же транзакции; запросы `GetAttachment`/`OpenAttachment` с `409` до `ready` и вариантами опубликованной версии -- [ ] 2.4 Обработчики `ProcessJob` (захват, staging → оригинал, рендер, запись вариантов, фиксация с fencing, повтор/финальная ошибка), `PublishOutbox`, `RecoverExpiredLeases`, `CleanStaging` -- [ ] 2.5 Проверка группы: unit-тесты `fit`, `RetryPolicy`, сообщения задания против `contracts/jobs.schema.json` +- [x] 2.1 Домен: статусы `queued/processing/failed`, `Attachment.enqueue`, `Variant`, геометрия `fit`, `Job` и его сообщение по `jobs.schema.json`, `Lease`, `RetryPolicy`, `FailureCode`, `Outcome`, события `AttachmentQueued`, `AttachmentReady`, `AttachmentFailed`, `JobRescheduled`, ошибка `AttachmentNotReady` +- [x] 2.2 Порты: `ObjectStorage.read`/`copy`, `VariantRenderer`, `JobPublisher`, `Faults`, репозитории `jobs` и `outbox` в `AttachmentsUnitOfWork`, `AttachmentDirectory.visible` +- [x] 2.3 Upload: `queued` вместо `ready`, задание и outbox в той же транзакции; запросы `GetAttachment`/`OpenAttachment` с `409` до `ready` и вариантами опубликованной версии +- [x] 2.4 Обработчики `ProcessJob` (захват, staging → оригинал, рендер, запись вариантов, фиксация с fencing, повтор/финальная ошибка), `PublishOutbox`, `RecoverExpiredLeases`, `CleanStaging` +- [x] 2.5 Проверка группы: unit-тесты `fit`, `RetryPolicy`, сообщения задания против `contracts/jobs.schema.json` ## 3. Зона `attachments`: инфраструктура -- [ ] 3.1 PostgreSQL: вложение с ключами и вариантами, задания, outbox, события, захват аренды, фиксация с fencing и `published_version < v`, повтор, финальная ошибка, истёкшие аренды, staging к уборке -- [ ] 3.2 S3: `read` и `copy`; Pillow: `PillowVariantRenderer` (EXIF orientation, 16 бит, alpha на белом, `fit`, JPEG без метаданных, классификация `invalid_image`) -- [ ] 3.3 FastStream: адаптер `JobPublisher` (exchange без объявления, `persist`, `mandatory`, confirm, `message_id`), `FAULT_POINT` с точкой `transient_error`; зависимости `faststream[rabbit]`, `dishka-faststream` в `uv.lock` -- [ ] 3.4 Проверка группы: тест рендера (fixture 320×200 → 256×160, RGBA на белом, EXIF 6, 17×1000 → 4×256, битые данные → `invalid_image`) +- [x] 3.1 PostgreSQL: вложение с ключами и вариантами, задания, outbox, события, захват аренды, фиксация с fencing и `published_version < v`, повтор, финальная ошибка, истёкшие аренды, staging к уборке +- [x] 3.2 S3: `read` и `copy`; Pillow: `PillowVariantRenderer` (EXIF orientation, 16 бит, alpha на белом, `fit`, JPEG без метаданных, классификация `invalid_image`) +- [x] 3.3 FastStream: адаптер `JobPublisher` (exchange без объявления, `persist`, `mandatory`, confirm, `message_id`), `FAULT_POINT` с точкой `transient_error`; зависимости `faststream[rabbit]`, `dishka-faststream` в `uv.lock` +- [x] 3.4 Проверка группы: тест рендера (fixture 320×200 → 256×160, RGBA на белом, EXIF 6, 17×1000 → 4×256, битые данные → `invalid_image`) ## 4. Точки входа и сборка процессов -- [ ] 4.1 HTTP: `202` upload, `variant` из четырёх значений, `409`, схема `AttachmentOut` со статусом и вариантами, реестр ошибок, журнал -- [ ] 4.2 `PipelineSettings`, `PipelineProvider`, точка входа worker (FastStream-подписчик, проверка сообщения, `ack`/`reject`/`nack`), циклы relay, heartbeat -- [ ] 4.3 Сборки `relay.py` и `worker.py`, тест графа импортов: сборки процессов знают одну зону, `shared/` не импортирует FastStream -- [ ] 4.4 Проверка группы: тесты pipeline против одноразового PostgreSQL с памятью вместо S3 — `ready`, дубль после `ready` без новых строк, параллельные копии, временная ошибка → `queued` и новая строка outbox, исчерпание → `retry_exhausted`, `invalid_image` без повторов, fencing чужой аренды, возврат истёкшей аренды, уборка staging, публикация outbox +- [x] 4.1 HTTP: `202` upload, `variant` из четырёх значений, `409`, схема `AttachmentOut` со статусом и вариантами, реестр ошибок, журнал +- [x] 4.2 `PipelineSettings`, `PipelineProvider`, точка входа worker (FastStream-подписчик, проверка сообщения, `ack`/`reject`/`nack`), циклы relay, heartbeat +- [x] 4.3 Сборки `relay.py` и `worker.py`, тест графа импортов: сборки процессов знают одну зону, `shared/` не импортирует FastStream +- [x] 4.4 Проверка группы: тесты pipeline против одноразового PostgreSQL с памятью вместо S3 — `ready`, дубль после `ready` без новых строк, параллельные копии, временная ошибка → `queued` и новая строка outbox, исчерпание → `retry_exhausted`, `invalid_image` без повторов, fencing чужой аренды, возврат истёкшей аренды, уборка staging, публикация outbox ## 5. Связь зон diff --git a/pyproject.toml b/pyproject.toml index 1b2e3f49..d1b68b6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ dependencies = [ "redis>=8.1.0", "aiobotocore>=3.10.0", "pillow>=12.0.0", + "faststream[rabbit]>=0.7.2,<0.8", + "dishka-faststream>=0.1", ] [dependency-groups] @@ -88,6 +90,8 @@ ignore = [ "src/messenger/shared/config.py" = ["ARG002", "ARG003"] "src/messenger/main.py" = ["TID251"] "src/messenger/migrate.py" = ["TID251"] +"src/messenger/relay.py" = ["TID251"] +"src/messenger/worker.py" = ["TID251"] "src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"] [tool.ruff.lint.flake8-tidy-imports.banned-api] diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py index b42141fa..b7543f3d 100644 --- a/src/messenger/attachments/application/commands.py +++ b/src/messenger/attachments/application/commands.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from datetime import timedelta from uuid import UUID +from messenger.attachments.domain.jobs import Job from messenger.shared.application.commands import Command @@ -19,3 +20,23 @@ class UploadAttachment(Command): class SweepAbandonedUploads(Command): grace: timedelta limit: int + + +@dataclass(frozen=True, slots=True) +class ProcessJob(Command): + job: Job + + +@dataclass(frozen=True, slots=True) +class PublishOutbox(Command): + limit: int + + +@dataclass(frozen=True, slots=True) +class RecoverExpiredLeases(Command): + limit: int + + +@dataclass(frozen=True, slots=True) +class CleanStaging(Command): + limit: int diff --git a/src/messenger/attachments/application/handlers.py b/src/messenger/attachments/application/handlers.py index 36e767b5..0ff449aa 100644 --- a/src/messenger/attachments/application/handlers.py +++ b/src/messenger/attachments/application/handlers.py @@ -10,6 +10,7 @@ from messenger.attachments.application.ports import ( ObjectStorage, ) from messenger.attachments.domain.errors import ChatNotFound, PayloadTooLarge +from messenger.attachments.domain.jobs import Job from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageKind from messenger.shared.application.ports import Clock, IdGenerator, Logger @@ -35,7 +36,8 @@ async def read_limited(body: AsyncIterator[bytes], limit: int) -> tuple[bytes, s async def drop( uow: AttachmentsUnitOfWork, storage: ObjectStorage, attachment: Attachment, reason: str ) -> None: - await storage.delete(attachment.object_key) + if attachment.staging_key is not None: + await storage.delete(attachment.staging_key) attachment.discard(reason) await uow.attachments.remove(attachment) await uow.commit() @@ -49,6 +51,7 @@ class UploadAttachmentHandler: "_limit", "_logger", "_membership", + "_pipeline_version", "_slots", "_storage", "_uow", @@ -66,6 +69,7 @@ class UploadAttachmentHandler: limit: int, slots: UploadSlots, logger: Logger, + pipeline_version: int = 1, ) -> None: self._uow = uow self._membership = membership @@ -76,6 +80,7 @@ class UploadAttachmentHandler: self._limit = limit self._slots = slots self._logger = logger + self._pipeline_version = pipeline_version async def __call__(self, command: UploadAttachment) -> Attachment: if not await self._membership.is_member(command.chat_id, command.owner_id): @@ -84,10 +89,13 @@ class UploadAttachmentHandler: declared = ImageKind.declared(command.content_type) async with self._slots: attachment = await self._write(command, filename, declared) - attachment.store(self._clock.now()) + now = self._clock.now() + attachment.enqueue(now) + job = Job(self._ids.new_id(), attachment.id, self._pipeline_version, 1, now) try: async with self._uow.transaction(): await self._uow.attachments.save(attachment) + await self._uow.jobs.add(job) await self._uow.commit() except Exception: await self._compensate(attachment) @@ -112,7 +120,7 @@ class UploadAttachmentHandler: async with self._uow.transaction(): await self._uow.attachments.add(attachment) await self._uow.commit() - await self._storage.put(attachment.object_key, data, attachment.content_type) + await self._storage.put(str(attachment.staging_key), data, attachment.content_type) return attachment async def _compensate(self, staged: Attachment) -> None: @@ -120,7 +128,7 @@ class UploadAttachmentHandler: async with self._uow.transaction(): current = await self._uow.attachments.lock(staged.id) if current is None: - await self._storage.delete(staged.object_key) + await self._storage.delete(str(staged.staging_key)) elif current.status is AttachmentStatus.PENDING: await drop(self._uow, self._storage, current, "compensation") except Exception as error: diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py new file mode 100644 index 00000000..803c0961 --- /dev/null +++ b/src/messenger/attachments/application/pipeline.py @@ -0,0 +1,219 @@ +import asyncio +from datetime import timedelta +from hashlib import sha256 + +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.application.ports import ( + AttachmentsUnitOfWork, + Faults, + JobPublisher, + ObjectStorage, + VariantRenderer, +) +from messenger.attachments.domain.errors import UnreadableImage +from messenger.attachments.domain.events import JobSettled +from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy +from messenger.attachments.domain.models import original_key +from messenger.attachments.domain.variants import Variant, variant_key +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Clock, IdGenerator + +TRANSIENT_FAULT = "transient_error" + + +def settled( + job: Job, + outcome: Outcome, + *, + error_code: str | None = None, + delay_seconds: float | None = None, +) -> JobSettled: + return JobSettled( + job_id=job.id, + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + attempt=job.attempt, + outcome=outcome.value, + error_code=error_code, + delay_seconds=delay_seconds, + ) + + +class ProcessJobHandler: + __slots__ = ("_clock", "_faults", "_ids", "_lease", "_policy", "_renderer", "_storage", "_uow") + + def __init__( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + renderer: VariantRenderer, + faults: Faults, + *, + clock: Clock, + ids: IdGenerator, + policy: RetryPolicy, + lease: timedelta, + ) -> None: + self._uow = uow + self._storage = storage + self._renderer = renderer + self._faults = faults + self._clock = clock + self._ids = ids + self._policy = policy + self._lease = lease + + async def __call__(self, command: ProcessJob) -> Outcome: + async with self._uow.transaction(): + claim = await self._uow.jobs.claim(command.job, self._ids.new_id(), self._lease) + if claim is None: + await self._uow.jobs.note(command.job, Outcome.DUPLICATE) + self._uow.record(settled(command.job, Outcome.DUPLICATE)) + await self._uow.commit() + if claim is None: + return Outcome.DUPLICATE + try: + variants = await self._produce(claim) + except UnreadableImage as error: + return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error) + except Unavailable as error: + return await self._retry(claim.lease, error) + except Exception as error: + return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error) + return await self._complete(claim.lease, variants) + + async def _produce(self, claim: Claim) -> list[Variant]: + job = claim.lease.job + self._faults.reach(TRANSIENT_FAULT) + data = await self._storage.read(claim.source_key) + renditions = await self._renderer.render(data, job.pipeline_version) + if claim.staged: + await self._storage.copy(claim.source_key, original_key(job.attachment_id)) + variants: list[Variant] = [] + for rendition in renditions: + digest = sha256(rendition.data).hexdigest() + key = variant_key( + job.attachment_id, job.pipeline_version, rendition.name, digest, rendition.extension + ) + await self._storage.put(key, rendition.data, rendition.content_type) + variants.append( + Variant( + name=rendition.name, + content_type=rendition.content_type, + size_bytes=len(rendition.data), + width=rendition.width, + height=rendition.height, + sha256=digest, + object_key=key, + ) + ) + return variants + + async def _complete(self, lease: Lease, variants: list[Variant]) -> Outcome: + async with self._uow.transaction(): + outcome = await self._uow.jobs.complete( + lease, variants, original_key(lease.job.attachment_id) + ) + self._uow.record(settled(lease.job, outcome)) + await self._uow.commit() + return outcome + + async def _retry(self, lease: Lease, error: Exception) -> Outcome: + delay = self._policy.delay(lease.job.attempt) + if delay is None: + return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error) + async with self._uow.transaction(): + at = self._clock.now() + delay + job = await self._uow.jobs.reschedule( + lease, at, Outcome.RESCHEDULED, type(error).__name__ + ) + outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED + if job is None: + await self._uow.jobs.note(lease.job, outcome) + self._uow.record(settled(lease.job, outcome, delay_seconds=delay.total_seconds())) + await self._uow.commit() + return outcome + + async def _fail(self, lease: Lease, code: FailureCode, error: Exception) -> Outcome: + async with self._uow.transaction(): + done = await self._uow.jobs.fail(lease, code, type(error).__name__) + outcome = Outcome.FAILED if done else Outcome.FENCED + if not done: + await self._uow.jobs.note(lease.job, outcome) + self._uow.record(settled(lease.job, outcome, error_code=code.value)) + await self._uow.commit() + return outcome + + +class PublishOutboxHandler: + __slots__ = ("_publisher", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher) -> None: + self._uow = uow + self._publisher = publisher + + async def __call__(self, command: PublishOutbox) -> int: + failure: Unavailable | None = None + published: list[int] = [] + async with self._uow.transaction(): + for entry in await self._uow.outbox.pending(command.limit): + try: + await self._publisher.publish(entry.job_id, entry.body) + except Unavailable as error: + failure = error + break + published.append(entry.id) + if published: + await self._uow.outbox.published(published) + await self._uow.commit() + if failure is not None: + raise failure + return len(published) + + +class RecoverExpiredLeasesHandler: + __slots__ = ("_clock", "_policy", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, clock: Clock, policy: RetryPolicy) -> None: + self._uow = uow + self._clock = clock + self._policy = policy + + async def __call__(self, command: RecoverExpiredLeases) -> int: + async with self._uow.transaction(): + leases = await self._uow.jobs.expired(command.limit) + for lease in leases: + if self._policy.delay(lease.job.attempt) is None: + await self._uow.jobs.fail( + lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value + ) + outcome = Outcome.FAILED + else: + await self._uow.jobs.reschedule( + lease, self._clock.now(), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value + ) + outcome = Outcome.LEASE_EXPIRED + self._uow.record(settled(lease.job, outcome)) + await self._uow.commit() + return len(leases) + + +class CleanStagingHandler: + __slots__ = ("_storage", "_uow") + + def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage) -> None: + self._uow = uow + self._storage = storage + + async def __call__(self, command: CleanStaging) -> int: + async with self._uow.transaction(): + staged = await self._uow.attachments.staged(command.limit) + await asyncio.gather(*(self._storage.delete(item.key) for item in staged)) + await self._uow.attachments.unstage([item.attachment_id for item in staged]) + await self._uow.commit() + return len(staged) diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index b1339546..dd2760a9 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -1,11 +1,13 @@ -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from typing import Protocol, runtime_checkable from uuid import UUID +from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome from messenger.attachments.domain.models import Attachment from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.domain.variants import Variant from messenger.shared.application.ports import UnitOfWork @@ -15,12 +17,39 @@ class ObjectContent: chunks: AsyncIterator[bytes] +@dataclass(frozen=True, slots=True) +class Rendition: + name: str + content_type: str + extension: str + data: bytes + width: int + height: int + + +@dataclass(frozen=True, slots=True) +class OutboxEntry: + id: int + job_id: UUID + body: bytes + + +@dataclass(frozen=True, slots=True) +class StagedObject: + attachment_id: UUID + key: str + + @runtime_checkable class ObjectStorage(Protocol): async def put(self, key: str, data: bytes, content_type: str) -> None: ... async def open(self, key: str) -> ObjectContent: ... + async def read(self, key: str) -> bytes: ... + + async def copy(self, source: str, target: str) -> None: ... + async def delete(self, key: str) -> None: ... @@ -29,6 +58,21 @@ class ImageInspector(Protocol): async def inspect(self, data: bytes, declared: ImageKind) -> ImageInfo: ... +@runtime_checkable +class VariantRenderer(Protocol): + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: ... + + +@runtime_checkable +class JobPublisher(Protocol): + async def publish(self, job_id: UUID, body: bytes) -> None: ... + + +@runtime_checkable +class Faults(Protocol): + def reach(self, point: str) -> None: ... + + @runtime_checkable class ChatMembership(Protocol): async def is_member(self, chat_id: UUID, user_id: UUID) -> bool: ... @@ -46,13 +90,53 @@ class AttachmentRepository(Protocol): async def remove(self, attachment: Attachment) -> None: ... + async def staged(self, limit: int) -> list[StagedObject]: ... + + async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ... + + +@runtime_checkable +class JobRepository(Protocol): + async def add(self, job: Job) -> None: ... + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: ... + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: ... + + async def reschedule( + self, lease: Lease, at: datetime, kind: Outcome, error: str + ) -> Job | None: ... + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: ... + + async def note(self, job: Job, kind: Outcome) -> None: ... + + async def expired(self, limit: int) -> list[Lease]: ... + + +@runtime_checkable +class OutboxRepository(Protocol): + async def pending(self, limit: int) -> list[OutboxEntry]: ... + + async def published(self, entry_ids: Sequence[int]) -> None: ... + @runtime_checkable class AttachmentsUnitOfWork(UnitOfWork, Protocol): @property def attachments(self) -> AttachmentRepository: ... + @property + def jobs(self) -> JobRepository: ... + + @property + def outbox(self) -> OutboxRepository: ... + @runtime_checkable class AttachmentDirectory(Protocol): - async def ready(self, attachment_id: UUID) -> Attachment | None: ... + async def visible(self, attachment_id: UUID) -> Attachment | None: ... + + async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... diff --git a/src/messenger/attachments/application/queries.py b/src/messenger/attachments/application/queries.py index d88c6740..973d759d 100644 --- a/src/messenger/attachments/application/queries.py +++ b/src/messenger/attachments/application/queries.py @@ -7,10 +7,12 @@ from messenger.attachments.application.ports import ( ObjectContent, ObjectStorage, ) -from messenger.attachments.domain.errors import AttachmentNotFound +from messenger.attachments.domain.errors import AttachmentNotFound, AttachmentNotReady from messenger.attachments.domain.models import Attachment from messenger.shared.application.ports import Logger +ORIGINAL = "original" + class GetAttachment: __slots__ = ("_attachments", "_membership") @@ -20,7 +22,7 @@ class GetAttachment: self._membership = membership async def __call__(self, attachment_id: UUID, user_id: UUID) -> Attachment: - attachment = await self._attachments.ready(attachment_id) + attachment = await self._attachments.visible(attachment_id) if attachment is None or not await self._membership.is_member(attachment.chat_id, user_id): raise AttachmentNotFound("вложение не найдено") return attachment @@ -35,12 +37,21 @@ class OpenAttachment: self._logger = logger async def __call__( - self, attachment_id: UUID, user_id: UUID - ) -> tuple[Attachment, ObjectContent]: + self, attachment_id: UUID, user_id: UUID, variant: str = ORIGINAL + ) -> tuple[str, ObjectContent]: attachment = await self._get(attachment_id, user_id) + if not attachment.ready: + raise AttachmentNotReady("вложение ещё не обработано или обработка не удалась") + if variant == ORIGINAL: + key, content_type = attachment.original_key, attachment.content_type + else: + found = attachment.variant(variant) + if found is None: + raise AttachmentNotFound("у вложения нет такого варианта") + key, content_type = found.object_key, found.content_type try: - content = await self._storage.open(attachment.object_key) + content = await self._storage.open(str(key)) except ObjectMissingError as error: self._logger.warning("attachment.object_missing", attachment_id=str(attachment.id)) raise AttachmentNotFound("вложение не найдено") from error - return attachment, content + return content_type, content diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py index 956e4d95..458408e4 100644 --- a/src/messenger/attachments/config.py +++ b/src/messenger/attachments/config.py @@ -1,3 +1,5 @@ +from pathlib import Path + from pydantic import Field, SecretStr from messenger.shared.config import Settings @@ -22,3 +24,22 @@ class AttachmentsSettings(Settings): upload_grace_seconds: int = Field(default=900, ge=60) upload_sweep_interval_seconds: float = Field(default=30.0, gt=0) upload_sweep_batch: int = Field(default=100, ge=1, le=10_000) + + +class PipelineSettings(AttachmentsSettings): + broker_url: SecretStr = Field(min_length=8) + worker_concurrency: int = Field(default=2, ge=1, le=32) + job_max_attempts: int = Field(default=5, ge=1, le=20) + job_timeout_seconds: float = Field(default=60.0, ge=1, le=290) + job_backoff_base_seconds: float = Field(default=2.0, gt=0, le=60) + job_backoff_cap_seconds: float = Field(default=60.0, gt=0, le=3600) + staging_retention_seconds: int = Field(default=86_400, ge=60) + outbox_poll_seconds: float = Field(default=1.0, gt=0, le=10) + outbox_batch: int = Field(default=50, ge=1, le=1000) + sweep_interval_seconds: float = Field(default=5.0, gt=0, le=300) + sweep_batch: int = Field(default=100, ge=1, le=10_000) + publish_timeout_seconds: float = Field(default=5.0, gt=0, le=60) + shutdown_grace_seconds: float = Field(default=5.0, gt=0, le=9) + heartbeat_file: Path | None = None + heartbeat_interval_seconds: float = Field(default=5.0, gt=0, le=10) + fault_point: str = "" diff --git a/src/messenger/attachments/domain/errors.py b/src/messenger/attachments/domain/errors.py index bec7ffdf..f69bf779 100644 --- a/src/messenger/attachments/domain/errors.py +++ b/src/messenger/attachments/domain/errors.py @@ -25,5 +25,13 @@ class AttachmentNotFound(AttachmentError): pass +class AttachmentNotReady(AttachmentError): + pass + + class UploadAborted(AttachmentError): pass + + +class UnreadableImage(AttachmentError): + pass diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py index f12bbe97..58a7c53b 100644 --- a/src/messenger/attachments/domain/events.py +++ b/src/messenger/attachments/domain/events.py @@ -5,12 +5,12 @@ from messenger.shared.domain.events import DomainEvent @dataclass(frozen=True, slots=True) -class AttachmentStored(DomainEvent): +class AttachmentQueued(DomainEvent): attachment_id: UUID chat_id: UUID owner_id: UUID size_bytes: int - name: str = field(init=False, default="attachment.stored") + name: str = field(init=False, default="attachment.queued") @dataclass(frozen=True, slots=True) @@ -19,3 +19,15 @@ class AttachmentDiscarded(DomainEvent): chat_id: UUID reason: str name: str = field(init=False, default="attachment.discarded") + + +@dataclass(frozen=True, slots=True) +class JobSettled(DomainEvent): + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + outcome: str + error_code: str | None = None + delay_seconds: float | None = None + name: str = field(init=False, default="job.settled") diff --git a/src/messenger/attachments/domain/jobs.py b/src/messenger/attachments/domain/jobs.py new file mode 100644 index 00000000..e53d618d --- /dev/null +++ b/src/messenger/attachments/domain/jobs.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass, replace +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from typing import Any +from uuid import UUID + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 + + +class FailureCode(StrEnum): + INVALID_IMAGE = "invalid_image" + PROCESSING_FAILED = "processing_failed" + RETRY_EXHAUSTED = "retry_exhausted" + + +class Outcome(StrEnum): + DONE = "done" + DUPLICATE = "duplicate_noop" + FENCED = "fenced" + SUPERSEDED = "superseded" + RESCHEDULED = "retry_scheduled" + LEASE_EXPIRED = "lease_expired" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class Job: + id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + created_at: datetime + + def payload(self) -> dict[str, Any]: + stamp = self.created_at.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + return { + "schema_version": SCHEMA_VERSION, + "job_id": str(self.id), + "type": JOB_TYPE, + "attachment_id": str(self.attachment_id), + "pipeline_version": self.pipeline_version, + "attempt": self.attempt, + "created_at": stamp, + } + + def retried(self, attempt: int) -> Job: + return replace(self, attempt=attempt) + + +@dataclass(frozen=True, slots=True) +class Lease: + job: Job + token: UUID + + +@dataclass(frozen=True, slots=True) +class Claim: + lease: Lease + source_key: str + staged: bool + + +@dataclass(frozen=True, slots=True) +class RetryPolicy: + max_attempts: int + base_seconds: float + cap_seconds: float + + def delay(self, attempt: int) -> timedelta | None: + if attempt >= self.max_attempts: + return None + return timedelta(seconds=min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1))) diff --git a/src/messenger/attachments/domain/models.py b/src/messenger/attachments/domain/models.py index fef585eb..cc917040 100644 --- a/src/messenger/attachments/domain/models.py +++ b/src/messenger/attachments/domain/models.py @@ -3,16 +3,25 @@ from datetime import datetime from enum import StrEnum from uuid import UUID -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued from messenger.attachments.domain.values import Filename, ImageInfo +from messenger.attachments.domain.variants import Variant from messenger.shared.domain.events import DomainEvent -OBJECT_PREFIX = "attachments/" +STAGING_PREFIX = "staging/" +ORIGINALS_PREFIX = "originals/" class AttachmentStatus(StrEnum): PENDING = "pending" + QUEUED = "queued" + PROCESSING = "processing" READY = "ready" + FAILED = "failed" + + +def original_key(attachment_id: UUID) -> str: + return f"{ORIGINALS_PREFIX}{attachment_id}" @dataclass(slots=True) @@ -24,10 +33,14 @@ class Attachment: image: ImageInfo size_bytes: int sha256: str - object_key: str status: AttachmentStatus created_at: datetime updated_at: datetime + staging_key: str | None = None + original_key: str | None = None + published_version: int = 0 + error_code: str | None = None + variants: tuple[Variant, ...] = () _events: list[DomainEvent] = field(init=False, default_factory=list, repr=False, compare=False) @classmethod @@ -51,21 +64,28 @@ class Attachment: image=image, size_bytes=size_bytes, sha256=sha256, - object_key=f"{OBJECT_PREFIX}{attachment_id}", status=AttachmentStatus.PENDING, created_at=now, updated_at=now, + staging_key=f"{STAGING_PREFIX}{attachment_id}", ) @property def content_type(self) -> str: return self.image.kind.value - def store(self, now: datetime) -> None: - self.status = AttachmentStatus.READY + @property + def ready(self) -> bool: + return self.status is AttachmentStatus.READY + + def variant(self, name: str) -> Variant | None: + return next((variant for variant in self.variants if variant.name == name), None) + + def enqueue(self, now: datetime) -> None: + self.status = AttachmentStatus.QUEUED self.updated_at = max(now, self.created_at) self._events.append( - AttachmentStored( + AttachmentQueued( attachment_id=self.id, chat_id=self.chat_id, owner_id=self.owner_id, diff --git a/src/messenger/attachments/domain/variants.py b/src/messenger/attachments/domain/variants.py new file mode 100644 index 00000000..b435f9b4 --- /dev/null +++ b/src/messenger/attachments/domain/variants.py @@ -0,0 +1,31 @@ +from dataclasses import dataclass +from uuid import UUID + +THUMBNAIL = "thumbnail" +THUMBNAIL_BOX = 256 +VARIANTS_PREFIX = "variants/" + + +@dataclass(frozen=True, slots=True) +class Variant: + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + object_key: str + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + if max(width, height) <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def variant_key( + attachment_id: UUID, pipeline_version: int, name: str, sha256: str, extension: str +) -> str: + return f"{VARIANTS_PREFIX}{attachment_id}/{pipeline_version}/{name}-{sha256[:16]}.{extension}" diff --git a/src/messenger/attachments/entrypoints/errors.py b/src/messenger/attachments/entrypoints/errors.py index 56e3e8e0..374038cb 100644 --- a/src/messenger/attachments/entrypoints/errors.py +++ b/src/messenger/attachments/entrypoints/errors.py @@ -1,5 +1,6 @@ from messenger.attachments.domain.errors import ( AttachmentNotFound, + AttachmentNotReady, ChatNotFound, InvalidFilename, PayloadTooLarge, @@ -13,6 +14,7 @@ ERROR_CODES: dict[type[DomainError], tuple[str, int]] = { UploadAborted: ("invalid_request", 400), ChatNotFound: ("not_found", 404), AttachmentNotFound: ("not_found", 404), + AttachmentNotReady: ("conflict", 409), PayloadTooLarge: ("payload_too_large", 413), UnsupportedImage: ("unsupported_media_type", 415), } diff --git a/src/messenger/attachments/entrypoints/heartbeat.py b/src/messenger/attachments/entrypoints/heartbeat.py new file mode 100644 index 00000000..707ce11d --- /dev/null +++ b/src/messenger/attachments/entrypoints/heartbeat.py @@ -0,0 +1,44 @@ +import asyncio +from collections.abc import Awaitable, Callable +from contextlib import suppress +from pathlib import Path + +from messenger.shared.application.ports import Logger + + +class Heartbeat: + __slots__ = ("_interval", "_logger", "_path", "_probe", "_task") + + def __init__( + self, + path: Path | None, + probe: Callable[[], Awaitable[bool]], + interval: float, + logger: Logger, + ) -> None: + self._path = path + self._probe = probe + self._interval = interval + self._logger = logger + self._task: asyncio.Task[None] | None = None + + def start(self) -> None: + if self._path is not None: + self._task = asyncio.create_task(self._run(self._path)) + + async def stop(self) -> None: + if self._task is None: + return + self._task.cancel() + with suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _run(self, path: Path) -> None: + while True: + try: + if await self._probe(): + await asyncio.to_thread(path.touch) + except Exception as error: + self._logger.warning("heartbeat.failed", error=type(error).__name__) + await asyncio.sleep(self._interval) diff --git a/src/messenger/attachments/entrypoints/http.py b/src/messenger/attachments/entrypoints/http.py index ac31decd..9899fecd 100644 --- a/src/messenger/attachments/entrypoints/http.py +++ b/src/messenger/attachments/entrypoints/http.py @@ -1,6 +1,6 @@ import re from collections.abc import AsyncIterator -from typing import Annotated, Literal +from typing import Annotated from uuid import UUID from dishka.integrations.fastapi import FromDishka, inject @@ -12,7 +12,7 @@ from messenger.attachments.application.bus import AttachmentsCommandBus from messenger.attachments.application.commands import UploadAttachment from messenger.attachments.application.queries import GetAttachment, OpenAttachment from messenger.attachments.domain.errors import UploadAborted -from messenger.attachments.entrypoints.schemas import AttachmentOut +from messenger.attachments.entrypoints.schemas import AttachmentOut, VariantName from messenger.shared.entrypoints.actor import Actor UPLOAD_PATH = re.compile(r"/api/v1/chats/[^/]+/attachments/?") @@ -30,7 +30,7 @@ async def body_of(request: Request) -> AsyncIterator[bytes]: raise UploadAborted("клиент оборвал загрузку до конца тела") from error -@uploads_router.post("", status_code=status.HTTP_201_CREATED) +@uploads_router.post("", status_code=status.HTTP_202_ACCEPTED) @inject async def upload_attachment( chat_id: UUID, @@ -67,11 +67,11 @@ async def download_attachment( attachment_id: UUID, actor: Actor, query: FromDishka[OpenAttachment], - _variant: Annotated[Literal["original"], Query(alias="variant")] = "original", + variant: Annotated[VariantName, Query()] = "original", ) -> StreamingResponse: - attachment, content = await query(attachment_id, actor) + content_type, content = await query(attachment_id, actor, variant) return StreamingResponse( content.chunks, - media_type=attachment.content_type, + media_type=content_type, headers={**CONTENT_HEADERS, "Content-Length": str(content.size)}, ) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index a7ca5464..cd97fb07 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -1,28 +1,50 @@ from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import timedelta from dishka import Provider, Scope, provide +from faststream.rabbit import RabbitBroker from messenger.attachments.application.bus import AttachmentsCommandBus -from messenger.attachments.application.commands import SweepAbandonedUploads, UploadAttachment +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, + SweepAbandonedUploads, + UploadAttachment, +) from messenger.attachments.application.handlers import ( SweepAbandonedUploadsHandler, UploadAttachmentHandler, UploadSlots, ) +from messenger.attachments.application.pipeline import ( + CleanStagingHandler, + ProcessJobHandler, + PublishOutboxHandler, + RecoverExpiredLeasesHandler, +) from messenger.attachments.application.ports import ( AttachmentDirectory, AttachmentsUnitOfWork, ChatMembership, + Faults, ImageInspector, + JobPublisher, ObjectStorage, + VariantRenderer, ) from messenger.attachments.application.queries import GetAttachment, OpenAttachment -from messenger.attachments.config import AttachmentsSettings -from messenger.attachments.infrastructure.images import PillowImageInspector +from messenger.attachments.config import AttachmentsSettings, PipelineSettings +from messenger.attachments.domain.jobs import RetryPolicy +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.attachments.infrastructure.images import PillowImageInspector, PillowVariantRenderer from messenger.attachments.infrastructure.postgres import ( PostgresAttachmentDirectory, PostgresAttachmentsUnitOfWork, ) +from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client from messenger.shared.application.commands import Command from messenger.shared.application.messagebus import CommandHandler @@ -30,6 +52,20 @@ from messenger.shared.application.ports import Clock, EventPublisher, IdGenerato from messenger.shared.infrastructure.postgres import Database +@asynccontextmanager +async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]: + async with s3_client( + endpoint=settings.s3_endpoint, + region=settings.s3_region, + access_key_id=settings.s3_access_key_id.get_secret_value(), + secret_access_key=settings.s3_secret_access_key.get_secret_value(), + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + max_attempts=settings.s3_max_attempts, + ) as client: + yield S3ObjectStorage(client, settings.s3_bucket) + + class AttachmentsProvider(Provider): def __init__(self, settings: AttachmentsSettings) -> None: super().__init__() @@ -41,16 +77,8 @@ class AttachmentsProvider(Provider): @provide(scope=Scope.APP) async def storage(self, settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]: - async with s3_client( - endpoint=settings.s3_endpoint, - region=settings.s3_region, - access_key_id=settings.s3_access_key_id.get_secret_value(), - secret_access_key=settings.s3_secret_access_key.get_secret_value(), - connect_timeout=settings.s3_connect_timeout_seconds, - read_timeout=settings.s3_read_timeout_seconds, - max_attempts=settings.s3_max_attempts, - ) as client: - yield S3ObjectStorage(client, settings.s3_bucket) + async with object_storage(settings) as storage: + yield storage @provide(scope=Scope.APP) def images(self, settings: AttachmentsSettings) -> ImageInspector: @@ -110,3 +138,92 @@ class AttachmentsProvider(Provider): SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock), } return AttachmentsCommandBus(handlers, uow, publisher) + + +class PipelineProvider(Provider): + def __init__(self, settings: PipelineSettings) -> None: + super().__init__() + self._settings = settings + + @provide(scope=Scope.APP) + def settings(self) -> PipelineSettings: + return self._settings + + @provide(scope=Scope.APP) + async def storage(self, settings: PipelineSettings) -> AsyncIterator[ObjectStorage]: + async with object_storage(settings) as storage: + yield storage + + @provide(scope=Scope.APP) + def policy(self, settings: PipelineSettings) -> RetryPolicy: + return RetryPolicy( + settings.job_max_attempts, + settings.job_backoff_base_seconds, + settings.job_backoff_cap_seconds, + ) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(database) + + +class WorkerProvider(Provider): + @provide(scope=Scope.APP) + def renderer(self, settings: PipelineSettings) -> VariantRenderer: + return PillowVariantRenderer(settings.max_image_pixels, settings.worker_concurrency) + + @provide(scope=Scope.APP) + def faults(self, settings: PipelineSettings, logger: Logger) -> Faults: + return EnvironmentFaults(settings.fault_point, logger) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + renderer: VariantRenderer, + faults: Faults, + publisher: EventPublisher, + clock: Clock, + ids: IdGenerator, + policy: RetryPolicy, + settings: PipelineSettings, + ) -> AttachmentsCommandBus: + process = ProcessJobHandler( + uow, + storage, + renderer, + faults, + clock=clock, + ids=ids, + policy=policy, + lease=timedelta(seconds=settings.job_timeout_seconds), + ) + return AttachmentsCommandBus({ProcessJob: process}, uow, publisher) + + +class RelayProvider(Provider): + def __init__(self, broker: RabbitBroker) -> None: + super().__init__() + self._broker = broker + + @provide(scope=Scope.APP) + def publisher(self, settings: PipelineSettings) -> JobPublisher: + return RabbitJobPublisher(self._broker, settings.publish_timeout_seconds) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, + uow: AttachmentsUnitOfWork, + storage: ObjectStorage, + jobs: JobPublisher, + publisher: EventPublisher, + clock: Clock, + policy: RetryPolicy, + ) -> AttachmentsCommandBus: + handlers: dict[type[Command], CommandHandler] = { + PublishOutbox: PublishOutboxHandler(uow, jobs), + RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, clock, policy), + CleanStaging: CleanStagingHandler(uow, storage), + } + return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py index 55ba5e5a..9b771e8a 100644 --- a/src/messenger/attachments/entrypoints/journal.py +++ b/src/messenger/attachments/entrypoints/journal.py @@ -1,7 +1,22 @@ -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued, JobSettled +from messenger.shared.application.messagebus import EventBus from messenger.shared.application.ports import Logger -LOG_FIELDS = frozenset({"attachment_id", "chat_id", "actor_id", "size_bytes", "count"}) +LOG_FIELDS = frozenset( + { + "attachment_id", + "chat_id", + "actor_id", + "size_bytes", + "count", + "job_id", + "attempt", + "pipeline_version", + "outcome", + "error_code", + "delay_seconds", + } +) class AttachmentsJournal: @@ -10,9 +25,9 @@ class AttachmentsJournal: def __init__(self, logger: Logger) -> None: self._logger = logger - async def on_attachment_stored(self, event: AttachmentStored) -> None: + async def on_attachment_queued(self, event: AttachmentQueued) -> None: self._logger.info( - "audit.attachment_stored", + "audit.attachment_queued", attachment_id=str(event.attachment_id), chat_id=str(event.chat_id), actor_id=str(event.owner_id), @@ -26,3 +41,22 @@ class AttachmentsJournal: chat_id=str(event.chat_id), reason=event.reason, ) + + async def on_job_settled(self, event: JobSettled) -> None: + self._logger.info( + "audit.job_settled", + job_id=str(event.job_id), + attachment_id=str(event.attachment_id), + pipeline_version=event.pipeline_version, + attempt=event.attempt, + outcome=event.outcome, + error_code=event.error_code, + delay_seconds=event.delay_seconds, + ) + + +def subscribe_journal(bus: EventBus, logger: Logger) -> None: + journal = AttachmentsJournal(logger) + bus.subscribe(AttachmentQueued, journal.on_attachment_queued) + bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded) + bus.subscribe(JobSettled, journal.on_job_settled) diff --git a/src/messenger/attachments/entrypoints/relay.py b/src/messenger/attachments/entrypoints/relay.py new file mode 100644 index 00000000..fad85f7f --- /dev/null +++ b/src/messenger/attachments/entrypoints/relay.py @@ -0,0 +1,67 @@ +import asyncio +from collections.abc import Sequence +from contextlib import suppress + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ( + CleanStaging, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.config import PipelineSettings +from messenger.shared.application.commands import Command +from messenger.shared.application.ports import Logger + + +class Relay: + __slots__ = ("_container", "_logger", "_settings", "_tasks") + + def __init__( + self, container: AsyncContainer, settings: PipelineSettings, logger: Logger + ) -> None: + self._container = container + self._settings = settings + self._logger = logger + self._tasks: list[asyncio.Task[None]] = [] + + def start(self) -> None: + settings = self._settings + loops: tuple[tuple[float, Sequence[Command]], ...] = ( + (settings.outbox_poll_seconds, (PublishOutbox(settings.outbox_batch),)), + ( + settings.sweep_interval_seconds, + (RecoverExpiredLeases(settings.sweep_batch), CleanStaging(settings.sweep_batch)), + ), + ) + self._tasks = [asyncio.create_task(self._every(*loop)) for loop in loops] + + async def stop(self) -> None: + for task in self._tasks: + task.cancel() + for task in self._tasks: + with suppress(asyncio.CancelledError): + await task + self._tasks = [] + + async def run(self, command: Command) -> int: + async with self._container() as scope: + bus = await scope.get(AttachmentsCommandBus) + return await bus.execute(command) + + async def _every(self, interval: float, commands: Sequence[Command]) -> None: + while True: + for command in commands: + try: + done = await self.run(command) + except Exception as error: + self._logger.warning( + "relay.pass_failed", + reason=type(command).__name__, + error=type(error).__name__, + ) + continue + if done: + self._logger.info("relay.pass", reason=type(command).__name__, count=done) + await asyncio.sleep(interval) diff --git a/src/messenger/attachments/entrypoints/schemas.py b/src/messenger/attachments/entrypoints/schemas.py index 86daa6cd..f482d4a5 100644 --- a/src/messenger/attachments/entrypoints/schemas.py +++ b/src/messenger/attachments/entrypoints/schemas.py @@ -1,10 +1,34 @@ from datetime import datetime -from typing import Literal +from typing import Annotated, Literal from uuid import UUID -from pydantic import BaseModel +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field +from messenger.attachments.domain.jobs import Job from messenger.attachments.domain.models import Attachment +from messenger.attachments.domain.variants import Variant + +type VariantName = Literal["original", "thumbnail", "crop", "optimized"] + + +class VariantOut(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + @classmethod + def of(cls, variant: Variant) -> VariantOut: + return cls( + name=variant.name, + content_type=variant.content_type, + size_bytes=variant.size_bytes, + width=variant.width, + height=variant.height, + sha256=variant.sha256, + ) class AttachmentOut(BaseModel): @@ -15,9 +39,9 @@ class AttachmentOut(BaseModel): content_type: str size_bytes: int sha256: str - status: Literal["ready"] = "ready" - variants: tuple[()] = () - error_code: None = None + status: str + variants: list[VariantOut] + error_code: str | None created_at: datetime updated_at: datetime @@ -31,6 +55,30 @@ class AttachmentOut(BaseModel): content_type=attachment.content_type, size_bytes=attachment.size_bytes, sha256=attachment.sha256, + status=attachment.status.value, + variants=[VariantOut.of(variant) for variant in attachment.variants], + error_code=attachment.error_code, created_at=attachment.created_at, updated_at=attachment.updated_at, ) + + +class JobMessage(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal[1] + job_id: UUID + type: Literal["attachment.process.v1"] + attachment_id: UUID + pipeline_version: Annotated[int, Field(ge=1)] + attempt: Annotated[int, Field(ge=1)] + created_at: AwareDatetime + traceparent: Annotated[ + str | None, Field(pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + ] = None + tracestate: Annotated[str | None, Field(max_length=512)] = None + + def job(self) -> Job: + return Job( + self.job_id, self.attachment_id, self.pipeline_version, self.attempt, self.created_at + ) diff --git a/src/messenger/attachments/entrypoints/worker.py b/src/messenger/attachments/entrypoints/worker.py new file mode 100644 index 00000000..8de721bd --- /dev/null +++ b/src/messenger/attachments/entrypoints/worker.py @@ -0,0 +1,50 @@ +import asyncio + +from dishka import FromDishka +from faststream import AckPolicy +from faststream.rabbit import Channel, QueueType, RabbitBroker, RabbitQueue +from faststream.rabbit.annotations import RabbitMessage +from pydantic import ValidationError + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.rabbitmq import QUEUE +from messenger.shared.application.ports import Logger + +REQUEUE_DELAY_SECONDS = 1.0 +JOBS = RabbitQueue(QUEUE, queue_type=QueueType.QUORUM, declare=False) + + +async def raw(message: RabbitMessage) -> bytes: + return message.body + + +async def handle_job( + message: RabbitMessage, + bus: FromDishka[AttachmentsCommandBus], + logger: FromDishka[Logger], +) -> None: + try: + job = JobMessage.model_validate_json(message.body).job() + except ValidationError: + logger.warning("job.rejected", reason="message does not match jobs.schema.json") + await message.reject(requeue=False) + return + try: + await bus.execute(ProcessJob(job)) + except Exception as error: + logger.warning("job.requeued", job_id=str(job.id), error=type(error).__name__) + await asyncio.sleep(REQUEUE_DELAY_SECONDS) + await message.nack(requeue=True) + return + await message.ack() + + +def subscribe(broker: RabbitBroker, concurrency: int) -> None: + broker.subscriber( + JOBS, + channel=Channel(prefetch_count=concurrency), + ack_policy=AckPolicy.MANUAL, + decoder=raw, + )(handle_job) diff --git a/src/messenger/attachments/infrastructure/faults.py b/src/messenger/attachments/infrastructure/faults.py new file mode 100644 index 00000000..814d3ce1 --- /dev/null +++ b/src/messenger/attachments/infrastructure/faults.py @@ -0,0 +1,26 @@ +import os +import sys + +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +TRANSIENT = frozenset({"transient_error"}) +CRASH_EXIT_CODE = 137 + + +class EnvironmentFaults: + __slots__ = ("_logger", "_point") + + def __init__(self, point: str, logger: Logger) -> None: + self._point = point + self._logger = logger + + def reach(self, point: str) -> None: + if not self._point or point != self._point: + return + self._logger.warning("fault.injected", reason=point) + if point in TRANSIENT: + raise Unavailable("внедрённая временная ошибка") + sys.stdout.flush() + sys.stderr.flush() + os._exit(CRASH_EXIT_CODE) diff --git a/src/messenger/attachments/infrastructure/images.py b/src/messenger/attachments/infrastructure/images.py index 6d63f670..e7cf1213 100644 --- a/src/messenger/attachments/infrastructure/images.py +++ b/src/messenger/attachments/infrastructure/images.py @@ -2,14 +2,19 @@ import asyncio import warnings from io import BytesIO -from PIL import Image +from PIL import Image, ImageOps -from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.errors import PayloadTooLarge, UnreadableImage, UnsupportedImage from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.domain.variants import THUMBNAIL, THUMBNAIL_BOX, fit KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG} BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning) JPEG_SCALE = 8 +WIDE_MODES = frozenset({"I", "I;16", "I;16B", "I;16L", "I;16N"}) +ALPHA_MODES = frozenset({"LA", "La", "PA", "RGBa"}) +JPEG_QUALITY = 85 class PillowImageInspector: @@ -47,3 +52,51 @@ class PillowImageInspector: except Exception as error: raise UnsupportedImage("изображение повреждено или обрезано") from error return ImageInfo(kind, width, height) + + +def flatten(image: Image.Image) -> Image.Image: + if image.mode in WIDE_MODES: + image = image.convert("I").point(lambda value: value / 256, "L") + if image.mode == "P": + image = image.convert("RGBA" if "transparency" in image.info else "RGB") + if image.mode in ALPHA_MODES: + image = image.convert("RGBA") + if image.mode == "RGBA": + background = Image.new("RGB", image.size, "white") + background.paste(image, mask=image.getchannel("A")) + return background + return image if image.mode == "RGB" else image.convert("RGB") + + +def jpeg(name: str, image: Image.Image, box: int) -> Rendition: + size = fit(image.width, image.height, box) + scaled = image if size == image.size else image.resize(size, Image.Resampling.LANCZOS) + buffer = BytesIO() + scaled.save(buffer, "JPEG", quality=JPEG_QUALITY, optimize=True) + return Rendition(name, ImageKind.JPEG.value, "jpg", buffer.getvalue(), *size) + + +def render_variants(data: bytes, max_pixels: int) -> list[Rendition]: + try: + with Image.open(BytesIO(data), formats=tuple(KINDS)) as image: + if image.width * image.height > max_pixels: + raise UnreadableImage("изображение больше допустимого числа пикселей") + image.load() + upright = flatten(ImageOps.exif_transpose(image)) + except UnreadableImage: + raise + except Exception as error: + raise UnreadableImage("изображение не декодируется") from error + return [jpeg(THUMBNAIL, upright, THUMBNAIL_BOX)] + + +class PillowVariantRenderer: + __slots__ = ("_max_pixels", "_slots") + + def __init__(self, max_pixels: int, concurrency: int) -> None: + self._max_pixels = max_pixels + self._slots = asyncio.Semaphore(concurrency) + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + async with self._slots: + return await asyncio.to_thread(render_variants, data, self._max_pixels) diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index c47d0f18..20a9ad59 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -1,11 +1,16 @@ -from datetime import datetime +import json +from collections.abc import Sequence +from datetime import datetime, timedelta from uuid import UUID from asyncpg import ForeignKeyViolationError, Record +from messenger.attachments.application.ports import OutboxEntry, StagedObject from messenger.attachments.domain.errors import ChatNotFound +from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind +from messenger.attachments.domain.variants import Variant from messenger.shared.application.errors import Unavailable from messenger.shared.infrastructure.postgres import ( Connection, @@ -16,26 +21,151 @@ from messenger.shared.infrastructure.postgres import ( INSERT = """ INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - width, height, object_key, status, created_at, updated_at) + width, height, staging_key, status, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) """ MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'" COLUMNS = """ -SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, - object_key, status, created_at, updated_at -FROM attachments +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, + a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, + a.created_at, a.updated_at +FROM attachments a """ -READY = COLUMNS + "WHERE id = $1 AND status = 'ready'" -LOCK = COLUMNS + "WHERE id = $1 FOR UPDATE" +VISIBLE = """ +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width, + a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code, + a.created_at, a.updated_at, + (SELECT coalesce(jsonb_agg(jsonb_build_array(v.name, v.content_type, v.size_bytes, v.width, + v.height, v.sha256, v.object_key) + ORDER BY v.name), '[]') + FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.published_version) AS variants +FROM attachments a +WHERE a.id = $1 AND a.status <> 'pending' +""" +FAILED_AMONG = """ +SELECT EXISTS (SELECT 1 FROM attachments + WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'failed') +""" +LOCK = COLUMNS + "WHERE a.id = $1 FOR UPDATE" ABANDONED = ( COLUMNS - + """WHERE status = 'pending' AND created_at < $1 -ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" + + """WHERE a.status = 'pending' AND a.created_at < $1 +ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" ) REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" +STAGED = """ +SELECT id, staging_key FROM attachments +WHERE staging_key IS NOT NULL AND status = 'ready' +ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED +""" +UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])" + +ADD_JOB = """ +WITH job AS ( + INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, attempt, + created_at, updated_at) + VALUES ($1, $2, $3, 'queued', $4, $5, $5) + RETURNING job_id +), box AS ( + INSERT INTO outbox (job_id, payload, available_at) + SELECT job_id, $6::jsonb, now() FROM job +) +INSERT INTO job_events (job_id, attachment_id, kind, attempt) +SELECT job_id, $2, 'enqueued', $4 FROM job +""" +CLAIM = """ +WITH claimed AS ( + UPDATE jobs + SET status = 'processing', lease_token = $4, claims = claims + 1, updated_at = now(), + lease_until = now() + make_interval(secs => $5) + WHERE job_id = $1 AND attachment_id = $2 AND pipeline_version = $3 + AND ((status = 'queued' AND coalesce(next_attempt_at, '-infinity') <= now()) + OR (status = 'processing' AND lease_until < now())) + RETURNING job_id, attachment_id, attempt, created_at +), moved AS ( + UPDATE attachments a SET status = 'processing', updated_at = greatest(now(), a.created_at) + FROM claimed c WHERE a.id = c.attachment_id AND a.status = 'queued' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt) + SELECT job_id, attachment_id, 'claimed', attempt FROM claimed +) +SELECT c.attempt, c.created_at, a.staging_key, a.original_key +FROM claimed c JOIN attachments a ON a.id = c.attachment_id +""" +FINISH = """ +UPDATE jobs SET status = 'done', lease_token = NULL, lease_until = NULL, next_attempt_at = NULL, + last_error = NULL, updated_at = now() +WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 +""" +ADD_VARIANTS = """ +INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, + size_bytes, width, height, sha256, created_at) +SELECT $1, $2, v.name, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256, + now() +FROM unnest($3::text[], $4::text[], $5::text[], $6::bigint[], $7::int[], $8::int[], $9::text[]) + AS v (name, object_key, content_type, size_bytes, width, height, sha256) +ON CONFLICT DO NOTHING +""" +PUBLISH = """ +UPDATE attachments +SET status = 'ready', published_version = $2, error_code = NULL, original_key = $3, + updated_at = greatest(now(), created_at) +WHERE id = $1 AND published_version < $2 +""" +SUPERSEDE = "UPDATE jobs SET status = 'superseded', updated_at = now() WHERE job_id = $1" +RESCHEDULE = """ +WITH job AS ( + UPDATE jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL, + next_attempt_at = $3, last_error = $4, updated_at = now() + WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 + RETURNING job_id, attachment_id, attempt +), moved AS ( + UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status = 'processing' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, $5, attempt, + jsonb_build_object('error', $4::text, 'next_attempt_at', $3::timestamptz) + FROM job +) +SELECT attempt FROM job +""" +ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)" +FAIL = """ +WITH job AS ( + UPDATE jobs SET status = 'failed', lease_token = NULL, lease_until = NULL, last_error = $4, + updated_at = now() + WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 + RETURNING job_id, attachment_id, attempt +), moved AS ( + UPDATE attachments a SET status = 'failed', error_code = $3, + updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status IN ('queued', 'processing') +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, 'failed', attempt, + jsonb_build_object('error_code', $3::text, 'error', $4::text) + FROM job +) +SELECT attempt FROM job +""" +NOTE = "INSERT INTO job_events (job_id, attachment_id, kind, attempt) VALUES ($1, $2, $3, $4)" +EXPIRED = """ +SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token FROM jobs +WHERE status = 'processing' AND lease_until < now() +ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED +""" +PENDING = """ +SELECT id, job_id, payload::text AS body FROM outbox +WHERE published_at IS NULL AND available_at <= now() +ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED +""" +PUBLISHED = "UPDATE outbox SET published_at = now() WHERE id = ANY($1::bigint[])" -def attachment_of(row: Record) -> Attachment: +def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment: return Attachment( id=row["id"], chat_id=row["chat_id"], @@ -44,13 +174,21 @@ def attachment_of(row: Record) -> Attachment: image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]), size_bytes=row["size_bytes"], sha256=row["sha256"], - object_key=row["object_key"], status=AttachmentStatus(row["status"]), created_at=row["created_at"], updated_at=row["updated_at"], + staging_key=row["staging_key"], + original_key=row["original_key"], + published_version=row["published_version"], + error_code=row["error_code"], + variants=variants, ) +def payload(job: Job) -> str: + return json.dumps(job.payload(), separators=(",", ":")) + + class PostgresAttachmentRepository: __slots__ = ("_connection", "_seen") @@ -71,7 +209,7 @@ class PostgresAttachmentRepository: attachment.sha256, attachment.image.width, attachment.image.height, - attachment.object_key, + attachment.staging_key, attachment.status.value, attachment.created_at, attachment.updated_at, @@ -100,6 +238,127 @@ class PostgresAttachmentRepository: await self._connection.execute(REMOVE, attachment.id) self._seen.append(attachment) + async def staged(self, limit: int) -> list[StagedObject]: + rows = await self._connection.fetch(STAGED, limit) + return [StagedObject(row["id"], row["staging_key"]) for row in rows] + + async def unstage(self, attachment_ids: Sequence[UUID]) -> None: + if attachment_ids: + await self._connection.execute(UNSTAGE, list(attachment_ids)) + + +class PostgresJobRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def add(self, job: Job) -> None: + await self._connection.execute( + ADD_JOB, + job.id, + job.attachment_id, + job.pipeline_version, + job.attempt, + job.created_at, + payload(job), + ) + + async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None: + row = await self._connection.fetchrow( + CLAIM, + job.id, + job.attachment_id, + job.pipeline_version, + token, + lease.total_seconds(), + ) + if row is None: + return None + claimed = Job( + job.id, job.attachment_id, job.pipeline_version, row["attempt"], row["created_at"] + ) + staged = row["staging_key"] is not None + source = row["staging_key"] if staged else row["original_key"] + return Claim(Lease(claimed, token), source, staged) + + async def complete( + self, lease: Lease, variants: Sequence[Variant], original_key: str + ) -> Outcome: + job = lease.job + if await self._connection.execute(FINISH, job.id, lease.token) == "UPDATE 0": + await self.note(job, Outcome.FENCED) + return Outcome.FENCED + await self._connection.execute( + ADD_VARIANTS, + job.attachment_id, + job.pipeline_version, + [v.name for v in variants], + [v.object_key for v in variants], + [v.content_type for v in variants], + [v.size_bytes for v in variants], + [v.width for v in variants], + [v.height for v in variants], + [v.sha256 for v in variants], + ) + published = await self._connection.execute( + PUBLISH, job.attachment_id, job.pipeline_version, original_key + ) + outcome = Outcome.SUPERSEDED if published == "UPDATE 0" else Outcome.DONE + if outcome is Outcome.SUPERSEDED: + await self._connection.execute(SUPERSEDE, job.id) + await self.note(job, outcome) + return outcome + + async def reschedule(self, lease: Lease, at: datetime, kind: Outcome, error: str) -> Job | None: + attempt = await self._connection.fetchval( + RESCHEDULE, lease.job.id, lease.token, at, error, kind.value + ) + if attempt is None: + return None + job = lease.job.retried(attempt) + await self._connection.execute(ENQUEUE, job.id, payload(job), at) + return job + + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: + attempt = await self._connection.fetchval( + FAIL, lease.job.id, lease.token, code.value, error + ) + return attempt is not None + + async def note(self, job: Job, kind: Outcome) -> None: + await self._connection.execute(NOTE, job.id, job.attachment_id, kind.value, job.attempt) + + async def expired(self, limit: int) -> list[Lease]: + rows = await self._connection.fetch(EXPIRED, limit) + return [ + Lease( + Job( + row["job_id"], + row["attachment_id"], + row["pipeline_version"], + row["attempt"], + row["created_at"], + ), + row["lease_token"], + ) + for row in rows + ] + + +class PostgresOutboxRepository: + __slots__ = ("_connection",) + + def __init__(self, connection: Connection) -> None: + self._connection = connection + + async def pending(self, limit: int) -> list[OutboxEntry]: + rows = await self._connection.fetch(PENDING, limit) + return [OutboxEntry(row["id"], row["job_id"], row["body"].encode()) for row in rows] + + async def published(self, entry_ids: Sequence[int]) -> None: + await self._connection.execute(PUBLISHED, list(entry_ids)) + class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): __slots__ = () @@ -108,6 +367,18 @@ class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork): def attachments(self) -> PostgresAttachmentRepository: return PostgresAttachmentRepository(self.connection, self.seen) + @property + def jobs(self) -> PostgresJobRepository: + return PostgresJobRepository(self.connection) + + @property + def outbox(self) -> PostgresOutboxRepository: + return PostgresOutboxRepository(self.connection) + + +def variants_of(raw: str) -> tuple[Variant, ...]: + return tuple(Variant(*item) for item in json.loads(raw)) + class PostgresAttachmentDirectory: __slots__ = ("_database",) @@ -115,7 +386,11 @@ class PostgresAttachmentDirectory: def __init__(self, database: Database) -> None: self._database = database - async def ready(self, attachment_id: UUID) -> Attachment | None: + async def visible(self, attachment_id: UUID) -> Attachment | None: async with self._database.connection() as connection: - row = await connection.fetchrow(READY, attachment_id) - return None if row is None else attachment_of(row) + row = await connection.fetchrow(VISIBLE, attachment_id) + return None if row is None else attachment_of(row, variants_of(row["variants"])) + + async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: + async with self._database.connection() as connection: + return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id)) diff --git a/src/messenger/attachments/infrastructure/rabbitmq.py b/src/messenger/attachments/infrastructure/rabbitmq.py new file mode 100644 index 00000000..778c1af9 --- /dev/null +++ b/src/messenger/attachments/infrastructure/rabbitmq.py @@ -0,0 +1,37 @@ +from uuid import UUID + +from faststream.rabbit import RabbitBroker, RabbitExchange +from pamqp.commands import Basic + +from messenger.attachments.domain.jobs import JOB_TYPE +from messenger.shared.application.errors import Unavailable + +EXCHANGE = RabbitExchange("attachments", declare=False) +QUEUE = "attachments.process" +DEAD_LETTERS = "attachments.process.dlq" + + +class RabbitJobPublisher: + __slots__ = ("_broker", "_timeout") + + def __init__(self, broker: RabbitBroker, timeout: float) -> None: + self._broker = broker + self._timeout = timeout + + async def publish(self, job_id: UUID, body: bytes) -> None: + try: + confirmation = await self._broker.publish( + body, + exchange=EXCHANGE, + routing_key=JOB_TYPE, + mandatory=True, + persist=True, + message_id=str(job_id), + message_type=JOB_TYPE, + content_type="application/json", + timeout=self._timeout, + ) + except Exception as error: + raise Unavailable("брокер не принял задание") from error + if not isinstance(confirmation, Basic.Ack): + raise Unavailable("брокер не подтвердил задание") diff --git a/src/messenger/attachments/infrastructure/s3.py b/src/messenger/attachments/infrastructure/s3.py index 5d47ecf0..3acea375 100644 --- a/src/messenger/attachments/infrastructure/s3.py +++ b/src/messenger/attachments/infrastructure/s3.py @@ -77,6 +77,21 @@ class S3ObjectStorage: raise Unavailable("S3 недоступно") from error return ObjectContent(size=response["ContentLength"], chunks=self._chunks(response["Body"])) + async def read(self, key: str) -> bytes: + content = await self.open(key) + try: + return b"".join([chunk async for chunk in content.chunks]) + except Exception as error: + raise Unavailable("S3 оборвал чтение объекта") from error + + async def copy(self, source: str, target: str) -> None: + try: + await self._client.copy_object( + Bucket=self._bucket, Key=target, CopySource={"Bucket": self._bucket, "Key": source} + ) + except FAILURES as error: + raise Unavailable("S3 недоступно") from error + async def delete(self, key: str) -> None: try: await self._client.delete_object(Bucket=self._bucket, Key=key) diff --git a/src/messenger/main.py b/src/messenger/main.py index ddaf3c6b..cf02aa5e 100644 --- a/src/messenger/main.py +++ b/src/messenger/main.py @@ -8,12 +8,11 @@ from fastapi import FastAPI from messenger.attachments.application.ports import ChatMembership from messenger.attachments.config import AttachmentsSettings -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored from messenger.attachments.entrypoints.errors import ERROR_CODES as ATTACHMENT_ERRORS from messenger.attachments.entrypoints.http import UPLOAD_PATH, attachments_router, uploads_router from messenger.attachments.entrypoints.ioc import AttachmentsProvider from messenger.attachments.entrypoints.journal import LOG_FIELDS as ATTACHMENT_FIELDS -from messenger.attachments.entrypoints.journal import AttachmentsJournal +from messenger.attachments.entrypoints.journal import subscribe_journal from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.identity.config import IdentitySettings from messenger.identity.domain.events import SessionClosed, SessionOpened, UserRegistered @@ -75,9 +74,7 @@ def wire_events(bus: EventBus, logger: Logger) -> None: bus.subscribe(ChatCreated, messaging.on_chat_created) bus.subscribe(MessagePosted, messaging.on_message_posted) - attachments = AttachmentsJournal(logger) - bus.subscribe(AttachmentStored, attachments.on_attachment_stored) - bus.subscribe(AttachmentDiscarded, attachments.on_attachment_discarded) + subscribe_journal(bus, logger) def create_app( diff --git a/src/messenger/relay.py b/src/messenger/relay.py new file mode 100644 index 00000000..7db49563 --- /dev/null +++ b/src/messenger/relay.py @@ -0,0 +1,66 @@ +import asyncio +import signal + +from dishka import make_async_container +from faststream.rabbit import Channel, RabbitBroker + +from messenger.attachments.config import PipelineSettings +from messenger.attachments.entrypoints.heartbeat import Heartbeat +from messenger.attachments.entrypoints.ioc import PipelineProvider, RelayProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.attachments.entrypoints.relay import Relay +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +PING_TIMEOUT_SECONDS = 2.0 + + +async def serve() -> None: + platform = PlatformSettings() + pipeline = PipelineSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + broker = RabbitBroker( + pipeline.broker_url.get_secret_value(), + default_channel=Channel(publisher_confirms=True, on_return_raises=True), + logger=None, + ) + container = make_async_container( + PlatformProvider(platform, logger), PipelineProvider(pipeline), RelayProvider(broker) + ) + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + await broker.connect() + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + relay = Relay(container, pipeline, logger) + heartbeat = Heartbeat( + pipeline.heartbeat_file, + lambda: broker.ping(PING_TIMEOUT_SECONDS), + pipeline.heartbeat_interval_seconds, + logger, + ) + relay.start() + heartbeat.start() + logger.info("relay.started") + try: + await stop.wait() + finally: + await heartbeat.stop() + await relay.stop() + await bus.stop() + await broker.stop() + await container.close() + logger.info("relay.stopped") + + +def main() -> None: + asyncio.run(serve()) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 00000000..a18902c1 --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,78 @@ +import asyncio +from collections.abc import Sequence + +from dishka import Provider, make_async_container +from dishka_faststream import setup_dishka +from faststream import FastStream +from faststream.rabbit import RabbitBroker + +from messenger.attachments.config import PipelineSettings +from messenger.attachments.entrypoints.heartbeat import Heartbeat +from messenger.attachments.entrypoints.ioc import PipelineProvider, WorkerProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.attachments.entrypoints.worker import subscribe +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + +PING_TIMEOUT_SECONDS = 2.0 + + +def create_worker( + *, + platform: PlatformSettings | None = None, + pipeline: PipelineSettings | None = None, + overrides: Sequence[Provider] = (), +) -> FastStream: + platform = platform or PlatformSettings() + pipeline = pipeline or PipelineSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + broker = RabbitBroker( + pipeline.broker_url.get_secret_value(), + graceful_timeout=pipeline.shutdown_grace_seconds, + logger=None, + ) + subscribe(broker, pipeline.worker_concurrency) + container = make_async_container( + PlatformProvider(platform, logger), + PipelineProvider(pipeline), + WorkerProvider(), + *overrides, + ) + heartbeat = Heartbeat( + pipeline.heartbeat_file, + lambda: broker.ping(PING_TIMEOUT_SECONDS), + pipeline.heartbeat_interval_seconds, + logger, + ) + app = FastStream(broker, logger=None) + + @app.on_startup + async def start_bus() -> None: + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + + @app.after_startup + async def started() -> None: + heartbeat.start() + logger.info("worker.started", limit=pipeline.worker_concurrency) + + @app.after_shutdown + async def stopped() -> None: + await heartbeat.stop() + await (await container.get(EventBus)).stop() + await container.close() + logger.info("worker.stopped") + + setup_dishka(container, app, auto_inject=True) + return app + + +def main() -> None: + asyncio.run(create_worker().run()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/adapters/test_rabbitmq.py b/student-tests/adapters/test_rabbitmq.py new file mode 100644 index 00000000..6f800555 --- /dev/null +++ b/student-tests/adapters/test_rabbitmq.py @@ -0,0 +1,259 @@ +import asyncio +import base64 +import hashlib +import json +import os +import re +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +import pytest +from dishka import Provider, Scope, make_async_container, provide +from dishka_faststream import setup_dishka +from faststream.rabbit import Channel, RabbitBroker + +from conftest import ROOT, STARTUP_SECONDS, container, image, published +from fakes import NullLogger +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ProcessJob +from messenger.attachments.domain.jobs import Job, Outcome +from messenger.attachments.entrypoints.worker import subscribe +from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher +from messenger.shared.application.errors import Unavailable +from messenger.shared.application.ports import Logger + +RABBITMQ_IMAGE = re.compile(r"image: (rabbitmq:\S+)") +PASSWORDS = {role: os.urandom(12).hex() for role in ("api", "worker", "ops")} +GRANTS = { + "api": ("^$", "^attachments$", "^$"), + "worker": ("^$", "^$", "^attachments\\.process$"), + "ops": (".*", ".*", ".*"), +} + + +def user(role: str) -> dict[str, Any]: + salt = os.urandom(4) + digest = hashlib.sha256(salt + PASSWORDS[role].encode()).digest() + return { + "name": f"messenger-{role}", + "password_hash": base64.b64encode(salt + digest).decode(), + "hashing_algorithm": "rabbit_password_hashing_sha256", + "tags": ["administrator"] if role == "ops" else [], + } + + +def definitions() -> dict[str, Any]: + return { + "users": [user(role) for role in PASSWORDS], + "permissions": [ + { + "user": f"messenger-{role}", + "vhost": "messenger", + **dict(zip(("configure", "write", "read"), grant, strict=True)), + } + for role, grant in GRANTS.items() + ], + } + + +def ready(name: str) -> bool: + probe = ["docker", "exec", name, "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + return subprocess.run(probe, capture_output=True, check=False).returncode == 0 + + +@pytest.fixture(scope="session") +def rabbitmq() -> Iterator[dict[str, Any]]: + name = f"mtusi-lab5-test-rabbitmq-{uuid4().hex[:8]}" + folder = ROOT / "build" / name + folder.mkdir(parents=True) + users = folder / "20-users.json" + users.write_text(json.dumps(definitions())) + users.chmod(0o644) + infra = ROOT / "infra" / "rabbitmq" + mounts = [ + f"{infra / 'rabbitmq.conf'}:/etc/rabbitmq/conf.d/20-messenger.conf:ro", + f"{infra / 'enabled_plugins'}:/etc/rabbitmq/enabled_plugins:ro", + f"{infra / 'definitions.json'}:/etc/rabbitmq/definitions.d/10-topology.json:ro", + f"{users}:/etc/rabbitmq/definitions.d/20-users.json:ro", + ] + volumes = [part for mount in mounts for part in ("-v", mount)] + container( + name, + "--hostname", + "broker", + "--user", + "100:101", + *volumes, + "-p", + "127.0.0.1::5672", + image(RABBITMQ_IMAGE), + ) + try: + deadline = time.monotonic() + STARTUP_SECONDS * 2 + while not ready(name): + assert time.monotonic() < deadline, "RabbitMQ не стал готов" + time.sleep(1) + port = published(name, 5672) + urls = { + role: f"amqp://messenger-{role}:{secret}@127.0.0.1:{port}/messenger" + for role, secret in PASSWORDS.items() + } + yield {"name": name, "urls": urls} + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + users.unlink() + folder.rmdir() + + +def depth(broker: dict[str, Any], queue: str) -> int: + listing = subprocess.run( + [ + "docker", + "exec", + broker["name"], + "rabbitmqctl", + "-q", + "list_queues", + "-p", + "messenger", + "name", + "messages", + "--formatter", + "json", + ], + capture_output=True, + check=True, + text=True, + ).stdout + return {row["name"]: row["messages"] for row in json.loads(listing)}[queue] + + +async def wait_for(probe: Any, seconds: float = 20) -> None: + deadline = time.monotonic() + seconds + while not probe(): + assert time.monotonic() < deadline + await asyncio.sleep(0.1) + + +class NoEvents: + @asynccontextmanager + async def transaction(self) -> AsyncIterator[None]: + yield + + async def commit(self) -> None: ... + + def record(self, event: object) -> None: ... + + def collect_new_events(self) -> list[Any]: + return [] + + +class Recorder: + def __init__(self) -> None: + self.jobs: list[Job] = [] + self.failures = 0 + + async def __call__(self, command: ProcessJob) -> Outcome: + if self.failures: + self.failures -= 1 + raise Unavailable("PostgreSQL недоступен") + self.jobs.append(command.job) + return Outcome.DONE + + +class FakeBus(Provider): + def __init__(self, recorder: Recorder) -> None: + super().__init__() + self._recorder = recorder + + @provide(scope=Scope.APP) + def logger(self) -> Logger: + return NullLogger() + + @provide(scope=Scope.REQUEST) + def bus(self) -> AttachmentsCommandBus: + return AttachmentsCommandBus({ProcessJob: self._recorder}, NoEvents(), NoPublisher()) + + +class NoPublisher: + async def publish(self, events: Any) -> None: ... + + +@asynccontextmanager +async def worker(rabbitmq: dict[str, Any], recorder: Recorder) -> AsyncIterator[None]: + broker = RabbitBroker(rabbitmq["urls"]["worker"], logger=None, graceful_timeout=2) + subscribe(broker, 2) + setup_dishka(make_async_container(FakeBus(recorder)), broker=broker, auto_inject=True) + await broker.start() + try: + yield + finally: + await broker.stop() + + +@asynccontextmanager +async def publisher( + rabbitmq: dict[str, Any], role: str = "api" +) -> AsyncIterator[RabbitJobPublisher]: + broker = RabbitBroker( + rabbitmq["urls"][role], + default_channel=Channel(publisher_confirms=True, on_return_raises=True), + logger=None, + ) + await broker.connect() + try: + yield RabbitJobPublisher(broker, 5) + finally: + await broker.stop() + + +def job() -> Job: + return Job(uuid4(), uuid4(), 1, 1, datetime.now(UTC)) + + +def body(payload: dict[str, Any]) -> bytes: + return json.dumps(payload).encode() + + +async def test_confirmed_job_is_consumed_and_acked(rabbitmq: dict[str, Any]) -> None: + recorder, sent = Recorder(), job() + async with publisher(rabbitmq) as jobs: + await jobs.publish(sent.id, body(sent.payload())) + await wait_for(lambda: depth(rabbitmq, "attachments.process") == 1) + async with worker(rabbitmq, recorder): + await wait_for(lambda: recorder.jobs) + await wait_for(lambda: depth(rabbitmq, "attachments.process") == 0) + assert recorder.jobs == [sent] + + +async def test_message_off_the_schema_goes_to_the_dead_letter_queue( + rabbitmq: dict[str, Any], +) -> None: + before = depth(rabbitmq, "attachments.process.dlq") + async with publisher(rabbitmq) as jobs: + await jobs.publish(uuid4(), body({"schema_version": 2, "job_id": str(uuid4())})) + async with worker(rabbitmq, Recorder()): + await wait_for(lambda: depth(rabbitmq, "attachments.process.dlq") == before + 1) + assert depth(rabbitmq, "attachments.process") == 0 + + +async def test_failed_processing_is_requeued_and_retried(rabbitmq: dict[str, Any]) -> None: + recorder, sent = Recorder(), job() + recorder.failures = 1 + async with publisher(rabbitmq) as jobs: + await jobs.publish(sent.id, body(sent.payload())) + async with worker(rabbitmq, recorder): + await wait_for(lambda: recorder.jobs) + assert recorder.jobs == [sent] + + +async def test_worker_account_cannot_publish(rabbitmq: dict[str, Any]) -> None: + sent = job() + with pytest.raises(Unavailable): + async with publisher(rabbitmq, "worker") as jobs: + await jobs.publish(sent.id, body(sent.payload())) diff --git a/student-tests/api/test_attachments.py b/student-tests/api/test_attachments.py index 9f4ad41c..30f667bc 100644 --- a/student-tests/api/test_attachments.py +++ b/student-tests/api/test_attachments.py @@ -11,6 +11,7 @@ from conftest import actor, make_chat, register, resolve, scalar from fakes import MemoryObjectStorage from messenger.attachments.application.ports import ObjectStorage from pictures import GIF, JPEG, SAMPLE, WEBP, png +from pipeline import settle LIMIT = 10_485_760 PNG = {"Content-Type": "image/png"} @@ -47,7 +48,7 @@ async def trio(http: AsyncClient) -> tuple[dict[str, Any], ...]: async def test_member_uploads_and_another_member_downloads(client: AsyncClient) -> None: alice, bob, eve, chat = await trio(client) response = await upload(client, alice, chat) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text attachment = response.json() assert set(attachment) == { "id", @@ -70,13 +71,21 @@ async def test_member_uploads_and_another_member_downloads(client: AsyncClient) assert attachment["size_bytes"] == len(SAMPLE) == 111817 assert attachment["sha256"] == sha256(SAMPLE).hexdigest() assert (attachment["status"], attachment["variants"], attachment["error_code"]) == ( - "ready", + "queued", [], None, ) assert attachment["created_at"].endswith("Z") path = f"/api/v1/attachments/{attachment['id']}" assert (await client.get(path, headers=actor(bob))).json() == attachment + await settle(client) + ready = (await client.get(path, headers=actor(bob))).json() + assert (ready["status"], ready["error_code"], ready["sha256"]) == ( + "ready", + None, + attachment["sha256"], + ) + assert [variant["name"] for variant in ready["variants"]] == ["thumbnail"] content = await client.get( f"{path}/content", headers={**actor(bob), "Accept": "application/json"} ) @@ -93,19 +102,24 @@ async def test_member_uploads_and_another_member_downloads(client: AsyncClient) assert stranger.json()["error"]["code"] == "not_found" -async def test_object_key_is_assigned_by_server(client: AsyncClient) -> None: +async def test_object_keys_are_assigned_by_server(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) attachment = (await upload(client, alice, chat, filename="a.png")).json() - key = await scalar("SELECT object_key FROM attachments WHERE id = $1", attachment["id"]) - assert key == f"attachments/{attachment['id']}" - assert set((await storage(client)).objects) >= {key} + staging = await scalar("SELECT staging_key FROM attachments WHERE id = $1", attachment["id"]) + assert staging == f"staging/{attachment['id']}" + assert set((await storage(client)).objects) >= {staging} + await settle(client) + original = await scalar("SELECT original_key FROM attachments WHERE id = $1", attachment["id"]) + assert original == f"originals/{attachment['id']}" + assert (await storage(client)).objects[original][0] == SAMPLE async def test_jpeg_is_accepted_by_content(client: AsyncClient) -> None: alice, bob, _, chat = await trio(client) response = await upload(client, alice, chat, JPEG, headers={"Content-Type": "IMAGE/JPEG"}) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()["content_type"] == "image/jpeg" + await settle(client) download = await client.get( f"/api/v1/attachments/{response.json()['id']}/content", headers=actor(bob) ) @@ -127,7 +141,7 @@ async def test_access_is_checked_before_content(client: AsyncClient) -> None: async def test_exact_limit_is_accepted(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) response = await upload(client, alice, chat, png(16, 16, size=LIMIT)) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()["size_bytes"] == LIMIT @@ -154,7 +168,7 @@ async def test_one_byte_over_chunked(client: AsyncClient) -> None: @pytest.mark.parametrize( ("width", "height", "status"), - [(8000, 5000, 201), (8001, 5000, 413), (100_000, 100_000, 413)], + [(8000, 5000, 202), (8001, 5000, 413), (100_000, 100_000, 413)], ) async def test_pixel_limit_is_read_from_header( client: AsyncClient, width: int, height: int, status: int @@ -224,16 +238,33 @@ async def test_dangerous_filename_is_rejected(client: AsyncClient, filename: str async def test_display_name_is_kept(client: AsyncClient, filename: str) -> None: alice, _, _, chat = await trio(client) response = await upload(client, alice, chat, filename=filename) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()["filename"] == filename -async def test_only_original_variant_exists(client: AsyncClient) -> None: - alice, _, _, chat = await trio(client) +async def test_content_waits_for_ready_and_serves_level_three_variants( + client: AsyncClient, +) -> None: + alice, bob, _, chat = await trio(client) attachment = (await upload(client, alice, chat)).json() path = f"/api/v1/attachments/{attachment['id']}/content" + for variant in ("original", "thumbnail", "crop"): + early = await client.get(f"{path}?variant={variant}", headers=actor(bob)) + assert early.status_code == 409, early.text + assert early.json()["error"]["code"] == "conflict" + await settle(client) assert (await client.get(f"{path}?variant=original", headers=actor(alice))).status_code == 200 - assert (await client.get(f"{path}?variant=thumbnail", headers=actor(alice))).status_code == 400 + thumbnail = await client.get(f"{path}?variant=thumbnail", headers=actor(bob)) + meta = (await client.get(path.removesuffix("/content"), headers=actor(bob))).json() + (variant,) = meta["variants"] + assert thumbnail.headers["content-type"] == "image/jpeg" + assert len(thumbnail.content) == variant["size_bytes"] + assert sha256(thumbnail.content).hexdigest() == variant["sha256"] + assert (variant["width"], variant["height"]) == (256, 160) + for absent in ("crop", "optimized"): + missing = await client.get(f"{path}?variant={absent}", headers=actor(alice)) + assert missing.status_code == 404 + assert (await client.get(f"{path}?variant=preview", headers=actor(alice))).status_code == 400 assert ( await client.get("/api/v1/attachments/x/content", headers=actor(alice)) ).status_code == 400 @@ -244,6 +275,7 @@ async def test_only_original_variant_exists(client: AsyncClient) -> None: async def test_storage_failure_leaves_nothing_visible(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) ready = await upload(client, alice, chat) + await settle(client) store = await storage(client) store.failing = True try: @@ -271,6 +303,7 @@ async def test_storage_failure_leaves_nothing_visible(client: AsyncClient) -> No async def test_lost_object_is_not_found(client: AsyncClient) -> None: alice, _, _, chat = await trio(client) attachment = (await upload(client, alice, chat)).json() + await settle(client) (await storage(client)).objects.clear() response = await client.get( f"/api/v1/attachments/{attachment['id']}/content", headers=actor(alice) diff --git a/student-tests/api/test_bounded_contexts.py b/student-tests/api/test_bounded_contexts.py index 6ae98776..9299c528 100644 --- a/student-tests/api/test_bounded_contexts.py +++ b/student-tests/api/test_bounded_contexts.py @@ -154,22 +154,46 @@ def test_every_zone_owns_its_http_layer() -> None: assert (SOURCE / zone / "entrypoints" / part).is_file(), f"{zone}: нет {part}" +PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,)} + + def test_only_main_knows_several_zones() -> None: offenders: dict[str, list[str]] = {} for path in SOURCE.rglob("*.py"): relative = path.relative_to(SOURCE) if path.name in {"__init__.py", "main.py"} or relative.parts[0] in ZONES: continue + allowed = PROCESSES.get(str(relative), ()) borrowed = [ module for module in imports_of(path) if module.startswith((IDENTITY, MESSAGING, ATTACHMENTS)) + and not module.startswith(allowed or ("-",)) ] if borrowed: offenders[str(relative)] = borrowed assert not offenders +def test_broker_framework_stays_out_of_shared_and_core_layers() -> None: + offenders: dict[str, list[str]] = {} + for path in [ + *modules_under("shared"), + *( + p + for zone in ZONES + for layer in ("domain", "application") + for p in modules_under(zone, layer) + ), + ]: + borrowed = [ + m for m in imports_of(path) if m.split(".")[0] in {"faststream", "aio_pika", "aiormq"} + ] + if borrowed: + offenders[str(path.relative_to(SOURCE))] = borrowed + assert not offenders + + def test_application_has_no_separate_entrypoints_root() -> None: assert not (SOURCE / "entrypoints").exists() assert (SOURCE / "main.py").is_file() @@ -233,7 +257,7 @@ def test_zone_errors_do_not_leak_transport_codes() -> None: IDENTITY_TABLES = ("users", "sessions") MESSAGING_TABLES = ("chats", "chat_members", "messages", "idempotency_keys", "message_attachments") -ATTACHMENT_TABLES = ("attachments",) +ATTACHMENT_TABLES = ("attachments", "attachment_variants", "jobs", "outbox", "job_events") def sql_tables(zone: str, tables: tuple[str, ...]) -> set[str]: diff --git a/student-tests/api/test_message_attachments.py b/student-tests/api/test_message_attachments.py index dc95b5aa..494a2fb5 100644 --- a/student-tests/api/test_message_attachments.py +++ b/student-tests/api/test_message_attachments.py @@ -14,7 +14,7 @@ async def attach(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) content=SAMPLE, headers={**actor(user), "Content-Type": "image/png"}, ) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text return response.json()["id"] diff --git a/student-tests/api/test_pipeline.py b/student-tests/api/test_pipeline.py new file mode 100644 index 00000000..91f98761 --- /dev/null +++ b/student-tests/api/test_pipeline.py @@ -0,0 +1,244 @@ +import asyncio +import json +from typing import Any +from uuid import uuid4 + +import asyncpg +import pytest +from httpx import AsyncClient +from test_attachments import storage, trio, upload + +from conftest import DATABASE, actor +from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.shared.application.errors import Unavailable +from pictures import SAMPLE +from pipeline import pipeline_of + + +async def rows(sql: str, *args: Any) -> list[dict[str, Any]]: + connection = await asyncpg.connect(DATABASE["url"]) + try: + return [dict(row) for row in await connection.fetch(sql, *args)] + finally: + await connection.close() + + +async def accepted(http: AsyncClient) -> tuple[dict[str, Any], dict[str, Any], str]: + alice, bob, _, chat = await trio(http) + response = await upload(http, alice, chat) + assert response.status_code == 202, response.text + return alice, bob, response.json()["id"] + + +async def state(attachment_id: str) -> dict[str, Any]: + (row,) = await rows( + "SELECT a.status, a.error_code, a.published_version, a.staging_key, a.original_key, " + "j.status AS job_status, j.attempt, j.claims FROM attachments a " + "JOIN jobs j ON j.attachment_id = a.id WHERE a.id = $1", + attachment_id, + ) + return row + + +async def variants(attachment_id: str) -> list[dict[str, Any]]: + return await rows( + "SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants " + "WHERE attachment_id = $1 ORDER BY name", + attachment_id, + ) + + +async def events(attachment_id: str) -> list[str]: + found = await rows( + "SELECT kind FROM job_events WHERE attachment_id = $1 ORDER BY id", attachment_id + ) + return [row["kind"] for row in found] + + +async def job_of(attachment_id: str) -> Job: + (row,) = await rows( + "SELECT payload::text AS body FROM outbox o JOIN jobs j USING (job_id) " + "WHERE j.attachment_id = $1 ORDER BY o.id LIMIT 1", + attachment_id, + ) + return JobMessage.model_validate_json(row["body"]).job() + + +async def test_job_is_processed_once_and_staging_is_cleaned(client: AsyncClient) -> None: + _, bob, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + outcomes = await pipeline.drain() + assert Outcome.DONE in outcomes + done = await state(attachment_id) + assert (done["status"], done["job_status"], done["published_version"]) == ("ready", "done", 1) + assert done["original_key"] == f"originals/{attachment_id}" + assert done["staging_key"] == f"staging/{attachment_id}" + (thumbnail,) = await variants(attachment_id) + assert thumbnail["object_key"].startswith(f"variants/{attachment_id}/1/thumbnail-") + assert await pipeline.clean() >= 1 + assert (await state(attachment_id))["staging_key"] is None + objects = (await storage(client)).objects + assert f"staging/{attachment_id}" not in objects + assert objects[f"originals/{attachment_id}"][0] == SAMPLE + content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(bob)) + assert content.content == SAMPLE + assert await events(attachment_id) == ["enqueued", "claimed", "done"] + + +async def test_message_is_a_contract_job(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + mine = [ + json.loads(body) + for body in pipeline.publisher.bodies + if json.loads(body)["attachment_id"] == attachment_id + ] + assert len(mine) == 1 + assert set(mine[0]) == { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + } + assert (mine[0]["pipeline_version"], mine[0]["attempt"]) == (1, 1) + assert await pipeline.publish() == [] + + +async def test_redelivery_after_ready_changes_nothing(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.drain() + before, job = await variants(attachment_id), await job_of(attachment_id) + assert await pipeline.process(job) is Outcome.DUPLICATE + assert await variants(attachment_id) == before + assert (await state(attachment_id))["status"] == "ready" + assert (await events(attachment_id))[-1] == "duplicate_noop" + + +async def test_two_copies_in_flight_give_one_result(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + outcomes = await asyncio.gather(pipeline.process(job), pipeline.process(job)) + assert sorted(outcomes) == sorted([Outcome.DONE, Outcome.DUPLICATE]) + assert len(await variants(attachment_id)) == 1 + assert (await state(attachment_id))["claims"] == 1 + + +async def test_transient_error_reschedules_with_the_same_job(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + store = await storage(client) + store.failing = True + try: + assert await pipeline.process(job) is Outcome.RESCHEDULED + finally: + store.failing = False + queued = await state(attachment_id) + assert (queued["status"], queued["job_status"], queued["attempt"]) == ("queued", "queued", 2) + (retry,) = [j for j in await pipeline.publish() if str(j.attachment_id) == attachment_id] + assert (retry.id, retry.attempt) == (job.id, 2) + assert await pipeline.process(retry) is Outcome.DONE + assert (await state(attachment_id))["status"] == "ready" + + +async def test_attempts_are_bounded(client: AsyncClient) -> None: + alice, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.faults = FailingFaults() + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["job_status"]) == ( + "failed", + "retry_exhausted", + "failed", + ) + assert failed["attempt"] == pipeline.policy.max_attempts + assert (await events(attachment_id)).count("retry_scheduled") == 4 + content = await client.get(f"/api/v1/attachments/{attachment_id}/content", headers=actor(alice)) + assert content.status_code == 409 + + +async def test_undecodable_image_fails_without_retries(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + (await storage(client)).objects[f"staging/{attachment_id}"] = (b"not a png", "image/png") + pipeline = await pipeline_of(client) + await pipeline.drain() + failed = await state(attachment_id) + assert (failed["status"], failed["error_code"], failed["attempt"]) == ( + "failed", + "invalid_image", + 1, + ) + + +async def test_expired_lease_is_recovered_and_old_holder_is_fenced(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + uow = pipeline.uow() + async with uow.transaction(): + claim = await uow.jobs.claim(job, uuid4(), pipeline.lease) + await uow.commit() + assert claim is not None + assert (await state(attachment_id))["status"] == "processing" + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1", + attachment_id, + ) + assert await pipeline.recover() >= 1 + assert (await state(attachment_id))["status"] == "queued" + late = pipeline.uow() + async with late.transaction(): + outcome = await late.jobs.complete(claim.lease, [], f"originals/{attachment_id}") + await late.commit() + assert outcome is Outcome.FENCED + assert await variants(attachment_id) == [] + await pipeline.drain() + recovered = await state(attachment_id) + assert (recovered["status"], recovered["attempt"]) == ("ready", 2) + assert {"lease_expired", "fenced"} <= set(await events(attachment_id)) + + +async def test_unpublished_rows_stay_when_the_broker_refuses(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.publisher.failing = True + with pytest.raises(Unavailable): + await pipeline.publish() + pending = await rows( + "SELECT o.published_at FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id = $1", + attachment_id, + ) + assert pending == [{"published_at": None}] + pipeline.publisher.failing = False + assert any(str(job.attachment_id) == attachment_id for job in await pipeline.publish()) + + +async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, policy=RetryPolicy(5, 30, 60)) + pipeline.faults = FailingFaults() + await pipeline.drain() + queued = await state(attachment_id) + assert (queued["status"], queued["attempt"]) == ("queued", 2) + (row,) = await rows( + "SELECT extract(epoch FROM next_attempt_at - now()) AS wait FROM jobs " + "WHERE attachment_id = $1", + attachment_id, + ) + assert 25 < row["wait"] <= 30 + + +class FailingFaults: + def reach(self, point: str) -> None: + raise Unavailable(point) diff --git a/student-tests/api/test_upload_consistency.py b/student-tests/api/test_upload_consistency.py index 30bc91cf..b65bb084 100644 --- a/student-tests/api/test_upload_consistency.py +++ b/student-tests/api/test_upload_consistency.py @@ -13,6 +13,7 @@ from messenger.attachments.domain.events import AttachmentDiscarded from messenger.attachments.entrypoints.sweeper import UploadSweeper from messenger.shared.application.messagebus import EventBus from pictures import SAMPLE +from pipeline import settle AGED = "UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = $1" @@ -51,7 +52,7 @@ async def intent(http: AsyncClient, user: dict[str, Any], chat: dict[str, Any]) "SELECT id::text FROM attachments WHERE filename = $1 AND status = 'pending'", name ) assert found is not None - store.objects[f"attachments/{found}"] = (SAMPLE, "image/png") + store.objects[f"staging/{found}"] = (SAMPLE, "image/png") return found @@ -68,7 +69,7 @@ async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> N f"""CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'injected' USING ERRCODE = '55P03'; END $$; CREATE TRIGGER {trigger} BEFORE UPDATE ON attachments FOR EACH ROW - WHEN (NEW.filename = '{name}' AND NEW.status = 'ready') EXECUTE FUNCTION {trigger}()""" + WHEN (NEW.filename = '{name}' AND NEW.status = 'queued') EXECUTE FUNCTION {trigger}()""" ) before, seen = set(store.objects), await discarded(client) try: @@ -78,6 +79,14 @@ async def test_commit_failure_after_put_is_compensated(client: AsyncClient) -> N assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 + assert ( + await scalar( + "SELECT count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id " + "WHERE a.filename = $1", + name, + ) + == 0 + ) assert set(store.objects) == before assert [event.reason for event in seen] == ["compensation"] @@ -87,7 +96,7 @@ async def test_lost_intent_gives_503_and_no_object(client: AsyncClient) -> None: store = await storage(client) async def swept_meanwhile(key: str) -> None: - await execute("DELETE FROM attachments WHERE object_key = $1", key) + await execute("DELETE FROM attachments WHERE staging_key = $1", key) store.after_put = swept_meanwhile before = set(store.objects) @@ -107,12 +116,13 @@ async def test_failed_put_leaves_an_intent_until_grace_ends(client: AsyncClient) await execute(AGED, pending) assert await sweeper(client).sweep() >= 1 assert not await exists(pending) - assert f"attachments/{pending}" not in (await storage(client)).objects + assert f"staging/{pending}" not in (await storage(client)).objects async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClient) -> None: alice, bob, _, chat = await trio(client) ready = (await upload(client, alice, chat)).json() + await settle(client) sent = await client.post( f"/api/v1/chats/{chat['id']}/messages", json={"text": "", "attachment_ids": [ready["id"]]}, @@ -125,9 +135,9 @@ async def test_sweep_keeps_fresh_intents_and_ready_attachments(client: AsyncClie assert await sweeper(client).sweep() == 1 objects = (await storage(client)).objects assert not await exists(old) - assert f"attachments/{old}" not in objects + assert f"staging/{old}" not in objects assert await exists(fresh) - assert f"attachments/{fresh}" in objects + assert f"staging/{fresh}" in objects download = await client.get(f"/api/v1/attachments/{ready['id']}/content", headers=actor(bob)) assert download.status_code == 200 assert download.content == SAMPLE @@ -146,7 +156,7 @@ async def test_parallel_sweeps_share_the_work(client: AsyncClient) -> None: assert {event.reason for event in seen} == {"abandoned"} assert await sweeper(client).sweep() == 0 objects = (await storage(client)).objects - assert not any(f"attachments/{attachment_id}" in objects for attachment_id in pending) + assert not any(f"staging/{attachment_id}" in objects for attachment_id in pending) async def test_background_sweeper_removes_abandoned_intents() -> None: @@ -183,7 +193,7 @@ async def test_upload_waits_for_a_free_slot() -> None: await asyncio.sleep(0.3) assert read == ["first"] gate.set() - assert [(await first).status_code, (await second).status_code] == [201, 201] + assert [(await first).status_code, (await second).status_code] == [202, 202] assert read == ["first", "second"] @@ -241,4 +251,4 @@ async def test_stalled_upload_times_out_and_frees_the_slot() -> None: assert response.status_code == 503 assert response.json()["error"]["code"] == "unavailable" assert await scalar("SELECT count(*) FROM attachments WHERE filename = $1", name) == 0 - assert (await upload(http, alice, chat)).status_code == 201 + assert (await upload(http, alice, chat)).status_code == 202 diff --git a/student-tests/fakes.py b/student-tests/fakes.py index e0d3633e..2b0e23f8 100644 --- a/student-tests/fakes.py +++ b/student-tests/fakes.py @@ -58,7 +58,20 @@ class MemoryObjectStorage: data = self.objects[key][0] return ObjectContent(size=len(data), chunks=pieces(data)) + async def read(self, key: str) -> bytes: + content = await self.open(key) + return b"".join([chunk async for chunk in content.chunks]) + + async def copy(self, source: str, target: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") + if source not in self.objects: + raise ObjectMissingError(source) + self.objects[target] = self.objects[source] + async def delete(self, key: str) -> None: + if self.failing: + raise Unavailable("S3 недоступно") self.objects.pop(key, None) diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py new file mode 100644 index 00000000..f3bfcae5 --- /dev/null +++ b/student-tests/pipeline.py @@ -0,0 +1,117 @@ +from datetime import timedelta +from uuid import UUID + +from httpx import AsyncClient + +from conftest import resolve +from fakes import NullLogger +from messenger.attachments.application.commands import ( + CleanStaging, + ProcessJob, + PublishOutbox, + RecoverExpiredLeases, +) +from messenger.attachments.application.pipeline import ( + CleanStagingHandler, + ProcessJobHandler, + PublishOutboxHandler, + RecoverExpiredLeasesHandler, +) +from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer +from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy +from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.attachments.infrastructure.images import PillowVariantRenderer +from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork +from messenger.shared.application.errors import Unavailable +from messenger.shared.infrastructure.clock import MonotonicUtcClock +from messenger.shared.infrastructure.ids import Uuid7Generator +from messenger.shared.infrastructure.postgres import Database + +FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001) + + +class CapturingPublisher: + __slots__ = ("bodies", "failing", "job_ids") + + def __init__(self) -> None: + self.bodies: list[bytes] = [] + self.job_ids: list[UUID] = [] + self.failing = False + + async def publish(self, job_id: UUID, body: bytes) -> None: + if self.failing: + raise Unavailable("брокер недоступен") + self.job_ids.append(job_id) + self.bodies.append(body) + + +class InlinePipeline: + __slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage") + + def __init__( + self, + database: Database, + storage: ObjectStorage, + *, + policy: RetryPolicy = FAST, + lease: timedelta = timedelta(seconds=60), + ) -> None: + self._database = database + self.storage = storage + self.policy = policy + self.lease = lease + self.publisher = CapturingPublisher() + self.renderer: VariantRenderer = PillowVariantRenderer(40_000_000, 2) + self.faults: Faults = EnvironmentFaults("", NullLogger()) + + def uow(self) -> PostgresAttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(self._database) + + async def publish(self) -> list[Job]: + start = len(self.publisher.bodies) + await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100)) + return [ + JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:] + ] + + async def process(self, job: Job) -> Outcome: + handler = ProcessJobHandler( + self.uow(), + self.storage, + self.renderer, + self.faults, + clock=MonotonicUtcClock(), + ids=Uuid7Generator(), + policy=self.policy, + lease=self.lease, + ) + return await handler(ProcessJob(job)) + + async def drain(self, rounds: int = 20) -> list[Outcome]: + outcomes: list[Outcome] = [] + for _ in range(rounds): + jobs = await self.publish() + if not jobs: + return outcomes + outcomes.extend([await self.process(job) for job in jobs]) + return outcomes + + async def recover(self) -> int: + handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy) + return await handler(RecoverExpiredLeases(100)) + + async def clean(self) -> int: + return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100)) + + +async def pipeline_of( + http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60) +) -> InlinePipeline: + database = await resolve(http, Database) + storage = await resolve(http, ObjectStorage) + return InlinePipeline(database, storage, policy=policy, lease=lease) + + +async def settle(http: AsyncClient) -> list[Outcome]: + return await (await pipeline_of(http)).drain() diff --git a/student-tests/unit/test_attachments_domain.py b/student-tests/unit/test_attachments_domain.py index e26d3101..db91c483 100644 --- a/student-tests/unit/test_attachments_domain.py +++ b/student-tests/unit/test_attachments_domain.py @@ -4,7 +4,7 @@ from uuid import uuid4 import pytest from messenger.attachments.domain.errors import InvalidFilename, UnsupportedImage -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentStored +from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind @@ -46,7 +46,7 @@ def test_declared_type_accepts_only_png_and_jpeg(header: str | None) -> None: ImageKind.declared(header) -def test_attachment_is_pending_until_stored() -> None: +def test_attachment_is_pending_until_queued() -> None: attachment_id = uuid4() attachment = Attachment.stage( attachment_id=attachment_id, @@ -59,13 +59,19 @@ def test_attachment_is_pending_until_stored() -> None: now=MOMENT, ) assert attachment.status is AttachmentStatus.PENDING - assert attachment.object_key == f"attachments/{attachment_id}" + assert attachment.staging_key == f"staging/{attachment_id}" + assert (attachment.original_key, attachment.published_version, attachment.variants) == ( + None, + 0, + (), + ) assert attachment.collect_events() == [] - attachment.store(MOMENT - timedelta(seconds=1)) - assert attachment.status is AttachmentStatus.READY + attachment.enqueue(MOMENT - timedelta(seconds=1)) + assert attachment.status is AttachmentStatus.QUEUED + assert not attachment.ready assert attachment.updated_at == MOMENT events = attachment.collect_events() - assert [type(event) for event in events] == [AttachmentStored] + assert [type(event) for event in events] == [AttachmentQueued] assert attachment.collect_events() == [] diff --git a/student-tests/unit/test_pipeline_domain.py b/student-tests/unit/test_pipeline_domain.py new file mode 100644 index 00000000..c801899f --- /dev/null +++ b/student-tests/unit/test_pipeline_domain.py @@ -0,0 +1,86 @@ +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import uuid4 + +import pytest + +from messenger.attachments.domain.jobs import Job, RetryPolicy +from messenger.attachments.domain.variants import fit, variant_key +from messenger.attachments.entrypoints.schemas import JobMessage + +SCHEMA = json.loads( + (Path(__file__).resolve().parents[2] / "contracts" / "jobs.schema.json").read_text() +) + + +@pytest.mark.parametrize( + ("size", "fitted"), + [ + ((320, 200), (256, 160)), + ((300, 200), (256, 170)), + ((801, 601), (256, 192)), + ((17, 1000), (4, 256)), + ((3000, 2), (256, 1)), + ((2, 3000), (1, 256)), + ((64, 64), (64, 64)), + ((256, 100), (256, 100)), + ((257, 257), (256, 256)), + ], +) +def test_thumbnail_fits_the_box_rounding_down( + size: tuple[int, int], fitted: tuple[int, int] +) -> None: + assert fit(*size, 256) == fitted + + +def test_retry_policy_doubles_up_to_the_cap_and_stops_at_the_limit() -> None: + policy = RetryPolicy(max_attempts=5, base_seconds=2, cap_seconds=10) + delays = [policy.delay(attempt) for attempt in range(1, 6)] + assert delays == [ + timedelta(seconds=2), + timedelta(seconds=4), + timedelta(seconds=8), + timedelta(seconds=10), + None, + ] + + +def test_job_message_follows_the_contract_schema() -> None: + job = Job(uuid4(), uuid4(), 1, 3, datetime(2026, 9, 1, 12, 0, tzinfo=UTC)) + payload = job.payload() + assert set(SCHEMA["required"]) <= set(payload) <= set(SCHEMA["properties"]) + assert payload["schema_version"] == SCHEMA["properties"]["schema_version"]["const"] + assert payload["type"] == SCHEMA["properties"]["type"]["const"] + assert payload["created_at"] == "2026-09-01T12:00:00.000000Z" + assert JobMessage.model_validate_json(json.dumps(payload)).job() == job + assert job.retried(4).payload()["job_id"] == payload["job_id"] + + +@pytest.mark.parametrize("example", SCHEMA["examples"]) +def test_contract_examples_are_accepted(example: dict[str, object]) -> None: + assert JobMessage.model_validate_json(json.dumps(example)).attempt == example["attempt"] + + +@pytest.mark.parametrize( + "change", + [ + {"schema_version": 2}, + {"type": "attachment.process.v2"}, + {"attempt": 0}, + {"pipeline_version": 0}, + {"filename": "a.png"}, + {"traceparent": "01-xyz"}, + {"created_at": "2026-09-01T12:00:00"}, + ], +) +def test_messages_off_the_schema_are_rejected(change: dict[str, object]) -> None: + message = {**SCHEMA["examples"][0], **change} + with pytest.raises(ValueError, match="validation error"): + JobMessage.model_validate_json(json.dumps(message)) + + +def test_variant_keys_carry_version_and_content_hash() -> None: + attachment = uuid4() + key = variant_key(attachment, 2, "thumbnail", "ab" * 32, "jpg") + assert key == f"variants/{attachment}/2/thumbnail-{'ab' * 8}.jpg" diff --git a/student-tests/unit/test_renderer.py b/student-tests/unit/test_renderer.py new file mode 100644 index 00000000..471fccdd --- /dev/null +++ b/student-tests/unit/test_renderer.py @@ -0,0 +1,89 @@ +from io import BytesIO + +import pytest +from PIL import Image + +from messenger.attachments.domain.errors import UnreadableImage +from messenger.attachments.infrastructure.images import render_variants +from pictures import SAMPLE + +LIMIT = 40_000_000 + + +def encoded(image: Image.Image, kind: str = "PNG", **options: object) -> bytes: + buffer = BytesIO() + image.save(buffer, kind, **options) + return buffer.getvalue() + + +def thumbnail(data: bytes) -> Image.Image: + (rendition,) = render_variants(data, LIMIT) + assert (rendition.name, rendition.content_type, rendition.extension) == ( + "thumbnail", + "image/jpeg", + "jpg", + ) + image = Image.open(BytesIO(rendition.data)) + assert image.format == "JPEG" + assert image.size == (rendition.width, rendition.height) + assert not image.getexif() + return image + + +def test_fixture_gives_256_by_160() -> None: + assert thumbnail(SAMPLE).size == (256, 160) + + +@pytest.mark.parametrize( + ("mode", "size", "fitted"), + [ + ("L", (17, 1000), (4, 256)), + ("RGB", (64, 64), (64, 64)), + ("RGB", (3000, 2), (256, 1)), + ("I;16", (300, 200), (256, 170)), + ("CMYK", (801, 601), (256, 192)), + ], +) +def test_sizes_round_down_and_never_upscale( + mode: str, size: tuple[int, int], fitted: tuple[int, int] +) -> None: + kind = "JPEG" if mode == "CMYK" else "PNG" + assert thumbnail(encoded(Image.new(mode, size), kind)).size == fitted + + +@pytest.mark.parametrize("mode", ["RGBA", "LA"]) +def test_transparency_is_composed_on_white(mode: str) -> None: + transparent = Image.new(mode, (300, 300), (0,) * len(mode)) + assert thumbnail(encoded(transparent)).convert("RGB").getpixel((5, 5)) == (255, 255, 255) + + +def test_palette_transparency_is_composed_on_white() -> None: + palette = Image.new("P", (200, 100)) + image = thumbnail(encoded(palette, transparency=0)) + assert image.size == (200, 100) + assert image.convert("RGB").getpixel((5, 5)) == (255, 255, 255) + + +@pytest.mark.parametrize(("orientation", "fitted"), [(6, (128, 256)), (3, (256, 128))]) +def test_exif_orientation_is_applied_before_sizing( + orientation: int, fitted: tuple[int, int] +) -> None: + exif = Image.Exif() + exif[0x0112] = orientation + data = encoded(Image.new("RGB", (400, 200), (10, 200, 10)), "JPEG", exif=exif) + assert thumbnail(data).size == fitted + + +@pytest.mark.parametrize( + "data", + [b"", b"garbage", SAMPLE[: len(SAMPLE) // 2], b"GIF89a" + bytes(20)], + ids=["empty", "garbage", "truncated", "gif"], +) +def test_undecodable_data_is_a_permanent_error(data: bytes) -> None: + with pytest.raises(UnreadableImage): + render_variants(data, LIMIT) + + +def test_pixel_limit_is_checked_before_decoding() -> None: + with pytest.raises(UnreadableImage): + render_variants(SAMPLE, 320 * 200 - 1) diff --git a/uv.lock b/uv.lock index e2f4b436..5c98efc5 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,19 @@ version = 1 revision = 5 requires-python = "==3.14.*" +[[package]] +name = "aio-pika" +version = "10.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiormq" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/31/70f0d7a9fe1affc3241d86d24c24714a4a2ac648961efd3d835ef734a6a1/aio_pika-10.2.1.tar.gz", hash = "sha256:e4f77afd9179a3cca5a904c3b0a1190256a36d9f90650db908b1b2245eb27487", size = 101040, upload-time = "2026-10-25T11:47:24.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/ae/f69eb62f240f743d5d3aff659a6ed76077f73b722a0d8f9f9559ab825836/aio_pika-10.2.1-py3-none-any.whl", hash = "sha256:66c85d311e30f1dd3e28109d686c9bf1d9dd1a74582004c7cb7bed284d8fe3b1", size = 61568, upload-time = "2026-10-25T11:47:23.119Z" }, +] + [[package]] name = "aiobotocore" version = "3.10.0" @@ -97,6 +110,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, ] +[[package]] +name = "aiormq" +version = "7.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pamqp" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/11/8b93eac7630c5087c34df3d34ce0bdf50d6ea2fb83061f17e8c73877141d/aiormq-7.2.4.tar.gz", hash = "sha256:5fd20bb9583d73de85572198496b65cf028f27527c1733c9840d458e0b40e8fa", size = 81569, upload-time = "2026-10-24T12:36:22.703Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/a7/346b74a5f19d0acbd9d428f5f7b070537a8a079e45a74fba58c7831e058d/aiormq-7.2.4-py3-none-any.whl", hash = "sha256:2b9d6dade171318185ab8f507717b630f20b8b2303a075f9293d3f6b0d166f1d", size = 40694, upload-time = "2026-10-24T12:36:21.054Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -304,6 +330,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/08/dd3259a319ba45c48d975caf237c830c7873a203288cb4ab186cc5bef585/dishka-1.10.1-py3-none-any.whl", hash = "sha256:84f40c1115eb391ce3fb558fd7946c5bcaf6361c35b77b7a185b7697a5720ec7", size = 120062, upload-time = "2026-04-25T12:07:59.153Z" }, ] +[[package]] +name = "dishka-faststream" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dishka" }, + { name = "faststream" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/32/c8df3791072a5ea11abaa8e142536fe8cd7745f3fe25e4be4fdf3ac112f5/dishka_faststream-0.7.0.tar.gz", hash = "sha256:b8c6e95bf30f32fc67b3289697cee51c3047778afe195aa9a1062d21e64bfb7b", size = 118092, upload-time = "2026-06-04T17:34:30.856Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/9f/b1918e3ee6814f1b94489ff367b9178e2730a3d23370c24f2f2bce5886e5/dishka_faststream-0.7.0-py3-none-any.whl", hash = "sha256:6f68ce1520b9f15478c8e8407e0d36cdcf1261d237a81e0090fb34ab4775e67e", size = 10551, upload-time = "2026-06-04T17:34:29.953Z" }, +] + +[[package]] +name = "fast-depends" +version = "3.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/76/1e67c34a18a75aff77f25de2d56de8ad3d38c05915ae513ba35293fd920b/fast_depends-3.0.10.tar.gz", hash = "sha256:234c1dcfcdbc1cdb0ebe543fbf8e8234c069ba39518fbdf1f71dad6e3e6111ad", size = 19807, upload-time = "2026-10-28T07:36:21.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/af/af933c80bd2c60070dec7ea324350ce698d579bada6ab6511800f749ab57/fast_depends-3.0.10-py3-none-any.whl", hash = "sha256:e49306f54b691a59076e6fe1fdae9418bff4af80d1c9d9fb82a6d517b4d4c04c", size = 25535, upload-time = "2026-10-28T07:36:20.374Z" }, +] + +[package.optional-dependencies] +pydantic = [ + { name = "pydantic" }, +] + [[package]] name = "fastapi" version = "0.141.1" @@ -320,6 +377,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] +[[package]] +name = "faststream" +version = "0.7.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "fast-depends", extra = ["pydantic"] }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/b6/d15b1c88f1adc3bf11e4e23f8a057e489e2e60a47f7d4b821a3ae264f4c2/faststream-0.7.8.tar.gz", hash = "sha256:9e367d9ba8b59eeafad19f46641b11f20d23a7ed75ea27b2503856a8ae0c075a", size = 392145, upload-time = "2026-10-22T20:50:29.387Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/79/daae6f842665756f5fa45683d4762333348c4ad4dba2e9d9f37644717d74/faststream-0.7.8-py3-none-any.whl", hash = "sha256:ffe412278cf116216078dcf29dcec93274994a98d036be2bc388071dff7e259e", size = 618852, upload-time = "2026-10-22T20:50:27.388Z" }, +] + +[package.optional-dependencies] +rabbit = [ + { name = "aio-pika" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -465,7 +541,9 @@ dependencies = [ { name = "argon2-cffi" }, { name = "asyncpg" }, { name = "dishka" }, + { name = "dishka-faststream" }, { name = "fastapi" }, + { name = "faststream", extra = ["rabbit"] }, { name = "gunicorn" }, { name = "orjson" }, { name = "pillow" }, @@ -492,7 +570,9 @@ requires-dist = [ { name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "asyncpg", specifier = ">=0.32.0" }, { name = "dishka", specifier = ">=1.7.0" }, + { name = "dishka-faststream", specifier = ">=0.1" }, { name = "fastapi", specifier = ">=0.118.0" }, + { name = "faststream", extras = ["rabbit"], specifier = ">=0.7.2,<0.8" }, { name = "gunicorn", specifier = ">=26.2.0" }, { name = "orjson", specifier = ">=3.12.0" }, { name = "pillow", specifier = ">=12.0.0" }, @@ -593,6 +673,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pamqp" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/31/4c/33a0ddaaac7bc42f9a542dbaaee8b580ceca3f89bf5da7c498d1fa97ff9a/pamqp-4.0.1.tar.gz", hash = "sha256:9dd13b828e346622793981f14a5df817fce5de998c746209d6c0154eb8403970", size = 137192, upload-time = "2026-07-06T16:37:51.732Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/14/1dfc08b743ba995a38dee0ea09beb46a05c7fe8ac53d729095905f7bf11d/pamqp-4.0.1-py3-none-any.whl", hash = "sha256:a547f45128b06e42ce8d7a739b0cfcc40f2c724770622eaaff4a3f587b1cf7d0", size = 32773, upload-time = "2026-07-06T16:37:50.623Z" }, +] + [[package]] name = "pillow" version = "13.0.0"