Files
mtusi2026lab2/student-tests/pipeline.py
T
ArtyomandClaude Opus 5.5 b152b6e07e Process attachment jobs through an outbox, a relay and a worker
Upload now stages the checked original under staging/, and one
transaction moves the intent to queued, inserts the job (pipeline
version 1, attempt 1), its outbox row and an audit event; the 202 is
the in-memory snapshot, never a reread. Content answers 409 until the
attachment is ready, the thumbnail variant is served from the
published version and absent variants are 404.

The relay publishes outbox rows through FastStream with publisher
confirms, mandatory routing and persistent delivery, marks a row only
after the broker acked it, returns jobs whose lease expired and deletes
the staging object of ready attachments. The worker consumes the
quorum queue through a FastStream subscriber that declares nothing,
takes prefetch from WORKER_CONCURRENCY and acks manually after the
result commit, rejects messages off jobs.schema.json to the dead letter
exchange and requeues after an infrastructure error. A job is claimed
under a lease token; the original is copied to originals/ and a
256 px JPEG thumbnail is written under a content-addressed key; the
commit is fenced by the token, inserts variants with ON CONFLICT DO
NOTHING and publishes only a newer pipeline version. A transient error
reschedules the same job with an exponential delay and a new outbox
row, the last attempt fails with retry_exhausted, an undecodable image
fails at once with invalid_image, and a redelivered finished job is a
recorded no-op.

The worker and the relay are assembled in worker.py and relay.py with
the dishka FastStream integration and the platform providers of the
API; the shared core imports neither FastStream nor aio-pika. Tests
cover the thumbnail geometry and rendering, the job message against
the contract, the pipeline against PostgreSQL (duplicates, parallel
copies, retries, the attempt bound, leases and fencing) and the broker
adapters against a throwaway RabbitMQ with the stand's topology and
permissions.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 01:08:06 +03:00

118 lines
4.1 KiB
Python

from datetime import timedelta
from uuid import UUID
from httpx import AsyncClient
from conftest import resolve
from fakes import NullLogger
from messenger.attachments.application.commands import (
CleanStaging,
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
)
from messenger.attachments.application.pipeline import (
CleanStagingHandler,
ProcessJobHandler,
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
)
from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import PillowVariantRenderer
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.clock import MonotonicUtcClock
from messenger.shared.infrastructure.ids import Uuid7Generator
from messenger.shared.infrastructure.postgres import Database
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
class CapturingPublisher:
__slots__ = ("bodies", "failing", "job_ids")
def __init__(self) -> None:
self.bodies: list[bytes] = []
self.job_ids: list[UUID] = []
self.failing = False
async def publish(self, job_id: UUID, body: bytes) -> None:
if self.failing:
raise Unavailable("брокер недоступен")
self.job_ids.append(job_id)
self.bodies.append(body)
class InlinePipeline:
__slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage")
def __init__(
self,
database: Database,
storage: ObjectStorage,
*,
policy: RetryPolicy = FAST,
lease: timedelta = timedelta(seconds=60),
) -> None:
self._database = database
self.storage = storage
self.policy = policy
self.lease = lease
self.publisher = CapturingPublisher()
self.renderer: VariantRenderer = PillowVariantRenderer(40_000_000, 2)
self.faults: Faults = EnvironmentFaults("", NullLogger())
def uow(self) -> PostgresAttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(self._database)
async def publish(self) -> list[Job]:
start = len(self.publisher.bodies)
await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100))
return [
JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:]
]
async def process(self, job: Job) -> Outcome:
handler = ProcessJobHandler(
self.uow(),
self.storage,
self.renderer,
self.faults,
clock=MonotonicUtcClock(),
ids=Uuid7Generator(),
policy=self.policy,
lease=self.lease,
)
return await handler(ProcessJob(job))
async def drain(self, rounds: int = 20) -> list[Outcome]:
outcomes: list[Outcome] = []
for _ in range(rounds):
jobs = await self.publish()
if not jobs:
return outcomes
outcomes.extend([await self.process(job) for job in jobs])
return outcomes
async def recover(self) -> int:
handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy)
return await handler(RecoverExpiredLeases(100))
async def clean(self) -> int:
return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100))
async def pipeline_of(
http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60)
) -> InlinePipeline:
database = await resolve(http, Database)
storage = await resolve(http, ObjectStorage)
return InlinePipeline(database, storage, policy=policy, lease=lease)
async def settle(http: AsyncClient) -> list[Outcome]:
return await (await pipeline_of(http)).drain()