Upload now stages the checked original under staging/, and one transaction moves the intent to queued, inserts the job (pipeline version 1, attempt 1), its outbox row and an audit event; the 202 is the in-memory snapshot, never a reread. Content answers 409 until the attachment is ready, the thumbnail variant is served from the published version and absent variants are 404. The relay publishes outbox rows through FastStream with publisher confirms, mandatory routing and persistent delivery, marks a row only after the broker acked it, returns jobs whose lease expired and deletes the staging object of ready attachments. The worker consumes the quorum queue through a FastStream subscriber that declares nothing, takes prefetch from WORKER_CONCURRENCY and acks manually after the result commit, rejects messages off jobs.schema.json to the dead letter exchange and requeues after an infrastructure error. A job is claimed under a lease token; the original is copied to originals/ and a 256 px JPEG thumbnail is written under a content-addressed key; the commit is fenced by the token, inserts variants with ON CONFLICT DO NOTHING and publishes only a newer pipeline version. A transient error reschedules the same job with an exponential delay and a new outbox row, the last attempt fails with retry_exhausted, an undecodable image fails at once with invalid_image, and a redelivered finished job is a recorded no-op. The worker and the relay are assembled in worker.py and relay.py with the dishka FastStream integration and the platform providers of the API; the shared core imports neither FastStream nor aio-pika. Tests cover the thumbnail geometry and rendering, the job message against the contract, the pipeline against PostgreSQL (duplicates, parallel copies, retries, the attempt bound, leases and fencing) and the broker adapters against a throwaway RabbitMQ with the stand's topology and permissions. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
118 lines
4.1 KiB
Python
118 lines
4.1 KiB
Python
from datetime import timedelta
|
|
from uuid import UUID
|
|
|
|
from httpx import AsyncClient
|
|
|
|
from conftest import resolve
|
|
from fakes import NullLogger
|
|
from messenger.attachments.application.commands import (
|
|
CleanStaging,
|
|
ProcessJob,
|
|
PublishOutbox,
|
|
RecoverExpiredLeases,
|
|
)
|
|
from messenger.attachments.application.pipeline import (
|
|
CleanStagingHandler,
|
|
ProcessJobHandler,
|
|
PublishOutboxHandler,
|
|
RecoverExpiredLeasesHandler,
|
|
)
|
|
from messenger.attachments.application.ports import Faults, ObjectStorage, VariantRenderer
|
|
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
|
from messenger.attachments.entrypoints.schemas import JobMessage
|
|
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
|
from messenger.attachments.infrastructure.images import PillowVariantRenderer
|
|
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
|
from messenger.shared.infrastructure.ids import Uuid7Generator
|
|
from messenger.shared.infrastructure.postgres import Database
|
|
|
|
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
|
|
|
|
|
|
class CapturingPublisher:
|
|
__slots__ = ("bodies", "failing", "job_ids")
|
|
|
|
def __init__(self) -> None:
|
|
self.bodies: list[bytes] = []
|
|
self.job_ids: list[UUID] = []
|
|
self.failing = False
|
|
|
|
async def publish(self, job_id: UUID, body: bytes) -> None:
|
|
if self.failing:
|
|
raise Unavailable("брокер недоступен")
|
|
self.job_ids.append(job_id)
|
|
self.bodies.append(body)
|
|
|
|
|
|
class InlinePipeline:
|
|
__slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage")
|
|
|
|
def __init__(
|
|
self,
|
|
database: Database,
|
|
storage: ObjectStorage,
|
|
*,
|
|
policy: RetryPolicy = FAST,
|
|
lease: timedelta = timedelta(seconds=60),
|
|
) -> None:
|
|
self._database = database
|
|
self.storage = storage
|
|
self.policy = policy
|
|
self.lease = lease
|
|
self.publisher = CapturingPublisher()
|
|
self.renderer: VariantRenderer = PillowVariantRenderer(40_000_000, 2)
|
|
self.faults: Faults = EnvironmentFaults("", NullLogger())
|
|
|
|
def uow(self) -> PostgresAttachmentsUnitOfWork:
|
|
return PostgresAttachmentsUnitOfWork(self._database)
|
|
|
|
async def publish(self) -> list[Job]:
|
|
start = len(self.publisher.bodies)
|
|
await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100))
|
|
return [
|
|
JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:]
|
|
]
|
|
|
|
async def process(self, job: Job) -> Outcome:
|
|
handler = ProcessJobHandler(
|
|
self.uow(),
|
|
self.storage,
|
|
self.renderer,
|
|
self.faults,
|
|
clock=MonotonicUtcClock(),
|
|
ids=Uuid7Generator(),
|
|
policy=self.policy,
|
|
lease=self.lease,
|
|
)
|
|
return await handler(ProcessJob(job))
|
|
|
|
async def drain(self, rounds: int = 20) -> list[Outcome]:
|
|
outcomes: list[Outcome] = []
|
|
for _ in range(rounds):
|
|
jobs = await self.publish()
|
|
if not jobs:
|
|
return outcomes
|
|
outcomes.extend([await self.process(job) for job in jobs])
|
|
return outcomes
|
|
|
|
async def recover(self) -> int:
|
|
handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy)
|
|
return await handler(RecoverExpiredLeases(100))
|
|
|
|
async def clean(self) -> int:
|
|
return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100))
|
|
|
|
|
|
async def pipeline_of(
|
|
http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60)
|
|
) -> InlinePipeline:
|
|
database = await resolve(http, Database)
|
|
storage = await resolve(http, ObjectStorage)
|
|
return InlinePipeline(database, storage, policy=policy, lease=lease)
|
|
|
|
|
|
async def settle(http: AsyncClient) -> list[Outcome]:
|
|
return await (await pipeline_of(http)).drain()
|