Files
mtusi2026lab5/src/messenger/attachments/entrypoints/ioc.py
T
ArtyomandClaude Opus 5.5 b568fcf9c9 Add the failed registry, an explicit replay and failed staging retention
python -m messenger.admin is a new process assembly over the attachments
zone with only a database: "failed" lists failed jobs (served by the new
partial index jobs_failed_idx), "replay --attachment-id" moves a failed
attachment back to queued with the same job_id, attempt 1 and a fresh
outbox row, writing a replayed event with the operator. A ready, queued
or processing attachment is a noop (replay_noop event), a failed one
whose staging is already gone is expired. Every request is audited.

The relay now also removes the staging object of a failed attachment
once STAGING_RETENTION_SECONDS have passed since it failed; replay and
the cleanup take the same row lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:12:19 +03:00

269 lines
9.1 KiB
Python

from collections.abc import AsyncIterator, Iterator
from contextlib import asynccontextmanager
from datetime import timedelta
from dishka import Provider, Scope, provide
from faststream.rabbit import RabbitBroker
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import (
CleanStaging,
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
SweepAbandonedUploads,
UploadAttachment,
)
from messenger.attachments.application.handlers import (
SweepAbandonedUploadsHandler,
UploadAttachmentHandler,
UploadSlots,
)
from messenger.attachments.application.pipeline import (
CleanStagingHandler,
ProcessJobHandler,
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
ReplayAttachmentHandler,
)
from messenger.attachments.application.ports import (
AttachmentDirectory,
AttachmentsUnitOfWork,
ChatMembership,
Faults,
ImageInspector,
Jitter,
JobPublisher,
ObjectStorage,
VariantRenderer,
)
from messenger.attachments.application.queries import GetAttachment, OpenAttachment
from messenger.attachments.config import AttachmentsSettings, PipelineSettings
from messenger.attachments.domain.jobs import RetryPolicy
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import (
PillowImageInspector,
ProcessVariantRenderer,
RenderLimits,
)
from messenger.attachments.infrastructure.jitter import RandomJitter
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentDirectory,
PostgresAttachmentsUnitOfWork,
)
from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.shared.application.commands import Command
from messenger.shared.application.messagebus import CommandHandler
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
from messenger.shared.infrastructure.postgres import Database
@asynccontextmanager
async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]:
async with s3_client(
endpoint=settings.s3_endpoint,
region=settings.s3_region,
access_key_id=settings.s3_access_key_id.get_secret_value(),
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
max_attempts=settings.s3_max_attempts,
) as client:
yield S3ObjectStorage(client, settings.s3_bucket)
class AttachmentsProvider(Provider):
def __init__(self, settings: AttachmentsSettings) -> None:
super().__init__()
self._settings = settings
@provide(scope=Scope.APP)
def settings(self) -> AttachmentsSettings:
return self._settings
@provide(scope=Scope.APP)
async def storage(self, settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]:
async with object_storage(settings) as storage:
yield storage
@provide(scope=Scope.APP)
def images(self, settings: AttachmentsSettings) -> ImageInspector:
return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency)
@provide(scope=Scope.APP)
def slots(self, settings: AttachmentsSettings) -> UploadSlots:
return UploadSlots(settings.upload_concurrency)
@provide(scope=Scope.APP)
def directory(self, database: Database) -> AttachmentDirectory:
return PostgresAttachmentDirectory(database)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(database)
@provide(scope=Scope.REQUEST)
def attachment(
self, directory: AttachmentDirectory, membership: ChatMembership
) -> GetAttachment:
return GetAttachment(directory, membership)
@provide(scope=Scope.REQUEST)
def content(
self, attachment: GetAttachment, storage: ObjectStorage, logger: Logger
) -> OpenAttachment:
return OpenAttachment(attachment, storage, logger)
@provide(scope=Scope.REQUEST)
def command_bus(
self,
uow: AttachmentsUnitOfWork,
membership: ChatMembership,
images: ImageInspector,
storage: ObjectStorage,
publisher: EventPublisher,
clock: Clock,
ids: IdGenerator,
settings: AttachmentsSettings,
slots: UploadSlots,
logger: Logger,
) -> AttachmentsCommandBus:
upload = UploadAttachmentHandler(
uow,
membership,
images,
storage,
clock=clock,
ids=ids,
limit=settings.max_upload_bytes,
slots=slots,
logger=logger,
)
handlers: dict[type[Command], CommandHandler] = {
UploadAttachment: upload,
SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock),
}
return AttachmentsCommandBus(handlers, uow, publisher)
class PipelineProvider(Provider):
def __init__(self, settings: PipelineSettings) -> None:
super().__init__()
self._settings = settings
@provide(scope=Scope.APP)
def settings(self) -> PipelineSettings:
return self._settings
@provide(scope=Scope.APP)
async def storage(self, settings: PipelineSettings) -> AsyncIterator[ObjectStorage]:
async with object_storage(settings) as storage:
yield storage
@provide(scope=Scope.APP)
def policy(self, settings: PipelineSettings) -> RetryPolicy:
return RetryPolicy(
settings.job_max_attempts,
settings.job_backoff_base_seconds,
settings.job_backoff_cap_seconds,
)
@provide(scope=Scope.APP)
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
return EnvironmentFaults(settings.fault_point, logger, settings.fault_stall_seconds)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(database)
class WorkerProvider(Provider):
@provide(scope=Scope.APP)
def renderer(self, settings: PipelineSettings) -> Iterator[VariantRenderer]:
renderer = ProcessVariantRenderer(
RenderLimits(settings.max_upload_bytes, settings.max_image_pixels),
concurrency=settings.worker_concurrency,
timeout=settings.render_timeout_seconds,
memory_bytes=settings.render_memory_mb << 20,
tasks_per_child=settings.render_tasks_per_child,
)
yield renderer
renderer.close()
@provide(scope=Scope.APP)
def jitter(self) -> Jitter:
return RandomJitter()
@provide(scope=Scope.REQUEST)
def command_bus(
self,
uow: AttachmentsUnitOfWork,
storage: ObjectStorage,
renderer: VariantRenderer,
faults: Faults,
publisher: EventPublisher,
ids: IdGenerator,
jitter: Jitter,
policy: RetryPolicy,
settings: PipelineSettings,
) -> AttachmentsCommandBus:
process = ProcessJobHandler(
uow,
storage,
renderer,
faults,
ids=ids,
jitter=jitter,
policy=policy,
lease=timedelta(seconds=settings.job_timeout_seconds),
)
return AttachmentsCommandBus({ProcessJob: process}, uow, publisher)
class RelayProvider(Provider):
def __init__(self, broker: RabbitBroker) -> None:
super().__init__()
self._broker = broker
@provide(scope=Scope.APP)
def publisher(self, settings: PipelineSettings) -> JobPublisher:
return RabbitJobPublisher(self._broker, settings.publish_timeout_seconds)
@provide(scope=Scope.REQUEST)
def command_bus(
self,
uow: AttachmentsUnitOfWork,
storage: ObjectStorage,
jobs: JobPublisher,
faults: Faults,
publisher: EventPublisher,
policy: RetryPolicy,
) -> AttachmentsCommandBus:
handlers: dict[type[Command], CommandHandler] = {
PublishOutbox: PublishOutboxHandler(uow, jobs, faults),
RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, policy),
CleanStaging: CleanStagingHandler(uow, storage),
}
return AttachmentsCommandBus(handlers, uow, publisher)
class AdminProvider(Provider):
@provide(scope=Scope.APP)
def directory(self, database: Database) -> AttachmentDirectory:
return PostgresAttachmentDirectory(database)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(database)
@provide(scope=Scope.REQUEST)
def command_bus(
self, uow: AttachmentsUnitOfWork, publisher: EventPublisher
) -> AttachmentsCommandBus:
handlers: dict[type[Command], CommandHandler] = {
ReplayAttachment: ReplayAttachmentHandler(uow)
}
return AttachmentsCommandBus(handlers, uow, publisher)