python -m messenger.admin is a new process assembly over the attachments zone with only a database: "failed" lists failed jobs (served by the new partial index jobs_failed_idx), "replay --attachment-id" moves a failed attachment back to queued with the same job_id, attempt 1 and a fresh outbox row, writing a replayed event with the operator. A ready, queued or processing attachment is a noop (replay_noop event), a failed one whose staging is already gone is expired. Every request is audited. The relay now also removes the staging object of a failed attachment once STAGING_RETENTION_SECONDS have passed since it failed; replay and the cleanup take the same row lock. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
269 lines
9.1 KiB
Python
269 lines
9.1 KiB
Python
from collections.abc import AsyncIterator, Iterator
|
|
from contextlib import asynccontextmanager
|
|
from datetime import timedelta
|
|
|
|
from dishka import Provider, Scope, provide
|
|
from faststream.rabbit import RabbitBroker
|
|
|
|
from messenger.attachments.application.bus import AttachmentsCommandBus
|
|
from messenger.attachments.application.commands import (
|
|
CleanStaging,
|
|
ProcessJob,
|
|
PublishOutbox,
|
|
RecoverExpiredLeases,
|
|
ReplayAttachment,
|
|
SweepAbandonedUploads,
|
|
UploadAttachment,
|
|
)
|
|
from messenger.attachments.application.handlers import (
|
|
SweepAbandonedUploadsHandler,
|
|
UploadAttachmentHandler,
|
|
UploadSlots,
|
|
)
|
|
from messenger.attachments.application.pipeline import (
|
|
CleanStagingHandler,
|
|
ProcessJobHandler,
|
|
PublishOutboxHandler,
|
|
RecoverExpiredLeasesHandler,
|
|
ReplayAttachmentHandler,
|
|
)
|
|
from messenger.attachments.application.ports import (
|
|
AttachmentDirectory,
|
|
AttachmentsUnitOfWork,
|
|
ChatMembership,
|
|
Faults,
|
|
ImageInspector,
|
|
Jitter,
|
|
JobPublisher,
|
|
ObjectStorage,
|
|
VariantRenderer,
|
|
)
|
|
from messenger.attachments.application.queries import GetAttachment, OpenAttachment
|
|
from messenger.attachments.config import AttachmentsSettings, PipelineSettings
|
|
from messenger.attachments.domain.jobs import RetryPolicy
|
|
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
|
from messenger.attachments.infrastructure.images import (
|
|
PillowImageInspector,
|
|
ProcessVariantRenderer,
|
|
RenderLimits,
|
|
)
|
|
from messenger.attachments.infrastructure.jitter import RandomJitter
|
|
from messenger.attachments.infrastructure.postgres import (
|
|
PostgresAttachmentDirectory,
|
|
PostgresAttachmentsUnitOfWork,
|
|
)
|
|
from messenger.attachments.infrastructure.rabbitmq import RabbitJobPublisher
|
|
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
|
|
from messenger.shared.application.commands import Command
|
|
from messenger.shared.application.messagebus import CommandHandler
|
|
from messenger.shared.application.ports import Clock, EventPublisher, IdGenerator, Logger
|
|
from messenger.shared.infrastructure.postgres import Database
|
|
|
|
|
|
@asynccontextmanager
|
|
async def object_storage(settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]:
|
|
async with s3_client(
|
|
endpoint=settings.s3_endpoint,
|
|
region=settings.s3_region,
|
|
access_key_id=settings.s3_access_key_id.get_secret_value(),
|
|
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
|
|
connect_timeout=settings.s3_connect_timeout_seconds,
|
|
read_timeout=settings.s3_read_timeout_seconds,
|
|
max_attempts=settings.s3_max_attempts,
|
|
) as client:
|
|
yield S3ObjectStorage(client, settings.s3_bucket)
|
|
|
|
|
|
class AttachmentsProvider(Provider):
|
|
def __init__(self, settings: AttachmentsSettings) -> None:
|
|
super().__init__()
|
|
self._settings = settings
|
|
|
|
@provide(scope=Scope.APP)
|
|
def settings(self) -> AttachmentsSettings:
|
|
return self._settings
|
|
|
|
@provide(scope=Scope.APP)
|
|
async def storage(self, settings: AttachmentsSettings) -> AsyncIterator[ObjectStorage]:
|
|
async with object_storage(settings) as storage:
|
|
yield storage
|
|
|
|
@provide(scope=Scope.APP)
|
|
def images(self, settings: AttachmentsSettings) -> ImageInspector:
|
|
return PillowImageInspector(settings.max_image_pixels, settings.image_check_concurrency)
|
|
|
|
@provide(scope=Scope.APP)
|
|
def slots(self, settings: AttachmentsSettings) -> UploadSlots:
|
|
return UploadSlots(settings.upload_concurrency)
|
|
|
|
@provide(scope=Scope.APP)
|
|
def directory(self, database: Database) -> AttachmentDirectory:
|
|
return PostgresAttachmentDirectory(database)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
|
|
return PostgresAttachmentsUnitOfWork(database)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def attachment(
|
|
self, directory: AttachmentDirectory, membership: ChatMembership
|
|
) -> GetAttachment:
|
|
return GetAttachment(directory, membership)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def content(
|
|
self, attachment: GetAttachment, storage: ObjectStorage, logger: Logger
|
|
) -> OpenAttachment:
|
|
return OpenAttachment(attachment, storage, logger)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def command_bus(
|
|
self,
|
|
uow: AttachmentsUnitOfWork,
|
|
membership: ChatMembership,
|
|
images: ImageInspector,
|
|
storage: ObjectStorage,
|
|
publisher: EventPublisher,
|
|
clock: Clock,
|
|
ids: IdGenerator,
|
|
settings: AttachmentsSettings,
|
|
slots: UploadSlots,
|
|
logger: Logger,
|
|
) -> AttachmentsCommandBus:
|
|
upload = UploadAttachmentHandler(
|
|
uow,
|
|
membership,
|
|
images,
|
|
storage,
|
|
clock=clock,
|
|
ids=ids,
|
|
limit=settings.max_upload_bytes,
|
|
slots=slots,
|
|
logger=logger,
|
|
)
|
|
handlers: dict[type[Command], CommandHandler] = {
|
|
UploadAttachment: upload,
|
|
SweepAbandonedUploads: SweepAbandonedUploadsHandler(uow, storage, clock),
|
|
}
|
|
return AttachmentsCommandBus(handlers, uow, publisher)
|
|
|
|
|
|
class PipelineProvider(Provider):
|
|
def __init__(self, settings: PipelineSettings) -> None:
|
|
super().__init__()
|
|
self._settings = settings
|
|
|
|
@provide(scope=Scope.APP)
|
|
def settings(self) -> PipelineSettings:
|
|
return self._settings
|
|
|
|
@provide(scope=Scope.APP)
|
|
async def storage(self, settings: PipelineSettings) -> AsyncIterator[ObjectStorage]:
|
|
async with object_storage(settings) as storage:
|
|
yield storage
|
|
|
|
@provide(scope=Scope.APP)
|
|
def policy(self, settings: PipelineSettings) -> RetryPolicy:
|
|
return RetryPolicy(
|
|
settings.job_max_attempts,
|
|
settings.job_backoff_base_seconds,
|
|
settings.job_backoff_cap_seconds,
|
|
)
|
|
|
|
@provide(scope=Scope.APP)
|
|
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
|
|
return EnvironmentFaults(settings.fault_point, logger, settings.fault_stall_seconds)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
|
|
return PostgresAttachmentsUnitOfWork(database)
|
|
|
|
|
|
class WorkerProvider(Provider):
|
|
@provide(scope=Scope.APP)
|
|
def renderer(self, settings: PipelineSettings) -> Iterator[VariantRenderer]:
|
|
renderer = ProcessVariantRenderer(
|
|
RenderLimits(settings.max_upload_bytes, settings.max_image_pixels),
|
|
concurrency=settings.worker_concurrency,
|
|
timeout=settings.render_timeout_seconds,
|
|
memory_bytes=settings.render_memory_mb << 20,
|
|
tasks_per_child=settings.render_tasks_per_child,
|
|
)
|
|
yield renderer
|
|
renderer.close()
|
|
|
|
@provide(scope=Scope.APP)
|
|
def jitter(self) -> Jitter:
|
|
return RandomJitter()
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def command_bus(
|
|
self,
|
|
uow: AttachmentsUnitOfWork,
|
|
storage: ObjectStorage,
|
|
renderer: VariantRenderer,
|
|
faults: Faults,
|
|
publisher: EventPublisher,
|
|
ids: IdGenerator,
|
|
jitter: Jitter,
|
|
policy: RetryPolicy,
|
|
settings: PipelineSettings,
|
|
) -> AttachmentsCommandBus:
|
|
process = ProcessJobHandler(
|
|
uow,
|
|
storage,
|
|
renderer,
|
|
faults,
|
|
ids=ids,
|
|
jitter=jitter,
|
|
policy=policy,
|
|
lease=timedelta(seconds=settings.job_timeout_seconds),
|
|
)
|
|
return AttachmentsCommandBus({ProcessJob: process}, uow, publisher)
|
|
|
|
|
|
class RelayProvider(Provider):
|
|
def __init__(self, broker: RabbitBroker) -> None:
|
|
super().__init__()
|
|
self._broker = broker
|
|
|
|
@provide(scope=Scope.APP)
|
|
def publisher(self, settings: PipelineSettings) -> JobPublisher:
|
|
return RabbitJobPublisher(self._broker, settings.publish_timeout_seconds)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def command_bus(
|
|
self,
|
|
uow: AttachmentsUnitOfWork,
|
|
storage: ObjectStorage,
|
|
jobs: JobPublisher,
|
|
faults: Faults,
|
|
publisher: EventPublisher,
|
|
policy: RetryPolicy,
|
|
) -> AttachmentsCommandBus:
|
|
handlers: dict[type[Command], CommandHandler] = {
|
|
PublishOutbox: PublishOutboxHandler(uow, jobs, faults),
|
|
RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, policy),
|
|
CleanStaging: CleanStagingHandler(uow, storage),
|
|
}
|
|
return AttachmentsCommandBus(handlers, uow, publisher)
|
|
|
|
|
|
class AdminProvider(Provider):
|
|
@provide(scope=Scope.APP)
|
|
def directory(self, database: Database) -> AttachmentDirectory:
|
|
return PostgresAttachmentDirectory(database)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
|
|
return PostgresAttachmentsUnitOfWork(database)
|
|
|
|
@provide(scope=Scope.REQUEST)
|
|
def command_bus(
|
|
self, uow: AttachmentsUnitOfWork, publisher: EventPublisher
|
|
) -> AttachmentsCommandBus:
|
|
handlers: dict[type[Command], CommandHandler] = {
|
|
ReplayAttachment: ReplayAttachmentHandler(uow)
|
|
}
|
|
return AttachmentsCommandBus(handlers, uow, publisher)
|