python -m messenger.admin is a new process assembly over the attachments zone with only a database: "failed" lists failed jobs (served by the new partial index jobs_failed_idx), "replay --attachment-id" moves a failed attachment back to queued with the same job_id, attempt 1 and a fresh outbox row, writing a replayed event with the operator. A ready, queued or processing attachment is a noop (replay_noop event), a failed one whose staging is already gone is expired. Every request is audited. The relay now also removes the staging object of a failed attachment once STAGING_RETENTION_SECONDS have passed since it failed; replay and the cleanup take the same row lock. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
165 lines
5.1 KiB
Python
165 lines
5.1 KiB
Python
import asyncio
|
|
from datetime import timedelta
|
|
from uuid import UUID
|
|
|
|
from httpx import AsyncClient
|
|
|
|
from conftest import resolve
|
|
from fakes import NullLogger
|
|
from messenger.attachments.application.commands import (
|
|
CleanStaging,
|
|
ProcessJob,
|
|
PublishOutbox,
|
|
RecoverExpiredLeases,
|
|
ReplayAttachment,
|
|
)
|
|
from messenger.attachments.application.pipeline import (
|
|
CleanStagingHandler,
|
|
ProcessJobHandler,
|
|
PublishOutboxHandler,
|
|
RecoverExpiredLeasesHandler,
|
|
ReplayAttachmentHandler,
|
|
)
|
|
from messenger.attachments.application.ports import (
|
|
Faults,
|
|
Jitter,
|
|
ObjectStorage,
|
|
Rendition,
|
|
VariantRenderer,
|
|
)
|
|
from messenger.attachments.domain.jobs import Job, Outcome, ReplayResult, RetryPolicy
|
|
from messenger.attachments.entrypoints.schemas import JobMessage
|
|
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
|
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
|
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.infrastructure.ids import Uuid7Generator
|
|
from messenger.shared.infrastructure.postgres import Database
|
|
|
|
IDLE_SECONDS = 0.1
|
|
HALF = 0.5
|
|
LIMITS = RenderLimits(10_485_760, 40_000_000)
|
|
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
|
|
|
|
|
|
class FixedJitter:
|
|
__slots__ = ("_spread",)
|
|
|
|
def __init__(self, spread: float) -> None:
|
|
self._spread = spread
|
|
|
|
def spread(self) -> float:
|
|
return self._spread
|
|
|
|
|
|
class InlineRenderer:
|
|
__slots__ = ()
|
|
|
|
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
|
return render_variants(data, LIMITS)
|
|
|
|
|
|
class CapturingPublisher:
|
|
__slots__ = ("bodies", "failing", "job_ids")
|
|
|
|
def __init__(self) -> None:
|
|
self.bodies: list[bytes] = []
|
|
self.job_ids: list[UUID] = []
|
|
self.failing = False
|
|
|
|
async def publish(self, job_id: UUID, body: bytes) -> None:
|
|
if self.failing:
|
|
raise Unavailable("брокер недоступен")
|
|
self.job_ids.append(job_id)
|
|
self.bodies.append(body)
|
|
|
|
|
|
class InlinePipeline:
|
|
__slots__ = (
|
|
"_database",
|
|
"faults",
|
|
"jitter",
|
|
"lease",
|
|
"policy",
|
|
"publisher",
|
|
"renderer",
|
|
"storage",
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
database: Database,
|
|
storage: ObjectStorage,
|
|
*,
|
|
policy: RetryPolicy = FAST,
|
|
lease: timedelta = timedelta(seconds=60),
|
|
) -> None:
|
|
self._database = database
|
|
self.storage = storage
|
|
self.policy = policy
|
|
self.lease = lease
|
|
self.publisher = CapturingPublisher()
|
|
self.renderer: VariantRenderer = InlineRenderer()
|
|
self.faults: Faults = EnvironmentFaults("", NullLogger())
|
|
self.jitter: Jitter = FixedJitter(HALF)
|
|
|
|
def uow(self) -> PostgresAttachmentsUnitOfWork:
|
|
return PostgresAttachmentsUnitOfWork(self._database)
|
|
|
|
async def publish(self) -> list[Job]:
|
|
start = len(self.publisher.bodies)
|
|
await PublishOutboxHandler(self.uow(), self.publisher, self.faults)(PublishOutbox(100))
|
|
return [
|
|
JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:]
|
|
]
|
|
|
|
async def process(self, job: Job) -> Outcome:
|
|
handler = ProcessJobHandler(
|
|
self.uow(),
|
|
self.storage,
|
|
self.renderer,
|
|
self.faults,
|
|
ids=Uuid7Generator(),
|
|
jitter=self.jitter,
|
|
policy=self.policy,
|
|
lease=self.lease,
|
|
)
|
|
return await handler(ProcessJob(job))
|
|
|
|
async def drain(self, rounds: int = 20, idle: int = 3) -> list[Outcome]:
|
|
outcomes: list[Outcome] = []
|
|
quiet = 0
|
|
for _ in range(rounds):
|
|
jobs = await self.publish()
|
|
quiet = 0 if jobs else quiet + 1
|
|
if quiet == idle:
|
|
return outcomes
|
|
if not jobs:
|
|
await asyncio.sleep(IDLE_SECONDS)
|
|
outcomes.extend([await self.process(job) for job in jobs])
|
|
return outcomes
|
|
|
|
async def recover(self) -> int:
|
|
handler = RecoverExpiredLeasesHandler(self.uow(), self.policy)
|
|
return await handler(RecoverExpiredLeases(100))
|
|
|
|
async def clean(self, retention: timedelta = timedelta(days=1)) -> int:
|
|
command = CleanStaging(100, retention)
|
|
return await CleanStagingHandler(self.uow(), self.storage)(command)
|
|
|
|
async def replay(self, attachment_id: UUID, operator: str = "test") -> ReplayResult:
|
|
command = ReplayAttachment(attachment_id, operator)
|
|
return await ReplayAttachmentHandler(self.uow())(command)
|
|
|
|
|
|
async def pipeline_of(
|
|
http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60)
|
|
) -> InlinePipeline:
|
|
database = await resolve(http, Database)
|
|
storage = await resolve(http, ObjectStorage)
|
|
return InlinePipeline(database, storage, policy=policy, lease=lease)
|
|
|
|
|
|
async def settle(http: AsyncClient) -> list[Outcome]:
|
|
return await (await pipeline_of(http)).drain()
|