Files
mtusi2026lab2/student-tests/pipeline.py
T

169 lines
5.3 KiB
Python

import asyncio
from datetime import timedelta
from uuid import UUID
from httpx import AsyncClient
from conftest import resolve
from fakes import NullLogger
from messenger.attachments.application.commands import (
CleanStaging,
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
)
from messenger.attachments.application.pipeline import (
CleanStagingHandler,
ProcessJobHandler,
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
ReplayAttachmentHandler,
)
from messenger.attachments.application.ports import (
Faults,
Jitter,
ObjectStorage,
Rendition,
VariantRenderer,
)
from messenger.attachments.domain.jobs import Job, Outcome, ReplayResult, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.ids import Uuid7Generator
from messenger.shared.infrastructure.postgres import Database
IDLE_SECONDS = 0.1
HALF = 0.5
LIMITS = RenderLimits(10_485_760, 40_000_000)
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
class FixedJitter:
__slots__ = ("_spread",)
def __init__(self, spread: float) -> None:
self._spread = spread
def spread(self) -> float:
return self._spread
class InlineRenderer:
__slots__ = ()
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
return render_variants(data, LIMITS, pipeline_version)
class CapturingPublisher:
__slots__ = ("bodies", "contexts", "failing", "job_ids")
def __init__(self) -> None:
self.bodies: list[bytes] = []
self.contexts: list[dict[str, str]] = []
self.job_ids: list[UUID] = []
self.failing = False
async def publish(
self, job_id: UUID, body: bytes, trace_context: dict[str, str] | None = None
) -> None:
if self.failing:
raise Unavailable("брокер недоступен")
self.job_ids.append(job_id)
self.bodies.append(body)
self.contexts.append(trace_context or {})
class InlinePipeline:
__slots__ = (
"_database",
"faults",
"jitter",
"lease",
"policy",
"publisher",
"renderer",
"storage",
)
def __init__(
self,
database: Database,
storage: ObjectStorage,
*,
policy: RetryPolicy = FAST,
lease: timedelta = timedelta(seconds=60),
) -> None:
self._database = database
self.storage = storage
self.policy = policy
self.lease = lease
self.publisher = CapturingPublisher()
self.renderer: VariantRenderer = InlineRenderer()
self.faults: Faults = EnvironmentFaults("", NullLogger())
self.jitter: Jitter = FixedJitter(HALF)
def uow(self) -> PostgresAttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(self._database)
async def publish(self) -> list[Job]:
start = len(self.publisher.bodies)
await PublishOutboxHandler(self.uow(), self.publisher, self.faults)(PublishOutbox(100))
return [
JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:]
]
async def process(self, job: Job) -> Outcome:
handler = ProcessJobHandler(
self.uow(),
self.storage,
self.renderer,
self.faults,
ids=Uuid7Generator(),
jitter=self.jitter,
policy=self.policy,
lease=self.lease,
)
return await handler(ProcessJob(job))
async def drain(self, rounds: int = 20, idle: int = 3) -> list[Outcome]:
outcomes: list[Outcome] = []
quiet = 0
for _ in range(rounds):
jobs = await self.publish()
quiet = 0 if jobs else quiet + 1
if quiet == idle:
return outcomes
if not jobs:
await asyncio.sleep(IDLE_SECONDS)
outcomes.extend([await self.process(job) for job in jobs])
return outcomes
async def recover(self) -> int:
handler = RecoverExpiredLeasesHandler(self.uow(), self.policy)
return await handler(RecoverExpiredLeases(100))
async def clean(self, retention: timedelta = timedelta(days=1)) -> int:
command = CleanStaging(100, retention)
return await CleanStagingHandler(self.uow(), self.storage)(command)
async def replay(self, attachment_id: UUID, operator: str = "test") -> ReplayResult:
command = ReplayAttachment(attachment_id, operator)
return await ReplayAttachmentHandler(self.uow())(command)
async def pipeline_of(
http: AsyncClient, *, policy: RetryPolicy = FAST, lease: timedelta = timedelta(seconds=60)
) -> InlinePipeline:
database = await resolve(http, Database)
storage = await resolve(http, ObjectStorage)
return InlinePipeline(database, storage, policy=policy, lease=lease)
async def settle(http: AsyncClient) -> list[Outcome]:
return await (await pipeline_of(http)).drain()