Files
mtusi2026lab2/src/messenger/attachments/application/pipeline.py
T
ArtyomandClaude Opus 5.5 b568fcf9c9 Add the failed registry, an explicit replay and failed staging retention
python -m messenger.admin is a new process assembly over the attachments
zone with only a database: "failed" lists failed jobs (served by the new
partial index jobs_failed_idx), "replay --attachment-id" moves a failed
attachment back to queued with the same job_id, attempt 1 and a fresh
outbox row, writing a replayed event with the operator. A ready, queued
or processing attachment is a noop (replay_noop event), a failed one
whose staging is already gone is expired. Every request is audited.

The relay now also removes the staging object of a failed attachment
once STAGING_RETENTION_SECONDS have passed since it failed; replay and
the cleanup take the same row lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:12:19 +03:00

302 lines
11 KiB
Python

import asyncio
from datetime import timedelta
from hashlib import sha256
from messenger.attachments.application.commands import (
CleanStaging,
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
)
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
Faults,
Jitter,
JobPublisher,
ObjectStorage,
VariantRenderer,
)
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled
from messenger.attachments.domain.jobs import (
Claim,
FailureCode,
Job,
Lease,
Outcome,
ReplayResult,
RetryPolicy,
)
from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key
from messenger.attachments.domain.variants import Variant, variant_key
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import IdGenerator
TRANSIENT_ERROR = "transient_error"
STALL_AFTER_CLAIM = "stall_after_claim"
AFTER_COMMIT_BEFORE_PUBLISH = "after_commit_before_publish"
AFTER_PUBLISH_BEFORE_CONFIRM = "after_publish_before_confirm"
AFTER_CLAIM = "after_claim_before_side_effect"
AFTER_S3_WRITE = "after_s3_write_before_commit"
AFTER_COMMIT_BEFORE_ACK = "after_commit_before_ack"
RENEWALS = 3
def settled(
job: Job,
outcome: Outcome,
*,
error_code: str | None = None,
delay_seconds: float | None = None,
) -> JobSettled:
return JobSettled(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt,
outcome=outcome.value,
error_code=error_code,
delay_seconds=delay_seconds,
)
class ProcessJobHandler:
__slots__ = ("_faults", "_ids", "_jitter", "_lease", "_policy", "_renderer", "_storage", "_uow")
def __init__(
self,
uow: AttachmentsUnitOfWork,
storage: ObjectStorage,
renderer: VariantRenderer,
faults: Faults,
*,
ids: IdGenerator,
jitter: Jitter,
policy: RetryPolicy,
lease: timedelta,
) -> None:
self._uow = uow
self._storage = storage
self._renderer = renderer
self._faults = faults
self._ids = ids
self._jitter = jitter
self._policy = policy
self._lease = lease
async def __call__(self, command: ProcessJob) -> Outcome:
async with self._uow.transaction():
claim = await self._uow.jobs.claim(command.job, self._ids.new_id(), self._lease)
if claim is None:
await self._uow.jobs.note(command.job, Outcome.DUPLICATE)
self._uow.record(settled(command.job, Outcome.DUPLICATE))
await self._uow.commit()
if claim is None:
return Outcome.DUPLICATE
await self._faults.reach(AFTER_CLAIM)
await self._faults.reach(STALL_AFTER_CLAIM)
try:
variants = await self._held(claim)
except LeaseLost:
return await self._fenced(claim.lease)
except UnreadableImage as error:
return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error)
except Unavailable as error:
return await self._retry(claim.lease, error)
except Exception as error:
return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error)
await self._faults.reach(AFTER_S3_WRITE)
outcome = await self._complete(claim.lease, variants)
await self._faults.reach(AFTER_COMMIT_BEFORE_ACK)
return outcome
async def _held(self, claim: Claim) -> list[Variant]:
await self._renew(claim.lease)
work = asyncio.ensure_future(self._produce(claim))
try:
while not work.done():
await asyncio.wait({work}, timeout=self._lease.total_seconds() / RENEWALS)
if not work.done():
await self._renew(claim.lease)
except BaseException:
work.cancel()
await asyncio.wait({work})
raise
return work.result()
async def _renew(self, lease: Lease) -> None:
async with self._uow.transaction():
held = await self._uow.jobs.renew(lease, self._lease)
await self._uow.commit()
if not held:
raise LeaseLost("аренду задания забрал другой держатель")
async def _produce(self, claim: Claim) -> list[Variant]:
job = claim.lease.job
await self._faults.reach(TRANSIENT_ERROR)
data = await self._storage.read(claim.source_key)
renditions = await self._renderer.render(data, job.pipeline_version)
if claim.staged:
await self._storage.copy(claim.source_key, original_key(job.attachment_id))
variants: list[Variant] = []
for rendition in renditions:
digest = sha256(rendition.data).hexdigest()
key = variant_key(
job.attachment_id, job.pipeline_version, rendition.name, digest, rendition.extension
)
await self._storage.put(key, rendition.data, rendition.content_type)
variants.append(
Variant(
name=rendition.name,
content_type=rendition.content_type,
size_bytes=len(rendition.data),
width=rendition.width,
height=rendition.height,
sha256=digest,
object_key=key,
)
)
return variants
async def _complete(self, lease: Lease, variants: list[Variant]) -> Outcome:
async with self._uow.transaction():
outcome = await self._uow.jobs.complete(
lease, variants, original_key(lease.job.attachment_id)
)
self._uow.record(settled(lease.job, outcome))
await self._uow.commit()
return outcome
async def _fenced(self, lease: Lease) -> Outcome:
async with self._uow.transaction():
await self._uow.jobs.note(lease.job, Outcome.FENCED)
self._uow.record(settled(lease.job, Outcome.FENCED))
await self._uow.commit()
return Outcome.FENCED
async def _retry(self, lease: Lease, error: Exception) -> Outcome:
delay = self._policy.delay(lease.job.attempt, self._jitter.spread())
if delay is None:
return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error)
async with self._uow.transaction():
job = await self._uow.jobs.reschedule(
lease, delay, Outcome.RESCHEDULED, type(error).__name__
)
outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED
if job is None:
await self._uow.jobs.note(lease.job, outcome)
self._uow.record(settled(lease.job, outcome, delay_seconds=delay.total_seconds()))
await self._uow.commit()
return outcome
async def _fail(self, lease: Lease, code: FailureCode, error: Exception) -> Outcome:
async with self._uow.transaction():
done = await self._uow.jobs.fail(lease, code, type(error).__name__)
outcome = Outcome.FAILED if done else Outcome.FENCED
if not done:
await self._uow.jobs.note(lease.job, outcome)
self._uow.record(settled(lease.job, outcome, error_code=code.value))
await self._uow.commit()
return outcome
class PublishOutboxHandler:
__slots__ = ("_faults", "_publisher", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher, faults: Faults) -> None:
self._uow = uow
self._publisher = publisher
self._faults = faults
async def __call__(self, command: PublishOutbox) -> int:
failure: Unavailable | None = None
published: list[int] = []
async with self._uow.transaction():
for entry in await self._uow.outbox.pending(command.limit):
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
try:
await self._publisher.publish(entry.job_id, entry.body)
except Unavailable as error:
failure = error
break
await self._faults.reach(AFTER_PUBLISH_BEFORE_CONFIRM)
published.append(entry.id)
if published:
await self._uow.outbox.published(published)
await self._uow.commit()
if failure is not None:
raise failure
return len(published)
class RecoverExpiredLeasesHandler:
__slots__ = ("_policy", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, policy: RetryPolicy) -> None:
self._uow = uow
self._policy = policy
async def __call__(self, command: RecoverExpiredLeases) -> int:
async with self._uow.transaction():
leases = await self._uow.jobs.expired(command.limit)
for lease in leases:
if self._policy.exhausted(lease.job.attempt):
await self._uow.jobs.fail(
lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value
)
outcome = Outcome.FAILED
else:
await self._uow.jobs.reschedule(
lease, timedelta(0), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value
)
outcome = Outcome.LEASE_EXPIRED
self._uow.record(settled(lease.job, outcome))
await self._uow.commit()
return len(leases)
class CleanStagingHandler:
__slots__ = ("_storage", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage) -> None:
self._uow = uow
self._storage = storage
async def __call__(self, command: CleanStaging) -> int:
async with self._uow.transaction():
staged = await self._uow.attachments.staged(command.limit, command.retention)
await asyncio.gather(*(self._storage.delete(item.key) for item in staged))
await self._uow.attachments.unstage([item.attachment_id for item in staged])
await self._uow.commit()
return len(staged)
class ReplayAttachmentHandler:
__slots__ = ("_uow",)
def __init__(self, uow: AttachmentsUnitOfWork) -> None:
self._uow = uow
async def __call__(self, command: ReplayAttachment) -> ReplayResult:
async with self._uow.transaction():
attachment = await self._uow.attachments.lock(command.attachment_id)
result = await self._replay(attachment, command.operator)
if attachment is not None and result is not ReplayResult.REPLAYED:
await self._uow.jobs.note_replay(attachment.id, command.operator, result.value)
self._uow.record(
AttachmentReplayRequested(command.attachment_id, command.operator, result.value)
)
await self._uow.commit()
return result
async def _replay(self, attachment: Attachment | None, operator: str) -> ReplayResult:
if attachment is None:
return ReplayResult.NOT_FOUND
if attachment.status is not AttachmentStatus.FAILED:
return ReplayResult.NOOP
if attachment.staging_key is None:
return ReplayResult.EXPIRED
job = await self._uow.jobs.replay(attachment, operator)
return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED