python -m messenger.admin is a new process assembly over the attachments zone with only a database: "failed" lists failed jobs (served by the new partial index jobs_failed_idx), "replay --attachment-id" moves a failed attachment back to queued with the same job_id, attempt 1 and a fresh outbox row, writing a replayed event with the operator. A ready, queued or processing attachment is a noop (replay_noop event), a failed one whose staging is already gone is expired. Every request is audited. The relay now also removes the staging object of a failed attachment once STAGING_RETENTION_SECONDS have passed since it failed; replay and the cleanup take the same row lock. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
302 lines
11 KiB
Python
302 lines
11 KiB
Python
import asyncio
|
|
from datetime import timedelta
|
|
from hashlib import sha256
|
|
|
|
from messenger.attachments.application.commands import (
|
|
CleanStaging,
|
|
ProcessJob,
|
|
PublishOutbox,
|
|
RecoverExpiredLeases,
|
|
ReplayAttachment,
|
|
)
|
|
from messenger.attachments.application.ports import (
|
|
AttachmentsUnitOfWork,
|
|
Faults,
|
|
Jitter,
|
|
JobPublisher,
|
|
ObjectStorage,
|
|
VariantRenderer,
|
|
)
|
|
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
|
|
from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled
|
|
from messenger.attachments.domain.jobs import (
|
|
Claim,
|
|
FailureCode,
|
|
Job,
|
|
Lease,
|
|
Outcome,
|
|
ReplayResult,
|
|
RetryPolicy,
|
|
)
|
|
from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key
|
|
from messenger.attachments.domain.variants import Variant, variant_key
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.application.ports import IdGenerator
|
|
|
|
TRANSIENT_ERROR = "transient_error"
|
|
STALL_AFTER_CLAIM = "stall_after_claim"
|
|
AFTER_COMMIT_BEFORE_PUBLISH = "after_commit_before_publish"
|
|
AFTER_PUBLISH_BEFORE_CONFIRM = "after_publish_before_confirm"
|
|
AFTER_CLAIM = "after_claim_before_side_effect"
|
|
AFTER_S3_WRITE = "after_s3_write_before_commit"
|
|
AFTER_COMMIT_BEFORE_ACK = "after_commit_before_ack"
|
|
RENEWALS = 3
|
|
|
|
|
|
def settled(
|
|
job: Job,
|
|
outcome: Outcome,
|
|
*,
|
|
error_code: str | None = None,
|
|
delay_seconds: float | None = None,
|
|
) -> JobSettled:
|
|
return JobSettled(
|
|
job_id=job.id,
|
|
attachment_id=job.attachment_id,
|
|
pipeline_version=job.pipeline_version,
|
|
attempt=job.attempt,
|
|
outcome=outcome.value,
|
|
error_code=error_code,
|
|
delay_seconds=delay_seconds,
|
|
)
|
|
|
|
|
|
class ProcessJobHandler:
|
|
__slots__ = ("_faults", "_ids", "_jitter", "_lease", "_policy", "_renderer", "_storage", "_uow")
|
|
|
|
def __init__(
|
|
self,
|
|
uow: AttachmentsUnitOfWork,
|
|
storage: ObjectStorage,
|
|
renderer: VariantRenderer,
|
|
faults: Faults,
|
|
*,
|
|
ids: IdGenerator,
|
|
jitter: Jitter,
|
|
policy: RetryPolicy,
|
|
lease: timedelta,
|
|
) -> None:
|
|
self._uow = uow
|
|
self._storage = storage
|
|
self._renderer = renderer
|
|
self._faults = faults
|
|
self._ids = ids
|
|
self._jitter = jitter
|
|
self._policy = policy
|
|
self._lease = lease
|
|
|
|
async def __call__(self, command: ProcessJob) -> Outcome:
|
|
async with self._uow.transaction():
|
|
claim = await self._uow.jobs.claim(command.job, self._ids.new_id(), self._lease)
|
|
if claim is None:
|
|
await self._uow.jobs.note(command.job, Outcome.DUPLICATE)
|
|
self._uow.record(settled(command.job, Outcome.DUPLICATE))
|
|
await self._uow.commit()
|
|
if claim is None:
|
|
return Outcome.DUPLICATE
|
|
await self._faults.reach(AFTER_CLAIM)
|
|
await self._faults.reach(STALL_AFTER_CLAIM)
|
|
try:
|
|
variants = await self._held(claim)
|
|
except LeaseLost:
|
|
return await self._fenced(claim.lease)
|
|
except UnreadableImage as error:
|
|
return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error)
|
|
except Unavailable as error:
|
|
return await self._retry(claim.lease, error)
|
|
except Exception as error:
|
|
return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error)
|
|
await self._faults.reach(AFTER_S3_WRITE)
|
|
outcome = await self._complete(claim.lease, variants)
|
|
await self._faults.reach(AFTER_COMMIT_BEFORE_ACK)
|
|
return outcome
|
|
|
|
async def _held(self, claim: Claim) -> list[Variant]:
|
|
await self._renew(claim.lease)
|
|
work = asyncio.ensure_future(self._produce(claim))
|
|
try:
|
|
while not work.done():
|
|
await asyncio.wait({work}, timeout=self._lease.total_seconds() / RENEWALS)
|
|
if not work.done():
|
|
await self._renew(claim.lease)
|
|
except BaseException:
|
|
work.cancel()
|
|
await asyncio.wait({work})
|
|
raise
|
|
return work.result()
|
|
|
|
async def _renew(self, lease: Lease) -> None:
|
|
async with self._uow.transaction():
|
|
held = await self._uow.jobs.renew(lease, self._lease)
|
|
await self._uow.commit()
|
|
if not held:
|
|
raise LeaseLost("аренду задания забрал другой держатель")
|
|
|
|
async def _produce(self, claim: Claim) -> list[Variant]:
|
|
job = claim.lease.job
|
|
await self._faults.reach(TRANSIENT_ERROR)
|
|
data = await self._storage.read(claim.source_key)
|
|
renditions = await self._renderer.render(data, job.pipeline_version)
|
|
if claim.staged:
|
|
await self._storage.copy(claim.source_key, original_key(job.attachment_id))
|
|
variants: list[Variant] = []
|
|
for rendition in renditions:
|
|
digest = sha256(rendition.data).hexdigest()
|
|
key = variant_key(
|
|
job.attachment_id, job.pipeline_version, rendition.name, digest, rendition.extension
|
|
)
|
|
await self._storage.put(key, rendition.data, rendition.content_type)
|
|
variants.append(
|
|
Variant(
|
|
name=rendition.name,
|
|
content_type=rendition.content_type,
|
|
size_bytes=len(rendition.data),
|
|
width=rendition.width,
|
|
height=rendition.height,
|
|
sha256=digest,
|
|
object_key=key,
|
|
)
|
|
)
|
|
return variants
|
|
|
|
async def _complete(self, lease: Lease, variants: list[Variant]) -> Outcome:
|
|
async with self._uow.transaction():
|
|
outcome = await self._uow.jobs.complete(
|
|
lease, variants, original_key(lease.job.attachment_id)
|
|
)
|
|
self._uow.record(settled(lease.job, outcome))
|
|
await self._uow.commit()
|
|
return outcome
|
|
|
|
async def _fenced(self, lease: Lease) -> Outcome:
|
|
async with self._uow.transaction():
|
|
await self._uow.jobs.note(lease.job, Outcome.FENCED)
|
|
self._uow.record(settled(lease.job, Outcome.FENCED))
|
|
await self._uow.commit()
|
|
return Outcome.FENCED
|
|
|
|
async def _retry(self, lease: Lease, error: Exception) -> Outcome:
|
|
delay = self._policy.delay(lease.job.attempt, self._jitter.spread())
|
|
if delay is None:
|
|
return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error)
|
|
async with self._uow.transaction():
|
|
job = await self._uow.jobs.reschedule(
|
|
lease, delay, Outcome.RESCHEDULED, type(error).__name__
|
|
)
|
|
outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED
|
|
if job is None:
|
|
await self._uow.jobs.note(lease.job, outcome)
|
|
self._uow.record(settled(lease.job, outcome, delay_seconds=delay.total_seconds()))
|
|
await self._uow.commit()
|
|
return outcome
|
|
|
|
async def _fail(self, lease: Lease, code: FailureCode, error: Exception) -> Outcome:
|
|
async with self._uow.transaction():
|
|
done = await self._uow.jobs.fail(lease, code, type(error).__name__)
|
|
outcome = Outcome.FAILED if done else Outcome.FENCED
|
|
if not done:
|
|
await self._uow.jobs.note(lease.job, outcome)
|
|
self._uow.record(settled(lease.job, outcome, error_code=code.value))
|
|
await self._uow.commit()
|
|
return outcome
|
|
|
|
|
|
class PublishOutboxHandler:
|
|
__slots__ = ("_faults", "_publisher", "_uow")
|
|
|
|
def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher, faults: Faults) -> None:
|
|
self._uow = uow
|
|
self._publisher = publisher
|
|
self._faults = faults
|
|
|
|
async def __call__(self, command: PublishOutbox) -> int:
|
|
failure: Unavailable | None = None
|
|
published: list[int] = []
|
|
async with self._uow.transaction():
|
|
for entry in await self._uow.outbox.pending(command.limit):
|
|
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
|
|
try:
|
|
await self._publisher.publish(entry.job_id, entry.body)
|
|
except Unavailable as error:
|
|
failure = error
|
|
break
|
|
await self._faults.reach(AFTER_PUBLISH_BEFORE_CONFIRM)
|
|
published.append(entry.id)
|
|
if published:
|
|
await self._uow.outbox.published(published)
|
|
await self._uow.commit()
|
|
if failure is not None:
|
|
raise failure
|
|
return len(published)
|
|
|
|
|
|
class RecoverExpiredLeasesHandler:
|
|
__slots__ = ("_policy", "_uow")
|
|
|
|
def __init__(self, uow: AttachmentsUnitOfWork, policy: RetryPolicy) -> None:
|
|
self._uow = uow
|
|
self._policy = policy
|
|
|
|
async def __call__(self, command: RecoverExpiredLeases) -> int:
|
|
async with self._uow.transaction():
|
|
leases = await self._uow.jobs.expired(command.limit)
|
|
for lease in leases:
|
|
if self._policy.exhausted(lease.job.attempt):
|
|
await self._uow.jobs.fail(
|
|
lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value
|
|
)
|
|
outcome = Outcome.FAILED
|
|
else:
|
|
await self._uow.jobs.reschedule(
|
|
lease, timedelta(0), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value
|
|
)
|
|
outcome = Outcome.LEASE_EXPIRED
|
|
self._uow.record(settled(lease.job, outcome))
|
|
await self._uow.commit()
|
|
return len(leases)
|
|
|
|
|
|
class CleanStagingHandler:
|
|
__slots__ = ("_storage", "_uow")
|
|
|
|
def __init__(self, uow: AttachmentsUnitOfWork, storage: ObjectStorage) -> None:
|
|
self._uow = uow
|
|
self._storage = storage
|
|
|
|
async def __call__(self, command: CleanStaging) -> int:
|
|
async with self._uow.transaction():
|
|
staged = await self._uow.attachments.staged(command.limit, command.retention)
|
|
await asyncio.gather(*(self._storage.delete(item.key) for item in staged))
|
|
await self._uow.attachments.unstage([item.attachment_id for item in staged])
|
|
await self._uow.commit()
|
|
return len(staged)
|
|
|
|
|
|
class ReplayAttachmentHandler:
|
|
__slots__ = ("_uow",)
|
|
|
|
def __init__(self, uow: AttachmentsUnitOfWork) -> None:
|
|
self._uow = uow
|
|
|
|
async def __call__(self, command: ReplayAttachment) -> ReplayResult:
|
|
async with self._uow.transaction():
|
|
attachment = await self._uow.attachments.lock(command.attachment_id)
|
|
result = await self._replay(attachment, command.operator)
|
|
if attachment is not None and result is not ReplayResult.REPLAYED:
|
|
await self._uow.jobs.note_replay(attachment.id, command.operator, result.value)
|
|
self._uow.record(
|
|
AttachmentReplayRequested(command.attachment_id, command.operator, result.value)
|
|
)
|
|
await self._uow.commit()
|
|
return result
|
|
|
|
async def _replay(self, attachment: Attachment | None, operator: str) -> ReplayResult:
|
|
if attachment is None:
|
|
return ReplayResult.NOT_FOUND
|
|
if attachment.status is not AttachmentStatus.FAILED:
|
|
return ReplayResult.NOOP
|
|
if attachment.staging_key is None:
|
|
return ReplayResult.EXPIRED
|
|
job = await self._uow.jobs.replay(attachment, operator)
|
|
return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED
|