Renew job leases, spread retries with jitter and add crash points
Leases: the worker renews its lease before the first side effect and every JOB_TIMEOUT_SECONDS/3 while the variants are produced; a renewal that finds the lease taken cancels the work (and its render process) and settles the job as fenced without writing anything. Retries: the delay before attempt n+1 is drawn from [d/2, d] with d = min(cap, base * 2^(n-1)) (equal jitter), the next attempt time is computed by PostgreSQL and the delay is kept in the job event. Faults: the Faults port is async; relay reaches after_commit_before_publish and after_publish_before_confirm, worker reaches after_claim_before_side_effect, stall_after_claim (once, FAULT_STALL_SECONDS), after_s3_write_before_commit and after_commit_before_ack. Crashes are os._exit(137). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f2242eb39d
commit
ecb4827e75
@@ -12,15 +12,15 @@
|
||||
|
||||
## 3. Повторы и аренда
|
||||
|
||||
- [ ] 3.1 `RetryPolicy.delay(attempt, spread)` с equal jitter, порт `Jitter` и адаптер, задержка в журнале задания
|
||||
- [ ] 3.2 `JobRepository.renew`, продление аренды на время побочных эффектов, проверка аренды до первой записи, отмена при потере аренды → `fenced`
|
||||
- [ ] 3.3 Проверка группы: unit-тесты границ jitter; тесты против PostgreSQL — долгий рендер не теряет аренду, потерянная аренда не пишет объектов
|
||||
- [x] 3.1 `RetryPolicy.delay(attempt, spread)` с equal jitter, порт `Jitter` и адаптер, задержка в журнале задания
|
||||
- [x] 3.2 `JobRepository.renew`, продление аренды на время побочных эффектов, проверка аренды до первой записи, отмена при потере аренды → `fenced`
|
||||
- [x] 3.3 Проверка группы: unit-тесты границ jitter; тесты против PostgreSQL — долгий рендер не теряет аренду, потерянная аренда не пишет объектов
|
||||
|
||||
## 4. Точки отказа
|
||||
|
||||
- [ ] 4.1 Асинхронный порт `Faults`, адаптер с падением `os._exit(137)`, однократным зависанием и временной ошибкой; `FAULT_STALL_SECONDS`
|
||||
- [ ] 4.2 Точки в `PublishOutbox` (окна 2 и 3) и `ProcessJob` (после захвата, зависание, окна 4 и 5); `Faults` в провайдере relay
|
||||
- [ ] 4.3 Проверка группы: тесты — каждая точка вызывается в своём месте и порядке, после «падения» состояние совпадает с таблицей окон
|
||||
- [x] 4.1 Асинхронный порт `Faults`, адаптер с падением `os._exit(137)`, однократным зависанием и временной ошибкой; `FAULT_STALL_SECONDS`
|
||||
- [x] 4.2 Точки в `PublishOutbox` (окна 2 и 3) и `ProcessJob` (после захвата, зависание, окна 4 и 5); `Faults` в провайдере relay
|
||||
- [x] 4.3 Проверка группы: тесты — каждая точка вызывается в своём месте и порядке, после «падения» состояние совпадает с таблицей окон
|
||||
|
||||
## 5. Реестр, replay, retention
|
||||
|
||||
|
||||
@@ -11,19 +11,27 @@ from messenger.attachments.application.commands import (
|
||||
from messenger.attachments.application.ports import (
|
||||
AttachmentsUnitOfWork,
|
||||
Faults,
|
||||
Jitter,
|
||||
JobPublisher,
|
||||
ObjectStorage,
|
||||
VariantRenderer,
|
||||
)
|
||||
from messenger.attachments.domain.errors import UnreadableImage
|
||||
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
|
||||
from messenger.attachments.domain.events import JobSettled
|
||||
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy
|
||||
from messenger.attachments.domain.models import original_key
|
||||
from messenger.attachments.domain.variants import Variant, variant_key
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.ports import Clock, IdGenerator
|
||||
from messenger.shared.application.ports import IdGenerator
|
||||
|
||||
TRANSIENT_FAULT = "transient_error"
|
||||
TRANSIENT_ERROR = "transient_error"
|
||||
STALL_AFTER_CLAIM = "stall_after_claim"
|
||||
AFTER_COMMIT_BEFORE_PUBLISH = "after_commit_before_publish"
|
||||
AFTER_PUBLISH_BEFORE_CONFIRM = "after_publish_before_confirm"
|
||||
AFTER_CLAIM = "after_claim_before_side_effect"
|
||||
AFTER_S3_WRITE = "after_s3_write_before_commit"
|
||||
AFTER_COMMIT_BEFORE_ACK = "after_commit_before_ack"
|
||||
RENEWALS = 3
|
||||
|
||||
|
||||
def settled(
|
||||
@@ -45,7 +53,7 @@ def settled(
|
||||
|
||||
|
||||
class ProcessJobHandler:
|
||||
__slots__ = ("_clock", "_faults", "_ids", "_lease", "_policy", "_renderer", "_storage", "_uow")
|
||||
__slots__ = ("_faults", "_ids", "_jitter", "_lease", "_policy", "_renderer", "_storage", "_uow")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -54,8 +62,8 @@ class ProcessJobHandler:
|
||||
renderer: VariantRenderer,
|
||||
faults: Faults,
|
||||
*,
|
||||
clock: Clock,
|
||||
ids: IdGenerator,
|
||||
jitter: Jitter,
|
||||
policy: RetryPolicy,
|
||||
lease: timedelta,
|
||||
) -> None:
|
||||
@@ -63,8 +71,8 @@ class ProcessJobHandler:
|
||||
self._storage = storage
|
||||
self._renderer = renderer
|
||||
self._faults = faults
|
||||
self._clock = clock
|
||||
self._ids = ids
|
||||
self._jitter = jitter
|
||||
self._policy = policy
|
||||
self._lease = lease
|
||||
|
||||
@@ -77,19 +85,47 @@ class ProcessJobHandler:
|
||||
await self._uow.commit()
|
||||
if claim is None:
|
||||
return Outcome.DUPLICATE
|
||||
await self._faults.reach(AFTER_CLAIM)
|
||||
await self._faults.reach(STALL_AFTER_CLAIM)
|
||||
try:
|
||||
variants = await self._produce(claim)
|
||||
variants = await self._held(claim)
|
||||
except LeaseLost:
|
||||
return await self._fenced(claim.lease)
|
||||
except UnreadableImage as error:
|
||||
return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error)
|
||||
except Unavailable as error:
|
||||
return await self._retry(claim.lease, error)
|
||||
except Exception as error:
|
||||
return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error)
|
||||
return await self._complete(claim.lease, variants)
|
||||
await self._faults.reach(AFTER_S3_WRITE)
|
||||
outcome = await self._complete(claim.lease, variants)
|
||||
await self._faults.reach(AFTER_COMMIT_BEFORE_ACK)
|
||||
return outcome
|
||||
|
||||
async def _held(self, claim: Claim) -> list[Variant]:
|
||||
await self._renew(claim.lease)
|
||||
work = asyncio.ensure_future(self._produce(claim))
|
||||
try:
|
||||
while not work.done():
|
||||
await asyncio.wait({work}, timeout=self._lease.total_seconds() / RENEWALS)
|
||||
if not work.done():
|
||||
await self._renew(claim.lease)
|
||||
except BaseException:
|
||||
work.cancel()
|
||||
await asyncio.wait({work})
|
||||
raise
|
||||
return work.result()
|
||||
|
||||
async def _renew(self, lease: Lease) -> None:
|
||||
async with self._uow.transaction():
|
||||
held = await self._uow.jobs.renew(lease, self._lease)
|
||||
await self._uow.commit()
|
||||
if not held:
|
||||
raise LeaseLost("аренду задания забрал другой держатель")
|
||||
|
||||
async def _produce(self, claim: Claim) -> list[Variant]:
|
||||
job = claim.lease.job
|
||||
self._faults.reach(TRANSIENT_FAULT)
|
||||
await self._faults.reach(TRANSIENT_ERROR)
|
||||
data = await self._storage.read(claim.source_key)
|
||||
renditions = await self._renderer.render(data, job.pipeline_version)
|
||||
if claim.staged:
|
||||
@@ -123,14 +159,20 @@ class ProcessJobHandler:
|
||||
await self._uow.commit()
|
||||
return outcome
|
||||
|
||||
async def _fenced(self, lease: Lease) -> Outcome:
|
||||
async with self._uow.transaction():
|
||||
await self._uow.jobs.note(lease.job, Outcome.FENCED)
|
||||
self._uow.record(settled(lease.job, Outcome.FENCED))
|
||||
await self._uow.commit()
|
||||
return Outcome.FENCED
|
||||
|
||||
async def _retry(self, lease: Lease, error: Exception) -> Outcome:
|
||||
delay = self._policy.delay(lease.job.attempt)
|
||||
delay = self._policy.delay(lease.job.attempt, self._jitter.spread())
|
||||
if delay is None:
|
||||
return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error)
|
||||
async with self._uow.transaction():
|
||||
at = self._clock.now() + delay
|
||||
job = await self._uow.jobs.reschedule(
|
||||
lease, at, Outcome.RESCHEDULED, type(error).__name__
|
||||
lease, delay, Outcome.RESCHEDULED, type(error).__name__
|
||||
)
|
||||
outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED
|
||||
if job is None:
|
||||
@@ -151,22 +193,25 @@ class ProcessJobHandler:
|
||||
|
||||
|
||||
class PublishOutboxHandler:
|
||||
__slots__ = ("_publisher", "_uow")
|
||||
__slots__ = ("_faults", "_publisher", "_uow")
|
||||
|
||||
def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher) -> None:
|
||||
def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher, faults: Faults) -> None:
|
||||
self._uow = uow
|
||||
self._publisher = publisher
|
||||
self._faults = faults
|
||||
|
||||
async def __call__(self, command: PublishOutbox) -> int:
|
||||
failure: Unavailable | None = None
|
||||
published: list[int] = []
|
||||
async with self._uow.transaction():
|
||||
for entry in await self._uow.outbox.pending(command.limit):
|
||||
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
|
||||
try:
|
||||
await self._publisher.publish(entry.job_id, entry.body)
|
||||
except Unavailable as error:
|
||||
failure = error
|
||||
break
|
||||
await self._faults.reach(AFTER_PUBLISH_BEFORE_CONFIRM)
|
||||
published.append(entry.id)
|
||||
if published:
|
||||
await self._uow.outbox.published(published)
|
||||
@@ -177,25 +222,24 @@ class PublishOutboxHandler:
|
||||
|
||||
|
||||
class RecoverExpiredLeasesHandler:
|
||||
__slots__ = ("_clock", "_policy", "_uow")
|
||||
__slots__ = ("_policy", "_uow")
|
||||
|
||||
def __init__(self, uow: AttachmentsUnitOfWork, clock: Clock, policy: RetryPolicy) -> None:
|
||||
def __init__(self, uow: AttachmentsUnitOfWork, policy: RetryPolicy) -> None:
|
||||
self._uow = uow
|
||||
self._clock = clock
|
||||
self._policy = policy
|
||||
|
||||
async def __call__(self, command: RecoverExpiredLeases) -> int:
|
||||
async with self._uow.transaction():
|
||||
leases = await self._uow.jobs.expired(command.limit)
|
||||
for lease in leases:
|
||||
if self._policy.delay(lease.job.attempt) is None:
|
||||
if self._policy.exhausted(lease.job.attempt):
|
||||
await self._uow.jobs.fail(
|
||||
lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value
|
||||
)
|
||||
outcome = Outcome.FAILED
|
||||
else:
|
||||
await self._uow.jobs.reschedule(
|
||||
lease, self._clock.now(), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value
|
||||
lease, timedelta(0), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value
|
||||
)
|
||||
outcome = Outcome.LEASE_EXPIRED
|
||||
self._uow.record(settled(lease.job, outcome))
|
||||
|
||||
@@ -70,7 +70,12 @@ class JobPublisher(Protocol):
|
||||
|
||||
@runtime_checkable
|
||||
class Faults(Protocol):
|
||||
def reach(self, point: str) -> None: ...
|
||||
async def reach(self, point: str) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Jitter(Protocol):
|
||||
def spread(self) -> float: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
@@ -106,9 +111,11 @@ class JobRepository(Protocol):
|
||||
) -> Outcome: ...
|
||||
|
||||
async def reschedule(
|
||||
self, lease: Lease, at: datetime, kind: Outcome, error: str
|
||||
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
|
||||
) -> Job | None: ...
|
||||
|
||||
async def renew(self, lease: Lease, duration: timedelta) -> bool: ...
|
||||
|
||||
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: ...
|
||||
|
||||
async def note(self, job: Job, kind: Outcome) -> None: ...
|
||||
|
||||
@@ -33,7 +33,7 @@ class PipelineSettings(AttachmentsSettings):
|
||||
job_timeout_seconds: float = Field(default=60.0, ge=1, le=290)
|
||||
job_backoff_base_seconds: float = Field(default=2.0, gt=0, le=60)
|
||||
job_backoff_cap_seconds: float = Field(default=60.0, gt=0, le=3600)
|
||||
staging_retention_seconds: int = Field(default=86_400, ge=60)
|
||||
staging_retention_seconds: int = Field(default=86_400, ge=1)
|
||||
outbox_poll_seconds: float = Field(default=1.0, gt=0, le=10)
|
||||
outbox_batch: int = Field(default=50, ge=1, le=1000)
|
||||
sweep_interval_seconds: float = Field(default=5.0, gt=0, le=300)
|
||||
@@ -46,3 +46,4 @@ class PipelineSettings(AttachmentsSettings):
|
||||
render_memory_mb: int = Field(default=448, ge=0, le=8192)
|
||||
render_tasks_per_child: int = Field(default=100, ge=1, le=10_000)
|
||||
fault_point: str = ""
|
||||
fault_stall_seconds: float = Field(default=15.0, ge=0, le=300)
|
||||
|
||||
@@ -39,3 +39,7 @@ class UnreadableImage(AttachmentError):
|
||||
|
||||
class ImageTooDemanding(AttachmentError):
|
||||
pass
|
||||
|
||||
|
||||
class LeaseLost(AttachmentError):
|
||||
pass
|
||||
|
||||
@@ -67,7 +67,11 @@ class RetryPolicy:
|
||||
base_seconds: float
|
||||
cap_seconds: float
|
||||
|
||||
def delay(self, attempt: int) -> timedelta | None:
|
||||
if attempt >= self.max_attempts:
|
||||
def exhausted(self, attempt: int) -> bool:
|
||||
return attempt >= self.max_attempts
|
||||
|
||||
def delay(self, attempt: int, spread: float) -> timedelta | None:
|
||||
if self.exhausted(attempt):
|
||||
return None
|
||||
return timedelta(seconds=min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1)))
|
||||
ceiling = min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1))
|
||||
return timedelta(seconds=ceiling / 2 * (1 + spread))
|
||||
|
||||
@@ -31,6 +31,7 @@ from messenger.attachments.application.ports import (
|
||||
ChatMembership,
|
||||
Faults,
|
||||
ImageInspector,
|
||||
Jitter,
|
||||
JobPublisher,
|
||||
ObjectStorage,
|
||||
VariantRenderer,
|
||||
@@ -44,6 +45,7 @@ from messenger.attachments.infrastructure.images import (
|
||||
ProcessVariantRenderer,
|
||||
RenderLimits,
|
||||
)
|
||||
from messenger.attachments.infrastructure.jitter import RandomJitter
|
||||
from messenger.attachments.infrastructure.postgres import (
|
||||
PostgresAttachmentDirectory,
|
||||
PostgresAttachmentsUnitOfWork,
|
||||
@@ -166,6 +168,10 @@ class PipelineProvider(Provider):
|
||||
settings.job_backoff_cap_seconds,
|
||||
)
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
|
||||
return EnvironmentFaults(settings.fault_point, logger, settings.fault_stall_seconds)
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
|
||||
return PostgresAttachmentsUnitOfWork(database)
|
||||
@@ -185,8 +191,8 @@ class WorkerProvider(Provider):
|
||||
renderer.close()
|
||||
|
||||
@provide(scope=Scope.APP)
|
||||
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
|
||||
return EnvironmentFaults(settings.fault_point, logger)
|
||||
def jitter(self) -> Jitter:
|
||||
return RandomJitter()
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def command_bus(
|
||||
@@ -196,8 +202,8 @@ class WorkerProvider(Provider):
|
||||
renderer: VariantRenderer,
|
||||
faults: Faults,
|
||||
publisher: EventPublisher,
|
||||
clock: Clock,
|
||||
ids: IdGenerator,
|
||||
jitter: Jitter,
|
||||
policy: RetryPolicy,
|
||||
settings: PipelineSettings,
|
||||
) -> AttachmentsCommandBus:
|
||||
@@ -206,8 +212,8 @@ class WorkerProvider(Provider):
|
||||
storage,
|
||||
renderer,
|
||||
faults,
|
||||
clock=clock,
|
||||
ids=ids,
|
||||
jitter=jitter,
|
||||
policy=policy,
|
||||
lease=timedelta(seconds=settings.job_timeout_seconds),
|
||||
)
|
||||
@@ -229,13 +235,13 @@ class RelayProvider(Provider):
|
||||
uow: AttachmentsUnitOfWork,
|
||||
storage: ObjectStorage,
|
||||
jobs: JobPublisher,
|
||||
faults: Faults,
|
||||
publisher: EventPublisher,
|
||||
clock: Clock,
|
||||
policy: RetryPolicy,
|
||||
) -> AttachmentsCommandBus:
|
||||
handlers: dict[type[Command], CommandHandler] = {
|
||||
PublishOutbox: PublishOutboxHandler(uow, jobs),
|
||||
RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, clock, policy),
|
||||
PublishOutbox: PublishOutboxHandler(uow, jobs, faults),
|
||||
RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, policy),
|
||||
CleanStaging: CleanStagingHandler(uow, storage),
|
||||
}
|
||||
return AttachmentsCommandBus(handlers, uow, publisher)
|
||||
|
||||
@@ -1,26 +1,32 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
from messenger.attachments.application.pipeline import STALL_AFTER_CLAIM, TRANSIENT_ERROR
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.ports import Logger
|
||||
|
||||
TRANSIENT = frozenset({"transient_error"})
|
||||
CRASH_EXIT_CODE = 137
|
||||
|
||||
|
||||
class EnvironmentFaults:
|
||||
__slots__ = ("_logger", "_point")
|
||||
__slots__ = ("_logger", "_point", "_stall_seconds")
|
||||
|
||||
def __init__(self, point: str, logger: Logger) -> None:
|
||||
def __init__(self, point: str, logger: Logger, stall_seconds: float = 0.0) -> None:
|
||||
self._point = point
|
||||
self._logger = logger
|
||||
self._stall_seconds = stall_seconds
|
||||
|
||||
def reach(self, point: str) -> None:
|
||||
async def reach(self, point: str) -> None:
|
||||
if not self._point or point != self._point:
|
||||
return
|
||||
self._logger.warning("fault.injected", reason=point)
|
||||
if point in TRANSIENT:
|
||||
if point == TRANSIENT_ERROR:
|
||||
raise Unavailable("внедрённая временная ошибка")
|
||||
if point == STALL_AFTER_CLAIM:
|
||||
self._point = ""
|
||||
await asyncio.sleep(self._stall_seconds)
|
||||
return
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
os._exit(CRASH_EXIT_CODE)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
from secrets import SystemRandom
|
||||
|
||||
SOURCE = SystemRandom()
|
||||
|
||||
|
||||
class RandomJitter:
|
||||
__slots__ = ()
|
||||
|
||||
def spread(self) -> float:
|
||||
return SOURCE.random()
|
||||
@@ -118,19 +118,24 @@ RESCHEDULE = """
|
||||
WITH job AS (
|
||||
UPDATE jobs
|
||||
SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL,
|
||||
next_attempt_at = $3, last_error = $4, updated_at = now()
|
||||
next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now()
|
||||
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
|
||||
RETURNING job_id, attachment_id, attempt
|
||||
RETURNING job_id, attachment_id, attempt, next_attempt_at
|
||||
), moved AS (
|
||||
UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at)
|
||||
FROM job WHERE a.id = job.attachment_id AND a.status = 'processing'
|
||||
), audit AS (
|
||||
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
||||
SELECT job_id, attachment_id, $5, attempt,
|
||||
jsonb_build_object('error', $4::text, 'next_attempt_at', $3::timestamptz)
|
||||
jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at,
|
||||
'delay_seconds', $3::float8)
|
||||
FROM job
|
||||
)
|
||||
SELECT attempt FROM job
|
||||
SELECT attempt, next_attempt_at FROM job
|
||||
"""
|
||||
RENEW = """
|
||||
UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now()
|
||||
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
||||
"""
|
||||
ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)"
|
||||
FAIL = """
|
||||
@@ -310,16 +315,24 @@ class PostgresJobRepository:
|
||||
await self.note(job, outcome)
|
||||
return outcome
|
||||
|
||||
async def reschedule(self, lease: Lease, at: datetime, kind: Outcome, error: str) -> Job | None:
|
||||
attempt = await self._connection.fetchval(
|
||||
RESCHEDULE, lease.job.id, lease.token, at, error, kind.value
|
||||
async def reschedule(
|
||||
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
|
||||
) -> Job | None:
|
||||
row = await self._connection.fetchrow(
|
||||
RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value
|
||||
)
|
||||
if attempt is None:
|
||||
if row is None:
|
||||
return None
|
||||
job = lease.job.retried(attempt)
|
||||
await self._connection.execute(ENQUEUE, job.id, payload(job), at)
|
||||
job = lease.job.retried(row["attempt"])
|
||||
await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"])
|
||||
return job
|
||||
|
||||
async def renew(self, lease: Lease, duration: timedelta) -> bool:
|
||||
done = await self._connection.execute(
|
||||
RENEW, lease.job.id, lease.token, duration.total_seconds()
|
||||
)
|
||||
return done == "UPDATE 1"
|
||||
|
||||
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool:
|
||||
attempt = await self._connection.fetchval(
|
||||
FAIL, lease.job.id, lease.token, code.value, error
|
||||
|
||||
@@ -10,6 +10,7 @@ from httpx import AsyncClient
|
||||
from test_attachments import storage, trio, upload
|
||||
|
||||
from conftest import DATABASE, actor
|
||||
from messenger.attachments.application.pipeline import TRANSIENT_ERROR
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.errors import ImageTooDemanding
|
||||
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
||||
@@ -287,9 +288,16 @@ async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
|
||||
"WHERE attachment_id = $1",
|
||||
attachment_id,
|
||||
)
|
||||
assert 25 < row["wait"] <= 30
|
||||
assert 20 < row["wait"] <= 22.5
|
||||
(event,) = await rows(
|
||||
"SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events "
|
||||
"WHERE attachment_id = $1 AND kind = 'retry_scheduled'",
|
||||
attachment_id,
|
||||
)
|
||||
assert event["delay"] == 22.5
|
||||
|
||||
|
||||
class FailingFaults:
|
||||
def reach(self, point: str) -> None:
|
||||
raise Unavailable(point)
|
||||
async def reach(self, point: str) -> None:
|
||||
if point == TRANSIENT_ERROR:
|
||||
raise Unavailable(point)
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
import asyncio
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
from test_attachments import storage
|
||||
from test_pipeline import accepted, events, job_of, rows, state, variants
|
||||
|
||||
from fakes import NullLogger
|
||||
from messenger.attachments.application.pipeline import (
|
||||
AFTER_CLAIM,
|
||||
AFTER_COMMIT_BEFORE_ACK,
|
||||
AFTER_COMMIT_BEFORE_PUBLISH,
|
||||
AFTER_PUBLISH_BEFORE_CONFIRM,
|
||||
AFTER_S3_WRITE,
|
||||
STALL_AFTER_CLAIM,
|
||||
TRANSIENT_ERROR,
|
||||
)
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.jobs import Outcome
|
||||
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
|
||||
|
||||
WORKER_POINTS = [
|
||||
AFTER_CLAIM,
|
||||
STALL_AFTER_CLAIM,
|
||||
TRANSIENT_ERROR,
|
||||
AFTER_S3_WRITE,
|
||||
AFTER_COMMIT_BEFORE_ACK,
|
||||
]
|
||||
|
||||
|
||||
class Crash(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
class RecordingFaults:
|
||||
__slots__ = ("crash_at", "reached")
|
||||
|
||||
def __init__(self, crash_at: str = "") -> None:
|
||||
self.crash_at = crash_at
|
||||
self.reached: list[str] = []
|
||||
|
||||
async def reach(self, point: str) -> None:
|
||||
self.reached.append(point)
|
||||
if point == self.crash_at:
|
||||
raise Crash(point)
|
||||
|
||||
|
||||
class SlowRenderer:
|
||||
__slots__ = ("seconds",)
|
||||
|
||||
def __init__(self, seconds: float) -> None:
|
||||
self.seconds = seconds
|
||||
|
||||
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
|
||||
await asyncio.sleep(self.seconds)
|
||||
return await InlineRenderer().render(data, pipeline_version)
|
||||
|
||||
|
||||
class ExpiringFaults:
|
||||
__slots__ = ("attachment_id", "pipeline")
|
||||
|
||||
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
|
||||
self.pipeline = pipeline
|
||||
self.attachment_id = attachment_id
|
||||
|
||||
async def reach(self, point: str) -> None:
|
||||
if point == STALL_AFTER_CLAIM:
|
||||
await rows(
|
||||
"UPDATE jobs SET lease_until = now() - interval '1 second' "
|
||||
"WHERE attachment_id = $1",
|
||||
self.attachment_id,
|
||||
)
|
||||
await self.pipeline.recover()
|
||||
|
||||
|
||||
async def outbox(attachment_id: str) -> list[bool]:
|
||||
found = await rows(
|
||||
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
|
||||
"WHERE j.attachment_id = $1 ORDER BY o.id",
|
||||
attachment_id,
|
||||
)
|
||||
return [row["published"] for row in found]
|
||||
|
||||
|
||||
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
|
||||
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
|
||||
|
||||
|
||||
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client)
|
||||
faults = RecordingFaults()
|
||||
pipeline.faults = faults
|
||||
published = await pipeline.publish()
|
||||
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
|
||||
faults.reached.clear()
|
||||
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
|
||||
assert await pipeline.process(job) is Outcome.DONE
|
||||
assert faults.reached == WORKER_POINTS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
|
||||
async def test_relay_crash_leaves_the_row_for_the_next_relay(
|
||||
client: AsyncClient, point: str
|
||||
) -> None:
|
||||
pipeline = await pipeline_of(client)
|
||||
await pipeline.drain()
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline.faults = RecordingFaults(point)
|
||||
with pytest.raises(Crash):
|
||||
await pipeline.publish()
|
||||
assert await outbox(attachment_id) == [False]
|
||||
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
|
||||
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
|
||||
pipeline.faults = RecordingFaults()
|
||||
outcomes = await pipeline.drain()
|
||||
assert Outcome.DONE in outcomes
|
||||
assert await outbox(attachment_id) == [True]
|
||||
assert (await state(attachment_id))["status"] == "ready"
|
||||
|
||||
|
||||
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
|
||||
await pipeline.publish()
|
||||
job = await job_of(attachment_id)
|
||||
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
|
||||
with pytest.raises(Crash):
|
||||
await pipeline.process(job)
|
||||
objects = (await storage(client)).objects
|
||||
written = stored(objects, attachment_id)
|
||||
assert len(written) == 3
|
||||
assert await variants(attachment_id) == []
|
||||
assert (await state(attachment_id))["status"] == "processing"
|
||||
await rows(
|
||||
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
|
||||
attachment_id,
|
||||
)
|
||||
pipeline.faults = RecordingFaults()
|
||||
assert await pipeline.recover() >= 1
|
||||
await pipeline.drain()
|
||||
done = await state(attachment_id)
|
||||
assert (done["status"], done["attempt"]) == ("ready", 2)
|
||||
assert stored(objects, attachment_id) == written
|
||||
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
|
||||
|
||||
|
||||
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client)
|
||||
await pipeline.publish()
|
||||
job = await job_of(attachment_id)
|
||||
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
|
||||
with pytest.raises(Crash):
|
||||
await pipeline.process(job)
|
||||
before = await variants(attachment_id)
|
||||
assert (await state(attachment_id))["status"] == "ready"
|
||||
pipeline.faults = RecordingFaults()
|
||||
assert await pipeline.process(job) is Outcome.DUPLICATE
|
||||
assert await variants(attachment_id) == before
|
||||
|
||||
|
||||
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
|
||||
pipeline.renderer = SlowRenderer(2.5)
|
||||
await pipeline.publish()
|
||||
job = await job_of(attachment_id)
|
||||
|
||||
async def sweep() -> None:
|
||||
for _ in range(4):
|
||||
await asyncio.sleep(0.6)
|
||||
await pipeline.recover()
|
||||
|
||||
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
|
||||
assert outcome is Outcome.DONE
|
||||
done = await state(attachment_id)
|
||||
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
|
||||
assert "lease_expired" not in await events(attachment_id)
|
||||
|
||||
|
||||
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client)
|
||||
await pipeline.publish()
|
||||
job = await job_of(attachment_id)
|
||||
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
|
||||
assert await pipeline.process(job) is Outcome.FENCED
|
||||
objects = (await storage(client)).objects
|
||||
assert stored(objects, attachment_id) == []
|
||||
assert f"originals/{attachment_id}" not in objects
|
||||
assert (await state(attachment_id))["status"] == "queued"
|
||||
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
|
||||
pipeline.faults = RecordingFaults()
|
||||
await pipeline.drain()
|
||||
assert (await state(attachment_id))["status"] == "ready"
|
||||
|
||||
|
||||
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
|
||||
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
|
||||
started = asyncio.get_running_loop().time()
|
||||
await stall.reach(AFTER_CLAIM)
|
||||
await stall.reach(STALL_AFTER_CLAIM)
|
||||
await stall.reach(STALL_AFTER_CLAIM)
|
||||
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
|
||||
with pytest.raises(Unavailable):
|
||||
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
|
||||
|
||||
|
||||
def test_environment_fault_crashes_the_process_with_137() -> None:
|
||||
script = (
|
||||
"import asyncio\n"
|
||||
"from fakes import NullLogger\n"
|
||||
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
|
||||
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
|
||||
".reach('after_commit_before_ack'))\n"
|
||||
"print('survived')\n"
|
||||
)
|
||||
done = subprocess.run(
|
||||
[sys.executable, "-c", script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
env={"PYTHONPATH": ":".join(sys.path)},
|
||||
)
|
||||
assert (done.returncode, done.stdout) == (137, "")
|
||||
@@ -20,6 +20,7 @@ from messenger.attachments.application.pipeline import (
|
||||
)
|
||||
from messenger.attachments.application.ports import (
|
||||
Faults,
|
||||
Jitter,
|
||||
ObjectStorage,
|
||||
Rendition,
|
||||
VariantRenderer,
|
||||
@@ -30,15 +31,25 @@ from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.infrastructure.clock import MonotonicUtcClock
|
||||
from messenger.shared.infrastructure.ids import Uuid7Generator
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
IDLE_SECONDS = 0.1
|
||||
HALF = 0.5
|
||||
LIMITS = RenderLimits(10_485_760, 40_000_000)
|
||||
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
|
||||
|
||||
|
||||
class FixedJitter:
|
||||
__slots__ = ("_spread",)
|
||||
|
||||
def __init__(self, spread: float) -> None:
|
||||
self._spread = spread
|
||||
|
||||
def spread(self) -> float:
|
||||
return self._spread
|
||||
|
||||
|
||||
class InlineRenderer:
|
||||
__slots__ = ()
|
||||
|
||||
@@ -62,7 +73,16 @@ class CapturingPublisher:
|
||||
|
||||
|
||||
class InlinePipeline:
|
||||
__slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage")
|
||||
__slots__ = (
|
||||
"_database",
|
||||
"faults",
|
||||
"jitter",
|
||||
"lease",
|
||||
"policy",
|
||||
"publisher",
|
||||
"renderer",
|
||||
"storage",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -79,13 +99,14 @@ class InlinePipeline:
|
||||
self.publisher = CapturingPublisher()
|
||||
self.renderer: VariantRenderer = InlineRenderer()
|
||||
self.faults: Faults = EnvironmentFaults("", NullLogger())
|
||||
self.jitter: Jitter = FixedJitter(HALF)
|
||||
|
||||
def uow(self) -> PostgresAttachmentsUnitOfWork:
|
||||
return PostgresAttachmentsUnitOfWork(self._database)
|
||||
|
||||
async def publish(self) -> list[Job]:
|
||||
start = len(self.publisher.bodies)
|
||||
await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100))
|
||||
await PublishOutboxHandler(self.uow(), self.publisher, self.faults)(PublishOutbox(100))
|
||||
return [
|
||||
JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:]
|
||||
]
|
||||
@@ -96,8 +117,8 @@ class InlinePipeline:
|
||||
self.storage,
|
||||
self.renderer,
|
||||
self.faults,
|
||||
clock=MonotonicUtcClock(),
|
||||
ids=Uuid7Generator(),
|
||||
jitter=self.jitter,
|
||||
policy=self.policy,
|
||||
lease=self.lease,
|
||||
)
|
||||
@@ -117,7 +138,7 @@ class InlinePipeline:
|
||||
return outcomes
|
||||
|
||||
async def recover(self) -> int:
|
||||
handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy)
|
||||
handler = RecoverExpiredLeasesHandler(self.uow(), self.policy)
|
||||
return await handler(RecoverExpiredLeases(100))
|
||||
|
||||
async def clean(self) -> int:
|
||||
|
||||
@@ -8,6 +8,7 @@ import pytest
|
||||
from messenger.attachments.domain.jobs import Job, RetryPolicy
|
||||
from messenger.attachments.domain.variants import fit, variant_key
|
||||
from messenger.attachments.entrypoints.schemas import JobMessage
|
||||
from messenger.attachments.infrastructure.jitter import RandomJitter
|
||||
|
||||
SCHEMA = json.loads(
|
||||
(Path(__file__).resolve().parents[2] / "contracts" / "jobs.schema.json").read_text()
|
||||
@@ -34,16 +35,22 @@ def test_thumbnail_fits_the_box_rounding_down(
|
||||
assert fit(*size, 256) == fitted
|
||||
|
||||
|
||||
def test_retry_policy_doubles_up_to_the_cap_and_stops_at_the_limit() -> None:
|
||||
@pytest.mark.parametrize(("spread", "share"), [(0.0, 0.5), (0.5, 0.75), (0.999, 0.9995)])
|
||||
def test_retry_delay_has_equal_jitter_up_to_the_cap(spread: float, share: float) -> None:
|
||||
policy = RetryPolicy(max_attempts=5, base_seconds=2, cap_seconds=10)
|
||||
delays = [policy.delay(attempt) for attempt in range(1, 6)]
|
||||
assert delays == [
|
||||
timedelta(seconds=2),
|
||||
timedelta(seconds=4),
|
||||
timedelta(seconds=8),
|
||||
timedelta(seconds=10),
|
||||
None,
|
||||
]
|
||||
delays = [policy.delay(attempt, spread) for attempt in range(1, 6)]
|
||||
ceilings = [2, 4, 8, 10]
|
||||
assert delays[:4] == [timedelta(seconds=ceiling * share) for ceiling in ceilings]
|
||||
assert delays[4] is None
|
||||
assert policy.exhausted(5)
|
||||
assert not policy.exhausted(4)
|
||||
|
||||
|
||||
def test_jitter_spreads_retries_between_half_and_full_delay() -> None:
|
||||
policy, jitter = RetryPolicy(5, 2, 60), RandomJitter()
|
||||
delays = {policy.delay(3, jitter.spread()) for _ in range(200)}
|
||||
assert all(timedelta(seconds=4) <= delay < timedelta(seconds=8) for delay in delays if delay)
|
||||
assert len(delays) > 150
|
||||
|
||||
|
||||
def test_job_message_follows_the_contract_schema() -> None:
|
||||
|
||||
Reference in New Issue
Block a user