Renew job leases, spread retries with jitter and add crash points

Leases: the worker renews its lease before the first side effect and
every JOB_TIMEOUT_SECONDS/3 while the variants are produced; a renewal
that finds the lease taken cancels the work (and its render process)
and settles the job as fenced without writing anything.

Retries: the delay before attempt n+1 is drawn from [d/2, d] with
d = min(cap, base * 2^(n-1)) (equal jitter), the next attempt time is
computed by PostgreSQL and the delay is kept in the job event.

Faults: the Faults port is async; relay reaches after_commit_before_publish
and after_publish_before_confirm, worker reaches after_claim_before_side_effect,
stall_after_claim (once, FAULT_STALL_SECONDS), after_s3_write_before_commit
and after_commit_before_ack. Crashes are os._exit(137).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 03:08:31 +03:00
co-authored by Claude Opus 5.5
parent f2242eb39d
commit ecb4827e75
14 changed files with 432 additions and 70 deletions
@@ -12,15 +12,15 @@
## 3. Повторы и аренда
- [ ] 3.1 `RetryPolicy.delay(attempt, spread)` с equal jitter, порт `Jitter` и адаптер, задержка в журнале задания
- [ ] 3.2 `JobRepository.renew`, продление аренды на время побочных эффектов, проверка аренды до первой записи, отмена при потере аренды → `fenced`
- [ ] 3.3 Проверка группы: unit-тесты границ jitter; тесты против PostgreSQL — долгий рендер не теряет аренду, потерянная аренда не пишет объектов
- [x] 3.1 `RetryPolicy.delay(attempt, spread)` с equal jitter, порт `Jitter` и адаптер, задержка в журнале задания
- [x] 3.2 `JobRepository.renew`, продление аренды на время побочных эффектов, проверка аренды до первой записи, отмена при потере аренды → `fenced`
- [x] 3.3 Проверка группы: unit-тесты границ jitter; тесты против PostgreSQL — долгий рендер не теряет аренду, потерянная аренда не пишет объектов
## 4. Точки отказа
- [ ] 4.1 Асинхронный порт `Faults`, адаптер с падением `os._exit(137)`, однократным зависанием и временной ошибкой; `FAULT_STALL_SECONDS`
- [ ] 4.2 Точки в `PublishOutbox` (окна 2 и 3) и `ProcessJob` (после захвата, зависание, окна 4 и 5); `Faults` в провайдере relay
- [ ] 4.3 Проверка группы: тесты — каждая точка вызывается в своём месте и порядке, после «падения» состояние совпадает с таблицей окон
- [x] 4.1 Асинхронный порт `Faults`, адаптер с падением `os._exit(137)`, однократным зависанием и временной ошибкой; `FAULT_STALL_SECONDS`
- [x] 4.2 Точки в `PublishOutbox` (окна 2 и 3) и `ProcessJob` (после захвата, зависание, окна 4 и 5); `Faults` в провайдере relay
- [x] 4.3 Проверка группы: тесты — каждая точка вызывается в своём месте и порядке, после «падения» состояние совпадает с таблицей окон
## 5. Реестр, replay, retention
@@ -11,19 +11,27 @@ from messenger.attachments.application.commands import (
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
Faults,
Jitter,
JobPublisher,
ObjectStorage,
VariantRenderer,
)
from messenger.attachments.domain.errors import UnreadableImage
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
from messenger.attachments.domain.events import JobSettled
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy
from messenger.attachments.domain.models import original_key
from messenger.attachments.domain.variants import Variant, variant_key
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Clock, IdGenerator
from messenger.shared.application.ports import IdGenerator
TRANSIENT_FAULT = "transient_error"
TRANSIENT_ERROR = "transient_error"
STALL_AFTER_CLAIM = "stall_after_claim"
AFTER_COMMIT_BEFORE_PUBLISH = "after_commit_before_publish"
AFTER_PUBLISH_BEFORE_CONFIRM = "after_publish_before_confirm"
AFTER_CLAIM = "after_claim_before_side_effect"
AFTER_S3_WRITE = "after_s3_write_before_commit"
AFTER_COMMIT_BEFORE_ACK = "after_commit_before_ack"
RENEWALS = 3
def settled(
@@ -45,7 +53,7 @@ def settled(
class ProcessJobHandler:
__slots__ = ("_clock", "_faults", "_ids", "_lease", "_policy", "_renderer", "_storage", "_uow")
__slots__ = ("_faults", "_ids", "_jitter", "_lease", "_policy", "_renderer", "_storage", "_uow")
def __init__(
self,
@@ -54,8 +62,8 @@ class ProcessJobHandler:
renderer: VariantRenderer,
faults: Faults,
*,
clock: Clock,
ids: IdGenerator,
jitter: Jitter,
policy: RetryPolicy,
lease: timedelta,
) -> None:
@@ -63,8 +71,8 @@ class ProcessJobHandler:
self._storage = storage
self._renderer = renderer
self._faults = faults
self._clock = clock
self._ids = ids
self._jitter = jitter
self._policy = policy
self._lease = lease
@@ -77,19 +85,47 @@ class ProcessJobHandler:
await self._uow.commit()
if claim is None:
return Outcome.DUPLICATE
await self._faults.reach(AFTER_CLAIM)
await self._faults.reach(STALL_AFTER_CLAIM)
try:
variants = await self._produce(claim)
variants = await self._held(claim)
except LeaseLost:
return await self._fenced(claim.lease)
except UnreadableImage as error:
return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error)
except Unavailable as error:
return await self._retry(claim.lease, error)
except Exception as error:
return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error)
return await self._complete(claim.lease, variants)
await self._faults.reach(AFTER_S3_WRITE)
outcome = await self._complete(claim.lease, variants)
await self._faults.reach(AFTER_COMMIT_BEFORE_ACK)
return outcome
async def _held(self, claim: Claim) -> list[Variant]:
await self._renew(claim.lease)
work = asyncio.ensure_future(self._produce(claim))
try:
while not work.done():
await asyncio.wait({work}, timeout=self._lease.total_seconds() / RENEWALS)
if not work.done():
await self._renew(claim.lease)
except BaseException:
work.cancel()
await asyncio.wait({work})
raise
return work.result()
async def _renew(self, lease: Lease) -> None:
async with self._uow.transaction():
held = await self._uow.jobs.renew(lease, self._lease)
await self._uow.commit()
if not held:
raise LeaseLost("аренду задания забрал другой держатель")
async def _produce(self, claim: Claim) -> list[Variant]:
job = claim.lease.job
self._faults.reach(TRANSIENT_FAULT)
await self._faults.reach(TRANSIENT_ERROR)
data = await self._storage.read(claim.source_key)
renditions = await self._renderer.render(data, job.pipeline_version)
if claim.staged:
@@ -123,14 +159,20 @@ class ProcessJobHandler:
await self._uow.commit()
return outcome
async def _fenced(self, lease: Lease) -> Outcome:
async with self._uow.transaction():
await self._uow.jobs.note(lease.job, Outcome.FENCED)
self._uow.record(settled(lease.job, Outcome.FENCED))
await self._uow.commit()
return Outcome.FENCED
async def _retry(self, lease: Lease, error: Exception) -> Outcome:
delay = self._policy.delay(lease.job.attempt)
delay = self._policy.delay(lease.job.attempt, self._jitter.spread())
if delay is None:
return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error)
async with self._uow.transaction():
at = self._clock.now() + delay
job = await self._uow.jobs.reschedule(
lease, at, Outcome.RESCHEDULED, type(error).__name__
lease, delay, Outcome.RESCHEDULED, type(error).__name__
)
outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED
if job is None:
@@ -151,22 +193,25 @@ class ProcessJobHandler:
class PublishOutboxHandler:
__slots__ = ("_publisher", "_uow")
__slots__ = ("_faults", "_publisher", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher) -> None:
def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher, faults: Faults) -> None:
self._uow = uow
self._publisher = publisher
self._faults = faults
async def __call__(self, command: PublishOutbox) -> int:
failure: Unavailable | None = None
published: list[int] = []
async with self._uow.transaction():
for entry in await self._uow.outbox.pending(command.limit):
await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH)
try:
await self._publisher.publish(entry.job_id, entry.body)
except Unavailable as error:
failure = error
break
await self._faults.reach(AFTER_PUBLISH_BEFORE_CONFIRM)
published.append(entry.id)
if published:
await self._uow.outbox.published(published)
@@ -177,25 +222,24 @@ class PublishOutboxHandler:
class RecoverExpiredLeasesHandler:
__slots__ = ("_clock", "_policy", "_uow")
__slots__ = ("_policy", "_uow")
def __init__(self, uow: AttachmentsUnitOfWork, clock: Clock, policy: RetryPolicy) -> None:
def __init__(self, uow: AttachmentsUnitOfWork, policy: RetryPolicy) -> None:
self._uow = uow
self._clock = clock
self._policy = policy
async def __call__(self, command: RecoverExpiredLeases) -> int:
async with self._uow.transaction():
leases = await self._uow.jobs.expired(command.limit)
for lease in leases:
if self._policy.delay(lease.job.attempt) is None:
if self._policy.exhausted(lease.job.attempt):
await self._uow.jobs.fail(
lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value
)
outcome = Outcome.FAILED
else:
await self._uow.jobs.reschedule(
lease, self._clock.now(), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value
lease, timedelta(0), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value
)
outcome = Outcome.LEASE_EXPIRED
self._uow.record(settled(lease.job, outcome))
@@ -70,7 +70,12 @@ class JobPublisher(Protocol):
@runtime_checkable
class Faults(Protocol):
def reach(self, point: str) -> None: ...
async def reach(self, point: str) -> None: ...
@runtime_checkable
class Jitter(Protocol):
def spread(self) -> float: ...
@runtime_checkable
@@ -106,9 +111,11 @@ class JobRepository(Protocol):
) -> Outcome: ...
async def reschedule(
self, lease: Lease, at: datetime, kind: Outcome, error: str
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
) -> Job | None: ...
async def renew(self, lease: Lease, duration: timedelta) -> bool: ...
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: ...
async def note(self, job: Job, kind: Outcome) -> None: ...
+2 -1
View File
@@ -33,7 +33,7 @@ class PipelineSettings(AttachmentsSettings):
job_timeout_seconds: float = Field(default=60.0, ge=1, le=290)
job_backoff_base_seconds: float = Field(default=2.0, gt=0, le=60)
job_backoff_cap_seconds: float = Field(default=60.0, gt=0, le=3600)
staging_retention_seconds: int = Field(default=86_400, ge=60)
staging_retention_seconds: int = Field(default=86_400, ge=1)
outbox_poll_seconds: float = Field(default=1.0, gt=0, le=10)
outbox_batch: int = Field(default=50, ge=1, le=1000)
sweep_interval_seconds: float = Field(default=5.0, gt=0, le=300)
@@ -46,3 +46,4 @@ class PipelineSettings(AttachmentsSettings):
render_memory_mb: int = Field(default=448, ge=0, le=8192)
render_tasks_per_child: int = Field(default=100, ge=1, le=10_000)
fault_point: str = ""
fault_stall_seconds: float = Field(default=15.0, ge=0, le=300)
@@ -39,3 +39,7 @@ class UnreadableImage(AttachmentError):
class ImageTooDemanding(AttachmentError):
pass
class LeaseLost(AttachmentError):
pass
+7 -3
View File
@@ -67,7 +67,11 @@ class RetryPolicy:
base_seconds: float
cap_seconds: float
def delay(self, attempt: int) -> timedelta | None:
if attempt >= self.max_attempts:
def exhausted(self, attempt: int) -> bool:
return attempt >= self.max_attempts
def delay(self, attempt: int, spread: float) -> timedelta | None:
if self.exhausted(attempt):
return None
return timedelta(seconds=min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1)))
ceiling = min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1))
return timedelta(seconds=ceiling / 2 * (1 + spread))
+13 -7
View File
@@ -31,6 +31,7 @@ from messenger.attachments.application.ports import (
ChatMembership,
Faults,
ImageInspector,
Jitter,
JobPublisher,
ObjectStorage,
VariantRenderer,
@@ -44,6 +45,7 @@ from messenger.attachments.infrastructure.images import (
ProcessVariantRenderer,
RenderLimits,
)
from messenger.attachments.infrastructure.jitter import RandomJitter
from messenger.attachments.infrastructure.postgres import (
PostgresAttachmentDirectory,
PostgresAttachmentsUnitOfWork,
@@ -166,6 +168,10 @@ class PipelineProvider(Provider):
settings.job_backoff_cap_seconds,
)
@provide(scope=Scope.APP)
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
return EnvironmentFaults(settings.fault_point, logger, settings.fault_stall_seconds)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(database)
@@ -185,8 +191,8 @@ class WorkerProvider(Provider):
renderer.close()
@provide(scope=Scope.APP)
def faults(self, settings: PipelineSettings, logger: Logger) -> Faults:
return EnvironmentFaults(settings.fault_point, logger)
def jitter(self) -> Jitter:
return RandomJitter()
@provide(scope=Scope.REQUEST)
def command_bus(
@@ -196,8 +202,8 @@ class WorkerProvider(Provider):
renderer: VariantRenderer,
faults: Faults,
publisher: EventPublisher,
clock: Clock,
ids: IdGenerator,
jitter: Jitter,
policy: RetryPolicy,
settings: PipelineSettings,
) -> AttachmentsCommandBus:
@@ -206,8 +212,8 @@ class WorkerProvider(Provider):
storage,
renderer,
faults,
clock=clock,
ids=ids,
jitter=jitter,
policy=policy,
lease=timedelta(seconds=settings.job_timeout_seconds),
)
@@ -229,13 +235,13 @@ class RelayProvider(Provider):
uow: AttachmentsUnitOfWork,
storage: ObjectStorage,
jobs: JobPublisher,
faults: Faults,
publisher: EventPublisher,
clock: Clock,
policy: RetryPolicy,
) -> AttachmentsCommandBus:
handlers: dict[type[Command], CommandHandler] = {
PublishOutbox: PublishOutboxHandler(uow, jobs),
RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, clock, policy),
PublishOutbox: PublishOutboxHandler(uow, jobs, faults),
RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, policy),
CleanStaging: CleanStagingHandler(uow, storage),
}
return AttachmentsCommandBus(handlers, uow, publisher)
@@ -1,26 +1,32 @@
import asyncio
import os
import sys
from messenger.attachments.application.pipeline import STALL_AFTER_CLAIM, TRANSIENT_ERROR
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import Logger
TRANSIENT = frozenset({"transient_error"})
CRASH_EXIT_CODE = 137
class EnvironmentFaults:
__slots__ = ("_logger", "_point")
__slots__ = ("_logger", "_point", "_stall_seconds")
def __init__(self, point: str, logger: Logger) -> None:
def __init__(self, point: str, logger: Logger, stall_seconds: float = 0.0) -> None:
self._point = point
self._logger = logger
self._stall_seconds = stall_seconds
def reach(self, point: str) -> None:
async def reach(self, point: str) -> None:
if not self._point or point != self._point:
return
self._logger.warning("fault.injected", reason=point)
if point in TRANSIENT:
if point == TRANSIENT_ERROR:
raise Unavailable("внедрённая временная ошибка")
if point == STALL_AFTER_CLAIM:
self._point = ""
await asyncio.sleep(self._stall_seconds)
return
sys.stdout.flush()
sys.stderr.flush()
os._exit(CRASH_EXIT_CODE)
@@ -0,0 +1,10 @@
from secrets import SystemRandom
SOURCE = SystemRandom()
class RandomJitter:
__slots__ = ()
def spread(self) -> float:
return SOURCE.random()
@@ -118,19 +118,24 @@ RESCHEDULE = """
WITH job AS (
UPDATE jobs
SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL,
next_attempt_at = $3, last_error = $4, updated_at = now()
next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
RETURNING job_id, attachment_id, attempt
RETURNING job_id, attachment_id, attempt, next_attempt_at
), moved AS (
UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at)
FROM job WHERE a.id = job.attachment_id AND a.status = 'processing'
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, $5, attempt,
jsonb_build_object('error', $4::text, 'next_attempt_at', $3::timestamptz)
jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at,
'delay_seconds', $3::float8)
FROM job
)
SELECT attempt FROM job
SELECT attempt, next_attempt_at FROM job
"""
RENEW = """
UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now()
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
"""
ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)"
FAIL = """
@@ -310,16 +315,24 @@ class PostgresJobRepository:
await self.note(job, outcome)
return outcome
async def reschedule(self, lease: Lease, at: datetime, kind: Outcome, error: str) -> Job | None:
attempt = await self._connection.fetchval(
RESCHEDULE, lease.job.id, lease.token, at, error, kind.value
async def reschedule(
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
) -> Job | None:
row = await self._connection.fetchrow(
RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value
)
if attempt is None:
if row is None:
return None
job = lease.job.retried(attempt)
await self._connection.execute(ENQUEUE, job.id, payload(job), at)
job = lease.job.retried(row["attempt"])
await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"])
return job
async def renew(self, lease: Lease, duration: timedelta) -> bool:
done = await self._connection.execute(
RENEW, lease.job.id, lease.token, duration.total_seconds()
)
return done == "UPDATE 1"
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool:
attempt = await self._connection.fetchval(
FAIL, lease.job.id, lease.token, code.value, error
+11 -3
View File
@@ -10,6 +10,7 @@ from httpx import AsyncClient
from test_attachments import storage, trio, upload
from conftest import DATABASE, actor
from messenger.attachments.application.pipeline import TRANSIENT_ERROR
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.errors import ImageTooDemanding
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
@@ -287,9 +288,16 @@ async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None:
"WHERE attachment_id = $1",
attachment_id,
)
assert 25 < row["wait"] <= 30
assert 20 < row["wait"] <= 22.5
(event,) = await rows(
"SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events "
"WHERE attachment_id = $1 AND kind = 'retry_scheduled'",
attachment_id,
)
assert event["delay"] == 22.5
class FailingFaults:
def reach(self, point: str) -> None:
raise Unavailable(point)
async def reach(self, point: str) -> None:
if point == TRANSIENT_ERROR:
raise Unavailable(point)
+231
View File
@@ -0,0 +1,231 @@
import asyncio
import subprocess
import sys
from datetime import timedelta
import pytest
from httpx import AsyncClient
from test_attachments import storage
from test_pipeline import accepted, events, job_of, rows, state, variants
from fakes import NullLogger
from messenger.attachments.application.pipeline import (
AFTER_CLAIM,
AFTER_COMMIT_BEFORE_ACK,
AFTER_COMMIT_BEFORE_PUBLISH,
AFTER_PUBLISH_BEFORE_CONFIRM,
AFTER_S3_WRITE,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
)
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.jobs import Outcome
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.shared.application.errors import Unavailable
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
WORKER_POINTS = [
AFTER_CLAIM,
STALL_AFTER_CLAIM,
TRANSIENT_ERROR,
AFTER_S3_WRITE,
AFTER_COMMIT_BEFORE_ACK,
]
class Crash(BaseException):
pass
class RecordingFaults:
__slots__ = ("crash_at", "reached")
def __init__(self, crash_at: str = "") -> None:
self.crash_at = crash_at
self.reached: list[str] = []
async def reach(self, point: str) -> None:
self.reached.append(point)
if point == self.crash_at:
raise Crash(point)
class SlowRenderer:
__slots__ = ("seconds",)
def __init__(self, seconds: float) -> None:
self.seconds = seconds
async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]:
await asyncio.sleep(self.seconds)
return await InlineRenderer().render(data, pipeline_version)
class ExpiringFaults:
__slots__ = ("attachment_id", "pipeline")
def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None:
self.pipeline = pipeline
self.attachment_id = attachment_id
async def reach(self, point: str) -> None:
if point == STALL_AFTER_CLAIM:
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' "
"WHERE attachment_id = $1",
self.attachment_id,
)
await self.pipeline.recover()
async def outbox(attachment_id: str) -> list[bool]:
found = await rows(
"SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) "
"WHERE j.attachment_id = $1 ORDER BY o.id",
attachment_id,
)
return [row["published"] for row in found]
def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]:
return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/"))
async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
faults = RecordingFaults()
pipeline.faults = faults
published = await pipeline.publish()
assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]
faults.reached.clear()
(job,) = [job for job in published if str(job.attachment_id) == attachment_id]
assert await pipeline.process(job) is Outcome.DONE
assert faults.reached == WORKER_POINTS
@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM])
async def test_relay_crash_leaves_the_row_for_the_next_relay(
client: AsyncClient, point: str
) -> None:
pipeline = await pipeline_of(client)
await pipeline.drain()
_, _, attachment_id = await accepted(client)
pipeline.faults = RecordingFaults(point)
with pytest.raises(Crash):
await pipeline.publish()
assert await outbox(attachment_id) == [False]
sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body]
assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM)
pipeline.faults = RecordingFaults()
outcomes = await pipeline.drain()
assert Outcome.DONE in outcomes
assert await outbox(attachment_id) == [True]
assert (await state(attachment_id))["status"] == "ready"
async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=60))
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_S3_WRITE)
with pytest.raises(Crash):
await pipeline.process(job)
objects = (await storage(client)).objects
written = stored(objects, attachment_id)
assert len(written) == 3
assert await variants(attachment_id) == []
assert (await state(attachment_id))["status"] == "processing"
await rows(
"UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1",
attachment_id,
)
pipeline.faults = RecordingFaults()
assert await pipeline.recover() >= 1
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 2)
assert stored(objects, attachment_id) == written
assert sorted(row["object_key"] for row in await variants(attachment_id)) == written
async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK)
with pytest.raises(Crash):
await pipeline.process(job)
before = await variants(attachment_id)
assert (await state(attachment_id))["status"] == "ready"
pipeline.faults = RecordingFaults()
assert await pipeline.process(job) is Outcome.DUPLICATE
assert await variants(attachment_id) == before
async def test_long_render_keeps_its_lease(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client, lease=timedelta(seconds=1))
pipeline.renderer = SlowRenderer(2.5)
await pipeline.publish()
job = await job_of(attachment_id)
async def sweep() -> None:
for _ in range(4):
await asyncio.sleep(0.6)
await pipeline.recover()
outcome, _ = await asyncio.gather(pipeline.process(job), sweep())
assert outcome is Outcome.DONE
done = await state(attachment_id)
assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1)
assert "lease_expired" not in await events(attachment_id)
async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
await pipeline.publish()
job = await job_of(attachment_id)
pipeline.faults = ExpiringFaults(pipeline, attachment_id)
assert await pipeline.process(job) is Outcome.FENCED
objects = (await storage(client)).objects
assert stored(objects, attachment_id) == []
assert f"originals/{attachment_id}" not in objects
assert (await state(attachment_id))["status"] == "queued"
assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"]
pipeline.faults = RecordingFaults()
await pipeline.drain()
assert (await state(attachment_id))["status"] == "ready"
async def test_environment_faults_stall_once_and_fail_transiently() -> None:
stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3)
started = asyncio.get_running_loop().time()
await stall.reach(AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
await stall.reach(STALL_AFTER_CLAIM)
assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6
with pytest.raises(Unavailable):
await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR)
def test_environment_fault_crashes_the_process_with_137() -> None:
script = (
"import asyncio\n"
"from fakes import NullLogger\n"
"from messenger.attachments.infrastructure.faults import EnvironmentFaults\n"
"asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())"
".reach('after_commit_before_ack'))\n"
"print('survived')\n"
)
done = subprocess.run(
[sys.executable, "-c", script],
capture_output=True,
text=True,
check=False,
env={"PYTHONPATH": ":".join(sys.path)},
)
assert (done.returncode, done.stdout) == (137, "")
+26 -5
View File
@@ -20,6 +20,7 @@ from messenger.attachments.application.pipeline import (
)
from messenger.attachments.application.ports import (
Faults,
Jitter,
ObjectStorage,
Rendition,
VariantRenderer,
@@ -30,15 +31,25 @@ from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.clock import MonotonicUtcClock
from messenger.shared.infrastructure.ids import Uuid7Generator
from messenger.shared.infrastructure.postgres import Database
IDLE_SECONDS = 0.1
HALF = 0.5
LIMITS = RenderLimits(10_485_760, 40_000_000)
FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001)
class FixedJitter:
__slots__ = ("_spread",)
def __init__(self, spread: float) -> None:
self._spread = spread
def spread(self) -> float:
return self._spread
class InlineRenderer:
__slots__ = ()
@@ -62,7 +73,16 @@ class CapturingPublisher:
class InlinePipeline:
__slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage")
__slots__ = (
"_database",
"faults",
"jitter",
"lease",
"policy",
"publisher",
"renderer",
"storage",
)
def __init__(
self,
@@ -79,13 +99,14 @@ class InlinePipeline:
self.publisher = CapturingPublisher()
self.renderer: VariantRenderer = InlineRenderer()
self.faults: Faults = EnvironmentFaults("", NullLogger())
self.jitter: Jitter = FixedJitter(HALF)
def uow(self) -> PostgresAttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(self._database)
async def publish(self) -> list[Job]:
start = len(self.publisher.bodies)
await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100))
await PublishOutboxHandler(self.uow(), self.publisher, self.faults)(PublishOutbox(100))
return [
JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:]
]
@@ -96,8 +117,8 @@ class InlinePipeline:
self.storage,
self.renderer,
self.faults,
clock=MonotonicUtcClock(),
ids=Uuid7Generator(),
jitter=self.jitter,
policy=self.policy,
lease=self.lease,
)
@@ -117,7 +138,7 @@ class InlinePipeline:
return outcomes
async def recover(self) -> int:
handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy)
handler = RecoverExpiredLeasesHandler(self.uow(), self.policy)
return await handler(RecoverExpiredLeases(100))
async def clean(self) -> int:
+16 -9
View File
@@ -8,6 +8,7 @@ import pytest
from messenger.attachments.domain.jobs import Job, RetryPolicy
from messenger.attachments.domain.variants import fit, variant_key
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.attachments.infrastructure.jitter import RandomJitter
SCHEMA = json.loads(
(Path(__file__).resolve().parents[2] / "contracts" / "jobs.schema.json").read_text()
@@ -34,16 +35,22 @@ def test_thumbnail_fits_the_box_rounding_down(
assert fit(*size, 256) == fitted
def test_retry_policy_doubles_up_to_the_cap_and_stops_at_the_limit() -> None:
@pytest.mark.parametrize(("spread", "share"), [(0.0, 0.5), (0.5, 0.75), (0.999, 0.9995)])
def test_retry_delay_has_equal_jitter_up_to_the_cap(spread: float, share: float) -> None:
policy = RetryPolicy(max_attempts=5, base_seconds=2, cap_seconds=10)
delays = [policy.delay(attempt) for attempt in range(1, 6)]
assert delays == [
timedelta(seconds=2),
timedelta(seconds=4),
timedelta(seconds=8),
timedelta(seconds=10),
None,
]
delays = [policy.delay(attempt, spread) for attempt in range(1, 6)]
ceilings = [2, 4, 8, 10]
assert delays[:4] == [timedelta(seconds=ceiling * share) for ceiling in ceilings]
assert delays[4] is None
assert policy.exhausted(5)
assert not policy.exhausted(4)
def test_jitter_spreads_retries_between_half_and_full_delay() -> None:
policy, jitter = RetryPolicy(5, 2, 60), RandomJitter()
delays = {policy.delay(3, jitter.spread()) for _ in range(200)}
assert all(timedelta(seconds=4) <= delay < timedelta(seconds=8) for delay in delays if delay)
assert len(delays) > 150
def test_job_message_follows_the_contract_schema() -> None: