From ecb4827e75121e988b54e65c97ce86174e04dafd Mon Sep 17 00:00:00 2001 From: Artyom Date: Thu, 29 Oct 2026 03:08:31 +0300 Subject: [PATCH] Renew job leases, spread retries with jitter and add crash points Leases: the worker renews its lease before the first side effect and every JOB_TIMEOUT_SECONDS/3 while the variants are produced; a renewal that finds the lease taken cancels the work (and its render process) and settles the job as fenced without writing anything. Retries: the delay before attempt n+1 is drawn from [d/2, d] with d = min(cap, base * 2^(n-1)) (equal jitter), the next attempt time is computed by PostgreSQL and the delay is kept in the job event. Faults: the Faults port is async; relay reaches after_commit_before_publish and after_publish_before_confirm, worker reaches after_claim_before_side_effect, stall_after_claim (once, FAULT_STALL_SECONDS), after_s3_write_before_commit and after_commit_before_ack. Crashes are os._exit(137). Co-Authored-By: Claude Opus 5.5 (1M context) --- .../add-lab5-variants-and-recovery/tasks.md | 12 +- .../attachments/application/pipeline.py | 82 +++++-- .../attachments/application/ports.py | 11 +- src/messenger/attachments/config.py | 3 +- src/messenger/attachments/domain/errors.py | 4 + src/messenger/attachments/domain/jobs.py | 10 +- src/messenger/attachments/entrypoints/ioc.py | 20 +- .../attachments/infrastructure/faults.py | 16 +- .../attachments/infrastructure/jitter.py | 10 + .../attachments/infrastructure/postgres.py | 33 ++- student-tests/api/test_pipeline.py | 14 +- student-tests/api/test_recovery.py | 231 ++++++++++++++++++ student-tests/pipeline.py | 31 ++- student-tests/unit/test_pipeline_domain.py | 25 +- 14 files changed, 432 insertions(+), 70 deletions(-) create mode 100644 src/messenger/attachments/infrastructure/jitter.py create mode 100644 student-tests/api/test_recovery.py diff --git a/openspec/changes/add-lab5-variants-and-recovery/tasks.md b/openspec/changes/add-lab5-variants-and-recovery/tasks.md index 77029b0..d35d47b 100644 --- a/openspec/changes/add-lab5-variants-and-recovery/tasks.md +++ b/openspec/changes/add-lab5-variants-and-recovery/tasks.md @@ -12,15 +12,15 @@ ## 3. Повторы и аренда -- [ ] 3.1 `RetryPolicy.delay(attempt, spread)` с equal jitter, порт `Jitter` и адаптер, задержка в журнале задания -- [ ] 3.2 `JobRepository.renew`, продление аренды на время побочных эффектов, проверка аренды до первой записи, отмена при потере аренды → `fenced` -- [ ] 3.3 Проверка группы: unit-тесты границ jitter; тесты против PostgreSQL — долгий рендер не теряет аренду, потерянная аренда не пишет объектов +- [x] 3.1 `RetryPolicy.delay(attempt, spread)` с equal jitter, порт `Jitter` и адаптер, задержка в журнале задания +- [x] 3.2 `JobRepository.renew`, продление аренды на время побочных эффектов, проверка аренды до первой записи, отмена при потере аренды → `fenced` +- [x] 3.3 Проверка группы: unit-тесты границ jitter; тесты против PostgreSQL — долгий рендер не теряет аренду, потерянная аренда не пишет объектов ## 4. Точки отказа -- [ ] 4.1 Асинхронный порт `Faults`, адаптер с падением `os._exit(137)`, однократным зависанием и временной ошибкой; `FAULT_STALL_SECONDS` -- [ ] 4.2 Точки в `PublishOutbox` (окна 2 и 3) и `ProcessJob` (после захвата, зависание, окна 4 и 5); `Faults` в провайдере relay -- [ ] 4.3 Проверка группы: тесты — каждая точка вызывается в своём месте и порядке, после «падения» состояние совпадает с таблицей окон +- [x] 4.1 Асинхронный порт `Faults`, адаптер с падением `os._exit(137)`, однократным зависанием и временной ошибкой; `FAULT_STALL_SECONDS` +- [x] 4.2 Точки в `PublishOutbox` (окна 2 и 3) и `ProcessJob` (после захвата, зависание, окна 4 и 5); `Faults` в провайдере relay +- [x] 4.3 Проверка группы: тесты — каждая точка вызывается в своём месте и порядке, после «падения» состояние совпадает с таблицей окон ## 5. Реестр, replay, retention diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py index 803c096..cf6f20b 100644 --- a/src/messenger/attachments/application/pipeline.py +++ b/src/messenger/attachments/application/pipeline.py @@ -11,19 +11,27 @@ from messenger.attachments.application.commands import ( from messenger.attachments.application.ports import ( AttachmentsUnitOfWork, Faults, + Jitter, JobPublisher, ObjectStorage, VariantRenderer, ) -from messenger.attachments.domain.errors import UnreadableImage +from messenger.attachments.domain.errors import LeaseLost, UnreadableImage from messenger.attachments.domain.events import JobSettled from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy from messenger.attachments.domain.models import original_key from messenger.attachments.domain.variants import Variant, variant_key from messenger.shared.application.errors import Unavailable -from messenger.shared.application.ports import Clock, IdGenerator +from messenger.shared.application.ports import IdGenerator -TRANSIENT_FAULT = "transient_error" +TRANSIENT_ERROR = "transient_error" +STALL_AFTER_CLAIM = "stall_after_claim" +AFTER_COMMIT_BEFORE_PUBLISH = "after_commit_before_publish" +AFTER_PUBLISH_BEFORE_CONFIRM = "after_publish_before_confirm" +AFTER_CLAIM = "after_claim_before_side_effect" +AFTER_S3_WRITE = "after_s3_write_before_commit" +AFTER_COMMIT_BEFORE_ACK = "after_commit_before_ack" +RENEWALS = 3 def settled( @@ -45,7 +53,7 @@ def settled( class ProcessJobHandler: - __slots__ = ("_clock", "_faults", "_ids", "_lease", "_policy", "_renderer", "_storage", "_uow") + __slots__ = ("_faults", "_ids", "_jitter", "_lease", "_policy", "_renderer", "_storage", "_uow") def __init__( self, @@ -54,8 +62,8 @@ class ProcessJobHandler: renderer: VariantRenderer, faults: Faults, *, - clock: Clock, ids: IdGenerator, + jitter: Jitter, policy: RetryPolicy, lease: timedelta, ) -> None: @@ -63,8 +71,8 @@ class ProcessJobHandler: self._storage = storage self._renderer = renderer self._faults = faults - self._clock = clock self._ids = ids + self._jitter = jitter self._policy = policy self._lease = lease @@ -77,19 +85,47 @@ class ProcessJobHandler: await self._uow.commit() if claim is None: return Outcome.DUPLICATE + await self._faults.reach(AFTER_CLAIM) + await self._faults.reach(STALL_AFTER_CLAIM) try: - variants = await self._produce(claim) + variants = await self._held(claim) + except LeaseLost: + return await self._fenced(claim.lease) except UnreadableImage as error: return await self._fail(claim.lease, FailureCode.INVALID_IMAGE, error) except Unavailable as error: return await self._retry(claim.lease, error) except Exception as error: return await self._fail(claim.lease, FailureCode.PROCESSING_FAILED, error) - return await self._complete(claim.lease, variants) + await self._faults.reach(AFTER_S3_WRITE) + outcome = await self._complete(claim.lease, variants) + await self._faults.reach(AFTER_COMMIT_BEFORE_ACK) + return outcome + + async def _held(self, claim: Claim) -> list[Variant]: + await self._renew(claim.lease) + work = asyncio.ensure_future(self._produce(claim)) + try: + while not work.done(): + await asyncio.wait({work}, timeout=self._lease.total_seconds() / RENEWALS) + if not work.done(): + await self._renew(claim.lease) + except BaseException: + work.cancel() + await asyncio.wait({work}) + raise + return work.result() + + async def _renew(self, lease: Lease) -> None: + async with self._uow.transaction(): + held = await self._uow.jobs.renew(lease, self._lease) + await self._uow.commit() + if not held: + raise LeaseLost("аренду задания забрал другой держатель") async def _produce(self, claim: Claim) -> list[Variant]: job = claim.lease.job - self._faults.reach(TRANSIENT_FAULT) + await self._faults.reach(TRANSIENT_ERROR) data = await self._storage.read(claim.source_key) renditions = await self._renderer.render(data, job.pipeline_version) if claim.staged: @@ -123,14 +159,20 @@ class ProcessJobHandler: await self._uow.commit() return outcome + async def _fenced(self, lease: Lease) -> Outcome: + async with self._uow.transaction(): + await self._uow.jobs.note(lease.job, Outcome.FENCED) + self._uow.record(settled(lease.job, Outcome.FENCED)) + await self._uow.commit() + return Outcome.FENCED + async def _retry(self, lease: Lease, error: Exception) -> Outcome: - delay = self._policy.delay(lease.job.attempt) + delay = self._policy.delay(lease.job.attempt, self._jitter.spread()) if delay is None: return await self._fail(lease, FailureCode.RETRY_EXHAUSTED, error) async with self._uow.transaction(): - at = self._clock.now() + delay job = await self._uow.jobs.reschedule( - lease, at, Outcome.RESCHEDULED, type(error).__name__ + lease, delay, Outcome.RESCHEDULED, type(error).__name__ ) outcome = Outcome.FENCED if job is None else Outcome.RESCHEDULED if job is None: @@ -151,22 +193,25 @@ class ProcessJobHandler: class PublishOutboxHandler: - __slots__ = ("_publisher", "_uow") + __slots__ = ("_faults", "_publisher", "_uow") - def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher) -> None: + def __init__(self, uow: AttachmentsUnitOfWork, publisher: JobPublisher, faults: Faults) -> None: self._uow = uow self._publisher = publisher + self._faults = faults async def __call__(self, command: PublishOutbox) -> int: failure: Unavailable | None = None published: list[int] = [] async with self._uow.transaction(): for entry in await self._uow.outbox.pending(command.limit): + await self._faults.reach(AFTER_COMMIT_BEFORE_PUBLISH) try: await self._publisher.publish(entry.job_id, entry.body) except Unavailable as error: failure = error break + await self._faults.reach(AFTER_PUBLISH_BEFORE_CONFIRM) published.append(entry.id) if published: await self._uow.outbox.published(published) @@ -177,25 +222,24 @@ class PublishOutboxHandler: class RecoverExpiredLeasesHandler: - __slots__ = ("_clock", "_policy", "_uow") + __slots__ = ("_policy", "_uow") - def __init__(self, uow: AttachmentsUnitOfWork, clock: Clock, policy: RetryPolicy) -> None: + def __init__(self, uow: AttachmentsUnitOfWork, policy: RetryPolicy) -> None: self._uow = uow - self._clock = clock self._policy = policy async def __call__(self, command: RecoverExpiredLeases) -> int: async with self._uow.transaction(): leases = await self._uow.jobs.expired(command.limit) for lease in leases: - if self._policy.delay(lease.job.attempt) is None: + if self._policy.exhausted(lease.job.attempt): await self._uow.jobs.fail( lease, FailureCode.RETRY_EXHAUSTED, Outcome.LEASE_EXPIRED.value ) outcome = Outcome.FAILED else: await self._uow.jobs.reschedule( - lease, self._clock.now(), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value + lease, timedelta(0), Outcome.LEASE_EXPIRED, Outcome.LEASE_EXPIRED.value ) outcome = Outcome.LEASE_EXPIRED self._uow.record(settled(lease.job, outcome)) diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index dd2760a..a0d04d4 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -70,7 +70,12 @@ class JobPublisher(Protocol): @runtime_checkable class Faults(Protocol): - def reach(self, point: str) -> None: ... + async def reach(self, point: str) -> None: ... + + +@runtime_checkable +class Jitter(Protocol): + def spread(self) -> float: ... @runtime_checkable @@ -106,9 +111,11 @@ class JobRepository(Protocol): ) -> Outcome: ... async def reschedule( - self, lease: Lease, at: datetime, kind: Outcome, error: str + self, lease: Lease, delay: timedelta, kind: Outcome, error: str ) -> Job | None: ... + async def renew(self, lease: Lease, duration: timedelta) -> bool: ... + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: ... async def note(self, job: Job, kind: Outcome) -> None: ... diff --git a/src/messenger/attachments/config.py b/src/messenger/attachments/config.py index 7d72de4..e16e768 100644 --- a/src/messenger/attachments/config.py +++ b/src/messenger/attachments/config.py @@ -33,7 +33,7 @@ class PipelineSettings(AttachmentsSettings): job_timeout_seconds: float = Field(default=60.0, ge=1, le=290) job_backoff_base_seconds: float = Field(default=2.0, gt=0, le=60) job_backoff_cap_seconds: float = Field(default=60.0, gt=0, le=3600) - staging_retention_seconds: int = Field(default=86_400, ge=60) + staging_retention_seconds: int = Field(default=86_400, ge=1) outbox_poll_seconds: float = Field(default=1.0, gt=0, le=10) outbox_batch: int = Field(default=50, ge=1, le=1000) sweep_interval_seconds: float = Field(default=5.0, gt=0, le=300) @@ -46,3 +46,4 @@ class PipelineSettings(AttachmentsSettings): render_memory_mb: int = Field(default=448, ge=0, le=8192) render_tasks_per_child: int = Field(default=100, ge=1, le=10_000) fault_point: str = "" + fault_stall_seconds: float = Field(default=15.0, ge=0, le=300) diff --git a/src/messenger/attachments/domain/errors.py b/src/messenger/attachments/domain/errors.py index 060d6f3..dfb12d8 100644 --- a/src/messenger/attachments/domain/errors.py +++ b/src/messenger/attachments/domain/errors.py @@ -39,3 +39,7 @@ class UnreadableImage(AttachmentError): class ImageTooDemanding(AttachmentError): pass + + +class LeaseLost(AttachmentError): + pass diff --git a/src/messenger/attachments/domain/jobs.py b/src/messenger/attachments/domain/jobs.py index e53d618..70b56b7 100644 --- a/src/messenger/attachments/domain/jobs.py +++ b/src/messenger/attachments/domain/jobs.py @@ -67,7 +67,11 @@ class RetryPolicy: base_seconds: float cap_seconds: float - def delay(self, attempt: int) -> timedelta | None: - if attempt >= self.max_attempts: + def exhausted(self, attempt: int) -> bool: + return attempt >= self.max_attempts + + def delay(self, attempt: int, spread: float) -> timedelta | None: + if self.exhausted(attempt): return None - return timedelta(seconds=min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1))) + ceiling = min(self.cap_seconds, self.base_seconds * 2 ** (attempt - 1)) + return timedelta(seconds=ceiling / 2 * (1 + spread)) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index 4585dc5..dcc50c0 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -31,6 +31,7 @@ from messenger.attachments.application.ports import ( ChatMembership, Faults, ImageInspector, + Jitter, JobPublisher, ObjectStorage, VariantRenderer, @@ -44,6 +45,7 @@ from messenger.attachments.infrastructure.images import ( ProcessVariantRenderer, RenderLimits, ) +from messenger.attachments.infrastructure.jitter import RandomJitter from messenger.attachments.infrastructure.postgres import ( PostgresAttachmentDirectory, PostgresAttachmentsUnitOfWork, @@ -166,6 +168,10 @@ class PipelineProvider(Provider): settings.job_backoff_cap_seconds, ) + @provide(scope=Scope.APP) + def faults(self, settings: PipelineSettings, logger: Logger) -> Faults: + return EnvironmentFaults(settings.fault_point, logger, settings.fault_stall_seconds) + @provide(scope=Scope.REQUEST) def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: return PostgresAttachmentsUnitOfWork(database) @@ -185,8 +191,8 @@ class WorkerProvider(Provider): renderer.close() @provide(scope=Scope.APP) - def faults(self, settings: PipelineSettings, logger: Logger) -> Faults: - return EnvironmentFaults(settings.fault_point, logger) + def jitter(self) -> Jitter: + return RandomJitter() @provide(scope=Scope.REQUEST) def command_bus( @@ -196,8 +202,8 @@ class WorkerProvider(Provider): renderer: VariantRenderer, faults: Faults, publisher: EventPublisher, - clock: Clock, ids: IdGenerator, + jitter: Jitter, policy: RetryPolicy, settings: PipelineSettings, ) -> AttachmentsCommandBus: @@ -206,8 +212,8 @@ class WorkerProvider(Provider): storage, renderer, faults, - clock=clock, ids=ids, + jitter=jitter, policy=policy, lease=timedelta(seconds=settings.job_timeout_seconds), ) @@ -229,13 +235,13 @@ class RelayProvider(Provider): uow: AttachmentsUnitOfWork, storage: ObjectStorage, jobs: JobPublisher, + faults: Faults, publisher: EventPublisher, - clock: Clock, policy: RetryPolicy, ) -> AttachmentsCommandBus: handlers: dict[type[Command], CommandHandler] = { - PublishOutbox: PublishOutboxHandler(uow, jobs), - RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, clock, policy), + PublishOutbox: PublishOutboxHandler(uow, jobs, faults), + RecoverExpiredLeases: RecoverExpiredLeasesHandler(uow, policy), CleanStaging: CleanStagingHandler(uow, storage), } return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/infrastructure/faults.py b/src/messenger/attachments/infrastructure/faults.py index 814d3ce..9797c73 100644 --- a/src/messenger/attachments/infrastructure/faults.py +++ b/src/messenger/attachments/infrastructure/faults.py @@ -1,26 +1,32 @@ +import asyncio import os import sys +from messenger.attachments.application.pipeline import STALL_AFTER_CLAIM, TRANSIENT_ERROR from messenger.shared.application.errors import Unavailable from messenger.shared.application.ports import Logger -TRANSIENT = frozenset({"transient_error"}) CRASH_EXIT_CODE = 137 class EnvironmentFaults: - __slots__ = ("_logger", "_point") + __slots__ = ("_logger", "_point", "_stall_seconds") - def __init__(self, point: str, logger: Logger) -> None: + def __init__(self, point: str, logger: Logger, stall_seconds: float = 0.0) -> None: self._point = point self._logger = logger + self._stall_seconds = stall_seconds - def reach(self, point: str) -> None: + async def reach(self, point: str) -> None: if not self._point or point != self._point: return self._logger.warning("fault.injected", reason=point) - if point in TRANSIENT: + if point == TRANSIENT_ERROR: raise Unavailable("внедрённая временная ошибка") + if point == STALL_AFTER_CLAIM: + self._point = "" + await asyncio.sleep(self._stall_seconds) + return sys.stdout.flush() sys.stderr.flush() os._exit(CRASH_EXIT_CODE) diff --git a/src/messenger/attachments/infrastructure/jitter.py b/src/messenger/attachments/infrastructure/jitter.py new file mode 100644 index 0000000..4207230 --- /dev/null +++ b/src/messenger/attachments/infrastructure/jitter.py @@ -0,0 +1,10 @@ +from secrets import SystemRandom + +SOURCE = SystemRandom() + + +class RandomJitter: + __slots__ = () + + def spread(self) -> float: + return SOURCE.random() diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index 20a9ad5..32483b5 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -118,19 +118,24 @@ RESCHEDULE = """ WITH job AS ( UPDATE jobs SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL, - next_attempt_at = $3, last_error = $4, updated_at = now() + next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now() WHERE job_id = $1 AND status = 'processing' AND lease_token = $2 - RETURNING job_id, attachment_id, attempt + RETURNING job_id, attachment_id, attempt, next_attempt_at ), moved AS ( UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at) FROM job WHERE a.id = job.attachment_id AND a.status = 'processing' ), audit AS ( INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) SELECT job_id, attachment_id, $5, attempt, - jsonb_build_object('error', $4::text, 'next_attempt_at', $3::timestamptz) + jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at, + 'delay_seconds', $3::float8) FROM job ) -SELECT attempt FROM job +SELECT attempt, next_attempt_at FROM job +""" +RENEW = """ +UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now() +WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' """ ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)" FAIL = """ @@ -310,16 +315,24 @@ class PostgresJobRepository: await self.note(job, outcome) return outcome - async def reschedule(self, lease: Lease, at: datetime, kind: Outcome, error: str) -> Job | None: - attempt = await self._connection.fetchval( - RESCHEDULE, lease.job.id, lease.token, at, error, kind.value + async def reschedule( + self, lease: Lease, delay: timedelta, kind: Outcome, error: str + ) -> Job | None: + row = await self._connection.fetchrow( + RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value ) - if attempt is None: + if row is None: return None - job = lease.job.retried(attempt) - await self._connection.execute(ENQUEUE, job.id, payload(job), at) + job = lease.job.retried(row["attempt"]) + await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"]) return job + async def renew(self, lease: Lease, duration: timedelta) -> bool: + done = await self._connection.execute( + RENEW, lease.job.id, lease.token, duration.total_seconds() + ) + return done == "UPDATE 1" + async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool: attempt = await self._connection.fetchval( FAIL, lease.job.id, lease.token, code.value, error diff --git a/student-tests/api/test_pipeline.py b/student-tests/api/test_pipeline.py index 58e1cb3..811abcf 100644 --- a/student-tests/api/test_pipeline.py +++ b/student-tests/api/test_pipeline.py @@ -10,6 +10,7 @@ from httpx import AsyncClient from test_attachments import storage, trio, upload from conftest import DATABASE, actor +from messenger.attachments.application.pipeline import TRANSIENT_ERROR from messenger.attachments.application.ports import Rendition from messenger.attachments.domain.errors import ImageTooDemanding from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy @@ -287,9 +288,16 @@ async def test_backoff_delays_the_next_attempt(client: AsyncClient) -> None: "WHERE attachment_id = $1", attachment_id, ) - assert 25 < row["wait"] <= 30 + assert 20 < row["wait"] <= 22.5 + (event,) = await rows( + "SELECT (detail->>'delay_seconds')::float8 AS delay FROM job_events " + "WHERE attachment_id = $1 AND kind = 'retry_scheduled'", + attachment_id, + ) + assert event["delay"] == 22.5 class FailingFaults: - def reach(self, point: str) -> None: - raise Unavailable(point) + async def reach(self, point: str) -> None: + if point == TRANSIENT_ERROR: + raise Unavailable(point) diff --git a/student-tests/api/test_recovery.py b/student-tests/api/test_recovery.py new file mode 100644 index 0000000..9d92a86 --- /dev/null +++ b/student-tests/api/test_recovery.py @@ -0,0 +1,231 @@ +import asyncio +import subprocess +import sys +from datetime import timedelta + +import pytest +from httpx import AsyncClient +from test_attachments import storage +from test_pipeline import accepted, events, job_of, rows, state, variants + +from fakes import NullLogger +from messenger.attachments.application.pipeline import ( + AFTER_CLAIM, + AFTER_COMMIT_BEFORE_ACK, + AFTER_COMMIT_BEFORE_PUBLISH, + AFTER_PUBLISH_BEFORE_CONFIRM, + AFTER_S3_WRITE, + STALL_AFTER_CLAIM, + TRANSIENT_ERROR, +) +from messenger.attachments.application.ports import Rendition +from messenger.attachments.domain.jobs import Outcome +from messenger.attachments.infrastructure.faults import EnvironmentFaults +from messenger.shared.application.errors import Unavailable +from pipeline import InlinePipeline, InlineRenderer, pipeline_of + +WORKER_POINTS = [ + AFTER_CLAIM, + STALL_AFTER_CLAIM, + TRANSIENT_ERROR, + AFTER_S3_WRITE, + AFTER_COMMIT_BEFORE_ACK, +] + + +class Crash(BaseException): + pass + + +class RecordingFaults: + __slots__ = ("crash_at", "reached") + + def __init__(self, crash_at: str = "") -> None: + self.crash_at = crash_at + self.reached: list[str] = [] + + async def reach(self, point: str) -> None: + self.reached.append(point) + if point == self.crash_at: + raise Crash(point) + + +class SlowRenderer: + __slots__ = ("seconds",) + + def __init__(self, seconds: float) -> None: + self.seconds = seconds + + async def render(self, data: bytes, pipeline_version: int) -> list[Rendition]: + await asyncio.sleep(self.seconds) + return await InlineRenderer().render(data, pipeline_version) + + +class ExpiringFaults: + __slots__ = ("attachment_id", "pipeline") + + def __init__(self, pipeline: InlinePipeline, attachment_id: str) -> None: + self.pipeline = pipeline + self.attachment_id = attachment_id + + async def reach(self, point: str) -> None: + if point == STALL_AFTER_CLAIM: + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' " + "WHERE attachment_id = $1", + self.attachment_id, + ) + await self.pipeline.recover() + + +async def outbox(attachment_id: str) -> list[bool]: + found = await rows( + "SELECT o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id) " + "WHERE j.attachment_id = $1 ORDER BY o.id", + attachment_id, + ) + return [row["published"] for row in found] + + +def stored(objects: dict[str, tuple[bytes, str]], attachment_id: str) -> list[str]: + return sorted(key for key in objects if key.startswith(f"variants/{attachment_id}/")) + + +async def test_fault_points_are_reached_in_order(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + faults = RecordingFaults() + pipeline.faults = faults + published = await pipeline.publish() + assert faults.reached[:2] == [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM] + faults.reached.clear() + (job,) = [job for job in published if str(job.attachment_id) == attachment_id] + assert await pipeline.process(job) is Outcome.DONE + assert faults.reached == WORKER_POINTS + + +@pytest.mark.parametrize("point", [AFTER_COMMIT_BEFORE_PUBLISH, AFTER_PUBLISH_BEFORE_CONFIRM]) +async def test_relay_crash_leaves_the_row_for_the_next_relay( + client: AsyncClient, point: str +) -> None: + pipeline = await pipeline_of(client) + await pipeline.drain() + _, _, attachment_id = await accepted(client) + pipeline.faults = RecordingFaults(point) + with pytest.raises(Crash): + await pipeline.publish() + assert await outbox(attachment_id) == [False] + sent = [body for body in pipeline.publisher.bodies if attachment_id.encode() in body] + assert len(sent) == (point == AFTER_PUBLISH_BEFORE_CONFIRM) + pipeline.faults = RecordingFaults() + outcomes = await pipeline.drain() + assert Outcome.DONE in outcomes + assert await outbox(attachment_id) == [True] + assert (await state(attachment_id))["status"] == "ready" + + +async def test_crash_after_s3_write_is_finished_by_the_next_attempt(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, lease=timedelta(seconds=60)) + await pipeline.publish() + job = await job_of(attachment_id) + pipeline.faults = RecordingFaults(AFTER_S3_WRITE) + with pytest.raises(Crash): + await pipeline.process(job) + objects = (await storage(client)).objects + written = stored(objects, attachment_id) + assert len(written) == 3 + assert await variants(attachment_id) == [] + assert (await state(attachment_id))["status"] == "processing" + await rows( + "UPDATE jobs SET lease_until = now() - interval '1 second' WHERE attachment_id = $1", + attachment_id, + ) + pipeline.faults = RecordingFaults() + assert await pipeline.recover() >= 1 + await pipeline.drain() + done = await state(attachment_id) + assert (done["status"], done["attempt"]) == ("ready", 2) + assert stored(objects, attachment_id) == written + assert sorted(row["object_key"] for row in await variants(attachment_id)) == written + + +async def test_crash_after_commit_is_a_duplicate_on_redelivery(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + pipeline.faults = RecordingFaults(AFTER_COMMIT_BEFORE_ACK) + with pytest.raises(Crash): + await pipeline.process(job) + before = await variants(attachment_id) + assert (await state(attachment_id))["status"] == "ready" + pipeline.faults = RecordingFaults() + assert await pipeline.process(job) is Outcome.DUPLICATE + assert await variants(attachment_id) == before + + +async def test_long_render_keeps_its_lease(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client, lease=timedelta(seconds=1)) + pipeline.renderer = SlowRenderer(2.5) + await pipeline.publish() + job = await job_of(attachment_id) + + async def sweep() -> None: + for _ in range(4): + await asyncio.sleep(0.6) + await pipeline.recover() + + outcome, _ = await asyncio.gather(pipeline.process(job), sweep()) + assert outcome is Outcome.DONE + done = await state(attachment_id) + assert (done["status"], done["attempt"], done["claims"]) == ("ready", 1, 1) + assert "lease_expired" not in await events(attachment_id) + + +async def test_lost_lease_stops_before_side_effects(client: AsyncClient) -> None: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + await pipeline.publish() + job = await job_of(attachment_id) + pipeline.faults = ExpiringFaults(pipeline, attachment_id) + assert await pipeline.process(job) is Outcome.FENCED + objects = (await storage(client)).objects + assert stored(objects, attachment_id) == [] + assert f"originals/{attachment_id}" not in objects + assert (await state(attachment_id))["status"] == "queued" + assert (await events(attachment_id))[-2:] == ["lease_expired", "fenced"] + pipeline.faults = RecordingFaults() + await pipeline.drain() + assert (await state(attachment_id))["status"] == "ready" + + +async def test_environment_faults_stall_once_and_fail_transiently() -> None: + stall = EnvironmentFaults(STALL_AFTER_CLAIM, NullLogger(), stall_seconds=0.3) + started = asyncio.get_running_loop().time() + await stall.reach(AFTER_CLAIM) + await stall.reach(STALL_AFTER_CLAIM) + await stall.reach(STALL_AFTER_CLAIM) + assert 0.3 <= asyncio.get_running_loop().time() - started < 0.6 + with pytest.raises(Unavailable): + await EnvironmentFaults(TRANSIENT_ERROR, NullLogger()).reach(TRANSIENT_ERROR) + + +def test_environment_fault_crashes_the_process_with_137() -> None: + script = ( + "import asyncio\n" + "from fakes import NullLogger\n" + "from messenger.attachments.infrastructure.faults import EnvironmentFaults\n" + "asyncio.run(EnvironmentFaults('after_commit_before_ack', NullLogger())" + ".reach('after_commit_before_ack'))\n" + "print('survived')\n" + ) + done = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + env={"PYTHONPATH": ":".join(sys.path)}, + ) + assert (done.returncode, done.stdout) == (137, "") diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py index 718d0c6..914c978 100644 --- a/student-tests/pipeline.py +++ b/student-tests/pipeline.py @@ -20,6 +20,7 @@ from messenger.attachments.application.pipeline import ( ) from messenger.attachments.application.ports import ( Faults, + Jitter, ObjectStorage, Rendition, VariantRenderer, @@ -30,15 +31,25 @@ from messenger.attachments.infrastructure.faults import EnvironmentFaults from messenger.attachments.infrastructure.images import RenderLimits, render_variants from messenger.attachments.infrastructure.postgres import PostgresAttachmentsUnitOfWork from messenger.shared.application.errors import Unavailable -from messenger.shared.infrastructure.clock import MonotonicUtcClock from messenger.shared.infrastructure.ids import Uuid7Generator from messenger.shared.infrastructure.postgres import Database IDLE_SECONDS = 0.1 +HALF = 0.5 LIMITS = RenderLimits(10_485_760, 40_000_000) FAST = RetryPolicy(max_attempts=5, base_seconds=0.001, cap_seconds=0.001) +class FixedJitter: + __slots__ = ("_spread",) + + def __init__(self, spread: float) -> None: + self._spread = spread + + def spread(self) -> float: + return self._spread + + class InlineRenderer: __slots__ = () @@ -62,7 +73,16 @@ class CapturingPublisher: class InlinePipeline: - __slots__ = ("_database", "faults", "lease", "policy", "publisher", "renderer", "storage") + __slots__ = ( + "_database", + "faults", + "jitter", + "lease", + "policy", + "publisher", + "renderer", + "storage", + ) def __init__( self, @@ -79,13 +99,14 @@ class InlinePipeline: self.publisher = CapturingPublisher() self.renderer: VariantRenderer = InlineRenderer() self.faults: Faults = EnvironmentFaults("", NullLogger()) + self.jitter: Jitter = FixedJitter(HALF) def uow(self) -> PostgresAttachmentsUnitOfWork: return PostgresAttachmentsUnitOfWork(self._database) async def publish(self) -> list[Job]: start = len(self.publisher.bodies) - await PublishOutboxHandler(self.uow(), self.publisher)(PublishOutbox(100)) + await PublishOutboxHandler(self.uow(), self.publisher, self.faults)(PublishOutbox(100)) return [ JobMessage.model_validate_json(body).job() for body in self.publisher.bodies[start:] ] @@ -96,8 +117,8 @@ class InlinePipeline: self.storage, self.renderer, self.faults, - clock=MonotonicUtcClock(), ids=Uuid7Generator(), + jitter=self.jitter, policy=self.policy, lease=self.lease, ) @@ -117,7 +138,7 @@ class InlinePipeline: return outcomes async def recover(self) -> int: - handler = RecoverExpiredLeasesHandler(self.uow(), MonotonicUtcClock(), self.policy) + handler = RecoverExpiredLeasesHandler(self.uow(), self.policy) return await handler(RecoverExpiredLeases(100)) async def clean(self) -> int: diff --git a/student-tests/unit/test_pipeline_domain.py b/student-tests/unit/test_pipeline_domain.py index c801899..6093584 100644 --- a/student-tests/unit/test_pipeline_domain.py +++ b/student-tests/unit/test_pipeline_domain.py @@ -8,6 +8,7 @@ import pytest from messenger.attachments.domain.jobs import Job, RetryPolicy from messenger.attachments.domain.variants import fit, variant_key from messenger.attachments.entrypoints.schemas import JobMessage +from messenger.attachments.infrastructure.jitter import RandomJitter SCHEMA = json.loads( (Path(__file__).resolve().parents[2] / "contracts" / "jobs.schema.json").read_text() @@ -34,16 +35,22 @@ def test_thumbnail_fits_the_box_rounding_down( assert fit(*size, 256) == fitted -def test_retry_policy_doubles_up_to_the_cap_and_stops_at_the_limit() -> None: +@pytest.mark.parametrize(("spread", "share"), [(0.0, 0.5), (0.5, 0.75), (0.999, 0.9995)]) +def test_retry_delay_has_equal_jitter_up_to_the_cap(spread: float, share: float) -> None: policy = RetryPolicy(max_attempts=5, base_seconds=2, cap_seconds=10) - delays = [policy.delay(attempt) for attempt in range(1, 6)] - assert delays == [ - timedelta(seconds=2), - timedelta(seconds=4), - timedelta(seconds=8), - timedelta(seconds=10), - None, - ] + delays = [policy.delay(attempt, spread) for attempt in range(1, 6)] + ceilings = [2, 4, 8, 10] + assert delays[:4] == [timedelta(seconds=ceiling * share) for ceiling in ceilings] + assert delays[4] is None + assert policy.exhausted(5) + assert not policy.exhausted(4) + + +def test_jitter_spreads_retries_between_half_and_full_delay() -> None: + policy, jitter = RetryPolicy(5, 2, 60), RandomJitter() + delays = {policy.delay(3, jitter.spread()) for _ in range(200)} + assert all(timedelta(seconds=4) <= delay < timedelta(seconds=8) for delay in delays if delay) + assert len(delays) > 150 def test_job_message_follows_the_contract_schema() -> None: