diff --git a/openspec/changes/add-lab5-variants-and-recovery/tasks.md b/openspec/changes/add-lab5-variants-and-recovery/tasks.md index d35d47b..6a0cc7a 100644 --- a/openspec/changes/add-lab5-variants-and-recovery/tasks.md +++ b/openspec/changes/add-lab5-variants-and-recovery/tasks.md @@ -24,11 +24,11 @@ ## 5. Реестр, replay, retention -- [ ] 5.1 Миграция `0005_failed_registry.sql` (частичный индекс `failed`-заданий) -- [ ] 5.2 Команда `ReplayAttachment` и запрос реестра в зоне `attachments`, событие `JobReplayed` в журнале аудита -- [ ] 5.3 Точка входа `attachments/entrypoints/admin.py` и сборка `messenger/admin.py` (`failed`, `replay`), тест графа импортов -- [ ] 5.4 `CleanStaging` с retention `failed`, `STAGING_RETENTION_SECONDS` от 1 с -- [ ] 5.5 Проверка группы: тесты против PostgreSQL — replay `failed` → `ready` с тем же `job_id` и `attempt` 1, replay `ready` → `noop` без outbox, после retention → `expired`, staging `queued` не трогается; тест миграций +- [x] 5.1 Миграция `0005_failed_registry.sql` (частичный индекс `failed`-заданий) +- [x] 5.2 Команда `ReplayAttachment` и запрос реестра в зоне `attachments`, событие `JobReplayed` в журнале аудита +- [x] 5.3 Точка входа `attachments/entrypoints/admin.py` и сборка `messenger/admin.py` (`failed`, `replay`), тест графа импортов +- [x] 5.4 `CleanStaging` с retention `failed`, `STAGING_RETENTION_SECONDS` от 1 с +- [x] 5.5 Проверка группы: тесты против PostgreSQL — replay `failed` → `ready` с тем же `job_id` и `attempt` 1, replay `ready` → `noop` без outbox, после retention → `expired`, staging `queued` не трогается; тест миграций ## 6. Стенд и обвязка diff --git a/pyproject.toml b/pyproject.toml index a518e85..bb26586 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,7 @@ ignore = [ "src/messenger/migrate.py" = ["TID251"] "src/messenger/relay.py" = ["TID251"] "src/messenger/worker.py" = ["TID251"] +"src/messenger/admin.py" = ["TID251"] "src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"] [tool.ruff.lint.flake8-tidy-imports.banned-api] diff --git a/src/messenger/admin.py b/src/messenger/admin.py new file mode 100644 index 0000000..4c6f3cb --- /dev/null +++ b/src/messenger/admin.py @@ -0,0 +1,37 @@ +import asyncio +import sys + +from dishka import make_async_container + +from messenger.attachments.domain.jobs import ReplayResult +from messenger.attachments.entrypoints.admin import rendered, run +from messenger.attachments.entrypoints.ioc import AdminProvider +from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal +from messenger.shared.application.messagebus import EventBus +from messenger.shared.config import PlatformSettings +from messenger.shared.entrypoints.ioc import PlatformProvider +from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger + + +async def serve(argv: list[str]) -> int: + platform = PlatformSettings() + logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS) + container = make_async_container(PlatformProvider(platform, logger), AdminProvider()) + bus = await container.get(EventBus) + subscribe_journal(bus, logger) + bus.start() + try: + report = await run(container, argv) + finally: + await bus.stop() + await container.close() + sys.stdout.write(rendered(report) + "\n") + return int(report.get("result") == ReplayResult.NOT_FOUND) + + +def main() -> None: + sys.exit(asyncio.run(serve(sys.argv[1:]))) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/attachments/application/commands.py b/src/messenger/attachments/application/commands.py index b7543f3..af89a4b 100644 --- a/src/messenger/attachments/application/commands.py +++ b/src/messenger/attachments/application/commands.py @@ -40,3 +40,10 @@ class RecoverExpiredLeases(Command): @dataclass(frozen=True, slots=True) class CleanStaging(Command): limit: int + retention: timedelta + + +@dataclass(frozen=True, slots=True) +class ReplayAttachment(Command): + attachment_id: UUID + operator: str diff --git a/src/messenger/attachments/application/pipeline.py b/src/messenger/attachments/application/pipeline.py index cf6f20b..acbeebb 100644 --- a/src/messenger/attachments/application/pipeline.py +++ b/src/messenger/attachments/application/pipeline.py @@ -7,6 +7,7 @@ from messenger.attachments.application.commands import ( ProcessJob, PublishOutbox, RecoverExpiredLeases, + ReplayAttachment, ) from messenger.attachments.application.ports import ( AttachmentsUnitOfWork, @@ -17,9 +18,17 @@ from messenger.attachments.application.ports import ( VariantRenderer, ) from messenger.attachments.domain.errors import LeaseLost, UnreadableImage -from messenger.attachments.domain.events import JobSettled -from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy -from messenger.attachments.domain.models import original_key +from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled +from messenger.attachments.domain.jobs import ( + Claim, + FailureCode, + Job, + Lease, + Outcome, + ReplayResult, + RetryPolicy, +) +from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key from messenger.attachments.domain.variants import Variant, variant_key from messenger.shared.application.errors import Unavailable from messenger.shared.application.ports import IdGenerator @@ -256,8 +265,37 @@ class CleanStagingHandler: async def __call__(self, command: CleanStaging) -> int: async with self._uow.transaction(): - staged = await self._uow.attachments.staged(command.limit) + staged = await self._uow.attachments.staged(command.limit, command.retention) await asyncio.gather(*(self._storage.delete(item.key) for item in staged)) await self._uow.attachments.unstage([item.attachment_id for item in staged]) await self._uow.commit() return len(staged) + + +class ReplayAttachmentHandler: + __slots__ = ("_uow",) + + def __init__(self, uow: AttachmentsUnitOfWork) -> None: + self._uow = uow + + async def __call__(self, command: ReplayAttachment) -> ReplayResult: + async with self._uow.transaction(): + attachment = await self._uow.attachments.lock(command.attachment_id) + result = await self._replay(attachment, command.operator) + if attachment is not None and result is not ReplayResult.REPLAYED: + await self._uow.jobs.note_replay(attachment.id, command.operator, result.value) + self._uow.record( + AttachmentReplayRequested(command.attachment_id, command.operator, result.value) + ) + await self._uow.commit() + return result + + async def _replay(self, attachment: Attachment | None, operator: str) -> ReplayResult: + if attachment is None: + return ReplayResult.NOT_FOUND + if attachment.status is not AttachmentStatus.FAILED: + return ReplayResult.NOOP + if attachment.staging_key is None: + return ReplayResult.EXPIRED + job = await self._uow.jobs.replay(attachment, operator) + return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED diff --git a/src/messenger/attachments/application/ports.py b/src/messenger/attachments/application/ports.py index a0d04d4..e32f42c 100644 --- a/src/messenger/attachments/application/ports.py +++ b/src/messenger/attachments/application/ports.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta from typing import Protocol, runtime_checkable from uuid import UUID -from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome +from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome from messenger.attachments.domain.models import Attachment from messenger.attachments.domain.values import ImageInfo, ImageKind from messenger.attachments.domain.variants import Variant @@ -95,7 +95,7 @@ class AttachmentRepository(Protocol): async def remove(self, attachment: Attachment) -> None: ... - async def staged(self, limit: int) -> list[StagedObject]: ... + async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ... async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ... @@ -122,6 +122,10 @@ class JobRepository(Protocol): async def expired(self, limit: int) -> list[Lease]: ... + async def replay(self, attachment: Attachment, operator: str) -> Job | None: ... + + async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: ... + @runtime_checkable class OutboxRepository(Protocol): @@ -147,3 +151,5 @@ class AttachmentDirectory(Protocol): async def visible(self, attachment_id: UUID) -> Attachment | None: ... async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ... + + async def failed_jobs(self, limit: int) -> list[FailedJob]: ... diff --git a/src/messenger/attachments/domain/events.py b/src/messenger/attachments/domain/events.py index 58a7c53..4c5eae1 100644 --- a/src/messenger/attachments/domain/events.py +++ b/src/messenger/attachments/domain/events.py @@ -31,3 +31,11 @@ class JobSettled(DomainEvent): error_code: str | None = None delay_seconds: float | None = None name: str = field(init=False, default="job.settled") + + +@dataclass(frozen=True, slots=True) +class AttachmentReplayRequested(DomainEvent): + attachment_id: UUID + operator: str + result: str + name: str = field(init=False, default="attachment.replay_requested") diff --git a/src/messenger/attachments/domain/jobs.py b/src/messenger/attachments/domain/jobs.py index 70b56b7..9c526e5 100644 --- a/src/messenger/attachments/domain/jobs.py +++ b/src/messenger/attachments/domain/jobs.py @@ -14,6 +14,13 @@ class FailureCode(StrEnum): RETRY_EXHAUSTED = "retry_exhausted" +class ReplayResult(StrEnum): + REPLAYED = "replayed" + NOOP = "noop" + EXPIRED = "expired" + NOT_FOUND = "not_found" + + class Outcome(StrEnum): DONE = "done" DUPLICATE = "duplicate_noop" @@ -61,6 +68,17 @@ class Claim: staged: bool +@dataclass(frozen=True, slots=True) +class FailedJob: + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + error_code: str | None + last_error: str | None + failed_at: datetime + + @dataclass(frozen=True, slots=True) class RetryPolicy: max_attempts: int diff --git a/src/messenger/attachments/entrypoints/admin.py b/src/messenger/attachments/entrypoints/admin.py new file mode 100644 index 0000000..6eec7ad --- /dev/null +++ b/src/messenger/attachments/entrypoints/admin.py @@ -0,0 +1,45 @@ +import argparse +import json +import os +from dataclasses import asdict +from typing import Any +from uuid import UUID + +from dishka import AsyncContainer + +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import ReplayAttachment +from messenger.attachments.application.ports import AttachmentDirectory +from messenger.attachments.domain.jobs import ReplayResult + +REGISTRY_LIMIT = 20 + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser( + prog="python -m messenger.admin", description="Реестр failed-заданий и явный replay" + ) + commands = root.add_subparsers(dest="command", required=True) + failed = commands.add_parser("failed", help="последние задания в failed") + failed.add_argument("--limit", type=int, default=REGISTRY_LIMIT) + replay = commands.add_parser("replay", help="вернуть failed-вложение в очередь") + replay.add_argument("--attachment-id", type=UUID, required=True) + replay.add_argument("--operator", default=os.environ.get("OPERATOR", "cli")) + return root + + +async def run(container: AsyncContainer, argv: list[str]) -> dict[str, Any]: + args = parser().parse_args(argv) + async with container() as scope: + if args.command == "failed": + directory = await scope.get(AttachmentDirectory) + return {"failed": [asdict(job) for job in await directory.failed_jobs(args.limit)]} + bus = await scope.get(AttachmentsCommandBus) + result: ReplayResult = await bus.execute( + ReplayAttachment(args.attachment_id, args.operator) + ) + return {"result": result.value, "attachment_id": args.attachment_id, "operator": args.operator} + + +def rendered(report: dict[str, Any]) -> str: + return json.dumps(report, default=str, ensure_ascii=False) diff --git a/src/messenger/attachments/entrypoints/ioc.py b/src/messenger/attachments/entrypoints/ioc.py index dcc50c0..c661dd1 100644 --- a/src/messenger/attachments/entrypoints/ioc.py +++ b/src/messenger/attachments/entrypoints/ioc.py @@ -11,6 +11,7 @@ from messenger.attachments.application.commands import ( ProcessJob, PublishOutbox, RecoverExpiredLeases, + ReplayAttachment, SweepAbandonedUploads, UploadAttachment, ) @@ -24,6 +25,7 @@ from messenger.attachments.application.pipeline import ( ProcessJobHandler, PublishOutboxHandler, RecoverExpiredLeasesHandler, + ReplayAttachmentHandler, ) from messenger.attachments.application.ports import ( AttachmentDirectory, @@ -245,3 +247,22 @@ class RelayProvider(Provider): CleanStaging: CleanStagingHandler(uow, storage), } return AttachmentsCommandBus(handlers, uow, publisher) + + +class AdminProvider(Provider): + @provide(scope=Scope.APP) + def directory(self, database: Database) -> AttachmentDirectory: + return PostgresAttachmentDirectory(database) + + @provide(scope=Scope.REQUEST) + def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork: + return PostgresAttachmentsUnitOfWork(database) + + @provide(scope=Scope.REQUEST) + def command_bus( + self, uow: AttachmentsUnitOfWork, publisher: EventPublisher + ) -> AttachmentsCommandBus: + handlers: dict[type[Command], CommandHandler] = { + ReplayAttachment: ReplayAttachmentHandler(uow) + } + return AttachmentsCommandBus(handlers, uow, publisher) diff --git a/src/messenger/attachments/entrypoints/journal.py b/src/messenger/attachments/entrypoints/journal.py index 9b771e8..014f23f 100644 --- a/src/messenger/attachments/entrypoints/journal.py +++ b/src/messenger/attachments/entrypoints/journal.py @@ -1,4 +1,9 @@ -from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued, JobSettled +from messenger.attachments.domain.events import ( + AttachmentDiscarded, + AttachmentQueued, + AttachmentReplayRequested, + JobSettled, +) from messenger.shared.application.messagebus import EventBus from messenger.shared.application.ports import Logger @@ -15,6 +20,8 @@ LOG_FIELDS = frozenset( "outcome", "error_code", "delay_seconds", + "operator", + "result", } ) @@ -54,9 +61,18 @@ class AttachmentsJournal: delay_seconds=event.delay_seconds, ) + async def on_replay_requested(self, event: AttachmentReplayRequested) -> None: + self._logger.info( + "audit.attachment_replay", + attachment_id=str(event.attachment_id), + operator=event.operator, + result=event.result, + ) + def subscribe_journal(bus: EventBus, logger: Logger) -> None: journal = AttachmentsJournal(logger) bus.subscribe(AttachmentQueued, journal.on_attachment_queued) bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded) bus.subscribe(JobSettled, journal.on_job_settled) + bus.subscribe(AttachmentReplayRequested, journal.on_replay_requested) diff --git a/src/messenger/attachments/entrypoints/relay.py b/src/messenger/attachments/entrypoints/relay.py index fad85f7..9c3436f 100644 --- a/src/messenger/attachments/entrypoints/relay.py +++ b/src/messenger/attachments/entrypoints/relay.py @@ -1,6 +1,7 @@ import asyncio from collections.abc import Sequence from contextlib import suppress +from datetime import timedelta from dishka import AsyncContainer @@ -32,7 +33,12 @@ class Relay: (settings.outbox_poll_seconds, (PublishOutbox(settings.outbox_batch),)), ( settings.sweep_interval_seconds, - (RecoverExpiredLeases(settings.sweep_batch), CleanStaging(settings.sweep_batch)), + ( + RecoverExpiredLeases(settings.sweep_batch), + CleanStaging( + settings.sweep_batch, timedelta(seconds=settings.staging_retention_seconds) + ), + ), ), ) self._tasks = [asyncio.create_task(self._every(*loop)) for loop in loops] diff --git a/src/messenger/attachments/infrastructure/postgres.py b/src/messenger/attachments/infrastructure/postgres.py index 32483b5..1af88b2 100644 --- a/src/messenger/attachments/infrastructure/postgres.py +++ b/src/messenger/attachments/infrastructure/postgres.py @@ -7,7 +7,7 @@ from asyncpg import ForeignKeyViolationError, Record from messenger.attachments.application.ports import OutboxEntry, StagedObject from messenger.attachments.domain.errors import ChatNotFound -from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome +from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome from messenger.attachments.domain.models import Attachment, AttachmentStatus from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind from messenger.attachments.domain.variants import Variant @@ -56,9 +56,17 @@ ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED""" REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'" STAGED = """ SELECT id, staging_key FROM attachments -WHERE staging_key IS NOT NULL AND status = 'ready' +WHERE staging_key IS NOT NULL + AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2))) ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED """ +FAILED_JOBS = """ +SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error, + j.updated_at AS failed_at +FROM jobs j JOIN attachments a ON a.id = j.attachment_id +WHERE j.status = 'failed' +ORDER BY j.updated_at DESC LIMIT $1 +""" UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])" ADD_JOB = """ @@ -162,6 +170,33 @@ SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token WHERE status = 'processing' AND lease_until < now() ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED """ +REPLAY = """ +WITH job AS ( + UPDATE jobs + SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL, + next_attempt_at = NULL, last_error = NULL, updated_at = now() + WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed' + ORDER BY pipeline_version DESC LIMIT 1) + RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at +), moved AS ( + UPDATE attachments a SET status = 'queued', error_code = NULL, + updated_at = greatest(now(), a.created_at) + FROM job WHERE a.id = job.attachment_id AND a.status = 'failed' +), audit AS ( + INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) + SELECT job_id, attachment_id, 'replayed', 1, + jsonb_build_object('operator', $2::text, 'error_code', $3::text) + FROM job +) +SELECT job_id, pipeline_version, created_at, updated_at FROM job +""" +NOTE_REPLAY = """ +INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail) +SELECT job_id, attachment_id, 'replay_noop', attempt, + jsonb_build_object('operator', $2::text, 'result', $3::text) +FROM jobs WHERE attachment_id = $1 +ORDER BY pipeline_version DESC LIMIT 1 +""" PENDING = """ SELECT id, job_id, payload::text AS body FROM outbox WHERE published_at IS NULL AND available_at <= now() @@ -243,8 +278,8 @@ class PostgresAttachmentRepository: await self._connection.execute(REMOVE, attachment.id) self._seen.append(attachment) - async def staged(self, limit: int) -> list[StagedObject]: - rows = await self._connection.fetch(STAGED, limit) + async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: + rows = await self._connection.fetch(STAGED, limit, retention.total_seconds()) return [StagedObject(row["id"], row["staging_key"]) for row in rows] async def unstage(self, attachment_ids: Sequence[UUID]) -> None: @@ -358,6 +393,19 @@ class PostgresJobRepository: for row in rows ] + async def replay(self, attachment: Attachment, operator: str) -> Job | None: + row = await self._connection.fetchrow( + REPLAY, attachment.id, operator, attachment.error_code + ) + if row is None: + return None + job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"]) + await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"]) + return job + + async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: + await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result) + class PostgresOutboxRepository: __slots__ = ("_connection",) @@ -407,3 +455,7 @@ class PostgresAttachmentDirectory: async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: async with self._database.connection() as connection: return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id)) + + async def failed_jobs(self, limit: int) -> list[FailedJob]: + async with self._database.connection() as connection: + return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)] diff --git a/src/messenger/migrations/0005_failed_registry.sql b/src/messenger/migrations/0005_failed_registry.sql new file mode 100644 index 0000000..e3244d5 --- /dev/null +++ b/src/messenger/migrations/0005_failed_registry.sql @@ -0,0 +1 @@ +CREATE INDEX jobs_failed_idx ON jobs (updated_at DESC) WHERE status = 'failed'; diff --git a/student-tests/api/test_bounded_contexts.py b/student-tests/api/test_bounded_contexts.py index 9299c52..b588703 100644 --- a/student-tests/api/test_bounded_contexts.py +++ b/student-tests/api/test_bounded_contexts.py @@ -154,7 +154,7 @@ def test_every_zone_owns_its_http_layer() -> None: assert (SOURCE / zone / "entrypoints" / part).is_file(), f"{zone}: нет {part}" -PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,)} +PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,), "admin.py": (ATTACHMENTS,)} def test_only_main_knows_several_zones() -> None: diff --git a/student-tests/api/test_migrations.py b/student-tests/api/test_migrations.py index db11500..50af1a5 100644 --- a/student-tests/api/test_migrations.py +++ b/student-tests/api/test_migrations.py @@ -48,12 +48,15 @@ async def counts() -> list[int]: async def test_migrations_are_ordered_and_versioned() -> None: versions = [version for version, _ in migrations()] assert versions == sorted(versions) - assert versions[:4] == [ + assert versions[:5] == [ "0001_init", "0002_idempotency_keys", "0003_attachments", "0004_jobs", + "0005_failed_registry", ] + plan = await scalar("SELECT indexdef FROM pg_indexes WHERE indexname = 'jobs_failed_idx'") + assert "WHERE (status = 'failed'::text)" in plan applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations") assert applied == versions diff --git a/student-tests/api/test_recovery.py b/student-tests/api/test_recovery.py index 9d92a86..e40b1e4 100644 --- a/student-tests/api/test_recovery.py +++ b/student-tests/api/test_recovery.py @@ -1,14 +1,18 @@ import asyncio +import json import subprocess import sys from datetime import timedelta +from uuid import UUID, uuid4 import pytest from httpx import AsyncClient from test_attachments import storage -from test_pipeline import accepted, events, job_of, rows, state, variants +from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants +from conftest import DATABASE from fakes import NullLogger +from messenger.admin import serve from messenger.attachments.application.pipeline import ( AFTER_CLAIM, AFTER_COMMIT_BEFORE_ACK, @@ -19,7 +23,7 @@ from messenger.attachments.application.pipeline import ( TRANSIENT_ERROR, ) from messenger.attachments.application.ports import Rendition -from messenger.attachments.domain.jobs import Outcome +from messenger.attachments.domain.jobs import Outcome, ReplayResult from messenger.attachments.infrastructure.faults import EnvironmentFaults from messenger.shared.application.errors import Unavailable from pipeline import InlinePipeline, InlineRenderer, pipeline_of @@ -229,3 +233,74 @@ def test_environment_fault_crashes_the_process_with_137() -> None: env={"PYTHONPATH": ":".join(sys.path)}, ) assert (done.returncode, done.stdout) == (137, "") + + +async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]: + _, _, attachment_id = await accepted(client) + pipeline = await pipeline_of(client) + pipeline.faults = FailingFaults() + await pipeline.drain() + pipeline.faults = RecordingFaults() + assert (await state(attachment_id))["error_code"] == "retry_exhausted" + return pipeline, attachment_id + + +async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None: + pipeline, attachment_id = await failed(client) + job = await job_of(attachment_id) + assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED + queued = await state(attachment_id) + assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == ( + "queued", + "queued", + 1, + None, + ) + assert await outbox(attachment_id) == [True] * 5 + [False] + await pipeline.drain() + done = await state(attachment_id) + assert (done["status"], done["attempt"]) == ("ready", 1) + assert (await job_of(attachment_id)).id == job.id + before = await variants(attachment_id) + assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP + assert await outbox(attachment_id) == [True] * 6 + assert await variants(attachment_id) == before + assert (await events(attachment_id))[-1] == "replay_noop" + assert "replayed" in await events(attachment_id) + assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND + + +async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None: + pipeline, attachment_id = await failed(client) + _, _, waiting = await accepted(client) + objects = (await storage(client)).objects + await pipeline.clean(timedelta(days=1)) + assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}" + assert f"staging/{attachment_id}" in objects + await pipeline.clean(timedelta(0)) + assert (await state(attachment_id))["staging_key"] is None + assert f"staging/{attachment_id}" not in objects + assert (await state(waiting))["staging_key"] == f"staging/{waiting}" + assert f"staging/{waiting}" in objects + assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED + assert (await state(attachment_id))["status"] == "failed" + + +async def test_operator_cli_lists_failed_jobs_and_replays( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + _, attachment_id = await failed(client) + monkeypatch.setenv("DATABASE_URL", DATABASE["url"]) + monkeypatch.setenv("LOG_LEVEL", "WARNING") + assert await serve(["failed", "--limit", "500"]) == 0 + registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"] + (mine,) = [row for row in registry if row["attachment_id"] == attachment_id] + assert (mine["error_code"], mine["attempt"], mine["last_error"]) == ( + "retry_exhausted", + 5, + "Unavailable", + ) + assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0 + report = json.loads(capsys.readouterr().out.splitlines()[-1]) + assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"} + assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1 diff --git a/student-tests/pipeline.py b/student-tests/pipeline.py index 914c978..bd91fde 100644 --- a/student-tests/pipeline.py +++ b/student-tests/pipeline.py @@ -11,12 +11,14 @@ from messenger.attachments.application.commands import ( ProcessJob, PublishOutbox, RecoverExpiredLeases, + ReplayAttachment, ) from messenger.attachments.application.pipeline import ( CleanStagingHandler, ProcessJobHandler, PublishOutboxHandler, RecoverExpiredLeasesHandler, + ReplayAttachmentHandler, ) from messenger.attachments.application.ports import ( Faults, @@ -25,7 +27,7 @@ from messenger.attachments.application.ports import ( Rendition, VariantRenderer, ) -from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy +from messenger.attachments.domain.jobs import Job, Outcome, ReplayResult, RetryPolicy from messenger.attachments.entrypoints.schemas import JobMessage from messenger.attachments.infrastructure.faults import EnvironmentFaults from messenger.attachments.infrastructure.images import RenderLimits, render_variants @@ -141,8 +143,13 @@ class InlinePipeline: handler = RecoverExpiredLeasesHandler(self.uow(), self.policy) return await handler(RecoverExpiredLeases(100)) - async def clean(self) -> int: - return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100)) + async def clean(self, retention: timedelta = timedelta(days=1)) -> int: + command = CleanStaging(100, retention) + return await CleanStagingHandler(self.uow(), self.storage)(command) + + async def replay(self, attachment_id: UUID, operator: str = "test") -> ReplayResult: + command = ReplayAttachment(attachment_id, operator) + return await ReplayAttachmentHandler(self.uow())(command) async def pipeline_of(