Add the failed registry, an explicit replay and failed staging retention

python -m messenger.admin is a new process assembly over the attachments
zone with only a database: "failed" lists failed jobs (served by the new
partial index jobs_failed_idx), "replay --attachment-id" moves a failed
attachment back to queued with the same job_id, attempt 1 and a fresh
outbox row, writing a replayed event with the operator. A ready, queued
or processing attachment is a noop (replay_noop event), a failed one
whose staging is already gone is expired. Every request is audited.

The relay now also removes the staging object of a failed attachment
once STAGING_RETENTION_SECONDS have passed since it failed; replay and
the cleanup take the same row lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 03:12:19 +03:00
co-authored by Claude Opus 5.5
parent ecb4827e75
commit b568fcf9c9
18 changed files with 365 additions and 24 deletions
@@ -24,11 +24,11 @@
## 5. Реестр, replay, retention
- [ ] 5.1 Миграция `0005_failed_registry.sql` (частичный индекс `failed`-заданий)
- [ ] 5.2 Команда `ReplayAttachment` и запрос реестра в зоне `attachments`, событие `JobReplayed` в журнале аудита
- [ ] 5.3 Точка входа `attachments/entrypoints/admin.py` и сборка `messenger/admin.py` (`failed`, `replay`), тест графа импортов
- [ ] 5.4 `CleanStaging` с retention `failed`, `STAGING_RETENTION_SECONDS` от 1 с
- [ ] 5.5 Проверка группы: тесты против PostgreSQL — replay `failed` → `ready` с тем же `job_id` и `attempt` 1, replay `ready` → `noop` без outbox, после retention → `expired`, staging `queued` не трогается; тест миграций
- [x] 5.1 Миграция `0005_failed_registry.sql` (частичный индекс `failed`-заданий)
- [x] 5.2 Команда `ReplayAttachment` и запрос реестра в зоне `attachments`, событие `JobReplayed` в журнале аудита
- [x] 5.3 Точка входа `attachments/entrypoints/admin.py` и сборка `messenger/admin.py` (`failed`, `replay`), тест графа импортов
- [x] 5.4 `CleanStaging` с retention `failed`, `STAGING_RETENTION_SECONDS` от 1 с
- [x] 5.5 Проверка группы: тесты против PostgreSQL — replay `failed` → `ready` с тем же `job_id` и `attempt` 1, replay `ready` → `noop` без outbox, после retention → `expired`, staging `queued` не трогается; тест миграций
## 6. Стенд и обвязка
+1
View File
@@ -94,6 +94,7 @@ ignore = [
"src/messenger/migrate.py" = ["TID251"]
"src/messenger/relay.py" = ["TID251"]
"src/messenger/worker.py" = ["TID251"]
"src/messenger/admin.py" = ["TID251"]
"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"]
[tool.ruff.lint.flake8-tidy-imports.banned-api]
+37
View File
@@ -0,0 +1,37 @@
import asyncio
import sys
from dishka import make_async_container
from messenger.attachments.domain.jobs import ReplayResult
from messenger.attachments.entrypoints.admin import rendered, run
from messenger.attachments.entrypoints.ioc import AdminProvider
from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal
from messenger.shared.application.messagebus import EventBus
from messenger.shared.config import PlatformSettings
from messenger.shared.entrypoints.ioc import PlatformProvider
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
async def serve(argv: list[str]) -> int:
platform = PlatformSettings()
logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS)
container = make_async_container(PlatformProvider(platform, logger), AdminProvider())
bus = await container.get(EventBus)
subscribe_journal(bus, logger)
bus.start()
try:
report = await run(container, argv)
finally:
await bus.stop()
await container.close()
sys.stdout.write(rendered(report) + "\n")
return int(report.get("result") == ReplayResult.NOT_FOUND)
def main() -> None:
sys.exit(asyncio.run(serve(sys.argv[1:])))
if __name__ == "__main__":
main()
@@ -40,3 +40,10 @@ class RecoverExpiredLeases(Command):
@dataclass(frozen=True, slots=True)
class CleanStaging(Command):
limit: int
retention: timedelta
@dataclass(frozen=True, slots=True)
class ReplayAttachment(Command):
attachment_id: UUID
operator: str
@@ -7,6 +7,7 @@ from messenger.attachments.application.commands import (
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
)
from messenger.attachments.application.ports import (
AttachmentsUnitOfWork,
@@ -17,9 +18,17 @@ from messenger.attachments.application.ports import (
VariantRenderer,
)
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
from messenger.attachments.domain.events import JobSettled
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy
from messenger.attachments.domain.models import original_key
from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled
from messenger.attachments.domain.jobs import (
Claim,
FailureCode,
Job,
Lease,
Outcome,
ReplayResult,
RetryPolicy,
)
from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key
from messenger.attachments.domain.variants import Variant, variant_key
from messenger.shared.application.errors import Unavailable
from messenger.shared.application.ports import IdGenerator
@@ -256,8 +265,37 @@ class CleanStagingHandler:
async def __call__(self, command: CleanStaging) -> int:
async with self._uow.transaction():
staged = await self._uow.attachments.staged(command.limit)
staged = await self._uow.attachments.staged(command.limit, command.retention)
await asyncio.gather(*(self._storage.delete(item.key) for item in staged))
await self._uow.attachments.unstage([item.attachment_id for item in staged])
await self._uow.commit()
return len(staged)
class ReplayAttachmentHandler:
__slots__ = ("_uow",)
def __init__(self, uow: AttachmentsUnitOfWork) -> None:
self._uow = uow
async def __call__(self, command: ReplayAttachment) -> ReplayResult:
async with self._uow.transaction():
attachment = await self._uow.attachments.lock(command.attachment_id)
result = await self._replay(attachment, command.operator)
if attachment is not None and result is not ReplayResult.REPLAYED:
await self._uow.jobs.note_replay(attachment.id, command.operator, result.value)
self._uow.record(
AttachmentReplayRequested(command.attachment_id, command.operator, result.value)
)
await self._uow.commit()
return result
async def _replay(self, attachment: Attachment | None, operator: str) -> ReplayResult:
if attachment is None:
return ReplayResult.NOT_FOUND
if attachment.status is not AttachmentStatus.FAILED:
return ReplayResult.NOOP
if attachment.staging_key is None:
return ReplayResult.EXPIRED
job = await self._uow.jobs.replay(attachment, operator)
return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED
@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
from typing import Protocol, runtime_checkable
from uuid import UUID
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.models import Attachment
from messenger.attachments.domain.values import ImageInfo, ImageKind
from messenger.attachments.domain.variants import Variant
@@ -95,7 +95,7 @@ class AttachmentRepository(Protocol):
async def remove(self, attachment: Attachment) -> None: ...
async def staged(self, limit: int) -> list[StagedObject]: ...
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ...
async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ...
@@ -122,6 +122,10 @@ class JobRepository(Protocol):
async def expired(self, limit: int) -> list[Lease]: ...
async def replay(self, attachment: Attachment, operator: str) -> Job | None: ...
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: ...
@runtime_checkable
class OutboxRepository(Protocol):
@@ -147,3 +151,5 @@ class AttachmentDirectory(Protocol):
async def visible(self, attachment_id: UUID) -> Attachment | None: ...
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ...
async def failed_jobs(self, limit: int) -> list[FailedJob]: ...
@@ -31,3 +31,11 @@ class JobSettled(DomainEvent):
error_code: str | None = None
delay_seconds: float | None = None
name: str = field(init=False, default="job.settled")
@dataclass(frozen=True, slots=True)
class AttachmentReplayRequested(DomainEvent):
attachment_id: UUID
operator: str
result: str
name: str = field(init=False, default="attachment.replay_requested")
+18
View File
@@ -14,6 +14,13 @@ class FailureCode(StrEnum):
RETRY_EXHAUSTED = "retry_exhausted"
class ReplayResult(StrEnum):
REPLAYED = "replayed"
NOOP = "noop"
EXPIRED = "expired"
NOT_FOUND = "not_found"
class Outcome(StrEnum):
DONE = "done"
DUPLICATE = "duplicate_noop"
@@ -61,6 +68,17 @@ class Claim:
staged: bool
@dataclass(frozen=True, slots=True)
class FailedJob:
job_id: UUID
attachment_id: UUID
pipeline_version: int
attempt: int
error_code: str | None
last_error: str | None
failed_at: datetime
@dataclass(frozen=True, slots=True)
class RetryPolicy:
max_attempts: int
@@ -0,0 +1,45 @@
import argparse
import json
import os
from dataclasses import asdict
from typing import Any
from uuid import UUID
from dishka import AsyncContainer
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import ReplayAttachment
from messenger.attachments.application.ports import AttachmentDirectory
from messenger.attachments.domain.jobs import ReplayResult
REGISTRY_LIMIT = 20
def parser() -> argparse.ArgumentParser:
root = argparse.ArgumentParser(
prog="python -m messenger.admin", description="Реестр failed-заданий и явный replay"
)
commands = root.add_subparsers(dest="command", required=True)
failed = commands.add_parser("failed", help="последние задания в failed")
failed.add_argument("--limit", type=int, default=REGISTRY_LIMIT)
replay = commands.add_parser("replay", help="вернуть failed-вложение в очередь")
replay.add_argument("--attachment-id", type=UUID, required=True)
replay.add_argument("--operator", default=os.environ.get("OPERATOR", "cli"))
return root
async def run(container: AsyncContainer, argv: list[str]) -> dict[str, Any]:
args = parser().parse_args(argv)
async with container() as scope:
if args.command == "failed":
directory = await scope.get(AttachmentDirectory)
return {"failed": [asdict(job) for job in await directory.failed_jobs(args.limit)]}
bus = await scope.get(AttachmentsCommandBus)
result: ReplayResult = await bus.execute(
ReplayAttachment(args.attachment_id, args.operator)
)
return {"result": result.value, "attachment_id": args.attachment_id, "operator": args.operator}
def rendered(report: dict[str, Any]) -> str:
return json.dumps(report, default=str, ensure_ascii=False)
@@ -11,6 +11,7 @@ from messenger.attachments.application.commands import (
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
SweepAbandonedUploads,
UploadAttachment,
)
@@ -24,6 +25,7 @@ from messenger.attachments.application.pipeline import (
ProcessJobHandler,
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
ReplayAttachmentHandler,
)
from messenger.attachments.application.ports import (
AttachmentDirectory,
@@ -245,3 +247,22 @@ class RelayProvider(Provider):
CleanStaging: CleanStagingHandler(uow, storage),
}
return AttachmentsCommandBus(handlers, uow, publisher)
class AdminProvider(Provider):
@provide(scope=Scope.APP)
def directory(self, database: Database) -> AttachmentDirectory:
return PostgresAttachmentDirectory(database)
@provide(scope=Scope.REQUEST)
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
return PostgresAttachmentsUnitOfWork(database)
@provide(scope=Scope.REQUEST)
def command_bus(
self, uow: AttachmentsUnitOfWork, publisher: EventPublisher
) -> AttachmentsCommandBus:
handlers: dict[type[Command], CommandHandler] = {
ReplayAttachment: ReplayAttachmentHandler(uow)
}
return AttachmentsCommandBus(handlers, uow, publisher)
@@ -1,4 +1,9 @@
from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued, JobSettled
from messenger.attachments.domain.events import (
AttachmentDiscarded,
AttachmentQueued,
AttachmentReplayRequested,
JobSettled,
)
from messenger.shared.application.messagebus import EventBus
from messenger.shared.application.ports import Logger
@@ -15,6 +20,8 @@ LOG_FIELDS = frozenset(
"outcome",
"error_code",
"delay_seconds",
"operator",
"result",
}
)
@@ -54,9 +61,18 @@ class AttachmentsJournal:
delay_seconds=event.delay_seconds,
)
async def on_replay_requested(self, event: AttachmentReplayRequested) -> None:
self._logger.info(
"audit.attachment_replay",
attachment_id=str(event.attachment_id),
operator=event.operator,
result=event.result,
)
def subscribe_journal(bus: EventBus, logger: Logger) -> None:
journal = AttachmentsJournal(logger)
bus.subscribe(AttachmentQueued, journal.on_attachment_queued)
bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded)
bus.subscribe(JobSettled, journal.on_job_settled)
bus.subscribe(AttachmentReplayRequested, journal.on_replay_requested)
@@ -1,6 +1,7 @@
import asyncio
from collections.abc import Sequence
from contextlib import suppress
from datetime import timedelta
from dishka import AsyncContainer
@@ -32,7 +33,12 @@ class Relay:
(settings.outbox_poll_seconds, (PublishOutbox(settings.outbox_batch),)),
(
settings.sweep_interval_seconds,
(RecoverExpiredLeases(settings.sweep_batch), CleanStaging(settings.sweep_batch)),
(
RecoverExpiredLeases(settings.sweep_batch),
CleanStaging(
settings.sweep_batch, timedelta(seconds=settings.staging_retention_seconds)
),
),
),
)
self._tasks = [asyncio.create_task(self._every(*loop)) for loop in loops]
@@ -7,7 +7,7 @@ from asyncpg import ForeignKeyViolationError, Record
from messenger.attachments.application.ports import OutboxEntry, StagedObject
from messenger.attachments.domain.errors import ChatNotFound
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.models import Attachment, AttachmentStatus
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
from messenger.attachments.domain.variants import Variant
@@ -56,9 +56,17 @@ ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
STAGED = """
SELECT id, staging_key FROM attachments
WHERE staging_key IS NOT NULL AND status = 'ready'
WHERE staging_key IS NOT NULL
AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED
"""
FAILED_JOBS = """
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error,
j.updated_at AS failed_at
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed'
ORDER BY j.updated_at DESC LIMIT $1
"""
UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])"
ADD_JOB = """
@@ -162,6 +170,33 @@ SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token
WHERE status = 'processing' AND lease_until < now()
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED
"""
REPLAY = """
WITH job AS (
UPDATE jobs
SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL,
next_attempt_at = NULL, last_error = NULL, updated_at = now()
WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed'
ORDER BY pipeline_version DESC LIMIT 1)
RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at
), moved AS (
UPDATE attachments a SET status = 'queued', error_code = NULL,
updated_at = greatest(now(), a.created_at)
FROM job WHERE a.id = job.attachment_id AND a.status = 'failed'
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'replayed', 1,
jsonb_build_object('operator', $2::text, 'error_code', $3::text)
FROM job
)
SELECT job_id, pipeline_version, created_at, updated_at FROM job
"""
NOTE_REPLAY = """
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'replay_noop', attempt,
jsonb_build_object('operator', $2::text, 'result', $3::text)
FROM jobs WHERE attachment_id = $1
ORDER BY pipeline_version DESC LIMIT 1
"""
PENDING = """
SELECT id, job_id, payload::text AS body FROM outbox
WHERE published_at IS NULL AND available_at <= now()
@@ -243,8 +278,8 @@ class PostgresAttachmentRepository:
await self._connection.execute(REMOVE, attachment.id)
self._seen.append(attachment)
async def staged(self, limit: int) -> list[StagedObject]:
rows = await self._connection.fetch(STAGED, limit)
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
rows = await self._connection.fetch(STAGED, limit, retention.total_seconds())
return [StagedObject(row["id"], row["staging_key"]) for row in rows]
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
@@ -358,6 +393,19 @@ class PostgresJobRepository:
for row in rows
]
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
row = await self._connection.fetchrow(
REPLAY, attachment.id, operator, attachment.error_code
)
if row is None:
return None
job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"])
await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"])
return job
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result)
class PostgresOutboxRepository:
__slots__ = ("_connection",)
@@ -407,3 +455,7 @@ class PostgresAttachmentDirectory:
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
async with self._database.connection() as connection:
return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id))
async def failed_jobs(self, limit: int) -> list[FailedJob]:
async with self._database.connection() as connection:
return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)]
@@ -0,0 +1 @@
CREATE INDEX jobs_failed_idx ON jobs (updated_at DESC) WHERE status = 'failed';
+1 -1
View File
@@ -154,7 +154,7 @@ def test_every_zone_owns_its_http_layer() -> None:
assert (SOURCE / zone / "entrypoints" / part).is_file(), f"{zone}: нет {part}"
PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,)}
PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,), "admin.py": (ATTACHMENTS,)}
def test_only_main_knows_several_zones() -> None:
+4 -1
View File
@@ -48,12 +48,15 @@ async def counts() -> list[int]:
async def test_migrations_are_ordered_and_versioned() -> None:
versions = [version for version, _ in migrations()]
assert versions == sorted(versions)
assert versions[:4] == [
assert versions[:5] == [
"0001_init",
"0002_idempotency_keys",
"0003_attachments",
"0004_jobs",
"0005_failed_registry",
]
plan = await scalar("SELECT indexdef FROM pg_indexes WHERE indexname = 'jobs_failed_idx'")
assert "WHERE (status = 'failed'::text)" in plan
applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations")
assert applied == versions
+77 -2
View File
@@ -1,14 +1,18 @@
import asyncio
import json
import subprocess
import sys
from datetime import timedelta
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from test_attachments import storage
from test_pipeline import accepted, events, job_of, rows, state, variants
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
from conftest import DATABASE
from fakes import NullLogger
from messenger.admin import serve
from messenger.attachments.application.pipeline import (
AFTER_CLAIM,
AFTER_COMMIT_BEFORE_ACK,
@@ -19,7 +23,7 @@ from messenger.attachments.application.pipeline import (
TRANSIENT_ERROR,
)
from messenger.attachments.application.ports import Rendition
from messenger.attachments.domain.jobs import Outcome
from messenger.attachments.domain.jobs import Outcome, ReplayResult
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.shared.application.errors import Unavailable
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
@@ -229,3 +233,74 @@ def test_environment_fault_crashes_the_process_with_137() -> None:
env={"PYTHONPATH": ":".join(sys.path)},
)
assert (done.returncode, done.stdout) == (137, "")
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
_, _, attachment_id = await accepted(client)
pipeline = await pipeline_of(client)
pipeline.faults = FailingFaults()
await pipeline.drain()
pipeline.faults = RecordingFaults()
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
return pipeline, attachment_id
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
job = await job_of(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
queued = await state(attachment_id)
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
"queued",
"queued",
1,
None,
)
assert await outbox(attachment_id) == [True] * 5 + [False]
await pipeline.drain()
done = await state(attachment_id)
assert (done["status"], done["attempt"]) == ("ready", 1)
assert (await job_of(attachment_id)).id == job.id
before = await variants(attachment_id)
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
assert await outbox(attachment_id) == [True] * 6
assert await variants(attachment_id) == before
assert (await events(attachment_id))[-1] == "replay_noop"
assert "replayed" in await events(attachment_id)
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
pipeline, attachment_id = await failed(client)
_, _, waiting = await accepted(client)
objects = (await storage(client)).objects
await pipeline.clean(timedelta(days=1))
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
assert f"staging/{attachment_id}" in objects
await pipeline.clean(timedelta(0))
assert (await state(attachment_id))["staging_key"] is None
assert f"staging/{attachment_id}" not in objects
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
assert f"staging/{waiting}" in objects
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
assert (await state(attachment_id))["status"] == "failed"
async def test_operator_cli_lists_failed_jobs_and_replays(
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
_, attachment_id = await failed(client)
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
monkeypatch.setenv("LOG_LEVEL", "WARNING")
assert await serve(["failed", "--limit", "500"]) == 0
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
"retry_exhausted",
5,
"Unavailable",
)
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
report = json.loads(capsys.readouterr().out.splitlines()[-1])
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1
+10 -3
View File
@@ -11,12 +11,14 @@ from messenger.attachments.application.commands import (
ProcessJob,
PublishOutbox,
RecoverExpiredLeases,
ReplayAttachment,
)
from messenger.attachments.application.pipeline import (
CleanStagingHandler,
ProcessJobHandler,
PublishOutboxHandler,
RecoverExpiredLeasesHandler,
ReplayAttachmentHandler,
)
from messenger.attachments.application.ports import (
Faults,
@@ -25,7 +27,7 @@ from messenger.attachments.application.ports import (
Rendition,
VariantRenderer,
)
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
from messenger.attachments.domain.jobs import Job, Outcome, ReplayResult, RetryPolicy
from messenger.attachments.entrypoints.schemas import JobMessage
from messenger.attachments.infrastructure.faults import EnvironmentFaults
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
@@ -141,8 +143,13 @@ class InlinePipeline:
handler = RecoverExpiredLeasesHandler(self.uow(), self.policy)
return await handler(RecoverExpiredLeases(100))
async def clean(self) -> int:
return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100))
async def clean(self, retention: timedelta = timedelta(days=1)) -> int:
command = CleanStaging(100, retention)
return await CleanStagingHandler(self.uow(), self.storage)(command)
async def replay(self, attachment_id: UUID, operator: str = "test") -> ReplayResult:
command = ReplayAttachment(attachment_id, operator)
return await ReplayAttachmentHandler(self.uow())(command)
async def pipeline_of(