Add the failed registry, an explicit replay and failed staging retention
python -m messenger.admin is a new process assembly over the attachments zone with only a database: "failed" lists failed jobs (served by the new partial index jobs_failed_idx), "replay --attachment-id" moves a failed attachment back to queued with the same job_id, attempt 1 and a fresh outbox row, writing a replayed event with the operator. A ready, queued or processing attachment is a noop (replay_noop event), a failed one whose staging is already gone is expired. Every request is audited. The relay now also removes the staging object of a failed attachment once STAGING_RETENTION_SECONDS have passed since it failed; replay and the cleanup take the same row lock. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ecb4827e75
commit
b568fcf9c9
@@ -24,11 +24,11 @@
|
||||
|
||||
## 5. Реестр, replay, retention
|
||||
|
||||
- [ ] 5.1 Миграция `0005_failed_registry.sql` (частичный индекс `failed`-заданий)
|
||||
- [ ] 5.2 Команда `ReplayAttachment` и запрос реестра в зоне `attachments`, событие `JobReplayed` в журнале аудита
|
||||
- [ ] 5.3 Точка входа `attachments/entrypoints/admin.py` и сборка `messenger/admin.py` (`failed`, `replay`), тест графа импортов
|
||||
- [ ] 5.4 `CleanStaging` с retention `failed`, `STAGING_RETENTION_SECONDS` от 1 с
|
||||
- [ ] 5.5 Проверка группы: тесты против PostgreSQL — replay `failed` → `ready` с тем же `job_id` и `attempt` 1, replay `ready` → `noop` без outbox, после retention → `expired`, staging `queued` не трогается; тест миграций
|
||||
- [x] 5.1 Миграция `0005_failed_registry.sql` (частичный индекс `failed`-заданий)
|
||||
- [x] 5.2 Команда `ReplayAttachment` и запрос реестра в зоне `attachments`, событие `JobReplayed` в журнале аудита
|
||||
- [x] 5.3 Точка входа `attachments/entrypoints/admin.py` и сборка `messenger/admin.py` (`failed`, `replay`), тест графа импортов
|
||||
- [x] 5.4 `CleanStaging` с retention `failed`, `STAGING_RETENTION_SECONDS` от 1 с
|
||||
- [x] 5.5 Проверка группы: тесты против PostgreSQL — replay `failed` → `ready` с тем же `job_id` и `attempt` 1, replay `ready` → `noop` без outbox, после retention → `expired`, staging `queued` не трогается; тест миграций
|
||||
|
||||
## 6. Стенд и обвязка
|
||||
|
||||
|
||||
@@ -94,6 +94,7 @@ ignore = [
|
||||
"src/messenger/migrate.py" = ["TID251"]
|
||||
"src/messenger/relay.py" = ["TID251"]
|
||||
"src/messenger/worker.py" = ["TID251"]
|
||||
"src/messenger/admin.py" = ["TID251"]
|
||||
"src/messenger/gunicorn_conf.py" = ["TID251", "ARG002", "RUF012"]
|
||||
|
||||
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from dishka import make_async_container
|
||||
|
||||
from messenger.attachments.domain.jobs import ReplayResult
|
||||
from messenger.attachments.entrypoints.admin import rendered, run
|
||||
from messenger.attachments.entrypoints.ioc import AdminProvider
|
||||
from messenger.attachments.entrypoints.journal import LOG_FIELDS, subscribe_journal
|
||||
from messenger.shared.application.messagebus import EventBus
|
||||
from messenger.shared.config import PlatformSettings
|
||||
from messenger.shared.entrypoints.ioc import PlatformProvider
|
||||
from messenger.shared.infrastructure.logging import PLATFORM_FIELDS, StructlogLogger
|
||||
|
||||
|
||||
async def serve(argv: list[str]) -> int:
|
||||
platform = PlatformSettings()
|
||||
logger = StructlogLogger(level=platform.log_level, allowed_fields=PLATFORM_FIELDS | LOG_FIELDS)
|
||||
container = make_async_container(PlatformProvider(platform, logger), AdminProvider())
|
||||
bus = await container.get(EventBus)
|
||||
subscribe_journal(bus, logger)
|
||||
bus.start()
|
||||
try:
|
||||
report = await run(container, argv)
|
||||
finally:
|
||||
await bus.stop()
|
||||
await container.close()
|
||||
sys.stdout.write(rendered(report) + "\n")
|
||||
return int(report.get("result") == ReplayResult.NOT_FOUND)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
sys.exit(asyncio.run(serve(sys.argv[1:])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -40,3 +40,10 @@ class RecoverExpiredLeases(Command):
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CleanStaging(Command):
|
||||
limit: int
|
||||
retention: timedelta
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ReplayAttachment(Command):
|
||||
attachment_id: UUID
|
||||
operator: str
|
||||
|
||||
@@ -7,6 +7,7 @@ from messenger.attachments.application.commands import (
|
||||
ProcessJob,
|
||||
PublishOutbox,
|
||||
RecoverExpiredLeases,
|
||||
ReplayAttachment,
|
||||
)
|
||||
from messenger.attachments.application.ports import (
|
||||
AttachmentsUnitOfWork,
|
||||
@@ -17,9 +18,17 @@ from messenger.attachments.application.ports import (
|
||||
VariantRenderer,
|
||||
)
|
||||
from messenger.attachments.domain.errors import LeaseLost, UnreadableImage
|
||||
from messenger.attachments.domain.events import JobSettled
|
||||
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome, RetryPolicy
|
||||
from messenger.attachments.domain.models import original_key
|
||||
from messenger.attachments.domain.events import AttachmentReplayRequested, JobSettled
|
||||
from messenger.attachments.domain.jobs import (
|
||||
Claim,
|
||||
FailureCode,
|
||||
Job,
|
||||
Lease,
|
||||
Outcome,
|
||||
ReplayResult,
|
||||
RetryPolicy,
|
||||
)
|
||||
from messenger.attachments.domain.models import Attachment, AttachmentStatus, original_key
|
||||
from messenger.attachments.domain.variants import Variant, variant_key
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.application.ports import IdGenerator
|
||||
@@ -256,8 +265,37 @@ class CleanStagingHandler:
|
||||
|
||||
async def __call__(self, command: CleanStaging) -> int:
|
||||
async with self._uow.transaction():
|
||||
staged = await self._uow.attachments.staged(command.limit)
|
||||
staged = await self._uow.attachments.staged(command.limit, command.retention)
|
||||
await asyncio.gather(*(self._storage.delete(item.key) for item in staged))
|
||||
await self._uow.attachments.unstage([item.attachment_id for item in staged])
|
||||
await self._uow.commit()
|
||||
return len(staged)
|
||||
|
||||
|
||||
class ReplayAttachmentHandler:
|
||||
__slots__ = ("_uow",)
|
||||
|
||||
def __init__(self, uow: AttachmentsUnitOfWork) -> None:
|
||||
self._uow = uow
|
||||
|
||||
async def __call__(self, command: ReplayAttachment) -> ReplayResult:
|
||||
async with self._uow.transaction():
|
||||
attachment = await self._uow.attachments.lock(command.attachment_id)
|
||||
result = await self._replay(attachment, command.operator)
|
||||
if attachment is not None and result is not ReplayResult.REPLAYED:
|
||||
await self._uow.jobs.note_replay(attachment.id, command.operator, result.value)
|
||||
self._uow.record(
|
||||
AttachmentReplayRequested(command.attachment_id, command.operator, result.value)
|
||||
)
|
||||
await self._uow.commit()
|
||||
return result
|
||||
|
||||
async def _replay(self, attachment: Attachment | None, operator: str) -> ReplayResult:
|
||||
if attachment is None:
|
||||
return ReplayResult.NOT_FOUND
|
||||
if attachment.status is not AttachmentStatus.FAILED:
|
||||
return ReplayResult.NOOP
|
||||
if attachment.staging_key is None:
|
||||
return ReplayResult.EXPIRED
|
||||
job = await self._uow.jobs.replay(attachment, operator)
|
||||
return ReplayResult.NOOP if job is None else ReplayResult.REPLAYED
|
||||
|
||||
@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
|
||||
from typing import Protocol, runtime_checkable
|
||||
from uuid import UUID
|
||||
|
||||
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome
|
||||
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
|
||||
from messenger.attachments.domain.models import Attachment
|
||||
from messenger.attachments.domain.values import ImageInfo, ImageKind
|
||||
from messenger.attachments.domain.variants import Variant
|
||||
@@ -95,7 +95,7 @@ class AttachmentRepository(Protocol):
|
||||
|
||||
async def remove(self, attachment: Attachment) -> None: ...
|
||||
|
||||
async def staged(self, limit: int) -> list[StagedObject]: ...
|
||||
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]: ...
|
||||
|
||||
async def unstage(self, attachment_ids: Sequence[UUID]) -> None: ...
|
||||
|
||||
@@ -122,6 +122,10 @@ class JobRepository(Protocol):
|
||||
|
||||
async def expired(self, limit: int) -> list[Lease]: ...
|
||||
|
||||
async def replay(self, attachment: Attachment, operator: str) -> Job | None: ...
|
||||
|
||||
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class OutboxRepository(Protocol):
|
||||
@@ -147,3 +151,5 @@ class AttachmentDirectory(Protocol):
|
||||
async def visible(self, attachment_id: UUID) -> Attachment | None: ...
|
||||
|
||||
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool: ...
|
||||
|
||||
async def failed_jobs(self, limit: int) -> list[FailedJob]: ...
|
||||
|
||||
@@ -31,3 +31,11 @@ class JobSettled(DomainEvent):
|
||||
error_code: str | None = None
|
||||
delay_seconds: float | None = None
|
||||
name: str = field(init=False, default="job.settled")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AttachmentReplayRequested(DomainEvent):
|
||||
attachment_id: UUID
|
||||
operator: str
|
||||
result: str
|
||||
name: str = field(init=False, default="attachment.replay_requested")
|
||||
|
||||
@@ -14,6 +14,13 @@ class FailureCode(StrEnum):
|
||||
RETRY_EXHAUSTED = "retry_exhausted"
|
||||
|
||||
|
||||
class ReplayResult(StrEnum):
|
||||
REPLAYED = "replayed"
|
||||
NOOP = "noop"
|
||||
EXPIRED = "expired"
|
||||
NOT_FOUND = "not_found"
|
||||
|
||||
|
||||
class Outcome(StrEnum):
|
||||
DONE = "done"
|
||||
DUPLICATE = "duplicate_noop"
|
||||
@@ -61,6 +68,17 @@ class Claim:
|
||||
staged: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FailedJob:
|
||||
job_id: UUID
|
||||
attachment_id: UUID
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
error_code: str | None
|
||||
last_error: str | None
|
||||
failed_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RetryPolicy:
|
||||
max_attempts: int
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from dishka import AsyncContainer
|
||||
|
||||
from messenger.attachments.application.bus import AttachmentsCommandBus
|
||||
from messenger.attachments.application.commands import ReplayAttachment
|
||||
from messenger.attachments.application.ports import AttachmentDirectory
|
||||
from messenger.attachments.domain.jobs import ReplayResult
|
||||
|
||||
REGISTRY_LIMIT = 20
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser(
|
||||
prog="python -m messenger.admin", description="Реестр failed-заданий и явный replay"
|
||||
)
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
failed = commands.add_parser("failed", help="последние задания в failed")
|
||||
failed.add_argument("--limit", type=int, default=REGISTRY_LIMIT)
|
||||
replay = commands.add_parser("replay", help="вернуть failed-вложение в очередь")
|
||||
replay.add_argument("--attachment-id", type=UUID, required=True)
|
||||
replay.add_argument("--operator", default=os.environ.get("OPERATOR", "cli"))
|
||||
return root
|
||||
|
||||
|
||||
async def run(container: AsyncContainer, argv: list[str]) -> dict[str, Any]:
|
||||
args = parser().parse_args(argv)
|
||||
async with container() as scope:
|
||||
if args.command == "failed":
|
||||
directory = await scope.get(AttachmentDirectory)
|
||||
return {"failed": [asdict(job) for job in await directory.failed_jobs(args.limit)]}
|
||||
bus = await scope.get(AttachmentsCommandBus)
|
||||
result: ReplayResult = await bus.execute(
|
||||
ReplayAttachment(args.attachment_id, args.operator)
|
||||
)
|
||||
return {"result": result.value, "attachment_id": args.attachment_id, "operator": args.operator}
|
||||
|
||||
|
||||
def rendered(report: dict[str, Any]) -> str:
|
||||
return json.dumps(report, default=str, ensure_ascii=False)
|
||||
@@ -11,6 +11,7 @@ from messenger.attachments.application.commands import (
|
||||
ProcessJob,
|
||||
PublishOutbox,
|
||||
RecoverExpiredLeases,
|
||||
ReplayAttachment,
|
||||
SweepAbandonedUploads,
|
||||
UploadAttachment,
|
||||
)
|
||||
@@ -24,6 +25,7 @@ from messenger.attachments.application.pipeline import (
|
||||
ProcessJobHandler,
|
||||
PublishOutboxHandler,
|
||||
RecoverExpiredLeasesHandler,
|
||||
ReplayAttachmentHandler,
|
||||
)
|
||||
from messenger.attachments.application.ports import (
|
||||
AttachmentDirectory,
|
||||
@@ -245,3 +247,22 @@ class RelayProvider(Provider):
|
||||
CleanStaging: CleanStagingHandler(uow, storage),
|
||||
}
|
||||
return AttachmentsCommandBus(handlers, uow, publisher)
|
||||
|
||||
|
||||
class AdminProvider(Provider):
|
||||
@provide(scope=Scope.APP)
|
||||
def directory(self, database: Database) -> AttachmentDirectory:
|
||||
return PostgresAttachmentDirectory(database)
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def unit_of_work(self, database: Database) -> AttachmentsUnitOfWork:
|
||||
return PostgresAttachmentsUnitOfWork(database)
|
||||
|
||||
@provide(scope=Scope.REQUEST)
|
||||
def command_bus(
|
||||
self, uow: AttachmentsUnitOfWork, publisher: EventPublisher
|
||||
) -> AttachmentsCommandBus:
|
||||
handlers: dict[type[Command], CommandHandler] = {
|
||||
ReplayAttachment: ReplayAttachmentHandler(uow)
|
||||
}
|
||||
return AttachmentsCommandBus(handlers, uow, publisher)
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
from messenger.attachments.domain.events import AttachmentDiscarded, AttachmentQueued, JobSettled
|
||||
from messenger.attachments.domain.events import (
|
||||
AttachmentDiscarded,
|
||||
AttachmentQueued,
|
||||
AttachmentReplayRequested,
|
||||
JobSettled,
|
||||
)
|
||||
from messenger.shared.application.messagebus import EventBus
|
||||
from messenger.shared.application.ports import Logger
|
||||
|
||||
@@ -15,6 +20,8 @@ LOG_FIELDS = frozenset(
|
||||
"outcome",
|
||||
"error_code",
|
||||
"delay_seconds",
|
||||
"operator",
|
||||
"result",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -54,9 +61,18 @@ class AttachmentsJournal:
|
||||
delay_seconds=event.delay_seconds,
|
||||
)
|
||||
|
||||
async def on_replay_requested(self, event: AttachmentReplayRequested) -> None:
|
||||
self._logger.info(
|
||||
"audit.attachment_replay",
|
||||
attachment_id=str(event.attachment_id),
|
||||
operator=event.operator,
|
||||
result=event.result,
|
||||
)
|
||||
|
||||
|
||||
def subscribe_journal(bus: EventBus, logger: Logger) -> None:
|
||||
journal = AttachmentsJournal(logger)
|
||||
bus.subscribe(AttachmentQueued, journal.on_attachment_queued)
|
||||
bus.subscribe(AttachmentDiscarded, journal.on_attachment_discarded)
|
||||
bus.subscribe(JobSettled, journal.on_job_settled)
|
||||
bus.subscribe(AttachmentReplayRequested, journal.on_replay_requested)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
from collections.abc import Sequence
|
||||
from contextlib import suppress
|
||||
from datetime import timedelta
|
||||
|
||||
from dishka import AsyncContainer
|
||||
|
||||
@@ -32,7 +33,12 @@ class Relay:
|
||||
(settings.outbox_poll_seconds, (PublishOutbox(settings.outbox_batch),)),
|
||||
(
|
||||
settings.sweep_interval_seconds,
|
||||
(RecoverExpiredLeases(settings.sweep_batch), CleanStaging(settings.sweep_batch)),
|
||||
(
|
||||
RecoverExpiredLeases(settings.sweep_batch),
|
||||
CleanStaging(
|
||||
settings.sweep_batch, timedelta(seconds=settings.staging_retention_seconds)
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
self._tasks = [asyncio.create_task(self._every(*loop)) for loop in loops]
|
||||
|
||||
@@ -7,7 +7,7 @@ from asyncpg import ForeignKeyViolationError, Record
|
||||
|
||||
from messenger.attachments.application.ports import OutboxEntry, StagedObject
|
||||
from messenger.attachments.domain.errors import ChatNotFound
|
||||
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome
|
||||
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
|
||||
from messenger.attachments.domain.models import Attachment, AttachmentStatus
|
||||
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
|
||||
from messenger.attachments.domain.variants import Variant
|
||||
@@ -56,9 +56,17 @@ ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
|
||||
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
|
||||
STAGED = """
|
||||
SELECT id, staging_key FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND status = 'ready'
|
||||
WHERE staging_key IS NOT NULL
|
||||
AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
|
||||
ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
FAILED_JOBS = """
|
||||
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error,
|
||||
j.updated_at AS failed_at
|
||||
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.status = 'failed'
|
||||
ORDER BY j.updated_at DESC LIMIT $1
|
||||
"""
|
||||
UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])"
|
||||
|
||||
ADD_JOB = """
|
||||
@@ -162,6 +170,33 @@ SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token
|
||||
WHERE status = 'processing' AND lease_until < now()
|
||||
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
REPLAY = """
|
||||
WITH job AS (
|
||||
UPDATE jobs
|
||||
SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL,
|
||||
next_attempt_at = NULL, last_error = NULL, updated_at = now()
|
||||
WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed'
|
||||
ORDER BY pipeline_version DESC LIMIT 1)
|
||||
RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at
|
||||
), moved AS (
|
||||
UPDATE attachments a SET status = 'queued', error_code = NULL,
|
||||
updated_at = greatest(now(), a.created_at)
|
||||
FROM job WHERE a.id = job.attachment_id AND a.status = 'failed'
|
||||
), audit AS (
|
||||
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
||||
SELECT job_id, attachment_id, 'replayed', 1,
|
||||
jsonb_build_object('operator', $2::text, 'error_code', $3::text)
|
||||
FROM job
|
||||
)
|
||||
SELECT job_id, pipeline_version, created_at, updated_at FROM job
|
||||
"""
|
||||
NOTE_REPLAY = """
|
||||
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
||||
SELECT job_id, attachment_id, 'replay_noop', attempt,
|
||||
jsonb_build_object('operator', $2::text, 'result', $3::text)
|
||||
FROM jobs WHERE attachment_id = $1
|
||||
ORDER BY pipeline_version DESC LIMIT 1
|
||||
"""
|
||||
PENDING = """
|
||||
SELECT id, job_id, payload::text AS body FROM outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()
|
||||
@@ -243,8 +278,8 @@ class PostgresAttachmentRepository:
|
||||
await self._connection.execute(REMOVE, attachment.id)
|
||||
self._seen.append(attachment)
|
||||
|
||||
async def staged(self, limit: int) -> list[StagedObject]:
|
||||
rows = await self._connection.fetch(STAGED, limit)
|
||||
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
|
||||
rows = await self._connection.fetch(STAGED, limit, retention.total_seconds())
|
||||
return [StagedObject(row["id"], row["staging_key"]) for row in rows]
|
||||
|
||||
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
|
||||
@@ -358,6 +393,19 @@ class PostgresJobRepository:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
|
||||
row = await self._connection.fetchrow(
|
||||
REPLAY, attachment.id, operator, attachment.error_code
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"])
|
||||
await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"])
|
||||
return job
|
||||
|
||||
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
|
||||
await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result)
|
||||
|
||||
|
||||
class PostgresOutboxRepository:
|
||||
__slots__ = ("_connection",)
|
||||
@@ -407,3 +455,7 @@ class PostgresAttachmentDirectory:
|
||||
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
|
||||
async with self._database.connection() as connection:
|
||||
return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id))
|
||||
|
||||
async def failed_jobs(self, limit: int) -> list[FailedJob]:
|
||||
async with self._database.connection() as connection:
|
||||
return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)]
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX jobs_failed_idx ON jobs (updated_at DESC) WHERE status = 'failed';
|
||||
@@ -154,7 +154,7 @@ def test_every_zone_owns_its_http_layer() -> None:
|
||||
assert (SOURCE / zone / "entrypoints" / part).is_file(), f"{zone}: нет {part}"
|
||||
|
||||
|
||||
PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,)}
|
||||
PROCESSES = {"relay.py": (ATTACHMENTS,), "worker.py": (ATTACHMENTS,), "admin.py": (ATTACHMENTS,)}
|
||||
|
||||
|
||||
def test_only_main_knows_several_zones() -> None:
|
||||
|
||||
@@ -48,12 +48,15 @@ async def counts() -> list[int]:
|
||||
async def test_migrations_are_ordered_and_versioned() -> None:
|
||||
versions = [version for version, _ in migrations()]
|
||||
assert versions == sorted(versions)
|
||||
assert versions[:4] == [
|
||||
assert versions[:5] == [
|
||||
"0001_init",
|
||||
"0002_idempotency_keys",
|
||||
"0003_attachments",
|
||||
"0004_jobs",
|
||||
"0005_failed_registry",
|
||||
]
|
||||
plan = await scalar("SELECT indexdef FROM pg_indexes WHERE indexname = 'jobs_failed_idx'")
|
||||
assert "WHERE (status = 'failed'::text)" in plan
|
||||
applied = await scalar("SELECT array_agg(version ORDER BY version) FROM schema_migrations")
|
||||
assert applied == versions
|
||||
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
from test_attachments import storage
|
||||
from test_pipeline import accepted, events, job_of, rows, state, variants
|
||||
from test_pipeline import FailingFaults, accepted, events, job_of, rows, state, variants
|
||||
|
||||
from conftest import DATABASE
|
||||
from fakes import NullLogger
|
||||
from messenger.admin import serve
|
||||
from messenger.attachments.application.pipeline import (
|
||||
AFTER_CLAIM,
|
||||
AFTER_COMMIT_BEFORE_ACK,
|
||||
@@ -19,7 +23,7 @@ from messenger.attachments.application.pipeline import (
|
||||
TRANSIENT_ERROR,
|
||||
)
|
||||
from messenger.attachments.application.ports import Rendition
|
||||
from messenger.attachments.domain.jobs import Outcome
|
||||
from messenger.attachments.domain.jobs import Outcome, ReplayResult
|
||||
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from pipeline import InlinePipeline, InlineRenderer, pipeline_of
|
||||
@@ -229,3 +233,74 @@ def test_environment_fault_crashes_the_process_with_137() -> None:
|
||||
env={"PYTHONPATH": ":".join(sys.path)},
|
||||
)
|
||||
assert (done.returncode, done.stdout) == (137, "")
|
||||
|
||||
|
||||
async def failed(client: AsyncClient) -> tuple[InlinePipeline, str]:
|
||||
_, _, attachment_id = await accepted(client)
|
||||
pipeline = await pipeline_of(client)
|
||||
pipeline.faults = FailingFaults()
|
||||
await pipeline.drain()
|
||||
pipeline.faults = RecordingFaults()
|
||||
assert (await state(attachment_id))["error_code"] == "retry_exhausted"
|
||||
return pipeline, attachment_id
|
||||
|
||||
|
||||
async def test_replay_of_failed_reaches_ready_once(client: AsyncClient) -> None:
|
||||
pipeline, attachment_id = await failed(client)
|
||||
job = await job_of(attachment_id)
|
||||
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.REPLAYED
|
||||
queued = await state(attachment_id)
|
||||
assert (queued["status"], queued["job_status"], queued["attempt"], queued["error_code"]) == (
|
||||
"queued",
|
||||
"queued",
|
||||
1,
|
||||
None,
|
||||
)
|
||||
assert await outbox(attachment_id) == [True] * 5 + [False]
|
||||
await pipeline.drain()
|
||||
done = await state(attachment_id)
|
||||
assert (done["status"], done["attempt"]) == ("ready", 1)
|
||||
assert (await job_of(attachment_id)).id == job.id
|
||||
before = await variants(attachment_id)
|
||||
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.NOOP
|
||||
assert await outbox(attachment_id) == [True] * 6
|
||||
assert await variants(attachment_id) == before
|
||||
assert (await events(attachment_id))[-1] == "replay_noop"
|
||||
assert "replayed" in await events(attachment_id)
|
||||
assert await pipeline.replay(uuid4()) is ReplayResult.NOT_FOUND
|
||||
|
||||
|
||||
async def test_failed_staging_is_kept_for_the_retention_period(client: AsyncClient) -> None:
|
||||
pipeline, attachment_id = await failed(client)
|
||||
_, _, waiting = await accepted(client)
|
||||
objects = (await storage(client)).objects
|
||||
await pipeline.clean(timedelta(days=1))
|
||||
assert (await state(attachment_id))["staging_key"] == f"staging/{attachment_id}"
|
||||
assert f"staging/{attachment_id}" in objects
|
||||
await pipeline.clean(timedelta(0))
|
||||
assert (await state(attachment_id))["staging_key"] is None
|
||||
assert f"staging/{attachment_id}" not in objects
|
||||
assert (await state(waiting))["staging_key"] == f"staging/{waiting}"
|
||||
assert f"staging/{waiting}" in objects
|
||||
assert await pipeline.replay(UUID(attachment_id)) is ReplayResult.EXPIRED
|
||||
assert (await state(attachment_id))["status"] == "failed"
|
||||
|
||||
|
||||
async def test_operator_cli_lists_failed_jobs_and_replays(
|
||||
client: AsyncClient, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
_, attachment_id = await failed(client)
|
||||
monkeypatch.setenv("DATABASE_URL", DATABASE["url"])
|
||||
monkeypatch.setenv("LOG_LEVEL", "WARNING")
|
||||
assert await serve(["failed", "--limit", "500"]) == 0
|
||||
registry = json.loads(capsys.readouterr().out.splitlines()[-1])["failed"]
|
||||
(mine,) = [row for row in registry if row["attachment_id"] == attachment_id]
|
||||
assert (mine["error_code"], mine["attempt"], mine["last_error"]) == (
|
||||
"retry_exhausted",
|
||||
5,
|
||||
"Unavailable",
|
||||
)
|
||||
assert await serve(["replay", "--attachment-id", attachment_id, "--operator", "alice"]) == 0
|
||||
report = json.loads(capsys.readouterr().out.splitlines()[-1])
|
||||
assert report == {"result": "replayed", "attachment_id": attachment_id, "operator": "alice"}
|
||||
assert await serve(["replay", "--attachment-id", str(uuid4())]) == 1
|
||||
|
||||
@@ -11,12 +11,14 @@ from messenger.attachments.application.commands import (
|
||||
ProcessJob,
|
||||
PublishOutbox,
|
||||
RecoverExpiredLeases,
|
||||
ReplayAttachment,
|
||||
)
|
||||
from messenger.attachments.application.pipeline import (
|
||||
CleanStagingHandler,
|
||||
ProcessJobHandler,
|
||||
PublishOutboxHandler,
|
||||
RecoverExpiredLeasesHandler,
|
||||
ReplayAttachmentHandler,
|
||||
)
|
||||
from messenger.attachments.application.ports import (
|
||||
Faults,
|
||||
@@ -25,7 +27,7 @@ from messenger.attachments.application.ports import (
|
||||
Rendition,
|
||||
VariantRenderer,
|
||||
)
|
||||
from messenger.attachments.domain.jobs import Job, Outcome, RetryPolicy
|
||||
from messenger.attachments.domain.jobs import Job, Outcome, ReplayResult, RetryPolicy
|
||||
from messenger.attachments.entrypoints.schemas import JobMessage
|
||||
from messenger.attachments.infrastructure.faults import EnvironmentFaults
|
||||
from messenger.attachments.infrastructure.images import RenderLimits, render_variants
|
||||
@@ -141,8 +143,13 @@ class InlinePipeline:
|
||||
handler = RecoverExpiredLeasesHandler(self.uow(), self.policy)
|
||||
return await handler(RecoverExpiredLeases(100))
|
||||
|
||||
async def clean(self) -> int:
|
||||
return await CleanStagingHandler(self.uow(), self.storage)(CleanStaging(100))
|
||||
async def clean(self, retention: timedelta = timedelta(days=1)) -> int:
|
||||
command = CleanStaging(100, retention)
|
||||
return await CleanStagingHandler(self.uow(), self.storage)(command)
|
||||
|
||||
async def replay(self, attachment_id: UUID, operator: str = "test") -> ReplayResult:
|
||||
command = ReplayAttachment(attachment_id, operator)
|
||||
return await ReplayAttachmentHandler(self.uow())(command)
|
||||
|
||||
|
||||
async def pipeline_of(
|
||||
|
||||
Reference in New Issue
Block a user