Add the failed registry, an explicit replay and failed staging retention

python -m messenger.admin is a new process assembly over the attachments
zone with only a database: "failed" lists failed jobs (served by the new
partial index jobs_failed_idx), "replay --attachment-id" moves a failed
attachment back to queued with the same job_id, attempt 1 and a fresh
outbox row, writing a replayed event with the operator. A ready, queued
or processing attachment is a noop (replay_noop event), a failed one
whose staging is already gone is expired. Every request is audited.

The relay now also removes the staging object of a failed attachment
once STAGING_RETENTION_SECONDS have passed since it failed; replay and
the cleanup take the same row lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-29 03:12:19 +03:00
co-authored by Claude Opus 5.5
parent ecb4827e75
commit b568fcf9c9
18 changed files with 365 additions and 24 deletions
@@ -7,7 +7,7 @@ from asyncpg import ForeignKeyViolationError, Record
from messenger.attachments.application.ports import OutboxEntry, StagedObject
from messenger.attachments.domain.errors import ChatNotFound
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.models import Attachment, AttachmentStatus
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
from messenger.attachments.domain.variants import Variant
@@ -56,9 +56,17 @@ ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
STAGED = """
SELECT id, staging_key FROM attachments
WHERE staging_key IS NOT NULL AND status = 'ready'
WHERE staging_key IS NOT NULL
AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED
"""
FAILED_JOBS = """
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error,
j.updated_at AS failed_at
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed'
ORDER BY j.updated_at DESC LIMIT $1
"""
UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])"
ADD_JOB = """
@@ -162,6 +170,33 @@ SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token
WHERE status = 'processing' AND lease_until < now()
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED
"""
REPLAY = """
WITH job AS (
UPDATE jobs
SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL,
next_attempt_at = NULL, last_error = NULL, updated_at = now()
WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed'
ORDER BY pipeline_version DESC LIMIT 1)
RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at
), moved AS (
UPDATE attachments a SET status = 'queued', error_code = NULL,
updated_at = greatest(now(), a.created_at)
FROM job WHERE a.id = job.attachment_id AND a.status = 'failed'
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'replayed', 1,
jsonb_build_object('operator', $2::text, 'error_code', $3::text)
FROM job
)
SELECT job_id, pipeline_version, created_at, updated_at FROM job
"""
NOTE_REPLAY = """
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'replay_noop', attempt,
jsonb_build_object('operator', $2::text, 'result', $3::text)
FROM jobs WHERE attachment_id = $1
ORDER BY pipeline_version DESC LIMIT 1
"""
PENDING = """
SELECT id, job_id, payload::text AS body FROM outbox
WHERE published_at IS NULL AND available_at <= now()
@@ -243,8 +278,8 @@ class PostgresAttachmentRepository:
await self._connection.execute(REMOVE, attachment.id)
self._seen.append(attachment)
async def staged(self, limit: int) -> list[StagedObject]:
rows = await self._connection.fetch(STAGED, limit)
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
rows = await self._connection.fetch(STAGED, limit, retention.total_seconds())
return [StagedObject(row["id"], row["staging_key"]) for row in rows]
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
@@ -358,6 +393,19 @@ class PostgresJobRepository:
for row in rows
]
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
row = await self._connection.fetchrow(
REPLAY, attachment.id, operator, attachment.error_code
)
if row is None:
return None
job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"])
await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"])
return job
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result)
class PostgresOutboxRepository:
__slots__ = ("_connection",)
@@ -407,3 +455,7 @@ class PostgresAttachmentDirectory:
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
async with self._database.connection() as connection:
return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id))
async def failed_jobs(self, limit: int) -> list[FailedJob]:
async with self._database.connection() as connection:
return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)]