Add the failed registry, an explicit replay and failed staging retention
python -m messenger.admin is a new process assembly over the attachments zone with only a database: "failed" lists failed jobs (served by the new partial index jobs_failed_idx), "replay --attachment-id" moves a failed attachment back to queued with the same job_id, attempt 1 and a fresh outbox row, writing a replayed event with the operator. A ready, queued or processing attachment is a noop (replay_noop event), a failed one whose staging is already gone is expired. Every request is audited. The relay now also removes the staging object of a failed attachment once STAGING_RETENTION_SECONDS have passed since it failed; replay and the cleanup take the same row lock. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ecb4827e75
commit
b568fcf9c9
@@ -7,7 +7,7 @@ from asyncpg import ForeignKeyViolationError, Record
|
||||
|
||||
from messenger.attachments.application.ports import OutboxEntry, StagedObject
|
||||
from messenger.attachments.domain.errors import ChatNotFound
|
||||
from messenger.attachments.domain.jobs import Claim, FailureCode, Job, Lease, Outcome
|
||||
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
|
||||
from messenger.attachments.domain.models import Attachment, AttachmentStatus
|
||||
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
|
||||
from messenger.attachments.domain.variants import Variant
|
||||
@@ -56,9 +56,17 @@ ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
|
||||
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
|
||||
STAGED = """
|
||||
SELECT id, staging_key FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND status = 'ready'
|
||||
WHERE staging_key IS NOT NULL
|
||||
AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
|
||||
ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
FAILED_JOBS = """
|
||||
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error,
|
||||
j.updated_at AS failed_at
|
||||
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.status = 'failed'
|
||||
ORDER BY j.updated_at DESC LIMIT $1
|
||||
"""
|
||||
UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])"
|
||||
|
||||
ADD_JOB = """
|
||||
@@ -162,6 +170,33 @@ SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token
|
||||
WHERE status = 'processing' AND lease_until < now()
|
||||
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
REPLAY = """
|
||||
WITH job AS (
|
||||
UPDATE jobs
|
||||
SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL,
|
||||
next_attempt_at = NULL, last_error = NULL, updated_at = now()
|
||||
WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed'
|
||||
ORDER BY pipeline_version DESC LIMIT 1)
|
||||
RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at
|
||||
), moved AS (
|
||||
UPDATE attachments a SET status = 'queued', error_code = NULL,
|
||||
updated_at = greatest(now(), a.created_at)
|
||||
FROM job WHERE a.id = job.attachment_id AND a.status = 'failed'
|
||||
), audit AS (
|
||||
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
||||
SELECT job_id, attachment_id, 'replayed', 1,
|
||||
jsonb_build_object('operator', $2::text, 'error_code', $3::text)
|
||||
FROM job
|
||||
)
|
||||
SELECT job_id, pipeline_version, created_at, updated_at FROM job
|
||||
"""
|
||||
NOTE_REPLAY = """
|
||||
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
||||
SELECT job_id, attachment_id, 'replay_noop', attempt,
|
||||
jsonb_build_object('operator', $2::text, 'result', $3::text)
|
||||
FROM jobs WHERE attachment_id = $1
|
||||
ORDER BY pipeline_version DESC LIMIT 1
|
||||
"""
|
||||
PENDING = """
|
||||
SELECT id, job_id, payload::text AS body FROM outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()
|
||||
@@ -243,8 +278,8 @@ class PostgresAttachmentRepository:
|
||||
await self._connection.execute(REMOVE, attachment.id)
|
||||
self._seen.append(attachment)
|
||||
|
||||
async def staged(self, limit: int) -> list[StagedObject]:
|
||||
rows = await self._connection.fetch(STAGED, limit)
|
||||
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
|
||||
rows = await self._connection.fetch(STAGED, limit, retention.total_seconds())
|
||||
return [StagedObject(row["id"], row["staging_key"]) for row in rows]
|
||||
|
||||
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
|
||||
@@ -358,6 +393,19 @@ class PostgresJobRepository:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
|
||||
row = await self._connection.fetchrow(
|
||||
REPLAY, attachment.id, operator, attachment.error_code
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"])
|
||||
await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"])
|
||||
return job
|
||||
|
||||
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
|
||||
await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result)
|
||||
|
||||
|
||||
class PostgresOutboxRepository:
|
||||
__slots__ = ("_connection",)
|
||||
@@ -407,3 +455,7 @@ class PostgresAttachmentDirectory:
|
||||
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
|
||||
async with self._database.connection() as connection:
|
||||
return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id))
|
||||
|
||||
async def failed_jobs(self, limit: int) -> list[FailedJob]:
|
||||
async with self._database.connection() as connection:
|
||||
return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)]
|
||||
|
||||
Reference in New Issue
Block a user