python -m messenger.admin is a new process assembly over the attachments zone with only a database: "failed" lists failed jobs (served by the new partial index jobs_failed_idx), "replay --attachment-id" moves a failed attachment back to queued with the same job_id, attempt 1 and a fresh outbox row, writing a replayed event with the operator. A ready, queued or processing attachment is a noop (replay_noop event), a failed one whose staging is already gone is expired. Every request is audited. The relay now also removes the staging object of a failed attachment once STAGING_RETENTION_SECONDS have passed since it failed; replay and the cleanup take the same row lock. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
462 lines
18 KiB
Python
462 lines
18 KiB
Python
import json
|
|
from collections.abc import Sequence
|
|
from datetime import datetime, timedelta
|
|
from uuid import UUID
|
|
|
|
from asyncpg import ForeignKeyViolationError, Record
|
|
|
|
from messenger.attachments.application.ports import OutboxEntry, StagedObject
|
|
from messenger.attachments.domain.errors import ChatNotFound
|
|
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
|
|
from messenger.attachments.domain.models import Attachment, AttachmentStatus
|
|
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
|
|
from messenger.attachments.domain.variants import Variant
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.infrastructure.postgres import (
|
|
Connection,
|
|
Database,
|
|
EventSource,
|
|
PostgresUnitOfWork,
|
|
)
|
|
|
|
INSERT = """
|
|
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
|
width, height, staging_key, status, created_at, updated_at)
|
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
|
"""
|
|
MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'"
|
|
COLUMNS = """
|
|
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width,
|
|
a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code,
|
|
a.created_at, a.updated_at
|
|
FROM attachments a
|
|
"""
|
|
VISIBLE = """
|
|
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width,
|
|
a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code,
|
|
a.created_at, a.updated_at,
|
|
(SELECT coalesce(jsonb_agg(jsonb_build_array(v.name, v.content_type, v.size_bytes, v.width,
|
|
v.height, v.sha256, v.object_key)
|
|
ORDER BY v.name), '[]')
|
|
FROM attachment_variants v
|
|
WHERE v.attachment_id = a.id AND v.pipeline_version = a.published_version) AS variants
|
|
FROM attachments a
|
|
WHERE a.id = $1 AND a.status <> 'pending'
|
|
"""
|
|
FAILED_AMONG = """
|
|
SELECT EXISTS (SELECT 1 FROM attachments
|
|
WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'failed')
|
|
"""
|
|
LOCK = COLUMNS + "WHERE a.id = $1 FOR UPDATE"
|
|
ABANDONED = (
|
|
COLUMNS
|
|
+ """WHERE a.status = 'pending' AND a.created_at < $1
|
|
ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
|
|
)
|
|
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
|
|
STAGED = """
|
|
SELECT id, staging_key FROM attachments
|
|
WHERE staging_key IS NOT NULL
|
|
AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
|
|
ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED
|
|
"""
|
|
FAILED_JOBS = """
|
|
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error,
|
|
j.updated_at AS failed_at
|
|
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
|
WHERE j.status = 'failed'
|
|
ORDER BY j.updated_at DESC LIMIT $1
|
|
"""
|
|
UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])"
|
|
|
|
ADD_JOB = """
|
|
WITH job AS (
|
|
INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, attempt,
|
|
created_at, updated_at)
|
|
VALUES ($1, $2, $3, 'queued', $4, $5, $5)
|
|
RETURNING job_id
|
|
), box AS (
|
|
INSERT INTO outbox (job_id, payload, available_at)
|
|
SELECT job_id, $6::jsonb, now() FROM job
|
|
)
|
|
INSERT INTO job_events (job_id, attachment_id, kind, attempt)
|
|
SELECT job_id, $2, 'enqueued', $4 FROM job
|
|
"""
|
|
CLAIM = """
|
|
WITH claimed AS (
|
|
UPDATE jobs
|
|
SET status = 'processing', lease_token = $4, claims = claims + 1, updated_at = now(),
|
|
lease_until = now() + make_interval(secs => $5)
|
|
WHERE job_id = $1 AND attachment_id = $2 AND pipeline_version = $3
|
|
AND ((status = 'queued' AND coalesce(next_attempt_at, '-infinity') <= now())
|
|
OR (status = 'processing' AND lease_until < now()))
|
|
RETURNING job_id, attachment_id, attempt, created_at
|
|
), moved AS (
|
|
UPDATE attachments a SET status = 'processing', updated_at = greatest(now(), a.created_at)
|
|
FROM claimed c WHERE a.id = c.attachment_id AND a.status = 'queued'
|
|
), audit AS (
|
|
INSERT INTO job_events (job_id, attachment_id, kind, attempt)
|
|
SELECT job_id, attachment_id, 'claimed', attempt FROM claimed
|
|
)
|
|
SELECT c.attempt, c.created_at, a.staging_key, a.original_key
|
|
FROM claimed c JOIN attachments a ON a.id = c.attachment_id
|
|
"""
|
|
FINISH = """
|
|
UPDATE jobs SET status = 'done', lease_token = NULL, lease_until = NULL, next_attempt_at = NULL,
|
|
last_error = NULL, updated_at = now()
|
|
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
|
|
"""
|
|
ADD_VARIANTS = """
|
|
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,
|
|
size_bytes, width, height, sha256, created_at)
|
|
SELECT $1, $2, v.name, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256,
|
|
now()
|
|
FROM unnest($3::text[], $4::text[], $5::text[], $6::bigint[], $7::int[], $8::int[], $9::text[])
|
|
AS v (name, object_key, content_type, size_bytes, width, height, sha256)
|
|
ON CONFLICT DO NOTHING
|
|
"""
|
|
PUBLISH = """
|
|
UPDATE attachments
|
|
SET status = 'ready', published_version = $2, error_code = NULL, original_key = $3,
|
|
updated_at = greatest(now(), created_at)
|
|
WHERE id = $1 AND published_version < $2
|
|
"""
|
|
SUPERSEDE = "UPDATE jobs SET status = 'superseded', updated_at = now() WHERE job_id = $1"
|
|
RESCHEDULE = """
|
|
WITH job AS (
|
|
UPDATE jobs
|
|
SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL,
|
|
next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now()
|
|
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
|
|
RETURNING job_id, attachment_id, attempt, next_attempt_at
|
|
), moved AS (
|
|
UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at)
|
|
FROM job WHERE a.id = job.attachment_id AND a.status = 'processing'
|
|
), audit AS (
|
|
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
|
SELECT job_id, attachment_id, $5, attempt,
|
|
jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at,
|
|
'delay_seconds', $3::float8)
|
|
FROM job
|
|
)
|
|
SELECT attempt, next_attempt_at FROM job
|
|
"""
|
|
RENEW = """
|
|
UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now()
|
|
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
|
|
"""
|
|
ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)"
|
|
FAIL = """
|
|
WITH job AS (
|
|
UPDATE jobs SET status = 'failed', lease_token = NULL, lease_until = NULL, last_error = $4,
|
|
updated_at = now()
|
|
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
|
|
RETURNING job_id, attachment_id, attempt
|
|
), moved AS (
|
|
UPDATE attachments a SET status = 'failed', error_code = $3,
|
|
updated_at = greatest(now(), a.created_at)
|
|
FROM job WHERE a.id = job.attachment_id AND a.status IN ('queued', 'processing')
|
|
), audit AS (
|
|
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
|
SELECT job_id, attachment_id, 'failed', attempt,
|
|
jsonb_build_object('error_code', $3::text, 'error', $4::text)
|
|
FROM job
|
|
)
|
|
SELECT attempt FROM job
|
|
"""
|
|
NOTE = "INSERT INTO job_events (job_id, attachment_id, kind, attempt) VALUES ($1, $2, $3, $4)"
|
|
EXPIRED = """
|
|
SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token FROM jobs
|
|
WHERE status = 'processing' AND lease_until < now()
|
|
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED
|
|
"""
|
|
REPLAY = """
|
|
WITH job AS (
|
|
UPDATE jobs
|
|
SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL,
|
|
next_attempt_at = NULL, last_error = NULL, updated_at = now()
|
|
WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed'
|
|
ORDER BY pipeline_version DESC LIMIT 1)
|
|
RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at
|
|
), moved AS (
|
|
UPDATE attachments a SET status = 'queued', error_code = NULL,
|
|
updated_at = greatest(now(), a.created_at)
|
|
FROM job WHERE a.id = job.attachment_id AND a.status = 'failed'
|
|
), audit AS (
|
|
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
|
SELECT job_id, attachment_id, 'replayed', 1,
|
|
jsonb_build_object('operator', $2::text, 'error_code', $3::text)
|
|
FROM job
|
|
)
|
|
SELECT job_id, pipeline_version, created_at, updated_at FROM job
|
|
"""
|
|
NOTE_REPLAY = """
|
|
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
|
|
SELECT job_id, attachment_id, 'replay_noop', attempt,
|
|
jsonb_build_object('operator', $2::text, 'result', $3::text)
|
|
FROM jobs WHERE attachment_id = $1
|
|
ORDER BY pipeline_version DESC LIMIT 1
|
|
"""
|
|
PENDING = """
|
|
SELECT id, job_id, payload::text AS body FROM outbox
|
|
WHERE published_at IS NULL AND available_at <= now()
|
|
ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED
|
|
"""
|
|
PUBLISHED = "UPDATE outbox SET published_at = now() WHERE id = ANY($1::bigint[])"
|
|
|
|
|
|
def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment:
|
|
return Attachment(
|
|
id=row["id"],
|
|
chat_id=row["chat_id"],
|
|
owner_id=row["owner_id"],
|
|
filename=Filename(row["filename"]),
|
|
image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]),
|
|
size_bytes=row["size_bytes"],
|
|
sha256=row["sha256"],
|
|
status=AttachmentStatus(row["status"]),
|
|
created_at=row["created_at"],
|
|
updated_at=row["updated_at"],
|
|
staging_key=row["staging_key"],
|
|
original_key=row["original_key"],
|
|
published_version=row["published_version"],
|
|
error_code=row["error_code"],
|
|
variants=variants,
|
|
)
|
|
|
|
|
|
def payload(job: Job) -> str:
|
|
return json.dumps(job.payload(), separators=(",", ":"))
|
|
|
|
|
|
class PostgresAttachmentRepository:
|
|
__slots__ = ("_connection", "_seen")
|
|
|
|
def __init__(self, connection: Connection, seen: list[EventSource]) -> None:
|
|
self._connection = connection
|
|
self._seen = seen
|
|
|
|
async def add(self, attachment: Attachment) -> None:
|
|
try:
|
|
await self._connection.execute(
|
|
INSERT,
|
|
attachment.id,
|
|
attachment.chat_id,
|
|
attachment.owner_id,
|
|
attachment.filename.value,
|
|
attachment.content_type,
|
|
attachment.size_bytes,
|
|
attachment.sha256,
|
|
attachment.image.width,
|
|
attachment.image.height,
|
|
attachment.staging_key,
|
|
attachment.status.value,
|
|
attachment.created_at,
|
|
attachment.updated_at,
|
|
)
|
|
except ForeignKeyViolationError as error:
|
|
raise ChatNotFound("чат не найден") from error
|
|
self._seen.append(attachment)
|
|
|
|
async def save(self, attachment: Attachment) -> None:
|
|
done = await self._connection.execute(
|
|
MARK, attachment.id, attachment.status.value, attachment.updated_at
|
|
)
|
|
if done == "UPDATE 0":
|
|
raise Unavailable("намерение загрузки уже снято уборкой")
|
|
self._seen.append(attachment)
|
|
|
|
async def lock(self, attachment_id: UUID) -> Attachment | None:
|
|
row = await self._connection.fetchrow(LOCK, attachment_id)
|
|
return None if row is None else attachment_of(row)
|
|
|
|
async def next_abandoned(self, before: datetime) -> Attachment | None:
|
|
row = await self._connection.fetchrow(ABANDONED, before)
|
|
return None if row is None else attachment_of(row)
|
|
|
|
async def remove(self, attachment: Attachment) -> None:
|
|
await self._connection.execute(REMOVE, attachment.id)
|
|
self._seen.append(attachment)
|
|
|
|
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
|
|
rows = await self._connection.fetch(STAGED, limit, retention.total_seconds())
|
|
return [StagedObject(row["id"], row["staging_key"]) for row in rows]
|
|
|
|
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
|
|
if attachment_ids:
|
|
await self._connection.execute(UNSTAGE, list(attachment_ids))
|
|
|
|
|
|
class PostgresJobRepository:
|
|
__slots__ = ("_connection",)
|
|
|
|
def __init__(self, connection: Connection) -> None:
|
|
self._connection = connection
|
|
|
|
async def add(self, job: Job) -> None:
|
|
await self._connection.execute(
|
|
ADD_JOB,
|
|
job.id,
|
|
job.attachment_id,
|
|
job.pipeline_version,
|
|
job.attempt,
|
|
job.created_at,
|
|
payload(job),
|
|
)
|
|
|
|
async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None:
|
|
row = await self._connection.fetchrow(
|
|
CLAIM,
|
|
job.id,
|
|
job.attachment_id,
|
|
job.pipeline_version,
|
|
token,
|
|
lease.total_seconds(),
|
|
)
|
|
if row is None:
|
|
return None
|
|
claimed = Job(
|
|
job.id, job.attachment_id, job.pipeline_version, row["attempt"], row["created_at"]
|
|
)
|
|
staged = row["staging_key"] is not None
|
|
source = row["staging_key"] if staged else row["original_key"]
|
|
return Claim(Lease(claimed, token), source, staged)
|
|
|
|
async def complete(
|
|
self, lease: Lease, variants: Sequence[Variant], original_key: str
|
|
) -> Outcome:
|
|
job = lease.job
|
|
if await self._connection.execute(FINISH, job.id, lease.token) == "UPDATE 0":
|
|
await self.note(job, Outcome.FENCED)
|
|
return Outcome.FENCED
|
|
await self._connection.execute(
|
|
ADD_VARIANTS,
|
|
job.attachment_id,
|
|
job.pipeline_version,
|
|
[v.name for v in variants],
|
|
[v.object_key for v in variants],
|
|
[v.content_type for v in variants],
|
|
[v.size_bytes for v in variants],
|
|
[v.width for v in variants],
|
|
[v.height for v in variants],
|
|
[v.sha256 for v in variants],
|
|
)
|
|
published = await self._connection.execute(
|
|
PUBLISH, job.attachment_id, job.pipeline_version, original_key
|
|
)
|
|
outcome = Outcome.SUPERSEDED if published == "UPDATE 0" else Outcome.DONE
|
|
if outcome is Outcome.SUPERSEDED:
|
|
await self._connection.execute(SUPERSEDE, job.id)
|
|
await self.note(job, outcome)
|
|
return outcome
|
|
|
|
async def reschedule(
|
|
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
|
|
) -> Job | None:
|
|
row = await self._connection.fetchrow(
|
|
RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value
|
|
)
|
|
if row is None:
|
|
return None
|
|
job = lease.job.retried(row["attempt"])
|
|
await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"])
|
|
return job
|
|
|
|
async def renew(self, lease: Lease, duration: timedelta) -> bool:
|
|
done = await self._connection.execute(
|
|
RENEW, lease.job.id, lease.token, duration.total_seconds()
|
|
)
|
|
return done == "UPDATE 1"
|
|
|
|
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool:
|
|
attempt = await self._connection.fetchval(
|
|
FAIL, lease.job.id, lease.token, code.value, error
|
|
)
|
|
return attempt is not None
|
|
|
|
async def note(self, job: Job, kind: Outcome) -> None:
|
|
await self._connection.execute(NOTE, job.id, job.attachment_id, kind.value, job.attempt)
|
|
|
|
async def expired(self, limit: int) -> list[Lease]:
|
|
rows = await self._connection.fetch(EXPIRED, limit)
|
|
return [
|
|
Lease(
|
|
Job(
|
|
row["job_id"],
|
|
row["attachment_id"],
|
|
row["pipeline_version"],
|
|
row["attempt"],
|
|
row["created_at"],
|
|
),
|
|
row["lease_token"],
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
|
|
row = await self._connection.fetchrow(
|
|
REPLAY, attachment.id, operator, attachment.error_code
|
|
)
|
|
if row is None:
|
|
return None
|
|
job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"])
|
|
await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"])
|
|
return job
|
|
|
|
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
|
|
await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result)
|
|
|
|
|
|
class PostgresOutboxRepository:
|
|
__slots__ = ("_connection",)
|
|
|
|
def __init__(self, connection: Connection) -> None:
|
|
self._connection = connection
|
|
|
|
async def pending(self, limit: int) -> list[OutboxEntry]:
|
|
rows = await self._connection.fetch(PENDING, limit)
|
|
return [OutboxEntry(row["id"], row["job_id"], row["body"].encode()) for row in rows]
|
|
|
|
async def published(self, entry_ids: Sequence[int]) -> None:
|
|
await self._connection.execute(PUBLISHED, list(entry_ids))
|
|
|
|
|
|
class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork):
|
|
__slots__ = ()
|
|
|
|
@property
|
|
def attachments(self) -> PostgresAttachmentRepository:
|
|
return PostgresAttachmentRepository(self.connection, self.seen)
|
|
|
|
@property
|
|
def jobs(self) -> PostgresJobRepository:
|
|
return PostgresJobRepository(self.connection)
|
|
|
|
@property
|
|
def outbox(self) -> PostgresOutboxRepository:
|
|
return PostgresOutboxRepository(self.connection)
|
|
|
|
|
|
def variants_of(raw: str) -> tuple[Variant, ...]:
|
|
return tuple(Variant(*item) for item in json.loads(raw))
|
|
|
|
|
|
class PostgresAttachmentDirectory:
|
|
__slots__ = ("_database",)
|
|
|
|
def __init__(self, database: Database) -> None:
|
|
self._database = database
|
|
|
|
async def visible(self, attachment_id: UUID) -> Attachment | None:
|
|
async with self._database.connection() as connection:
|
|
row = await connection.fetchrow(VISIBLE, attachment_id)
|
|
return None if row is None else attachment_of(row, variants_of(row["variants"]))
|
|
|
|
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
|
|
async with self._database.connection() as connection:
|
|
return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id))
|
|
|
|
async def failed_jobs(self, limit: int) -> list[FailedJob]:
|
|
async with self._database.connection() as connection:
|
|
return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)]
|