Files
mtusi2026lab2/src/messenger/attachments/infrastructure/postgres.py
T
ArtyomandClaude Opus 5.5 b568fcf9c9 Add the failed registry, an explicit replay and failed staging retention
python -m messenger.admin is a new process assembly over the attachments
zone with only a database: "failed" lists failed jobs (served by the new
partial index jobs_failed_idx), "replay --attachment-id" moves a failed
attachment back to queued with the same job_id, attempt 1 and a fresh
outbox row, writing a replayed event with the operator. A ready, queued
or processing attachment is a noop (replay_noop event), a failed one
whose staging is already gone is expired. Every request is audited.

The relay now also removes the staging object of a failed attachment
once STAGING_RETENTION_SECONDS have passed since it failed; replay and
the cleanup take the same row lock.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:12:19 +03:00

462 lines
18 KiB
Python

import json
from collections.abc import Sequence
from datetime import datetime, timedelta
from uuid import UUID
from asyncpg import ForeignKeyViolationError, Record
from messenger.attachments.application.ports import OutboxEntry, StagedObject
from messenger.attachments.domain.errors import ChatNotFound
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
from messenger.attachments.domain.models import Attachment, AttachmentStatus
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
from messenger.attachments.domain.variants import Variant
from messenger.shared.application.errors import Unavailable
from messenger.shared.infrastructure.postgres import (
Connection,
Database,
EventSource,
PostgresUnitOfWork,
)
INSERT = """
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, staging_key, status, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
"""
MARK = "UPDATE attachments SET status = $2, updated_at = $3 WHERE id = $1 AND status = 'pending'"
COLUMNS = """
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width,
a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code,
a.created_at, a.updated_at
FROM attachments a
"""
VISIBLE = """
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.width,
a.height, a.status, a.staging_key, a.original_key, a.published_version, a.error_code,
a.created_at, a.updated_at,
(SELECT coalesce(jsonb_agg(jsonb_build_array(v.name, v.content_type, v.size_bytes, v.width,
v.height, v.sha256, v.object_key)
ORDER BY v.name), '[]')
FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.published_version) AS variants
FROM attachments a
WHERE a.id = $1 AND a.status <> 'pending'
"""
FAILED_AMONG = """
SELECT EXISTS (SELECT 1 FROM attachments
WHERE id = ANY($1::uuid[]) AND chat_id = $2 AND status = 'failed')
"""
LOCK = COLUMNS + "WHERE a.id = $1 FOR UPDATE"
ABANDONED = (
COLUMNS
+ """WHERE a.status = 'pending' AND a.created_at < $1
ORDER BY a.created_at LIMIT 1 FOR UPDATE SKIP LOCKED"""
)
REMOVE = "DELETE FROM attachments WHERE id = $1 AND status = 'pending'"
STAGED = """
SELECT id, staging_key FROM attachments
WHERE staging_key IS NOT NULL
AND (status = 'ready' OR (status = 'failed' AND updated_at < now() - make_interval(secs => $2)))
ORDER BY updated_at LIMIT $1 FOR UPDATE SKIP LOCKED
"""
FAILED_JOBS = """
SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, a.error_code, j.last_error,
j.updated_at AS failed_at
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
WHERE j.status = 'failed'
ORDER BY j.updated_at DESC LIMIT $1
"""
UNSTAGE = "UPDATE attachments SET staging_key = NULL WHERE id = ANY($1::uuid[])"
ADD_JOB = """
WITH job AS (
INSERT INTO jobs (job_id, attachment_id, pipeline_version, status, attempt,
created_at, updated_at)
VALUES ($1, $2, $3, 'queued', $4, $5, $5)
RETURNING job_id
), box AS (
INSERT INTO outbox (job_id, payload, available_at)
SELECT job_id, $6::jsonb, now() FROM job
)
INSERT INTO job_events (job_id, attachment_id, kind, attempt)
SELECT job_id, $2, 'enqueued', $4 FROM job
"""
CLAIM = """
WITH claimed AS (
UPDATE jobs
SET status = 'processing', lease_token = $4, claims = claims + 1, updated_at = now(),
lease_until = now() + make_interval(secs => $5)
WHERE job_id = $1 AND attachment_id = $2 AND pipeline_version = $3
AND ((status = 'queued' AND coalesce(next_attempt_at, '-infinity') <= now())
OR (status = 'processing' AND lease_until < now()))
RETURNING job_id, attachment_id, attempt, created_at
), moved AS (
UPDATE attachments a SET status = 'processing', updated_at = greatest(now(), a.created_at)
FROM claimed c WHERE a.id = c.attachment_id AND a.status = 'queued'
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt)
SELECT job_id, attachment_id, 'claimed', attempt FROM claimed
)
SELECT c.attempt, c.created_at, a.staging_key, a.original_key
FROM claimed c JOIN attachments a ON a.id = c.attachment_id
"""
FINISH = """
UPDATE jobs SET status = 'done', lease_token = NULL, lease_until = NULL, next_attempt_at = NULL,
last_error = NULL, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
"""
ADD_VARIANTS = """
INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,
size_bytes, width, height, sha256, created_at)
SELECT $1, $2, v.name, v.object_key, v.content_type, v.size_bytes, v.width, v.height, v.sha256,
now()
FROM unnest($3::text[], $4::text[], $5::text[], $6::bigint[], $7::int[], $8::int[], $9::text[])
AS v (name, object_key, content_type, size_bytes, width, height, sha256)
ON CONFLICT DO NOTHING
"""
PUBLISH = """
UPDATE attachments
SET status = 'ready', published_version = $2, error_code = NULL, original_key = $3,
updated_at = greatest(now(), created_at)
WHERE id = $1 AND published_version < $2
"""
SUPERSEDE = "UPDATE jobs SET status = 'superseded', updated_at = now() WHERE job_id = $1"
RESCHEDULE = """
WITH job AS (
UPDATE jobs
SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_until = NULL,
next_attempt_at = now() + make_interval(secs => $3), last_error = $4, updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
RETURNING job_id, attachment_id, attempt, next_attempt_at
), moved AS (
UPDATE attachments a SET status = 'queued', updated_at = greatest(now(), a.created_at)
FROM job WHERE a.id = job.attachment_id AND a.status = 'processing'
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, $5, attempt,
jsonb_build_object('error', $4::text, 'next_attempt_at', next_attempt_at,
'delay_seconds', $3::float8)
FROM job
)
SELECT attempt, next_attempt_at FROM job
"""
RENEW = """
UPDATE jobs SET lease_until = now() + make_interval(secs => $3), updated_at = now()
WHERE job_id = $1 AND lease_token = $2 AND status = 'processing'
"""
ENQUEUE = "INSERT INTO outbox (job_id, payload, available_at) VALUES ($1, $2::jsonb, $3)"
FAIL = """
WITH job AS (
UPDATE jobs SET status = 'failed', lease_token = NULL, lease_until = NULL, last_error = $4,
updated_at = now()
WHERE job_id = $1 AND status = 'processing' AND lease_token = $2
RETURNING job_id, attachment_id, attempt
), moved AS (
UPDATE attachments a SET status = 'failed', error_code = $3,
updated_at = greatest(now(), a.created_at)
FROM job WHERE a.id = job.attachment_id AND a.status IN ('queued', 'processing')
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'failed', attempt,
jsonb_build_object('error_code', $3::text, 'error', $4::text)
FROM job
)
SELECT attempt FROM job
"""
NOTE = "INSERT INTO job_events (job_id, attachment_id, kind, attempt) VALUES ($1, $2, $3, $4)"
EXPIRED = """
SELECT job_id, attachment_id, pipeline_version, attempt, created_at, lease_token FROM jobs
WHERE status = 'processing' AND lease_until < now()
ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED
"""
REPLAY = """
WITH job AS (
UPDATE jobs
SET status = 'queued', attempt = 1, lease_token = NULL, lease_until = NULL,
next_attempt_at = NULL, last_error = NULL, updated_at = now()
WHERE job_id = (SELECT job_id FROM jobs WHERE attachment_id = $1 AND status = 'failed'
ORDER BY pipeline_version DESC LIMIT 1)
RETURNING job_id, attachment_id, pipeline_version, created_at, updated_at
), moved AS (
UPDATE attachments a SET status = 'queued', error_code = NULL,
updated_at = greatest(now(), a.created_at)
FROM job WHERE a.id = job.attachment_id AND a.status = 'failed'
), audit AS (
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'replayed', 1,
jsonb_build_object('operator', $2::text, 'error_code', $3::text)
FROM job
)
SELECT job_id, pipeline_version, created_at, updated_at FROM job
"""
NOTE_REPLAY = """
INSERT INTO job_events (job_id, attachment_id, kind, attempt, detail)
SELECT job_id, attachment_id, 'replay_noop', attempt,
jsonb_build_object('operator', $2::text, 'result', $3::text)
FROM jobs WHERE attachment_id = $1
ORDER BY pipeline_version DESC LIMIT 1
"""
PENDING = """
SELECT id, job_id, payload::text AS body FROM outbox
WHERE published_at IS NULL AND available_at <= now()
ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED
"""
PUBLISHED = "UPDATE outbox SET published_at = now() WHERE id = ANY($1::bigint[])"
def attachment_of(row: Record, variants: tuple[Variant, ...] = ()) -> Attachment:
return Attachment(
id=row["id"],
chat_id=row["chat_id"],
owner_id=row["owner_id"],
filename=Filename(row["filename"]),
image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]),
size_bytes=row["size_bytes"],
sha256=row["sha256"],
status=AttachmentStatus(row["status"]),
created_at=row["created_at"],
updated_at=row["updated_at"],
staging_key=row["staging_key"],
original_key=row["original_key"],
published_version=row["published_version"],
error_code=row["error_code"],
variants=variants,
)
def payload(job: Job) -> str:
return json.dumps(job.payload(), separators=(",", ":"))
class PostgresAttachmentRepository:
__slots__ = ("_connection", "_seen")
def __init__(self, connection: Connection, seen: list[EventSource]) -> None:
self._connection = connection
self._seen = seen
async def add(self, attachment: Attachment) -> None:
try:
await self._connection.execute(
INSERT,
attachment.id,
attachment.chat_id,
attachment.owner_id,
attachment.filename.value,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
attachment.image.width,
attachment.image.height,
attachment.staging_key,
attachment.status.value,
attachment.created_at,
attachment.updated_at,
)
except ForeignKeyViolationError as error:
raise ChatNotFound("чат не найден") from error
self._seen.append(attachment)
async def save(self, attachment: Attachment) -> None:
done = await self._connection.execute(
MARK, attachment.id, attachment.status.value, attachment.updated_at
)
if done == "UPDATE 0":
raise Unavailable("намерение загрузки уже снято уборкой")
self._seen.append(attachment)
async def lock(self, attachment_id: UUID) -> Attachment | None:
row = await self._connection.fetchrow(LOCK, attachment_id)
return None if row is None else attachment_of(row)
async def next_abandoned(self, before: datetime) -> Attachment | None:
row = await self._connection.fetchrow(ABANDONED, before)
return None if row is None else attachment_of(row)
async def remove(self, attachment: Attachment) -> None:
await self._connection.execute(REMOVE, attachment.id)
self._seen.append(attachment)
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
rows = await self._connection.fetch(STAGED, limit, retention.total_seconds())
return [StagedObject(row["id"], row["staging_key"]) for row in rows]
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
if attachment_ids:
await self._connection.execute(UNSTAGE, list(attachment_ids))
class PostgresJobRepository:
__slots__ = ("_connection",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
async def add(self, job: Job) -> None:
await self._connection.execute(
ADD_JOB,
job.id,
job.attachment_id,
job.pipeline_version,
job.attempt,
job.created_at,
payload(job),
)
async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None:
row = await self._connection.fetchrow(
CLAIM,
job.id,
job.attachment_id,
job.pipeline_version,
token,
lease.total_seconds(),
)
if row is None:
return None
claimed = Job(
job.id, job.attachment_id, job.pipeline_version, row["attempt"], row["created_at"]
)
staged = row["staging_key"] is not None
source = row["staging_key"] if staged else row["original_key"]
return Claim(Lease(claimed, token), source, staged)
async def complete(
self, lease: Lease, variants: Sequence[Variant], original_key: str
) -> Outcome:
job = lease.job
if await self._connection.execute(FINISH, job.id, lease.token) == "UPDATE 0":
await self.note(job, Outcome.FENCED)
return Outcome.FENCED
await self._connection.execute(
ADD_VARIANTS,
job.attachment_id,
job.pipeline_version,
[v.name for v in variants],
[v.object_key for v in variants],
[v.content_type for v in variants],
[v.size_bytes for v in variants],
[v.width for v in variants],
[v.height for v in variants],
[v.sha256 for v in variants],
)
published = await self._connection.execute(
PUBLISH, job.attachment_id, job.pipeline_version, original_key
)
outcome = Outcome.SUPERSEDED if published == "UPDATE 0" else Outcome.DONE
if outcome is Outcome.SUPERSEDED:
await self._connection.execute(SUPERSEDE, job.id)
await self.note(job, outcome)
return outcome
async def reschedule(
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
) -> Job | None:
row = await self._connection.fetchrow(
RESCHEDULE, lease.job.id, lease.token, delay.total_seconds(), error, kind.value
)
if row is None:
return None
job = lease.job.retried(row["attempt"])
await self._connection.execute(ENQUEUE, job.id, payload(job), row["next_attempt_at"])
return job
async def renew(self, lease: Lease, duration: timedelta) -> bool:
done = await self._connection.execute(
RENEW, lease.job.id, lease.token, duration.total_seconds()
)
return done == "UPDATE 1"
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool:
attempt = await self._connection.fetchval(
FAIL, lease.job.id, lease.token, code.value, error
)
return attempt is not None
async def note(self, job: Job, kind: Outcome) -> None:
await self._connection.execute(NOTE, job.id, job.attachment_id, kind.value, job.attempt)
async def expired(self, limit: int) -> list[Lease]:
rows = await self._connection.fetch(EXPIRED, limit)
return [
Lease(
Job(
row["job_id"],
row["attachment_id"],
row["pipeline_version"],
row["attempt"],
row["created_at"],
),
row["lease_token"],
)
for row in rows
]
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
row = await self._connection.fetchrow(
REPLAY, attachment.id, operator, attachment.error_code
)
if row is None:
return None
job = Job(row["job_id"], attachment.id, row["pipeline_version"], 1, row["created_at"])
await self._connection.execute(ENQUEUE, job.id, payload(job), row["updated_at"])
return job
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
await self._connection.execute(NOTE_REPLAY, attachment_id, operator, result)
class PostgresOutboxRepository:
__slots__ = ("_connection",)
def __init__(self, connection: Connection) -> None:
self._connection = connection
async def pending(self, limit: int) -> list[OutboxEntry]:
rows = await self._connection.fetch(PENDING, limit)
return [OutboxEntry(row["id"], row["job_id"], row["body"].encode()) for row in rows]
async def published(self, entry_ids: Sequence[int]) -> None:
await self._connection.execute(PUBLISHED, list(entry_ids))
class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork):
__slots__ = ()
@property
def attachments(self) -> PostgresAttachmentRepository:
return PostgresAttachmentRepository(self.connection, self.seen)
@property
def jobs(self) -> PostgresJobRepository:
return PostgresJobRepository(self.connection)
@property
def outbox(self) -> PostgresOutboxRepository:
return PostgresOutboxRepository(self.connection)
def variants_of(raw: str) -> tuple[Variant, ...]:
return tuple(Variant(*item) for item in json.loads(raw))
class PostgresAttachmentDirectory:
__slots__ = ("_database",)
def __init__(self, database: Database) -> None:
self._database = database
async def visible(self, attachment_id: UUID) -> Attachment | None:
async with self._database.connection() as connection:
row = await connection.fetchrow(VISIBLE, attachment_id)
return None if row is None else attachment_of(row, variants_of(row["variants"]))
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
async with self._database.connection() as connection:
return bool(await connection.fetchval(FAILED_AMONG, list(attachment_ids), chat_id))
async def failed_jobs(self, limit: int) -> list[FailedJob]:
async with self._database.connection() as connection:
return [FailedJob(**row) for row in await connection.fetch(FAILED_JOBS, limit)]