forked from mtusi2026/mtusi2026lab6
848 lines
31 KiB
Python
848 lines
31 KiB
Python
import json
|
|
from collections.abc import AsyncIterator, Sequence
|
|
from contextlib import asynccontextmanager
|
|
from datetime import datetime, timedelta
|
|
from typing import Any, cast
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import (
|
|
BigInteger,
|
|
CheckConstraint,
|
|
DateTime,
|
|
ForeignKey,
|
|
ForeignKeyConstraint,
|
|
Identity,
|
|
Index,
|
|
Integer,
|
|
RowMapping,
|
|
Select,
|
|
Table,
|
|
Text,
|
|
UniqueConstraint,
|
|
delete,
|
|
func,
|
|
insert,
|
|
or_,
|
|
select,
|
|
update,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from messenger.attachments.application.ports import OutboxEntry, StagedObject
|
|
from messenger.attachments.domain.errors import ChatNotFound
|
|
from messenger.attachments.domain.jobs import Claim, FailedJob, FailureCode, Job, Lease, Outcome
|
|
from messenger.attachments.domain.models import Attachment, AttachmentStatus, Quarantine
|
|
from messenger.attachments.domain.values import Filename, ImageInfo, ImageKind
|
|
from messenger.attachments.domain.variants import Variant
|
|
from messenger.shared.application.errors import Unavailable
|
|
from messenger.shared.infrastructure.postgres import (
|
|
Base,
|
|
Database,
|
|
DatabaseSession,
|
|
EventSource,
|
|
PostgresUnitOfWork,
|
|
)
|
|
from messenger.shared.infrastructure.trace_context import capture_context
|
|
|
|
|
|
class AttachmentRow(Base):
|
|
__tablename__ = "attachments"
|
|
__table_args__ = (
|
|
CheckConstraint(
|
|
"char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check"
|
|
),
|
|
CheckConstraint(
|
|
"content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check"
|
|
),
|
|
CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check"),
|
|
CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"),
|
|
CheckConstraint("width > 0", name="attachments_width_check"),
|
|
CheckConstraint("height > 0", name="attachments_height_check"),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'queued', 'processing', 'ready', 'failed')",
|
|
name="attachments_status_check",
|
|
),
|
|
CheckConstraint(
|
|
"quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')",
|
|
name="attachments_quarantine_reason_check",
|
|
),
|
|
CheckConstraint(
|
|
"(quarantine_reason IS NULL) = (quarantined_at IS NULL)",
|
|
name="attachments_quarantine_check",
|
|
),
|
|
UniqueConstraint("object_key", name="attachments_object_key_key"),
|
|
CheckConstraint("published_version >= 0", name="attachments_published_version_check"),
|
|
CheckConstraint(
|
|
"error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')",
|
|
name="attachments_error_code_check",
|
|
),
|
|
UniqueConstraint("id", "chat_id", name="attachments_chat_key"),
|
|
ForeignKeyConstraint(
|
|
["chat_id", "owner_id"],
|
|
["chat_members.chat_id", "chat_members.user_id"],
|
|
name="attachments_owner_fkey",
|
|
),
|
|
)
|
|
|
|
id: Mapped[UUID] = mapped_column(primary_key=True)
|
|
chat_id: Mapped[UUID]
|
|
owner_id: Mapped[UUID]
|
|
filename: Mapped[str]
|
|
content_type: Mapped[str]
|
|
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
|
sha256: Mapped[str]
|
|
width: Mapped[int] = mapped_column(Integer)
|
|
height: Mapped[int] = mapped_column(Integer)
|
|
object_key: Mapped[str | None] = mapped_column(Text(collation="C"))
|
|
status: Mapped[str]
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
staging_key: Mapped[str | None] = mapped_column(Text(collation="C"), unique=True)
|
|
original_key: Mapped[str | None] = mapped_column(Text(collation="C"), unique=True)
|
|
published_version: Mapped[int] = mapped_column(server_default="0")
|
|
error_code: Mapped[str | None]
|
|
quarantine_reason: Mapped[str | None]
|
|
quarantined_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
Index(
|
|
"attachments_pending_idx",
|
|
AttachmentRow.created_at,
|
|
postgresql_where=AttachmentRow.status == "pending",
|
|
)
|
|
|
|
|
|
class VariantRow(Base):
|
|
__tablename__ = "attachment_variants"
|
|
__table_args__ = (
|
|
CheckConstraint("pipeline_version >= 1"),
|
|
CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')"),
|
|
CheckConstraint("content_type IN ('image/jpeg', 'image/png', 'image/webp')"),
|
|
CheckConstraint("size_bytes >= 1"),
|
|
CheckConstraint("width >= 1"),
|
|
CheckConstraint("height >= 1"),
|
|
CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'"),
|
|
)
|
|
|
|
attachment_id: Mapped[UUID] = mapped_column(ForeignKey("attachments.id"), primary_key=True)
|
|
pipeline_version: Mapped[int] = mapped_column(primary_key=True)
|
|
name: Mapped[str] = mapped_column(primary_key=True)
|
|
object_key: Mapped[str] = mapped_column(Text(collation="C"), unique=True)
|
|
content_type: Mapped[str]
|
|
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
|
width: Mapped[int]
|
|
height: Mapped[int]
|
|
sha256: Mapped[str]
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
class JobRow(Base):
|
|
__tablename__ = "jobs"
|
|
__table_args__ = (
|
|
CheckConstraint("pipeline_version >= 1"),
|
|
CheckConstraint("status IN ('queued', 'processing', 'done', 'failed', 'superseded')"),
|
|
CheckConstraint("attempt >= 1"),
|
|
CheckConstraint("claims >= 0"),
|
|
UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"),
|
|
)
|
|
|
|
job_id: Mapped[UUID] = mapped_column(primary_key=True)
|
|
attachment_id: Mapped[UUID] = mapped_column(ForeignKey("attachments.id"))
|
|
pipeline_version: Mapped[int]
|
|
status: Mapped[str]
|
|
attempt: Mapped[int]
|
|
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
|
|
claims: Mapped[int] = mapped_column(server_default="0")
|
|
lease_token: Mapped[UUID | None]
|
|
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
last_error: Mapped[str | None]
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
class OutboxRow(Base):
|
|
__tablename__ = "outbox"
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
|
|
job_id: Mapped[UUID] = mapped_column(ForeignKey("jobs.job_id"))
|
|
payload: Mapped[dict[str, Any]] = mapped_column(JSONB)
|
|
trace_context: Mapped[dict[str, str]] = mapped_column(JSONB, server_default="{}")
|
|
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
class JobEventRow(Base):
|
|
__tablename__ = "job_events"
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
|
|
job_id: Mapped[UUID]
|
|
attachment_id: Mapped[UUID]
|
|
kind: Mapped[str]
|
|
attempt: Mapped[int | None]
|
|
detail: Mapped[dict[str, Any]] = mapped_column(JSONB, server_default="{}")
|
|
at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
|
|
|
|
Index("jobs_lease_idx", JobRow.lease_until, postgresql_where=JobRow.status == "processing")
|
|
Index("jobs_failed_idx", JobRow.updated_at.desc(), postgresql_where=JobRow.status == "failed")
|
|
Index(
|
|
"outbox_pending_idx",
|
|
OutboxRow.available_at,
|
|
OutboxRow.id,
|
|
postgresql_where=OutboxRow.published_at.is_(None),
|
|
)
|
|
Index("outbox_job_idx", OutboxRow.job_id)
|
|
Index("job_events_attachment_idx", JobEventRow.attachment_id, JobEventRow.id)
|
|
Index(
|
|
"attachments_staged_idx",
|
|
AttachmentRow.updated_at,
|
|
postgresql_where=AttachmentRow.staging_key.is_not(None),
|
|
)
|
|
|
|
|
|
COLUMNS = select(AttachmentRow.__table__)
|
|
|
|
|
|
def attachment_of(row: RowMapping, variants: tuple[Variant, ...] = ()) -> Attachment:
|
|
return Attachment(
|
|
id=row["id"],
|
|
chat_id=row["chat_id"],
|
|
owner_id=row["owner_id"],
|
|
filename=Filename(row["filename"]),
|
|
image=ImageInfo(ImageKind(row["content_type"]), row["width"], row["height"]),
|
|
size_bytes=row["size_bytes"],
|
|
sha256=row["sha256"],
|
|
staging_key=row["staging_key"]
|
|
or (row["object_key"] if row["status"] == "pending" else None),
|
|
original_key=row["original_key"]
|
|
or (row["object_key"] if row["status"] == "ready" else None),
|
|
published_version=row["published_version"],
|
|
error_code=row["error_code"],
|
|
variants=variants,
|
|
status=AttachmentStatus(row["status"]),
|
|
created_at=row["created_at"],
|
|
updated_at=row["updated_at"],
|
|
quarantine=Quarantine(row["quarantine_reason"]) if row["quarantine_reason"] else None,
|
|
)
|
|
|
|
|
|
class PostgresAttachmentRepository:
|
|
__slots__ = ("_seen", "_session")
|
|
|
|
def __init__(self, session: DatabaseSession, seen: list[EventSource]) -> None:
|
|
self._session = session
|
|
self._seen = seen
|
|
|
|
async def add(self, attachment: Attachment) -> None:
|
|
try:
|
|
await self._session.execute(
|
|
insert(AttachmentRow).values(
|
|
id=attachment.id,
|
|
chat_id=attachment.chat_id,
|
|
owner_id=attachment.owner_id,
|
|
filename=attachment.filename.value,
|
|
content_type=attachment.content_type,
|
|
size_bytes=attachment.size_bytes,
|
|
sha256=attachment.sha256,
|
|
width=attachment.image.width,
|
|
height=attachment.image.height,
|
|
staging_key=attachment.staging_key,
|
|
original_key=attachment.original_key,
|
|
published_version=attachment.published_version,
|
|
status=attachment.status.value,
|
|
created_at=attachment.created_at,
|
|
updated_at=attachment.updated_at,
|
|
)
|
|
)
|
|
except IntegrityError as error:
|
|
if getattr(error.orig, "sqlstate", None) != "23503":
|
|
raise
|
|
raise ChatNotFound("чат не найден") from error
|
|
self._seen.append(attachment)
|
|
|
|
async def save(self, attachment: Attachment) -> None:
|
|
done = await self._session.scalar(
|
|
update(AttachmentRow)
|
|
.where(AttachmentRow.id == attachment.id, AttachmentRow.status == "pending")
|
|
.values(status=attachment.status.value, updated_at=attachment.updated_at)
|
|
.returning(AttachmentRow.id)
|
|
)
|
|
if done is None:
|
|
raise Unavailable("намерение загрузки уже снято уборкой")
|
|
self._seen.append(attachment)
|
|
|
|
async def _one(self, statement: Select) -> Attachment | None:
|
|
row = (await self._session.execute(statement)).mappings().first()
|
|
return None if row is None else attachment_of(row)
|
|
|
|
async def lock(self, attachment_id: UUID) -> Attachment | None:
|
|
return await self._one(COLUMNS.where(AttachmentRow.id == attachment_id).with_for_update())
|
|
|
|
async def next_abandoned(self, before: datetime) -> Attachment | None:
|
|
return await self._one(
|
|
COLUMNS.where(AttachmentRow.status == "pending", AttachmentRow.created_at < before)
|
|
.order_by(AttachmentRow.created_at)
|
|
.limit(1)
|
|
.with_for_update(skip_locked=True)
|
|
)
|
|
|
|
async def remove(self, attachment: Attachment) -> None:
|
|
await self._session.execute(
|
|
delete(AttachmentRow).where(
|
|
AttachmentRow.id == attachment.id, AttachmentRow.status == "pending"
|
|
)
|
|
)
|
|
self._seen.append(attachment)
|
|
|
|
async def staged(self, limit: int, retention: timedelta) -> list[StagedObject]:
|
|
rows = await self._session.execute(
|
|
select(AttachmentRow.id, AttachmentRow.staging_key)
|
|
.where(
|
|
AttachmentRow.staging_key.is_not(None),
|
|
~select(JobRow.job_id)
|
|
.where(
|
|
JobRow.attachment_id == AttachmentRow.id,
|
|
JobRow.status.in_(["queued", "processing"]),
|
|
)
|
|
.exists(),
|
|
or_(
|
|
AttachmentRow.status == "ready",
|
|
(AttachmentRow.status == "failed")
|
|
& (AttachmentRow.updated_at < func.now() - retention),
|
|
),
|
|
)
|
|
.order_by(AttachmentRow.updated_at)
|
|
.limit(limit)
|
|
.with_for_update(skip_locked=True)
|
|
)
|
|
return [StagedObject(row.id, row.staging_key) for row in rows]
|
|
|
|
async def unstage(self, attachment_ids: Sequence[UUID]) -> None:
|
|
if attachment_ids:
|
|
await self._session.execute(
|
|
update(AttachmentRow)
|
|
.where(AttachmentRow.id.in_(attachment_ids))
|
|
.values(staging_key=None)
|
|
)
|
|
|
|
async def page_after(self, key: str, limit: int) -> Sequence[Attachment]:
|
|
rows = await self._session.execute(
|
|
COLUMNS.where(AttachmentRow.object_key > key)
|
|
.order_by(AttachmentRow.object_key)
|
|
.limit(limit)
|
|
)
|
|
return [attachment_of(row) for row in rows.mappings()]
|
|
|
|
async def by_key(self, key: str) -> Attachment | None:
|
|
return await self._one(COLUMNS.where(AttachmentRow.object_key == key))
|
|
|
|
async def claim(self, attachment_id: UUID) -> Attachment | None:
|
|
return await self._one(
|
|
COLUMNS.where(AttachmentRow.id == attachment_id).with_for_update(skip_locked=True)
|
|
)
|
|
|
|
async def mark(self, attachment: Attachment) -> None:
|
|
reason = None if attachment.quarantine is None else attachment.quarantine.value
|
|
await self._session.execute(
|
|
update(AttachmentRow)
|
|
.where(AttachmentRow.id == attachment.id, AttachmentRow.status == "ready")
|
|
.values(quarantine_reason=reason, quarantined_at=None if reason is None else func.now())
|
|
)
|
|
|
|
|
|
class PostgresAttachmentsUnitOfWork(PostgresUnitOfWork):
|
|
__slots__ = ()
|
|
|
|
@property
|
|
def attachments(self) -> PostgresAttachmentRepository:
|
|
return PostgresAttachmentRepository(self.session, self.seen)
|
|
|
|
@property
|
|
def jobs(self) -> PostgresJobRepository:
|
|
return PostgresJobRepository(self.session)
|
|
|
|
@property
|
|
def outbox(self) -> PostgresOutboxRepository:
|
|
return PostgresOutboxRepository(self.session)
|
|
|
|
|
|
class PostgresAttachmentDirectory:
|
|
__slots__ = ("_database",)
|
|
|
|
def __init__(self, database: Database) -> None:
|
|
self._database = database
|
|
|
|
async def visible(self, attachment_id: UUID) -> Attachment | None:
|
|
async with self._database.session() as session:
|
|
row = (
|
|
(
|
|
await session.execute(
|
|
COLUMNS.where(
|
|
AttachmentRow.id == attachment_id, AttachmentRow.status != "pending"
|
|
)
|
|
)
|
|
)
|
|
.mappings()
|
|
.first()
|
|
)
|
|
if row is None:
|
|
return None
|
|
rows = (
|
|
await session.execute(
|
|
select(
|
|
VariantRow.name,
|
|
VariantRow.content_type,
|
|
VariantRow.size_bytes,
|
|
VariantRow.width,
|
|
VariantRow.height,
|
|
VariantRow.sha256,
|
|
VariantRow.object_key,
|
|
)
|
|
.where(
|
|
VariantRow.attachment_id == attachment_id,
|
|
VariantRow.pipeline_version == row["published_version"],
|
|
)
|
|
.order_by(VariantRow.name)
|
|
)
|
|
).all()
|
|
return attachment_of(row, tuple(Variant(*item) for item in rows))
|
|
|
|
async def failed_among(self, chat_id: UUID, attachment_ids: Sequence[UUID]) -> bool:
|
|
async with self._database.session() as session:
|
|
return bool(
|
|
await session.scalar(
|
|
select(
|
|
select(AttachmentRow.id)
|
|
.where(
|
|
AttachmentRow.id.in_(attachment_ids),
|
|
AttachmentRow.chat_id == chat_id,
|
|
AttachmentRow.status == "failed",
|
|
)
|
|
.exists()
|
|
)
|
|
)
|
|
)
|
|
|
|
async def failed_jobs(self, limit: int) -> list[FailedJob]:
|
|
async with self._database.session() as session:
|
|
rows = await session.execute(
|
|
select(
|
|
JobRow.job_id,
|
|
JobRow.attachment_id,
|
|
JobRow.pipeline_version,
|
|
JobRow.attempt,
|
|
AttachmentRow.error_code,
|
|
JobRow.last_error,
|
|
JobRow.updated_at.label("failed_at"),
|
|
)
|
|
.join(AttachmentRow, AttachmentRow.id == JobRow.attachment_id)
|
|
.where(JobRow.status == "failed")
|
|
.order_by(JobRow.updated_at.desc())
|
|
.limit(limit)
|
|
)
|
|
return [FailedJob(**row) for row in rows.mappings()]
|
|
|
|
|
|
class PostgresSingleRun:
|
|
__slots__ = ("_database", "_key")
|
|
|
|
def __init__(self, database: Database, key: int) -> None:
|
|
self._database = database
|
|
self._key = key
|
|
|
|
@asynccontextmanager
|
|
async def hold(self) -> AsyncIterator[bool]:
|
|
async with self._database.session() as session:
|
|
connection = await session.connection(
|
|
execution_options={"isolation_level": "AUTOCOMMIT"}
|
|
)
|
|
held = bool(await connection.scalar(select(func.pg_try_advisory_lock(self._key))))
|
|
try:
|
|
yield held
|
|
finally:
|
|
if held:
|
|
await connection.execute(select(func.pg_advisory_unlock(self._key)))
|
|
|
|
|
|
class PostgresJobRepository:
|
|
__slots__ = ("_session",)
|
|
|
|
def __init__(self, session: DatabaseSession) -> None:
|
|
self._session = session
|
|
|
|
async def _enqueue(self, job: Job, available_at: datetime | None = None) -> None:
|
|
context = (
|
|
capture_context()
|
|
or await self._session.scalar(
|
|
select(JobRow.trace_context).where(JobRow.job_id == job.id)
|
|
)
|
|
or {}
|
|
)
|
|
await self._session.execute(
|
|
insert(OutboxRow).values(
|
|
job_id=job.id,
|
|
payload=job.payload(),
|
|
trace_context=context,
|
|
available_at=func.now() if available_at is None else available_at,
|
|
)
|
|
)
|
|
|
|
async def _note(self, job: Job, kind: str, detail: dict[str, Any] | None = None) -> None:
|
|
await self._session.execute(
|
|
insert(JobEventRow).values(
|
|
job_id=job.id,
|
|
attachment_id=job.attachment_id,
|
|
kind=kind,
|
|
attempt=job.attempt,
|
|
detail=detail or {},
|
|
)
|
|
)
|
|
|
|
async def _move(
|
|
self, attachment_id: UUID, previous: Sequence[str], status: str, **values: Any
|
|
) -> None:
|
|
await self._session.execute(
|
|
update(AttachmentRow)
|
|
.where(AttachmentRow.id == attachment_id, AttachmentRow.status.in_(previous))
|
|
.values(
|
|
status=status,
|
|
updated_at=func.greatest(func.now(), AttachmentRow.created_at),
|
|
**values,
|
|
)
|
|
)
|
|
|
|
def _owned(self, lease: Lease):
|
|
return update(cast(Table, JobRow.__table__)).where(
|
|
JobRow.job_id == lease.job.id,
|
|
JobRow.status == "processing",
|
|
JobRow.lease_token == lease.token,
|
|
)
|
|
|
|
async def add(self, job: Job) -> None:
|
|
await self._session.execute(
|
|
insert(JobRow).values(
|
|
job_id=job.id,
|
|
attachment_id=job.attachment_id,
|
|
pipeline_version=job.pipeline_version,
|
|
status="queued",
|
|
trace_context=capture_context(),
|
|
attempt=job.attempt,
|
|
created_at=job.created_at,
|
|
updated_at=job.created_at,
|
|
)
|
|
)
|
|
await self._enqueue(job)
|
|
await self._note(job, "enqueued")
|
|
|
|
async def reprocess(self, job: Job, operator: str) -> bool:
|
|
existing = await self._session.scalar(
|
|
select(JobRow.job_id).where(
|
|
JobRow.attachment_id == job.attachment_id,
|
|
JobRow.pipeline_version == job.pipeline_version,
|
|
)
|
|
)
|
|
if existing is not None:
|
|
return False
|
|
await self.add(job)
|
|
await self._note(job, "reprocess_requested", {"operator": operator})
|
|
return True
|
|
|
|
async def claim(self, job: Job, token: UUID, lease: timedelta) -> Claim | None:
|
|
row = (
|
|
await self._session.execute(
|
|
update(cast(Table, JobRow.__table__))
|
|
.where(
|
|
JobRow.job_id == job.id,
|
|
JobRow.attachment_id == job.attachment_id,
|
|
JobRow.pipeline_version == job.pipeline_version,
|
|
or_(
|
|
(JobRow.status == "queued")
|
|
& or_(
|
|
JobRow.next_attempt_at.is_(None), JobRow.next_attempt_at <= func.now()
|
|
),
|
|
(JobRow.status == "processing") & (JobRow.lease_until < func.now()),
|
|
),
|
|
)
|
|
.values(
|
|
status="processing",
|
|
lease_token=token,
|
|
claims=JobRow.claims + 1,
|
|
updated_at=func.now(),
|
|
lease_until=func.now() + lease,
|
|
)
|
|
.returning(JobRow.attempt, JobRow.created_at)
|
|
)
|
|
).first()
|
|
if row is None:
|
|
return None
|
|
claimed = Job(job.id, job.attachment_id, job.pipeline_version, row.attempt, row.created_at)
|
|
await self._move(job.attachment_id, ["queued"], "processing")
|
|
await self._note(claimed, "claimed")
|
|
source = (
|
|
await self._session.execute(
|
|
select(
|
|
AttachmentRow.status, AttachmentRow.staging_key, AttachmentRow.original_key
|
|
).where(AttachmentRow.id == job.attachment_id)
|
|
)
|
|
).one()
|
|
staged = source.status != "ready" and source.staging_key is not None
|
|
return Claim(
|
|
Lease(claimed, token), source.staging_key if staged else source.original_key, staged
|
|
)
|
|
|
|
async def complete(
|
|
self, lease: Lease, variants: Sequence[Variant], original_key: str
|
|
) -> Outcome:
|
|
job = lease.job
|
|
done = await self._session.scalar(
|
|
self._owned(lease)
|
|
.values(
|
|
status="done",
|
|
lease_token=None,
|
|
lease_until=None,
|
|
next_attempt_at=None,
|
|
last_error=None,
|
|
updated_at=func.now(),
|
|
)
|
|
.returning(JobRow.job_id)
|
|
)
|
|
if done is None:
|
|
await self.note(job, Outcome.FENCED)
|
|
return Outcome.FENCED
|
|
if variants:
|
|
await self._session.execute(
|
|
pg_insert(VariantRow)
|
|
.values(
|
|
[
|
|
{
|
|
"attachment_id": job.attachment_id,
|
|
"pipeline_version": job.pipeline_version,
|
|
"name": v.name,
|
|
"object_key": v.object_key,
|
|
"content_type": v.content_type,
|
|
"size_bytes": v.size_bytes,
|
|
"width": v.width,
|
|
"height": v.height,
|
|
"sha256": v.sha256,
|
|
"created_at": func.now(),
|
|
}
|
|
for v in variants
|
|
]
|
|
)
|
|
.on_conflict_do_nothing()
|
|
)
|
|
published = await self._session.scalar(
|
|
update(AttachmentRow)
|
|
.where(
|
|
AttachmentRow.id == job.attachment_id,
|
|
AttachmentRow.published_version < job.pipeline_version,
|
|
)
|
|
.values(
|
|
status="ready",
|
|
published_version=job.pipeline_version,
|
|
error_code=None,
|
|
original_key=original_key,
|
|
updated_at=func.greatest(func.now(), AttachmentRow.created_at),
|
|
)
|
|
.returning(AttachmentRow.id)
|
|
)
|
|
outcome = Outcome.SUPERSEDED if published is None else Outcome.DONE
|
|
if outcome is Outcome.SUPERSEDED:
|
|
await self._session.execute(
|
|
update(cast(Table, JobRow.__table__))
|
|
.where(JobRow.job_id == job.id)
|
|
.values(status="superseded", updated_at=func.now())
|
|
)
|
|
await self.note(job, outcome)
|
|
return outcome
|
|
|
|
async def reschedule(
|
|
self, lease: Lease, delay: timedelta, kind: Outcome, error: str
|
|
) -> Job | None:
|
|
row = (
|
|
await self._session.execute(
|
|
self._owned(lease)
|
|
.values(
|
|
status="queued",
|
|
attempt=JobRow.attempt + 1,
|
|
lease_token=None,
|
|
lease_until=None,
|
|
next_attempt_at=func.now() + delay,
|
|
last_error=error,
|
|
updated_at=func.now(),
|
|
)
|
|
.returning(JobRow.attempt, JobRow.next_attempt_at)
|
|
)
|
|
).first()
|
|
if row is None:
|
|
return None
|
|
job = lease.job.retried(row.attempt)
|
|
await self._move(job.attachment_id, ["processing"], "queued")
|
|
await self._note(
|
|
job,
|
|
kind.value,
|
|
{
|
|
"error": error,
|
|
"next_attempt_at": row.next_attempt_at.isoformat(),
|
|
"delay_seconds": delay.total_seconds(),
|
|
},
|
|
)
|
|
await self._enqueue(job, row.next_attempt_at)
|
|
return job
|
|
|
|
async def renew(self, lease: Lease, duration: timedelta) -> bool:
|
|
return (
|
|
await self._session.scalar(
|
|
self._owned(lease)
|
|
.values(lease_until=func.now() + duration, updated_at=func.now())
|
|
.returning(JobRow.job_id)
|
|
)
|
|
is not None
|
|
)
|
|
|
|
async def fail(self, lease: Lease, code: FailureCode, error: str) -> bool:
|
|
attempt = await self._session.scalar(
|
|
self._owned(lease)
|
|
.values(
|
|
status="failed",
|
|
lease_token=None,
|
|
lease_until=None,
|
|
last_error=error,
|
|
updated_at=func.now(),
|
|
)
|
|
.returning(JobRow.attempt)
|
|
)
|
|
if attempt is None:
|
|
return False
|
|
await self._session.execute(
|
|
select(AttachmentRow.id)
|
|
.where(AttachmentRow.id == lease.job.attachment_id)
|
|
.with_for_update()
|
|
)
|
|
active = (
|
|
select(JobRow.job_id)
|
|
.where(
|
|
JobRow.attachment_id == lease.job.attachment_id,
|
|
JobRow.status.in_(["queued", "processing"]),
|
|
)
|
|
.exists()
|
|
)
|
|
await self._session.execute(
|
|
update(AttachmentRow)
|
|
.where(
|
|
AttachmentRow.id == lease.job.attachment_id,
|
|
AttachmentRow.status.in_(["queued", "processing"]),
|
|
~active,
|
|
)
|
|
.values(
|
|
status="failed",
|
|
error_code=code.value,
|
|
updated_at=func.greatest(func.now(), AttachmentRow.created_at),
|
|
)
|
|
)
|
|
await self._note(
|
|
lease.job.retried(attempt), "failed", {"error_code": code.value, "error": error}
|
|
)
|
|
return True
|
|
|
|
async def note(self, job: Job, kind: Outcome) -> None:
|
|
await self._note(job, kind.value)
|
|
|
|
async def expired(self, limit: int) -> list[Lease]:
|
|
rows = await self._session.execute(
|
|
select(JobRow.__table__)
|
|
.where(JobRow.status == "processing", JobRow.lease_until < func.now())
|
|
.order_by(JobRow.lease_until)
|
|
.limit(limit)
|
|
.with_for_update(skip_locked=True)
|
|
)
|
|
return [
|
|
Lease(
|
|
Job(
|
|
row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at
|
|
),
|
|
row.lease_token,
|
|
)
|
|
for row in rows
|
|
]
|
|
|
|
async def replay(self, attachment: Attachment, operator: str) -> Job | None:
|
|
job_id = (
|
|
select(JobRow.job_id)
|
|
.where(JobRow.attachment_id == attachment.id, JobRow.status == "failed")
|
|
.order_by(JobRow.pipeline_version.desc())
|
|
.limit(1)
|
|
.scalar_subquery()
|
|
)
|
|
row = (
|
|
await self._session.execute(
|
|
update(cast(Table, JobRow.__table__))
|
|
.where(JobRow.job_id == job_id)
|
|
.values(
|
|
status="queued",
|
|
attempt=1,
|
|
lease_token=None,
|
|
lease_until=None,
|
|
next_attempt_at=None,
|
|
last_error=None,
|
|
updated_at=func.now(),
|
|
)
|
|
.returning(
|
|
JobRow.job_id, JobRow.pipeline_version, JobRow.created_at, JobRow.updated_at
|
|
)
|
|
)
|
|
).first()
|
|
if row is None:
|
|
return None
|
|
job = Job(row.job_id, attachment.id, row.pipeline_version, 1, row.created_at)
|
|
await self._move(attachment.id, ["failed"], "queued", error_code=None)
|
|
await self._note(
|
|
job, "replayed", {"operator": operator, "error_code": attachment.error_code}
|
|
)
|
|
await self._enqueue(job, row.updated_at)
|
|
return job
|
|
|
|
async def note_replay(self, attachment_id: UUID, operator: str, result: str) -> None:
|
|
row = (
|
|
await self._session.execute(
|
|
select(JobRow.__table__)
|
|
.where(JobRow.attachment_id == attachment_id)
|
|
.order_by(JobRow.pipeline_version.desc())
|
|
.limit(1)
|
|
)
|
|
).first()
|
|
if row is not None:
|
|
job = Job(
|
|
row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at
|
|
)
|
|
await self._note(job, "replay_noop", {"operator": operator, "result": result})
|
|
|
|
|
|
class PostgresOutboxRepository:
|
|
__slots__ = ("_session",)
|
|
|
|
def __init__(self, session: DatabaseSession) -> None:
|
|
self._session = session
|
|
|
|
async def pending(self, limit: int) -> list[OutboxEntry]:
|
|
rows = await self._session.execute(
|
|
select(OutboxRow.__table__)
|
|
.where(OutboxRow.published_at.is_(None), OutboxRow.available_at <= func.now())
|
|
.order_by(OutboxRow.available_at, OutboxRow.id)
|
|
.limit(limit)
|
|
.with_for_update(skip_locked=True)
|
|
)
|
|
return [
|
|
OutboxEntry(row.id, row.job_id, json.dumps(row.payload).encode(), row.trace_context)
|
|
for row in rows
|
|
]
|
|
|
|
async def published(self, entry_ids: Sequence[int]) -> None:
|
|
await self._session.execute(
|
|
update(OutboxRow).where(OutboxRow.id.in_(entry_ids)).values(published_at=func.now())
|
|
)
|