161 lines
4.7 KiB
Python
161 lines
4.7 KiB
Python
"""SQL и изменения jobs/attachments; commit остаётся в domain."""
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy import and_, or_, func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.attachment import Attachment
|
|
from app.models.job import AttachmentJob
|
|
from app.schemas.attachment import VARIANT_FILENAMES
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class JobWork:
|
|
job_id: UUID
|
|
attachment_id: UUID
|
|
lease_token: UUID
|
|
pipeline_version: int
|
|
staging_key: str
|
|
content_type: str
|
|
size_bytes: int
|
|
sha256: str
|
|
|
|
@property
|
|
def original_key(self) -> str:
|
|
return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}"
|
|
|
|
def variant_key(self, name: str) -> str:
|
|
filename = VARIANT_FILENAMES[name]
|
|
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
|
|
|
|
|
|
async def create_job(
|
|
session: AsyncSession, attachment_id: UUID, staging_key: str
|
|
) -> UUID:
|
|
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
|
|
session.add(job)
|
|
await session.flush()
|
|
return job.id
|
|
|
|
|
|
async def lock_job(session: AsyncSession, job_id: UUID):
|
|
result = await session.execute(
|
|
select(AttachmentJob, Attachment)
|
|
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
|
.where(AttachmentJob.id == job_id)
|
|
.with_for_update(of=AttachmentJob)
|
|
.execution_options(populate_existing=True)
|
|
)
|
|
return result.one_or_none()
|
|
|
|
|
|
async def database_time(session: AsyncSession) -> datetime:
|
|
return await session.scalar(select(func.clock_timestamp()))
|
|
|
|
|
|
async def expired_jobs(session: AsyncSession, limit: int = 20):
|
|
result = await session.execute(
|
|
select(AttachmentJob, Attachment)
|
|
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
|
.where(
|
|
Attachment.status == "processing",
|
|
AttachmentJob.lease_until < func.clock_timestamp(),
|
|
)
|
|
.order_by(AttachmentJob.lease_until)
|
|
.limit(limit)
|
|
.with_for_update(of=AttachmentJob, skip_locked=True)
|
|
)
|
|
return result.all()
|
|
|
|
|
|
def start_job(
|
|
job, attachment, token: UUID, now: datetime, lease_seconds: int
|
|
) -> JobWork:
|
|
job.attempt += 1
|
|
job.lease_token = token
|
|
job.lease_until = now + timedelta(seconds=lease_seconds)
|
|
attachment.status = "processing"
|
|
attachment.updated_at = now
|
|
return JobWork(
|
|
job.id,
|
|
attachment.id,
|
|
token,
|
|
job.pipeline_version,
|
|
job.staging_key,
|
|
attachment.content_type,
|
|
attachment.size_bytes,
|
|
attachment.sha256,
|
|
)
|
|
|
|
|
|
def record_failure(
|
|
job,
|
|
attachment,
|
|
now: datetime,
|
|
error_code: str,
|
|
*,
|
|
terminal: bool,
|
|
retry_seconds: float,
|
|
) -> None:
|
|
job.lease_token = None
|
|
job.lease_until = None
|
|
job.last_error_code = error_code
|
|
job.next_attempt_at = now + timedelta(seconds=retry_seconds)
|
|
job.finished_at = now if terminal else None
|
|
attachment.status = "failed" if terminal else "queued"
|
|
attachment.error_code = error_code if terminal else None
|
|
attachment.updated_at = now
|
|
|
|
|
|
def complete_job(
|
|
job, attachment, work: JobWork, variants: list[dict], now: datetime
|
|
) -> None:
|
|
attachment.object_key = work.original_key
|
|
attachment.thumbnail_key = work.variant_key("thumbnail")
|
|
attachment.crop_key = work.variant_key("crop")
|
|
attachment.optimized_key = work.variant_key("optimized")
|
|
attachment.variants = variants
|
|
attachment.status = "ready"
|
|
attachment.error_code = None
|
|
attachment.updated_at = now
|
|
job.lease_token = None
|
|
job.lease_until = None
|
|
job.last_error_code = None
|
|
job.last_error_permanent = None
|
|
job.finished_at = now
|
|
|
|
|
|
async def terminal_staging(
|
|
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
|
|
):
|
|
result = await session.execute(
|
|
select(AttachmentJob, Attachment)
|
|
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
|
.where(
|
|
AttachmentJob.staging_key.is_not(None),
|
|
or_(
|
|
and_(
|
|
Attachment.status == "ready",
|
|
AttachmentJob.finished_at
|
|
< func.clock_timestamp() - timedelta(seconds=retention),
|
|
),
|
|
and_(
|
|
Attachment.status == "failed",
|
|
AttachmentJob.finished_at
|
|
< func.clock_timestamp() - timedelta(seconds=failed_retention),
|
|
),
|
|
),
|
|
)
|
|
.order_by(AttachmentJob.finished_at)
|
|
.limit(limit)
|
|
.with_for_update(of=AttachmentJob, skip_locked=True)
|
|
)
|
|
return result.all()
|
|
|
|
|
|
def clear_staging(job) -> None:
|
|
job.staging_key = None
|