Files
mtusi2026lab5-fork/app/storage/jobs.py
T

161 lines
4.7 KiB
Python

"""SQL и изменения jobs/attachments; commit остаётся в domain."""
from dataclasses import dataclass
from datetime import datetime, timedelta
from uuid import UUID
from sqlalchemy import and_, or_, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.attachment import Attachment
from app.models.job import AttachmentJob
from app.schemas.attachment import VARIANT_FILENAMES
@dataclass(frozen=True)
class JobWork:
job_id: UUID
attachment_id: UUID
lease_token: UUID
pipeline_version: int
staging_key: str
content_type: str
size_bytes: int
sha256: str
@property
def original_key(self) -> str:
return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}"
def variant_key(self, name: str) -> str:
filename = VARIANT_FILENAMES[name]
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
async def create_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
session.add(job)
await session.flush()
return job.id
async def lock_job(session: AsyncSession, job_id: UUID):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(AttachmentJob.id == job_id)
.with_for_update(of=AttachmentJob)
.execution_options(populate_existing=True)
)
return result.one_or_none()
async def database_time(session: AsyncSession) -> datetime:
return await session.scalar(select(func.clock_timestamp()))
async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
Attachment.status == "processing",
AttachmentJob.lease_until < func.clock_timestamp(),
)
.order_by(AttachmentJob.lease_until)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
return result.all()
def start_job(
job, attachment, token: UUID, now: datetime, lease_seconds: int
) -> JobWork:
job.attempt += 1
job.lease_token = token
job.lease_until = now + timedelta(seconds=lease_seconds)
attachment.status = "processing"
attachment.updated_at = now
return JobWork(
job.id,
attachment.id,
token,
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
def record_failure(
job,
attachment,
now: datetime,
error_code: str,
*,
terminal: bool,
retry_seconds: float,
) -> None:
job.lease_token = None
job.lease_until = None
job.last_error_code = error_code
job.next_attempt_at = now + timedelta(seconds=retry_seconds)
job.finished_at = now if terminal else None
attachment.status = "failed" if terminal else "queued"
attachment.error_code = error_code if terminal else None
attachment.updated_at = now
def complete_job(
job, attachment, work: JobWork, variants: list[dict], now: datetime
) -> None:
attachment.object_key = work.original_key
attachment.thumbnail_key = work.variant_key("thumbnail")
attachment.crop_key = work.variant_key("crop")
attachment.optimized_key = work.variant_key("optimized")
attachment.variants = variants
attachment.status = "ready"
attachment.error_code = None
attachment.updated_at = now
job.lease_token = None
job.lease_until = None
job.last_error_code = None
job.last_error_permanent = None
job.finished_at = now
async def terminal_staging(
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
AttachmentJob.staging_key.is_not(None),
or_(
and_(
Attachment.status == "ready",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=retention),
),
and_(
Attachment.status == "failed",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=failed_retention),
),
),
)
.order_by(AttachmentJob.finished_at)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
)
return result.all()
def clear_staging(job) -> None:
job.staging_key = None