formatted project by python-black
This commit is contained in:
+23
-7
@@ -28,26 +28,42 @@ class JobBroker:
|
||||
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
|
||||
|
||||
async def ensure_ready(self) -> None:
|
||||
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
|
||||
if (
|
||||
self.connection is None
|
||||
or self.connection.is_closed
|
||||
or self.channel is None
|
||||
or self.channel.is_closed
|
||||
):
|
||||
await self.aclose()
|
||||
self.connection = await aio_pika.connect(
|
||||
host=self.settings.broker_host, port=self.settings.broker_port,
|
||||
host=self.settings.broker_host,
|
||||
port=self.settings.broker_port,
|
||||
login=self.settings.broker_user,
|
||||
password=self.settings.broker_password_file.read_text().strip(),
|
||||
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
|
||||
virtualhost=self.settings.broker_vhost,
|
||||
timeout=5,
|
||||
heartbeat=30,
|
||||
)
|
||||
self.channel = await self.connection.channel(
|
||||
publisher_confirms=True, on_return_raises=True
|
||||
)
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
await self.channel.set_qos(prefetch_count=1)
|
||||
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
|
||||
self.queue = await self.channel.declare_queue(
|
||||
self.settings.broker_queue, durable=True
|
||||
)
|
||||
|
||||
async def publish_job(self, message: JobMessage) -> None:
|
||||
await self.ensure_ready()
|
||||
await self.channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=message.model_dump_json(exclude_none=True).encode(),
|
||||
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=f"{message.job_id}:{message.attempt}",
|
||||
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
|
||||
),
|
||||
routing_key=self.settings.broker_queue,
|
||||
mandatory=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
async def receive(self, consumer: str) -> Delivery:
|
||||
|
||||
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
async def check_database(engine: AsyncEngine) -> bool:
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
await connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,8 @@ async def get_active_key(
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
) == tuple_(*scope),
|
||||
)
|
||||
== tuple_(*scope),
|
||||
IdempotencyKey.created_at
|
||||
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
|
||||
)
|
||||
@@ -45,14 +46,20 @@ async def save_result(
|
||||
) -> None:
|
||||
"""Сохраняет новый результат или заменяет истёкший."""
|
||||
stmt = insert(IdempotencyKey).values(
|
||||
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
|
||||
request_hash=request_hash, message_id=message_id,
|
||||
user_id=scope[0],
|
||||
method=scope[1],
|
||||
path=scope[2],
|
||||
key=scope[3],
|
||||
request_hash=request_hash,
|
||||
message_id=message_id,
|
||||
)
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=[
|
||||
IdempotencyKey.user_id, IdempotencyKey.method,
|
||||
IdempotencyKey.path, IdempotencyKey.key,
|
||||
IdempotencyKey.user_id,
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
],
|
||||
set_={
|
||||
"request_hash": request_hash,
|
||||
|
||||
@@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int):
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "failed")
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
|
||||
.limit(limit)
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@@ -23,12 +24,24 @@ async def replay_record(session, request_id):
|
||||
|
||||
|
||||
async def insert_replay(session, **values) -> bool:
|
||||
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
|
||||
result = await session.scalar(
|
||||
insert(JobReplay)
|
||||
.values(**values)
|
||||
.on_conflict_do_nothing(index_elements=[JobReplay.id])
|
||||
.returning(JobReplay.id)
|
||||
)
|
||||
return result is not None
|
||||
|
||||
|
||||
async def replay_history(session, job_id):
|
||||
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
|
||||
return (
|
||||
await session.scalars(
|
||||
select(JobReplay)
|
||||
.where(JobReplay.job_id == job_id)
|
||||
.order_by(JobReplay.created_at)
|
||||
.limit(100)
|
||||
)
|
||||
).all()
|
||||
|
||||
|
||||
def reset_failed(job, attachment, now, attempt_limit):
|
||||
|
||||
+43
-10
@@ -32,7 +32,9 @@ class JobWork:
|
||||
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
|
||||
|
||||
|
||||
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
async def create_job(
|
||||
session: AsyncSession, attachment_id: UUID, staging_key: str
|
||||
) -> UUID:
|
||||
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
@@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
|
||||
.where(
|
||||
Attachment.status == "processing",
|
||||
AttachmentJob.lease_until < func.clock_timestamp(),
|
||||
)
|
||||
.order_by(AttachmentJob.lease_until)
|
||||
.limit(limit)
|
||||
.with_for_update(of=AttachmentJob, skip_locked=True)
|
||||
@@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
return result.all()
|
||||
|
||||
|
||||
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
|
||||
def start_job(
|
||||
job, attachment, token: UUID, now: datetime, lease_seconds: int
|
||||
) -> JobWork:
|
||||
job.attempt += 1
|
||||
job.lease_token = token
|
||||
job.lease_until = now + timedelta(seconds=lease_seconds)
|
||||
attachment.status = "processing"
|
||||
attachment.updated_at = now
|
||||
return JobWork(
|
||||
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256,
|
||||
job.id,
|
||||
attachment.id,
|
||||
token,
|
||||
job.pipeline_version,
|
||||
job.staging_key,
|
||||
attachment.content_type,
|
||||
attachment.size_bytes,
|
||||
attachment.sha256,
|
||||
)
|
||||
|
||||
|
||||
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
|
||||
def record_failure(
|
||||
job,
|
||||
attachment,
|
||||
now: datetime,
|
||||
error_code: str,
|
||||
*,
|
||||
terminal: bool,
|
||||
retry_seconds: float,
|
||||
) -> None:
|
||||
job.lease_token = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = error_code
|
||||
@@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal:
|
||||
attachment.updated_at = now
|
||||
|
||||
|
||||
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
|
||||
def complete_job(
|
||||
job, attachment, work: JobWork, variants: list[dict], now: datetime
|
||||
) -> None:
|
||||
attachment.object_key = work.original_key
|
||||
attachment.thumbnail_key = work.variant_key("thumbnail")
|
||||
attachment.crop_key = work.variant_key("crop")
|
||||
@@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date
|
||||
job.finished_at = now
|
||||
|
||||
|
||||
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
|
||||
async def terminal_staging(
|
||||
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
|
||||
):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(
|
||||
AttachmentJob.staging_key.is_not(None),
|
||||
or_(
|
||||
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
|
||||
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
|
||||
and_(
|
||||
Attachment.status == "ready",
|
||||
AttachmentJob.finished_at
|
||||
< func.clock_timestamp() - timedelta(seconds=retention),
|
||||
),
|
||||
and_(
|
||||
Attachment.status == "failed",
|
||||
AttachmentJob.finished_at
|
||||
< func.clock_timestamp() - timedelta(seconds=failed_retention),
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(AttachmentJob.finished_at)
|
||||
|
||||
@@ -9,7 +9,9 @@ from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
|
||||
|
||||
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
|
||||
async def next_unpublished_job(
|
||||
session: AsyncSession, *, attachment_id: UUID | None = None
|
||||
) -> AttachmentJob | None:
|
||||
statement = (
|
||||
select(AttachmentJob)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
|
||||
Reference in New Issue
Block a user