formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+23 -7
View File
@@ -28,26 +28,42 @@ class JobBroker:
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
async def ensure_ready(self) -> None:
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
if (
self.connection is None
or self.connection.is_closed
or self.channel is None
or self.channel.is_closed
):
await self.aclose()
self.connection = await aio_pika.connect(
host=self.settings.broker_host, port=self.settings.broker_port,
host=self.settings.broker_host,
port=self.settings.broker_port,
login=self.settings.broker_user,
password=self.settings.broker_password_file.read_text().strip(),
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
virtualhost=self.settings.broker_vhost,
timeout=5,
heartbeat=30,
)
self.channel = await self.connection.channel(
publisher_confirms=True, on_return_raises=True
)
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
self.queue = await self.channel.declare_queue(
self.settings.broker_queue, durable=True
)
async def publish_job(self, message: JobMessage) -> None:
await self.ensure_ready()
await self.channel.default_exchange.publish(
aio_pika.Message(
body=message.model_dump_json(exclude_none=True).encode(),
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=f"{message.job_id}:{message.attempt}",
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
),
routing_key=self.settings.broker_queue,
mandatory=True,
timeout=10,
)
async def receive(self, consumer: str) -> Delivery:
+1 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
async def check_database(engine: AsyncEngine) -> bool:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
await connection.execute(text("SELECT 1"))
return True
+12 -5
View File
@@ -33,7 +33,8 @@ async def get_active_key(
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
) == tuple_(*scope),
)
== tuple_(*scope),
IdempotencyKey.created_at
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
)
@@ -45,14 +46,20 @@ async def save_result(
) -> None:
"""Сохраняет новый результат или заменяет истёкший."""
stmt = insert(IdempotencyKey).values(
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
request_hash=request_hash, message_id=message_id,
user_id=scope[0],
method=scope[1],
path=scope[2],
key=scope[3],
request_hash=request_hash,
message_id=message_id,
)
await session.execute(
stmt.on_conflict_do_update(
index_elements=[
IdempotencyKey.user_id, IdempotencyKey.method,
IdempotencyKey.path, IdempotencyKey.key,
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
],
set_={
"request_hash": request_hash,
+16 -3
View File
@@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int):
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "failed")
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
.limit(limit)
)
return result.all()
@@ -23,12 +24,24 @@ async def replay_record(session, request_id):
async def insert_replay(session, **values) -> bool:
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
result = await session.scalar(
insert(JobReplay)
.values(**values)
.on_conflict_do_nothing(index_elements=[JobReplay.id])
.returning(JobReplay.id)
)
return result is not None
async def replay_history(session, job_id):
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
return (
await session.scalars(
select(JobReplay)
.where(JobReplay.job_id == job_id)
.order_by(JobReplay.created_at)
.limit(100)
)
).all()
def reset_failed(job, attachment, now, attempt_limit):
+43 -10
View File
@@ -32,7 +32,9 @@ class JobWork:
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
async def create_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
session.add(job)
await session.flush()
@@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
.where(
Attachment.status == "processing",
AttachmentJob.lease_until < func.clock_timestamp(),
)
.order_by(AttachmentJob.lease_until)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
@@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
return result.all()
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
def start_job(
job, attachment, token: UUID, now: datetime, lease_seconds: int
) -> JobWork:
job.attempt += 1
job.lease_token = token
job.lease_until = now + timedelta(seconds=lease_seconds)
attachment.status = "processing"
attachment.updated_at = now
return JobWork(
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256,
job.id,
attachment.id,
token,
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
def record_failure(
job,
attachment,
now: datetime,
error_code: str,
*,
terminal: bool,
retry_seconds: float,
) -> None:
job.lease_token = None
job.lease_until = None
job.last_error_code = error_code
@@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal:
attachment.updated_at = now
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
def complete_job(
job, attachment, work: JobWork, variants: list[dict], now: datetime
) -> None:
attachment.object_key = work.original_key
attachment.thumbnail_key = work.variant_key("thumbnail")
attachment.crop_key = work.variant_key("crop")
@@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date
job.finished_at = now
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
async def terminal_staging(
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
AttachmentJob.staging_key.is_not(None),
or_(
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
and_(
Attachment.status == "ready",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=retention),
),
and_(
Attachment.status == "failed",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=failed_retention),
),
),
)
.order_by(AttachmentJob.finished_at)
+3 -1
View File
@@ -9,7 +9,9 @@ from app.models.attachment import Attachment
from app.models.job import AttachmentJob
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
async def next_unpublished_job(
session: AsyncSession, *, attachment_id: UUID | None = None
) -> AttachmentJob | None:
statement = (
select(AttachmentJob)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)