formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+8 -1
View File
@@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
from sqlalchemy.exc import InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
TRANSIENT_JOB_ERRORS = (
AMQPException,
ChannelInvalidStateError,
InterfaceError,
OperationalError,
PoolTimeoutError,
OSError,
)
logger = logging.getLogger(__name__)
+3 -1
View File
@@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
staging = await cleanup_staging(session, storage, jobs, apply=apply)
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
print(
json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
+3 -1
View File
@@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
class ImageSettings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
model_config = SettingsConfigDict(
env_prefix="IMAGE_", env_file=".env", extra="ignore"
)
jpeg_quality: int = Field(default=85, ge=1, le=95)
webp_quality: int = Field(default=80, ge=1, le=100)
+22 -5
View File
@@ -9,7 +9,13 @@ from app.core.background import log_background_error
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
from app.domain.job_admin import (
REPLAY_REASONS,
history,
list_failed,
replay_failed_job,
show_job,
)
from app.errors.exceptions import JobOperationError
from app.storage.s3 import create_s3_storage
@@ -29,8 +35,15 @@ async def execute(args):
result = await history(session, args.job_id)
else:
storage = create_s3_storage(settings.s3)
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
request_id=args.request_id, actor=args.actor, reason=args.reason)
result = await replay_failed_job(
session,
storage,
JobSettings(),
args.job_id,
request_id=args.request_id,
actor=args.actor,
reason=args.reason,
)
print(json.dumps(result, default=str), flush=True)
finally:
if storage is not None:
@@ -39,10 +52,14 @@ async def execute(args):
def main():
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
parser = argparse.ArgumentParser(
description="Реестр failed и явный replay с аудитом"
)
commands = parser.add_subparsers(dest="command", required=True)
listing = commands.add_parser("list-failed")
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
listing.add_argument(
"--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
)
for name in ("show", "history", "replay"):
command = commands.add_parser(name)
command.add_argument("job_id", type=UUID)
+42 -9
View File
@@ -12,7 +12,12 @@ from uuid import UUID, uuid4
from PIL import features
from pydantic import ValidationError
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
@@ -34,7 +39,10 @@ class Faults:
object_name: str = "optimized"
def kill(self, attachment_id: UUID, phase: str) -> None:
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
print(
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
def after_object(self, attachment_id: UUID, name: str) -> None:
@@ -52,7 +60,14 @@ class Faults:
raise ProcessingError("injected_permanent")
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
async def consume_connected(
factory,
storage,
broker: JobBroker,
settings: JobSettings,
stop: asyncio.Event,
faults: Faults,
) -> None:
consumer = uuid4().hex
while not stop.is_set():
try:
@@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe
try:
async with factory() as session:
acknowledge = await process_attachment_job(
session, storage, message, settings,
after_object=faults.after_object, before_processing=faults.before_processing,
session,
storage,
message,
settings,
after_object=faults.after_object,
before_processing=faults.before_processing,
)
except ProcessingError:
await broker.reject_invalid(delivery.id)
@@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None:
storage = create_s3_storage(settings.s3)
factory = create_session_factory(engine)
async with asyncio.TaskGroup() as group:
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
tasks = [
group.create_task(consume(factory, storage, jobs, stop, faults))
for _ in range(jobs.worker_concurrency)
]
await stop.wait()
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
for task in tasks:
@@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
parser = argparse.ArgumentParser(
description="Image worker с явными fault-injection hooks"
)
modes = parser.add_mutually_exclusive_group()
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
for flag in (
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
):
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
parser.add_argument(
"--object-name",
choices=("original", "thumbnail", "crop", "optimized"),
default="optimized",
)
args = parser.parse_args()
faults = Faults(**vars(args))
try:
+3 -1
View File
@@ -29,7 +29,9 @@ class JobSettings(BaseSettings):
@model_validator(mode="after")
def validate_lease(self) -> Self:
if self.job_lease_seconds < self.job_timeout_seconds + 30:
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
raise ValueError(
"Lease должен превышать timeout задания минимум на 30 секунд"
)
if self.job_retry_max_seconds < self.job_retry_seconds:
raise ValueError("Retry maximum must be at least the base delay")
return self
+24 -6
View File
@@ -8,7 +8,12 @@ import os
import signal
from uuid import UUID
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
@@ -18,7 +23,10 @@ from app.storage.broker import JobBroker
def crash_before_publish(attachment_id: UUID) -> None:
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
print(
json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
@@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
async with factory() as session:
await recover_expired_jobs(session, jobs)
published = await publish_next_job(
session, broker, attachment_id=crash_attachment_id,
before_publish=crash_before_publish if crash_attachment_id is not None else None,
session,
broker,
attachment_id=crash_attachment_id,
before_publish=(
crash_before_publish
if crash_attachment_id is not None
else None
),
)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Publisher deferred", error)
@@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
parser = argparse.ArgumentParser(
description="Публикация устойчивых jobs в RabbitMQ"
)
parser.add_argument(
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
"--crash-before-publish",
type=UUID,
metavar="ATTACHMENT_ID",
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
)
args = parser.parse_args()
+1 -2
View File
@@ -4,6 +4,7 @@ from redis.backoff import NoBackoff
from app.core.config import Settings
def create_redis_client(settings: Settings) -> Redis:
password = (
settings.redis_password.get_secret_value() if settings.redis_password else None
@@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis:
decode_responses=True,
)
return Redis.from_pool(pool)
+16 -4
View File
@@ -6,6 +6,7 @@ from urllib.parse import urlsplit
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class S3Settings(BaseSettings):
endpoint: str = "http://minio:9000"
region: str = "us-east-1"
@@ -24,12 +25,23 @@ class S3Settings(BaseSettings):
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
cleanup_interval_seconds: int = Field(default=300, ge=1)
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
model_config = SettingsConfigDict(
env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
)
@model_validator(mode="after")
def validate_endpoint(self):
url = urlsplit(self.endpoint)
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
if (
url.scheme not in {"http", "https"}
or not url.hostname
or url.username
or url.password
or url.path not in {"", "/"}
or url.query
or url.fragment
):
raise ValueError(
"S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
)
return self
+6 -2
View File
@@ -13,11 +13,15 @@ def main() -> int:
settings = ImageSettings()
memory = settings.memory_mib * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
resource.setrlimit(
resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
)
source, directory, metadata, content_type, maximum = sys.argv[1:]
code = 0
try:
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
result = create_variants(
Path(source), Path(directory), content_type, int(maximum), settings
)
Path(metadata).write_text(result.model_dump_json())
except IMAGE_WORKER_ERRORS as error:
code = pipeline_worker_error_code(error)
+95 -28
View File
@@ -29,13 +29,17 @@ class Claim:
acknowledge: bool = False
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
async def enqueue_attachment_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
"""Включить намерение обработки в транзакцию queued metadata; без commit."""
return await jobs.create_job(session, attachment_id, staging_key)
async def publish_next_job(
session: AsyncSession, broker: JobBroker, *,
session: AsyncSession,
broker: JobBroker,
*,
attachment_id: UUID | None = None,
before_publish: Callable[[UUID], None] | None = None,
) -> bool:
@@ -45,24 +49,33 @@ async def publish_next_job(
if job is not None:
if before_publish is not None:
before_publish(job.attachment_id)
await broker.publish_job(JobMessage(
job_id=job.id, attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version, attempt=job.attempt + 1,
created_at=job.created_at,
))
await broker.publish_job(
JobMessage(
job_id=job.id,
attachment_id=job.attachment_id,
pipeline_version=job.pipeline_version,
attempt=job.attempt + 1,
created_at=job.created_at,
)
)
outbox.mark_published(job)
await session.commit()
return published
async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim:
async def claim_job(
session: AsyncSession, message: JobMessage, settings: JobSettings
) -> Claim:
"""Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
row = await jobs.lock_job(session, message.job_id)
result = Claim()
if row is None:
raise ProcessingError("unknown_job")
job, attachment = row
if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version:
if (
message.attachment_id != attachment.id
or message.pipeline_version != job.pipeline_version
):
raise ProcessingError("invalid_job")
now = await jobs.database_time(session)
if job.attempt_limit is None:
@@ -77,16 +90,32 @@ async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSet
elif message.attempt > job.attempt + 1:
raise ProcessingError("invalid_attempt")
elif attachment.status == "queued" and job.next_attempt_at <= now:
if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None:
jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0)
if (
job.attempt >= (job.attempt_limit or settings.job_max_attempts)
or job.staging_key is None
):
jobs.record_failure(
job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0
)
result = Claim(acknowledge=True)
else:
result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds))
result = Claim(
work=jobs.start_job(
job, attachment, uuid4(), now, settings.job_lease_seconds
)
)
await session.commit()
return result
async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None:
async def save_failure(
session: AsyncSession,
work: JobWork,
settings: JobSettings,
code: str,
*,
permanent: bool,
) -> None:
# Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
await session.rollback()
row = await jobs.lock_job(session, work.job_id)
@@ -95,10 +124,14 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
if attachment.status == "processing" and job.lease_token == work.lease_token:
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job, attachment, await jobs.database_time(session),
job,
attachment,
await jobs.database_time(session),
"retry_exhausted" if exhausted and not permanent else code,
terminal=permanent or exhausted,
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
retry_seconds=retry_delay(
settings, job.attempt - job.cycle_start_attempt
),
)
job.last_error_code = code
job.last_error_permanent = permanent
@@ -106,7 +139,9 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
async def build_result(
session: AsyncSession, storage: MinioStorage, work: JobWork,
session: AsyncSession,
storage: MinioStorage,
work: JobWork,
after_object: Callable[[UUID, str], None] | None = None,
) -> bool:
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
@@ -129,26 +164,42 @@ async def build_result(
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
await storage.put_file(
work.original_key, source, size_bytes=work.size_bytes,
content_type=work.content_type, sha256=work.sha256,
work.original_key,
source,
size_bytes=work.size_bytes,
content_type=work.content_type,
sha256=work.sha256,
)
if after_object is not None:
after_object(work.attachment_id, "original")
for variant in result.variants:
await storage.put_file(
work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256,
work.variant_key(variant.name),
Path(directory) / VARIANT_FILENAMES[variant.name],
size_bytes=variant.size_bytes,
content_type=variant.content_type,
sha256=variant.sha256,
)
if after_object is not None:
after_object(work.attachment_id, variant.name)
jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session))
jobs.complete_job(
job,
attachment,
work,
[variant.model_dump() for variant in result.variants],
await jobs.database_time(session),
)
await session.commit()
return True
async def process_attachment_job(
session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings,
*, after_object: Callable[[UUID, str], None] | None = None,
session: AsyncSession,
storage: MinioStorage,
message: JobMessage,
settings: JobSettings,
*,
after_object: Callable[[UUID, str], None] | None = None,
before_processing: Callable[[UUID], None] | None = None,
) -> bool:
claim = await claim_job(session, message, settings)
@@ -160,7 +211,13 @@ async def process_attachment_job(
if before_processing is not None:
before_processing(work.attachment_id)
acknowledge = await build_result(session, storage, work, after_object)
except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error:
except (
ProcessingError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
) as error:
code, permanent = classify_failure(error)
await save_failure(session, work, settings, code, permanent=permanent)
acknowledge = True
@@ -176,8 +233,12 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
job.attempt_limit = settings.job_max_attempts
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
jobs.record_failure(
job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout",
terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
job,
attachment,
now,
"retry_exhausted" if exhausted else "worker_timeout",
terminal=exhausted,
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
)
job.last_error_code = "worker_timeout"
job.last_error_permanent = False
@@ -185,8 +246,14 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
return len(rows)
async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int:
rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds)
async def cleanup_staging(
session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool
) -> int:
rows = await jobs.terminal_staging(
session,
settings.staging_retention_seconds,
settings.failed_staging_retention_seconds,
)
if apply:
for job, attachment in rows:
if job.staging_key != attachment.object_key:
+11 -3
View File
@@ -276,12 +276,20 @@ async def download_attachment(
key = record.object_key
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
else:
selected = next((item for item in result.variants if item.name == variant), None)
keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key}
selected = next(
(item for item in result.variants if item.name == variant), None
)
keys = {
"thumbnail": record.thumbnail_key,
"crop": record.crop_key,
"optimized": record.optimized_key,
}
key = keys.get(variant)
if selected is None or key is None:
raise not_found()
info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes)
info = DownloadInfo(
VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
)
await session.rollback()
stored = await storage.stat_object(key)
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
+7 -4
View File
@@ -14,12 +14,15 @@ from app.storage.s3 import MinioStorage
def liveness() -> Health:
return Health(status='ok')
return Health(status="ok")
async def readiness(
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
lifecycle: Lifecycle, timeout_seconds: float,
engine: AsyncEngine,
redis: Redis,
storage: MinioStorage | None,
lifecycle: Lifecycle,
timeout_seconds: float,
) -> Health:
if lifecycle.draining:
raise unavailable()
@@ -30,4 +33,4 @@ async def readiness(
dependencies = await asyncio.gather(*checks, return_exceptions=True)
if not all(result is True for result in dependencies):
raise unavailable()
return Health(status='ok')
return Health(status="ok")
+9 -3
View File
@@ -57,16 +57,22 @@ async def send_message_idempotent(
raise conflict("Idempotency-Key уже использован для другого сообщения")
message = await message_storage.get_message(session, previous.message_id)
else:
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
await validate_message_attachments(
session, sender, chat_id, new_message.attachment_ids
)
if not await message_storage.reserve_message_slot(session, max_messages):
raise unavailable("Достигнут лимит количества сообщений")
message = await message_storage.create_message(
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
)
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
await attachment_storage.bind_attachments(
session, message.id, new_message.attachment_ids
)
await key_storage.save_result(session, scope, fingerprint, message.id)
links = await attachment_storage.message_attachment_ids(session, [message.id])
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
result = Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]}
)
await session.commit()
return result
+37 -11
View File
@@ -15,18 +15,27 @@ def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
longest = max(size)
result = size
if longest > maximum:
result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest))
result = (
max(1, size[0] * maximum // longest),
max(1, size[1] * maximum // longest),
)
return result
def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant:
def describe_variant(
name: str, content_type: str, path: Path, size: tuple[int, int]
) -> Variant:
checksum = hashlib.sha256()
with path.open("rb") as source:
for chunk in iter(lambda: source.read(65536), b""):
checksum.update(chunk)
return Variant(
name=name, content_type=content_type, size_bytes=path.stat().st_size,
width=size[0], height=size[1], sha256=checksum.hexdigest(),
name=name,
content_type=content_type,
size_bytes=path.stat().st_size,
width=size[0],
height=size[1],
sha256=checksum.hexdigest(),
)
@@ -47,32 +56,49 @@ def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Var
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
"""Центрированный JPEG 128×128; маленький исходник допускает увеличение."""
with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop:
with ImageOps.fit(
image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
) as crop:
save_jpeg(crop, destination, quality)
return describe_variant("crop", "image/jpeg", destination, (128, 128))
def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant:
def create_optimized(
image: Image.Image, destination: Path, settings: ImageSettings
) -> Variant:
"""WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются."""
size = fitted_size(image.size, 1280)
with image.resize(size, Image.Resampling.LANCZOS) as resized:
with Image.new("RGBA", size) as clean:
clean.paste(resized)
clean.save(
destination, format="WEBP", lossless=False,
quality=settings.webp_quality, method=settings.webp_method,
exif=b"", icc_profile=b"", xmp=b"",
destination,
format="WEBP",
lossless=False,
quality=settings.webp_quality,
method=settings.webp_method,
exif=b"",
icc_profile=b"",
xmp=b"",
)
return describe_variant("optimized", "image/webp", destination, size)
def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult:
def create_variants(
source: Path,
directory: Path,
content_type: str,
max_pixels: int,
settings: ImageSettings,
) -> PipelineResult:
decode_image(source, content_type, max_pixels)
with Image.open(source, formats=["PNG", "JPEG"]) as image:
ImageOps.exif_transpose(image, in_place=True)
with image.convert("RGBA") as pixels:
variants = [
create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality),
create_thumbnail(
pixels, directory / "thumbnail.jpg", settings.jpeg_quality
),
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
create_optimized(pixels, directory / "optimized.webp", settings),
]
+71 -19
View File
@@ -17,15 +17,28 @@ REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
def snapshot(job, attachment):
return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status,
attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code, last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at)
return dict(
job_id=str(job.id),
attachment_id=str(attachment.id),
status=attachment.status,
attempt=job.attempt,
attempt_limit=job.attempt_limit,
cycle_start_attempt=job.cycle_start_attempt,
error_code=attachment.error_code,
last_error_code=job.last_error_code,
last_error_permanent=job.last_error_permanent,
staging_retained=job.staging_key is not None,
next_attempt_at=job.next_attempt_at,
lease_until=job.lease_until,
finished_at=job.finished_at,
)
async def list_failed(session, limit):
return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)]
return [
snapshot(job, attachment)
for job, attachment in await job_admin.failed_jobs(session, limit)
]
async def show_job(session, job_id):
@@ -38,20 +51,37 @@ async def show_job(session, job_id):
async def history(session, job_id):
return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason,
previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code, created_at=row.created_at)
for row in await job_admin.replay_history(session, job_id)]
return [
dict(
request_id=str(row.id),
actor=row.actor,
reason=row.reason,
previous_attempt=row.previous_attempt,
attempt_limit=row.attempt_limit,
previous_error_code=row.previous_error_code,
created_at=row.created_at,
)
for row in await job_admin.replay_history(session, job_id)
]
def same_request(record, job_id, actor, reason):
if record.job_id != job_id or record.actor != actor or record.reason != reason:
raise JobOperationError("request_conflict")
return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True}
return {
"request_id": str(record.id),
"job_id": str(job_id),
"already_applied": True,
}
async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason):
if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS:
async def replay_failed_job(
session, storage, settings, job_id, *, request_id, actor, reason
):
if (
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
or reason not in REPLAY_REASONS
):
raise JobOperationError("invalid_audit_fields")
row = await jobs.lock_job(session, job_id)
if row is None:
@@ -67,8 +97,16 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
if job.staging_key is None:
raise JobOperationError("source_expired")
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256)
work = JobWork(
job.id,
attachment.id,
uuid4(),
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
try:
async with asyncio.timeout(settings.job_timeout_seconds):
with TemporaryDirectory(prefix="replay-source-") as directory:
@@ -81,13 +119,27 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
raise
limit = job.attempt + settings.job_max_attempts
inserted = await job_admin.insert_replay(
session, id=request_id, job_id=job_id, actor=actor, reason=reason,
previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code,
session,
id=request_id,
job_id=job_id,
actor=actor,
reason=reason,
previous_attempt=job.attempt,
attempt_limit=limit,
previous_error_code=job.last_error_code,
)
if not inserted:
result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason)
result = same_request(
await job_admin.replay_record(session, request_id), job_id, actor, reason
)
await session.rollback()
return result
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
await session.commit()
return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit}
return {
"request_id": str(request_id),
"job_id": str(job_id),
"already_applied": False,
"status": "queued",
"attempt_limit": limit,
}
+16 -5
View File
@@ -14,7 +14,9 @@ from app.storage.jobs import JobWork
from app.storage.s3 import MinioStorage
async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None:
async def fetch_original(
storage: MinioStorage, work: JobWork, destination: Path
) -> None:
if work.size_bytes > storage.settings.max_upload_bytes:
raise ProcessingError("image_limits")
checksum = hashlib.sha256()
@@ -34,12 +36,21 @@ async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path
raise ProcessingError("source_mismatch")
async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult:
async def render_variants(
storage: MinioStorage, work: JobWork, source: Path, directory: Path
) -> PipelineResult:
metadata = directory / "variants.json"
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory),
str(metadata), work.content_type, str(storage.settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
sys.executable,
"-m",
"app.core.thumbnail_worker",
str(source),
str(directory),
str(metadata),
work.content_type,
str(storage.settings.max_image_pixels),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
try:
code = await process.wait()
+23 -4
View File
@@ -10,7 +10,10 @@ from app.errors.exceptions import ProcessingError
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)))
ceiling = min(
settings.job_retry_max_seconds,
settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)),
)
return random.uniform(ceiling / 2, ceiling)
@@ -20,13 +23,29 @@ def classify_failure(error: Exception) -> tuple[str, bool]:
if isinstance(error, S3Error):
if error.code in {"NoSuchKey", "NoSuchObject"}:
return "source_missing", True
if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}:
if error.code in {
"AccessDenied",
"InvalidAccessKeyId",
"SignatureDoesNotMatch",
"NoSuchBucket",
}:
return "storage_configuration", True
if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}:
if error.code in {
"InternalError",
"SlowDown",
"ServiceUnavailable",
"RequestTimeout",
"Throttling",
}:
return "storage_unavailable", False
return "storage_rejected", True
if isinstance(error, TimeoutError):
return "processing_timeout", False
if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
if isinstance(error, OSError) and error.errno in {
errno.EACCES,
errno.EPERM,
errno.EROFS,
errno.ENOENT,
}:
return "processing_configuration", True
return "processing_unavailable", False
+9 -4
View File
@@ -32,10 +32,15 @@ async def list_messages(
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
page_rows, has_next = split_page(rows, limit)
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
items = [Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
) for message in page_rows]
links = await attachments_storage.message_attachment_ids(
session, [message.id for message in page_rows]
)
items = [
Message.model_validate(message).model_copy(
update={"attachment_ids": links[message.id]},
)
for message in page_rows
]
page = MessagePage(
items=items,
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
+39 -17
View File
@@ -12,41 +12,63 @@ from app.domain import attachments
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
from app.schemas.attachment import Attachment
router = APIRouter(prefix='/api/v1', tags=['attachments'])
router = APIRouter(prefix="/api/v1", tags=["attachments"])
@router.post('/chats/{chat_id}/attachments', status_code=202)
@router.post("/chats/{chat_id}/attachments", status_code=202)
async def upload(
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
storage: S3Storage, capacity: UploadCapacity,
request: Request,
chat_id: UUID,
user: CurrentUser,
session: DbSession,
storage: S3Storage,
capacity: UploadCapacity,
filename: Annotated[str, Query(min_length=1, max_length=255)],
) -> Attachment:
return await attachments.receive_attachment(
session, storage, user, chat_id, capacity=capacity, filename=filename,
content_type=request.headers.get('content-type', '').lower(),
declared_length=request.headers.get('content-length'), chunks=request.stream(),
session,
storage,
user,
chat_id,
capacity=capacity,
filename=filename,
content_type=request.headers.get("content-type", "").lower(),
declared_length=request.headers.get("content-length"),
chunks=request.stream(),
)
@router.get('/attachments/{attachment_id}')
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
@router.get("/attachments/{attachment_id}")
async def metadata(
attachment_id: UUID, user: CurrentUser, session: DbSession
) -> Attachment:
return await attachments.get_attachment(session, user, attachment_id)
@router.get('/attachments/{attachment_id}/content')
@router.get("/attachments/{attachment_id}/content")
async def content(
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
variant: Annotated[str, Query()] = 'original',
attachment_id: UUID,
user: CurrentUser,
session: DbSession,
storage: S3Storage,
variant: Annotated[str, Query()] = "original",
) -> StreamingResponse:
record, stream = await attachments.download_attachment(
session, storage, user, attachment_id, variant=variant,
session,
storage,
user,
attachment_id,
variant=variant,
)
return StreamingResponse(
stream, media_type=record.content_type,
stream,
media_type=record.content_type,
headers={
'Content-Length': str(record.size_bytes),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
"Content-Length": str(record.size_bytes),
"Content-Disposition": "attachment; filename*=UTF-8''"
+ quote(record.filename, safe=""),
"X-Content-Type-Options": "nosniff",
"Cache-Control": "private, no-store",
},
background=BackgroundTask(stream.aclose),
)
+7 -4
View File
@@ -3,18 +3,21 @@ from fastapi import APIRouter, Request
from app.domain import health
from app.schemas.base import Health
router = APIRouter(prefix='/health', tags=['health'])
router = APIRouter(prefix="/health", tags=["health"])
@router.get('/live')
@router.get("/live")
async def liveness() -> Health:
return health.liveness()
@router.get('/ready')
@router.get("/ready")
async def readiness(request: Request) -> Health:
state = request.app.state
return await health.readiness(
state.engine, state.redis, state.s3, state.lifecycle,
state.engine,
state.redis,
state.s3,
state.lifecycle,
state.settings.readiness_timeout_seconds,
)
+6 -2
View File
@@ -21,9 +21,13 @@ async def send_new_message(
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
) -> Message:
return await send_message_idempotent(
session, sender, chat_id, new_message,
session,
sender,
chat_id,
new_message,
validate_idempotency_key(idempotency_key),
settings.max_messages, settings.idempotency_ttl_seconds,
settings.max_messages,
settings.idempotency_ttl_seconds,
)
+12 -3
View File
@@ -90,14 +90,23 @@ def internal_error() -> ApiError:
IMAGE_WORKER_ERRORS = (
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
ApiError,
Image.DecompressionBombError,
Image.DecompressionBombWarning,
MemoryError,
UnidentifiedImageError,
OSError,
ValueError,
SyntaxError,
)
def image_worker_error_code(error: Exception) -> int:
code = 2
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
resource_error = isinstance(
error,
(Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError),
)
declared_limit = isinstance(error, ApiError) and error.status_code == 413
if resource_error or declared_limit:
code = 3
+17 -4
View File
@@ -18,14 +18,25 @@ from urllib3.exceptions import HTTPError as StorageHTTPError
from app.core.instance import INSTANCE_ID_HEADER
from app.core.request_id import REQUEST_ID_HEADER
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
from app.errors.exceptions import (
ApiError,
internal_error,
invalid_request,
not_found,
unavailable,
)
logger = logging.getLogger(__name__)
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
TRANSIENT_SQLSTATES = frozenset({"55P03"})
RECOVERABLE_STORAGE_ERRORS = (
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
ApiError,
SQLAlchemyError,
MinioException,
StorageHTTPError,
OSError,
TimeoutError,
)
@@ -40,12 +51,14 @@ async def operation_timeout(seconds: float, error: ApiError):
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
return await api_error_handler(
request, invalid_request("Загрузка прервана клиентом")
)
def log_recoverable_error(operation: str, error: Exception) -> None:
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
logger.warning('%s deferred: %s', operation, type(error).__name__)
logger.warning("%s deferred: %s", operation, type(error).__name__)
def _get_request_id(request: Request) -> str:
+3 -1
View File
@@ -52,7 +52,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
application.state.session_factory = create_session_factory(engine)
application.state.redis = redis
application.state.s3 = s3
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
application.state.upload_capacity = create_request_capacity(
settings.s3.max_concurrent_uploads
)
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
application.state.instance_id = resolve_instance_id(settings.instance_id)
application.state.lifecycle = Lifecycle()
+14 -1
View File
@@ -9,4 +9,17 @@ from app.models.idempotency_key import IdempotencyKey
from app.models.message import Message
from app.models.user import User
__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
__all__ = [
"JobReplay",
"AttachmentJob",
"Base",
"Attachment",
"MessageAttachment",
"UploadIntent",
"Chat",
"ChatMember",
"Counter",
"IdempotencyKey",
"Message",
"User",
]
+10 -3
View File
@@ -18,8 +18,13 @@ class Attachment(Base):
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
),
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"),
CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"),
CheckConstraint(
"status IN ('queued', 'processing', 'ready', 'failed')", name="status"
),
CheckConstraint(
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
name="error_code",
),
Index("ix_attachments_chat_id", "chat_id"),
)
@@ -36,7 +41,9 @@ class Attachment(Base):
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
crop_key: Mapped[str | None] = mapped_column(String(128))
optimized_key: Mapped[str | None] = mapped_column(String(128))
variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb"))
variants: Mapped[list[dict]] = mapped_column(
JSONB, default=list, server_default=text("'[]'::jsonb")
)
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+5 -2
View File
@@ -20,7 +20,8 @@ class AttachmentJob(Base):
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
attachment_id: Mapped[UUID] = mapped_column(
ForeignKey("attachments.id"), unique=True,
ForeignKey("attachments.id"),
unique=True,
)
staging_key: Mapped[str | None] = mapped_column(String(128))
pipeline_version: Mapped[int] = mapped_column(server_default="1")
@@ -29,7 +30,9 @@ class AttachmentJob(Base):
attempt_limit: Mapped[int | None]
last_error_permanent: Mapped[bool | None]
published_attempt: Mapped[int] = mapped_column(server_default="0")
next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
next_attempt_at: Mapped[datetime] = mapped_column(
server_default=func.clock_timestamp()
)
lease_token: Mapped[UUID | None]
lease_until: Mapped[datetime | None]
last_error_code: Mapped[str | None] = mapped_column(String(64))
+3 -1
View File
@@ -35,7 +35,9 @@ class Attachment(Base):
VARIANT_FILENAMES = {
"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp",
"thumbnail": "thumbnail.jpg",
"crop": "crop.jpg",
"optimized": "optimized.webp",
}
+3 -1
View File
@@ -16,5 +16,7 @@ class JobMessage(BaseModel):
pipeline_version: int = Field(default=1, ge=1)
attempt: int = Field(ge=1)
created_at: AwareDatetime
traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
traceparent: str | None = Field(
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
)
tracestate: str | None = Field(default=None, max_length=512)
+23 -7
View File
@@ -28,26 +28,42 @@ class JobBroker:
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
async def ensure_ready(self) -> None:
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
if (
self.connection is None
or self.connection.is_closed
or self.channel is None
or self.channel.is_closed
):
await self.aclose()
self.connection = await aio_pika.connect(
host=self.settings.broker_host, port=self.settings.broker_port,
host=self.settings.broker_host,
port=self.settings.broker_port,
login=self.settings.broker_user,
password=self.settings.broker_password_file.read_text().strip(),
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
virtualhost=self.settings.broker_vhost,
timeout=5,
heartbeat=30,
)
self.channel = await self.connection.channel(
publisher_confirms=True, on_return_raises=True
)
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
self.queue = await self.channel.declare_queue(
self.settings.broker_queue, durable=True
)
async def publish_job(self, message: JobMessage) -> None:
await self.ensure_ready()
await self.channel.default_exchange.publish(
aio_pika.Message(
body=message.model_dump_json(exclude_none=True).encode(),
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
content_type="application/json",
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
message_id=f"{message.job_id}:{message.attempt}",
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
),
routing_key=self.settings.broker_queue,
mandatory=True,
timeout=10,
)
async def receive(self, consumer: str) -> Delivery:
+1 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
async def check_database(engine: AsyncEngine) -> bool:
async with engine.connect() as connection:
await connection.execute(text('SELECT 1'))
await connection.execute(text("SELECT 1"))
return True
+12 -5
View File
@@ -33,7 +33,8 @@ async def get_active_key(
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
) == tuple_(*scope),
)
== tuple_(*scope),
IdempotencyKey.created_at
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
)
@@ -45,14 +46,20 @@ async def save_result(
) -> None:
"""Сохраняет новый результат или заменяет истёкший."""
stmt = insert(IdempotencyKey).values(
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
request_hash=request_hash, message_id=message_id,
user_id=scope[0],
method=scope[1],
path=scope[2],
key=scope[3],
request_hash=request_hash,
message_id=message_id,
)
await session.execute(
stmt.on_conflict_do_update(
index_elements=[
IdempotencyKey.user_id, IdempotencyKey.method,
IdempotencyKey.path, IdempotencyKey.key,
IdempotencyKey.user_id,
IdempotencyKey.method,
IdempotencyKey.path,
IdempotencyKey.key,
],
set_={
"request_hash": request_hash,
+16 -3
View File
@@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int):
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "failed")
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
.limit(limit)
)
return result.all()
@@ -23,12 +24,24 @@ async def replay_record(session, request_id):
async def insert_replay(session, **values) -> bool:
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
result = await session.scalar(
insert(JobReplay)
.values(**values)
.on_conflict_do_nothing(index_elements=[JobReplay.id])
.returning(JobReplay.id)
)
return result is not None
async def replay_history(session, job_id):
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
return (
await session.scalars(
select(JobReplay)
.where(JobReplay.job_id == job_id)
.order_by(JobReplay.created_at)
.limit(100)
)
).all()
def reset_failed(job, attachment, now, attempt_limit):
+43 -10
View File
@@ -32,7 +32,9 @@ class JobWork:
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
async def create_job(
session: AsyncSession, attachment_id: UUID, staging_key: str
) -> UUID:
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
session.add(job)
await session.flush()
@@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
.where(
Attachment.status == "processing",
AttachmentJob.lease_until < func.clock_timestamp(),
)
.order_by(AttachmentJob.lease_until)
.limit(limit)
.with_for_update(of=AttachmentJob, skip_locked=True)
@@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
return result.all()
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
def start_job(
job, attachment, token: UUID, now: datetime, lease_seconds: int
) -> JobWork:
job.attempt += 1
job.lease_token = token
job.lease_until = now + timedelta(seconds=lease_seconds)
attachment.status = "processing"
attachment.updated_at = now
return JobWork(
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
attachment.content_type, attachment.size_bytes, attachment.sha256,
job.id,
attachment.id,
token,
job.pipeline_version,
job.staging_key,
attachment.content_type,
attachment.size_bytes,
attachment.sha256,
)
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
def record_failure(
job,
attachment,
now: datetime,
error_code: str,
*,
terminal: bool,
retry_seconds: float,
) -> None:
job.lease_token = None
job.lease_until = None
job.last_error_code = error_code
@@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal:
attachment.updated_at = now
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
def complete_job(
job, attachment, work: JobWork, variants: list[dict], now: datetime
) -> None:
attachment.object_key = work.original_key
attachment.thumbnail_key = work.variant_key("thumbnail")
attachment.crop_key = work.variant_key("crop")
@@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date
job.finished_at = now
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
async def terminal_staging(
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
):
result = await session.execute(
select(AttachmentJob, Attachment)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
.where(
AttachmentJob.staging_key.is_not(None),
or_(
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
and_(
Attachment.status == "ready",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=retention),
),
and_(
Attachment.status == "failed",
AttachmentJob.finished_at
< func.clock_timestamp() - timedelta(seconds=failed_retention),
),
),
)
.order_by(AttachmentJob.finished_at)
+3 -1
View File
@@ -9,7 +9,9 @@ from app.models.attachment import Attachment
from app.models.job import AttachmentJob
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
async def next_unpublished_job(
session: AsyncSession, *, attachment_id: UUID | None = None
) -> AttachmentJob | None:
statement = (
select(AttachmentJob)
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)