forked from mtusi2026/mtusi2026lab5
formatted project by python-black
This commit is contained in:
@@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
|
||||
from sqlalchemy.exc import InterfaceError, OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||
|
||||
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
|
||||
TRANSIENT_JOB_ERRORS = (
|
||||
AMQPException,
|
||||
ChannelInvalidStateError,
|
||||
InterfaceError,
|
||||
OperationalError,
|
||||
PoolTimeoutError,
|
||||
OSError,
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool
|
||||
async with factory() as session:
|
||||
result = await reconcile_attachments(session, storage, dry_run=not apply)
|
||||
staging = await cleanup_staging(session, storage, jobs, apply=apply)
|
||||
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
|
||||
print(
|
||||
json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
|
||||
)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error("Reconciliation", error)
|
||||
completed = False
|
||||
|
||||
@@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class ImageSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="IMAGE_", env_file=".env", extra="ignore"
|
||||
)
|
||||
|
||||
jpeg_quality: int = Field(default=85, ge=1, le=95)
|
||||
webp_quality: int = Field(default=80, ge=1, le=100)
|
||||
|
||||
+22
-5
@@ -9,7 +9,13 @@ from app.core.background import log_background_error
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
|
||||
from app.domain.job_admin import (
|
||||
REPLAY_REASONS,
|
||||
history,
|
||||
list_failed,
|
||||
replay_failed_job,
|
||||
show_job,
|
||||
)
|
||||
from app.errors.exceptions import JobOperationError
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
@@ -29,8 +35,15 @@ async def execute(args):
|
||||
result = await history(session, args.job_id)
|
||||
else:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
|
||||
request_id=args.request_id, actor=args.actor, reason=args.reason)
|
||||
result = await replay_failed_job(
|
||||
session,
|
||||
storage,
|
||||
JobSettings(),
|
||||
args.job_id,
|
||||
request_id=args.request_id,
|
||||
actor=args.actor,
|
||||
reason=args.reason,
|
||||
)
|
||||
print(json.dumps(result, default=str), flush=True)
|
||||
finally:
|
||||
if storage is not None:
|
||||
@@ -39,10 +52,14 @@ async def execute(args):
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Реестр failed и явный replay с аудитом"
|
||||
)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
listing = commands.add_parser("list-failed")
|
||||
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
|
||||
listing.add_argument(
|
||||
"--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
|
||||
)
|
||||
for name in ("show", "history", "replay"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("job_id", type=UUID)
|
||||
|
||||
+42
-9
@@ -12,7 +12,12 @@ from uuid import UUID, uuid4
|
||||
from PIL import features
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||||
from app.core.background import (
|
||||
TRANSIENT_JOB_ERRORS,
|
||||
log_background_error,
|
||||
shutdown_event,
|
||||
wait_or_stop,
|
||||
)
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
@@ -34,7 +39,10 @@ class Faults:
|
||||
object_name: str = "optimized"
|
||||
|
||||
def kill(self, attachment_id: UUID, phase: str) -> None:
|
||||
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
|
||||
print(
|
||||
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
|
||||
flush=True,
|
||||
)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
def after_object(self, attachment_id: UUID, name: str) -> None:
|
||||
@@ -52,7 +60,14 @@ class Faults:
|
||||
raise ProcessingError("injected_permanent")
|
||||
|
||||
|
||||
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
|
||||
async def consume_connected(
|
||||
factory,
|
||||
storage,
|
||||
broker: JobBroker,
|
||||
settings: JobSettings,
|
||||
stop: asyncio.Event,
|
||||
faults: Faults,
|
||||
) -> None:
|
||||
consumer = uuid4().hex
|
||||
while not stop.is_set():
|
||||
try:
|
||||
@@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe
|
||||
try:
|
||||
async with factory() as session:
|
||||
acknowledge = await process_attachment_job(
|
||||
session, storage, message, settings,
|
||||
after_object=faults.after_object, before_processing=faults.before_processing,
|
||||
session,
|
||||
storage,
|
||||
message,
|
||||
settings,
|
||||
after_object=faults.after_object,
|
||||
before_processing=faults.before_processing,
|
||||
)
|
||||
except ProcessingError:
|
||||
await broker.reject_invalid(delivery.id)
|
||||
@@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
factory = create_session_factory(engine)
|
||||
async with asyncio.TaskGroup() as group:
|
||||
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
|
||||
tasks = [
|
||||
group.create_task(consume(factory, storage, jobs, stop, faults))
|
||||
for _ in range(jobs.worker_concurrency)
|
||||
]
|
||||
await stop.wait()
|
||||
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
|
||||
for task in tasks:
|
||||
@@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Image worker с явными fault-injection hooks"
|
||||
)
|
||||
modes = parser.add_mutually_exclusive_group()
|
||||
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
|
||||
for flag in (
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent",
|
||||
):
|
||||
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
|
||||
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
|
||||
parser.add_argument(
|
||||
"--object-name",
|
||||
choices=("original", "thumbnail", "crop", "optimized"),
|
||||
default="optimized",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
faults = Faults(**vars(args))
|
||||
try:
|
||||
|
||||
+3
-1
@@ -29,7 +29,9 @@ class JobSettings(BaseSettings):
|
||||
@model_validator(mode="after")
|
||||
def validate_lease(self) -> Self:
|
||||
if self.job_lease_seconds < self.job_timeout_seconds + 30:
|
||||
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
|
||||
raise ValueError(
|
||||
"Lease должен превышать timeout задания минимум на 30 секунд"
|
||||
)
|
||||
if self.job_retry_max_seconds < self.job_retry_seconds:
|
||||
raise ValueError("Retry maximum must be at least the base delay")
|
||||
return self
|
||||
|
||||
@@ -8,7 +8,12 @@ import os
|
||||
import signal
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||||
from app.core.background import (
|
||||
TRANSIENT_JOB_ERRORS,
|
||||
log_background_error,
|
||||
shutdown_event,
|
||||
wait_or_stop,
|
||||
)
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
@@ -18,7 +23,10 @@ from app.storage.broker import JobBroker
|
||||
|
||||
def crash_before_publish(attachment_id: UUID) -> None:
|
||||
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
|
||||
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
|
||||
print(
|
||||
json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
|
||||
flush=True,
|
||||
)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
|
||||
@@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
async with factory() as session:
|
||||
await recover_expired_jobs(session, jobs)
|
||||
published = await publish_next_job(
|
||||
session, broker, attachment_id=crash_attachment_id,
|
||||
before_publish=crash_before_publish if crash_attachment_id is not None else None,
|
||||
session,
|
||||
broker,
|
||||
attachment_id=crash_attachment_id,
|
||||
before_publish=(
|
||||
crash_before_publish
|
||||
if crash_attachment_id is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
except TRANSIENT_JOB_ERRORS as error:
|
||||
log_background_error("Publisher deferred", error)
|
||||
@@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Публикация устойчивых jobs в RabbitMQ"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
|
||||
"--crash-before-publish",
|
||||
type=UUID,
|
||||
metavar="ATTACHMENT_ID",
|
||||
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
+1
-2
@@ -4,6 +4,7 @@ from redis.backoff import NoBackoff
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
def create_redis_client(settings: Settings) -> Redis:
|
||||
password = (
|
||||
settings.redis_password.get_secret_value() if settings.redis_password else None
|
||||
@@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis:
|
||||
decode_responses=True,
|
||||
)
|
||||
return Redis.from_pool(pool)
|
||||
|
||||
|
||||
|
||||
+16
-4
@@ -6,6 +6,7 @@ from urllib.parse import urlsplit
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class S3Settings(BaseSettings):
|
||||
endpoint: str = "http://minio:9000"
|
||||
region: str = "us-east-1"
|
||||
@@ -24,12 +25,23 @@ class S3Settings(BaseSettings):
|
||||
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
|
||||
cleanup_interval_seconds: int = Field(default=300, ge=1)
|
||||
|
||||
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_endpoint(self):
|
||||
url = urlsplit(self.endpoint)
|
||||
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
|
||||
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
|
||||
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
|
||||
if (
|
||||
url.scheme not in {"http", "https"}
|
||||
or not url.hostname
|
||||
or url.username
|
||||
or url.password
|
||||
or url.path not in {"", "/"}
|
||||
or url.query
|
||||
or url.fragment
|
||||
):
|
||||
raise ValueError(
|
||||
"S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
|
||||
)
|
||||
return self
|
||||
|
||||
@@ -13,11 +13,15 @@ def main() -> int:
|
||||
settings = ImageSettings()
|
||||
memory = settings.memory_mib * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
|
||||
)
|
||||
source, directory, metadata, content_type, maximum = sys.argv[1:]
|
||||
code = 0
|
||||
try:
|
||||
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
|
||||
result = create_variants(
|
||||
Path(source), Path(directory), content_type, int(maximum), settings
|
||||
)
|
||||
Path(metadata).write_text(result.model_dump_json())
|
||||
except IMAGE_WORKER_ERRORS as error:
|
||||
code = pipeline_worker_error_code(error)
|
||||
|
||||
@@ -29,13 +29,17 @@ class Claim:
|
||||
acknowledge: bool = False
|
||||
|
||||
|
||||
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
async def enqueue_attachment_job(
|
||||
session: AsyncSession, attachment_id: UUID, staging_key: str
|
||||
) -> UUID:
|
||||
"""Включить намерение обработки в транзакцию queued metadata; без commit."""
|
||||
return await jobs.create_job(session, attachment_id, staging_key)
|
||||
|
||||
|
||||
async def publish_next_job(
|
||||
session: AsyncSession, broker: JobBroker, *,
|
||||
session: AsyncSession,
|
||||
broker: JobBroker,
|
||||
*,
|
||||
attachment_id: UUID | None = None,
|
||||
before_publish: Callable[[UUID], None] | None = None,
|
||||
) -> bool:
|
||||
@@ -45,24 +49,33 @@ async def publish_next_job(
|
||||
if job is not None:
|
||||
if before_publish is not None:
|
||||
before_publish(job.attachment_id)
|
||||
await broker.publish_job(JobMessage(
|
||||
job_id=job.id, attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version, attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
))
|
||||
await broker.publish_job(
|
||||
JobMessage(
|
||||
job_id=job.id,
|
||||
attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version,
|
||||
attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
)
|
||||
)
|
||||
outbox.mark_published(job)
|
||||
await session.commit()
|
||||
return published
|
||||
|
||||
|
||||
async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim:
|
||||
async def claim_job(
|
||||
session: AsyncSession, message: JobMessage, settings: JobSettings
|
||||
) -> Claim:
|
||||
"""Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
|
||||
row = await jobs.lock_job(session, message.job_id)
|
||||
result = Claim()
|
||||
if row is None:
|
||||
raise ProcessingError("unknown_job")
|
||||
job, attachment = row
|
||||
if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version:
|
||||
if (
|
||||
message.attachment_id != attachment.id
|
||||
or message.pipeline_version != job.pipeline_version
|
||||
):
|
||||
raise ProcessingError("invalid_job")
|
||||
now = await jobs.database_time(session)
|
||||
if job.attempt_limit is None:
|
||||
@@ -77,16 +90,32 @@ async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSet
|
||||
elif message.attempt > job.attempt + 1:
|
||||
raise ProcessingError("invalid_attempt")
|
||||
elif attachment.status == "queued" and job.next_attempt_at <= now:
|
||||
if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None:
|
||||
jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0)
|
||||
if (
|
||||
job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
or job.staging_key is None
|
||||
):
|
||||
jobs.record_failure(
|
||||
job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0
|
||||
)
|
||||
result = Claim(acknowledge=True)
|
||||
else:
|
||||
result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds))
|
||||
result = Claim(
|
||||
work=jobs.start_job(
|
||||
job, attachment, uuid4(), now, settings.job_lease_seconds
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
|
||||
async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None:
|
||||
async def save_failure(
|
||||
session: AsyncSession,
|
||||
work: JobWork,
|
||||
settings: JobSettings,
|
||||
code: str,
|
||||
*,
|
||||
permanent: bool,
|
||||
) -> None:
|
||||
# Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
|
||||
await session.rollback()
|
||||
row = await jobs.lock_job(session, work.job_id)
|
||||
@@ -95,10 +124,14 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
|
||||
if attachment.status == "processing" and job.lease_token == work.lease_token:
|
||||
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
jobs.record_failure(
|
||||
job, attachment, await jobs.database_time(session),
|
||||
job,
|
||||
attachment,
|
||||
await jobs.database_time(session),
|
||||
"retry_exhausted" if exhausted and not permanent else code,
|
||||
terminal=permanent or exhausted,
|
||||
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
retry_seconds=retry_delay(
|
||||
settings, job.attempt - job.cycle_start_attempt
|
||||
),
|
||||
)
|
||||
job.last_error_code = code
|
||||
job.last_error_permanent = permanent
|
||||
@@ -106,7 +139,9 @@ async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettin
|
||||
|
||||
|
||||
async def build_result(
|
||||
session: AsyncSession, storage: MinioStorage, work: JobWork,
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
work: JobWork,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
) -> bool:
|
||||
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
|
||||
@@ -129,26 +164,42 @@ async def build_result(
|
||||
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
|
||||
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
|
||||
await storage.put_file(
|
||||
work.original_key, source, size_bytes=work.size_bytes,
|
||||
content_type=work.content_type, sha256=work.sha256,
|
||||
work.original_key,
|
||||
source,
|
||||
size_bytes=work.size_bytes,
|
||||
content_type=work.content_type,
|
||||
sha256=work.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, "original")
|
||||
for variant in result.variants:
|
||||
await storage.put_file(
|
||||
work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256,
|
||||
work.variant_key(variant.name),
|
||||
Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes,
|
||||
content_type=variant.content_type,
|
||||
sha256=variant.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, variant.name)
|
||||
jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session))
|
||||
jobs.complete_job(
|
||||
job,
|
||||
attachment,
|
||||
work,
|
||||
[variant.model_dump() for variant in result.variants],
|
||||
await jobs.database_time(session),
|
||||
)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def process_attachment_job(
|
||||
session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings,
|
||||
*, after_object: Callable[[UUID, str], None] | None = None,
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
message: JobMessage,
|
||||
settings: JobSettings,
|
||||
*,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
before_processing: Callable[[UUID], None] | None = None,
|
||||
) -> bool:
|
||||
claim = await claim_job(session, message, settings)
|
||||
@@ -160,7 +211,13 @@ async def process_attachment_job(
|
||||
if before_processing is not None:
|
||||
before_processing(work.attachment_id)
|
||||
acknowledge = await build_result(session, storage, work, after_object)
|
||||
except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error:
|
||||
except (
|
||||
ProcessingError,
|
||||
MinioException,
|
||||
StorageHTTPError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
) as error:
|
||||
code, permanent = classify_failure(error)
|
||||
await save_failure(session, work, settings, code, permanent=permanent)
|
||||
acknowledge = True
|
||||
@@ -176,8 +233,12 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
|
||||
job.attempt_limit = settings.job_max_attempts
|
||||
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
jobs.record_failure(
|
||||
job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout",
|
||||
terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
job,
|
||||
attachment,
|
||||
now,
|
||||
"retry_exhausted" if exhausted else "worker_timeout",
|
||||
terminal=exhausted,
|
||||
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
)
|
||||
job.last_error_code = "worker_timeout"
|
||||
job.last_error_permanent = False
|
||||
@@ -185,8 +246,14 @@ async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) ->
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int:
|
||||
rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds)
|
||||
async def cleanup_staging(
|
||||
session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool
|
||||
) -> int:
|
||||
rows = await jobs.terminal_staging(
|
||||
session,
|
||||
settings.staging_retention_seconds,
|
||||
settings.failed_staging_retention_seconds,
|
||||
)
|
||||
if apply:
|
||||
for job, attachment in rows:
|
||||
if job.staging_key != attachment.object_key:
|
||||
|
||||
@@ -276,12 +276,20 @@ async def download_attachment(
|
||||
key = record.object_key
|
||||
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
|
||||
else:
|
||||
selected = next((item for item in result.variants if item.name == variant), None)
|
||||
keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key}
|
||||
selected = next(
|
||||
(item for item in result.variants if item.name == variant), None
|
||||
)
|
||||
keys = {
|
||||
"thumbnail": record.thumbnail_key,
|
||||
"crop": record.crop_key,
|
||||
"optimized": record.optimized_key,
|
||||
}
|
||||
key = keys.get(variant)
|
||||
if selected is None or key is None:
|
||||
raise not_found()
|
||||
info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes)
|
||||
info = DownloadInfo(
|
||||
VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
|
||||
)
|
||||
await session.rollback()
|
||||
stored = await storage.stat_object(key)
|
||||
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
|
||||
|
||||
@@ -14,12 +14,15 @@ from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
def liveness() -> Health:
|
||||
return Health(status='ok')
|
||||
return Health(status="ok")
|
||||
|
||||
|
||||
async def readiness(
|
||||
engine: AsyncEngine, redis: Redis, storage: MinioStorage | None,
|
||||
lifecycle: Lifecycle, timeout_seconds: float,
|
||||
engine: AsyncEngine,
|
||||
redis: Redis,
|
||||
storage: MinioStorage | None,
|
||||
lifecycle: Lifecycle,
|
||||
timeout_seconds: float,
|
||||
) -> Health:
|
||||
if lifecycle.draining:
|
||||
raise unavailable()
|
||||
@@ -30,4 +33,4 @@ async def readiness(
|
||||
dependencies = await asyncio.gather(*checks, return_exceptions=True)
|
||||
if not all(result is True for result in dependencies):
|
||||
raise unavailable()
|
||||
return Health(status='ok')
|
||||
return Health(status="ok")
|
||||
|
||||
@@ -57,16 +57,22 @@ async def send_message_idempotent(
|
||||
raise conflict("Idempotency-Key уже использован для другого сообщения")
|
||||
message = await message_storage.get_message(session, previous.message_id)
|
||||
else:
|
||||
await validate_message_attachments(session, sender, chat_id, new_message.attachment_ids)
|
||||
await validate_message_attachments(
|
||||
session, sender, chat_id, new_message.attachment_ids
|
||||
)
|
||||
if not await message_storage.reserve_message_slot(session, max_messages):
|
||||
raise unavailable("Достигнут лимит количества сообщений")
|
||||
message = await message_storage.create_message(
|
||||
session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
|
||||
)
|
||||
await attachment_storage.bind_attachments(session, message.id, new_message.attachment_ids)
|
||||
await attachment_storage.bind_attachments(
|
||||
session, message.id, new_message.attachment_ids
|
||||
)
|
||||
await key_storage.save_result(session, scope, fingerprint, message.id)
|
||||
|
||||
links = await attachment_storage.message_attachment_ids(session, [message.id])
|
||||
result = Message.model_validate(message).model_copy(update={"attachment_ids": links[message.id]})
|
||||
result = Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]}
|
||||
)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
@@ -15,18 +15,27 @@ def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
|
||||
longest = max(size)
|
||||
result = size
|
||||
if longest > maximum:
|
||||
result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest))
|
||||
result = (
|
||||
max(1, size[0] * maximum // longest),
|
||||
max(1, size[1] * maximum // longest),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant:
|
||||
def describe_variant(
|
||||
name: str, content_type: str, path: Path, size: tuple[int, int]
|
||||
) -> Variant:
|
||||
checksum = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(65536), b""):
|
||||
checksum.update(chunk)
|
||||
return Variant(
|
||||
name=name, content_type=content_type, size_bytes=path.stat().st_size,
|
||||
width=size[0], height=size[1], sha256=checksum.hexdigest(),
|
||||
name=name,
|
||||
content_type=content_type,
|
||||
size_bytes=path.stat().st_size,
|
||||
width=size[0],
|
||||
height=size[1],
|
||||
sha256=checksum.hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
@@ -47,32 +56,49 @@ def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Var
|
||||
|
||||
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
|
||||
"""Центрированный JPEG 128×128; маленький исходник допускает увеличение."""
|
||||
with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop:
|
||||
with ImageOps.fit(
|
||||
image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
|
||||
) as crop:
|
||||
save_jpeg(crop, destination, quality)
|
||||
return describe_variant("crop", "image/jpeg", destination, (128, 128))
|
||||
|
||||
|
||||
def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant:
|
||||
def create_optimized(
|
||||
image: Image.Image, destination: Path, settings: ImageSettings
|
||||
) -> Variant:
|
||||
"""WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются."""
|
||||
size = fitted_size(image.size, 1280)
|
||||
with image.resize(size, Image.Resampling.LANCZOS) as resized:
|
||||
with Image.new("RGBA", size) as clean:
|
||||
clean.paste(resized)
|
||||
clean.save(
|
||||
destination, format="WEBP", lossless=False,
|
||||
quality=settings.webp_quality, method=settings.webp_method,
|
||||
exif=b"", icc_profile=b"", xmp=b"",
|
||||
destination,
|
||||
format="WEBP",
|
||||
lossless=False,
|
||||
quality=settings.webp_quality,
|
||||
method=settings.webp_method,
|
||||
exif=b"",
|
||||
icc_profile=b"",
|
||||
xmp=b"",
|
||||
)
|
||||
return describe_variant("optimized", "image/webp", destination, size)
|
||||
|
||||
|
||||
def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult:
|
||||
def create_variants(
|
||||
source: Path,
|
||||
directory: Path,
|
||||
content_type: str,
|
||||
max_pixels: int,
|
||||
settings: ImageSettings,
|
||||
) -> PipelineResult:
|
||||
decode_image(source, content_type, max_pixels)
|
||||
with Image.open(source, formats=["PNG", "JPEG"]) as image:
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
with image.convert("RGBA") as pixels:
|
||||
variants = [
|
||||
create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality),
|
||||
create_thumbnail(
|
||||
pixels, directory / "thumbnail.jpg", settings.jpeg_quality
|
||||
),
|
||||
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
|
||||
create_optimized(pixels, directory / "optimized.webp", settings),
|
||||
]
|
||||
|
||||
+71
-19
@@ -17,15 +17,28 @@ REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
|
||||
|
||||
|
||||
def snapshot(job, attachment):
|
||||
return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status,
|
||||
attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt,
|
||||
error_code=attachment.error_code, last_error_code=job.last_error_code,
|
||||
last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None,
|
||||
next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at)
|
||||
return dict(
|
||||
job_id=str(job.id),
|
||||
attachment_id=str(attachment.id),
|
||||
status=attachment.status,
|
||||
attempt=job.attempt,
|
||||
attempt_limit=job.attempt_limit,
|
||||
cycle_start_attempt=job.cycle_start_attempt,
|
||||
error_code=attachment.error_code,
|
||||
last_error_code=job.last_error_code,
|
||||
last_error_permanent=job.last_error_permanent,
|
||||
staging_retained=job.staging_key is not None,
|
||||
next_attempt_at=job.next_attempt_at,
|
||||
lease_until=job.lease_until,
|
||||
finished_at=job.finished_at,
|
||||
)
|
||||
|
||||
|
||||
async def list_failed(session, limit):
|
||||
return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)]
|
||||
return [
|
||||
snapshot(job, attachment)
|
||||
for job, attachment in await job_admin.failed_jobs(session, limit)
|
||||
]
|
||||
|
||||
|
||||
async def show_job(session, job_id):
|
||||
@@ -38,20 +51,37 @@ async def show_job(session, job_id):
|
||||
|
||||
|
||||
async def history(session, job_id):
|
||||
return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason,
|
||||
previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit,
|
||||
previous_error_code=row.previous_error_code, created_at=row.created_at)
|
||||
for row in await job_admin.replay_history(session, job_id)]
|
||||
return [
|
||||
dict(
|
||||
request_id=str(row.id),
|
||||
actor=row.actor,
|
||||
reason=row.reason,
|
||||
previous_attempt=row.previous_attempt,
|
||||
attempt_limit=row.attempt_limit,
|
||||
previous_error_code=row.previous_error_code,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
for row in await job_admin.replay_history(session, job_id)
|
||||
]
|
||||
|
||||
|
||||
def same_request(record, job_id, actor, reason):
|
||||
if record.job_id != job_id or record.actor != actor or record.reason != reason:
|
||||
raise JobOperationError("request_conflict")
|
||||
return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True}
|
||||
return {
|
||||
"request_id": str(record.id),
|
||||
"job_id": str(job_id),
|
||||
"already_applied": True,
|
||||
}
|
||||
|
||||
|
||||
async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason):
|
||||
if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS:
|
||||
async def replay_failed_job(
|
||||
session, storage, settings, job_id, *, request_id, actor, reason
|
||||
):
|
||||
if (
|
||||
re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
|
||||
or reason not in REPLAY_REASONS
|
||||
):
|
||||
raise JobOperationError("invalid_audit_fields")
|
||||
row = await jobs.lock_job(session, job_id)
|
||||
if row is None:
|
||||
@@ -67,8 +97,16 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
|
||||
if job.staging_key is None:
|
||||
raise JobOperationError("source_expired")
|
||||
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
|
||||
work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256)
|
||||
work = JobWork(
|
||||
job.id,
|
||||
attachment.id,
|
||||
uuid4(),
|
||||
job.pipeline_version,
|
||||
job.staging_key,
|
||||
attachment.content_type,
|
||||
attachment.size_bytes,
|
||||
attachment.sha256,
|
||||
)
|
||||
try:
|
||||
async with asyncio.timeout(settings.job_timeout_seconds):
|
||||
with TemporaryDirectory(prefix="replay-source-") as directory:
|
||||
@@ -81,13 +119,27 @@ async def replay_failed_job(session, storage, settings, job_id, *, request_id, a
|
||||
raise
|
||||
limit = job.attempt + settings.job_max_attempts
|
||||
inserted = await job_admin.insert_replay(
|
||||
session, id=request_id, job_id=job_id, actor=actor, reason=reason,
|
||||
previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code,
|
||||
session,
|
||||
id=request_id,
|
||||
job_id=job_id,
|
||||
actor=actor,
|
||||
reason=reason,
|
||||
previous_attempt=job.attempt,
|
||||
attempt_limit=limit,
|
||||
previous_error_code=job.last_error_code,
|
||||
)
|
||||
if not inserted:
|
||||
result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason)
|
||||
result = same_request(
|
||||
await job_admin.replay_record(session, request_id), job_id, actor, reason
|
||||
)
|
||||
await session.rollback()
|
||||
return result
|
||||
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
|
||||
await session.commit()
|
||||
return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit}
|
||||
return {
|
||||
"request_id": str(request_id),
|
||||
"job_id": str(job_id),
|
||||
"already_applied": False,
|
||||
"status": "queued",
|
||||
"attempt_limit": limit,
|
||||
}
|
||||
|
||||
+16
-5
@@ -14,7 +14,9 @@ from app.storage.jobs import JobWork
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None:
|
||||
async def fetch_original(
|
||||
storage: MinioStorage, work: JobWork, destination: Path
|
||||
) -> None:
|
||||
if work.size_bytes > storage.settings.max_upload_bytes:
|
||||
raise ProcessingError("image_limits")
|
||||
checksum = hashlib.sha256()
|
||||
@@ -34,12 +36,21 @@ async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path
|
||||
raise ProcessingError("source_mismatch")
|
||||
|
||||
|
||||
async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult:
|
||||
async def render_variants(
|
||||
storage: MinioStorage, work: JobWork, source: Path, directory: Path
|
||||
) -> PipelineResult:
|
||||
metadata = directory / "variants.json"
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory),
|
||||
str(metadata), work.content_type, str(storage.settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
||||
sys.executable,
|
||||
"-m",
|
||||
"app.core.thumbnail_worker",
|
||||
str(source),
|
||||
str(directory),
|
||||
str(metadata),
|
||||
work.content_type,
|
||||
str(storage.settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
code = await process.wait()
|
||||
|
||||
+23
-4
@@ -10,7 +10,10 @@ from app.errors.exceptions import ProcessingError
|
||||
|
||||
|
||||
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
|
||||
ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)))
|
||||
ceiling = min(
|
||||
settings.job_retry_max_seconds,
|
||||
settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)),
|
||||
)
|
||||
return random.uniform(ceiling / 2, ceiling)
|
||||
|
||||
|
||||
@@ -20,13 +23,29 @@ def classify_failure(error: Exception) -> tuple[str, bool]:
|
||||
if isinstance(error, S3Error):
|
||||
if error.code in {"NoSuchKey", "NoSuchObject"}:
|
||||
return "source_missing", True
|
||||
if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}:
|
||||
if error.code in {
|
||||
"AccessDenied",
|
||||
"InvalidAccessKeyId",
|
||||
"SignatureDoesNotMatch",
|
||||
"NoSuchBucket",
|
||||
}:
|
||||
return "storage_configuration", True
|
||||
if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}:
|
||||
if error.code in {
|
||||
"InternalError",
|
||||
"SlowDown",
|
||||
"ServiceUnavailable",
|
||||
"RequestTimeout",
|
||||
"Throttling",
|
||||
}:
|
||||
return "storage_unavailable", False
|
||||
return "storage_rejected", True
|
||||
if isinstance(error, TimeoutError):
|
||||
return "processing_timeout", False
|
||||
if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
|
||||
if isinstance(error, OSError) and error.errno in {
|
||||
errno.EACCES,
|
||||
errno.EPERM,
|
||||
errno.EROFS,
|
||||
errno.ENOENT,
|
||||
}:
|
||||
return "processing_configuration", True
|
||||
return "processing_unavailable", False
|
||||
|
||||
@@ -32,10 +32,15 @@ async def list_messages(
|
||||
|
||||
rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
|
||||
page_rows, has_next = split_page(rows, limit)
|
||||
links = await attachments_storage.message_attachment_ids(session, [message.id for message in page_rows])
|
||||
items = [Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]},
|
||||
) for message in page_rows]
|
||||
links = await attachments_storage.message_attachment_ids(
|
||||
session, [message.id for message in page_rows]
|
||||
)
|
||||
items = [
|
||||
Message.model_validate(message).model_copy(
|
||||
update={"attachment_ids": links[message.id]},
|
||||
)
|
||||
for message in page_rows
|
||||
]
|
||||
page = MessagePage(
|
||||
items=items,
|
||||
next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
|
||||
|
||||
@@ -12,41 +12,63 @@ from app.domain import attachments
|
||||
from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
|
||||
from app.schemas.attachment import Attachment
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
router = APIRouter(prefix="/api/v1", tags=["attachments"])
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=202)
|
||||
@router.post("/chats/{chat_id}/attachments", status_code=202)
|
||||
async def upload(
|
||||
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
|
||||
storage: S3Storage, capacity: UploadCapacity,
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
user: CurrentUser,
|
||||
session: DbSession,
|
||||
storage: S3Storage,
|
||||
capacity: UploadCapacity,
|
||||
filename: Annotated[str, Query(min_length=1, max_length=255)],
|
||||
) -> Attachment:
|
||||
return await attachments.receive_attachment(
|
||||
session, storage, user, chat_id, capacity=capacity, filename=filename,
|
||||
content_type=request.headers.get('content-type', '').lower(),
|
||||
declared_length=request.headers.get('content-length'), chunks=request.stream(),
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
chat_id,
|
||||
capacity=capacity,
|
||||
filename=filename,
|
||||
content_type=request.headers.get("content-type", "").lower(),
|
||||
declared_length=request.headers.get("content-length"),
|
||||
chunks=request.stream(),
|
||||
)
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}')
|
||||
async def metadata(attachment_id: UUID, user: CurrentUser, session: DbSession) -> Attachment:
|
||||
@router.get("/attachments/{attachment_id}")
|
||||
async def metadata(
|
||||
attachment_id: UUID, user: CurrentUser, session: DbSession
|
||||
) -> Attachment:
|
||||
return await attachments.get_attachment(session, user, attachment_id)
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}/content')
|
||||
@router.get("/attachments/{attachment_id}/content")
|
||||
async def content(
|
||||
attachment_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage,
|
||||
variant: Annotated[str, Query()] = 'original',
|
||||
attachment_id: UUID,
|
||||
user: CurrentUser,
|
||||
session: DbSession,
|
||||
storage: S3Storage,
|
||||
variant: Annotated[str, Query()] = "original",
|
||||
) -> StreamingResponse:
|
||||
record, stream = await attachments.download_attachment(
|
||||
session, storage, user, attachment_id, variant=variant,
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
attachment_id,
|
||||
variant=variant,
|
||||
)
|
||||
return StreamingResponse(
|
||||
stream, media_type=record.content_type,
|
||||
stream,
|
||||
media_type=record.content_type,
|
||||
headers={
|
||||
'Content-Length': str(record.size_bytes),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(record.filename, safe=''),
|
||||
'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'private, no-store',
|
||||
"Content-Length": str(record.size_bytes),
|
||||
"Content-Disposition": "attachment; filename*=UTF-8''"
|
||||
+ quote(record.filename, safe=""),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, no-store",
|
||||
},
|
||||
background=BackgroundTask(stream.aclose),
|
||||
)
|
||||
|
||||
@@ -3,18 +3,21 @@ from fastapi import APIRouter, Request
|
||||
from app.domain import health
|
||||
from app.schemas.base import Health
|
||||
|
||||
router = APIRouter(prefix='/health', tags=['health'])
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
|
||||
@router.get('/live')
|
||||
@router.get("/live")
|
||||
async def liveness() -> Health:
|
||||
return health.liveness()
|
||||
|
||||
|
||||
@router.get('/ready')
|
||||
@router.get("/ready")
|
||||
async def readiness(request: Request) -> Health:
|
||||
state = request.app.state
|
||||
return await health.readiness(
|
||||
state.engine, state.redis, state.s3, state.lifecycle,
|
||||
state.engine,
|
||||
state.redis,
|
||||
state.s3,
|
||||
state.lifecycle,
|
||||
state.settings.readiness_timeout_seconds,
|
||||
)
|
||||
|
||||
@@ -21,9 +21,13 @@ async def send_new_message(
|
||||
idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
|
||||
) -> Message:
|
||||
return await send_message_idempotent(
|
||||
session, sender, chat_id, new_message,
|
||||
session,
|
||||
sender,
|
||||
chat_id,
|
||||
new_message,
|
||||
validate_idempotency_key(idempotency_key),
|
||||
settings.max_messages, settings.idempotency_ttl_seconds,
|
||||
settings.max_messages,
|
||||
settings.idempotency_ttl_seconds,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -90,14 +90,23 @@ def internal_error() -> ApiError:
|
||||
|
||||
|
||||
IMAGE_WORKER_ERRORS = (
|
||||
ApiError, Image.DecompressionBombError, Image.DecompressionBombWarning,
|
||||
MemoryError, UnidentifiedImageError, OSError, ValueError, SyntaxError,
|
||||
ApiError,
|
||||
Image.DecompressionBombError,
|
||||
Image.DecompressionBombWarning,
|
||||
MemoryError,
|
||||
UnidentifiedImageError,
|
||||
OSError,
|
||||
ValueError,
|
||||
SyntaxError,
|
||||
)
|
||||
|
||||
|
||||
def image_worker_error_code(error: Exception) -> int:
|
||||
code = 2
|
||||
resource_error = isinstance(error, (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError))
|
||||
resource_error = isinstance(
|
||||
error,
|
||||
(Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError),
|
||||
)
|
||||
declared_limit = isinstance(error, ApiError) and error.status_code == 413
|
||||
if resource_error or declared_limit:
|
||||
code = 3
|
||||
|
||||
+17
-4
@@ -18,14 +18,25 @@ from urllib3.exceptions import HTTPError as StorageHTTPError
|
||||
|
||||
from app.core.instance import INSTANCE_ID_HEADER
|
||||
from app.core.request_id import REQUEST_ID_HEADER
|
||||
from app.errors.exceptions import ApiError, internal_error, invalid_request, not_found, unavailable
|
||||
from app.errors.exceptions import (
|
||||
ApiError,
|
||||
internal_error,
|
||||
invalid_request,
|
||||
not_found,
|
||||
unavailable,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
|
||||
TRANSIENT_SQLSTATES = frozenset({"55P03"})
|
||||
RECOVERABLE_STORAGE_ERRORS = (
|
||||
ApiError, SQLAlchemyError, MinioException, StorageHTTPError, OSError, TimeoutError,
|
||||
ApiError,
|
||||
SQLAlchemyError,
|
||||
MinioException,
|
||||
StorageHTTPError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,12 +51,14 @@ async def operation_timeout(seconds: float, error: ApiError):
|
||||
|
||||
|
||||
async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return await api_error_handler(request, invalid_request("Загрузка прервана клиентом"))
|
||||
return await api_error_handler(
|
||||
request, invalid_request("Загрузка прервана клиентом")
|
||||
)
|
||||
|
||||
|
||||
def log_recoverable_error(operation: str, error: Exception) -> None:
|
||||
"""Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
|
||||
logger.warning('%s deferred: %s', operation, type(error).__name__)
|
||||
logger.warning("%s deferred: %s", operation, type(error).__name__)
|
||||
|
||||
|
||||
def _get_request_id(request: Request) -> str:
|
||||
|
||||
+3
-1
@@ -52,7 +52,9 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
application.state.session_factory = create_session_factory(engine)
|
||||
application.state.redis = redis
|
||||
application.state.s3 = s3
|
||||
application.state.upload_capacity = create_request_capacity(settings.s3.max_concurrent_uploads)
|
||||
application.state.upload_capacity = create_request_capacity(
|
||||
settings.s3.max_concurrent_uploads
|
||||
)
|
||||
application.state.request_capacity = create_request_capacity(settings.max_in_flight)
|
||||
application.state.instance_id = resolve_instance_id(settings.instance_id)
|
||||
application.state.lifecycle = Lifecycle()
|
||||
|
||||
+14
-1
@@ -9,4 +9,17 @@ from app.models.idempotency_key import IdempotencyKey
|
||||
from app.models.message import Message
|
||||
from app.models.user import User
|
||||
|
||||
__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
|
||||
__all__ = [
|
||||
"JobReplay",
|
||||
"AttachmentJob",
|
||||
"Base",
|
||||
"Attachment",
|
||||
"MessageAttachment",
|
||||
"UploadIntent",
|
||||
"Chat",
|
||||
"ChatMember",
|
||||
"Counter",
|
||||
"IdempotencyKey",
|
||||
"Message",
|
||||
"User",
|
||||
]
|
||||
|
||||
@@ -18,8 +18,13 @@ class Attachment(Base):
|
||||
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
|
||||
),
|
||||
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
|
||||
CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"),
|
||||
CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"),
|
||||
CheckConstraint(
|
||||
"status IN ('queued', 'processing', 'ready', 'failed')", name="status"
|
||||
),
|
||||
CheckConstraint(
|
||||
"(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
|
||||
name="error_code",
|
||||
),
|
||||
Index("ix_attachments_chat_id", "chat_id"),
|
||||
)
|
||||
|
||||
@@ -36,7 +41,9 @@ class Attachment(Base):
|
||||
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
|
||||
crop_key: Mapped[str | None] = mapped_column(String(128))
|
||||
optimized_key: Mapped[str | None] = mapped_column(String(128))
|
||||
variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb"))
|
||||
variants: Mapped[list[dict]] = mapped_column(
|
||||
JSONB, default=list, server_default=text("'[]'::jsonb")
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
+5
-2
@@ -20,7 +20,8 @@ class AttachmentJob(Base):
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("attachments.id"), unique=True,
|
||||
ForeignKey("attachments.id"),
|
||||
unique=True,
|
||||
)
|
||||
staging_key: Mapped[str | None] = mapped_column(String(128))
|
||||
pipeline_version: Mapped[int] = mapped_column(server_default="1")
|
||||
@@ -29,7 +30,9 @@ class AttachmentJob(Base):
|
||||
attempt_limit: Mapped[int | None]
|
||||
last_error_permanent: Mapped[bool | None]
|
||||
published_attempt: Mapped[int] = mapped_column(server_default="0")
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(
|
||||
server_default=func.clock_timestamp()
|
||||
)
|
||||
lease_token: Mapped[UUID | None]
|
||||
lease_until: Mapped[datetime | None]
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
@@ -35,7 +35,9 @@ class Attachment(Base):
|
||||
|
||||
|
||||
VARIANT_FILENAMES = {
|
||||
"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp",
|
||||
"thumbnail": "thumbnail.jpg",
|
||||
"crop": "crop.jpg",
|
||||
"optimized": "optimized.webp",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -16,5 +16,7 @@ class JobMessage(BaseModel):
|
||||
pipeline_version: int = Field(default=1, ge=1)
|
||||
attempt: int = Field(ge=1)
|
||||
created_at: AwareDatetime
|
||||
traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
traceparent: str | None = Field(
|
||||
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
|
||||
)
|
||||
tracestate: str | None = Field(default=None, max_length=512)
|
||||
|
||||
+23
-7
@@ -28,26 +28,42 @@ class JobBroker:
|
||||
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
|
||||
|
||||
async def ensure_ready(self) -> None:
|
||||
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
|
||||
if (
|
||||
self.connection is None
|
||||
or self.connection.is_closed
|
||||
or self.channel is None
|
||||
or self.channel.is_closed
|
||||
):
|
||||
await self.aclose()
|
||||
self.connection = await aio_pika.connect(
|
||||
host=self.settings.broker_host, port=self.settings.broker_port,
|
||||
host=self.settings.broker_host,
|
||||
port=self.settings.broker_port,
|
||||
login=self.settings.broker_user,
|
||||
password=self.settings.broker_password_file.read_text().strip(),
|
||||
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
|
||||
virtualhost=self.settings.broker_vhost,
|
||||
timeout=5,
|
||||
heartbeat=30,
|
||||
)
|
||||
self.channel = await self.connection.channel(
|
||||
publisher_confirms=True, on_return_raises=True
|
||||
)
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
await self.channel.set_qos(prefetch_count=1)
|
||||
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
|
||||
self.queue = await self.channel.declare_queue(
|
||||
self.settings.broker_queue, durable=True
|
||||
)
|
||||
|
||||
async def publish_job(self, message: JobMessage) -> None:
|
||||
await self.ensure_ready()
|
||||
await self.channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=message.model_dump_json(exclude_none=True).encode(),
|
||||
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=f"{message.job_id}:{message.attempt}",
|
||||
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
|
||||
),
|
||||
routing_key=self.settings.broker_queue,
|
||||
mandatory=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
async def receive(self, consumer: str) -> Delivery:
|
||||
|
||||
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
async def check_database(engine: AsyncEngine) -> bool:
|
||||
async with engine.connect() as connection:
|
||||
await connection.execute(text('SELECT 1'))
|
||||
await connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -33,7 +33,8 @@ async def get_active_key(
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
) == tuple_(*scope),
|
||||
)
|
||||
== tuple_(*scope),
|
||||
IdempotencyKey.created_at
|
||||
> func.clock_timestamp() - timedelta(seconds=ttl_seconds),
|
||||
)
|
||||
@@ -45,14 +46,20 @@ async def save_result(
|
||||
) -> None:
|
||||
"""Сохраняет новый результат или заменяет истёкший."""
|
||||
stmt = insert(IdempotencyKey).values(
|
||||
user_id=scope[0], method=scope[1], path=scope[2], key=scope[3],
|
||||
request_hash=request_hash, message_id=message_id,
|
||||
user_id=scope[0],
|
||||
method=scope[1],
|
||||
path=scope[2],
|
||||
key=scope[3],
|
||||
request_hash=request_hash,
|
||||
message_id=message_id,
|
||||
)
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=[
|
||||
IdempotencyKey.user_id, IdempotencyKey.method,
|
||||
IdempotencyKey.path, IdempotencyKey.key,
|
||||
IdempotencyKey.user_id,
|
||||
IdempotencyKey.method,
|
||||
IdempotencyKey.path,
|
||||
IdempotencyKey.key,
|
||||
],
|
||||
set_={
|
||||
"request_hash": request_hash,
|
||||
|
||||
@@ -13,7 +13,8 @@ async def failed_jobs(session, limit: int):
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "failed")
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
|
||||
.limit(limit)
|
||||
)
|
||||
return result.all()
|
||||
|
||||
@@ -23,12 +24,24 @@ async def replay_record(session, request_id):
|
||||
|
||||
|
||||
async def insert_replay(session, **values) -> bool:
|
||||
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
|
||||
result = await session.scalar(
|
||||
insert(JobReplay)
|
||||
.values(**values)
|
||||
.on_conflict_do_nothing(index_elements=[JobReplay.id])
|
||||
.returning(JobReplay.id)
|
||||
)
|
||||
return result is not None
|
||||
|
||||
|
||||
async def replay_history(session, job_id):
|
||||
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
|
||||
return (
|
||||
await session.scalars(
|
||||
select(JobReplay)
|
||||
.where(JobReplay.job_id == job_id)
|
||||
.order_by(JobReplay.created_at)
|
||||
.limit(100)
|
||||
)
|
||||
).all()
|
||||
|
||||
|
||||
def reset_failed(job, attachment, now, attempt_limit):
|
||||
|
||||
+43
-10
@@ -32,7 +32,9 @@ class JobWork:
|
||||
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
|
||||
|
||||
|
||||
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
async def create_job(
|
||||
session: AsyncSession, attachment_id: UUID, staging_key: str
|
||||
) -> UUID:
|
||||
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
@@ -58,7 +60,10 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
|
||||
.where(
|
||||
Attachment.status == "processing",
|
||||
AttachmentJob.lease_until < func.clock_timestamp(),
|
||||
)
|
||||
.order_by(AttachmentJob.lease_until)
|
||||
.limit(limit)
|
||||
.with_for_update(of=AttachmentJob, skip_locked=True)
|
||||
@@ -66,19 +71,35 @@ async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
return result.all()
|
||||
|
||||
|
||||
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
|
||||
def start_job(
|
||||
job, attachment, token: UUID, now: datetime, lease_seconds: int
|
||||
) -> JobWork:
|
||||
job.attempt += 1
|
||||
job.lease_token = token
|
||||
job.lease_until = now + timedelta(seconds=lease_seconds)
|
||||
attachment.status = "processing"
|
||||
attachment.updated_at = now
|
||||
return JobWork(
|
||||
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256,
|
||||
job.id,
|
||||
attachment.id,
|
||||
token,
|
||||
job.pipeline_version,
|
||||
job.staging_key,
|
||||
attachment.content_type,
|
||||
attachment.size_bytes,
|
||||
attachment.sha256,
|
||||
)
|
||||
|
||||
|
||||
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
|
||||
def record_failure(
|
||||
job,
|
||||
attachment,
|
||||
now: datetime,
|
||||
error_code: str,
|
||||
*,
|
||||
terminal: bool,
|
||||
retry_seconds: float,
|
||||
) -> None:
|
||||
job.lease_token = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = error_code
|
||||
@@ -89,7 +110,9 @@ def record_failure(job, attachment, now: datetime, error_code: str, *, terminal:
|
||||
attachment.updated_at = now
|
||||
|
||||
|
||||
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
|
||||
def complete_job(
|
||||
job, attachment, work: JobWork, variants: list[dict], now: datetime
|
||||
) -> None:
|
||||
attachment.object_key = work.original_key
|
||||
attachment.thumbnail_key = work.variant_key("thumbnail")
|
||||
attachment.crop_key = work.variant_key("crop")
|
||||
@@ -105,15 +128,25 @@ def complete_job(job, attachment, work: JobWork, variants: list[dict], now: date
|
||||
job.finished_at = now
|
||||
|
||||
|
||||
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
|
||||
async def terminal_staging(
|
||||
session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
|
||||
):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(
|
||||
AttachmentJob.staging_key.is_not(None),
|
||||
or_(
|
||||
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
|
||||
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
|
||||
and_(
|
||||
Attachment.status == "ready",
|
||||
AttachmentJob.finished_at
|
||||
< func.clock_timestamp() - timedelta(seconds=retention),
|
||||
),
|
||||
and_(
|
||||
Attachment.status == "failed",
|
||||
AttachmentJob.finished_at
|
||||
< func.clock_timestamp() - timedelta(seconds=failed_retention),
|
||||
),
|
||||
),
|
||||
)
|
||||
.order_by(AttachmentJob.finished_at)
|
||||
|
||||
@@ -9,7 +9,9 @@ from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
|
||||
|
||||
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
|
||||
async def next_unpublished_job(
|
||||
session: AsyncSession, *, attachment_id: UUID | None = None
|
||||
) -> AttachmentJob | None:
|
||||
statement = (
|
||||
select(AttachmentJob)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
|
||||
Reference in New Issue
Block a user