formatted project by python-black

This commit is contained in:
2026-11-10 09:52:42 +03:00
parent 7cd02da51d
commit 9899ee4676
66 changed files with 2517 additions and 944 deletions
+8 -1
View File
@@ -8,7 +8,14 @@ from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
from sqlalchemy.exc import InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
TRANSIENT_JOB_ERRORS = (
AMQPException,
ChannelInvalidStateError,
InterfaceError,
OperationalError,
PoolTimeoutError,
OSError,
)
logger = logging.getLogger(__name__)
+3 -1
View File
@@ -21,7 +21,9 @@ async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool
async with factory() as session:
result = await reconcile_attachments(session, storage, dry_run=not apply)
staging = await cleanup_staging(session, storage, jobs, apply=apply)
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
print(
json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
)
except RECOVERABLE_STORAGE_ERRORS as error:
log_recoverable_error("Reconciliation", error)
completed = False
+3 -1
View File
@@ -5,7 +5,9 @@ from pydantic_settings import BaseSettings, SettingsConfigDict
class ImageSettings(BaseSettings):
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
model_config = SettingsConfigDict(
env_prefix="IMAGE_", env_file=".env", extra="ignore"
)
jpeg_quality: int = Field(default=85, ge=1, le=95)
webp_quality: int = Field(default=80, ge=1, le=100)
+22 -5
View File
@@ -9,7 +9,13 @@ from app.core.background import log_background_error
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
from app.domain.job_admin import (
REPLAY_REASONS,
history,
list_failed,
replay_failed_job,
show_job,
)
from app.errors.exceptions import JobOperationError
from app.storage.s3 import create_s3_storage
@@ -29,8 +35,15 @@ async def execute(args):
result = await history(session, args.job_id)
else:
storage = create_s3_storage(settings.s3)
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
request_id=args.request_id, actor=args.actor, reason=args.reason)
result = await replay_failed_job(
session,
storage,
JobSettings(),
args.job_id,
request_id=args.request_id,
actor=args.actor,
reason=args.reason,
)
print(json.dumps(result, default=str), flush=True)
finally:
if storage is not None:
@@ -39,10 +52,14 @@ async def execute(args):
def main():
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
parser = argparse.ArgumentParser(
description="Реестр failed и явный replay с аудитом"
)
commands = parser.add_subparsers(dest="command", required=True)
listing = commands.add_parser("list-failed")
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
listing.add_argument(
"--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
)
for name in ("show", "history", "replay"):
command = commands.add_parser(name)
command.add_argument("job_id", type=UUID)
+42 -9
View File
@@ -12,7 +12,12 @@ from uuid import UUID, uuid4
from PIL import features
from pydantic import ValidationError
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
@@ -34,7 +39,10 @@ class Faults:
object_name: str = "optimized"
def kill(self, attachment_id: UUID, phase: str) -> None:
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
print(
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
def after_object(self, attachment_id: UUID, name: str) -> None:
@@ -52,7 +60,14 @@ class Faults:
raise ProcessingError("injected_permanent")
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
async def consume_connected(
factory,
storage,
broker: JobBroker,
settings: JobSettings,
stop: asyncio.Event,
faults: Faults,
) -> None:
consumer = uuid4().hex
while not stop.is_set():
try:
@@ -68,8 +83,12 @@ async def consume_connected(factory, storage, broker: JobBroker, settings: JobSe
try:
async with factory() as session:
acknowledge = await process_attachment_job(
session, storage, message, settings,
after_object=faults.after_object, before_processing=faults.before_processing,
session,
storage,
message,
settings,
after_object=faults.after_object,
before_processing=faults.before_processing,
)
except ProcessingError:
await broker.reject_invalid(delivery.id)
@@ -107,7 +126,10 @@ async def run_worker(faults: Faults) -> None:
storage = create_s3_storage(settings.s3)
factory = create_session_factory(engine)
async with asyncio.TaskGroup() as group:
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
tasks = [
group.create_task(consume(factory, storage, jobs, stop, faults))
for _ in range(jobs.worker_concurrency)
]
await stop.wait()
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
for task in tasks:
@@ -119,11 +141,22 @@ async def run_worker(faults: Faults) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
parser = argparse.ArgumentParser(
description="Image worker с явными fault-injection hooks"
)
modes = parser.add_mutually_exclusive_group()
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
for flag in (
"crash-after-object",
"crash-before-ack",
"fail-transient",
"fail-permanent",
):
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
parser.add_argument(
"--object-name",
choices=("original", "thumbnail", "crop", "optimized"),
default="optimized",
)
args = parser.parse_args()
faults = Faults(**vars(args))
try:
+3 -1
View File
@@ -29,7 +29,9 @@ class JobSettings(BaseSettings):
@model_validator(mode="after")
def validate_lease(self) -> Self:
if self.job_lease_seconds < self.job_timeout_seconds + 30:
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
raise ValueError(
"Lease должен превышать timeout задания минимум на 30 секунд"
)
if self.job_retry_max_seconds < self.job_retry_seconds:
raise ValueError("Retry maximum must be at least the base delay")
return self
+24 -6
View File
@@ -8,7 +8,12 @@ import os
import signal
from uuid import UUID
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
from app.core.background import (
TRANSIENT_JOB_ERRORS,
log_background_error,
shutdown_event,
wait_or_stop,
)
from app.core.config import Settings
from app.core.database import create_db_engine, create_session_factory
from app.core.jobs import JobSettings
@@ -18,7 +23,10 @@ from app.storage.broker import JobBroker
def crash_before_publish(attachment_id: UUID) -> None:
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
print(
json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
flush=True,
)
os.kill(os.getpid(), signal.SIGKILL)
@@ -39,8 +47,14 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
async with factory() as session:
await recover_expired_jobs(session, jobs)
published = await publish_next_job(
session, broker, attachment_id=crash_attachment_id,
before_publish=crash_before_publish if crash_attachment_id is not None else None,
session,
broker,
attachment_id=crash_attachment_id,
before_publish=(
crash_before_publish
if crash_attachment_id is not None
else None
),
)
except TRANSIENT_JOB_ERRORS as error:
log_background_error("Publisher deferred", error)
@@ -53,9 +67,13 @@ async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
def main() -> None:
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
parser = argparse.ArgumentParser(
description="Публикация устойчивых jobs в RabbitMQ"
)
parser.add_argument(
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
"--crash-before-publish",
type=UUID,
metavar="ATTACHMENT_ID",
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
)
args = parser.parse_args()
+1 -2
View File
@@ -4,6 +4,7 @@ from redis.backoff import NoBackoff
from app.core.config import Settings
def create_redis_client(settings: Settings) -> Redis:
password = (
settings.redis_password.get_secret_value() if settings.redis_password else None
@@ -21,5 +22,3 @@ def create_redis_client(settings: Settings) -> Redis:
decode_responses=True,
)
return Redis.from_pool(pool)
+16 -4
View File
@@ -6,6 +6,7 @@ from urllib.parse import urlsplit
from pydantic import Field, model_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
class S3Settings(BaseSettings):
endpoint: str = "http://minio:9000"
region: str = "us-east-1"
@@ -24,12 +25,23 @@ class S3Settings(BaseSettings):
cleanup_grace_seconds: int = Field(default=86_400, ge=300)
cleanup_interval_seconds: int = Field(default=300, ge=1)
model_config = SettingsConfigDict(env_prefix="S3_", env_file=".env", extra="ignore", frozen=True)
model_config = SettingsConfigDict(
env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
)
@model_validator(mode="after")
def validate_endpoint(self):
url = urlsplit(self.endpoint)
if (url.scheme not in {"http", "https"} or not url.hostname or url.username
or url.password or url.path not in {"", "/"} or url.query or url.fragment):
raise ValueError("S3_ENDPOINT должен быть http(s) адресом без credentials/пути")
if (
url.scheme not in {"http", "https"}
or not url.hostname
or url.username
or url.password
or url.path not in {"", "/"}
or url.query
or url.fragment
):
raise ValueError(
"S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
)
return self
+6 -2
View File
@@ -13,11 +13,15 @@ def main() -> int:
settings = ImageSettings()
memory = settings.memory_mib * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
resource.setrlimit(
resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
)
source, directory, metadata, content_type, maximum = sys.argv[1:]
code = 0
try:
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
result = create_variants(
Path(source), Path(directory), content_type, int(maximum), settings
)
Path(metadata).write_text(result.model_dump_json())
except IMAGE_WORKER_ERRORS as error:
code = pipeline_worker_error_code(error)