171 lines
5.6 KiB
Python
171 lines
5.6 KiB
Python
"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize."""
|
||
|
||
import argparse
|
||
import asyncio
|
||
from dataclasses import dataclass
|
||
import json
|
||
import logging
|
||
import os
|
||
import signal
|
||
from uuid import UUID, uuid4
|
||
|
||
from PIL import features
|
||
from pydantic import ValidationError
|
||
|
||
from app.core.background import (
|
||
TRANSIENT_JOB_ERRORS,
|
||
log_background_error,
|
||
shutdown_event,
|
||
wait_or_stop,
|
||
)
|
||
from app.core.config import Settings
|
||
from app.core.database import create_db_engine, create_session_factory
|
||
from app.core.jobs import JobSettings
|
||
from app.domain.attachment_jobs import process_attachment_job
|
||
from app.errors.exceptions import ProcessingError
|
||
from app.schemas.job import JobMessage
|
||
from app.storage.broker import JobBroker
|
||
from app.storage.s3 import create_s3_storage
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Faults:
|
||
"""Только явные CLI-флаги; обычный worker не включает инъекции сбоев."""
|
||
|
||
crash_after_object: UUID | None = None
|
||
crash_before_ack: UUID | None = None
|
||
fail_transient: UUID | None = None
|
||
fail_permanent: UUID | None = None
|
||
object_name: str = "optimized"
|
||
|
||
def kill(self, attachment_id: UUID, phase: str) -> None:
|
||
print(
|
||
json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
|
||
flush=True,
|
||
)
|
||
os.kill(os.getpid(), signal.SIGKILL)
|
||
|
||
def after_object(self, attachment_id: UUID, name: str) -> None:
|
||
if attachment_id == self.crash_after_object and name == self.object_name:
|
||
self.kill(attachment_id, "after_object:" + name)
|
||
|
||
def before_ack(self, attachment_id: UUID) -> None:
|
||
if attachment_id == self.crash_before_ack:
|
||
self.kill(attachment_id, "before_ack")
|
||
|
||
def before_processing(self, attachment_id: UUID) -> None:
|
||
if attachment_id == self.fail_transient:
|
||
raise TimeoutError("injected transient failure")
|
||
if attachment_id == self.fail_permanent:
|
||
raise ProcessingError("injected_permanent")
|
||
|
||
|
||
async def consume_connected(
|
||
factory,
|
||
storage,
|
||
broker: JobBroker,
|
||
settings: JobSettings,
|
||
stop: asyncio.Event,
|
||
faults: Faults,
|
||
) -> None:
|
||
consumer = uuid4().hex
|
||
while not stop.is_set():
|
||
try:
|
||
await broker.ensure_ready()
|
||
delivery = await broker.receive(consumer)
|
||
if delivery is None:
|
||
continue
|
||
try:
|
||
message = JobMessage.model_validate_json(delivery.payload)
|
||
except ValidationError:
|
||
await broker.reject_invalid(delivery.id)
|
||
continue
|
||
try:
|
||
async with factory() as session:
|
||
acknowledge = await process_attachment_job(
|
||
session,
|
||
storage,
|
||
message,
|
||
settings,
|
||
after_object=faults.after_object,
|
||
before_processing=faults.before_processing,
|
||
)
|
||
except ProcessingError:
|
||
await broker.reject_invalid(delivery.id)
|
||
else:
|
||
if acknowledge:
|
||
faults.before_ack(message.attachment_id)
|
||
await broker.acknowledge_job(delivery.id)
|
||
else:
|
||
await broker.defer_job(delivery.id)
|
||
except TRANSIENT_JOB_ERRORS as error:
|
||
log_background_error("Worker delivery deferred", error)
|
||
await broker.aclose()
|
||
await wait_or_stop(stop, settings.publisher_poll_seconds)
|
||
|
||
|
||
async def consume(factory, storage, settings, stop, faults) -> None:
|
||
broker = JobBroker(settings)
|
||
try:
|
||
await consume_connected(factory, storage, broker, settings, stop, faults)
|
||
finally:
|
||
await broker.aclose()
|
||
|
||
|
||
async def run_worker(faults: Faults) -> None:
|
||
settings = Settings()
|
||
jobs = JobSettings()
|
||
logging.basicConfig(level=settings.log_level)
|
||
if not features.check("webp"):
|
||
logging.error("Pillow requires WebP support for the image worker")
|
||
raise RuntimeError("WebP codec is unavailable")
|
||
stop = shutdown_event()
|
||
engine = create_db_engine(settings)
|
||
storage = None
|
||
try:
|
||
storage = create_s3_storage(settings.s3)
|
||
factory = create_session_factory(engine)
|
||
async with asyncio.TaskGroup() as group:
|
||
tasks = [
|
||
group.create_task(consume(factory, storage, jobs, stop, faults))
|
||
for _ in range(jobs.worker_concurrency)
|
||
]
|
||
await stop.wait()
|
||
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
|
||
for task in tasks:
|
||
task.cancel()
|
||
finally:
|
||
if storage is not None:
|
||
await storage.aclose()
|
||
await engine.dispose()
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(
|
||
description="Image worker с явными fault-injection hooks"
|
||
)
|
||
modes = parser.add_mutually_exclusive_group()
|
||
for flag in (
|
||
"crash-after-object",
|
||
"crash-before-ack",
|
||
"fail-transient",
|
||
"fail-permanent",
|
||
):
|
||
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
|
||
parser.add_argument(
|
||
"--object-name",
|
||
choices=("original", "thumbnail", "crop", "optimized"),
|
||
default="optimized",
|
||
)
|
||
args = parser.parse_args()
|
||
faults = Faults(**vars(args))
|
||
try:
|
||
asyncio.run(run_worker(faults))
|
||
except Exception as error:
|
||
log_background_error("Worker stopped", error)
|
||
raise SystemExit(1) from None
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|