138 lines
5.3 KiB
Python
138 lines
5.3 KiB
Python
"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize."""
|
||
|
||
import argparse
|
||
import asyncio
|
||
from dataclasses import dataclass
|
||
import json
|
||
import logging
|
||
import os
|
||
import signal
|
||
from uuid import UUID, uuid4
|
||
|
||
from PIL import features
|
||
from pydantic import ValidationError
|
||
|
||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||
from app.core.config import Settings
|
||
from app.core.database import create_db_engine, create_session_factory
|
||
from app.core.jobs import JobSettings
|
||
from app.domain.attachment_jobs import process_attachment_job
|
||
from app.errors.exceptions import ProcessingError
|
||
from app.schemas.job import JobMessage
|
||
from app.storage.broker import JobBroker
|
||
from app.storage.s3 import create_s3_storage
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Faults:
|
||
"""Только явные CLI-флаги; обычный worker не включает инъекции сбоев."""
|
||
|
||
crash_after_object: UUID | None = None
|
||
crash_before_ack: UUID | None = None
|
||
fail_transient: UUID | None = None
|
||
fail_permanent: UUID | None = None
|
||
object_name: str = "optimized"
|
||
|
||
def kill(self, attachment_id: UUID, phase: str) -> None:
|
||
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
|
||
os.kill(os.getpid(), signal.SIGKILL)
|
||
|
||
def after_object(self, attachment_id: UUID, name: str) -> None:
|
||
if attachment_id == self.crash_after_object and name == self.object_name:
|
||
self.kill(attachment_id, "after_object:" + name)
|
||
|
||
def before_ack(self, attachment_id: UUID) -> None:
|
||
if attachment_id == self.crash_before_ack:
|
||
self.kill(attachment_id, "before_ack")
|
||
|
||
def before_processing(self, attachment_id: UUID) -> None:
|
||
if attachment_id == self.fail_transient:
|
||
raise TimeoutError("injected transient failure")
|
||
if attachment_id == self.fail_permanent:
|
||
raise ProcessingError("injected_permanent")
|
||
|
||
|
||
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
|
||
consumer = uuid4().hex
|
||
while not stop.is_set():
|
||
try:
|
||
await broker.ensure_ready()
|
||
delivery = await broker.receive(consumer)
|
||
if delivery is None:
|
||
continue
|
||
try:
|
||
message = JobMessage.model_validate_json(delivery.payload)
|
||
except ValidationError:
|
||
await broker.reject_invalid(delivery.id)
|
||
continue
|
||
try:
|
||
async with factory() as session:
|
||
acknowledge = await process_attachment_job(
|
||
session, storage, message, settings,
|
||
after_object=faults.after_object, before_processing=faults.before_processing,
|
||
)
|
||
except ProcessingError:
|
||
await broker.reject_invalid(delivery.id)
|
||
else:
|
||
if acknowledge:
|
||
faults.before_ack(message.attachment_id)
|
||
await broker.acknowledge_job(delivery.id)
|
||
else:
|
||
await broker.defer_job(delivery.id)
|
||
except TRANSIENT_JOB_ERRORS as error:
|
||
log_background_error("Worker delivery deferred", error)
|
||
await broker.aclose()
|
||
await wait_or_stop(stop, settings.publisher_poll_seconds)
|
||
|
||
|
||
async def consume(factory, storage, settings, stop, faults) -> None:
|
||
broker = JobBroker(settings)
|
||
try:
|
||
await consume_connected(factory, storage, broker, settings, stop, faults)
|
||
finally:
|
||
await broker.aclose()
|
||
|
||
|
||
async def run_worker(faults: Faults) -> None:
|
||
settings = Settings()
|
||
jobs = JobSettings()
|
||
logging.basicConfig(level=settings.log_level)
|
||
if not features.check("webp"):
|
||
logging.error("Pillow requires WebP support for the image worker")
|
||
raise RuntimeError("WebP codec is unavailable")
|
||
stop = shutdown_event()
|
||
engine = create_db_engine(settings)
|
||
storage = None
|
||
try:
|
||
storage = create_s3_storage(settings.s3)
|
||
factory = create_session_factory(engine)
|
||
async with asyncio.TaskGroup() as group:
|
||
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
|
||
await stop.wait()
|
||
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
|
||
for task in tasks:
|
||
task.cancel()
|
||
finally:
|
||
if storage is not None:
|
||
await storage.aclose()
|
||
await engine.dispose()
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
|
||
modes = parser.add_mutually_exclusive_group()
|
||
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
|
||
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
|
||
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
|
||
args = parser.parse_args()
|
||
faults = Faults(**vars(args))
|
||
try:
|
||
asyncio.run(run_worker(faults))
|
||
except Exception as error:
|
||
log_background_error("Worker stopped", error)
|
||
raise SystemExit(1) from None
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|