"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose). Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1): одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди. Обработка одного сообщения: 1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ. 2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op. 3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает -> аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued. Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения. 4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД. 5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь). 6. PUT в конечный prefix: attachments//v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp. Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи. 7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT. 8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД. 9. Удаление исходника из staging (если не вышло — доделает janitor). Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД, таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed. """ import hashlib import json import logging import os import shutil import signal import subprocess import sys import tempfile import threading import time from dataclasses import dataclass from pathlib import Path import psycopg from psycopg.rows import dict_row import broker import faults import imaging import pipeline import settings import storage logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") log = logging.getLogger("messenger.worker") STOP = threading.Event() HERE = Path(__file__).resolve().parent VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large", imaging.EXIT_MEMORY: "resource_limit"} class JobError(Exception): def __init__(self, code: str, permanent: bool): super().__init__(code) self.code = code self.permanent = permanent class LeaseLost(Exception): pass def connect_db(): return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") def final_prefix(claim: pipeline.Claim) -> str: return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/" class LeaseKeeper(threading.Thread): """Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку.""" def __init__(self, claim: pipeline.Claim): super().__init__(daemon=True) self.claim = claim self.done = threading.Event() self.lost = threading.Event() self.frozen = threading.Event() # fault worker_freeze: перестать продлевать def run(self): interval = max(1.0, settings.JOB_LEASE_SECONDS / 3) conn = None while not self.done.wait(interval): if self.frozen.is_set(): continue try: if conn is None: conn = connect_db() if not pipeline.renew_lease(conn, self.claim): log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id) self.lost.set() return except psycopg.Error as exc: # Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT). log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__) if conn is not None: conn.close() conn = None if conn is not None: conn.close() def check(self): if self.lost.is_set(): raise LeaseLost() def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: # Дочернему процессу — только лимиты, никаких секретов окружения. env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1", "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} for fault in ("hang", "invalid", "oom"): if faults.hit("imaging_" + fault): env["FAULT_IMAGING"] = fault process = subprocess.Popen( [sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type, str(settings.MAX_IMAGE_PIXELS)], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir), ) deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS try: while True: try: output, _ = process.communicate(timeout=0.5) break except subprocess.TimeoutExpired: if time.monotonic() > deadline: raise JobError("processing_timeout", permanent=False) from None keeper.check() finally: if process.poll() is None: process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку process.wait() code = process.returncode if code in IMAGING_EXIT: raise JobError(IMAGING_EXIT[code], permanent=True) if code != 0: # Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены. raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False) return json.loads(output.decode("utf-8").strip().splitlines()[-1]) def sha256_file(path: Path) -> tuple[int, str]: digest = hashlib.sha256() size = 0 with open(path, "rb") as file: while chunk := file.read(1024 * 1024): size += len(chunk) digest.update(chunk) return size, digest.hexdigest() def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None: if faults.hit("worker_s3_error"): raise JobError("storage_unavailable", permanent=False) with open(path, "rb") as file: try: storage.put(key, file, size, content_type, sha256) except storage.StorageUnavailable: raise JobError("storage_unavailable", permanent=False) from None def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]: """Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов.""" source = workdir / "source" try: storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES) except storage.ObjectMissing: raise JobError("source_missing", permanent=True) from None except storage.StorageUnavailable: raise JobError("storage_unavailable", permanent=False) from None size, digest = sha256_file(source) if size != claim.size_bytes or digest != claim.sha256: # Staging не совпадает с тем, что принял API: публиковать нельзя. raise JobError("source_corrupted", permanent=True) keeper.check() out_dir = workdir / "out" out_dir.mkdir() result = run_imaging(source, out_dir, claim, keeper) keeper.check() prefix = final_prefix(claim) original_key = prefix + "original" put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes) variants = [] for item in result["variants"]: path = out_dir / item["file"] v_size, v_sha = sha256_file(path) key = prefix + VARIANT_KEYS[item["name"]] put_file(key, path, item["content_type"], v_sha, v_size) variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size, "width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key}) keeper.check() return original_key, variants @dataclass class Outcome: ack: bool = True # False -> nack с requeue (не смогли даже записать claim) reject: bool = False # True -> reject без requeue -> DLQ cleanup_key: str | None = None attachment_id: object = None class Consumer(threading.Thread): def __init__(self, number: int): super().__init__(name=f"consumer-{number}", daemon=True) self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}" self.db = None def database(self): if self.db is None or self.db.closed: self.db = connect_db() return self.db def handle(self, body: bytes, redelivered: bool) -> Outcome: try: message = pipeline.parse_message(body) except pipeline.InvalidMessage as exc: log.warning("invalid message -> DLQ (%s)", exc) return Outcome(reject=True) try: result = pipeline.claim(self.database(), message, self.owner, redelivered) except psycopg.Error as exc: log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__) self.db = None STOP.wait(2) return Outcome(ack=False) if isinstance(result, pipeline.Reject): log.warning("job %s: %s -> DLQ", message["job_id"], result.reason) return Outcome(reject=True) if isinstance(result, pipeline.Skip): log.info("job %s: no-op (%s), ack", message["job_id"], result.reason) return Outcome() return self.execute(result) def execute(self, claim: pipeline.Claim) -> Outcome: started = time.monotonic() log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt, claim.max_attempts, claim.attachment_id) keeper = LeaseKeeper(claim) keeper.start() workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR)) try: try: original_key, variants = process_job(claim, keeper, workdir) faults.crash("worker_crash_after_put") if faults.hit("worker_freeze"): keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят time.sleep(settings.JOB_LEASE_SECONDS * 3) committed = pipeline.finish_ready(self.database(), claim, original_key, variants, self.owner) except JobError as exc: outcome = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner) log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code, "permanent" if exc.permanent else "temporary", outcome) return Outcome() except LeaseLost: log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt) return Outcome() except psycopg.Error: raise except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt) pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner) return Outcome() except psycopg.Error as exc: # Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и # dispatcher вернёт job в очередь; если прошёл — повтор увидит ready. log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__) self.db = None return Outcome() finally: keeper.done.set() shutil.rmtree(workdir, ignore_errors=True) if not committed: log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id, claim.attempt) return Outcome() faults.crash("worker_crash_after_commit") log.info("job %s attempt %s: ready in %.0f ms", claim.job_id, claim.attempt, (time.monotonic() - started) * 1000) return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id) def cleanup_staging(self, outcome: Outcome) -> None: """После COMMIT ready и ack: исходник в staging больше не нужен.""" try: storage.delete(outcome.cleanup_key) self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' " "AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key)) except (storage.StorageUnavailable, psycopg.Error) as exc: log.info("staging cleanup postponed to janitor: %s", type(exc).__name__) self.db = None def consume_once(self) -> None: connection = broker.connect() try: channel = connection.channel() channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH) broker.check_topology(channel) log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH) for method, _properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1): if STOP.is_set(): break if method is None: continue box = {} runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle( body, method.redelivered)), daemon=True) runner.start() while runner.is_alive(): connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа outcome = box.get("outcome", Outcome(ack=False)) if outcome.reject: channel.basic_reject(method.delivery_tag, requeue=False) elif outcome.ack: channel.basic_ack(method.delivery_tag) else: channel.basic_nack(method.delivery_tag, requeue=True) if outcome.cleanup_key: self.cleanup_staging(outcome) try: channel.cancel() except broker.BrokerError: pass finally: try: connection.close() except broker.BrokerError: pass def safe_handle(self, body, redelivered) -> Outcome: try: return self.handle(body, redelivered) except Exception: # noqa: BLE001 log.exception("unexpected error while handling a message; requeue") self.db = None STOP.wait(2) return Outcome(ack=False) def run(self): pause = 1.0 while not STOP.is_set(): try: self.consume_once() pause = 1.0 except broker.BrokerError as exc: log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name, type(exc).__name__, pause) STOP.wait(pause) pause = min(pause * 2, 15) def main() -> None: if not settings.DATABASE_URL or not settings.BROKER_URL: raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True) for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda *_: STOP.set()) consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)] for consumer in consumers: consumer.start() log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, " "fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH, settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB, "ON" if settings.FAULT_INJECTION else "off") while not STOP.wait(2): if not all(consumer.is_alive() for consumer in consumers): log.error("a consumer thread died, exiting so that Docker restarts the worker") os._exit(1) try: Path(settings.ALIVE_FILE).touch() except OSError: pass # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят. for consumer in consumers: consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10) log.info("stopped") if __name__ == "__main__": main()