"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose). Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1): одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди. Обработка одного сообщения: 1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ. 2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op. 3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает -> аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued. Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения. 4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД. 5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь). 6. PUT в конечный prefix: attachments//v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp. Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи. 7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT. 8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД. 9. Удаление исходника из staging (если не вышло — доделает janitor). Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД, таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed. """ import hashlib import json import logging import os import shutil import signal import subprocess import sys import tempfile import threading import time from dataclasses import dataclass from pathlib import Path import psycopg from opentelemetry import trace from opentelemetry.trace import NonRecordingSpan, SpanKind from psycopg.rows import dict_row import settings import telemetry telemetry.setup("messenger-worker") import broker # noqa: E402 import faults # noqa: E402 import imaging # noqa: E402 import pipeline # noqa: E402 import storage # noqa: E402 log = logging.getLogger("messenger.worker") # lab6: метрики фоновой обработки (contracts/observability.md). Метки: тип задания, исход, очередь. JOB_ATTRS = {"job.type": pipeline.JOB_TYPE} JOB_DURATION = telemetry.meter.create_histogram( "messenger.job.duration", unit="s", description="Attempt of an attachment job from claim to result (outcome: ready/retry/failed/stale/lease_lost)") TIME_TO_READY = telemetry.meter.create_histogram( "messenger.attachment.time_to_ready", unit="s", description="From durable acceptance (job COMMIT in API) to ready COMMIT in worker") MESSAGES = telemetry.meter.create_counter( "messenger.messages.consumed", unit="{message}", description="Broker deliveries by outcome: processed (claimed), noop (duplicate/not due), dead_lettered, requeued") BUSY = {"value": 0} BUSY_LOCK = threading.Lock() def _observe_busy(_options): from opentelemetry.metrics import Observation return [Observation(BUSY["value"], {"messaging.destination.name": pipeline.QUEUE})] def _observe_capacity(_options): from opentelemetry.metrics import Observation return [Observation(settings.WORKER_CONCURRENCY, {"messaging.destination.name": pipeline.QUEUE})] telemetry.meter.create_observable_up_down_counter( "messenger.worker.busy", [_observe_busy], unit="{consumer}", description="Consumers processing a job now") telemetry.meter.create_observable_up_down_counter( "messenger.worker.capacity", [_observe_capacity], unit="{consumer}", description="Consumers of this worker process (WORKER_CONCURRENCY)") def busy(delta: int) -> None: with BUSY_LOCK: BUSY["value"] += delta STOP = threading.Event() HERE = Path(__file__).resolve().parent VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large", imaging.EXIT_MEMORY: "resource_limit"} class JobError(Exception): def __init__(self, code: str, permanent: bool): super().__init__(code) self.code = code self.permanent = permanent class LeaseLost(Exception): pass def connect_db(): return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") def final_prefix(claim: pipeline.Claim) -> str: return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/" class LeaseKeeper(threading.Thread): """Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку.""" def __init__(self, claim: pipeline.Claim): super().__init__(daemon=True) self.claim = claim self.done = threading.Event() self.lost = threading.Event() self.frozen = threading.Event() # fault worker_freeze: перестать продлевать def run(self): interval = max(1.0, settings.JOB_LEASE_SECONDS / 3) conn = None while not self.done.wait(interval): if self.frozen.is_set(): continue try: if conn is None: conn = connect_db() if not pipeline.renew_lease(conn, self.claim): log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id, extra={"event": "job.lease_lost", "job.id": str(self.claim.job_id)}) self.lost.set() return except psycopg.Error as exc: # Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT). log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__) if conn is not None: conn.close() conn = None if conn is not None: conn.close() def check(self): if self.lost.is_set(): raise LeaseLost() def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: # Дочернему процессу — только лимиты, никаких секретов окружения. env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1", "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} for fault in ("hang", "invalid", "oom"): if faults.hit("imaging_" + fault): env["FAULT_IMAGING"] = fault with telemetry.internal_span("imaging", {"messenger.pipeline.version": claim.pipeline_version, "messenger.source.content_type": claim.content_type}) as span: result = _run_imaging(env, source, out_dir, claim, keeper) span.set_attribute("messenger.variants", len(result["variants"])) return result def _run_imaging(env: dict, source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: process = subprocess.Popen( [sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type, str(settings.MAX_IMAGE_PIXELS)], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir), ) deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS try: while True: try: output, _ = process.communicate(timeout=0.5) break except subprocess.TimeoutExpired: if time.monotonic() > deadline: raise JobError("processing_timeout", permanent=False) from None keeper.check() finally: if process.poll() is None: process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку process.wait() code = process.returncode if code in IMAGING_EXIT: raise JobError(IMAGING_EXIT[code], permanent=True) if code != 0: # Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены. raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False) return json.loads(output.decode("utf-8").strip().splitlines()[-1]) def sha256_file(path: Path) -> tuple[int, str]: digest = hashlib.sha256() size = 0 with open(path, "rb") as file: while chunk := file.read(1024 * 1024): size += len(chunk) digest.update(chunk) return size, digest.hexdigest() def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None: if faults.hit("worker_s3_error"): raise JobError("storage_unavailable", permanent=False) with open(path, "rb") as file: try: storage.put(key, file, size, content_type, sha256) except storage.StorageUnavailable: raise JobError("storage_unavailable", permanent=False) from None def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]: """Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов.""" source = workdir / "source" try: storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES) except storage.ObjectMissing: raise JobError("source_missing", permanent=True) from None except storage.StorageUnavailable: raise JobError("storage_unavailable", permanent=False) from None with telemetry.internal_span("verify source sha256"): size, digest = sha256_file(source) if size != claim.size_bytes or digest != claim.sha256: # Staging не совпадает с тем, что принял API: публиковать нельзя. raise JobError("source_corrupted", permanent=True) keeper.check() out_dir = workdir / "out" out_dir.mkdir() result = run_imaging(source, out_dir, claim, keeper) keeper.check() prefix = final_prefix(claim) original_key = prefix + "original" put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes) variants = [] for item in result["variants"]: path = out_dir / item["file"] v_size, v_sha = sha256_file(path) key = prefix + VARIANT_KEYS[item["name"]] put_file(key, path, item["content_type"], v_sha, v_size) variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size, "width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key}) keeper.check() return original_key, variants @dataclass class Outcome: ack: bool = True # False -> nack с requeue (не смогли даже записать claim) reject: bool = False # True -> reject без requeue -> DLQ cleanup_key: str | None = None attachment_id: object = None span_context: object = None # спан обработки: очистка staging после ack — в той же трассе delivery: str = "processed" # метка messenger.messages.consumed: processed / noop def carrier_from(headers, body: bytes) -> dict: """W3C контекст сообщения: native headers AMQP (основной carrier), иначе поле traceparent тела.""" carrier = {} for key in ("traceparent", "tracestate"): value = (headers or {}).get(key) if isinstance(value, bytes): value = value.decode("ascii", "replace") if isinstance(value, str): carrier[key] = value if "traceparent" not in carrier: try: message = json.loads(body.decode("utf-8")) if isinstance(message, dict) and isinstance(message.get("traceparent"), str): carrier["traceparent"] = message["traceparent"] except (UnicodeDecodeError, ValueError): pass return carrier class Consumer(threading.Thread): def __init__(self, number: int): super().__init__(name=f"consumer-{number}", daemon=True) self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}" self.db = None def database(self): if self.db is None or self.db.closed: self.db = connect_db() return self.db def handle(self, body: bytes, redelivered: bool, headers=None) -> Outcome: """Спан CONSUMER «process attachments.process.v1» — ребёнок спана publish из dispatcher.""" attributes = {"messaging.system": "rabbitmq", "messaging.operation.type": "process", "messaging.destination.name": pipeline.QUEUE, "messaging.consumer.group.name": pipeline.QUEUE, "messaging.rabbitmq.message.redelivered": redelivered} with telemetry.tracer.start_as_current_span( f"process {pipeline.QUEUE}", context=telemetry.extract(carrier_from(headers, body)), kind=SpanKind.CONSUMER, attributes=attributes, record_exception=False, set_status_on_exception=False) as span: outcome = self._handle(body, redelivered, span) outcome.span_context = span.get_span_context() kind = "dead_lettered" if outcome.reject else "requeued" if not outcome.ack else outcome.delivery span.set_attribute("messenger.delivery", kind) MESSAGES.add(1, {"messaging.destination.name": pipeline.QUEUE, "outcome": kind}) return outcome def _handle(self, body: bytes, redelivered: bool, span) -> Outcome: try: message = pipeline.parse_message(body) except pipeline.InvalidMessage as exc: log.warning("invalid message -> DLQ (%s)", exc, extra={"event": "job.invalid_message"}) telemetry.mark_error(span, "invalid_message") return Outcome(reject=True) span.set_attributes({"messaging.message.id": f"{message['job_id']}:{message['attempt']}", "messenger.job.id": message["job_id"], "messenger.attachment.id": message["attachment_id"], "messenger.message.attempt": message["attempt"]}) try: result = pipeline.claim(self.database(), message, self.owner, redelivered) except psycopg.Error as exc: log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__, extra={"event": "job.claim_failed", "job.id": message["job_id"], "error.type": type(exc).__name__}) telemetry.mark_error(span, type(exc).__name__) self.db = None STOP.wait(2) return Outcome(ack=False) if isinstance(result, pipeline.Reject): log.warning("job %s: %s -> DLQ", message["job_id"], result.reason, extra={"event": "job.rejected", "job.id": message["job_id"], "reason": result.reason}) telemetry.mark_error(span, result.reason) return Outcome(reject=True) if isinstance(result, pipeline.Skip): log.info("job %s: no-op (%s), ack", message["job_id"], result.reason, extra={"event": "job.noop", "job.id": message["job_id"], "reason": result.reason}) span.set_attribute("messenger.noop.reason", result.reason) return Outcome(delivery="noop") span.set_attribute("messenger.job.attempt", result.attempt) busy(+1) try: return self.execute(result, span) finally: busy(-1) def execute(self, claim: pipeline.Claim, span) -> Outcome: started = time.monotonic() fields = {"job.id": str(claim.job_id), "attachment.id": str(claim.attachment_id), "attempt": claim.attempt} log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt, claim.max_attempts, claim.attachment_id, extra={"event": "job.claimed", **fields}) keeper = LeaseKeeper(claim) keeper.start() workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR)) outcome_name = "ready" try: try: original_key, variants = process_job(claim, keeper, workdir) faults.crash("worker_crash_after_put") if faults.hit("worker_freeze"): keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят time.sleep(settings.JOB_LEASE_SECONDS * 3) with telemetry.internal_span("commit result"): time_to_ready = pipeline.finish_ready(self.database(), claim, original_key, variants, self.owner) if time_to_ready is None: outcome_name = "stale" # аренду перехватили: fencing отклонил результат except JobError as exc: outcome_name = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner) telemetry.mark_error(span, exc.code) log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code, "permanent" if exc.permanent else "temporary", outcome_name, extra={"event": f"job.{outcome_name}", "error.type": exc.code, "permanent": exc.permanent, **fields}) return Outcome() except LeaseLost: outcome_name = "lease_lost" telemetry.mark_error(span, "lease_lost") log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt, extra={"event": "job.lease_lost", **fields}) return Outcome() except psycopg.Error: raise except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt, extra={"event": "job.unexpected_error", **fields}) telemetry.mark_error(span, "processing_failed") outcome_name = pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner) return Outcome() except psycopg.Error as exc: # Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и # dispatcher вернёт job в очередь; если прошёл — повтор увидит ready. outcome_name = "db_error" telemetry.mark_error(span, type(exc).__name__) log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__, extra={"event": "job.db_error", "error.type": type(exc).__name__, **fields}) self.db = None return Outcome() finally: keeper.done.set() shutil.rmtree(workdir, ignore_errors=True) elapsed = time.monotonic() - started span.set_attribute("messenger.job.outcome", outcome_name) JOB_DURATION.record(elapsed, JOB_ATTRS | {"outcome": outcome_name}) if time_to_ready is None: telemetry.mark_error(span, "stale_result_rejected") log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id, claim.attempt, extra={"event": "job.stale_result_rejected", **fields}) return Outcome() TIME_TO_READY.record(time_to_ready, JOB_ATTRS) faults.crash("worker_crash_after_commit") log.info("job %s attempt %s: ready in %.0f ms (%.1f s since upload)", claim.job_id, claim.attempt, (time.monotonic() - started) * 1000, time_to_ready, extra={"event": "job.ready", "duration_ms": round((time.monotonic() - started) * 1000, 1), "time_to_ready_s": round(time_to_ready, 3), **fields}) return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id) def cleanup_staging(self, outcome: Outcome) -> None: """После COMMIT ready и ack: исходник в staging больше не нужен.""" parent = trace.set_span_in_context(NonRecordingSpan(outcome.span_context)) if outcome.span_context else None with telemetry.tracer.start_as_current_span("cleanup staging", context=parent): try: storage.delete(outcome.cleanup_key) self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' " "AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key)) except (storage.StorageUnavailable, psycopg.Error) as exc: log.info("staging cleanup postponed to janitor: %s", type(exc).__name__, extra={"event": "staging.cleanup_postponed", "error.type": type(exc).__name__}) self.db = None def consume_once(self) -> None: connection = broker.connect() try: channel = connection.channel() channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH) broker.check_topology(channel) log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH, extra={"event": "worker.consuming"}) for method, properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1): if STOP.is_set(): break if method is None: continue box = {} headers = properties.headers if properties is not None else None runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle( body, method.redelivered, headers)), daemon=True) runner.start() while runner.is_alive(): connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа outcome = box.get("outcome", Outcome(ack=False)) if outcome.reject: channel.basic_reject(method.delivery_tag, requeue=False) elif outcome.ack: channel.basic_ack(method.delivery_tag) else: channel.basic_nack(method.delivery_tag, requeue=True) if outcome.cleanup_key: self.cleanup_staging(outcome) try: channel.cancel() except broker.BrokerError: pass finally: try: connection.close() except broker.BrokerError: pass def safe_handle(self, body, redelivered, headers=None) -> Outcome: try: return self.handle(body, redelivered, headers) except Exception: # noqa: BLE001 log.exception("unexpected error while handling a message; requeue", extra={"event": "job.handler_crashed"}) self.db = None STOP.wait(2) return Outcome(ack=False) def run(self): pause = 1.0 while not STOP.is_set(): try: self.consume_once() pause = 1.0 except broker.BrokerError as exc: log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name, type(exc).__name__, pause, extra={"event": "broker.unavailable", "error.type": type(exc).__name__}) STOP.wait(pause) pause = min(pause * 2, 15) def main() -> None: if not settings.DATABASE_URL or not settings.BROKER_URL: raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True) for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda *_: STOP.set()) consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)] for consumer in consumers: consumer.start() log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, " "fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH, settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB, "ON" if settings.FAULT_INJECTION else "off", extra={"event": "service.started"}) while not STOP.wait(2): if not all(consumer.is_alive() for consumer in consumers): log.error("a consumer thread died, exiting so that Docker restarts the worker") os._exit(1) try: Path(settings.ALIVE_FILE).touch() except OSError: pass # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят. for consumer in consumers: consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10) log.info("stopped", extra={"event": "service.stopped"}) telemetry.shutdown() if __name__ == "__main__": main()