"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен.""" import logging import os import socket import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import pika import psycopg from psycopg.rows import dict_row from psycopg_pool import ConnectionPool from blobs import bucket_ready, read_secret from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log LOGGER = logging.getLogger("worker") SCHEMA_VERSION = "004_jobs" BUSY = 0 BUSY_LOCK = threading.Lock() def database_url(): return read_secret(os.environ["DATABASE_URL_FILE"]) def concurrency(): value = int(os.environ.get("WORKER_CONCURRENCY", "2")) if not 1 <= value <= 8: raise RuntimeError("WORKER_CONCURRENCY") return value def wait_schema(conninfo): last = "error" for _ in range(90): try: with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn: row = conn.execute( "SELECT 1 FROM schema_migrations WHERE version = %s", (SCHEMA_VERSION,), ).fetchone() if row is not None: return last = "migration missing" except psycopg.Error as exc: last = type(exc).__name__ time.sleep(1) LOGGER.error("schema not ready: %s", last) raise SystemExit(1) def wait_bucket(): for _ in range(60): if bucket_ready(): return time.sleep(1) LOGGER.error("object storage not ready") raise SystemExit(1) def wait_broker(): last = "error" for _ in range(60): try: with broker_channel(): return except Exception as exc: last = type(exc).__name__ time.sleep(1) LOGGER.error("broker not ready: %s", last) raise SystemExit(1) def make_pool(conninfo, slots): return ConnectionPool( conninfo, min_size=1, max_size=slots * 2 + 2, timeout=3, kwargs={ "connect_timeout": 3, "row_factory": dict_row, "application_name": "mtusi-lab6-worker", "options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000", }, open=True, ) def oldest_job_age_seconds(pool): try: with pool.connection() as conn: row = conn.execute( """ SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age FROM attachment_jobs WHERE status IN ('queued', 'processing') """ ).fetchone() if row is None or row["age"] is None: return 0.0 return float(row["age"]) except Exception: return 0.0 def refresh_runtime_stats(pool, slots): global BUSY try: depth = queue_depth(PROCESS_QUEUE) except Exception: depth = 0 try: retry_depth = queue_depth(RETRY_QUEUE) except Exception: retry_depth = 0 age = oldest_job_age_seconds(pool) set_queue_stats(PROCESS_QUEUE, depth, age) set_queue_stats(RETRY_QUEUE, retry_depth, 0.0) with BUSY_LOCK: busy = BUSY set_worker_stats(busy, slots) observe_process() try: stats = pool.get_stats() from telemetry import set_pool_stats used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0)) waiting = int(stats.get("requests_waiting", 0)) set_pool_stats(max(0, used), waiting) except Exception: pass def consume(pool, owner, slots): executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job") while True: connection = None try: connection = pika.BlockingConnection(_parameters()) channel = connection.channel() ensure_topology(channel) channel.basic_qos(prefetch_count=slots) def on_message(ch, method, _props, body): executor.submit(run_one, pool, owner, ch, method.delivery_tag, body) channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False) channel.start_consuming() except Exception as exc: LOGGER.warning("consumer %s", type(exc).__name__) time.sleep(1) finally: if connection is not None: try: connection.close() except Exception: pass def run_one(pool, owner, channel, tag, body): global BUSY with BUSY_LOCK: BUSY += 1 try: process_delivery(pool, body, owner) action = "ack" except Exception as exc: LOGGER.warning("delivery %s", type(exc).__name__) action = "nack" finally: with BUSY_LOCK: BUSY = max(0, BUSY - 1) def finish(): try: if action == "ack": channel.basic_ack(tag) else: channel.basic_nack(tag, requeue=True) except Exception: pass try: channel.connection.add_callback_threadsafe(finish) except Exception: pass def wait_bao(): """Fail-closed: worker не стартует без scoped secret из OpenBao.""" if not os.environ.get("BAO_ADDR"): return from bao import BaoError, login_and_read path = os.environ.get("BAO_SECRET_PATH", "secret/data/messenger/worker") last = None for _ in range(60): try: login_and_read(path) return except BaoError as exc: last = exc time.sleep(2) LOGGER.error("openbao unavailable: %s", last) raise SystemExit(1) def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker")) wait_bao() conninfo = database_url() wait_schema(conninfo) wait_bucket() wait_broker() slots = concurrency() pool = make_pool(conninfo, slots) owner = socket.gethostname() stop = threading.Event() set_worker_stats(0, slots) def drain(): while not stop.wait(1): try: with pool.connection() as conn: publish_pending(conn) except Exception as exc: LOGGER.warning("drain %s", type(exc).__name__) def sweep(): ticks = 0 while not stop.wait(5): ticks += 1 try: with pool.connection() as conn: release_expired(conn) if ticks % 12 == 0: cleanup_staging(conn) refresh_runtime_stats(pool, slots) except Exception as exc: LOGGER.warning("sweep %s", type(exc).__name__) threading.Thread(target=drain, name="drain", daemon=True).start() threading.Thread(target=sweep, name="sweep", daemon=True).start() try: with pool.connection() as conn: publish_pending(conn) except Exception as exc: LOGGER.warning("startup drain %s", type(exc).__name__) structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds()) LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds()) consume(pool, owner, slots) if __name__ == "__main__": main()