"""Worker вложений: python -m app.worker Конвейер задания (jobs.schema.json, состояния — contracts/attachment-states.md): claim (queued -> processing, lease) -> чтение staging -> варианты (Pillow, в потоках) -> запись в конечный S3-prefix -> ОДНА транзакция: варианты + ready -> ack -> очистка staging. Ack отправляется только после устойчивого эффекта (COMMIT результата). Повторная доставка уже готовой задачи — no-op с ack. Временные ошибки: возврат в queued с backoff (экспоненциальный + jitter) через outbox; после MAX_ATTEMPTS — failed/retry_exhausted. Постоянные ошибки (невалидное изображение): failed с error_code сразу. Зависшие processing возвращает sweeper по истечении lease. """ import asyncio import json import os import random import signal import tempfile import time import uuid from concurrent.futures import ThreadPoolExecutor import aio_pika from . import config, pipeline, telemetry from .broker import QUEUE, Broker from .db import Database from .outbox import Relay from .s3 import S3 from .web import log WORKER_ID = "%s-%s" % ((os.environ.get("HOSTNAME") or "worker")[:12], uuid.uuid4().hex[:6]) def backoff_seconds(attempt): """Экспоненциальная задержка с полным jitter: случайное значение в [0, min(cap, base * 2^(attempt-1))].""" ceiling = min(config.RETRY_MAX_DELAY, config.RETRY_BASE_DELAY * (2 ** max(0, attempt - 1))) return random.uniform(ceiling * 0.5, ceiling) class Worker: def __init__(self): self.db, self.s3, self.broker = Database(), S3(), Broker() self.relay = Relay(self.db, self.broker) self.pool = ThreadPoolExecutor(max_workers=config.WORKER_CONCURRENCY, thread_name_prefix="img") self.busy = 0 self.stopping = False self.tasks = [] telemetry.gauge("messenger.worker.busy", lambda: [(self.busy, {})], "{job}", "Задания в обработке") telemetry.gauge("messenger.worker.capacity", lambda: [(config.WORKER_CONCURRENCY, {})], "{job}", "Ёмкость worker") # --- claim / finalize --------------------------------------------------------- async def claim(self, job): return await self.db.fetchrow( "UPDATE attachments SET status = 'processing', lease_owner = $2, processing_started_at = now(), " "lease_expires_at = now() + make_interval(secs => $3), attempt = $4, job_id = $5, updated_at = now() " "WHERE id = $1 AND status = 'queued' AND pipeline_version <= $6 " "RETURNING id, chat_id, staging_key, s3_key, content_type, size_bytes, attempt", uuid.UUID(job["attachment_id"]), WORKER_ID, float(config.LEASE_SECONDS), job["attempt"], uuid.UUID(job["job_id"]), job["pipeline_version"], retry=False) async def fail(self, att_id, job, code, event="failed"): await self.db.execute( "WITH u AS (UPDATE attachments SET status = 'failed', error_code = $2, lease_owner = NULL, lease_expires_at = NULL, " "updated_at = now() WHERE id = $1 AND status = 'processing' AND lease_owner = $3 RETURNING id) " "INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $4, $5, $6, $2 FROM u", att_id, code, WORKER_ID, uuid.UUID(job["job_id"]), event, job["attempt"]) async def reschedule(self, att_id, job, reason): """Временная ошибка: processing -> queued, новая запись outbox с отложенной публикацией (backoff + jitter).""" nxt = job["attempt"] + 1 if nxt > config.MAX_ATTEMPTS: log("warning", "job_retry_exhausted", attachment_id=str(att_id), attempts=job["attempt"]) await self.fail(att_id, job, "retry_exhausted", "retry_exhausted") return delay = backoff_seconds(job["attempt"]) await self.db.execute( "WITH u AS (UPDATE attachments SET status = 'queued', lease_owner = NULL, lease_expires_at = NULL, updated_at = now() " "WHERE id = $1 AND status = 'processing' AND lease_owner = $2 RETURNING id, pipeline_version), " "o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, available_at, traceparent, tracestate) " "SELECT $3, id, pipeline_version, $4, now() + make_interval(secs => $5), $7, $8 FROM u RETURNING 1) " "INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, $3, 'retry_scheduled', $4, $6 FROM u", att_id, WORKER_ID, uuid.UUID(job["job_id"]), nxt, delay, reason, job.get("traceparent"), job.get("tracestate")) log("info", "job_rescheduled", attachment_id=str(att_id), next_attempt=nxt, delay_seconds=round(delay, 2), reason=reason) async def publish_result(self, row, job, variants): """Одна транзакция: варианты (идемпотентно) + ready. Только владелец lease может опубликовать результат.""" async with self.db.conn() as c: async with c.transaction(): owner = await c.fetchval("SELECT 1 FROM attachments WHERE id = $1 AND status = 'processing' AND lease_owner = $2 " "FOR UPDATE", row["id"], WORKER_ID) if owner is None: return False # lease потерян (sweeper вернул задачу): результат не публикуем, чужой повтор доделает for v in variants: await c.execute( "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, " "height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) " "ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING", row["id"], job["pipeline_version"], v["name"], v["content_type"], len(v["data"]), v["width"], v["height"], v["sha256"], "att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"])) await c.execute("UPDATE attachments SET status = 'ready', error_code = NULL, ready_at = now(), updated_at = now(), " "lease_owner = NULL, lease_expires_at = NULL WHERE id = $1", row["id"]) await c.execute("INSERT INTO job_events (attachment_id, job_id, event, attempt) VALUES ($1, $2, 'ready', $3)", row["id"], uuid.UUID(job["job_id"]), job["attempt"]) return True # --- обработка ---------------------------------------------------------------- async def process(self, job): att_id = uuid.UUID(job["attachment_id"]) row = await self.claim(job) if row is None: state = await self.db.fetchval("SELECT status FROM attachments WHERE id = $1", att_id) log("info", "job_noop", attachment_id=job["attachment_id"], status=state or "unknown") telemetry.record_job("attachment.process.v1", "noop", 0.0) return # ready/failed/processing у другого владельца/неизвестное вложение: идемпотентный no-op, ack started = time.perf_counter() outcome = "ready" try: await self.fault("after_claim") stream, _ = await self.s3.open_stream(row["staging_key"]) spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024) try: try: while True: chunk = await stream.read(256 * 1024) if not chunk: break spool.write(chunk) finally: stream.close() loop = asyncio.get_running_loop() variants = await asyncio.wait_for(loop.run_in_executor(self.pool, pipeline.process, spool), config.JOB_TIMEOUT) finally: spool.close() # оригинал и варианты в конечный prefix: ключи детерминированы, повторная запись перезаписывает те же байты await self.copy_original(row) for v in variants: await self.s3.put_bytes("att/%s/v%d/%s" % (row["id"], job["pipeline_version"], v["name"]), v["data"], v["content_type"]) await self.fault("after_s3_before_commit") if not await self.publish_result(row, job, variants): log("warning", "lease_lost", attachment_id=job["attachment_id"]) return await self.fault("after_commit_before_ack") log("info", "job_ready", attachment_id=job["attachment_id"], attempt=job["attempt"], duration_ms=round((time.perf_counter() - started) * 1000, 1)) try: await self.s3.delete(row["staging_key"]) # staging безопасно удалять после COMMIT результата except Exception as exc: log("warning", "staging_cleanup_failed", error_type=type(exc).__name__) except pipeline.PermanentError as exc: outcome = "failed" await self.fail(att_id, job, exc.code) log("warning", "job_failed", attachment_id=job["attachment_id"], error_code=exc.code) except MemoryError: outcome = "failed" await self.fail(att_id, job, "processing_failed") log("error", "job_failed_oom", attachment_id=job["attachment_id"]) except asyncio.CancelledError: raise except InjectedCrash: raise except Exception as exc: # S3/БД/таймаут/неожиданное: временная ошибка outcome = "retried" await self.reschedule(att_id, job, type(exc).__name__) finally: ttr = None if outcome == "ready": ttr = await self.db.fetchval("SELECT extract(epoch FROM ready_at - accepted_at) FROM attachments WHERE id = $1 " "AND status = 'ready'", att_id) telemetry.record_job("attachment.process.v1", outcome, time.perf_counter() - started, float(ttr) if ttr is not None else None) async def copy_original(self, row): stream, size = await self.s3.open_stream(row["staging_key"]) spool = tempfile.SpooledTemporaryFile(max_size=2 * 1024 * 1024) try: while True: chunk = await stream.read(256 * 1024) if not chunk: break spool.write(chunk) stream.close() spool.seek(0) await self.s3.put(row["s3_key"], spool, row["content_type"], size) finally: spool.close() async def fault(self, point): """Точки внедрения сбоев для экспериментов (FAULT_INJECTION=crash:<точка> => процесс умирает мгновенно).""" if "crash_once:" + point in config.FAULT_INJECTION: marker = "/tmp/fault-%s.done" % point # сохраняется при рестарте того же контейнера -> сбой ровно один раз if not os.path.exists(marker): open(marker, "w").close() log("error", "fault_crash_once", point=point) os._exit(137) if "crash:" + point in config.FAULT_INJECTION: log("error", "fault_crash", point=point) os._exit(137) if "fail:" + point in config.FAULT_INJECTION: raise OSError("injected failure at " + point) async def on_message(self, message): try: job = json.loads(message.body) if (job.get("schema_version") != 1 or job.get("type") != "attachment.process.v1" or not isinstance(job.get("attempt"), int) or not isinstance(job.get("pipeline_version"), int)): raise ValueError("bad job") uuid.UUID(job["attachment_id"]), uuid.UUID(job["job_id"]) except Exception: log("error", "job_unreadable") await message.reject(requeue=False) # -> DLQ return self.busy += 1 carrier = {k: job[k] for k in ("traceparent", "tracestate") if k in job} try: with telemetry.span("attachment.process process", "consumer", {"messaging.system": "rabbitmq", "messaging.destination.name": "attachment.process", "messaging.operation": "process", "job.type": job["type"], "job.attempt": job["attempt"]}, carrier): await self.process(job) await message.ack() except InjectedCrash: raise except Exception as exc: log("error", "job_unhandled", error_type=type(exc).__name__) await message.nack(requeue=True) finally: self.busy -= 1 # --- sweeper ------------------------------------------------------------------- async def sweep_once(self): """Истёкший lease -> queued (новая попытка) или failed/retry_exhausted; чистка старого staging.""" rows = await self.db.fetch( "WITH exp AS (SELECT id FROM attachments WHERE status = 'processing' AND lease_expires_at < now() FOR UPDATE SKIP LOCKED), " "u AS (UPDATE attachments a SET status = CASE WHEN a.attempt >= $1 THEN 'failed' ELSE 'queued' END, " "error_code = CASE WHEN a.attempt >= $1 THEN 'retry_exhausted' ELSE NULL END, lease_owner = NULL, lease_expires_at = NULL, " "updated_at = now() FROM exp WHERE a.id = exp.id RETURNING a.id, a.status, a.pipeline_version, a.attempt, a.job_id), " "o AS (INSERT INTO outbox (job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) " "SELECT COALESCE(job_id, gen_random_uuid()), id, pipeline_version, attempt + 1, " "(SELECT traceparent FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1), " "(SELECT tracestate FROM outbox WHERE attachment_id = u.id ORDER BY id DESC LIMIT 1) FROM u WHERE status = 'queued' RETURNING 1), " "e AS (INSERT INTO job_events (attachment_id, job_id, event, attempt, detail) SELECT id, job_id, 'lease_expired', attempt, status FROM u RETURNING 1) " "SELECT id, status FROM u", config.MAX_ATTEMPTS) for r in rows: log("warning", "lease_expired", attachment_id=str(r["id"]), new_status=r["status"]) if rows: self.relay.kick() async def sweeper(self): while not self.stopping: try: await self.sweep_once() except Exception as exc: log("warning", "sweeper_failed", error_type=type(exc).__name__) await asyncio.sleep(config.SWEEP_INTERVAL) async def staging_janitor(self): """Удаляет staging для ready (если не удалён после COMMIT) и для failed старше retention; текущую работу не трогает.""" while not self.stopping: await asyncio.sleep(max(30, config.SWEEP_INTERVAL * 6)) try: rows = await self.db.fetch( "SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (status = 'ready' OR " "(status = 'failed' AND updated_at < now() - make_interval(secs => $1))) LIMIT 200", float(config.STAGING_FAILED_RETENTION_SECONDS)) for r in rows: await self.s3.delete(r["staging_key"]) await self.db.execute("UPDATE attachments SET staging_key = NULL WHERE id = $1", r["id"]) except Exception as exc: log("warning", "staging_janitor_failed", error_type=type(exc).__name__) # --- жизненный цикл ------------------------------------------------------------ async def connect_broker(self): while not self.stopping: try: await self.broker.ensure(prefetch=config.WORKER_CONCURRENCY) return except Exception as exc: log("warning", "broker_unavailable", error_type=type(exc).__name__) await asyncio.sleep(2) async def consume(self): while not self.stopping: await self.connect_broker() try: queue = await self.broker.channel.declare_queue(QUEUE, durable=True, passive=True) tag = await queue.consume(self.on_message) log("info", "consuming", worker=WORKER_ID, prefetch=config.WORKER_CONCURRENCY) while not self.stopping and not self.broker.channel.is_closed: await asyncio.sleep(1) if self.stopping: await queue.cancel(tag) return except Exception as exc: log("warning", "consume_failed", error_type=type(exc).__name__) await asyncio.sleep(2) async def run(self): telemetry.setup(config.SERVICE_NAME) telemetry.process_metrics() await self.db.start() await self.s3.start() log("info", "worker_started", worker=WORKER_ID, concurrency=config.WORKER_CONCURRENCY, lease=config.LEASE_SECONDS, max_attempts=config.MAX_ATTEMPTS) loop = asyncio.get_running_loop() for sig in (signal.SIGTERM, signal.SIGINT): try: loop.add_signal_handler(sig, self.request_stop) except NotImplementedError: signal.signal(sig, lambda *_: self.request_stop()) self.tasks = [asyncio.create_task(self.consume()), asyncio.create_task(self.relay.run()), asyncio.create_task(self.sweeper()), asyncio.create_task(self.staging_janitor())] beat = 0.0 while not self.stopping: if time.monotonic() - beat > 5: # heartbeat для HEALTHCHECK контейнера beat = time.monotonic() try: with open("/tmp/worker.alive", "w") as fh: fh.write(str(time.time())) except OSError: pass await asyncio.sleep(0.5) # graceful: перестаём брать новые задания, ждём текущие до GRACEFUL_SHUTDOWN_SECONDS deadline = time.monotonic() + config.GRACEFUL_SHUTDOWN_SECONDS while self.busy and time.monotonic() < deadline: await asyncio.sleep(0.2) self.relay.stop() for t in self.tasks: t.cancel() await self.broker.stop() await self.s3.stop() await self.db.stop() log("info", "worker_stopped", unfinished=self.busy) telemetry.shutdown() def request_stop(self): log("info", "worker_stopping") self.stopping = True class InjectedCrash(Exception): pass def main(): asyncio.run(Worker().run()) if __name__ == "__main__": main()