"""Задания обработки вложений: формат сообщения, outbox, переходы состояний. Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат) и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT. Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать, решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op. Переходы (contracts/attachment-states.md): (нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202 queued -> processing claim worker, аренда lease_until + fencing-токен lease_token processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter) recover_lease аренда истекла (worker умер/завис), попытка засчитана processing -> ready finish_ready только при совпадении lease_token (fencing) processing -> failed finish_failed постоянная ошибка или попытки исчерпаны failed -> queued replay явная операция оператора (ops.py), с записью в журнал """ import json import random import re import secrets from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timezone from uuid import UUID, uuid4 from opentelemetry import trace from opentelemetry.trace import SpanKind from psycopg.types.json import Jsonb import settings import telemetry SCHEMA_VERSION = 1 JOB_TYPE = "attachment.process.v1" EXCHANGE = "attachments" ROUTING_KEY = "attachment.process.v1" QUEUE = "attachments.process.v1" DLX = "attachments.dlx" DLQ = "attachments.process.v1.dlq" DLQ_ROUTING_KEY = "attachment.process.v1.dlq" MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", "traceparent", "tracestate"} REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"} TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") CODE_RE = re.compile(r"^[a-z_]{1,64}$") # Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES. PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"} FINAL_CODES = PERMANENT | {"retry_exhausted"} class InvalidMessage(Exception): """Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ.""" def utc_text(moment: datetime) -> str: return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def new_traceparent() -> str: """Запасной traceparent, если телеметрия выключена (OTEL_SDK_DISABLED): поле схемы остаётся заполненным.""" return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01" def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, traceparent: str | None, tracestate: str | None = None) -> dict: """Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3. lab6: traceparent — контекст РЕАЛЬНОГО producer-спана, создавшего строку outbox (а не случайный ID). Dispatcher при публикации заменяет его контекстом своего спана publish (тот же trace_id). """ message = { "schema_version": SCHEMA_VERSION, "job_id": str(job_id), "type": JOB_TYPE, "attachment_id": str(attachment_id), "pipeline_version": pipeline_version, "attempt": attempt, "created_at": utc_text(datetime.now(timezone.utc)), } if traceparent: message["traceparent"] = traceparent if tracestate: message["tracestate"] = tracestate[:512] return message def messaging_attributes(job_id: UUID, attachment_id: UUID, attempt: int, operation: str) -> dict: """Атрибуты messaging-спанов (OTel semantic conventions) + технические ID задания.""" return {"messaging.system": "rabbitmq", "messaging.operation.type": operation, "messaging.destination.name": EXCHANGE, "messaging.rabbitmq.destination.routing_key": ROUTING_KEY, "messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id), "messenger.attachment.id": str(attachment_id), "messenger.job.attempt": attempt} @contextmanager def producer_span(job_id: UUID, attachment_id: UUID, attempt: int, parent_traceparent: str | None = None, links: list | None = None): """Спан «create attachments» (PRODUCER): строка outbox = сообщение, созданное в этой транзакции. Родитель — текущий спан (запрос API, попытка worker), а если его нет (фоновый dispatcher) — parent_traceparent из строки задания. Возвращает (traceparent, tracestate) для сообщения. """ current = trace.get_current_span().get_span_context() parent = None if current.is_valid else telemetry.context_from(parent_traceparent) with telemetry.tracer.start_as_current_span( f"create {EXCHANGE}", context=parent, kind=SpanKind.PRODUCER, links=links, attributes=messaging_attributes(job_id, attachment_id, attempt, "create")): carrier = telemetry.inject() yield carrier.get("traceparent") or new_traceparent(), carrier.get("tracestate") def parse_message(body: bytes) -> dict: """Строгая проверка по схеме (additionalProperties: false, const, типы, форматы).""" try: message = json.loads(body.decode("utf-8")) except (UnicodeDecodeError, ValueError): raise InvalidMessage("not json") from None if not isinstance(message, dict): raise InvalidMessage("not an object") if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message): raise InvalidMessage("fields") if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE: raise InvalidMessage("schema_version/type") for name in ("job_id", "attachment_id"): if not isinstance(message[name], str) or not UUID_RE.match(message[name]): raise InvalidMessage(name) for name in ("pipeline_version", "attempt"): value = message[name] if not isinstance(value, int) or isinstance(value, bool) or value < 1: raise InvalidMessage(name) try: created = message["created_at"] if not isinstance(created, str): raise ValueError datetime.fromisoformat(created.replace("Z", "+00:00")) except ValueError: raise InvalidMessage("created_at") from None if "traceparent" in message and (not isinstance(message["traceparent"], str) or not TRACEPARENT_RE.match(message["traceparent"])): raise InvalidMessage("traceparent") if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512): raise InvalidMessage("tracestate") return message def backoff_seconds(failed_attempt: int) -> float: """Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна. Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS. Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу). """ ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS, settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1)) return ceiling / 2 + random.uniform(0, ceiling / 2) def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None: conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)", (job_id, attempt, name, detail[:200] if detail else None, actor[:200])) def _insert_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, traceparent: str, tracestate: str | None, delay_seconds: float) -> None: payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate) conn.execute( """INSERT INTO outbox (job_id, attempt, payload, available_at) VALUES (%s, %s, %s, now() + make_interval(secs => %s))""", (job_id, attempt, Jsonb(payload), delay_seconds), ) conn.execute("NOTIFY outbox") def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, parent_traceparent: str | None, delay_seconds: float = 0, links: list | None = None) -> None: """Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT.""" with producer_span(job_id, attachment_id, attempt, parent_traceparent, links) as (traceparent, tracestate): _insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate, delay_seconds) def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID: """Новое задание для вложения, уже переведённого в queued (в транзакции API). attachment_jobs.traceparent — контекст исходной загрузки (спан create внутри запроса API): к нему привязываются восстановление аренды и сверка, а replay оператора ссылается на него через link. """ job_id = uuid4() with producer_span(job_id, attachment_id, 1) as (traceparent, tracestate): conn.execute( """INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent) VALUES (%s, %s, %s, 'queued', 1, %s, %s)""", (job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent), ) event(conn, job_id, 1, "created", actor) _insert_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent, tracestate, 0) return job_id # ---------- worker: захват и результат ---------- @dataclass class Claim: job_id: UUID attachment_id: UUID pipeline_version: int attempt: int max_attempts: int lease_token: UUID source_key: str content_type: str size_bytes: int sha256: str traceparent: str | None @dataclass class Skip: """Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker).""" reason: str @dataclass class Reject: """Сообщение не относится ни к одной job -> DLQ.""" reason: str def claim(conn, message: dict, owner: str, redelivered: bool): """queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject.""" job_id = UUID(message["job_id"]) with conn.transaction(): job = conn.execute( """SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""", (job_id,), ).fetchone() if job is None: return Reject("unknown_job") if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]: return Reject("job_mismatch") status = job["status"] if status in ("ready", "failed"): event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}") return Skip(f"already_{status}") attempt = job["attempt"] if status == "queued": if not job["due"]: return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox else: # processing if not job["lease_expired"]: event(conn, job_id, attempt, "duplicate_ignored", owner, "leased") return Skip("leased") # Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана. event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker") if attempt >= job["max_attempts"]: _fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired") return Skip("exhausted") attempt += 1 token = uuid4() conn.execute( """UPDATE attachment_jobs SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s""", (attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id), ) att = conn.execute( """UPDATE attachments SET state = 'processing', updated_at = now() WHERE id = %s AND state IN ('queued', 'processing') RETURNING source_key, content_type, size_bytes, sha256""", (job["attachment_id"],), ).fetchone() if att is None or att["source_key"] is None: raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат event(conn, job_id, attempt, "claimed", owner, f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}") return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token, att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"]) def renew_lease(conn, c: Claim) -> bool: """Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить.""" row = conn.execute( """UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""", (settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token), ).fetchone() return row is not None def _fenced(conn, c: Claim, owner: str) -> bool: """Fencing: результат принимается, только если lease_token всё ещё наш.""" row = conn.execute( """SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing' FOR UPDATE""", (c.job_id, c.lease_token), ).fetchone() return row is not None def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> float | None: """processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT. Возвращает время от устойчивого приёма (COMMIT задания в API) до ready в секундах по часам PostgreSQL (метрика messenger.attachment.time_to_ready) или None, если результат отклонён fencing. """ time_to_ready = None with conn.transaction(): if not _fenced(conn, c, owner): stale = True else: stale = False for v in variants: conn.execute( """INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, size_bytes, width, height, sha256, object_key) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""", (c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"], v["width"], v["height"], v["sha256"], v["object_key"]), ) # Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна). updated = conn.execute( """UPDATE attachments SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now() WHERE id = %s AND state = 'processing' AND (pipeline_version IS NULL OR pipeline_version <= %s) RETURNING id""", (original_key, c.pipeline_version, c.attachment_id, c.pipeline_version), ).fetchone() if updated is None: raise RuntimeError("attachment changed state during processing") time_to_ready = conn.execute( """UPDATE attachment_jobs SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, error_code = NULL, finished_at = now(), updated_at = now() WHERE job_id = %s RETURNING extract(epoch FROM clock_timestamp() - created_at)::float8 AS seconds""", (c.job_id,), ).fetchone()["seconds"] event(conn, c.job_id, c.attempt, "ready", owner) if stale: with conn.transaction(): event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost") return None return time_to_ready def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None): conn.execute( """UPDATE attachment_jobs SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, lease_until = NULL, finished_at = now(), updated_at = now() WHERE job_id = %s""", (code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id), ) conn.execute( """UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now() WHERE id = %s AND state IN ('queued', 'processing')""", (code, attachment_id), ) event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}") def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int, traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None: conn.execute( """UPDATE attachment_jobs SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s""", (next_attempt, reason, delay, job_id), ) conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'", (attachment_id,)) add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay) event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s") def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str: """Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'.""" with conn.transaction(): if not _fenced(conn, c, owner): outcome = "stale" elif permanent: _fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None) outcome = "failed" elif c.attempt >= c.max_attempts: _fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code) outcome = "failed" else: _requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent, backoff_seconds(c.attempt), code, owner, "retry_scheduled") outcome = "retry" if outcome == "stale": with conn.transaction(): event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code) return outcome # ---------- dispatcher: аренды и сверка ---------- def recover_expired_leases(conn, actor: str, limit: int = 100) -> int: """processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted).""" recovered = 0 with conn.transaction(): rows = conn.execute( """SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent FROM attachment_jobs WHERE status = 'processing' AND lease_until < now() ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""", (limit,), ).fetchall() for job in rows: # Спан в трассе исходной загрузки: видно, что попытку вернул dispatcher, а не worker. with telemetry.tracer.start_as_current_span( "recover expired lease", context=telemetry.context_from(job["traceparent"]), attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}): if job["attempt"] >= job["max_attempts"]: _fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor, "lease_expired") else: _requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1, job["traceparent"], backoff_seconds(job["attempt"]), "lease_expired", actor, "lease_expired") recovered += 1 return recovered def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int: """Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет. Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет. """ with conn.transaction(): rows = conn.execute( """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent FROM attachment_jobs j WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s) AND j.updated_at < now() - make_interval(secs => %s) AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND (o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s))) ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""", (older_than, older_than, older_than, limit), ).fetchall() for job in rows: with telemetry.tracer.start_as_current_span( "republish lost message", context=telemetry.context_from(job["traceparent"]), attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}): add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], job["traceparent"]) conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],)) event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight") return len(rows) # ---------- оператор: replay ---------- class ReplayRefused(Exception): pass def replay(conn, job_id: UUID, actor: str, reason: str) -> dict: """failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются.""" with conn.transaction(): job = conn.execute( """SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""", (job_id,), ).fetchone() if job is None: raise ReplayRefused("job not found") if job["status"] != "failed": # «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается. raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs") if job["source_key"] is None: raise ReplayRefused("source file is no longer in staging (retention expired)") attempt = job["attempt"] + 1 conn.execute( """UPDATE attachment_jobs SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL, available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now() WHERE job_id = %s""", (attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id), ) conn.execute( "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'", (job["attachment_id"],), ) event(conn, job_id, attempt, "replayed", actor, reason or "operator replay") # Replay — новое решение оператора, а не продолжение исходного запроса: новая трасса со # ссылкой (link) на исходную загрузку. Вызов из спана ops.py становится его родителем. add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, None, links=telemetry.link_to(job["traceparent"])) return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt}