Files
mtusi2026lab6/pipeline.py
T

503 lines
27 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Задания обработки вложений: формат сообщения, outbox, переходы состояний.
Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат)
и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции
вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT.
Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать,
решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение
безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op.
Переходы (contracts/attachment-states.md):
(нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202
queued -> processing claim worker, аренда lease_until + fencing-токен lease_token
processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter)
recover_lease аренда истекла (worker умер/завис), попытка засчитана
processing -> ready finish_ready только при совпадении lease_token (fencing)
processing -> failed finish_failed постоянная ошибка или попытки исчерпаны
failed -> queued replay явная операция оператора (ops.py), с записью в журнал
"""
import json
import random
import re
import secrets
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime, timezone
from uuid import UUID, uuid4
from opentelemetry import trace
from opentelemetry.trace import SpanKind
from psycopg.types.json import Jsonb
import settings
import telemetry
SCHEMA_VERSION = 1
JOB_TYPE = "attachment.process.v1"
EXCHANGE = "attachments"
ROUTING_KEY = "attachment.process.v1"
QUEUE = "attachments.process.v1"
DLX = "attachments.dlx"
DLQ = "attachments.process.v1.dlq"
DLQ_ROUTING_KEY = "attachment.process.v1.dlq"
MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt",
"created_at", "traceparent", "tracestate"}
REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"}
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
CODE_RE = re.compile(r"^[a-z_]{1,64}$")
# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES.
PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"}
FINAL_CODES = PERMANENT | {"retry_exhausted"}
class InvalidMessage(Exception):
"""Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ."""
def utc_text(moment: datetime) -> str:
return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def new_traceparent() -> str:
"""Запасной traceparent, если телеметрия выключена (OTEL_SDK_DISABLED): поле схемы остаётся заполненным."""
return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01"
def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
traceparent: str | None, tracestate: str | None = None) -> dict:
"""Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3.
lab6: traceparent — контекст РЕАЛЬНОГО producer-спана, создавшего строку outbox (а не случайный ID).
Dispatcher при публикации заменяет его контекстом своего спана publish (тот же trace_id).
"""
message = {
"schema_version": SCHEMA_VERSION,
"job_id": str(job_id),
"type": JOB_TYPE,
"attachment_id": str(attachment_id),
"pipeline_version": pipeline_version,
"attempt": attempt,
"created_at": utc_text(datetime.now(timezone.utc)),
}
if traceparent:
message["traceparent"] = traceparent
if tracestate:
message["tracestate"] = tracestate[:512]
return message
def messaging_attributes(job_id: UUID, attachment_id: UUID, attempt: int, operation: str) -> dict:
"""Атрибуты messaging-спанов (OTel semantic conventions) + технические ID задания."""
return {"messaging.system": "rabbitmq", "messaging.operation.type": operation,
"messaging.destination.name": EXCHANGE, "messaging.rabbitmq.destination.routing_key": ROUTING_KEY,
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
"messenger.attachment.id": str(attachment_id), "messenger.job.attempt": attempt}
@contextmanager
def producer_span(job_id: UUID, attachment_id: UUID, attempt: int, parent_traceparent: str | None = None,
links: list | None = None):
"""Спан «create attachments» (PRODUCER): строка outbox = сообщение, созданное в этой транзакции.
Родитель — текущий спан (запрос API, попытка worker), а если его нет (фоновый dispatcher) —
parent_traceparent из строки задания. Возвращает (traceparent, tracestate) для сообщения.
"""
current = trace.get_current_span().get_span_context()
parent = None if current.is_valid else telemetry.context_from(parent_traceparent)
with telemetry.tracer.start_as_current_span(
f"create {EXCHANGE}", context=parent, kind=SpanKind.PRODUCER, links=links,
attributes=messaging_attributes(job_id, attachment_id, attempt, "create")):
carrier = telemetry.inject()
yield carrier.get("traceparent") or new_traceparent(), carrier.get("tracestate")
def parse_message(body: bytes) -> dict:
"""Строгая проверка по схеме (additionalProperties: false, const, типы, форматы)."""
try:
message = json.loads(body.decode("utf-8"))
except (UnicodeDecodeError, ValueError):
raise InvalidMessage("not json") from None
if not isinstance(message, dict):
raise InvalidMessage("not an object")
if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message):
raise InvalidMessage("fields")
if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE:
raise InvalidMessage("schema_version/type")
for name in ("job_id", "attachment_id"):
if not isinstance(message[name], str) or not UUID_RE.match(message[name]):
raise InvalidMessage(name)
for name in ("pipeline_version", "attempt"):
value = message[name]
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
raise InvalidMessage(name)
try:
created = message["created_at"]
if not isinstance(created, str):
raise ValueError
datetime.fromisoformat(created.replace("Z", "+00:00"))
except ValueError:
raise InvalidMessage("created_at") from None
if "traceparent" in message and (not isinstance(message["traceparent"], str)
or not TRACEPARENT_RE.match(message["traceparent"])):
raise InvalidMessage("traceparent")
if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512):
raise InvalidMessage("tracestate")
return message
def backoff_seconds(failed_attempt: int) -> float:
"""Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна.
Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS.
Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу).
"""
ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS,
settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1))
return ceiling / 2 + random.uniform(0, ceiling / 2)
def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None:
conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)",
(job_id, attempt, name, detail[:200] if detail else None, actor[:200]))
def _insert_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
traceparent: str, tracestate: str | None, delay_seconds: float) -> None:
payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate)
conn.execute(
"""INSERT INTO outbox (job_id, attempt, payload, available_at)
VALUES (%s, %s, %s, now() + make_interval(secs => %s))""",
(job_id, attempt, Jsonb(payload), delay_seconds),
)
conn.execute("NOTIFY outbox")
def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
parent_traceparent: str | None, delay_seconds: float = 0, links: list | None = None) -> None:
"""Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT."""
with producer_span(job_id, attachment_id, attempt, parent_traceparent, links) as (traceparent, tracestate):
_insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate,
delay_seconds)
def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID:
"""Новое задание для вложения, уже переведённого в queued (в транзакции API).
attachment_jobs.traceparent — контекст исходной загрузки (спан create внутри запроса API): к нему
привязываются восстановление аренды и сверка, а replay оператора ссылается на него через link.
"""
job_id = uuid4()
with producer_span(job_id, attachment_id, 1) as (traceparent, tracestate):
conn.execute(
"""INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts,
traceparent)
VALUES (%s, %s, %s, 'queued', 1, %s, %s)""",
(job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent),
)
event(conn, job_id, 1, "created", actor)
_insert_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent, tracestate, 0)
return job_id
# ---------- worker: захват и результат ----------
@dataclass
class Claim:
job_id: UUID
attachment_id: UUID
pipeline_version: int
attempt: int
max_attempts: int
lease_token: UUID
source_key: str
content_type: str
size_bytes: int
sha256: str
traceparent: str | None
@dataclass
class Skip:
"""Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker)."""
reason: str
@dataclass
class Reject:
"""Сообщение не относится ни к одной job -> DLQ."""
reason: str
def claim(conn, message: dict, owner: str, redelivered: bool):
"""queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject."""
job_id = UUID(message["job_id"])
with conn.transaction():
job = conn.execute(
"""SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired
FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""",
(job_id,),
).fetchone()
if job is None:
return Reject("unknown_job")
if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]:
return Reject("job_mismatch")
status = job["status"]
if status in ("ready", "failed"):
event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}")
return Skip(f"already_{status}")
attempt = job["attempt"]
if status == "queued":
if not job["due"]:
return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox
else: # processing
if not job["lease_expired"]:
event(conn, job_id, attempt, "duplicate_ignored", owner, "leased")
return Skip("leased")
# Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана.
event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker")
if attempt >= job["max_attempts"]:
_fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired")
return Skip("exhausted")
attempt += 1
token = uuid4()
conn.execute(
"""UPDATE attachment_jobs
SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s,
lease_until = now() + make_interval(secs => %s), updated_at = now()
WHERE job_id = %s""",
(attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id),
)
att = conn.execute(
"""UPDATE attachments SET state = 'processing', updated_at = now()
WHERE id = %s AND state IN ('queued', 'processing')
RETURNING source_key, content_type, size_bytes, sha256""",
(job["attachment_id"],),
).fetchone()
if att is None or att["source_key"] is None:
raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат
event(conn, job_id, attempt, "claimed", owner,
f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}")
return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token,
att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"])
def renew_lease(conn, c: Claim) -> bool:
"""Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить."""
row = conn.execute(
"""UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now()
WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""",
(settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token),
).fetchone()
return row is not None
def _fenced(conn, c: Claim, owner: str) -> bool:
"""Fencing: результат принимается, только если lease_token всё ещё наш."""
row = conn.execute(
"""SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing'
FOR UPDATE""",
(c.job_id, c.lease_token),
).fetchone()
return row is not None
def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> float | None:
"""processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT.
Возвращает время от устойчивого приёма (COMMIT задания в API) до ready в секундах по часам
PostgreSQL (метрика messenger.attachment.time_to_ready) или None, если результат отклонён fencing.
"""
time_to_ready = None
with conn.transaction():
if not _fenced(conn, c, owner):
stale = True
else:
stale = False
for v in variants:
conn.execute(
"""INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type,
size_bytes, width, height, sha256, object_key)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""",
(c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"],
v["width"], v["height"], v["sha256"], v["object_key"]),
)
# Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна).
updated = conn.execute(
"""UPDATE attachments
SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now()
WHERE id = %s AND state = 'processing'
AND (pipeline_version IS NULL OR pipeline_version <= %s)
RETURNING id""",
(original_key, c.pipeline_version, c.attachment_id, c.pipeline_version),
).fetchone()
if updated is None:
raise RuntimeError("attachment changed state during processing")
time_to_ready = conn.execute(
"""UPDATE attachment_jobs
SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
error_code = NULL, finished_at = now(), updated_at = now()
WHERE job_id = %s
RETURNING extract(epoch FROM clock_timestamp() - created_at)::float8 AS seconds""",
(c.job_id,),
).fetchone()["seconds"]
event(conn, c.job_id, c.attempt, "ready", owner)
if stale:
with conn.transaction():
event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost")
return None
return time_to_ready
def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None):
conn.execute(
"""UPDATE attachment_jobs
SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL,
lease_until = NULL, finished_at = now(), updated_at = now()
WHERE job_id = %s""",
(code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id),
)
conn.execute(
"""UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now()
WHERE id = %s AND state IN ('queued', 'processing')""",
(code, attachment_id),
)
event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}")
def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int,
traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None:
conn.execute(
"""UPDATE attachment_jobs
SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL,
lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now()
WHERE job_id = %s""",
(next_attempt, reason, delay, job_id),
)
conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'",
(attachment_id,))
add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay)
event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s")
def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str:
"""Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'."""
with conn.transaction():
if not _fenced(conn, c, owner):
outcome = "stale"
elif permanent:
_fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None)
outcome = "failed"
elif c.attempt >= c.max_attempts:
_fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code)
outcome = "failed"
else:
_requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent,
backoff_seconds(c.attempt), code, owner, "retry_scheduled")
outcome = "retry"
if outcome == "stale":
with conn.transaction():
event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code)
return outcome
# ---------- dispatcher: аренды и сверка ----------
def recover_expired_leases(conn, actor: str, limit: int = 100) -> int:
"""processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted)."""
recovered = 0
with conn.transaction():
rows = conn.execute(
"""SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent
FROM attachment_jobs WHERE status = 'processing' AND lease_until < now()
ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""",
(limit,),
).fetchall()
for job in rows:
# Спан в трассе исходной загрузки: видно, что попытку вернул dispatcher, а не worker.
with telemetry.tracer.start_as_current_span(
"recover expired lease", context=telemetry.context_from(job["traceparent"]),
attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}):
if job["attempt"] >= job["max_attempts"]:
_fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor,
"lease_expired")
else:
_requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"],
job["attempt"] + 1, job["traceparent"], backoff_seconds(job["attempt"]),
"lease_expired", actor, "lease_expired")
recovered += 1
return recovered
def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int:
"""Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет.
Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его
подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет.
"""
with conn.transaction():
rows = conn.execute(
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent
FROM attachment_jobs j
WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s)
AND j.updated_at < now() - make_interval(secs => %s)
AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND
(o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s)))
ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""",
(older_than, older_than, older_than, limit),
).fetchall()
for job in rows:
with telemetry.tracer.start_as_current_span(
"republish lost message", context=telemetry.context_from(job["traceparent"]),
attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}):
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"],
job["traceparent"])
conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],))
event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight")
return len(rows)
# ---------- оператор: replay ----------
class ReplayRefused(Exception):
pass
def replay(conn, job_id: UUID, actor: str, reason: str) -> dict:
"""failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются."""
with conn.transaction():
job = conn.execute(
"""SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""",
(job_id,),
).fetchone()
if job is None:
raise ReplayRefused("job not found")
if job["status"] != "failed":
# «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается.
raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs")
if job["source_key"] is None:
raise ReplayRefused("source file is no longer in staging (retention expired)")
attempt = job["attempt"] + 1
conn.execute(
"""UPDATE attachment_jobs
SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL,
available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now()
WHERE job_id = %s""",
(attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id),
)
conn.execute(
"UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'",
(job["attachment_id"],),
)
event(conn, job_id, attempt, "replayed", actor, reason or "operator replay")
# Replay — новое решение оператора, а не продолжение исходного запроса: новая трасса со
# ссылкой (link) на исходную загрузку. Вызов из спана ops.py становится его родителем.
add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, None,
links=telemetry.link_to(job["traceparent"]))
return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt}