forked from mtusi2026/mtusi2026lab7
503 lines
27 KiB
Python
503 lines
27 KiB
Python
"""Задания обработки вложений: формат сообщения, outbox, переходы состояний.
|
||
|
||
Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат)
|
||
и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции
|
||
вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT.
|
||
|
||
Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать,
|
||
решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение
|
||
безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op.
|
||
|
||
Переходы (contracts/attachment-states.md):
|
||
(нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202
|
||
queued -> processing claim worker, аренда lease_until + fencing-токен lease_token
|
||
processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter)
|
||
recover_lease аренда истекла (worker умер/завис), попытка засчитана
|
||
processing -> ready finish_ready только при совпадении lease_token (fencing)
|
||
processing -> failed finish_failed постоянная ошибка или попытки исчерпаны
|
||
failed -> queued replay явная операция оператора (ops.py), с записью в журнал
|
||
"""
|
||
import json
|
||
import random
|
||
import re
|
||
import secrets
|
||
from contextlib import contextmanager
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timezone
|
||
from uuid import UUID, uuid4
|
||
|
||
from opentelemetry import trace
|
||
from opentelemetry.trace import SpanKind
|
||
from psycopg.types.json import Jsonb
|
||
|
||
import settings
|
||
import telemetry
|
||
|
||
SCHEMA_VERSION = 1
|
||
JOB_TYPE = "attachment.process.v1"
|
||
EXCHANGE = "attachments"
|
||
ROUTING_KEY = "attachment.process.v1"
|
||
QUEUE = "attachments.process.v1"
|
||
DLX = "attachments.dlx"
|
||
DLQ = "attachments.process.v1.dlq"
|
||
DLQ_ROUTING_KEY = "attachment.process.v1.dlq"
|
||
|
||
MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt",
|
||
"created_at", "traceparent", "tracestate"}
|
||
REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"}
|
||
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||
UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||
CODE_RE = re.compile(r"^[a-z_]{1,64}$")
|
||
|
||
# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES.
|
||
PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"}
|
||
FINAL_CODES = PERMANENT | {"retry_exhausted"}
|
||
|
||
|
||
class InvalidMessage(Exception):
|
||
"""Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ."""
|
||
|
||
|
||
def utc_text(moment: datetime) -> str:
|
||
return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
|
||
def new_traceparent() -> str:
|
||
"""Запасной traceparent, если телеметрия выключена (OTEL_SDK_DISABLED): поле схемы остаётся заполненным."""
|
||
return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01"
|
||
|
||
|
||
def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
|
||
traceparent: str | None, tracestate: str | None = None) -> dict:
|
||
"""Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3.
|
||
|
||
lab6: traceparent — контекст РЕАЛЬНОГО producer-спана, создавшего строку outbox (а не случайный ID).
|
||
Dispatcher при публикации заменяет его контекстом своего спана publish (тот же trace_id).
|
||
"""
|
||
message = {
|
||
"schema_version": SCHEMA_VERSION,
|
||
"job_id": str(job_id),
|
||
"type": JOB_TYPE,
|
||
"attachment_id": str(attachment_id),
|
||
"pipeline_version": pipeline_version,
|
||
"attempt": attempt,
|
||
"created_at": utc_text(datetime.now(timezone.utc)),
|
||
}
|
||
if traceparent:
|
||
message["traceparent"] = traceparent
|
||
if tracestate:
|
||
message["tracestate"] = tracestate[:512]
|
||
return message
|
||
|
||
|
||
def messaging_attributes(job_id: UUID, attachment_id: UUID, attempt: int, operation: str) -> dict:
|
||
"""Атрибуты messaging-спанов (OTel semantic conventions) + технические ID задания."""
|
||
return {"messaging.system": "rabbitmq", "messaging.operation.type": operation,
|
||
"messaging.destination.name": EXCHANGE, "messaging.rabbitmq.destination.routing_key": ROUTING_KEY,
|
||
"messaging.message.id": f"{job_id}:{attempt}", "messenger.job.id": str(job_id),
|
||
"messenger.attachment.id": str(attachment_id), "messenger.job.attempt": attempt}
|
||
|
||
|
||
@contextmanager
|
||
def producer_span(job_id: UUID, attachment_id: UUID, attempt: int, parent_traceparent: str | None = None,
|
||
links: list | None = None):
|
||
"""Спан «create attachments» (PRODUCER): строка outbox = сообщение, созданное в этой транзакции.
|
||
|
||
Родитель — текущий спан (запрос API, попытка worker), а если его нет (фоновый dispatcher) —
|
||
parent_traceparent из строки задания. Возвращает (traceparent, tracestate) для сообщения.
|
||
"""
|
||
current = trace.get_current_span().get_span_context()
|
||
parent = None if current.is_valid else telemetry.context_from(parent_traceparent)
|
||
with telemetry.tracer.start_as_current_span(
|
||
f"create {EXCHANGE}", context=parent, kind=SpanKind.PRODUCER, links=links,
|
||
attributes=messaging_attributes(job_id, attachment_id, attempt, "create")):
|
||
carrier = telemetry.inject()
|
||
yield carrier.get("traceparent") or new_traceparent(), carrier.get("tracestate")
|
||
|
||
|
||
def parse_message(body: bytes) -> dict:
|
||
"""Строгая проверка по схеме (additionalProperties: false, const, типы, форматы)."""
|
||
try:
|
||
message = json.loads(body.decode("utf-8"))
|
||
except (UnicodeDecodeError, ValueError):
|
||
raise InvalidMessage("not json") from None
|
||
if not isinstance(message, dict):
|
||
raise InvalidMessage("not an object")
|
||
if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message):
|
||
raise InvalidMessage("fields")
|
||
if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE:
|
||
raise InvalidMessage("schema_version/type")
|
||
for name in ("job_id", "attachment_id"):
|
||
if not isinstance(message[name], str) or not UUID_RE.match(message[name]):
|
||
raise InvalidMessage(name)
|
||
for name in ("pipeline_version", "attempt"):
|
||
value = message[name]
|
||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||
raise InvalidMessage(name)
|
||
try:
|
||
created = message["created_at"]
|
||
if not isinstance(created, str):
|
||
raise ValueError
|
||
datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||
except ValueError:
|
||
raise InvalidMessage("created_at") from None
|
||
if "traceparent" in message and (not isinstance(message["traceparent"], str)
|
||
or not TRACEPARENT_RE.match(message["traceparent"])):
|
||
raise InvalidMessage("traceparent")
|
||
if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512):
|
||
raise InvalidMessage("tracestate")
|
||
return message
|
||
|
||
|
||
def backoff_seconds(failed_attempt: int) -> float:
|
||
"""Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна.
|
||
|
||
Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS.
|
||
Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу).
|
||
"""
|
||
ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS,
|
||
settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1))
|
||
return ceiling / 2 + random.uniform(0, ceiling / 2)
|
||
|
||
|
||
def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None:
|
||
conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)",
|
||
(job_id, attempt, name, detail[:200] if detail else None, actor[:200]))
|
||
|
||
|
||
def _insert_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
|
||
traceparent: str, tracestate: str | None, delay_seconds: float) -> None:
|
||
payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate)
|
||
conn.execute(
|
||
"""INSERT INTO outbox (job_id, attempt, payload, available_at)
|
||
VALUES (%s, %s, %s, now() + make_interval(secs => %s))""",
|
||
(job_id, attempt, Jsonb(payload), delay_seconds),
|
||
)
|
||
conn.execute("NOTIFY outbox")
|
||
|
||
|
||
def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
|
||
parent_traceparent: str | None, delay_seconds: float = 0, links: list | None = None) -> None:
|
||
"""Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT."""
|
||
with producer_span(job_id, attachment_id, attempt, parent_traceparent, links) as (traceparent, tracestate):
|
||
_insert_outbox(conn, job_id, attachment_id, pipeline_version, attempt, traceparent, tracestate,
|
||
delay_seconds)
|
||
|
||
|
||
def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID:
|
||
"""Новое задание для вложения, уже переведённого в queued (в транзакции API).
|
||
|
||
attachment_jobs.traceparent — контекст исходной загрузки (спан create внутри запроса API): к нему
|
||
привязываются восстановление аренды и сверка, а replay оператора ссылается на него через link.
|
||
"""
|
||
job_id = uuid4()
|
||
with producer_span(job_id, attachment_id, 1) as (traceparent, tracestate):
|
||
conn.execute(
|
||
"""INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts,
|
||
traceparent)
|
||
VALUES (%s, %s, %s, 'queued', 1, %s, %s)""",
|
||
(job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent),
|
||
)
|
||
event(conn, job_id, 1, "created", actor)
|
||
_insert_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent, tracestate, 0)
|
||
return job_id
|
||
|
||
|
||
# ---------- worker: захват и результат ----------
|
||
|
||
@dataclass
|
||
class Claim:
|
||
job_id: UUID
|
||
attachment_id: UUID
|
||
pipeline_version: int
|
||
attempt: int
|
||
max_attempts: int
|
||
lease_token: UUID
|
||
source_key: str
|
||
content_type: str
|
||
size_bytes: int
|
||
sha256: str
|
||
traceparent: str | None
|
||
|
||
|
||
@dataclass
|
||
class Skip:
|
||
"""Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker)."""
|
||
reason: str
|
||
|
||
|
||
@dataclass
|
||
class Reject:
|
||
"""Сообщение не относится ни к одной job -> DLQ."""
|
||
reason: str
|
||
|
||
|
||
def claim(conn, message: dict, owner: str, redelivered: bool):
|
||
"""queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject."""
|
||
job_id = UUID(message["job_id"])
|
||
with conn.transaction():
|
||
job = conn.execute(
|
||
"""SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired
|
||
FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""",
|
||
(job_id,),
|
||
).fetchone()
|
||
if job is None:
|
||
return Reject("unknown_job")
|
||
if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]:
|
||
return Reject("job_mismatch")
|
||
status = job["status"]
|
||
if status in ("ready", "failed"):
|
||
event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}")
|
||
return Skip(f"already_{status}")
|
||
attempt = job["attempt"]
|
||
if status == "queued":
|
||
if not job["due"]:
|
||
return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox
|
||
else: # processing
|
||
if not job["lease_expired"]:
|
||
event(conn, job_id, attempt, "duplicate_ignored", owner, "leased")
|
||
return Skip("leased")
|
||
# Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана.
|
||
event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker")
|
||
if attempt >= job["max_attempts"]:
|
||
_fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired")
|
||
return Skip("exhausted")
|
||
attempt += 1
|
||
token = uuid4()
|
||
conn.execute(
|
||
"""UPDATE attachment_jobs
|
||
SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s,
|
||
lease_until = now() + make_interval(secs => %s), updated_at = now()
|
||
WHERE job_id = %s""",
|
||
(attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id),
|
||
)
|
||
att = conn.execute(
|
||
"""UPDATE attachments SET state = 'processing', updated_at = now()
|
||
WHERE id = %s AND state IN ('queued', 'processing')
|
||
RETURNING source_key, content_type, size_bytes, sha256""",
|
||
(job["attachment_id"],),
|
||
).fetchone()
|
||
if att is None or att["source_key"] is None:
|
||
raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат
|
||
event(conn, job_id, attempt, "claimed", owner,
|
||
f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}")
|
||
return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token,
|
||
att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"])
|
||
|
||
|
||
def renew_lease(conn, c: Claim) -> bool:
|
||
"""Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить."""
|
||
row = conn.execute(
|
||
"""UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now()
|
||
WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""",
|
||
(settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token),
|
||
).fetchone()
|
||
return row is not None
|
||
|
||
|
||
def _fenced(conn, c: Claim, owner: str) -> bool:
|
||
"""Fencing: результат принимается, только если lease_token всё ещё наш."""
|
||
row = conn.execute(
|
||
"""SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing'
|
||
FOR UPDATE""",
|
||
(c.job_id, c.lease_token),
|
||
).fetchone()
|
||
return row is not None
|
||
|
||
|
||
def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> float | None:
|
||
"""processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT.
|
||
|
||
Возвращает время от устойчивого приёма (COMMIT задания в API) до ready в секундах по часам
|
||
PostgreSQL (метрика messenger.attachment.time_to_ready) или None, если результат отклонён fencing.
|
||
"""
|
||
time_to_ready = None
|
||
with conn.transaction():
|
||
if not _fenced(conn, c, owner):
|
||
stale = True
|
||
else:
|
||
stale = False
|
||
for v in variants:
|
||
conn.execute(
|
||
"""INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type,
|
||
size_bytes, width, height, sha256, object_key)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""",
|
||
(c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"],
|
||
v["width"], v["height"], v["sha256"], v["object_key"]),
|
||
)
|
||
# Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна).
|
||
updated = conn.execute(
|
||
"""UPDATE attachments
|
||
SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now()
|
||
WHERE id = %s AND state = 'processing'
|
||
AND (pipeline_version IS NULL OR pipeline_version <= %s)
|
||
RETURNING id""",
|
||
(original_key, c.pipeline_version, c.attachment_id, c.pipeline_version),
|
||
).fetchone()
|
||
if updated is None:
|
||
raise RuntimeError("attachment changed state during processing")
|
||
time_to_ready = conn.execute(
|
||
"""UPDATE attachment_jobs
|
||
SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||
error_code = NULL, finished_at = now(), updated_at = now()
|
||
WHERE job_id = %s
|
||
RETURNING extract(epoch FROM clock_timestamp() - created_at)::float8 AS seconds""",
|
||
(c.job_id,),
|
||
).fetchone()["seconds"]
|
||
event(conn, c.job_id, c.attempt, "ready", owner)
|
||
if stale:
|
||
with conn.transaction():
|
||
event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost")
|
||
return None
|
||
return time_to_ready
|
||
|
||
|
||
def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None):
|
||
conn.execute(
|
||
"""UPDATE attachment_jobs
|
||
SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL,
|
||
lease_until = NULL, finished_at = now(), updated_at = now()
|
||
WHERE job_id = %s""",
|
||
(code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id),
|
||
)
|
||
conn.execute(
|
||
"""UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now()
|
||
WHERE id = %s AND state IN ('queued', 'processing')""",
|
||
(code, attachment_id),
|
||
)
|
||
event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}")
|
||
|
||
|
||
def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int,
|
||
traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None:
|
||
conn.execute(
|
||
"""UPDATE attachment_jobs
|
||
SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL,
|
||
lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now()
|
||
WHERE job_id = %s""",
|
||
(next_attempt, reason, delay, job_id),
|
||
)
|
||
conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'",
|
||
(attachment_id,))
|
||
add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay)
|
||
event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s")
|
||
|
||
|
||
def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str:
|
||
"""Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'."""
|
||
with conn.transaction():
|
||
if not _fenced(conn, c, owner):
|
||
outcome = "stale"
|
||
elif permanent:
|
||
_fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None)
|
||
outcome = "failed"
|
||
elif c.attempt >= c.max_attempts:
|
||
_fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code)
|
||
outcome = "failed"
|
||
else:
|
||
_requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent,
|
||
backoff_seconds(c.attempt), code, owner, "retry_scheduled")
|
||
outcome = "retry"
|
||
if outcome == "stale":
|
||
with conn.transaction():
|
||
event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code)
|
||
return outcome
|
||
|
||
|
||
# ---------- dispatcher: аренды и сверка ----------
|
||
|
||
def recover_expired_leases(conn, actor: str, limit: int = 100) -> int:
|
||
"""processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted)."""
|
||
recovered = 0
|
||
with conn.transaction():
|
||
rows = conn.execute(
|
||
"""SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent
|
||
FROM attachment_jobs WHERE status = 'processing' AND lease_until < now()
|
||
ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||
(limit,),
|
||
).fetchall()
|
||
for job in rows:
|
||
# Спан в трассе исходной загрузки: видно, что попытку вернул dispatcher, а не worker.
|
||
with telemetry.tracer.start_as_current_span(
|
||
"recover expired lease", context=telemetry.context_from(job["traceparent"]),
|
||
attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}):
|
||
if job["attempt"] >= job["max_attempts"]:
|
||
_fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor,
|
||
"lease_expired")
|
||
else:
|
||
_requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"],
|
||
job["attempt"] + 1, job["traceparent"], backoff_seconds(job["attempt"]),
|
||
"lease_expired", actor, "lease_expired")
|
||
recovered += 1
|
||
return recovered
|
||
|
||
|
||
def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int:
|
||
"""Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет.
|
||
|
||
Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его
|
||
подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет.
|
||
"""
|
||
with conn.transaction():
|
||
rows = conn.execute(
|
||
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent
|
||
FROM attachment_jobs j
|
||
WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s)
|
||
AND j.updated_at < now() - make_interval(secs => %s)
|
||
AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND
|
||
(o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s)))
|
||
ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""",
|
||
(older_than, older_than, older_than, limit),
|
||
).fetchall()
|
||
for job in rows:
|
||
with telemetry.tracer.start_as_current_span(
|
||
"republish lost message", context=telemetry.context_from(job["traceparent"]),
|
||
attributes={"messenger.job.id": str(job["job_id"]), "messenger.job.attempt": job["attempt"]}):
|
||
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"],
|
||
job["traceparent"])
|
||
conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],))
|
||
event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight")
|
||
return len(rows)
|
||
|
||
|
||
# ---------- оператор: replay ----------
|
||
|
||
class ReplayRefused(Exception):
|
||
pass
|
||
|
||
|
||
def replay(conn, job_id: UUID, actor: str, reason: str) -> dict:
|
||
"""failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются."""
|
||
with conn.transaction():
|
||
job = conn.execute(
|
||
"""SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j
|
||
JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""",
|
||
(job_id,),
|
||
).fetchone()
|
||
if job is None:
|
||
raise ReplayRefused("job not found")
|
||
if job["status"] != "failed":
|
||
# «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается.
|
||
raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs")
|
||
if job["source_key"] is None:
|
||
raise ReplayRefused("source file is no longer in staging (retention expired)")
|
||
attempt = job["attempt"] + 1
|
||
conn.execute(
|
||
"""UPDATE attachment_jobs
|
||
SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL,
|
||
available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now()
|
||
WHERE job_id = %s""",
|
||
(attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id),
|
||
)
|
||
conn.execute(
|
||
"UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'",
|
||
(job["attachment_id"],),
|
||
)
|
||
event(conn, job_id, attempt, "replayed", actor, reason or "operator replay")
|
||
# Replay — новое решение оператора, а не продолжение исходного запроса: новая трасса со
|
||
# ссылкой (link) на исходную загрузку. Вызов из спана ops.py становится его родителем.
|
||
add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, None,
|
||
links=telemetry.link_to(job["traceparent"]))
|
||
return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt}
|