Files
mtusi2026lab5/app/worker.py
T

585 lines
33 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Worker фоновой обработки вложений (lab5). Отдельный процесс/контейнер: `python -m app.worker`.
Поток одной задачи (сообщение брокера = только ID, см. app/broker.py):
XREADGROUP ─► claim в БД (FOR UPDATE): queued → processing, lease_token, lease_until
─► скачать оригинал из staging, сверить sha256 с БД (доверенные метаданные)
─► дочерний процесс: thumbnail / crop / optimized (лимиты памяти и времени)
─► S3: copy staging → originals/<id>, PUT variants/<id>/v1/* (детерминированные ключи)
─► COMMIT результата (только владелец аренды): variants ON CONFLICT DO NOTHING,
attachment и job → ready
─► XACK (+XDEL) ─► удалить staging (после безопасной фиксации)
Ошибки:
постоянные (битое изображение, sha256 не совпал, исходника нет) → failed + error_code;
временные (S3/БД недоступны, таймаут обработки) → queued, attempt+1, backoff с jitter
через outbox (available_at в будущем); попытки кончились → failed retry_exhausted.
Отказы worker:
умер до XACK → сообщение остаётся в PEL брокера; после рестарта worker читает свой PEL,
чужой брошенный PEL забирает XAUTOCLAIM (CLAIM_IDLE_SECONDS);
завис/умер с арендой → аренду никто не продлевает, после истечения задачу
забирает другой (lease takeover) или reaper переводит её в queued с новой попыткой.
Устаревший worker после пробуждения ничего не перезапишет: COMMIT проверяет lease_token.
Поток maintenance (в каждом worker, безопасно параллельно — FOR UPDATE SKIP LOCKED):
outbox relay, reaper просроченных аренд, очистка staging, heartbeat-файл для healthcheck.
"""
from __future__ import annotations
import hashlib
import json
import logging
import multiprocessing
import os
import random
import shutil
import signal
import tempfile
import threading
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
import psycopg
import redis
from psycopg_pool import ConnectionPool
from . import broker, imaging, storage
from .config import settings
log = logging.getLogger("messenger.worker")
class Transient(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail
class Permanent(Exception):
def __init__(self, code: str, detail: str = ""):
super().__init__(code)
self.code, self.detail = code, detail
class LeaseLost(Exception):
"""Аренду забрали (истекла и задачу взял другой) — результат фиксировать нельзя."""
@dataclass
class Job:
job_id: str
attachment_id: str
pipeline_version: int
attempt: int
max_attempts: int
token: str
staging_key: str | None
object_key: str
sha256: str
size_bytes: int
content_type: str
def backoff_seconds(attempt: int) -> float:
"""Экспоненциальный backoff с jitter («equal jitter»): половина задержки
фиксирована, половина случайна — повторы разных задач не идут синхронной волной."""
d = min(settings.job_backoff_max_seconds, settings.job_backoff_base_seconds * 2 ** (attempt - 1))
return d / 2 + random.uniform(0, d / 2)
class Worker:
def __init__(self) -> None:
self.stop = threading.Event()
self.name = settings.instance_id
self.pool = ConnectionPool(
conninfo=settings.database_url, min_size=1,
max_size=settings.worker_concurrency + 3, timeout=settings.db_pool_timeout,
kwargs={"connect_timeout": settings.db_connect_timeout,
"options": f"-c statement_timeout={settings.db_statement_timeout_ms}",
"application_name": "messenger-worker"},
check=ConnectionPool.check_connection, open=False)
self.r = broker.sync_client(read_timeout=settings.broker_timeout + 5)
self.mp = multiprocessing.get_context("forkserver")
self.mp.set_forkserver_preload(["app.imaging"])
self.work_root = Path(tempfile.gettempdir()) / "worker-jobs"
# ================================================================ запуск
def run(self) -> None:
signal.signal(signal.SIGTERM, lambda *_: self.stop.set())
signal.signal(signal.SIGINT, lambda *_: self.stop.set())
self.pool.open(wait=False)
storage.init()
shutil.rmtree(self.work_root, ignore_errors=True) # мусор прошлой жизни контейнера
self.work_root.mkdir(parents=True, exist_ok=True)
threads = [threading.Thread(target=self.maintenance_loop, name="maintenance", daemon=True)]
for i in range(settings.worker_concurrency):
threads.append(threading.Thread(target=self.consume_loop, args=(f"{self.name}-{i}",),
name=f"consumer-{i}", daemon=True))
for t in threads:
t.start()
log.info("worker %s started: concurrency=%d max_attempts=%d timeout=%ss lease=%ss",
self.name, settings.worker_concurrency, settings.job_max_attempts,
settings.job_timeout_seconds, settings.job_lease_seconds)
while not self.stop.is_set():
self.stop.wait(1)
log.info("worker %s stopping: finishing current jobs", self.name)
for t in threads[1:]:
t.join(timeout=settings.job_timeout_seconds + 10)
self.pool.close()
# ================================================================ потребитель
def consume_loop(self, consumer: str) -> None:
"""Ограниченная конкуренция: каждый поток берёт по ОДНОМУ сообщению (COUNT 1),
потоков WORKER_CONCURRENCY → не больше N задач на процесс (аналог prefetch)."""
last_scan = 0.0
while not self.stop.is_set():
try:
self._ensure_group()
batch = []
if time.monotonic() - last_scan > 10:
last_scan = time.monotonic()
# 1) свои неподтверждённые (worker перезапустился в том же контейнере);
resp = self.r.xreadgroup(settings.jobs_group, consumer,
{settings.jobs_stream: "0"}, count=10)
batch += [m for _, msgs in resp for m in msgs]
# 2) брошенные другими (их процесс умер до XACK).
claimed = self.r.xautoclaim(settings.jobs_stream, settings.jobs_group, consumer,
min_idle_time=int(settings.claim_idle_seconds * 1000),
start_id="0-0", count=5)
batch += [m for m in claimed[1] if m]
if not batch:
resp = self.r.xreadgroup(settings.jobs_group, consumer,
{settings.jobs_stream: ">"}, count=1, block=2000)
batch = [m for _, msgs in resp for m in msgs]
for msg_id, fields in batch:
if self.stop.is_set():
break
if self.handle(consumer, msg_id, fields) == "ack":
self._ack(msg_id)
except redis.ResponseError as exc:
if "NOGROUP" in str(exc): # брокер пересоздан без данных — создаём группу снова
self._group_ok = False
log.warning("broker error (%s), retry in 1s", str(exc)[:80])
self.stop.wait(1)
except redis.RedisError as exc:
log.warning("broker unavailable (%s), retry in 1s", type(exc).__name__)
self.stop.wait(1)
except Exception: # noqa: BLE001 — поток не должен умирать
log.exception("consumer loop error")
self.stop.wait(1)
def _ensure_group(self) -> None:
if getattr(self, "_group_ok", False):
return
broker.ensure_group(self.r)
self._group_ok = True
def _ack(self, msg_id: str) -> None:
# XACK = consumer ack: результат уже устойчиво зафиксирован в БД и S3.
self.r.xack(settings.jobs_stream, settings.jobs_group, msg_id)
self.r.xdel(settings.jobs_stream, msg_id)
# ================================================================ одна задача
def handle(self, consumer: str, msg_id: str, fields: dict | None) -> str:
raw = (fields or {}).get("job")
if raw is None: # запись удалена из stream (XDEL) — подтверждаем
return "ack"
try:
payload = broker.parse(raw)
except broker.BadMessage as exc:
log.error("bad message %s → DLQ: %s", msg_id, exc)
broker.to_dlq(self.r, raw, f"bad_message: {exc}", msg_id)
return "ack"
try:
decision, job = self.claim(payload, consumer)
except Exception as exc: # noqa: BLE001 — БД недоступна: не ack, повтор позже
log.warning("claim failed for job %s: %s", payload["job_id"], type(exc).__name__)
self.stop.wait(1)
return "leave"
if decision != "process":
log.info("job=%s attempt=%s msg=%s → %s", payload["job_id"], payload["attempt"],
msg_id, decision)
if decision == "poison":
broker.to_dlq(self.r, raw, "unknown job", msg_id)
return "leave" if decision == "busy" else "ack"
return self.process(job, msg_id)
def claim(self, p: dict, consumer: str) -> tuple[str, Job | None]:
"""Таблица решений (сообщение несёт attempt):
job нет / не то вложение → poison (DLQ, ack)
ready → already_ready (no-op, ack): повторная доставка
failed → failed (ack); вернуть может только replay
attempt сообщения ≠ attempt job → stale (ack): устаревшая попытка
queued → берём в работу
processing, аренда истекла → takeover той же попытки
processing, аренда жива → busy: не ack, проверим позже (другой работает)"""
with self.pool.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT j.status, j.attempt, j.max_attempts, j.lease_until < now(), "
"j.pipeline_version, j.attachment_id, a.staging_key, a.object_key, a.sha256, "
"a.size_bytes, a.content_type "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
"WHERE j.job_id = %s FOR UPDATE OF j", (p["job_id"],)).fetchone()
if row is None or str(row[5]) != p["attachment_id"] or row[4] != p["pipeline_version"]:
return "poison", None
status, attempt, max_attempts, expired = row[0], row[1], row[2], row[3]
if status == "ready":
return "already_ready", None
if status == "failed":
return "failed", None
if p["attempt"] != attempt:
return "stale", None
if status == "processing" and not expired:
return "busy", None
event = "lease_takeover" if status == "processing" else "claimed"
token = str(uuid.uuid4())
conn.execute(
"UPDATE attachment_jobs SET status = 'processing', lease_token = %s, lease_owner = %s, "
"lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s",
(token, consumer, settings.job_lease_seconds, p["job_id"]))
conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() "
"WHERE id = %s AND status <> 'ready'", (p["attachment_id"],))
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, %s, %s, %s)",
(p["job_id"], event, attempt, consumer))
return "process", Job(p["job_id"], p["attachment_id"], row[4], attempt, max_attempts, token,
row[6], row[7], row[8], row[9], row[10])
def process(self, job: Job, msg_id: str) -> str:
log.info("job=%s attachment=%s attempt=%d/%d msg=%s: processing", job.job_id,
job.attachment_id, job.attempt, job.max_attempts, msg_id)
lost = threading.Event()
frozen = threading.Event()
done = threading.Event()
hb = threading.Thread(target=self._heartbeat, args=(job, lost, frozen, done), daemon=True)
hb.start()
workdir = Path(tempfile.mkdtemp(prefix=f"{job.attachment_id}-", dir=self.work_root))
try:
if self._fault("freeze-after-claim"):
# Имитация зависшего worker: аренду не продлеваем, «спим» дольше аренды.
frozen.set()
log.warning("FAULT INJECTION: freeze-after-claim, sleeping %ss", 3 * settings.job_lease_seconds)
time.sleep(3 * settings.job_lease_seconds)
self._check_lease(job)
variants = self._build(job, workdir, lost)
self._check_lease(job) # не пишем в S3, если задачу уже забрали
self._publish_objects(job, variants)
if self._fault("crash-after-s3-write"):
log.warning("FAULT INJECTION: crash-after-s3-write (before COMMIT and ack)")
os._exit(1)
self._finalize(job, variants)
if self._fault("crash-after-commit-before-ack"):
log.warning("FAULT INJECTION: crash-after-commit-before-ack")
os._exit(1)
log.info("job=%s attachment=%s attempt=%d: ready", job.job_id, job.attachment_id, job.attempt)
done.set()
self._ack(msg_id)
self._drop_staging(job)
return "acked"
except LeaseLost:
log.warning("job=%s attempt=%d: lease lost, result discarded (another worker owns it)",
job.job_id, job.attempt)
return "ack"
except (Transient, Permanent) as exc:
return self._failure(job, exc, msg_id)
except storage.StorageUnavailable as exc:
return self._failure(job, Transient("storage_unavailable", type(exc).__name__), msg_id)
except storage.ObjectMissing:
return self._failure(job, Permanent("processing_failed", "staging object is missing"), msg_id)
except psycopg.Error as exc:
return self._failure(job, Transient("database_unavailable", type(exc).__name__), msg_id)
finally:
done.set()
hb.join(timeout=2)
shutil.rmtree(workdir, ignore_errors=True)
def _heartbeat(self, job: Job, lost: threading.Event, frozen: threading.Event,
done: threading.Event) -> None:
"""Живой worker продлевает аренду каждые lease/3 с — так «долгая» задача
отличается от «зависшей»: у долгой аренда свежая, у мёртвой/замёрзшей — истекает."""
interval = settings.job_lease_seconds / 3
while not done.wait(interval):
if frozen.is_set():
continue
try:
with self.pool.connection() as conn:
ok = conn.execute(
"UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), "
"updated_at = now() WHERE job_id = %s AND lease_token = %s "
"AND status = 'processing' RETURNING 1",
(settings.job_lease_seconds, job.job_id, job.token)).fetchone()
if ok is None:
lost.set()
return
except psycopg.Error as exc:
log.warning("lease renew failed: %s", type(exc).__name__)
# ---------------------------------------------------------------- шаги
def _check_lease(self, job: Job) -> None:
with self.pool.connection() as conn:
row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s",
(job.job_id,)).fetchone()
if row is None or row[0] != "processing" or str(row[1]) != job.token:
raise LeaseLost()
def _build(self, job: Job, workdir: Path, lost: threading.Event) -> dict:
if not job.staging_key:
raise Permanent("processing_failed", "staging source is gone")
if self._fault("transient-error", counter=True):
raise Transient("injected_transient", "fault injection")
src = workdir / "source"
with open(src, "wb") as f:
try:
storage.download_sync(job.staging_key, f, settings.max_upload_bytes)
except storage.ObjectMissing:
raise Permanent("processing_failed", "staging object is missing")
except ValueError:
raise Permanent("invalid_image", "source larger than limit")
h = hashlib.sha256()
with open(src, "rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
h.update(chunk)
if h.hexdigest() != job.sha256:
raise Permanent("processing_failed", "source checksum mismatch")
out = workdir / "out"
out.mkdir()
result_path = workdir / "result.json"
slow = float(self.r.get("fault:slow-seconds") or 0) if settings.fault_injection else 0.0
proc = self.mp.Process(target=imaging.child_main, args=(
str(src), str(out), job.content_type, settings.max_image_pixels,
settings.image_memory_mb, int(settings.job_timeout_seconds) + 5, str(result_path), slow))
proc.start()
deadline = time.monotonic() + settings.job_timeout_seconds
while proc.is_alive() and time.monotonic() < deadline and not lost.is_set():
proc.join(timeout=0.5)
if proc.is_alive():
proc.kill()
proc.join()
if lost.is_set():
raise LeaseLost()
raise Transient("processing_timeout", f"> {settings.job_timeout_seconds}s")
if lost.is_set():
raise LeaseLost()
if not result_path.exists():
raise Transient("processing_crashed", f"exit code {proc.exitcode}")
result = json.loads(result_path.read_text(encoding="utf-8"))
if not result["ok"]:
if result.get("trace"):
log.warning("job=%s child error: %s", job.job_id, result["trace"])
exc = Permanent if result["permanent"] else Transient
raise exc(result["code"], result.get("detail", ""))
variants = {}
for name in imaging.VARIANTS:
v = result["variants"][name]
data = Path(v["path"]).read_bytes()
v["size_bytes"] = len(data)
v["sha256"] = hashlib.sha256(data).hexdigest()
v["object_key"] = storage.variant_key(job.attachment_id, job.pipeline_version, name,
imaging.VARIANT_EXT[name])
variants[name] = v
return variants
def _publish_objects(self, job: Job, variants: dict) -> None:
"""S3-эффекты идемпотентны: те же ключи и те же байты при повторе.
Оригинал копируется на стороне S3 (staging → originals), без скачивания."""
storage.copy_sync(job.staging_key, job.object_key)
for name, v in variants.items():
storage.put_file_sync(v["object_key"], v["path"], v["content_type"], v["sha256"])
def _finalize(self, job: Job, variants: dict) -> None:
"""Атомарная публикация результата. Только владелец аренды (lease_token);
ready не откатывается; ключ эффекта (attachment, pipeline_version, variant)."""
with self.pool.connection() as conn, conn.transaction():
row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s "
"FOR UPDATE", (job.job_id,)).fetchone()
if row is None or row[0] != "processing" or str(row[1]) != job.token:
raise LeaseLost()
for name, v in variants.items():
conn.execute(
"INSERT INTO attachment_variants (attachment_id, pipeline_version, variant, "
"object_key, content_type, width, height, size_bytes, sha256, created_at) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) "
"ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING",
(job.attachment_id, job.pipeline_version, name, v["object_key"],
v["content_type"], v["width"], v["height"], v["size_bytes"], v["sha256"]))
conn.execute(
"UPDATE attachment_jobs SET status = 'ready', lease_token = NULL, lease_owner = NULL, "
"lease_until = NULL, error_code = NULL, finished_at = now(), updated_at = now() "
"WHERE job_id = %s", (job.job_id,))
conn.execute("UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now() "
"WHERE id = %s", (job.attachment_id,))
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) "
"VALUES (%s, 'ready', %s, %s)", (job.job_id, job.attempt, self.name))
def _drop_staging(self, job: Job) -> None:
"""Staging удаляется только ПОСЛЕ COMMIT ready и ack. Не вышло — подчистит maintenance."""
try:
storage.delete_sync(job.staging_key)
with self.pool.connection() as conn:
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'",
(job.attachment_id,))
except Exception as exc: # noqa: BLE001
log.info("staging cleanup deferred: %s", type(exc).__name__)
def _failure(self, job: Job, exc: Transient | Permanent, msg_id: str) -> str:
permanent = isinstance(exc, Permanent)
log.warning("job=%s attempt=%d/%d %s error: %s %s", job.job_id, job.attempt, job.max_attempts,
"permanent" if permanent else "transient", exc.code, exc.detail)
try:
outcome = schedule_retry_or_fail(self.pool, job.job_id, job.token, exc.code, exc.detail,
permanent, self.name)
except psycopg.Error as db_exc:
log.warning("cannot record failure (%s): leave message pending", type(db_exc).__name__)
return "leave"
if outcome == "failed":
try:
broker.to_dlq(self.r, broker.dumps({"job_id": job.job_id, "attachment_id": job.attachment_id}),
exc.code if permanent else f"retry_exhausted ({exc.code})", msg_id)
except redis.RedisError:
pass # реестр failed — в БД; DLQ-stream лишь копия для оператора
return "ack"
# ---------------------------------------------------------------- fault injection
def _fault(self, point: str, counter: bool = False) -> bool:
"""Только при FAULT_INJECTION=true (сервис worker-fault). Флаг ставит тест:
redis-cli SET fault:<point> 1 (одноразовый), для counter — число срабатываний."""
if not settings.fault_injection:
return False
key = f"fault:{point}"
try:
if counter:
left = self.r.decr(key)
if left < 0:
self.r.delete(key)
return False
return True
return self.r.getdel(key) is not None
except redis.RedisError:
return False
# ================================================================ maintenance
def maintenance_loop(self) -> None:
while not self.stop.is_set():
Path(settings.heartbeat_file).touch()
for step in (self.relay_outbox, self.reap_leases, self.sweep_staging):
try:
step()
except Exception as exc: # noqa: BLE001
log.warning("maintenance %s failed: %s", step.__name__, type(exc).__name__)
self.stop.wait(settings.maintenance_interval_seconds)
def relay_outbox(self) -> None:
"""Transactional outbox → брокер. Публикует строки, которые API не отметил
(упал между COMMIT и publish, брокер был недоступен) и отложенные retry.
SKIP LOCKED: несколько worker не публикуют одну строку одновременно.
Упали после XADD, до COMMIT → строка опубликуется ещё раз: дубль безопасен."""
with self.pool.connection() as conn, conn.transaction():
rows = conn.execute(
"SELECT id, payload FROM job_outbox WHERE published_at IS NULL "
"AND available_at <= now() - make_interval(secs => %s) "
"ORDER BY available_at, id LIMIT 100 FOR UPDATE SKIP LOCKED",
(settings.outbox_relay_delay_seconds,)).fetchall()
for oid, payload in rows:
try:
stream_id = broker.publish_sync(self.r, payload)
except broker.BrokerUnavailable as exc:
conn.execute("UPDATE job_outbox SET publish_attempts = publish_attempts + 1, "
"last_error = %s WHERE id = %s", (str(exc)[:100], oid))
break
conn.execute("UPDATE job_outbox SET published_at = now(), stream_id = %s, "
"publish_attempts = publish_attempts + 1 WHERE id = %s", (stream_id, oid))
log.info("relay: published outbox=%s job=%s attempt=%s", oid, payload["job_id"],
payload["attempt"])
def reap_leases(self) -> None:
"""Аренда истекла и за ещё одну аренду никто не перехватил (redelivery) →
worker мёртв: новая управляемая попытка через outbox (или failed, если попыток нет)."""
with self.pool.connection() as conn:
ids = [r[0] for r in conn.execute(
"SELECT job_id FROM attachment_jobs WHERE status = 'processing' "
"AND lease_until < now() - make_interval(secs => %s) LIMIT 20",
(settings.job_lease_seconds,)).fetchall()]
for job_id in ids:
outcome = schedule_retry_or_fail(self.pool, job_id, None, "lease_expired",
"worker stopped renewing lease", False, "reaper")
log.warning("reaper: job=%s lease expired → %s", job_id, outcome)
def sweep_staging(self) -> None:
"""Очистка staging: ready — сразу после фиксации (страховка, если worker не успел);
failed — после STAGING_RETENTION_SECONDS (до этого исходник нужен для replay)."""
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND ("
" (status = 'ready' AND updated_at < now() - interval '30 seconds') OR"
" (status = 'failed' AND updated_at < now() - make_interval(secs => %s))) LIMIT 50",
(settings.staging_retention_seconds,)).fetchall()
for aid, key in rows:
storage.delete_sync(key)
with self.pool.connection() as conn:
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s "
"AND status IN ('ready', 'failed')", (aid,))
log.info("staging cleaned for attachment %s", aid)
def schedule_retry_or_fail(pool: ConnectionPool, job_id, token: str | None, code: str, detail: str,
permanent: bool, actor: str) -> str:
"""processing → queued (новая попытка через outbox с backoff) или → failed.
token задан — проверяем владельца аренды; None — вызывает reaper (аренда уже истекла)."""
with pool.connection() as conn, conn.transaction():
row = conn.execute(
"SELECT status, attempt, max_attempts, lease_token, attachment_id, pipeline_version, "
"lease_until < now() - make_interval(secs => %s) FROM attachment_jobs "
"WHERE job_id = %s FOR UPDATE SKIP LOCKED", (settings.job_lease_seconds, job_id)).fetchone()
if row is None or row[0] != "processing":
return "skipped"
status, attempt, max_attempts, lease_token, attachment_id, pv, long_expired = row
if token is not None and str(lease_token) != token:
return "skipped" # аренду уже забрали
if token is None and not long_expired:
return "skipped"
if permanent or attempt >= max_attempts:
final = code if permanent else "retry_exhausted"
conn.execute(
"UPDATE attachment_jobs SET status = 'failed', error_code = %s, last_error = %s, "
"lease_token = NULL, lease_owner = NULL, lease_until = NULL, finished_at = now(), "
"updated_at = now() WHERE job_id = %s", (final, f"{code}: {detail}"[:300], job_id))
conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() "
"WHERE id = %s", (final, attachment_id))
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
"VALUES (%s, 'failed', %s, %s, %s)", (job_id, attempt, actor, f"{code}: {detail}"[:300]))
return "failed"
delay = backoff_seconds(attempt)
nxt = attempt + 1
conn.execute(
"UPDATE attachment_jobs SET status = 'queued', attempt = %s, last_error = %s, "
"next_attempt_at = now() + make_interval(secs => %s), lease_token = NULL, "
"lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE job_id = %s",
(nxt, f"{code}: {detail}"[:300], delay, job_id))
conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s",
(attachment_id,))
created = conn.execute("SELECT now()").fetchone()[0]
payload = broker.make_payload(job_id, attachment_id, pv, nxt, created)
conn.execute(
"INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
"VALUES (%s, %s, %s, now() + make_interval(secs => %s), now())",
(job_id, nxt, json.dumps(payload), delay))
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
"VALUES (%s, 'retry_scheduled', %s, %s, %s)",
(job_id, attempt, actor, f"{code}; next attempt {nxt} in {delay:.1f}s"))
return f"retry #{nxt} in {delay:.1f}s"
def main() -> None:
logging.basicConfig(level=settings.log_level.upper(),
format="%(asctime)s %(levelname)s %(name)s %(message)s")
logging.getLogger("botocore").setLevel(logging.WARNING)
Worker().run()
if __name__ == "__main__":
main()