""" Worker фоновой обработки вложений (lab5). Отдельный процесс/контейнер: `python -m app.worker`. Поток одной задачи (сообщение брокера = только ID, см. app/broker.py): XREADGROUP ─► claim в БД (FOR UPDATE): queued → processing, lease_token, lease_until ─► скачать оригинал из staging, сверить sha256 с БД (доверенные метаданные) ─► дочерний процесс: thumbnail / crop / optimized (лимиты памяти и времени) ─► S3: copy staging → originals/, PUT variants//v1/* (детерминированные ключи) ─► COMMIT результата (только владелец аренды): variants ON CONFLICT DO NOTHING, attachment и job → ready ─► XACK (+XDEL) ─► удалить staging (после безопасной фиксации) Ошибки: постоянные (битое изображение, sha256 не совпал, исходника нет) → failed + error_code; временные (S3/БД недоступны, таймаут обработки) → queued, attempt+1, backoff с jitter через outbox (available_at в будущем); попытки кончились → failed retry_exhausted. Отказы worker: умер до XACK → сообщение остаётся в PEL брокера; после рестарта worker читает свой PEL, чужой брошенный PEL забирает XAUTOCLAIM (CLAIM_IDLE_SECONDS); завис/умер с арендой → аренду никто не продлевает, после истечения задачу забирает другой (lease takeover) или reaper переводит её в queued с новой попыткой. Устаревший worker после пробуждения ничего не перезапишет: COMMIT проверяет lease_token. Поток maintenance (в каждом worker, безопасно параллельно — FOR UPDATE SKIP LOCKED): outbox relay, reaper просроченных аренд, очистка staging, heartbeat-файл для healthcheck. """ from __future__ import annotations import hashlib import json import logging import multiprocessing import os import random import shutil import signal import tempfile import threading import time import uuid from dataclasses import dataclass from pathlib import Path import psycopg import redis from psycopg_pool import ConnectionPool from . import broker, imaging, storage from .config import settings log = logging.getLogger("messenger.worker") class Transient(Exception): def __init__(self, code: str, detail: str = ""): super().__init__(code) self.code, self.detail = code, detail class Permanent(Exception): def __init__(self, code: str, detail: str = ""): super().__init__(code) self.code, self.detail = code, detail class LeaseLost(Exception): """Аренду забрали (истекла и задачу взял другой) — результат фиксировать нельзя.""" @dataclass class Job: job_id: str attachment_id: str pipeline_version: int attempt: int max_attempts: int token: str staging_key: str | None object_key: str sha256: str size_bytes: int content_type: str def backoff_seconds(attempt: int) -> float: """Экспоненциальный backoff с jitter («equal jitter»): половина задержки фиксирована, половина случайна — повторы разных задач не идут синхронной волной.""" d = min(settings.job_backoff_max_seconds, settings.job_backoff_base_seconds * 2 ** (attempt - 1)) return d / 2 + random.uniform(0, d / 2) class Worker: def __init__(self) -> None: self.stop = threading.Event() self.name = settings.instance_id self.pool = ConnectionPool( conninfo=settings.database_url, min_size=1, max_size=settings.worker_concurrency + 3, timeout=settings.db_pool_timeout, kwargs={"connect_timeout": settings.db_connect_timeout, "options": f"-c statement_timeout={settings.db_statement_timeout_ms}", "application_name": "messenger-worker"}, check=ConnectionPool.check_connection, open=False) self.r = broker.sync_client(read_timeout=settings.broker_timeout + 5) self.mp = multiprocessing.get_context("forkserver") self.mp.set_forkserver_preload(["app.imaging"]) self.work_root = Path(tempfile.gettempdir()) / "worker-jobs" # ================================================================ запуск def run(self) -> None: signal.signal(signal.SIGTERM, lambda *_: self.stop.set()) signal.signal(signal.SIGINT, lambda *_: self.stop.set()) self.pool.open(wait=False) storage.init() shutil.rmtree(self.work_root, ignore_errors=True) # мусор прошлой жизни контейнера self.work_root.mkdir(parents=True, exist_ok=True) threads = [threading.Thread(target=self.maintenance_loop, name="maintenance", daemon=True)] for i in range(settings.worker_concurrency): threads.append(threading.Thread(target=self.consume_loop, args=(f"{self.name}-{i}",), name=f"consumer-{i}", daemon=True)) for t in threads: t.start() log.info("worker %s started: concurrency=%d max_attempts=%d timeout=%ss lease=%ss", self.name, settings.worker_concurrency, settings.job_max_attempts, settings.job_timeout_seconds, settings.job_lease_seconds) while not self.stop.is_set(): self.stop.wait(1) log.info("worker %s stopping: finishing current jobs", self.name) for t in threads[1:]: t.join(timeout=settings.job_timeout_seconds + 10) self.pool.close() # ================================================================ потребитель def consume_loop(self, consumer: str) -> None: """Ограниченная конкуренция: каждый поток берёт по ОДНОМУ сообщению (COUNT 1), потоков WORKER_CONCURRENCY → не больше N задач на процесс (аналог prefetch).""" last_scan = 0.0 while not self.stop.is_set(): try: self._ensure_group() batch = [] if time.monotonic() - last_scan > 10: last_scan = time.monotonic() # 1) свои неподтверждённые (worker перезапустился в том же контейнере); resp = self.r.xreadgroup(settings.jobs_group, consumer, {settings.jobs_stream: "0"}, count=10) batch += [m for _, msgs in resp for m in msgs] # 2) брошенные другими (их процесс умер до XACK). claimed = self.r.xautoclaim(settings.jobs_stream, settings.jobs_group, consumer, min_idle_time=int(settings.claim_idle_seconds * 1000), start_id="0-0", count=5) batch += [m for m in claimed[1] if m] if not batch: resp = self.r.xreadgroup(settings.jobs_group, consumer, {settings.jobs_stream: ">"}, count=1, block=2000) batch = [m for _, msgs in resp for m in msgs] for msg_id, fields in batch: if self.stop.is_set(): break if self.handle(consumer, msg_id, fields) == "ack": self._ack(msg_id) except redis.ResponseError as exc: if "NOGROUP" in str(exc): # брокер пересоздан без данных — создаём группу снова self._group_ok = False log.warning("broker error (%s), retry in 1s", str(exc)[:80]) self.stop.wait(1) except redis.RedisError as exc: log.warning("broker unavailable (%s), retry in 1s", type(exc).__name__) self.stop.wait(1) except Exception: # noqa: BLE001 — поток не должен умирать log.exception("consumer loop error") self.stop.wait(1) def _ensure_group(self) -> None: if getattr(self, "_group_ok", False): return broker.ensure_group(self.r) self._group_ok = True def _ack(self, msg_id: str) -> None: # XACK = consumer ack: результат уже устойчиво зафиксирован в БД и S3. self.r.xack(settings.jobs_stream, settings.jobs_group, msg_id) self.r.xdel(settings.jobs_stream, msg_id) # ================================================================ одна задача def handle(self, consumer: str, msg_id: str, fields: dict | None) -> str: raw = (fields or {}).get("job") if raw is None: # запись удалена из stream (XDEL) — подтверждаем return "ack" try: payload = broker.parse(raw) except broker.BadMessage as exc: log.error("bad message %s → DLQ: %s", msg_id, exc) broker.to_dlq(self.r, raw, f"bad_message: {exc}", msg_id) return "ack" try: decision, job = self.claim(payload, consumer) except Exception as exc: # noqa: BLE001 — БД недоступна: не ack, повтор позже log.warning("claim failed for job %s: %s", payload["job_id"], type(exc).__name__) self.stop.wait(1) return "leave" if decision != "process": log.info("job=%s attempt=%s msg=%s → %s", payload["job_id"], payload["attempt"], msg_id, decision) if decision == "poison": broker.to_dlq(self.r, raw, "unknown job", msg_id) return "leave" if decision == "busy" else "ack" return self.process(job, msg_id) def claim(self, p: dict, consumer: str) -> tuple[str, Job | None]: """Таблица решений (сообщение несёт attempt): job нет / не то вложение → poison (DLQ, ack) ready → already_ready (no-op, ack): повторная доставка failed → failed (ack); вернуть может только replay attempt сообщения ≠ attempt job → stale (ack): устаревшая попытка queued → берём в работу processing, аренда истекла → takeover той же попытки processing, аренда жива → busy: не ack, проверим позже (другой работает)""" with self.pool.connection() as conn, conn.transaction(): row = conn.execute( "SELECT j.status, j.attempt, j.max_attempts, j.lease_until < now(), " "j.pipeline_version, j.attachment_id, a.staging_key, a.object_key, a.sha256, " "a.size_bytes, a.content_type " "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id " "WHERE j.job_id = %s FOR UPDATE OF j", (p["job_id"],)).fetchone() if row is None or str(row[5]) != p["attachment_id"] or row[4] != p["pipeline_version"]: return "poison", None status, attempt, max_attempts, expired = row[0], row[1], row[2], row[3] if status == "ready": return "already_ready", None if status == "failed": return "failed", None if p["attempt"] != attempt: return "stale", None if status == "processing" and not expired: return "busy", None event = "lease_takeover" if status == "processing" else "claimed" token = str(uuid.uuid4()) conn.execute( "UPDATE attachment_jobs SET status = 'processing', lease_token = %s, lease_owner = %s, " "lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s", (token, consumer, settings.job_lease_seconds, p["job_id"])) conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() " "WHERE id = %s AND status <> 'ready'", (p["attachment_id"],)) conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, %s, %s, %s)", (p["job_id"], event, attempt, consumer)) return "process", Job(p["job_id"], p["attachment_id"], row[4], attempt, max_attempts, token, row[6], row[7], row[8], row[9], row[10]) def process(self, job: Job, msg_id: str) -> str: log.info("job=%s attachment=%s attempt=%d/%d msg=%s: processing", job.job_id, job.attachment_id, job.attempt, job.max_attempts, msg_id) lost = threading.Event() frozen = threading.Event() done = threading.Event() hb = threading.Thread(target=self._heartbeat, args=(job, lost, frozen, done), daemon=True) hb.start() workdir = Path(tempfile.mkdtemp(prefix=f"{job.attachment_id}-", dir=self.work_root)) try: if self._fault("freeze-after-claim"): # Имитация зависшего worker: аренду не продлеваем, «спим» дольше аренды. frozen.set() log.warning("FAULT INJECTION: freeze-after-claim, sleeping %ss", 3 * settings.job_lease_seconds) time.sleep(3 * settings.job_lease_seconds) self._check_lease(job) variants = self._build(job, workdir, lost) self._check_lease(job) # не пишем в S3, если задачу уже забрали self._publish_objects(job, variants) if self._fault("crash-after-s3-write"): log.warning("FAULT INJECTION: crash-after-s3-write (before COMMIT and ack)") os._exit(1) self._finalize(job, variants) if self._fault("crash-after-commit-before-ack"): log.warning("FAULT INJECTION: crash-after-commit-before-ack") os._exit(1) log.info("job=%s attachment=%s attempt=%d: ready", job.job_id, job.attachment_id, job.attempt) done.set() self._ack(msg_id) self._drop_staging(job) return "acked" except LeaseLost: log.warning("job=%s attempt=%d: lease lost, result discarded (another worker owns it)", job.job_id, job.attempt) return "ack" except (Transient, Permanent) as exc: return self._failure(job, exc, msg_id) except storage.StorageUnavailable as exc: return self._failure(job, Transient("storage_unavailable", type(exc).__name__), msg_id) except storage.ObjectMissing: return self._failure(job, Permanent("processing_failed", "staging object is missing"), msg_id) except psycopg.Error as exc: return self._failure(job, Transient("database_unavailable", type(exc).__name__), msg_id) finally: done.set() hb.join(timeout=2) shutil.rmtree(workdir, ignore_errors=True) def _heartbeat(self, job: Job, lost: threading.Event, frozen: threading.Event, done: threading.Event) -> None: """Живой worker продлевает аренду каждые lease/3 с — так «долгая» задача отличается от «зависшей»: у долгой аренда свежая, у мёртвой/замёрзшей — истекает.""" interval = settings.job_lease_seconds / 3 while not done.wait(interval): if frozen.is_set(): continue try: with self.pool.connection() as conn: ok = conn.execute( "UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), " "updated_at = now() WHERE job_id = %s AND lease_token = %s " "AND status = 'processing' RETURNING 1", (settings.job_lease_seconds, job.job_id, job.token)).fetchone() if ok is None: lost.set() return except psycopg.Error as exc: log.warning("lease renew failed: %s", type(exc).__name__) # ---------------------------------------------------------------- шаги def _check_lease(self, job: Job) -> None: with self.pool.connection() as conn: row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s", (job.job_id,)).fetchone() if row is None or row[0] != "processing" or str(row[1]) != job.token: raise LeaseLost() def _build(self, job: Job, workdir: Path, lost: threading.Event) -> dict: if not job.staging_key: raise Permanent("processing_failed", "staging source is gone") if self._fault("transient-error", counter=True): raise Transient("injected_transient", "fault injection") src = workdir / "source" with open(src, "wb") as f: try: storage.download_sync(job.staging_key, f, settings.max_upload_bytes) except storage.ObjectMissing: raise Permanent("processing_failed", "staging object is missing") except ValueError: raise Permanent("invalid_image", "source larger than limit") h = hashlib.sha256() with open(src, "rb") as f: for chunk in iter(lambda: f.read(1024 * 1024), b""): h.update(chunk) if h.hexdigest() != job.sha256: raise Permanent("processing_failed", "source checksum mismatch") out = workdir / "out" out.mkdir() result_path = workdir / "result.json" slow = float(self.r.get("fault:slow-seconds") or 0) if settings.fault_injection else 0.0 proc = self.mp.Process(target=imaging.child_main, args=( str(src), str(out), job.content_type, settings.max_image_pixels, settings.image_memory_mb, int(settings.job_timeout_seconds) + 5, str(result_path), slow)) proc.start() deadline = time.monotonic() + settings.job_timeout_seconds while proc.is_alive() and time.monotonic() < deadline and not lost.is_set(): proc.join(timeout=0.5) if proc.is_alive(): proc.kill() proc.join() if lost.is_set(): raise LeaseLost() raise Transient("processing_timeout", f"> {settings.job_timeout_seconds}s") if lost.is_set(): raise LeaseLost() if not result_path.exists(): raise Transient("processing_crashed", f"exit code {proc.exitcode}") result = json.loads(result_path.read_text(encoding="utf-8")) if not result["ok"]: if result.get("trace"): log.warning("job=%s child error: %s", job.job_id, result["trace"]) exc = Permanent if result["permanent"] else Transient raise exc(result["code"], result.get("detail", "")) variants = {} for name in imaging.VARIANTS: v = result["variants"][name] data = Path(v["path"]).read_bytes() v["size_bytes"] = len(data) v["sha256"] = hashlib.sha256(data).hexdigest() v["object_key"] = storage.variant_key(job.attachment_id, job.pipeline_version, name, imaging.VARIANT_EXT[name]) variants[name] = v return variants def _publish_objects(self, job: Job, variants: dict) -> None: """S3-эффекты идемпотентны: те же ключи и те же байты при повторе. Оригинал копируется на стороне S3 (staging → originals), без скачивания.""" storage.copy_sync(job.staging_key, job.object_key) for name, v in variants.items(): storage.put_file_sync(v["object_key"], v["path"], v["content_type"], v["sha256"]) def _finalize(self, job: Job, variants: dict) -> None: """Атомарная публикация результата. Только владелец аренды (lease_token); ready не откатывается; ключ эффекта (attachment, pipeline_version, variant).""" with self.pool.connection() as conn, conn.transaction(): row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s " "FOR UPDATE", (job.job_id,)).fetchone() if row is None or row[0] != "processing" or str(row[1]) != job.token: raise LeaseLost() for name, v in variants.items(): conn.execute( "INSERT INTO attachment_variants (attachment_id, pipeline_version, variant, " "object_key, content_type, width, height, size_bytes, sha256, created_at) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) " "ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING", (job.attachment_id, job.pipeline_version, name, v["object_key"], v["content_type"], v["width"], v["height"], v["size_bytes"], v["sha256"])) conn.execute( "UPDATE attachment_jobs SET status = 'ready', lease_token = NULL, lease_owner = NULL, " "lease_until = NULL, error_code = NULL, finished_at = now(), updated_at = now() " "WHERE job_id = %s", (job.job_id,)) conn.execute("UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now() " "WHERE id = %s", (job.attachment_id,)) conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) " "VALUES (%s, 'ready', %s, %s)", (job.job_id, job.attempt, self.name)) def _drop_staging(self, job: Job) -> None: """Staging удаляется только ПОСЛЕ COMMIT ready и ack. Не вышло — подчистит maintenance.""" try: storage.delete_sync(job.staging_key) with self.pool.connection() as conn: conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'", (job.attachment_id,)) except Exception as exc: # noqa: BLE001 log.info("staging cleanup deferred: %s", type(exc).__name__) def _failure(self, job: Job, exc: Transient | Permanent, msg_id: str) -> str: permanent = isinstance(exc, Permanent) log.warning("job=%s attempt=%d/%d %s error: %s %s", job.job_id, job.attempt, job.max_attempts, "permanent" if permanent else "transient", exc.code, exc.detail) try: outcome = schedule_retry_or_fail(self.pool, job.job_id, job.token, exc.code, exc.detail, permanent, self.name) except psycopg.Error as db_exc: log.warning("cannot record failure (%s): leave message pending", type(db_exc).__name__) return "leave" if outcome == "failed": try: broker.to_dlq(self.r, broker.dumps({"job_id": job.job_id, "attachment_id": job.attachment_id}), exc.code if permanent else f"retry_exhausted ({exc.code})", msg_id) except redis.RedisError: pass # реестр failed — в БД; DLQ-stream лишь копия для оператора return "ack" # ---------------------------------------------------------------- fault injection def _fault(self, point: str, counter: bool = False) -> bool: """Только при FAULT_INJECTION=true (сервис worker-fault). Флаг ставит тест: redis-cli SET fault: 1 (одноразовый), для counter — число срабатываний.""" if not settings.fault_injection: return False key = f"fault:{point}" try: if counter: left = self.r.decr(key) if left < 0: self.r.delete(key) return False return True return self.r.getdel(key) is not None except redis.RedisError: return False # ================================================================ maintenance def maintenance_loop(self) -> None: while not self.stop.is_set(): Path(settings.heartbeat_file).touch() for step in (self.relay_outbox, self.reap_leases, self.sweep_staging): try: step() except Exception as exc: # noqa: BLE001 log.warning("maintenance %s failed: %s", step.__name__, type(exc).__name__) self.stop.wait(settings.maintenance_interval_seconds) def relay_outbox(self) -> None: """Transactional outbox → брокер. Публикует строки, которые API не отметил (упал между COMMIT и publish, брокер был недоступен) и отложенные retry. SKIP LOCKED: несколько worker не публикуют одну строку одновременно. Упали после XADD, до COMMIT → строка опубликуется ещё раз: дубль безопасен.""" with self.pool.connection() as conn, conn.transaction(): rows = conn.execute( "SELECT id, payload FROM job_outbox WHERE published_at IS NULL " "AND available_at <= now() - make_interval(secs => %s) " "ORDER BY available_at, id LIMIT 100 FOR UPDATE SKIP LOCKED", (settings.outbox_relay_delay_seconds,)).fetchall() for oid, payload in rows: try: stream_id = broker.publish_sync(self.r, payload) except broker.BrokerUnavailable as exc: conn.execute("UPDATE job_outbox SET publish_attempts = publish_attempts + 1, " "last_error = %s WHERE id = %s", (str(exc)[:100], oid)) break conn.execute("UPDATE job_outbox SET published_at = now(), stream_id = %s, " "publish_attempts = publish_attempts + 1 WHERE id = %s", (stream_id, oid)) log.info("relay: published outbox=%s job=%s attempt=%s", oid, payload["job_id"], payload["attempt"]) def reap_leases(self) -> None: """Аренда истекла и за ещё одну аренду никто не перехватил (redelivery) → worker мёртв: новая управляемая попытка через outbox (или failed, если попыток нет).""" with self.pool.connection() as conn: ids = [r[0] for r in conn.execute( "SELECT job_id FROM attachment_jobs WHERE status = 'processing' " "AND lease_until < now() - make_interval(secs => %s) LIMIT 20", (settings.job_lease_seconds,)).fetchall()] for job_id in ids: outcome = schedule_retry_or_fail(self.pool, job_id, None, "lease_expired", "worker stopped renewing lease", False, "reaper") log.warning("reaper: job=%s lease expired → %s", job_id, outcome) def sweep_staging(self) -> None: """Очистка staging: ready — сразу после фиксации (страховка, если worker не успел); failed — после STAGING_RETENTION_SECONDS (до этого исходник нужен для replay).""" with self.pool.connection() as conn: rows = conn.execute( "SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (" " (status = 'ready' AND updated_at < now() - interval '30 seconds') OR" " (status = 'failed' AND updated_at < now() - make_interval(secs => %s))) LIMIT 50", (settings.staging_retention_seconds,)).fetchall() for aid, key in rows: storage.delete_sync(key) with self.pool.connection() as conn: conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s " "AND status IN ('ready', 'failed')", (aid,)) log.info("staging cleaned for attachment %s", aid) def schedule_retry_or_fail(pool: ConnectionPool, job_id, token: str | None, code: str, detail: str, permanent: bool, actor: str) -> str: """processing → queued (новая попытка через outbox с backoff) или → failed. token задан — проверяем владельца аренды; None — вызывает reaper (аренда уже истекла).""" with pool.connection() as conn, conn.transaction(): row = conn.execute( "SELECT status, attempt, max_attempts, lease_token, attachment_id, pipeline_version, " "lease_until < now() - make_interval(secs => %s) FROM attachment_jobs " "WHERE job_id = %s FOR UPDATE SKIP LOCKED", (settings.job_lease_seconds, job_id)).fetchone() if row is None or row[0] != "processing": return "skipped" status, attempt, max_attempts, lease_token, attachment_id, pv, long_expired = row if token is not None and str(lease_token) != token: return "skipped" # аренду уже забрали if token is None and not long_expired: return "skipped" if permanent or attempt >= max_attempts: final = code if permanent else "retry_exhausted" conn.execute( "UPDATE attachment_jobs SET status = 'failed', error_code = %s, last_error = %s, " "lease_token = NULL, lease_owner = NULL, lease_until = NULL, finished_at = now(), " "updated_at = now() WHERE job_id = %s", (final, f"{code}: {detail}"[:300], job_id)) conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() " "WHERE id = %s", (final, attachment_id)) conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) " "VALUES (%s, 'failed', %s, %s, %s)", (job_id, attempt, actor, f"{code}: {detail}"[:300])) return "failed" delay = backoff_seconds(attempt) nxt = attempt + 1 conn.execute( "UPDATE attachment_jobs SET status = 'queued', attempt = %s, last_error = %s, " "next_attempt_at = now() + make_interval(secs => %s), lease_token = NULL, " "lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE job_id = %s", (nxt, f"{code}: {detail}"[:300], delay, job_id)) conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s", (attachment_id,)) created = conn.execute("SELECT now()").fetchone()[0] payload = broker.make_payload(job_id, attachment_id, pv, nxt, created) conn.execute( "INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) " "VALUES (%s, %s, %s, now() + make_interval(secs => %s), now())", (job_id, nxt, json.dumps(payload), delay)) conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) " "VALUES (%s, 'retry_scheduled', %s, %s, %s)", (job_id, attempt, actor, f"{code}; next attempt {nxt} in {delay:.1f}s")) return f"retry #{nxt} in {delay:.1f}s" def main() -> None: logging.basicConfig(level=settings.log_level.upper(), format="%(asctime)s %(levelname)s %(name)s %(message)s") logging.getLogger("botocore").setLevel(logging.WARNING) Worker().run() if __name__ == "__main__": main()