"""Durable job queue on Redis Streams (persisted via the same AOF-enabled Redis used for sessions). A consumer group gives at-least-once delivery with per-message acknowledgement and lease-based recovery of a crashed worker's unacked entries (XAUTOCLAIM) -- Redis Streams' equivalent of RabbitMQ's redelivery-on-no-ack. """ import json import redis.asyncio as redis STREAM = "attachment_jobs" GROUP = "workers" _client: redis.Redis | None = None def init(url: str) -> None: global _client _client = redis.from_url(url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5) def client() -> redis.Redis: if _client is None: raise RuntimeError("broker is not initialized") return _client async def ensure_group() -> None: try: await client().xgroup_create(STREAM, GROUP, id="0", mkstream=True) except redis.ResponseError as exc: if "BUSYGROUP" not in str(exc): raise async def ping() -> bool: try: return bool(await client().ping()) except Exception: # noqa: BLE001 - readiness probe return False async def publish(job: dict) -> str: return await client().xadd(STREAM, {"payload": json.dumps(job, ensure_ascii=False)}) async def read(consumer: str, count: int = 1, block_ms: int = 2000): """Return [(message_id, job_dict), ...] of newly delivered entries.""" resp = await client().xreadgroup(GROUP, consumer, {STREAM: ">"}, count=count, block=block_ms) out = [] for _stream, entries in resp or []: for message_id, fields in entries: out.append((message_id, json.loads(fields["payload"]))) return out async def ack(message_id: str) -> None: await client().xack(STREAM, GROUP, message_id) async def reclaim_stale(consumer: str, min_idle_ms: int, count: int = 10): """Hand this consumer any entries some other (likely dead) consumer has held unacked for longer than min_idle_ms -- recovery for a worker that died mid-processing, not just redelivery of a fresh message.""" _cursor, entries, _deleted = await client().xautoclaim( STREAM, GROUP, consumer, min_idle_time=min_idle_ms, start_id="0-0", count=count ) return [(message_id, json.loads(fields["payload"])) for message_id, fields in entries]