forked from mtusi2026/mtusi2026lab5
Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь 202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized. Transactional outbox закрывает разрыв COMMIT/publish; durable retry (job_retry_schedule) переживает падение worker во время backoff; lease recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator replay (scripts/replay_failed.py). make test/test-persistence/test-failover/ validate зелёные; в процессе экспериментов найдены и исправлены два реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов asyncpg в построении interval) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
69 lines
2.2 KiB
Python
69 lines
2.2 KiB
Python
"""Durable job queue on Redis Streams (persisted via the same AOF-enabled
|
|
Redis used for sessions). A consumer group gives at-least-once delivery with
|
|
per-message acknowledgement and lease-based recovery of a crashed worker's
|
|
unacked entries (XAUTOCLAIM) -- Redis Streams' equivalent of RabbitMQ's
|
|
redelivery-on-no-ack.
|
|
"""
|
|
import json
|
|
|
|
import redis.asyncio as redis
|
|
|
|
STREAM = "attachment_jobs"
|
|
GROUP = "workers"
|
|
|
|
_client: redis.Redis | None = None
|
|
|
|
|
|
def init(url: str) -> None:
|
|
global _client
|
|
_client = redis.from_url(url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5)
|
|
|
|
|
|
def client() -> redis.Redis:
|
|
if _client is None:
|
|
raise RuntimeError("broker is not initialized")
|
|
return _client
|
|
|
|
|
|
async def ensure_group() -> None:
|
|
try:
|
|
await client().xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if "BUSYGROUP" not in str(exc):
|
|
raise
|
|
|
|
|
|
async def ping() -> bool:
|
|
try:
|
|
return bool(await client().ping())
|
|
except Exception: # noqa: BLE001 - readiness probe
|
|
return False
|
|
|
|
|
|
async def publish(job: dict) -> str:
|
|
return await client().xadd(STREAM, {"payload": json.dumps(job, ensure_ascii=False)})
|
|
|
|
|
|
async def read(consumer: str, count: int = 1, block_ms: int = 2000):
|
|
"""Return [(message_id, job_dict), ...] of newly delivered entries."""
|
|
resp = await client().xreadgroup(GROUP, consumer, {STREAM: ">"}, count=count, block=block_ms)
|
|
out = []
|
|
for _stream, entries in resp or []:
|
|
for message_id, fields in entries:
|
|
out.append((message_id, json.loads(fields["payload"])))
|
|
return out
|
|
|
|
|
|
async def ack(message_id: str) -> None:
|
|
await client().xack(STREAM, GROUP, message_id)
|
|
|
|
|
|
async def reclaim_stale(consumer: str, min_idle_ms: int, count: int = 10):
|
|
"""Hand this consumer any entries some other (likely dead) consumer has
|
|
held unacked for longer than min_idle_ms -- recovery for a worker that
|
|
died mid-processing, not just redelivery of a fresh message."""
|
|
_cursor, entries, _deleted = await client().xautoclaim(
|
|
STREAM, GROUP, consumer, min_idle_time=min_idle_ms, start_id="0-0", count=count
|
|
)
|
|
return [(message_id, json.loads(fields["payload"])) for message_id, fields in entries]
|