Files
mtusi2026lab5/app/broker.py
T
Artu-RitoandClaude Sonnet 5 bdba93f022 Лаба 5: Redis Streams очередь + асинхронная обработка изображений (оценка 4)
Переиспользован стек lab4 (LB/2 реплики/Postgres/SeaweedFS). upload теперь
202 queued; отдельный worker-процесс генерирует thumbnail/crop/optimized.
Transactional outbox закрывает разрыв COMMIT/publish; durable retry
(job_retry_schedule) переживает падение worker во время backoff; lease
recovery через XAUTOCLAIM; DLQ-реестр (job_failures) с явным operator
replay (scripts/replay_failed.py). make test/test-persistence/test-failover/
validate зелёные; в процессе экспериментов найдены и исправлены два
реальных бага (неполный try/except вокруг S3-записи вариантов, ошибка типов
asyncpg в построении interval) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:21:42 +03:00

69 lines
2.2 KiB
Python

"""Durable job queue on Redis Streams (persisted via the same AOF-enabled
Redis used for sessions). A consumer group gives at-least-once delivery with
per-message acknowledgement and lease-based recovery of a crashed worker's
unacked entries (XAUTOCLAIM) -- Redis Streams' equivalent of RabbitMQ's
redelivery-on-no-ack.
"""
import json
import redis.asyncio as redis
STREAM = "attachment_jobs"
GROUP = "workers"
_client: redis.Redis | None = None
def init(url: str) -> None:
global _client
_client = redis.from_url(url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5)
def client() -> redis.Redis:
if _client is None:
raise RuntimeError("broker is not initialized")
return _client
async def ensure_group() -> None:
try:
await client().xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
async def ping() -> bool:
try:
return bool(await client().ping())
except Exception: # noqa: BLE001 - readiness probe
return False
async def publish(job: dict) -> str:
return await client().xadd(STREAM, {"payload": json.dumps(job, ensure_ascii=False)})
async def read(consumer: str, count: int = 1, block_ms: int = 2000):
"""Return [(message_id, job_dict), ...] of newly delivered entries."""
resp = await client().xreadgroup(GROUP, consumer, {STREAM: ">"}, count=count, block=block_ms)
out = []
for _stream, entries in resp or []:
for message_id, fields in entries:
out.append((message_id, json.loads(fields["payload"])))
return out
async def ack(message_id: str) -> None:
await client().xack(STREAM, GROUP, message_id)
async def reclaim_stale(consumer: str, min_idle_ms: int, count: int = 10):
"""Hand this consumer any entries some other (likely dead) consumer has
held unacked for longer than min_idle_ms -- recovery for a worker that
died mid-processing, not just redelivery of a fresh message."""
_cursor, entries, _deleted = await client().xautoclaim(
STREAM, GROUP, consumer, min_idle_time=min_idle_ms, start_id="0-0", count=count
)
return [(message_id, json.loads(fields["payload"])) for message_id, fields in entries]