forked from mtusi2026/mtusi2026lab6
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов, дашборда и alert rules в Git. W3C trace context переносится через поле traceparent в задании очереди (API producer span -> worker consumer span в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых инцидента прогнаны и задокументированы (worker down -> queue lag + alert, DB down отличим от медленного handler по error-span, telemetry backend down не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага (middleware после старта приложения, detached span context, замерзающая метрика очереди при падении её единственного источника, неатомарный XACK+XDEL) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
91 lines
3.0 KiB
Python
91 lines
3.0 KiB
Python
"""Durable job queue on Redis Streams (persisted via the same AOF-enabled
|
|
Redis used for sessions). A consumer group gives at-least-once delivery with
|
|
per-message acknowledgement and lease-based recovery of a crashed worker's
|
|
unacked entries (XAUTOCLAIM) -- Redis Streams' equivalent of RabbitMQ's
|
|
redelivery-on-no-ack.
|
|
"""
|
|
import json
|
|
|
|
import redis.asyncio as redis
|
|
|
|
STREAM = "attachment_jobs"
|
|
GROUP = "workers"
|
|
|
|
_client: redis.Redis | None = None
|
|
|
|
|
|
def init(url: str) -> None:
|
|
global _client
|
|
_client = redis.from_url(url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5)
|
|
|
|
|
|
def client() -> redis.Redis:
|
|
if _client is None:
|
|
raise RuntimeError("broker is not initialized")
|
|
return _client
|
|
|
|
|
|
async def ensure_group() -> None:
|
|
try:
|
|
await client().xgroup_create(STREAM, GROUP, id="0", mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if "BUSYGROUP" not in str(exc):
|
|
raise
|
|
|
|
|
|
async def ping() -> bool:
|
|
try:
|
|
return bool(await client().ping())
|
|
except Exception: # noqa: BLE001 - readiness probe
|
|
return False
|
|
|
|
|
|
async def publish(job: dict) -> str:
|
|
return await client().xadd(STREAM, {"payload": json.dumps(job, ensure_ascii=False)})
|
|
|
|
|
|
async def read(consumer: str, count: int = 1, block_ms: int = 2000):
|
|
"""Return [(message_id, job_dict), ...] of newly delivered entries."""
|
|
resp = await client().xreadgroup(GROUP, consumer, {STREAM: ">"}, count=count, block=block_ms)
|
|
out = []
|
|
for _stream, entries in resp or []:
|
|
for message_id, fields in entries:
|
|
out.append((message_id, json.loads(fields["payload"])))
|
|
return out
|
|
|
|
|
|
async def ack(message_id: str) -> None:
|
|
# XACK + XDEL as one pipeline so a hiccup between the two calls cannot
|
|
# leave an already-acked entry lingering in XRANGE (it would inflate the
|
|
# depth gauge with "phantom" entries that are done but never trimmed).
|
|
pipe = client().pipeline()
|
|
pipe.xack(STREAM, GROUP, message_id)
|
|
pipe.xdel(STREAM, message_id)
|
|
await pipe.execute()
|
|
|
|
|
|
async def depth_and_oldest_age_seconds():
|
|
"""(unprocessed count, age in seconds of the oldest unprocessed entry)."""
|
|
import time as _time
|
|
|
|
depth = await client().xlen(STREAM)
|
|
if depth == 0:
|
|
return 0, None
|
|
oldest = await client().xrange(STREAM, min="-", max="+", count=1)
|
|
if not oldest:
|
|
return depth, None
|
|
entry_id = oldest[0][0]
|
|
entry_ms = int(entry_id.split("-")[0])
|
|
age = _time.time() - entry_ms / 1000
|
|
return depth, age
|
|
|
|
|
|
async def reclaim_stale(consumer: str, min_idle_ms: int, count: int = 10):
|
|
"""Hand this consumer any entries some other (likely dead) consumer has
|
|
held unacked for longer than min_idle_ms -- recovery for a worker that
|
|
died mid-processing, not just redelivery of a fresh message."""
|
|
_cursor, entries, _deleted = await client().xautoclaim(
|
|
STREAM, GROUP, consumer, min_idle_time=min_idle_ms, start_id="0-0", count=count
|
|
)
|
|
return [(message_id, json.loads(fields["payload"])) for message_id, fields in entries]
|