Files
mtusi2026lab6/app/broker.py
T
Artu-RitoandClaude Sonnet 5 6c50a63743 Лаба 6: OpenTelemetry + Grafana наблюдаемость (оценка 4)
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в
otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов,
дашборда и alert rules в Git. W3C trace context переносится через поле
traceparent в задании очереди (API producer span -> worker consumer span
в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых
инцидента прогнаны и задокументированы (worker down -> queue lag + alert,
DB down отличим от медленного handler по error-span, telemetry backend down
не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага
(middleware после старта приложения, detached span context, замерзающая
метрика очереди при падении её единственного источника, неатомарный
XACK+XDEL) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:59:33 +03:00

91 lines
3.0 KiB
Python

"""Durable job queue on Redis Streams (persisted via the same AOF-enabled
Redis used for sessions). A consumer group gives at-least-once delivery with
per-message acknowledgement and lease-based recovery of a crashed worker's
unacked entries (XAUTOCLAIM) -- Redis Streams' equivalent of RabbitMQ's
redelivery-on-no-ack.
"""
import json
import redis.asyncio as redis
STREAM = "attachment_jobs"
GROUP = "workers"
_client: redis.Redis | None = None
def init(url: str) -> None:
global _client
_client = redis.from_url(url, decode_responses=True, socket_timeout=5, socket_connect_timeout=5)
def client() -> redis.Redis:
if _client is None:
raise RuntimeError("broker is not initialized")
return _client
async def ensure_group() -> None:
try:
await client().xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
async def ping() -> bool:
try:
return bool(await client().ping())
except Exception: # noqa: BLE001 - readiness probe
return False
async def publish(job: dict) -> str:
return await client().xadd(STREAM, {"payload": json.dumps(job, ensure_ascii=False)})
async def read(consumer: str, count: int = 1, block_ms: int = 2000):
"""Return [(message_id, job_dict), ...] of newly delivered entries."""
resp = await client().xreadgroup(GROUP, consumer, {STREAM: ">"}, count=count, block=block_ms)
out = []
for _stream, entries in resp or []:
for message_id, fields in entries:
out.append((message_id, json.loads(fields["payload"])))
return out
async def ack(message_id: str) -> None:
# XACK + XDEL as one pipeline so a hiccup between the two calls cannot
# leave an already-acked entry lingering in XRANGE (it would inflate the
# depth gauge with "phantom" entries that are done but never trimmed).
pipe = client().pipeline()
pipe.xack(STREAM, GROUP, message_id)
pipe.xdel(STREAM, message_id)
await pipe.execute()
async def depth_and_oldest_age_seconds():
"""(unprocessed count, age in seconds of the oldest unprocessed entry)."""
import time as _time
depth = await client().xlen(STREAM)
if depth == 0:
return 0, None
oldest = await client().xrange(STREAM, min="-", max="+", count=1)
if not oldest:
return depth, None
entry_id = oldest[0][0]
entry_ms = int(entry_id.split("-")[0])
age = _time.time() - entry_ms / 1000
return depth, age
async def reclaim_stale(consumer: str, min_idle_ms: int, count: int = 10):
"""Hand this consumer any entries some other (likely dead) consumer has
held unacked for longer than min_idle_ms -- recovery for a worker that
died mid-processing, not just redelivery of a fresh message."""
_cursor, entries, _deleted = await client().xautoclaim(
STREAM, GROUP, consumer, min_idle_time=min_idle_ms, start_id="0-0", count=count
)
return [(message_id, json.loads(fields["payload"])) for message_id, fields in entries]