Files
mtusi2026lab6-fork/src/dispatcher.py
T
2026-11-21 00:02:11 +03:00

139 lines
6.9 KiB
Python

"""Lab 6 dispatcher (lab 5 + OpenTelemetry): transactional-outbox relay + lease reaper + queue gauges.
* relay : outbox rows (committed together with the state change) -> Kafka (acks=all, idempotent producer, delivery report)
* reaper : processing job with an expired lease (worker died) -> queued again (or failed when attempts are exhausted)
"""
from __future__ import annotations
import json
import logging
import os
import time
from pathlib import Path
import psycopg
from confluent_kafka import Producer
from opentelemetry import metrics, trace
from opentelemetry.propagate import extract, inject
from opentelemetry.trace import SpanKind, Status, StatusCode
from . import jobs, telemetry
telemetry.init("messenger-dispatcher", instrument=("psycopg",))
log = logging.getLogger("dispatcher")
TRACER = telemetry.tracer()
# ---- Lab 6: queue gauges are computed HERE (not in the worker) so they stay visible when the worker is stopped. ----
QUEUE_SAMPLE_SECONDS = 5.0
_queue = {"depth": 0.0, "oldest": 0.0, "at": None}
def _queue_gauge(key: str):
def callback(_options):
at = _queue["at"]
if at is not None and time.monotonic() - at < 30: # stale (DB down) => no observation instead of a fake value
yield metrics.Observation(_queue[key], {"queue.name": jobs.JOBS_TOPIC})
return callback
telemetry.meter().create_observable_gauge("messenger.queue.depth", callbacks=[_queue_gauge("depth")], unit="{job}",
description="Jobs waiting for a worker: status=queued and due (backoff elapsed)")
telemetry.meter().create_observable_gauge("messenger.queue.oldest_job_age", callbacks=[_queue_gauge("oldest")], unit="s",
description="Age of the oldest waiting job (now - time it became due); 0 if the queue is empty")
telemetry.register_process_metrics()
def sample_queue(conn) -> None:
depth, oldest = conn.execute(
"SELECT count(*), COALESCE(EXTRACT(EPOCH FROM (now() - min(next_attempt_at))), 0) "
"FROM attachment_jobs WHERE status='queued' AND next_attempt_at <= now()").fetchone()
_queue.update(depth=float(depth), oldest=max(0.0, float(oldest)), at=time.monotonic())
POLL_SECONDS = float(os.getenv("DISPATCH_POLL_SECONDS", "1"))
BATCH = int(os.getenv("OUTBOX_BATCH", "100"))
FAULT_FLAG = Path("/tmp/dispatcher_fault_done")
FAULT_EXIT_BEFORE_PUBLISH = os.getenv("FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH", "0") == "1" # test hook: crash between COMMIT and publish
def relay(conn, producer: Producer) -> int:
with conn.transaction():
rows = conn.execute("SELECT id, topic, msg_key, payload::text FROM outbox WHERE published_at IS NULL AND available_at <= now() "
"ORDER BY id LIMIT %s FOR UPDATE SKIP LOCKED", (BATCH,)).fetchall()
if not rows:
return 0
if FAULT_EXIT_BEFORE_PUBLISH and not FAULT_FLAG.exists():
FAULT_FLAG.touch()
log.error("FAULT: exiting after reading the outbox, before publishing (COMMIT happened, publish did not)")
os._exit(137)
delivered, errors = set(), {}
for oid, topic, key, payload in rows:
msg = json.loads(payload)
# PRODUCER span: child of the span that wrote the outbox row (API upload / worker retry); the payload then
# carries THIS span's traceparent, so the worker's CONSUMER span becomes its child.
span = TRACER.start_span(f"{topic} publish", context=extract(msg), kind=SpanKind.PRODUCER,
attributes={"messaging.system": "kafka", "messaging.destination.name": topic,
"messaging.operation.type": "publish", "job.id": msg.get("job_id", ""),
"job.attempt": msg.get("attempt", 0)})
with trace.use_span(span, end_on_exit=False):
inject(msg)
def cb(err, _m, oid=oid, span=span):
if err:
errors[oid] = str(err)
span.set_status(Status(StatusCode.ERROR, "delivery not confirmed"))
else:
delivered.add(oid) # publisher confirm arrives here
span.end()
producer.produce(topic, key=key.encode(), value=json.dumps(msg).encode(), on_delivery=cb)
producer.flush(15)
if delivered:
conn.execute("UPDATE outbox SET published_at=now() WHERE id = ANY(%s)", (list(delivered),))
if errors:
telemetry.log_event(log, logging.WARNING, "outbox_publish_unconfirmed", f"{len(errors)} outbox rows not confirmed by the broker, will retry", count=len(errors))
return len(delivered)
def reap(conn) -> int:
n = 0
with conn.transaction():
for job_id, att_id, attempt, max_attempts, pv in conn.execute(
"SELECT job_id, attachment_id, attempt, max_attempts, pipeline_version FROM attachment_jobs "
"WHERE status='processing' AND lease_until < now() FOR UPDATE SKIP LOCKED").fetchall():
if attempt >= max_attempts:
jobs.fail_job(conn, job_id, att_id, "retry_exhausted", attempt)
else:
conn.execute("UPDATE attachment_jobs SET status='queued', lease_until=NULL, next_attempt_at=now(), updated_at=now() WHERE job_id=%s", (job_id,))
conn.execute("UPDATE attachments SET status='queued', updated_at=now() WHERE id=%s", (att_id,))
jobs.add_event(conn, job_id, "lease_expired", f"attempt={attempt}")
jobs.enqueue(conn, job_id, att_id, pv, attempt + 1)
n += 1
return n
def main() -> None:
producer = Producer({"bootstrap.servers": jobs.KAFKA_BOOTSTRAP, "acks": "all", "enable.idempotence": True,
"message.timeout.ms": 15000, "linger.ms": 5})
telemetry.log_event(log, logging.INFO, "dispatcher_started", f"dispatcher started (poll={POLL_SECONDS:.1f}s)")
last_sample = 0.0
while True:
try:
with psycopg.connect(connect_timeout=5, autocommit=True, options="-c statement_timeout=20000") as conn:
while True:
done = reap(conn)
sent = relay(conn, producer)
if done or sent:
telemetry.log_event(log, logging.INFO, "dispatcher_cycle", f"reaped={done} published={sent}", count=sent)
if time.monotonic() - last_sample >= QUEUE_SAMPLE_SECONDS:
sample_queue(conn)
last_sample = time.monotonic()
if not sent:
time.sleep(POLL_SECONDS)
except psycopg.Error as exc:
log.warning("database unavailable: %s", type(exc).__name__); time.sleep(2)
except Exception as exc:
log.warning("dispatcher iteration failed: %s", type(exc).__name__); time.sleep(2)
if __name__ == "__main__":
main()