forked from mtusi2026/mtusi2026lab6
139 lines
6.9 KiB
Python
139 lines
6.9 KiB
Python
"""Lab 6 dispatcher (lab 5 + OpenTelemetry): transactional-outbox relay + lease reaper + queue gauges.
|
|
|
|
* relay : outbox rows (committed together with the state change) -> Kafka (acks=all, idempotent producer, delivery report)
|
|
* reaper : processing job with an expired lease (worker died) -> queued again (or failed when attempts are exhausted)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import psycopg
|
|
from confluent_kafka import Producer
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry.propagate import extract, inject
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
from . import jobs, telemetry
|
|
|
|
telemetry.init("messenger-dispatcher", instrument=("psycopg",))
|
|
log = logging.getLogger("dispatcher")
|
|
TRACER = telemetry.tracer()
|
|
|
|
# ---- Lab 6: queue gauges are computed HERE (not in the worker) so they stay visible when the worker is stopped. ----
|
|
QUEUE_SAMPLE_SECONDS = 5.0
|
|
_queue = {"depth": 0.0, "oldest": 0.0, "at": None}
|
|
|
|
|
|
def _queue_gauge(key: str):
|
|
def callback(_options):
|
|
at = _queue["at"]
|
|
if at is not None and time.monotonic() - at < 30: # stale (DB down) => no observation instead of a fake value
|
|
yield metrics.Observation(_queue[key], {"queue.name": jobs.JOBS_TOPIC})
|
|
return callback
|
|
|
|
|
|
telemetry.meter().create_observable_gauge("messenger.queue.depth", callbacks=[_queue_gauge("depth")], unit="{job}",
|
|
description="Jobs waiting for a worker: status=queued and due (backoff elapsed)")
|
|
telemetry.meter().create_observable_gauge("messenger.queue.oldest_job_age", callbacks=[_queue_gauge("oldest")], unit="s",
|
|
description="Age of the oldest waiting job (now - time it became due); 0 if the queue is empty")
|
|
telemetry.register_process_metrics()
|
|
|
|
|
|
def sample_queue(conn) -> None:
|
|
depth, oldest = conn.execute(
|
|
"SELECT count(*), COALESCE(EXTRACT(EPOCH FROM (now() - min(next_attempt_at))), 0) "
|
|
"FROM attachment_jobs WHERE status='queued' AND next_attempt_at <= now()").fetchone()
|
|
_queue.update(depth=float(depth), oldest=max(0.0, float(oldest)), at=time.monotonic())
|
|
POLL_SECONDS = float(os.getenv("DISPATCH_POLL_SECONDS", "1"))
|
|
BATCH = int(os.getenv("OUTBOX_BATCH", "100"))
|
|
FAULT_FLAG = Path("/tmp/dispatcher_fault_done")
|
|
FAULT_EXIT_BEFORE_PUBLISH = os.getenv("FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH", "0") == "1" # test hook: crash between COMMIT and publish
|
|
|
|
|
|
def relay(conn, producer: Producer) -> int:
|
|
with conn.transaction():
|
|
rows = conn.execute("SELECT id, topic, msg_key, payload::text FROM outbox WHERE published_at IS NULL AND available_at <= now() "
|
|
"ORDER BY id LIMIT %s FOR UPDATE SKIP LOCKED", (BATCH,)).fetchall()
|
|
if not rows:
|
|
return 0
|
|
if FAULT_EXIT_BEFORE_PUBLISH and not FAULT_FLAG.exists():
|
|
FAULT_FLAG.touch()
|
|
log.error("FAULT: exiting after reading the outbox, before publishing (COMMIT happened, publish did not)")
|
|
os._exit(137)
|
|
delivered, errors = set(), {}
|
|
|
|
for oid, topic, key, payload in rows:
|
|
msg = json.loads(payload)
|
|
# PRODUCER span: child of the span that wrote the outbox row (API upload / worker retry); the payload then
|
|
# carries THIS span's traceparent, so the worker's CONSUMER span becomes its child.
|
|
span = TRACER.start_span(f"{topic} publish", context=extract(msg), kind=SpanKind.PRODUCER,
|
|
attributes={"messaging.system": "kafka", "messaging.destination.name": topic,
|
|
"messaging.operation.type": "publish", "job.id": msg.get("job_id", ""),
|
|
"job.attempt": msg.get("attempt", 0)})
|
|
with trace.use_span(span, end_on_exit=False):
|
|
inject(msg)
|
|
|
|
def cb(err, _m, oid=oid, span=span):
|
|
if err:
|
|
errors[oid] = str(err)
|
|
span.set_status(Status(StatusCode.ERROR, "delivery not confirmed"))
|
|
else:
|
|
delivered.add(oid) # publisher confirm arrives here
|
|
span.end()
|
|
producer.produce(topic, key=key.encode(), value=json.dumps(msg).encode(), on_delivery=cb)
|
|
producer.flush(15)
|
|
if delivered:
|
|
conn.execute("UPDATE outbox SET published_at=now() WHERE id = ANY(%s)", (list(delivered),))
|
|
if errors:
|
|
telemetry.log_event(log, logging.WARNING, "outbox_publish_unconfirmed", f"{len(errors)} outbox rows not confirmed by the broker, will retry", count=len(errors))
|
|
return len(delivered)
|
|
|
|
|
|
def reap(conn) -> int:
|
|
n = 0
|
|
with conn.transaction():
|
|
for job_id, att_id, attempt, max_attempts, pv in conn.execute(
|
|
"SELECT job_id, attachment_id, attempt, max_attempts, pipeline_version FROM attachment_jobs "
|
|
"WHERE status='processing' AND lease_until < now() FOR UPDATE SKIP LOCKED").fetchall():
|
|
if attempt >= max_attempts:
|
|
jobs.fail_job(conn, job_id, att_id, "retry_exhausted", attempt)
|
|
else:
|
|
conn.execute("UPDATE attachment_jobs SET status='queued', lease_until=NULL, next_attempt_at=now(), updated_at=now() WHERE job_id=%s", (job_id,))
|
|
conn.execute("UPDATE attachments SET status='queued', updated_at=now() WHERE id=%s", (att_id,))
|
|
jobs.add_event(conn, job_id, "lease_expired", f"attempt={attempt}")
|
|
jobs.enqueue(conn, job_id, att_id, pv, attempt + 1)
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def main() -> None:
|
|
producer = Producer({"bootstrap.servers": jobs.KAFKA_BOOTSTRAP, "acks": "all", "enable.idempotence": True,
|
|
"message.timeout.ms": 15000, "linger.ms": 5})
|
|
telemetry.log_event(log, logging.INFO, "dispatcher_started", f"dispatcher started (poll={POLL_SECONDS:.1f}s)")
|
|
last_sample = 0.0
|
|
while True:
|
|
try:
|
|
with psycopg.connect(connect_timeout=5, autocommit=True, options="-c statement_timeout=20000") as conn:
|
|
while True:
|
|
done = reap(conn)
|
|
sent = relay(conn, producer)
|
|
if done or sent:
|
|
telemetry.log_event(log, logging.INFO, "dispatcher_cycle", f"reaped={done} published={sent}", count=sent)
|
|
if time.monotonic() - last_sample >= QUEUE_SAMPLE_SECONDS:
|
|
sample_queue(conn)
|
|
last_sample = time.monotonic()
|
|
if not sent:
|
|
time.sleep(POLL_SECONDS)
|
|
except psycopg.Error as exc:
|
|
log.warning("database unavailable: %s", type(exc).__name__); time.sleep(2)
|
|
except Exception as exc:
|
|
log.warning("dispatcher iteration failed: %s", type(exc).__name__); time.sleep(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|