Files
mtusi2026lab6/src/dispatcher.py
T

63 lines
3.0 KiB
Python

"""Recover committed intent automatically and republish after uncertain acceptance."""
import json
import signal
import threading
from .broker import connect, ensure_group, publish
from .jobs import database, payload, recover_expired
from . import telemetry as tel
stop = threading.Event()
def tick():
with database() as db, connect() as client:
recover_expired(db)
ensure_group(client)
with db.transaction():
# Bounded batch and SKIP LOCKED allow another dispatcher without
# duplicate simultaneous publication. Crash after XADD may redeliver.
rows = db.execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id
WHERE j.status='queued' AND j.next_attempt_at<=clock_timestamp()
AND (o.published_at IS NULL OR o.published_at<clock_timestamp()-interval '30 seconds')
ORDER BY j.created_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED""").fetchall()
for job in rows:
with tel.span('attachment.publish',kind=tel.trace.SpanKind.PRODUCER,
carrier={key:job[key] for key in ('traceparent','tracestate') if job.get(key)},
attributes={'messaging.system':'redis','messaging.destination.name':'attachment-jobs'}):
item=payload(job); item.update(tel.carrier())
publish(client,item)
tel.event('job_published',job_id=str(job['id']),attempt=job['attempts']+1)
db.execute('UPDATE outbox SET published_at=clock_timestamp(),publications=publications+1 WHERE job_id=%s',
(job['id'],))
queue=db.execute("SELECT count(*) AS n,coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS age FROM jobs WHERE status IN ('queued','processing')").fetchone()
attrs={'queue':'attachment-jobs'}
tel.queue_depth.set(queue['n'],attrs); tel.queue_age.set(float(queue['age']),attrs)
for window in (60,300):
slo=db.execute("""SELECT count(*) AS total,count(*) FILTER (WHERE status!='ready'
OR updated_at>accepted_at+interval '30 seconds') AS bad FROM attachments
WHERE accepted_at<=clock_timestamp()-interval '30 seconds'
AND accepted_at>clock_timestamp()-((%s+30)*interval '1 second')""",(window,)).fetchone()
tel.slo_total.set(slo['total'],{'window':str(window)})
tel.slo_bad.set(slo['bad'],{'window':str(window)})
tel.process_metrics()
return len(rows)
def main():
tel.setup('messenger-dispatcher')
signal.signal(signal.SIGTERM, lambda *_: stop.set())
signal.signal(signal.SIGINT, lambda *_: stop.set())
while not stop.is_set():
try:
count = tick()
if count:
tel.event('jobs_published',count=count)
except Exception as exc:
tel.event('dispatch_retry',error_type=type(exc).__name__)
stop.wait(.5)
if __name__ == '__main__':
main()