63 lines
3.0 KiB
Python
63 lines
3.0 KiB
Python
"""Recover committed intent automatically and republish after uncertain acceptance."""
|
|
import json
|
|
import signal
|
|
import threading
|
|
|
|
from .broker import connect, ensure_group, publish
|
|
from .jobs import database, payload, recover_expired
|
|
from . import telemetry as tel
|
|
|
|
stop = threading.Event()
|
|
|
|
|
|
def tick():
|
|
with database() as db, connect() as client:
|
|
recover_expired(db)
|
|
ensure_group(client)
|
|
with db.transaction():
|
|
# Bounded batch and SKIP LOCKED allow another dispatcher without
|
|
# duplicate simultaneous publication. Crash after XADD may redeliver.
|
|
rows = db.execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id
|
|
WHERE j.status='queued' AND j.next_attempt_at<=clock_timestamp()
|
|
AND (o.published_at IS NULL OR o.published_at<clock_timestamp()-interval '30 seconds')
|
|
ORDER BY j.created_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED""").fetchall()
|
|
for job in rows:
|
|
with tel.span('attachment.publish',kind=tel.trace.SpanKind.PRODUCER,
|
|
carrier={key:job[key] for key in ('traceparent','tracestate') if job.get(key)},
|
|
attributes={'messaging.system':'redis','messaging.destination.name':'attachment-jobs'}):
|
|
item=payload(job); item.update(tel.carrier())
|
|
publish(client,item)
|
|
tel.event('job_published',job_id=str(job['id']),attempt=job['attempts']+1)
|
|
db.execute('UPDATE outbox SET published_at=clock_timestamp(),publications=publications+1 WHERE job_id=%s',
|
|
(job['id'],))
|
|
queue=db.execute("SELECT count(*) AS n,coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS age FROM jobs WHERE status IN ('queued','processing')").fetchone()
|
|
attrs={'queue':'attachment-jobs'}
|
|
tel.queue_depth.set(queue['n'],attrs); tel.queue_age.set(float(queue['age']),attrs)
|
|
for window in (60,300):
|
|
slo=db.execute("""SELECT count(*) AS total,count(*) FILTER (WHERE status!='ready'
|
|
OR updated_at>accepted_at+interval '30 seconds') AS bad FROM attachments
|
|
WHERE accepted_at<=clock_timestamp()-interval '30 seconds'
|
|
AND accepted_at>clock_timestamp()-((%s+30)*interval '1 second')""",(window,)).fetchone()
|
|
tel.slo_total.set(slo['total'],{'window':str(window)})
|
|
tel.slo_bad.set(slo['bad'],{'window':str(window)})
|
|
tel.process_metrics()
|
|
return len(rows)
|
|
|
|
|
|
def main():
|
|
tel.setup('messenger-dispatcher')
|
|
signal.signal(signal.SIGTERM, lambda *_: stop.set())
|
|
signal.signal(signal.SIGINT, lambda *_: stop.set())
|
|
while not stop.is_set():
|
|
try:
|
|
count = tick()
|
|
if count:
|
|
tel.event('jobs_published',count=count)
|
|
except Exception as exc:
|
|
tel.event('dispatch_retry',error_type=type(exc).__name__)
|
|
stop.wait(.5)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|