Files

46 lines
1.8 KiB
Python

"""Recoverable DB outbox -> persistent Valkey Streams (at least once)."""
import argparse
import json
import signal
import threading
from psycopg.rows import dict_row
from app import store
from pipeline import broker, envelope, STREAM
stop=threading.Event()
def publish_once(client):
with store.connect() as conn:
# Marking published happens AFTER XADD. A crash in between produces a
# duplicate. DB remains authoritative; no acknowledged intent is lost.
rows=conn.cursor(row_factory=dict_row).execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id
WHERE j.state IN ('queued','processing') AND j.next_at<=now()
AND (o.published_at IS NULL OR (j.state='processing' AND j.lease_until<now()
AND o.published_at<now()-interval '5 seconds'))
ORDER BY j.created_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED""").fetchall()
for row in rows:
client.xadd(STREAM,{'job':json.dumps(envelope(row))})
conn.execute('UPDATE outbox SET published_at=now(),deliveries=deliveries+1 WHERE job_id=%s',(row['id'],))
return len(rows)
if __name__=='__main__':
parser=argparse.ArgumentParser();parser.add_argument('--once',action='store_true');args=parser.parse_args()
store.pool.open(wait=True,timeout=30)
client=broker()
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda *_:stop.set())
try:
while not stop.is_set():
try:
count=publish_once(client)
if count:print(json.dumps({'published':count}),flush=True)
except Exception as exc:
print(json.dumps({'publisher_error':type(exc).__name__}),flush=True)
if args.once:raise
if args.once:break
stop.wait(.25)
finally:
client.close();store.pool.close()