46 lines
1.8 KiB
Python
46 lines
1.8 KiB
Python
"""Recoverable DB outbox -> persistent Valkey Streams (at least once)."""
|
|
import argparse
|
|
import json
|
|
import signal
|
|
import threading
|
|
from psycopg.rows import dict_row
|
|
from app import store
|
|
from pipeline import broker, envelope, STREAM
|
|
|
|
stop=threading.Event()
|
|
|
|
|
|
def publish_once(client):
|
|
with store.connect() as conn:
|
|
# Marking published happens AFTER XADD. A crash in between produces a
|
|
# duplicate. DB remains authoritative; no acknowledged intent is lost.
|
|
rows=conn.cursor(row_factory=dict_row).execute("""SELECT j.* FROM outbox o JOIN jobs j ON j.id=o.job_id
|
|
WHERE j.state IN ('queued','processing') AND j.next_at<=now()
|
|
AND (o.published_at IS NULL OR (j.state='processing' AND j.lease_until<now()
|
|
AND o.published_at<now()-interval '5 seconds'))
|
|
ORDER BY j.created_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED""").fetchall()
|
|
for row in rows:
|
|
client.xadd(STREAM,{'job':json.dumps(envelope(row))})
|
|
conn.execute('UPDATE outbox SET published_at=now(),deliveries=deliveries+1 WHERE job_id=%s',(row['id'],))
|
|
return len(rows)
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser();parser.add_argument('--once',action='store_true');args=parser.parse_args()
|
|
store.pool.open(wait=True,timeout=30)
|
|
client=broker()
|
|
for sig in (signal.SIGTERM,signal.SIGINT):
|
|
signal.signal(sig,lambda *_:stop.set())
|
|
try:
|
|
while not stop.is_set():
|
|
try:
|
|
count=publish_once(client)
|
|
if count:print(json.dumps({'published':count}),flush=True)
|
|
except Exception as exc:
|
|
print(json.dumps({'publisher_error':type(exc).__name__}),flush=True)
|
|
if args.once:raise
|
|
if args.once:break
|
|
stop.wait(.25)
|
|
finally:
|
|
client.close();store.pool.close()
|