forked from mtusi2026/mtusi2026lab5
115 lines
3.5 KiB
Python
115 lines
3.5 KiB
Python
"""Transactional outbox and operator controls shared by the API and the worker."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import uuid
|
|
|
|
from psycopg.types.json import Jsonb
|
|
|
|
from app import runtime
|
|
from app.broker import ROUTING_DLQ
|
|
from app.db import connection
|
|
from app.timeutil import iso
|
|
|
|
log = logging.getLogger("messenger.outbox")
|
|
|
|
|
|
def payload_for(job_id, attachment_id, pipeline_version: int, attempt: int, created_at) -> dict:
|
|
created = created_at if isinstance(created_at, str) else iso(created_at)
|
|
return {
|
|
"schema_version": 1,
|
|
"job_id": str(job_id),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(attachment_id),
|
|
"pipeline_version": int(pipeline_version),
|
|
"attempt": int(attempt),
|
|
"created_at": created,
|
|
}
|
|
|
|
|
|
def add_outbox(conn, job_id, attachment_id, pipeline_version: int, attempt: int, created_at) -> dict:
|
|
body = payload_for(job_id, attachment_id, pipeline_version, attempt, created_at)
|
|
conn.execute(
|
|
"INSERT INTO outbox (job_id, payload) VALUES (%s, %s)",
|
|
(job_id, Jsonb(body)),
|
|
)
|
|
return body
|
|
|
|
|
|
def audit(conn, job_id, attachment_id, action: str, detail: dict) -> None:
|
|
conn.execute(
|
|
"INSERT INTO job_audit (job_id, attachment_id, action, detail) VALUES (%s, %s, %s, %s)",
|
|
(job_id, attachment_id, action, Jsonb(detail)),
|
|
)
|
|
|
|
|
|
def outbox_paused(conn) -> bool:
|
|
row = conn.execute("SELECT value FROM runtime_control WHERE key='outbox_paused'").fetchone()
|
|
return bool(row and row["value"] is True)
|
|
|
|
|
|
def relay_tick(limit: int = 10) -> int:
|
|
"""Publish committed outbox rows. A crash after COMMIT and before confirm leaves the row unpublished."""
|
|
settings, _store, publisher = runtime.require()
|
|
published = 0
|
|
with connection() as conn:
|
|
if outbox_paused(conn):
|
|
return 0
|
|
conn.execute(
|
|
"""
|
|
UPDATE outbox AS o
|
|
SET published_at = clock_timestamp()
|
|
FROM jobs AS j
|
|
WHERE o.job_id = j.job_id
|
|
AND o.published_at IS NULL
|
|
AND j.status IN ('ready', 'failed', 'superseded')
|
|
"""
|
|
)
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT o.id, o.payload
|
|
FROM outbox AS o
|
|
JOIN jobs AS j ON j.job_id = o.job_id
|
|
WHERE o.published_at IS NULL
|
|
AND j.status = 'queued'
|
|
AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= clock_timestamp())
|
|
ORDER BY o.id
|
|
FOR UPDATE OF o SKIP LOCKED
|
|
LIMIT %s
|
|
""",
|
|
(limit,),
|
|
).fetchall()
|
|
for row in rows:
|
|
if publisher.publish(row["payload"]):
|
|
conn.execute("UPDATE outbox SET published_at = clock_timestamp() WHERE id = %s", (row["id"],))
|
|
published += 1
|
|
else:
|
|
break
|
|
if published:
|
|
log.info("published %s job message(s)", published)
|
|
return published
|
|
|
|
|
|
def publish_dlq(body: dict) -> None:
|
|
_settings, _store, publisher = runtime.require()
|
|
publisher.publish(body, routing_key=ROUTING_DLQ)
|
|
|
|
|
|
def start_relay() -> None:
|
|
import threading
|
|
import time
|
|
|
|
def loop() -> None:
|
|
while True:
|
|
try:
|
|
relay_tick()
|
|
except Exception:
|
|
log.warning("outbox relay iteration failed", exc_info=True)
|
|
time.sleep(0.4)
|
|
|
|
threading.Thread(target=loop, name="outbox-relay", daemon=True).start()
|
|
|
|
|
|
def new_job_id() -> uuid.UUID:
|
|
return uuid.uuid4()
|