Files
mtusi2026lab6/src/messenger/relay.py
T

107 lines
3.7 KiB
Python

"""Transactional outbox publisher and expired-lease watchdog, independently restartable."""
import signal
import time
from pathlib import Path
import redis
from opentelemetry.trace import SpanKind
from messenger import telemetry
from messenger.broker import Broker, Job
from messenger.domain import DomainError
from messenger.faults import checkpoint
from messenger.pipeline import Pipeline, event
from messenger.repository import PostgresRepository, stamp
from messenger.settings import Settings
def publish_one(db, broker, settings):
with db.connection(write=True) as conn:
row = conn.execute(
"SELECT o.job_id,j.attachment_id,j.pipeline_version,j.attempt,j.created_at,"
"j.traceparent,j.tracestate "
"FROM job_outbox o JOIN attachment_jobs j USING(job_id) "
"WHERE o.published_at IS NULL AND o.due_at<=clock_timestamp() AND j.state='queued' "
"ORDER BY o.due_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED"
).fetchone()
if not row:
return False
with telemetry.span("jobs.publish", context=telemetry.context(row), kind=SpanKind.PRODUCER):
job = Job(
schema_version=1,
type=telemetry.JOB_TYPE,
job_id=str(row["job_id"]),
attachment_id=str(row["attachment_id"]),
pipeline_version=row["pipeline_version"],
attempt=row["attempt"] + 1,
created_at=stamp(row["created_at"]),
**telemetry.carrier(),
)
stream_id = broker.publish(job)
telemetry.event("job_published", job_id=job.job_id, attempt=job.attempt)
# Commit-on-return may be lost here; repeat sends the same logical job_id.
checkpoint(db, settings, row["job_id"], "after_publish")
conn.execute(
"UPDATE job_outbox SET published_at=clock_timestamp(),stream_id=%s WHERE job_id=%s",
(stream_id, row["job_id"]),
)
event(conn, row["job_id"], "published", stream_id=stream_id, attempt=job.attempt)
return True
def sample_backlog(db):
with db.connection() as conn:
row = conn.execute(
"SELECT count(*) AS depth, "
"coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS age "
"FROM attachment_jobs WHERE state IN ('queued','processing')"
).fetchone()
telemetry.gauge("messenger.queue.depth", row["depth"], queue="attachment-jobs")
telemetry.gauge(
"messenger.queue.oldest", max(0, float(row["age"])), unit="s", queue="attachment-jobs"
)
def main():
with telemetry.active("relay"):
run()
def run():
settings = Settings.from_env()
db = PostgresRepository(settings.database_url, pool_size=4)
db.open()
broker = Broker(settings)
pipeline = Pipeline(db, settings)
running = True
next_sample = 0.0
def stop(*_):
nonlocal running
running = False
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
while running:
try:
if telemetry.runtime is not None and time.monotonic() >= next_sample:
sample_backlog(db)
next_sample = time.monotonic() + 2
pipeline.recover()
sent = publish_one(db, broker, settings)
Path("/tmp/relay-heartbeat").touch()
if not sent:
time.sleep(0.2)
except redis.RedisError, DomainError:
# Keep the transaction uncommitted and retry. No credentials/SQL in logs.
time.sleep(0.5)
finally:
broker.close()
db.close()
if __name__ == "__main__":
main()