forked from mtusi2026/mtusi2026lab6
471 lines
24 KiB
Python
471 lines
24 KiB
Python
"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
|
|
|
|
There is no exactly-once transport guarantee. We guarantee eventual recovery of
|
|
committed intent and idempotent DB results under at-least-once redelivery.
|
|
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
|
|
"""
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import threading
|
|
from . import telemetry as ot
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
import uuid
|
|
from datetime import datetime, timezone, timedelta
|
|
|
|
import pika
|
|
import psycopg
|
|
from pika.exceptions import AMQPError
|
|
from psycopg.rows import dict_row
|
|
|
|
log = logging.getLogger("lab5.jobs")
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
QUEUE = "attachment-processing.v1"
|
|
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
|
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
|
|
LEASE_SECONDS = JOB_TIMEOUT + 45
|
|
|
|
|
|
@contextmanager
|
|
def connect_db():
|
|
# No credentials, SQL text, or parameters in spans.
|
|
with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}):
|
|
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn:
|
|
yield conn
|
|
|
|
|
|
def job_payload(job_id, attachment_id, version, attempt):
|
|
return {
|
|
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
|
|
"attachment_id": str(attachment_id), "pipeline_version": version,
|
|
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
|
|
**ot.inject(),
|
|
}
|
|
|
|
|
|
@contextmanager
|
|
def connection():
|
|
params = pika.ConnectionParameters(
|
|
host=os.getenv("BROKER_HOST", "broker"), port=5672,
|
|
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
|
|
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
|
|
socket_timeout=5, connection_attempts=3, retry_delay=1)
|
|
mq = pika.BlockingConnection(params)
|
|
try:
|
|
yield mq
|
|
finally:
|
|
try:
|
|
if mq.is_open:
|
|
mq.close()
|
|
except AMQPError:
|
|
pass
|
|
|
|
|
|
def setup_channel(conn):
|
|
ch = conn.channel()
|
|
ch.queue_declare(queue=QUEUE, durable=True)
|
|
return ch
|
|
|
|
|
|
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
|
|
payload = job_payload(job_id, attachment_id, version, attempt)
|
|
if not payload.get('traceparent'):
|
|
# Lease rescue/replay has no active consumer context. Preserve the first
|
|
# acceptance context rather than inventing a new disconnected trace.
|
|
previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone()
|
|
if previous:
|
|
payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']})
|
|
db.execute("""
|
|
INSERT INTO job_outbox(id,job_id,payload,available_at)
|
|
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
|
|
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
|
|
|
|
|
|
def cleanup_abandoned_staging():
|
|
"""Best effort cleanup of retained stages; never delete current processing work."""
|
|
from .attachments import BUCKET, s3_client
|
|
client = s3_client()
|
|
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
|
|
with connect_db() as db:
|
|
rows = db.execute("""
|
|
SELECT id,staging_key FROM attachments
|
|
WHERE staging_key IS NOT NULL AND
|
|
((status='ready' AND updated_at < now()-interval '300 seconds') OR
|
|
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
|
|
LIMIT 50
|
|
""", (retention,)).fetchall()
|
|
for row in rows:
|
|
with connect_db() as db:
|
|
with db.transaction():
|
|
# Row-level lock serializes against operator replay.
|
|
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
|
|
if not current or current["staging_key"] != row["staging_key"]:
|
|
continue
|
|
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
|
|
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
|
|
continue
|
|
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
|
|
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
|
|
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
|
|
from botocore.exceptions import ClientError
|
|
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
|
|
for obj in page.get("Contents", []):
|
|
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
|
|
continue
|
|
key = obj["Key"]
|
|
with connect_db() as db:
|
|
with db.transaction():
|
|
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
|
|
if not referenced:
|
|
client.delete_object(Bucket=BUCKET, Key=key)
|
|
|
|
def rescue_expired_jobs(conn):
|
|
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
|
|
with conn.transaction():
|
|
rows = conn.execute("""
|
|
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
|
|
FROM processing_jobs j
|
|
WHERE j.status='processing' AND j.leased_until < now()
|
|
FOR UPDATE SKIP LOCKED LIMIT 20
|
|
""").fetchall()
|
|
for row in rows:
|
|
if row["attempt"] >= MAX_ATTEMPTS:
|
|
conn.execute("""UPDATE processing_jobs
|
|
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
|
|
WHERE id=%s""", (row["id"],))
|
|
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
|
|
WHERE id=%s""", (row["attachment_id"],))
|
|
else:
|
|
attempt = row["attempt"] + 1
|
|
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
|
|
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
|
|
WHERE id=%s""", (attempt, row["id"]))
|
|
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
|
|
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
|
|
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
|
|
|
|
|
|
def sample_backlog():
|
|
# Query durable *queued* jobs, including delayed retries and unpublished outbox.
|
|
# No broker membership scan; oldest age is wall clock since initial acceptance.
|
|
while True:
|
|
try:
|
|
with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db:
|
|
row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone()
|
|
ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age'])
|
|
except psycopg.Error:
|
|
ot.backlog_depth = ot.backlog_age = None
|
|
time.sleep(3)
|
|
|
|
|
|
def dispatch_loop():
|
|
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
|
|
threading.Thread(target=sample_backlog, daemon=True).start()
|
|
while True:
|
|
try:
|
|
with connection() as mq:
|
|
ch = setup_channel(mq)
|
|
ch.confirm_delivery()
|
|
last_clean = time.monotonic()
|
|
last_queue_sample = 0
|
|
while True:
|
|
# Periodic lease recovery works even if original message was consumed.
|
|
with connect_db() as conn:
|
|
rescue_expired_jobs(conn)
|
|
with conn.transaction():
|
|
rows = conn.execute("""
|
|
SELECT id,payload FROM job_outbox
|
|
WHERE published_at IS NULL AND available_at<=now()
|
|
ORDER BY available_at,created_at
|
|
FOR UPDATE SKIP LOCKED LIMIT 20
|
|
""").fetchall()
|
|
for row in rows:
|
|
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
|
|
# Persist one-time fault before hard exit; outbox still
|
|
# unmarked and dispatcher will publish on restart.
|
|
with connect_db() as faultdb:
|
|
inserted = faultdb.execute("""
|
|
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
|
|
ON CONFLICT DO NOTHING RETURNING job_id
|
|
""", (row["payload"]["job_id"],)).fetchone()
|
|
if inserted:
|
|
os._exit(78)
|
|
payload = row['payload']
|
|
with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload),
|
|
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer:
|
|
# Propagate actual producer span, while DB retains the original intent.
|
|
raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":"))
|
|
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
|
|
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
|
|
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
|
|
ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt'])
|
|
if time.monotonic() - last_queue_sample > 3:
|
|
q = ch.queue_declare(queue=QUEUE, passive=True).method
|
|
ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count
|
|
last_queue_sample = time.monotonic()
|
|
if time.monotonic() - last_clean > 60:
|
|
try:
|
|
cleanup_abandoned_staging()
|
|
except Exception:
|
|
log.warning("staging janitor failed; retry later")
|
|
last_clean = time.monotonic()
|
|
if not rows:
|
|
mq.sleep(1)
|
|
except (AMQPError, OSError, psycopg.Error) as exc:
|
|
ot.broker_ready = ot.broker_consumers = None
|
|
log.warning("dispatcher reconnect after %s", type(exc).__name__)
|
|
time.sleep(2)
|
|
|
|
|
|
def claim_job(db, message):
|
|
job_id = message["job_id"]
|
|
with db.transaction():
|
|
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
|
|
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
|
|
return "ignore", None
|
|
if job["status"] == "processing":
|
|
return "busy", None
|
|
if job["next_run_at"] > datetime.now(timezone.utc):
|
|
return "busy", None
|
|
lease = str(uuid.uuid4())
|
|
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
|
|
leased_until=now()+(%s*interval '1 second'),updated_at=now()
|
|
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
|
|
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
|
|
return "claimed", {"job": job, "lease": lease}
|
|
|
|
|
|
def publish_result(db, *, job, lease, variants):
|
|
"""Final result is committed atomically; stale leases/versions cannot publish."""
|
|
with db.transaction():
|
|
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
|
|
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
|
|
log.warning("stale worker result ignored %s", job["id"])
|
|
return False
|
|
for v in variants:
|
|
db.execute("""
|
|
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
|
|
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
|
|
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
|
|
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
|
|
width=EXCLUDED.width,height=EXCLUDED.height
|
|
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
|
|
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
|
|
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
|
|
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
|
|
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
|
|
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
|
|
return True
|
|
|
|
|
|
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
|
|
with db.transaction():
|
|
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
|
|
if current["status"] != "processing" or str(current["lease_token"]) != lease:
|
|
return "stale"
|
|
if permanent or job["attempt"] >= MAX_ATTEMPTS:
|
|
code = code if permanent else "retry_exhausted"
|
|
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
|
|
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
|
|
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
|
|
return "failed"
|
|
else:
|
|
attempt = job["attempt"] + 1
|
|
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
|
|
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
|
|
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
|
|
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
|
|
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
|
|
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
|
|
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
|
|
return "retry"
|
|
|
|
|
|
def cleanup_staging(job):
|
|
from .attachments import BUCKET, s3_client
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
try:
|
|
with connect_db() as conn:
|
|
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
|
|
if row and row["status"] == "ready" and row["staging_key"]:
|
|
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
|
|
with connect_db() as conn:
|
|
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
|
|
(job["attachment_id"], row["staging_key"]))
|
|
except (BotoCoreError, ClientError, psycopg.Error, OSError):
|
|
log.warning("staging cleanup deferred for %s", job["id"])
|
|
|
|
|
|
def process_message(raw):
|
|
try:
|
|
carrier = json.loads(raw)
|
|
if not isinstance(carrier, dict): carrier = {}
|
|
except (ValueError, TypeError):
|
|
carrier = {}
|
|
with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier),
|
|
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer:
|
|
return process_message_inner(raw, consumer)
|
|
|
|
|
|
def process_message_inner(raw, consumer):
|
|
try:
|
|
msg = json.loads(raw)
|
|
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
|
|
uuid.UUID(msg["job_id"])
|
|
uuid.UUID(msg["attachment_id"])
|
|
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
|
|
except (ValueError, KeyError, AssertionError, TypeError):
|
|
log.error("invalid internal broker message discarded")
|
|
return "ack" # no secrets printed, does not touch DB
|
|
with connect_db() as db:
|
|
status, state = claim_job(db, msg)
|
|
if status == "ignore":
|
|
return "ack"
|
|
if status == "busy":
|
|
return "requeue"
|
|
job, lease = state["job"], state["lease"]
|
|
consumer.set_attribute('job.attempt',job['attempt'])
|
|
consumer.set_attribute('job.id',str(job['id']))
|
|
ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt'])
|
|
start = time.monotonic()
|
|
ot.worker_busy = 1
|
|
def done(outcome):
|
|
attrs = {'job_type':'attachment.process.v1','outcome':outcome}
|
|
ot.count('lab_jobs',attrs)
|
|
ot.observe('lab_job_duration',time.monotonic()-start,attrs)
|
|
if outcome == 'retry':
|
|
ot.count('lab_job_retries',{'job_type':'attachment.process.v1'})
|
|
if outcome in ('retry','failed','requeue'):
|
|
consumer.set_status(Status(StatusCode.ERROR))
|
|
consumer.set_attribute('job.outcome',outcome)
|
|
ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt'])
|
|
try:
|
|
if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once':
|
|
with connect_db() as db:
|
|
inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone()
|
|
if inserted:
|
|
raise RuntimeError('injected transient retry')
|
|
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
|
|
with ot.span('image.subprocess'):
|
|
env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())}
|
|
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env)
|
|
if proc.returncode != 0:
|
|
with connect_db() as db:
|
|
outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20),
|
|
code="invalid_image" if proc.returncode == 20 else "processing_failed")
|
|
done(outcome)
|
|
return "ack" # next attempt is already safely in DB outbox
|
|
result = json.loads(proc.stdout)
|
|
variants = result["variants"]
|
|
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
|
|
raise ValueError("incomplete variants")
|
|
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
|
|
# One-time operator-only injected hard death, after all S3 writes and
|
|
# before DB result commit/ack; SQL PK prevents endless failure loop.
|
|
with connect_db() as db:
|
|
with db.transaction():
|
|
inserted = db.execute("""
|
|
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
|
|
ON CONFLICT DO NOTHING RETURNING job_id
|
|
""", (job["id"],)).fetchone()
|
|
if inserted:
|
|
os._exit(77)
|
|
with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
|
|
with connect_db() as db:
|
|
ok = publish_result(db, job=job, lease=lease, variants=variants)
|
|
if ok:
|
|
ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
|
|
cleanup_staging(job)
|
|
done('ready' if ok else 'stale')
|
|
return "ack"
|
|
except subprocess.TimeoutExpired:
|
|
with connect_db() as db:
|
|
outcome = fail_job(db, job, lease, code="processing_timeout")
|
|
done(outcome)
|
|
return "ack"
|
|
except Exception:
|
|
log.exception("job processing failure (internal details not sent to API)")
|
|
# Leave lease for dispatcher recovery if even DB unavailable.
|
|
try:
|
|
with connect_db() as db:
|
|
outcome = fail_job(db, job, lease)
|
|
done(outcome)
|
|
return "ack"
|
|
except psycopg.Error:
|
|
done("requeue")
|
|
return "requeue"
|
|
finally:
|
|
ot.worker_busy = 0
|
|
|
|
|
|
def worker_loop():
|
|
while True:
|
|
try:
|
|
with connection() as mq:
|
|
ch = setup_channel(mq)
|
|
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
|
|
|
|
def callback(channel, method, props, body):
|
|
outcome = process_message(body)
|
|
if outcome == "ack":
|
|
channel.basic_ack(method.delivery_tag)
|
|
else:
|
|
channel.basic_nack(method.delivery_tag, requeue=True)
|
|
time.sleep(2) # prevent tight redelivery spin on an active lease
|
|
|
|
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
|
|
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
|
|
ch.start_consuming()
|
|
except (AMQPError, OSError, psycopg.Error) as exc:
|
|
log.warning("worker reconnect after %s", type(exc).__name__)
|
|
time.sleep(2)
|
|
|
|
|
|
def replay_failed(ident):
|
|
with connect_db() as db:
|
|
with db.transaction():
|
|
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
|
|
if not row or row["status"] != "failed":
|
|
raise RuntimeError("Only failed jobs may be explicitly replayed")
|
|
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
|
|
if not attachment or not attachment["staging_key"]:
|
|
raise RuntimeError("Retained staging is required for replay")
|
|
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
|
|
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
|
|
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
|
|
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
|
|
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
|
|
print("Replayed failed job", ident)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
|
|
parser.add_argument("--id", help="UUID for explicit operator replay/status")
|
|
args = parser.parse_args()
|
|
if args.mode == "dispatch":
|
|
ot.init("dispatcher")
|
|
dispatch_loop()
|
|
elif args.mode == "worker":
|
|
ot.init("worker")
|
|
worker_loop()
|
|
elif args.mode in ("replay", "status"):
|
|
if not args.id:
|
|
parser.error("--id is required")
|
|
ident = str(uuid.UUID(args.id))
|
|
if args.mode == "replay":
|
|
replay_failed(ident)
|
|
else:
|
|
with connect_db() as db:
|
|
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|