Files

387 lines
19 KiB
Python

"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
There is no exactly-once transport guarantee. We guarantee eventual recovery of
committed intent and idempotent DB results under at-least-once redelivery.
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
"""
import argparse
from contextlib import contextmanager
import json
import logging
import os
import random
import subprocess
import sys
import time
import uuid
from datetime import datetime, timezone, timedelta
import pika
import psycopg
from pika.exceptions import AMQPError
from psycopg.rows import dict_row
log = logging.getLogger("lab5.jobs")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
QUEUE = "attachment-processing.v1"
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
LEASE_SECONDS = JOB_TIMEOUT + 45
def connect_db():
return psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5)
def job_payload(job_id, attachment_id, version, attempt):
return {
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": version,
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
}
@contextmanager
def connection():
params = pika.ConnectionParameters(
host=os.getenv("BROKER_HOST", "broker"), port=5672,
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
socket_timeout=5, connection_attempts=3, retry_delay=1)
mq = pika.BlockingConnection(params)
try:
yield mq
finally:
try:
if mq.is_open:
mq.close()
except AMQPError:
pass
def setup_channel(conn):
ch = conn.channel()
ch.queue_declare(queue=QUEUE, durable=True)
return ch
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
payload = job_payload(job_id, attachment_id, version, attempt)
db.execute("""
INSERT INTO job_outbox(id,job_id,payload,available_at)
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
def cleanup_abandoned_staging():
"""Best effort cleanup of retained stages; never delete current processing work."""
from .attachments import BUCKET, s3_client
client = s3_client()
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
with connect_db() as db:
rows = db.execute("""
SELECT id,staging_key FROM attachments
WHERE staging_key IS NOT NULL AND
((status='ready' AND updated_at < now()-interval '300 seconds') OR
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
LIMIT 50
""", (retention,)).fetchall()
for row in rows:
with connect_db() as db:
with db.transaction():
# Row-level lock serializes against operator replay.
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
if not current or current["staging_key"] != row["staging_key"]:
continue
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
continue
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
from botocore.exceptions import ClientError
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
continue
key = obj["Key"]
with connect_db() as db:
with db.transaction():
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
if not referenced:
client.delete_object(Bucket=BUCKET, Key=key)
def rescue_expired_jobs(conn):
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
with conn.transaction():
rows = conn.execute("""
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
FROM processing_jobs j
WHERE j.status='processing' AND j.leased_until < now()
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if row["attempt"] >= MAX_ATTEMPTS:
conn.execute("""UPDATE processing_jobs
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
WHERE id=%s""", (row["id"],))
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
WHERE id=%s""", (row["attachment_id"],))
else:
attempt = row["attempt"] + 1
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
WHERE id=%s""", (attempt, row["id"]))
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
def dispatch_loop():
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.confirm_delivery()
last_clean = time.monotonic()
while True:
# Periodic lease recovery works even if original message was consumed.
with connect_db() as conn:
rescue_expired_jobs(conn)
with conn.transaction():
rows = conn.execute("""
SELECT id,payload FROM job_outbox
WHERE published_at IS NULL AND available_at<=now()
ORDER BY available_at,created_at
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
# Persist one-time fault before hard exit; outbox still
# unmarked and dispatcher will publish on restart.
with connect_db() as faultdb:
inserted = faultdb.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
ON CONFLICT DO NOTHING RETURNING job_id
""", (row["payload"]["job_id"],)).fetchone()
if inserted:
os._exit(78)
raw = json.dumps(row["payload"], separators=(",", ":"))
# Mandatory + confirm; an ambiguous publish will be retried.
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
if time.monotonic() - last_clean > 60:
try:
cleanup_abandoned_staging()
except Exception:
log.warning("staging janitor failed; retry later")
last_clean = time.monotonic()
if not rows:
time.sleep(1)
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("dispatcher reconnect after %s", type(exc).__name__)
time.sleep(2)
def claim_job(db, message):
job_id = message["job_id"]
with db.transaction():
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
return "ignore", None
if job["status"] == "processing":
return "busy", None
if job["next_run_at"] > datetime.now(timezone.utc):
return "busy", None
lease = str(uuid.uuid4())
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
leased_until=now()+(%s*interval '1 second'),updated_at=now()
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
return "claimed", {"job": job, "lease": lease}
def publish_result(db, *, job, lease, variants):
"""Final result is committed atomically; stale leases/versions cannot publish."""
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
log.warning("stale worker result ignored %s", job["id"])
return False
for v in variants:
db.execute("""
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
width=EXCLUDED.width,height=EXCLUDED.height
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
return True
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease:
return
if permanent or job["attempt"] >= MAX_ATTEMPTS:
code = code if permanent else "retry_exhausted"
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
else:
attempt = job["attempt"] + 1
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
def cleanup_staging(job):
from .attachments import BUCKET, s3_client
from botocore.exceptions import BotoCoreError, ClientError
try:
with connect_db() as conn:
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
if row and row["status"] == "ready" and row["staging_key"]:
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
with connect_db() as conn:
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
(job["attachment_id"], row["staging_key"]))
except (BotoCoreError, ClientError, psycopg.Error, OSError):
log.warning("staging cleanup deferred for %s", job["id"])
def process_message(raw):
try:
msg = json.loads(raw)
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
uuid.UUID(msg["job_id"])
uuid.UUID(msg["attachment_id"])
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
except (ValueError, KeyError, AssertionError, TypeError):
log.error("invalid internal broker message discarded")
return "ack" # no secrets printed, does not touch DB
with connect_db() as db:
status, state = claim_job(db, msg)
if status == "ignore":
return "ack"
if status == "busy":
return "requeue"
job, lease = state["job"], state["lease"]
try:
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False)
if proc.returncode != 0:
with connect_db() as db:
fail_job(db, job, lease, permanent=(proc.returncode == 20),
code="invalid_image" if proc.returncode == 20 else "processing_failed")
return "ack" # next attempt is already safely in DB outbox
result = json.loads(proc.stdout)
variants = result["variants"]
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
raise ValueError("incomplete variants")
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
# One-time operator-only injected hard death, after all S3 writes and
# before DB result commit/ack; SQL PK prevents endless failure loop.
with connect_db() as db:
with db.transaction():
inserted = db.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
ON CONFLICT DO NOTHING RETURNING job_id
""", (job["id"],)).fetchone()
if inserted:
os._exit(77)
with connect_db() as db:
ok = publish_result(db, job=job, lease=lease, variants=variants)
if ok:
cleanup_staging(job)
return "ack"
except subprocess.TimeoutExpired:
with connect_db() as db:
fail_job(db, job, lease, code="processing_timeout")
return "ack"
except Exception:
log.exception("job processing failure (internal details not sent to API)")
# Leave lease for dispatcher recovery if even DB unavailable.
try:
with connect_db() as db:
fail_job(db, job, lease)
return "ack"
except psycopg.Error:
return "requeue"
def worker_loop():
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
def callback(channel, method, props, body):
outcome = process_message(body)
if outcome == "ack":
channel.basic_ack(method.delivery_tag)
else:
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2) # prevent tight redelivery spin on an active lease
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
ch.start_consuming()
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("worker reconnect after %s", type(exc).__name__)
time.sleep(2)
def replay_failed(ident):
with connect_db() as db:
with db.transaction():
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
if not row or row["status"] != "failed":
raise RuntimeError("Only failed jobs may be explicitly replayed")
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not attachment or not attachment["staging_key"]:
raise RuntimeError("Retained staging is required for replay")
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
print("Replayed failed job", ident)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
parser.add_argument("--id", help="UUID for explicit operator replay/status")
args = parser.parse_args()
if args.mode == "dispatch":
dispatch_loop()
elif args.mode == "worker":
worker_loop()
elif args.mode in ("replay", "status"):
if not args.id:
parser.error("--id is required")
ident = str(uuid.UUID(args.id))
if args.mode == "replay":
replay_failed(ident)
else:
with connect_db() as db:
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
if __name__ == "__main__":
main()