Files

471 lines
24 KiB
Python

"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases.
There is no exactly-once transport guarantee. We guarantee eventual recovery of
committed intent and idempotent DB results under at-least-once redelivery.
Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`.
"""
import argparse
from contextlib import contextmanager
import json
import logging
import os
import random
import subprocess
import sys
import time
import threading
from . import telemetry as ot
from opentelemetry.trace import SpanKind, Status, StatusCode
import uuid
from datetime import datetime, timezone, timedelta
import pika
import psycopg
from pika.exceptions import AMQPError
from psycopg.rows import dict_row
log = logging.getLogger("lab5.jobs")
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
QUEUE = "attachment-processing.v1"
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60"))
LEASE_SECONDS = JOB_TIMEOUT + 45
@contextmanager
def connect_db():
# No credentials, SQL text, or parameters in spans.
with ot.dependency_span('db.job.transaction', kind=SpanKind.CLIENT, attrs={'db.system.name':'postgresql'}):
with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5, options='-c statement_timeout=5000') as conn:
yield conn
def job_payload(job_id, attachment_id, version, attempt):
return {
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
"attachment_id": str(attachment_id), "pipeline_version": version,
"attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(),
**ot.inject(),
}
@contextmanager
def connection():
params = pika.ConnectionParameters(
host=os.getenv("BROKER_HOST", "broker"), port=5672,
credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]),
heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120,
socket_timeout=5, connection_attempts=3, retry_delay=1)
mq = pika.BlockingConnection(params)
try:
yield mq
finally:
try:
if mq.is_open:
mq.close()
except AMQPError:
pass
def setup_channel(conn):
ch = conn.channel()
ch.queue_declare(queue=QUEUE, durable=True)
return ch
def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None):
payload = job_payload(job_id, attachment_id, version, attempt)
if not payload.get('traceparent'):
# Lease rescue/replay has no active consumer context. Preserve the first
# acceptance context rather than inventing a new disconnected trace.
previous = db.execute("SELECT payload FROM job_outbox WHERE job_id=%s ORDER BY created_at,id LIMIT 1", (job_id,)).fetchone()
if previous:
payload.update({k: previous['payload'][k] for k in ('traceparent','tracestate') if k in previous['payload']})
db.execute("""
INSERT INTO job_outbox(id,job_id,payload,available_at)
VALUES (%s,%s,%s::jsonb,COALESCE(%s,now()))
""", (str(uuid.uuid4()), job_id, json.dumps(payload), available_at))
def cleanup_abandoned_staging():
"""Best effort cleanup of retained stages; never delete current processing work."""
from .attachments import BUCKET, s3_client
client = s3_client()
retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
with connect_db() as db:
rows = db.execute("""
SELECT id,staging_key FROM attachments
WHERE staging_key IS NOT NULL AND
((status='ready' AND updated_at < now()-interval '300 seconds') OR
(status='failed' AND updated_at < now()-(%s*interval '1 second')))
LIMIT 50
""", (retention,)).fetchall()
for row in rows:
with connect_db() as db:
with db.transaction():
# Row-level lock serializes against operator replay.
current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone()
if not current or current["staging_key"] != row["staging_key"]:
continue
age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds()
if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention):
continue
client.delete_object(Bucket=BUCKET, Key=row["staging_key"])
db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"]))
# Also prune staging uploads whose DB COMMIT failed, after a long retention.
from botocore.exceptions import ClientError
for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"):
for obj in page.get("Contents", []):
if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention:
continue
key = obj["Key"]
with connect_db() as db:
with db.transaction():
referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
if not referenced:
client.delete_object(Bucket=BUCKET, Key=key)
def rescue_expired_jobs(conn):
"""Requeue leased jobs whose worker died (durably, no manual task recreation)."""
with conn.transaction():
rows = conn.execute("""
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt
FROM processing_jobs j
WHERE j.status='processing' AND j.leased_until < now()
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if row["attempt"] >= MAX_ATTEMPTS:
conn.execute("""UPDATE processing_jobs
SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now()
WHERE id=%s""", (row["id"],))
conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now()
WHERE id=%s""", (row["attachment_id"],))
else:
attempt = row["attempt"] + 1
conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,
lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now()
WHERE id=%s""", (attempt, row["id"]))
conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],))
queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt)
log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt)
def sample_backlog():
# Query durable *queued* jobs, including delayed retries and unpublished outbox.
# No broker membership scan; oldest age is wall clock since initial acceptance.
while True:
try:
with psycopg.connect(os.environ['DATABASE_URL'], row_factory=dict_row, connect_timeout=3, options='-c statement_timeout=2000') as db:
row = db.execute("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)),0) AS age FROM processing_jobs WHERE status='queued'").fetchone()
ot.backlog_depth, ot.backlog_age = row['depth'], float(row['age'])
except psycopg.Error:
ot.backlog_depth = ot.backlog_age = None
time.sleep(3)
def dispatch_loop():
"""Publish outbox rows with publisher confirm, mark only AFTER confirm."""
threading.Thread(target=sample_backlog, daemon=True).start()
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.confirm_delivery()
last_clean = time.monotonic()
last_queue_sample = 0
while True:
# Periodic lease recovery works even if original message was consumed.
with connect_db() as conn:
rescue_expired_jobs(conn)
with conn.transaction():
rows = conn.execute("""
SELECT id,payload FROM job_outbox
WHERE published_at IS NULL AND available_at<=now()
ORDER BY available_at,created_at
FOR UPDATE SKIP LOCKED LIMIT 20
""").fetchall()
for row in rows:
if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once":
# Persist one-time fault before hard exit; outbox still
# unmarked and dispatcher will publish on restart.
with connect_db() as faultdb:
inserted = faultdb.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish')
ON CONFLICT DO NOTHING RETURNING job_id
""", (row["payload"]["job_id"],)).fetchone()
if inserted:
os._exit(78)
payload = row['payload']
with ot.span('rabbitmq.publish', kind=SpanKind.PRODUCER, context=ot.extract(payload),
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE,'job.attempt':payload['attempt']}) as producer:
# Propagate actual producer span, while DB retains the original intent.
raw = json.dumps({**payload, **ot.inject()}, separators=(",", ":"))
ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"),
mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2))
conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],))
ot.event('job.published', job_id=payload['job_id'], attempt=payload['attempt'])
if time.monotonic() - last_queue_sample > 3:
q = ch.queue_declare(queue=QUEUE, passive=True).method
ot.broker_ready, ot.broker_consumers = q.message_count, q.consumer_count
last_queue_sample = time.monotonic()
if time.monotonic() - last_clean > 60:
try:
cleanup_abandoned_staging()
except Exception:
log.warning("staging janitor failed; retry later")
last_clean = time.monotonic()
if not rows:
mq.sleep(1)
except (AMQPError, OSError, psycopg.Error) as exc:
ot.broker_ready = ot.broker_consumers = None
log.warning("dispatcher reconnect after %s", type(exc).__name__)
time.sleep(2)
def claim_job(db, message):
job_id = message["job_id"]
with db.transaction():
job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone()
if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]:
return "ignore", None
if job["status"] == "processing":
return "busy", None
if job["next_run_at"] > datetime.now(timezone.utc):
return "busy", None
lease = str(uuid.uuid4())
db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s,
leased_until=now()+(%s*interval '1 second'),updated_at=now()
WHERE id=%s""", (lease, LEASE_SECONDS, job_id))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],))
return "claimed", {"job": job, "lease": lease}
def publish_result(db, *, job, lease, variants):
"""Final result is committed atomically; stale leases/versions cannot publish."""
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]:
log.warning("stale worker result ignored %s", job["id"])
return False
for v in variants:
db.execute("""
INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET
object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type,
size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256,
width=EXCLUDED.width,height=EXCLUDED.height
""", (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"],
v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"]))
db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now()
WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"]))
db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL,
error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],))
return True
def fail_job(db, job, lease, *, permanent=False, code="processing_failed"):
with db.transaction():
current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone()
if current["status"] != "processing" or str(current["lease_token"]) != lease:
return "stale"
if permanent or job["attempt"] >= MAX_ATTEMPTS:
code = code if permanent else "retry_exhausted"
db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL,
leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"]))
db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"]))
return "failed"
else:
attempt = job["attempt"] + 1
delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0)
run_at = datetime.now(timezone.utc) + timedelta(seconds=delay)
db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s,
lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"]))
db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],))
queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at)
log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay)
return "retry"
def cleanup_staging(job):
from .attachments import BUCKET, s3_client
from botocore.exceptions import BotoCoreError, ClientError
try:
with connect_db() as conn:
row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone()
if row and row["status"] == "ready" and row["staging_key"]:
s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"])
with connect_db() as conn:
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s",
(job["attachment_id"], row["staging_key"]))
except (BotoCoreError, ClientError, psycopg.Error, OSError):
log.warning("staging cleanup deferred for %s", job["id"])
def process_message(raw):
try:
carrier = json.loads(raw)
if not isinstance(carrier, dict): carrier = {}
except (ValueError, TypeError):
carrier = {}
with ot.span('rabbitmq.consume', kind=SpanKind.CONSUMER, context=ot.extract(carrier),
attrs={'messaging.system':'rabbitmq','messaging.destination.name':QUEUE}) as consumer:
return process_message_inner(raw, consumer)
def process_message_inner(raw, consumer):
try:
msg = json.loads(raw)
assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1"
uuid.UUID(msg["job_id"])
uuid.UUID(msg["attachment_id"])
assert type(msg["attempt"]) is int and msg["attempt"] >= 1
except (ValueError, KeyError, AssertionError, TypeError):
log.error("invalid internal broker message discarded")
return "ack" # no secrets printed, does not touch DB
with connect_db() as db:
status, state = claim_job(db, msg)
if status == "ignore":
return "ack"
if status == "busy":
return "requeue"
job, lease = state["job"], state["lease"]
consumer.set_attribute('job.attempt',job['attempt'])
consumer.set_attribute('job.id',str(job['id']))
ot.event('job.claimed', job_id=str(job['id']), attempt=job['attempt'])
start = time.monotonic()
ot.worker_busy = 1
def done(outcome):
attrs = {'job_type':'attachment.process.v1','outcome':outcome}
ot.count('lab_jobs',attrs)
ot.observe('lab_job_duration',time.monotonic()-start,attrs)
if outcome == 'retry':
ot.count('lab_job_retries',{'job_type':'attachment.process.v1'})
if outcome in ('retry','failed','requeue'):
consumer.set_status(Status(StatusCode.ERROR))
consumer.set_attribute('job.outcome',outcome)
ot.event('job.'+outcome, level=logging.ERROR if outcome in ('failed','retry','requeue') else logging.INFO, job_id=str(job['id']), attempt=job['attempt'])
try:
if os.getenv('LAB6_FAULT_RETRY_ONCE') == 'once':
with connect_db() as db:
inserted = db.execute("INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'lab6_retry') ON CONFLICT DO NOTHING RETURNING job_id", (job['id'],)).fetchone()
if inserted:
raise RuntimeError('injected transient retry')
cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])]
with ot.span('image.subprocess'):
env = {**os.environ, 'LAB6_PARENT_CONTEXT': json.dumps(ot.inject())}
proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False, env=env)
if proc.returncode != 0:
with connect_db() as db:
outcome = fail_job(db, job, lease, permanent=(proc.returncode == 20),
code="invalid_image" if proc.returncode == 20 else "processing_failed")
done(outcome)
return "ack" # next attempt is already safely in DB outbox
result = json.loads(proc.stdout)
variants = result["variants"]
if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]:
raise ValueError("incomplete variants")
if os.getenv("LAB5_FAULT_AFTER_S3") == "once":
# One-time operator-only injected hard death, after all S3 writes and
# before DB result commit/ack; SQL PK prevents endless failure loop.
with connect_db() as db:
with db.transaction():
inserted = db.execute("""
INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3')
ON CONFLICT DO NOTHING RETURNING job_id
""", (job["id"],)).fetchone()
if inserted:
os._exit(77)
with ot.span("db.publish_result", kind=SpanKind.CLIENT, attrs={"db.system.name":"postgresql"}):
with connect_db() as db:
ok = publish_result(db, job=job, lease=lease, variants=variants)
if ok:
ot.observe('lab_ready_duration',(datetime.now(timezone.utc)-job['created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
cleanup_staging(job)
done('ready' if ok else 'stale')
return "ack"
except subprocess.TimeoutExpired:
with connect_db() as db:
outcome = fail_job(db, job, lease, code="processing_timeout")
done(outcome)
return "ack"
except Exception:
log.exception("job processing failure (internal details not sent to API)")
# Leave lease for dispatcher recovery if even DB unavailable.
try:
with connect_db() as db:
outcome = fail_job(db, job, lease)
done(outcome)
return "ack"
except psycopg.Error:
done("requeue")
return "requeue"
finally:
ot.worker_busy = 0
def worker_loop():
while True:
try:
with connection() as mq:
ch = setup_channel(mq)
ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1"))))
def callback(channel, method, props, body):
outcome = process_message(body)
if outcome == "ack":
channel.basic_ack(method.delivery_tag)
else:
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2) # prevent tight redelivery spin on an active lease
ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False)
log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT)
ch.start_consuming()
except (AMQPError, OSError, psycopg.Error) as exc:
log.warning("worker reconnect after %s", type(exc).__name__)
time.sleep(2)
def replay_failed(ident):
with connect_db() as db:
with db.transaction():
row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone()
if not row or row["status"] != "failed":
raise RuntimeError("Only failed jobs may be explicitly replayed")
attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone()
if not attachment or not attachment["staging_key"]:
raise RuntimeError("Retained staging is required for replay")
db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(),
lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],))
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,))
queue_outbox(db, row["id"], ident, row["pipeline_version"], 1)
db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"]))
print("Replayed failed job", ident)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"])
parser.add_argument("--id", help="UUID for explicit operator replay/status")
args = parser.parse_args()
if args.mode == "dispatch":
ot.init("dispatcher")
dispatch_loop()
elif args.mode == "worker":
ot.init("worker")
worker_loop()
elif args.mode in ("replay", "status"):
if not args.id:
parser.error("--id is required")
ident = str(uuid.UUID(args.id))
if args.mode == "replay":
replay_failed(ident)
else:
with connect_db() as db:
print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str))
if __name__ == "__main__":
main()