"""Lab 5 worker: Kafka consumer -> claim (lease) in PostgreSQL -> staging -> pipeline -> final S3 + COMMIT -> offset commit. PostgreSQL is the source of truth for job state; Kafka is the at-least-once transport. The Kafka offset is committed only AFTER the durable effect (final objects + COMMIT of the result) or after a durable decision (retry scheduled / failed). """ from __future__ import annotations import hashlib import json import logging import os import signal import socket import subprocess import sys import tempfile import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path import psycopg from botocore.exceptions import BotoCoreError, ClientError from confluent_kafka import Consumer, TopicPartition from . import jobs logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s worker %(message)s") log = logging.getLogger("worker") WORKER_ID = os.getenv("INSTANCE_ID") or socket.gethostname() CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2")) FAULT_EXIT_AFTER_S3 = os.getenv("FAULT_WORKER_EXIT_AFTER_S3", "0") == "1" # test hook: crash after the S3 write, before COMMIT/ack IMG_MEMORY_MB = os.getenv("IMG_MEMORY_MB", "1024") CODE_BY_EXIT = {2: "invalid_image", 3: "image_too_large", 4: "image_too_complex"} # permanent errors of imgproc STOP = False class Permanent(Exception): def __init__(self, code: str): super().__init__(code) self.code = code class Transient(Exception): pass def connect(): return psycopg.connect(connect_timeout=5, options="-c statement_timeout=10000") def parse_job(raw: bytes) -> dict | None: """Validate against jobs.schema.json (the subset that matters). Poison messages are logged and skipped.""" try: m = json.loads(raw) assert m["schema_version"] == 1 and m["type"] == jobs.JOB_TYPE uuid.UUID(m["job_id"]); uuid.UUID(m["attachment_id"]) assert isinstance(m["pipeline_version"], int) and m["pipeline_version"] >= 1 return m except Exception: log.error("poison message skipped (schema violation), size=%d", len(raw)) return None def claim(msg: dict) -> dict | None: """queued -> processing with a lease. Returns None when this delivery must be a no-op (ack).""" with connect() as conn: row = conn.execute( "SELECT status, attempt, max_attempts, attachment_id, pipeline_version, " "(next_attempt_at <= now()) AS due, (lease_until IS NOT NULL AND lease_until < now()) AS lease_expired " "FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (msg["job_id"],)).fetchone() if row is None: log.warning("job %s unknown, skipped", msg["job_id"]); return None status, attempt, max_attempts, att_id, pv, due, expired = row if pv != msg["pipeline_version"]: return None # stale job of another pipeline version never overwrites the newer one if status in ("ready", "failed"): log.info("job %s already %s: duplicate delivery is a no-op", msg["job_id"], status); return None if status == "queued" and not due: return None # early duplicate: the delayed outbox row will deliver it later if status == "processing" and not expired: log.info("job %s is leased by another worker: skip (dispatcher recovers it if that worker dies)", msg["job_id"]); return None if attempt >= max_attempts: # lease expired and no attempts left jobs.fail_job(conn, msg["job_id"], att_id, "retry_exhausted", attempt) return None attempt += 1 conn.execute("UPDATE attachment_jobs SET status='processing', attempt=%s, worker_id=%s, " "lease_until=now()+make_interval(secs=>%s), updated_at=now() WHERE job_id=%s", (attempt, WORKER_ID, jobs.LEASE_SECONDS, msg["job_id"])) conn.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,)) jobs.add_event(conn, msg["job_id"], "claimed", f"attempt={attempt} worker={WORKER_ID}") a = conn.execute("SELECT sha256, staging_key, s3_key, content_type, filename FROM attachments WHERE id=%s", (att_id,)).fetchone() return {"job_id": msg["job_id"], "attachment_id": att_id, "pv": pv, "attempt": attempt, "max_attempts": max_attempts, "sha256": a[0].strip(), "staging_key": a[1], "original_key": a[2], "content_type": a[3]} def run_pipeline(s3, c: dict) -> dict: """Download staging -> verify -> child process with limits -> idempotent PUTs of deterministic keys.""" if not c["staging_key"]: raise Permanent("staging_missing") with tempfile.TemporaryDirectory(prefix="job-") as tmp: src, out = Path(tmp) / "in.bin", Path(tmp) / "out" out.mkdir() try: s3.download_file(jobs.S3_BUCKET, c["staging_key"], str(src)) except ClientError as e: if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): raise Permanent("staging_missing") raise if hashlib.sha256(src.read_bytes()).hexdigest() != c["sha256"]: raise Permanent("integrity_error") # bytes differ from what the API accepted try: r = subprocess.run([sys.executable, "-m", "src.imgproc", str(src), str(out)], capture_output=True, text=True, timeout=jobs.JOB_TIMEOUT_SECONDS, env={"PATH": os.environ.get("PATH", ""), "IMG_MEMORY_MB": IMG_MEMORY_MB, "IMG_CPU_SECONDS": str(jobs.JOB_TIMEOUT_SECONDS), "MAX_IMAGE_PIXELS": os.getenv("MAX_IMAGE_PIXELS", "40000000")}) # no secrets in the child except subprocess.TimeoutExpired: raise Transient("job timeout") # child is killed by subprocess.run if r.returncode in CODE_BY_EXIT: raise Permanent(CODE_BY_EXIT[r.returncode]) if r.returncode != 0: raise Transient(f"imgproc exit {r.returncode}") meta = json.loads(r.stdout) files = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} # 1) original into the final prefix (same key every time => repeated PUT is harmless) s3.upload_file(str(src), jobs.S3_BUCKET, c["original_key"], ExtraArgs={"ContentType": c["content_type"]}) # 2) variants for name in jobs.VARIANT_ORDER: key = jobs.final_key(c["attachment_id"], c["pv"], name) s3.upload_file(str(out / files[name]), jobs.S3_BUCKET, key, ExtraArgs={"ContentType": meta[name]["content_type"]}) meta[name]["s3_key"] = key return meta def commit_ready(c: dict, meta: dict) -> bool: """Atomically publish the result. False if this attempt no longer owns the job (lease lost to a newer attempt).""" with connect() as conn: row = conn.execute("SELECT status, attempt FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (c["job_id"],)).fetchone() if row is None or row[0] != "processing" or row[1] != c["attempt"]: log.warning("job %s: attempt %s lost ownership, result not committed", c["job_id"], c["attempt"]); return False for name in jobs.VARIANT_ORDER: v = meta[name] conn.execute("INSERT INTO attachment_variants(attachment_id,pipeline_version,name,s3_key,content_type,size_bytes,width,height,sha256) " "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (attachment_id,pipeline_version,name) DO NOTHING", (c["attachment_id"], c["pv"], name, v["s3_key"], v["content_type"], v["size_bytes"], v["width"], v["height"], v["sha256"])) conn.execute("UPDATE attachment_jobs SET status='ready', error_code=NULL, lease_until=NULL, updated_at=now() WHERE job_id=%s", (c["job_id"],)) conn.execute("UPDATE attachments SET status='ready', error_code=NULL, staging_key=NULL, updated_at=now() WHERE id=%s", (c["attachment_id"],)) jobs.add_event(conn, c["job_id"], "ready", f"attempt={c['attempt']}") return True def record_failure(c: dict, exc: Exception) -> None: """Permanent -> failed in the PostgreSQL registry. Transient -> queued with backoff+jitter (delayed outbox row) or failed when attempts are exhausted.""" with connect() as conn: row = conn.execute("SELECT status, attempt FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (c["job_id"],)).fetchone() if row is None or row[0] != "processing" or row[1] != c["attempt"]: return if isinstance(exc, Permanent): jobs.fail_job(conn, c["job_id"], c["attachment_id"], exc.code, c["attempt"]) elif c["attempt"] >= c["max_attempts"]: jobs.fail_job(conn, c["job_id"], c["attachment_id"], "retry_exhausted", c["attempt"]) else: delay = jobs.backoff_seconds(c["attempt"]) conn.execute("UPDATE attachment_jobs SET status='queued', lease_until=NULL, next_attempt_at=now()+make_interval(secs=>%s), " "error_code=NULL, updated_at=now() WHERE job_id=%s", (delay, c["job_id"])) conn.execute("UPDATE attachments SET status='queued', updated_at=now() WHERE id=%s", (c["attachment_id"],)) jobs.add_event(conn, c["job_id"], "retry_scheduled", f"attempt={c['attempt']} delay={delay:.1f}s reason={type(exc).__name__}") jobs.enqueue(conn, c["job_id"], c["attachment_id"], c["pv"], c["attempt"] + 1, delay_seconds=delay) def handle(s3, raw: bytes) -> None: msg = parse_job(raw) if msg is None: return c = claim(msg) if c is None: return try: meta = run_pipeline(s3, c) if FAULT_EXIT_AFTER_S3 and c["attempt"] == 1: log.error("FAULT: exiting after S3 write, before COMMIT and offset commit") os._exit(137) if commit_ready(c, meta): try: s3.delete_object(Bucket=jobs.S3_BUCKET, Key=c["staging_key"]) # staging cleanup after the safe COMMIT except Exception: log.warning("staging cleanup failed; sweeper will remove it") except (Permanent, Transient, ClientError, BotoCoreError, OSError, json.JSONDecodeError) as exc: log.warning("job %s attempt %s failed: %s", c["job_id"], c["attempt"], type(exc).__name__) record_failure(c, exc) def main() -> None: def request_stop(*_): global STOP STOP = True signal.signal(signal.SIGTERM, request_stop) signal.signal(signal.SIGINT, request_stop) s3 = jobs.make_s3() consumer = Consumer({ "bootstrap.servers": jobs.KAFKA_BOOTSTRAP, "group.id": "attachment-workers", "enable.auto.commit": False, "auto.offset.reset": "earliest", "isolation.level": "read_committed", "max.poll.interval.ms": 600000, "session.timeout.ms": 30000, "queued.max.messages.kbytes": 1024, "fetch.max.bytes": 1048576, # bounded client-side prefetch }) consumer.subscribe([jobs.JOBS_TOPIC]) log.info("worker %s started, concurrency=%d, max_attempts=%d, lease=%ds", WORKER_ID, CONCURRENCY, jobs.JOB_MAX_ATTEMPTS, jobs.LEASE_SECONDS) with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: while not STOP: msgs = consumer.consume(num_messages=CONCURRENCY, timeout=1.0) # at most CONCURRENCY jobs in flight batch = [] for m in msgs: if m.error(): log.warning("kafka error: %s", m.error()); continue batch.append(m) if not batch: continue futures = [pool.submit(handle, s3, m.value()) for m in batch] for f in futures: f.result() # unexpected exception => process exits, offsets stay uncommitted, Kafka redelivers # ACK boundary: commit the highest processed offset+1 per partition, only after every job of the batch is durable top = {} for m in batch: top[(m.topic(), m.partition())] = max(top.get((m.topic(), m.partition()), -1), m.offset() + 1) consumer.commit(offsets=[TopicPartition(t, p, o) for (t, p), o in top.items()], asynchronous=False) consumer.close() if __name__ == "__main__": main()