231 lines
12 KiB
Python
231 lines
12 KiB
Python
"""Lab 5 worker: Kafka consumer -> claim (lease) in PostgreSQL -> staging -> pipeline -> final S3 + COMMIT -> offset commit.
|
|
|
|
PostgreSQL is the source of truth for job state; Kafka is the at-least-once transport. The Kafka offset is committed
|
|
only AFTER the durable effect (final objects + COMMIT of the result) or after a durable decision (retry scheduled / failed).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
import psycopg
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from confluent_kafka import Consumer, TopicPartition
|
|
|
|
from . import jobs
|
|
|
|
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s worker %(message)s")
|
|
log = logging.getLogger("worker")
|
|
WORKER_ID = os.getenv("INSTANCE_ID") or socket.gethostname()
|
|
CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2"))
|
|
FAULT_EXIT_AFTER_S3 = os.getenv("FAULT_WORKER_EXIT_AFTER_S3", "0") == "1" # test hook: crash after the S3 write, before COMMIT/ack
|
|
IMG_MEMORY_MB = os.getenv("IMG_MEMORY_MB", "1024")
|
|
CODE_BY_EXIT = {2: "invalid_image", 3: "image_too_large", 4: "image_too_complex"} # permanent errors of imgproc
|
|
STOP = False
|
|
|
|
|
|
class Permanent(Exception):
|
|
def __init__(self, code: str):
|
|
super().__init__(code)
|
|
self.code = code
|
|
|
|
|
|
class Transient(Exception):
|
|
pass
|
|
|
|
|
|
def connect():
|
|
return psycopg.connect(connect_timeout=5, options="-c statement_timeout=10000")
|
|
|
|
|
|
def parse_job(raw: bytes) -> dict | None:
|
|
"""Validate against jobs.schema.json (the subset that matters). Poison messages are logged and skipped."""
|
|
try:
|
|
m = json.loads(raw)
|
|
assert m["schema_version"] == 1 and m["type"] == jobs.JOB_TYPE
|
|
uuid.UUID(m["job_id"]); uuid.UUID(m["attachment_id"])
|
|
assert isinstance(m["pipeline_version"], int) and m["pipeline_version"] >= 1
|
|
return m
|
|
except Exception:
|
|
log.error("poison message skipped (schema violation), size=%d", len(raw))
|
|
return None
|
|
|
|
|
|
def claim(msg: dict) -> dict | None:
|
|
"""queued -> processing with a lease. Returns None when this delivery must be a no-op (ack)."""
|
|
with connect() as conn:
|
|
row = conn.execute(
|
|
"SELECT status, attempt, max_attempts, attachment_id, pipeline_version, "
|
|
"(next_attempt_at <= now()) AS due, (lease_until IS NOT NULL AND lease_until < now()) AS lease_expired "
|
|
"FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (msg["job_id"],)).fetchone()
|
|
if row is None:
|
|
log.warning("job %s unknown, skipped", msg["job_id"]); return None
|
|
status, attempt, max_attempts, att_id, pv, due, expired = row
|
|
if pv != msg["pipeline_version"]:
|
|
return None # stale job of another pipeline version never overwrites the newer one
|
|
if status in ("ready", "failed"):
|
|
log.info("job %s already %s: duplicate delivery is a no-op", msg["job_id"], status); return None
|
|
if status == "queued" and not due:
|
|
return None # early duplicate: the delayed outbox row will deliver it later
|
|
if status == "processing" and not expired:
|
|
log.info("job %s is leased by another worker: skip (dispatcher recovers it if that worker dies)", msg["job_id"]); return None
|
|
if attempt >= max_attempts: # lease expired and no attempts left
|
|
jobs.fail_job(conn, msg["job_id"], att_id, "retry_exhausted", attempt)
|
|
return None
|
|
attempt += 1
|
|
conn.execute("UPDATE attachment_jobs SET status='processing', attempt=%s, worker_id=%s, "
|
|
"lease_until=now()+make_interval(secs=>%s), updated_at=now() WHERE job_id=%s",
|
|
(attempt, WORKER_ID, jobs.LEASE_SECONDS, msg["job_id"]))
|
|
conn.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,))
|
|
jobs.add_event(conn, msg["job_id"], "claimed", f"attempt={attempt} worker={WORKER_ID}")
|
|
a = conn.execute("SELECT sha256, staging_key, s3_key, content_type, filename FROM attachments WHERE id=%s", (att_id,)).fetchone()
|
|
return {"job_id": msg["job_id"], "attachment_id": att_id, "pv": pv, "attempt": attempt, "max_attempts": max_attempts,
|
|
"sha256": a[0].strip(), "staging_key": a[1], "original_key": a[2], "content_type": a[3]}
|
|
|
|
|
|
def run_pipeline(s3, c: dict) -> dict:
|
|
"""Download staging -> verify -> child process with limits -> idempotent PUTs of deterministic keys."""
|
|
if not c["staging_key"]:
|
|
raise Permanent("staging_missing")
|
|
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
|
|
src, out = Path(tmp) / "in.bin", Path(tmp) / "out"
|
|
out.mkdir()
|
|
try:
|
|
s3.download_file(jobs.S3_BUCKET, c["staging_key"], str(src))
|
|
except ClientError as e:
|
|
if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
|
|
raise Permanent("staging_missing")
|
|
raise
|
|
if hashlib.sha256(src.read_bytes()).hexdigest() != c["sha256"]:
|
|
raise Permanent("integrity_error") # bytes differ from what the API accepted
|
|
try:
|
|
r = subprocess.run([sys.executable, "-m", "src.imgproc", str(src), str(out)], capture_output=True, text=True,
|
|
timeout=jobs.JOB_TIMEOUT_SECONDS, env={"PATH": os.environ.get("PATH", ""), "IMG_MEMORY_MB": IMG_MEMORY_MB,
|
|
"IMG_CPU_SECONDS": str(jobs.JOB_TIMEOUT_SECONDS),
|
|
"MAX_IMAGE_PIXELS": os.getenv("MAX_IMAGE_PIXELS", "40000000")}) # no secrets in the child
|
|
except subprocess.TimeoutExpired:
|
|
raise Transient("job timeout") # child is killed by subprocess.run
|
|
if r.returncode in CODE_BY_EXIT:
|
|
raise Permanent(CODE_BY_EXIT[r.returncode])
|
|
if r.returncode != 0:
|
|
raise Transient(f"imgproc exit {r.returncode}")
|
|
meta = json.loads(r.stdout)
|
|
files = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"}
|
|
# 1) original into the final prefix (same key every time => repeated PUT is harmless)
|
|
s3.upload_file(str(src), jobs.S3_BUCKET, c["original_key"], ExtraArgs={"ContentType": c["content_type"]})
|
|
# 2) variants
|
|
for name in jobs.VARIANT_ORDER:
|
|
key = jobs.final_key(c["attachment_id"], c["pv"], name)
|
|
s3.upload_file(str(out / files[name]), jobs.S3_BUCKET, key, ExtraArgs={"ContentType": meta[name]["content_type"]})
|
|
meta[name]["s3_key"] = key
|
|
return meta
|
|
|
|
|
|
def commit_ready(c: dict, meta: dict) -> bool:
|
|
"""Atomically publish the result. False if this attempt no longer owns the job (lease lost to a newer attempt)."""
|
|
with connect() as conn:
|
|
row = conn.execute("SELECT status, attempt FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (c["job_id"],)).fetchone()
|
|
if row is None or row[0] != "processing" or row[1] != c["attempt"]:
|
|
log.warning("job %s: attempt %s lost ownership, result not committed", c["job_id"], c["attempt"]); return False
|
|
for name in jobs.VARIANT_ORDER:
|
|
v = meta[name]
|
|
conn.execute("INSERT INTO attachment_variants(attachment_id,pipeline_version,name,s3_key,content_type,size_bytes,width,height,sha256) "
|
|
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (attachment_id,pipeline_version,name) DO NOTHING",
|
|
(c["attachment_id"], c["pv"], name, v["s3_key"], v["content_type"], v["size_bytes"], v["width"], v["height"], v["sha256"]))
|
|
conn.execute("UPDATE attachment_jobs SET status='ready', error_code=NULL, lease_until=NULL, updated_at=now() WHERE job_id=%s", (c["job_id"],))
|
|
conn.execute("UPDATE attachments SET status='ready', error_code=NULL, staging_key=NULL, updated_at=now() WHERE id=%s", (c["attachment_id"],))
|
|
jobs.add_event(conn, c["job_id"], "ready", f"attempt={c['attempt']}")
|
|
return True
|
|
|
|
|
|
def record_failure(c: dict, exc: Exception) -> None:
|
|
"""Permanent -> failed in the PostgreSQL registry. Transient -> queued with backoff+jitter (delayed outbox row) or failed when attempts are exhausted."""
|
|
with connect() as conn:
|
|
row = conn.execute("SELECT status, attempt FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (c["job_id"],)).fetchone()
|
|
if row is None or row[0] != "processing" or row[1] != c["attempt"]:
|
|
return
|
|
if isinstance(exc, Permanent):
|
|
jobs.fail_job(conn, c["job_id"], c["attachment_id"], exc.code, c["attempt"])
|
|
elif c["attempt"] >= c["max_attempts"]:
|
|
jobs.fail_job(conn, c["job_id"], c["attachment_id"], "retry_exhausted", c["attempt"])
|
|
else:
|
|
delay = jobs.backoff_seconds(c["attempt"])
|
|
conn.execute("UPDATE attachment_jobs SET status='queued', lease_until=NULL, next_attempt_at=now()+make_interval(secs=>%s), "
|
|
"error_code=NULL, updated_at=now() WHERE job_id=%s", (delay, c["job_id"]))
|
|
conn.execute("UPDATE attachments SET status='queued', updated_at=now() WHERE id=%s", (c["attachment_id"],))
|
|
jobs.add_event(conn, c["job_id"], "retry_scheduled", f"attempt={c['attempt']} delay={delay:.1f}s reason={type(exc).__name__}")
|
|
jobs.enqueue(conn, c["job_id"], c["attachment_id"], c["pv"], c["attempt"] + 1, delay_seconds=delay)
|
|
|
|
|
|
def handle(s3, raw: bytes) -> None:
|
|
msg = parse_job(raw)
|
|
if msg is None:
|
|
return
|
|
c = claim(msg)
|
|
if c is None:
|
|
return
|
|
try:
|
|
meta = run_pipeline(s3, c)
|
|
if FAULT_EXIT_AFTER_S3 and c["attempt"] == 1:
|
|
log.error("FAULT: exiting after S3 write, before COMMIT and offset commit")
|
|
os._exit(137)
|
|
if commit_ready(c, meta):
|
|
try:
|
|
s3.delete_object(Bucket=jobs.S3_BUCKET, Key=c["staging_key"]) # staging cleanup after the safe COMMIT
|
|
except Exception:
|
|
log.warning("staging cleanup failed; sweeper will remove it")
|
|
except (Permanent, Transient, ClientError, BotoCoreError, OSError, json.JSONDecodeError) as exc:
|
|
log.warning("job %s attempt %s failed: %s", c["job_id"], c["attempt"], type(exc).__name__)
|
|
record_failure(c, exc)
|
|
|
|
|
|
def main() -> None:
|
|
def request_stop(*_):
|
|
global STOP
|
|
STOP = True
|
|
signal.signal(signal.SIGTERM, request_stop)
|
|
signal.signal(signal.SIGINT, request_stop)
|
|
s3 = jobs.make_s3()
|
|
consumer = Consumer({
|
|
"bootstrap.servers": jobs.KAFKA_BOOTSTRAP, "group.id": "attachment-workers",
|
|
"enable.auto.commit": False, "auto.offset.reset": "earliest", "isolation.level": "read_committed",
|
|
"max.poll.interval.ms": 600000, "session.timeout.ms": 30000,
|
|
"queued.max.messages.kbytes": 1024, "fetch.max.bytes": 1048576, # bounded client-side prefetch
|
|
})
|
|
consumer.subscribe([jobs.JOBS_TOPIC])
|
|
log.info("worker %s started, concurrency=%d, max_attempts=%d, lease=%ds", WORKER_ID, CONCURRENCY, jobs.JOB_MAX_ATTEMPTS, jobs.LEASE_SECONDS)
|
|
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
|
|
while not STOP:
|
|
msgs = consumer.consume(num_messages=CONCURRENCY, timeout=1.0) # at most CONCURRENCY jobs in flight
|
|
batch = []
|
|
for m in msgs:
|
|
if m.error():
|
|
log.warning("kafka error: %s", m.error()); continue
|
|
batch.append(m)
|
|
if not batch:
|
|
continue
|
|
futures = [pool.submit(handle, s3, m.value()) for m in batch]
|
|
for f in futures:
|
|
f.result() # unexpected exception => process exits, offsets stay uncommitted, Kafka redelivers
|
|
# ACK boundary: commit the highest processed offset+1 per partition, only after every job of the batch is durable
|
|
top = {}
|
|
for m in batch:
|
|
top[(m.topic(), m.partition())] = max(top.get((m.topic(), m.partition()), -1), m.offset() + 1)
|
|
consumer.commit(offsets=[TopicPartition(t, p, o) for (t, p), o in top.items()], asynchronous=False)
|
|
consumer.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|