Files
mtusi2026lab5/src/worker.py
T
2026-11-07 19:45:45 +03:00

231 lines
12 KiB
Python

"""Lab 5 worker: Kafka consumer -> claim (lease) in PostgreSQL -> staging -> pipeline -> final S3 + COMMIT -> offset commit.
PostgreSQL is the source of truth for job state; Kafka is the at-least-once transport. The Kafka offset is committed
only AFTER the durable effect (final objects + COMMIT of the result) or after a durable decision (retry scheduled / failed).
"""
from __future__ import annotations
import hashlib
import json
import logging
import os
import signal
import socket
import subprocess
import sys
import tempfile
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import psycopg
from botocore.exceptions import BotoCoreError, ClientError
from confluent_kafka import Consumer, TopicPartition
from . import jobs
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s worker %(message)s")
log = logging.getLogger("worker")
WORKER_ID = os.getenv("INSTANCE_ID") or socket.gethostname()
CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2"))
FAULT_EXIT_AFTER_S3 = os.getenv("FAULT_WORKER_EXIT_AFTER_S3", "0") == "1" # test hook: crash after the S3 write, before COMMIT/ack
IMG_MEMORY_MB = os.getenv("IMG_MEMORY_MB", "1024")
CODE_BY_EXIT = {2: "invalid_image", 3: "image_too_large", 4: "image_too_complex"} # permanent errors of imgproc
STOP = False
class Permanent(Exception):
def __init__(self, code: str):
super().__init__(code)
self.code = code
class Transient(Exception):
pass
def connect():
return psycopg.connect(connect_timeout=5, options="-c statement_timeout=10000")
def parse_job(raw: bytes) -> dict | None:
"""Validate against jobs.schema.json (the subset that matters). Poison messages are logged and skipped."""
try:
m = json.loads(raw)
assert m["schema_version"] == 1 and m["type"] == jobs.JOB_TYPE
uuid.UUID(m["job_id"]); uuid.UUID(m["attachment_id"])
assert isinstance(m["pipeline_version"], int) and m["pipeline_version"] >= 1
return m
except Exception:
log.error("poison message skipped (schema violation), size=%d", len(raw))
return None
def claim(msg: dict) -> dict | None:
"""queued -> processing with a lease. Returns None when this delivery must be a no-op (ack)."""
with connect() as conn:
row = conn.execute(
"SELECT status, attempt, max_attempts, attachment_id, pipeline_version, "
"(next_attempt_at <= now()) AS due, (lease_until IS NOT NULL AND lease_until < now()) AS lease_expired "
"FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (msg["job_id"],)).fetchone()
if row is None:
log.warning("job %s unknown, skipped", msg["job_id"]); return None
status, attempt, max_attempts, att_id, pv, due, expired = row
if pv != msg["pipeline_version"]:
return None # stale job of another pipeline version never overwrites the newer one
if status in ("ready", "failed"):
log.info("job %s already %s: duplicate delivery is a no-op", msg["job_id"], status); return None
if status == "queued" and not due:
return None # early duplicate: the delayed outbox row will deliver it later
if status == "processing" and not expired:
log.info("job %s is leased by another worker: skip (dispatcher recovers it if that worker dies)", msg["job_id"]); return None
if attempt >= max_attempts: # lease expired and no attempts left
jobs.fail_job(conn, msg["job_id"], att_id, "retry_exhausted", attempt)
return None
attempt += 1
conn.execute("UPDATE attachment_jobs SET status='processing', attempt=%s, worker_id=%s, "
"lease_until=now()+make_interval(secs=>%s), updated_at=now() WHERE job_id=%s",
(attempt, WORKER_ID, jobs.LEASE_SECONDS, msg["job_id"]))
conn.execute("UPDATE attachments SET status='processing', updated_at=now() WHERE id=%s", (att_id,))
jobs.add_event(conn, msg["job_id"], "claimed", f"attempt={attempt} worker={WORKER_ID}")
a = conn.execute("SELECT sha256, staging_key, s3_key, content_type, filename FROM attachments WHERE id=%s", (att_id,)).fetchone()
return {"job_id": msg["job_id"], "attachment_id": att_id, "pv": pv, "attempt": attempt, "max_attempts": max_attempts,
"sha256": a[0].strip(), "staging_key": a[1], "original_key": a[2], "content_type": a[3]}
def run_pipeline(s3, c: dict) -> dict:
"""Download staging -> verify -> child process with limits -> idempotent PUTs of deterministic keys."""
if not c["staging_key"]:
raise Permanent("staging_missing")
with tempfile.TemporaryDirectory(prefix="job-") as tmp:
src, out = Path(tmp) / "in.bin", Path(tmp) / "out"
out.mkdir()
try:
s3.download_file(jobs.S3_BUCKET, c["staging_key"], str(src))
except ClientError as e:
if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"):
raise Permanent("staging_missing")
raise
if hashlib.sha256(src.read_bytes()).hexdigest() != c["sha256"]:
raise Permanent("integrity_error") # bytes differ from what the API accepted
try:
r = subprocess.run([sys.executable, "-m", "src.imgproc", str(src), str(out)], capture_output=True, text=True,
timeout=jobs.JOB_TIMEOUT_SECONDS, env={"PATH": os.environ.get("PATH", ""), "IMG_MEMORY_MB": IMG_MEMORY_MB,
"IMG_CPU_SECONDS": str(jobs.JOB_TIMEOUT_SECONDS),
"MAX_IMAGE_PIXELS": os.getenv("MAX_IMAGE_PIXELS", "40000000")}) # no secrets in the child
except subprocess.TimeoutExpired:
raise Transient("job timeout") # child is killed by subprocess.run
if r.returncode in CODE_BY_EXIT:
raise Permanent(CODE_BY_EXIT[r.returncode])
if r.returncode != 0:
raise Transient(f"imgproc exit {r.returncode}")
meta = json.loads(r.stdout)
files = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"}
# 1) original into the final prefix (same key every time => repeated PUT is harmless)
s3.upload_file(str(src), jobs.S3_BUCKET, c["original_key"], ExtraArgs={"ContentType": c["content_type"]})
# 2) variants
for name in jobs.VARIANT_ORDER:
key = jobs.final_key(c["attachment_id"], c["pv"], name)
s3.upload_file(str(out / files[name]), jobs.S3_BUCKET, key, ExtraArgs={"ContentType": meta[name]["content_type"]})
meta[name]["s3_key"] = key
return meta
def commit_ready(c: dict, meta: dict) -> bool:
"""Atomically publish the result. False if this attempt no longer owns the job (lease lost to a newer attempt)."""
with connect() as conn:
row = conn.execute("SELECT status, attempt FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (c["job_id"],)).fetchone()
if row is None or row[0] != "processing" or row[1] != c["attempt"]:
log.warning("job %s: attempt %s lost ownership, result not committed", c["job_id"], c["attempt"]); return False
for name in jobs.VARIANT_ORDER:
v = meta[name]
conn.execute("INSERT INTO attachment_variants(attachment_id,pipeline_version,name,s3_key,content_type,size_bytes,width,height,sha256) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (attachment_id,pipeline_version,name) DO NOTHING",
(c["attachment_id"], c["pv"], name, v["s3_key"], v["content_type"], v["size_bytes"], v["width"], v["height"], v["sha256"]))
conn.execute("UPDATE attachment_jobs SET status='ready', error_code=NULL, lease_until=NULL, updated_at=now() WHERE job_id=%s", (c["job_id"],))
conn.execute("UPDATE attachments SET status='ready', error_code=NULL, staging_key=NULL, updated_at=now() WHERE id=%s", (c["attachment_id"],))
jobs.add_event(conn, c["job_id"], "ready", f"attempt={c['attempt']}")
return True
def record_failure(c: dict, exc: Exception) -> None:
"""Permanent -> failed in the PostgreSQL registry. Transient -> queued with backoff+jitter (delayed outbox row) or failed when attempts are exhausted."""
with connect() as conn:
row = conn.execute("SELECT status, attempt FROM attachment_jobs WHERE job_id=%s FOR UPDATE", (c["job_id"],)).fetchone()
if row is None or row[0] != "processing" or row[1] != c["attempt"]:
return
if isinstance(exc, Permanent):
jobs.fail_job(conn, c["job_id"], c["attachment_id"], exc.code, c["attempt"])
elif c["attempt"] >= c["max_attempts"]:
jobs.fail_job(conn, c["job_id"], c["attachment_id"], "retry_exhausted", c["attempt"])
else:
delay = jobs.backoff_seconds(c["attempt"])
conn.execute("UPDATE attachment_jobs SET status='queued', lease_until=NULL, next_attempt_at=now()+make_interval(secs=>%s), "
"error_code=NULL, updated_at=now() WHERE job_id=%s", (delay, c["job_id"]))
conn.execute("UPDATE attachments SET status='queued', updated_at=now() WHERE id=%s", (c["attachment_id"],))
jobs.add_event(conn, c["job_id"], "retry_scheduled", f"attempt={c['attempt']} delay={delay:.1f}s reason={type(exc).__name__}")
jobs.enqueue(conn, c["job_id"], c["attachment_id"], c["pv"], c["attempt"] + 1, delay_seconds=delay)
def handle(s3, raw: bytes) -> None:
msg = parse_job(raw)
if msg is None:
return
c = claim(msg)
if c is None:
return
try:
meta = run_pipeline(s3, c)
if FAULT_EXIT_AFTER_S3 and c["attempt"] == 1:
log.error("FAULT: exiting after S3 write, before COMMIT and offset commit")
os._exit(137)
if commit_ready(c, meta):
try:
s3.delete_object(Bucket=jobs.S3_BUCKET, Key=c["staging_key"]) # staging cleanup after the safe COMMIT
except Exception:
log.warning("staging cleanup failed; sweeper will remove it")
except (Permanent, Transient, ClientError, BotoCoreError, OSError, json.JSONDecodeError) as exc:
log.warning("job %s attempt %s failed: %s", c["job_id"], c["attempt"], type(exc).__name__)
record_failure(c, exc)
def main() -> None:
def request_stop(*_):
global STOP
STOP = True
signal.signal(signal.SIGTERM, request_stop)
signal.signal(signal.SIGINT, request_stop)
s3 = jobs.make_s3()
consumer = Consumer({
"bootstrap.servers": jobs.KAFKA_BOOTSTRAP, "group.id": "attachment-workers",
"enable.auto.commit": False, "auto.offset.reset": "earliest", "isolation.level": "read_committed",
"max.poll.interval.ms": 600000, "session.timeout.ms": 30000,
"queued.max.messages.kbytes": 1024, "fetch.max.bytes": 1048576, # bounded client-side prefetch
})
consumer.subscribe([jobs.JOBS_TOPIC])
log.info("worker %s started, concurrency=%d, max_attempts=%d, lease=%ds", WORKER_ID, CONCURRENCY, jobs.JOB_MAX_ATTEMPTS, jobs.LEASE_SECONDS)
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
while not STOP:
msgs = consumer.consume(num_messages=CONCURRENCY, timeout=1.0) # at most CONCURRENCY jobs in flight
batch = []
for m in msgs:
if m.error():
log.warning("kafka error: %s", m.error()); continue
batch.append(m)
if not batch:
continue
futures = [pool.submit(handle, s3, m.value()) for m in batch]
for f in futures:
f.result() # unexpected exception => process exits, offsets stay uncommitted, Kafka redelivers
# ACK boundary: commit the highest processed offset+1 per partition, only after every job of the batch is durable
top = {}
for m in batch:
top[(m.topic(), m.partition())] = max(top.get((m.topic(), m.partition()), -1), m.offset() + 1)
consumer.commit(offsets=[TopicPartition(t, p, o) for (t, p), o in top.items()], asynchronous=False)
consumer.close()
if __name__ == "__main__":
main()