"""Standalone worker process: consumes attachment.process.v1 jobs, generates variants, writes results idempotently. Runs as its own container/command, separate from the API (contracts/attachment-states.md requires this split). """ import asyncio import hashlib import logging import os import random import socket import uuid from . import broker, db, pipeline, storage logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("worker") CONSUMER_NAME = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}" MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) JOB_TIMEOUT_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) RETRY_SCAN_INTERVAL_SECONDS = 1 def read_secret(file_env: str, inline_env: str) -> str: path = os.environ.get(file_env) if path and os.path.isfile(path): return open(path, encoding="utf-8").read().strip() value = os.environ.get(inline_env) if not value: raise RuntimeError(f"{file_env} or {inline_env} must be set") return value async def process_job(job: dict) -> None: attachment_id = uuid.UUID(job["attachment_id"]) job_id = uuid.UUID(job["job_id"]) attempt = job["attempt"] async with db.pool().acquire() as conn: record = await conn.fetchrow("SELECT * FROM attachments WHERE id=$1", attachment_id) if record is None: logger.warning("job %s references missing attachment %s; dropping", job_id, attachment_id) return if record["status"] in ("ready", "failed"): # Redelivery of an already-finished job, or a stale duplicate racing # a newer attempt: never roll a finished result backwards. logger.info("attachment %s already %s; no-op", attachment_id, record["status"]) return async with db.pool().acquire() as conn: await conn.execute( "UPDATE attachments SET status='processing', updated_at=now() WHERE id=$1 AND status != 'ready'", attachment_id, ) content_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} try: original = await storage.get_object(record["s3_key"]) generated = { "thumbnail": pipeline.thumbnail(original), "crop": pipeline.crop(original), "optimized": pipeline.optimized(original), } uploaded = [] for name, (data, width, height) in generated.items(): key = f"attachments/{attachment_id}/{name}" await storage.put_object(key, data, content_types[name]) # network call: can fail transiently too uploaded.append((name, key, data, width, height)) except Exception: # noqa: BLE001 - covers S3 reachability (get+put) and image decode, not just decode logger.exception("processing failed for attachment %s", attachment_id) await _handle_failure(job, "processing_failed") return async with db.pool().acquire() as conn: async with conn.transaction(): for name, key, data, width, height in uploaded: digest = hashlib.sha256(data).hexdigest() await conn.execute( """ INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (attachment_id, name) DO UPDATE SET content_type=EXCLUDED.content_type, size_bytes=EXCLUDED.size_bytes, width=EXCLUDED.width, height=EXCLUDED.height, sha256=EXCLUDED.sha256, s3_key=EXCLUDED.s3_key """, attachment_id, name, content_types[name], len(data), width, height, digest, key, ) await conn.execute( "UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=$1 AND status != 'ready'", attachment_id, ) logger.info("attachment %s ready (attempt %d)", attachment_id, attempt) async def _handle_failure(job: dict, error_code: str) -> None: attachment_id = uuid.UUID(job["attachment_id"]) job_id = uuid.UUID(job["job_id"]) attempt = job["attempt"] async with db.pool().acquire() as conn: if attempt >= MAX_ATTEMPTS: async with conn.transaction(): await conn.execute( "UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", attachment_id, error_code, ) await conn.execute( "INSERT INTO job_failures (attachment_id, job_id, attempt, error_code) VALUES ($1,$2,$3,$4) " "ON CONFLICT (attachment_id, job_id) DO NOTHING", attachment_id, job_id, attempt, error_code, ) logger.warning("attachment %s permanently failed after %d attempts: %s", attachment_id, attempt, error_code) return # Transient: schedule a durable delayed retry instead of sleeping in-process. delay = min(30, 2 ** (attempt - 1)) + random.uniform(0, 1) await conn.execute( """ INSERT INTO job_retry_schedule (attachment_id, job_id, attempt, next_attempt_at) VALUES ($1, $2, $3, now() + ($4 * interval '1 second')) ON CONFLICT (attachment_id, job_id) DO UPDATE SET attempt=EXCLUDED.attempt, next_attempt_at=EXCLUDED.next_attempt_at """, attachment_id, job_id, attempt + 1, delay, ) await conn.execute( "UPDATE attachments SET status='queued', updated_at=now() WHERE id=$1 AND status != 'ready'", attachment_id, ) logger.info("attachment %s scheduled for retry #%d in %.1fs", attachment_id, attempt + 1, delay) async def retry_scheduler_loop() -> None: while True: try: async with db.pool().acquire() as conn: due = await conn.fetch( "SELECT attachment_id, job_id, attempt FROM job_retry_schedule WHERE next_attempt_at <= now() LIMIT 50" ) for row in due: job = { "schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": 1, "attempt": row["attempt"], "created_at": "1970-01-01T00:00:00Z", # informational only; not used for ordering } await broker.publish(job) await conn.execute( "DELETE FROM job_retry_schedule WHERE attachment_id=$1 AND job_id=$2", row["attachment_id"], row["job_id"], ) except Exception: # noqa: BLE001 - scheduler must survive a transient hiccup logger.exception("retry scheduler tick failed") await asyncio.sleep(RETRY_SCAN_INTERVAL_SECONDS) async def reclaim_loop() -> None: """Hand this worker any job another (likely dead) worker left unacked past JOB_TIMEOUT_SECONDS -- lease recovery for a crashed worker.""" while True: try: for message_id, job in await broker.reclaim_stale(CONSUMER_NAME, JOB_TIMEOUT_SECONDS * 1000): logger.info("reclaimed stale job %s (message %s)", job["job_id"], message_id) await process_job(job) await broker.ack(message_id) except Exception: # noqa: BLE001 logger.exception("reclaim tick failed") await asyncio.sleep(JOB_TIMEOUT_SECONDS / 2) async def consume_loop() -> None: while True: try: for message_id, job in await broker.read(CONSUMER_NAME, count=1, block_ms=2000): await process_job(job) await broker.ack(message_id) except Exception: # noqa: BLE001 - keep the loop alive across transient errors logger.exception("consume tick failed") await asyncio.sleep(1) async def main() -> None: database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL") cache_url = read_secret("CACHE_URL_FILE", "CACHE_URL") await db.init_pool(database_url) broker.init(cache_url) await broker.ensure_group() storage.init( endpoint=os.environ["S3_ENDPOINT"], region=os.environ.get("S3_REGION", "us-east-1"), access_key=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), secret_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), bucket=os.environ.get("S3_BUCKET", "messenger-attachments"), ) logger.info("worker %s starting", CONSUMER_NAME) await asyncio.gather(consume_loop(), reclaim_loop(), retry_scheduler_loop()) if __name__ == "__main__": asyncio.run(main())