"""Standalone worker process: consumes attachment.process.v1 jobs, generates variants, writes results idempotently. Runs as its own container/command, separate from the API (contracts/attachment-states.md requires this split). """ import asyncio import datetime as dt import hashlib import logging import os import random import socket import time import uuid from opentelemetry import trace from opentelemetry.trace import SpanKind from . import broker, db, pipeline, storage, telemetry from . import metrics as app_metrics logger = logging.getLogger("worker") CONSUMER_NAME = os.environ.get("INSTANCE_ID") or f"{socket.gethostname()}-{os.getpid()}" MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) JOB_TIMEOUT_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) RETRY_SCAN_INTERVAL_SECONDS = 1 WORKER_CAPACITY = 1 # this process handles one job at a time (see REPORT.md limitations) _active_jobs = 0 _tracer = None def read_secret(file_env: str, inline_env: str) -> str: path = os.environ.get(file_env) if path and os.path.isfile(path): return open(path, encoding="utf-8").read().strip() value = os.environ.get(inline_env) if not value: raise RuntimeError(f"{file_env} or {inline_env} must be set") return value async def process_job(job: dict) -> None: global _active_jobs ctx = telemetry.extract_context(job) if job.get("traceparent") else None span_cm = _tracer.start_as_current_span("process_job", context=ctx, kind=SpanKind.CONSUMER) if _tracer else None if span_cm: span_cm.__enter__() started = time.monotonic() _active_jobs += 1 try: await _process_job_inner(job, started) finally: _active_jobs -= 1 if span_cm: span_cm.__exit__(None, None, None) async def _process_job_inner(job: dict, started: float) -> None: attachment_id = uuid.UUID(job["attachment_id"]) job_id = uuid.UUID(job["job_id"]) attempt = job["attempt"] async with db.pool().acquire() as conn: record = await conn.fetchrow("SELECT * FROM attachments WHERE id=$1", attachment_id) if record is None: logger.warning("job %s references missing attachment %s; dropping", job_id, attachment_id) return if record["status"] in ("ready", "failed"): # Redelivery of an already-finished job, or a stale duplicate racing # a newer attempt: never roll a finished result backwards. logger.info("attachment %s already %s; no-op", attachment_id, record["status"]) app_metrics.record_job("attachment.process.v1", "duplicate_noop", time.monotonic() - started) return async with db.pool().acquire() as conn: await conn.execute( "UPDATE attachments SET status='processing', updated_at=now() WHERE id=$1 AND status != 'ready'", attachment_id, ) content_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} try: original = await storage.get_object(record["s3_key"]) generated = { "thumbnail": pipeline.thumbnail(original), "crop": pipeline.crop(original), "optimized": pipeline.optimized(original), } uploaded = [] for name, (data, width, height) in generated.items(): key = f"attachments/{attachment_id}/{name}" await storage.put_object(key, data, content_types[name]) # network call: can fail transiently too uploaded.append((name, key, data, width, height)) except Exception: # noqa: BLE001 - covers S3 reachability (get+put) and image decode, not just decode logger.exception("processing failed for attachment %s", attachment_id) app_metrics.record_job("attachment.process.v1", "failed_attempt", time.monotonic() - started) await _handle_failure(job, "processing_failed") return async with db.pool().acquire() as conn: async with conn.transaction(): for name, key, data, width, height in uploaded: digest = hashlib.sha256(data).hexdigest() await conn.execute( """ INSERT INTO attachment_variants (attachment_id, name, content_type, size_bytes, width, height, sha256, s3_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (attachment_id, name) DO UPDATE SET content_type=EXCLUDED.content_type, size_bytes=EXCLUDED.size_bytes, width=EXCLUDED.width, height=EXCLUDED.height, sha256=EXCLUDED.sha256, s3_key=EXCLUDED.s3_key """, attachment_id, name, content_types[name], len(data), width, height, digest, key, ) await conn.execute( "UPDATE attachments SET status='ready', error_code=NULL, updated_at=now() WHERE id=$1 AND status != 'ready'", attachment_id, ) logger.info("attachment %s ready (attempt %d)", attachment_id, attempt) app_metrics.record_job("attachment.process.v1", "success", time.monotonic() - started) accepted_at = record["created_at"] app_metrics.record_time_to_ready((dt.datetime.now(dt.timezone.utc) - accepted_at).total_seconds()) async def _handle_failure(job: dict, error_code: str) -> None: attachment_id = uuid.UUID(job["attachment_id"]) job_id = uuid.UUID(job["job_id"]) attempt = job["attempt"] async with db.pool().acquire() as conn: if attempt >= MAX_ATTEMPTS: async with conn.transaction(): await conn.execute( "UPDATE attachments SET status='failed', error_code=$2, updated_at=now() WHERE id=$1", attachment_id, error_code, ) await conn.execute( "INSERT INTO job_failures (attachment_id, job_id, attempt, error_code) VALUES ($1,$2,$3,$4) " "ON CONFLICT (attachment_id, job_id) DO NOTHING", attachment_id, job_id, attempt, error_code, ) logger.warning("attachment %s permanently failed after %d attempts: %s", attachment_id, attempt, error_code) return # Transient: schedule a durable delayed retry instead of sleeping in-process. delay = min(30, 2 ** (attempt - 1)) + random.uniform(0, 1) await conn.execute( """ INSERT INTO job_retry_schedule (attachment_id, job_id, attempt, next_attempt_at) VALUES ($1, $2, $3, now() + ($4 * interval '1 second')) ON CONFLICT (attachment_id, job_id) DO UPDATE SET attempt=EXCLUDED.attempt, next_attempt_at=EXCLUDED.next_attempt_at """, attachment_id, job_id, attempt + 1, delay, ) await conn.execute( "UPDATE attachments SET status='queued', updated_at=now() WHERE id=$1 AND status != 'ready'", attachment_id, ) logger.info("attachment %s scheduled for retry #%d in %.1fs", attachment_id, attempt + 1, delay) async def retry_scheduler_loop() -> None: while True: try: async with db.pool().acquire() as conn: due = await conn.fetch( "SELECT attachment_id, job_id, attempt FROM job_retry_schedule WHERE next_attempt_at <= now() LIMIT 50" ) for row in due: job = { "schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": 1, "attempt": row["attempt"], "created_at": "1970-01-01T00:00:00Z", # informational only; not used for ordering } await broker.publish(job) await conn.execute( "DELETE FROM job_retry_schedule WHERE attachment_id=$1 AND job_id=$2", row["attachment_id"], row["job_id"], ) except Exception: # noqa: BLE001 - scheduler must survive a transient hiccup logger.exception("retry scheduler tick failed") await asyncio.sleep(RETRY_SCAN_INTERVAL_SECONDS) async def reclaim_loop() -> None: """Hand this worker any job another (likely dead) worker left unacked past JOB_TIMEOUT_SECONDS -- lease recovery for a crashed worker.""" while True: try: for message_id, job in await broker.reclaim_stale(CONSUMER_NAME, JOB_TIMEOUT_SECONDS * 1000): logger.info("reclaimed stale job %s (message %s)", job["job_id"], message_id) await process_job(job) await broker.ack(message_id) except Exception: # noqa: BLE001 logger.exception("reclaim tick failed") await asyncio.sleep(JOB_TIMEOUT_SECONDS / 2) async def consume_loop() -> None: while True: try: for message_id, job in await broker.read(CONSUMER_NAME, count=1, block_ms=2000): await process_job(job) await broker.ack(message_id) except Exception: # noqa: BLE001 - keep the loop alive across transient errors logger.exception("consume tick failed") await asyncio.sleep(1) async def main() -> None: global _tracer telemetry.init(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker")) app_metrics.init() _tracer = trace.get_tracer("messenger-worker") database_url = read_secret("DATABASE_URL_FILE", "DATABASE_URL") cache_url = read_secret("CACHE_URL_FILE", "CACHE_URL") await db.init_pool(database_url) broker.init(cache_url) await broker.ensure_group() storage.init( endpoint=os.environ["S3_ENDPOINT"], region=os.environ.get("S3_REGION", "us-east-1"), access_key=read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), secret_key=read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), bucket=os.environ.get("S3_BUCKET", "messenger-attachments"), ) loop = asyncio.get_event_loop() def _queue_sync(): future = asyncio.run_coroutine_threadsafe(broker.depth_and_oldest_age_seconds(), loop) return future.result(timeout=5) app_metrics.observable_gauges( queue_depth_getter=_queue_sync, worker_slots_getter=lambda: (_active_jobs, WORKER_CAPACITY), ) logger.info("worker %s starting", CONSUMER_NAME) await asyncio.gather(consume_loop(), reclaim_loop(), retry_scheduler_loop()) if __name__ == "__main__": asyncio.run(main())