#!/usr/bin/env python3 import io import json import logging import os import time import uuid from datetime import datetime, timedelta, timezone from PIL import Image, ImageOps import redis from db import Store from s3store import ObjectStore, S3Unavailable LOG = logging.getLogger("attachment-worker") STREAM = os.environ.get("BROKER_STREAM", "attachment.jobs") GROUP = os.environ.get("BROKER_GROUP", "attachment-workers") MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) PREFETCH = max(1, min(10, int(os.environ.get("BROKER_PREFETCH", "1")))) FAULT = os.environ.get("FAULT_INJECTION", "").strip() def inject_crash(point): if FAULT == point: LOG.error("fault injection crash point=%s", point) os._exit(42) def now(): return datetime.now(timezone.utc) def iso(value): return value.isoformat(timespec="microseconds").replace("+00:00", "Z") def broker_url(): path = os.environ.get("BROKER_URL_FILE") if path and os.path.isfile(path): return open(path, encoding="utf-8").read().strip() return os.environ.get("BROKER_URL", os.environ.get("CACHE_URL", "redis://redis:6379/0")) def image_bytes(image, format_name, **options): output = io.BytesIO() image.save(output, format=format_name, **options) return output.getvalue() def prepare_variants(raw, attachment_id): with Image.open(io.BytesIO(raw)) as source: image = ImageOps.exif_transpose(source) if image.mode in ("RGBA", "LA") or "transparency" in image.info: background = Image.new("RGB", image.size, "white") alpha = image.convert("RGBA") background.paste(alpha, mask=alpha.getchannel("A")) image = background else: image = image.convert("RGB") thumbnail = image.copy() thumbnail.thumbnail((256, 256), Image.Resampling.LANCZOS) crop_size = min(image.size) left = (image.width - crop_size) // 2 top = (image.height - crop_size) // 2 crop = image.crop((left, top, left + crop_size, top + crop_size)) crop = crop.resize((128, 128), Image.Resampling.LANCZOS) optimized = image.copy() optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) values = [ ("thumbnail", thumbnail, "JPEG", "image/jpeg", {"quality": 88, "optimize": True}), ("crop", crop, "JPEG", "image/jpeg", {"quality": 88, "optimize": True}), ("optimized", optimized, "WEBP", "image/webp", {"quality": 86, "method": 4}), ] result = [] for name, rendered, fmt, content_type, options in values: data = image_bytes(rendered, fmt, **options) result.append({ "name": name, "content_type": content_type, "size_bytes": len(data), "width": rendered.width, "height": rendered.height, "sha256": __import__("hashlib").sha256(data).hexdigest(), "_key": "objects/" + attachment_id + "/" + name, "_data": data, }) return result def public_variants(values): return [{key: value for key, value in item.items() if not key.startswith("_")} for item in values] class Worker: def __init__(self): self.store = Store.connect(run_migrations=False) self.objects = ObjectStore.from_env() self.redis = redis.from_url(broker_url(), decode_responses=True, socket_connect_timeout=2) self.consumer = os.environ.get("WORKER_ID", "worker-" + uuid.uuid4().hex[:8]) try: self.redis.xgroup_create(STREAM, GROUP, id="0", mkstream=True) except redis.ResponseError as error: if "BUSYGROUP" not in str(error): raise def publish_outbox(self): for row in self.store.pending_outbox(): fields = { "schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": 1, "created_at": iso(now()), } message_id = self.redis.xadd(STREAM, fields, maxlen=10000, approximate=True) self.store.mark_outbox_published(row["id"], iso(now())) LOG.info("published job=%s message=%s", fields["job_id"], message_id) def process(self, message_id, fields): job_id = fields.get("job_id") attachment_id = fields.get("attachment_id") if not job_id or not attachment_id: self.redis.xack(STREAM, GROUP, message_id) return claim = self.store.claim_job( job_id, iso(now() + timedelta(seconds=LEASE_SECONDS)), iso(now()) ) if claim is None: self.redis.xack(STREAM, GROUP, message_id) return inject_crash("before-side-effect") attachment = self.store.get_attachment_for_job(attachment_id) if not attachment: self.redis.xack(STREAM, GROUP, message_id) return if int(fields.get("pipeline_version", 1)) < int(attachment.get("pipeline_version", 1)): self.redis.xack(STREAM, GROUP, message_id) return try: staged = self.objects.get_object(attachment["staging_key"]) if staged is None: raise ValueError("staging_missing") raw = staged["Body"].read() staged["Body"].close() self.objects.put_file( attachment["object_key"], io.BytesIO(raw), len(raw), attachment["content_type"] ) inject_crash("after-original") rendered = prepare_variants(raw, attachment_id) for item in rendered: self.objects.put_file( item["_key"], io.BytesIO(item["_data"]), item["size_bytes"], item["content_type"] ) inject_crash("after-side-effect") self.store.set_attachment_variants(attachment_id, public_variants(rendered), iso(now())) try: self.objects.delete_object(attachment["staging_key"]) except S3Unavailable: LOG.warning("staging cleanup deferred attachment=%s", attachment_id) self.redis.xack(STREAM, GROUP, message_id) except (S3Unavailable, OSError, ValueError, RuntimeError) as error: attempt = claim["attempts"] code = "processing_failed" if not isinstance(error, ValueError) else str(error) if attempt >= MAX_ATTEMPTS: self.store.fail_job(job_id, attachment_id, "retry_exhausted", iso(now())) self.redis.xack(STREAM, GROUP, message_id) else: delay = min(300, 2 ** (attempt - 1)) + (hash(job_id) % 1000) / 1000 self.store.retry_job( job_id, attachment_id, iso(now() + timedelta(seconds=delay)), code, iso(now()) ) self.redis.xack(STREAM, GROUP, message_id) time.sleep(delay) self.redis.xadd(STREAM, fields, maxlen=10000, approximate=True) def run(self): while True: self.publish_outbox() pending = self.redis.xautoclaim( STREAM, GROUP, self.consumer, min_idle_time=LEASE_SECONDS * 1000, start_id="0-0", count=PREFETCH ) for message_id, fields in pending[1]: self.process(message_id, fields) result = self.redis.xreadgroup( GROUP, self.consumer, {STREAM: ">"}, count=PREFETCH, block=1000 ) for _, messages in result: for message_id, fields in messages: self.process(message_id, fields) def main(): logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s") worker = Worker() try: worker.run() finally: worker.store.close() worker.objects.client.close() if __name__ == "__main__": main()