import hashlib import io import json import random import time from PIL import Image, ImageOps from src.db import make_pool from src.storage import BUCKET, s3 from src.queue import ensure_group, client as broker, STREAM, GROUP MAX_ATTEMPTS = 5 def make_variant(image, name): image = ImageOps.exif_transpose(image) if name == 'thumbnail': image.thumbnail((256, 256), Image.Resampling.LANCZOS); target = image elif name == 'crop': target = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS) else: image.thumbnail((1280, 1280), Image.Resampling.LANCZOS); target = image if target.mode not in ('RGB', 'L'): bg = Image.new('RGB', target.size, 'white') bg.paste(target, mask=target.getchannel('A') if 'A' in target.getbands() else None) target = bg out = io.BytesIO() content_type = 'image/webp' if name == 'optimized' else 'image/jpeg' target.convert('RGB').save(out, format='WEBP' if name == 'optimized' else 'JPEG', quality=82, exif=b'') raw = out.getvalue() return raw, content_type, target.width, target.height def process(row, pool): raw = s3.get_object(Bucket=BUCKET, Key=row['object_key'])['Body'].read() with Image.open(io.BytesIO(raw)) as image: variants=[] for name in ('thumbnail','crop','optimized'): data, content_type, width, height = make_variant(image.copy(), name) s3.put_object(Bucket=BUCKET, Key=row['object_key']+'.'+name, Body=data, ContentType=content_type) variants.append({'name':name,'content_type':content_type,'size_bytes':len(data),'width':width,'height':height,'sha256':hashlib.sha256(data).hexdigest()}) with pool.connection() as conn: conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'", (json.dumps(variants),row['id'])) conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),locked_until=NULL WHERE attachment_id=%s',(row['id'],)) def loop(): pool=make_pool() ensure_group() while True: # Repair the transactional-outbox -> stream gap after a crash between commit and XADD. with pool.connection() as conn: pending = conn.execute("SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until'}, count=1, block=1000) if messages: for _, entries in messages: for message_id, fields in entries: broker.xack(STREAM, GROUP, message_id) with pool.connection() as conn: job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until= MAX_ATTEMPTS: conn.execute("UPDATE attachments SET status='failed',error_code='processing_failed',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),last_error=%s WHERE attachment_id=%s',(type(error).__name__,row['id'])) else: delay=min(60,2**job['attempts'])+random.random() conn.execute("UPDATE attachment_jobs SET available_at=clock_timestamp()+(%s*interval '1 second'),locked_until=NULL,last_error=%s WHERE attachment_id=%s",(delay,type(error).__name__,row['id'])) if __name__ == '__main__': loop()