"""Durable outbox relay and idempotent Redis Streams image worker.""" import io, json, os, time, uuid, hashlib, random, shutil from concurrent.futures import ThreadPoolExecutor import redis, boto3, psycopg from psycopg.rows import dict_row from psycopg_pool import ConnectionPool from botocore.config import Config from PIL import Image, ImageOps, UnidentifiedImageError from app.migrate import setting def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name] db=ConnectionPool(setting('DATABASE_URL'),min_size=1,max_size=8,kwargs={'row_factory':dict_row,'connect_timeout':3}) broker=redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=1,socket_timeout=2,decode_responses=True) s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2})) bucket=os.getenv('S3_BUCKET','messenger-attachments'); stream='attachment-jobs'; group='image-workers'; attempts=int(os.getenv('JOB_MAX_ATTEMPTS','5')); concurrency=int(os.getenv('WORKER_CONCURRENCY','2')); timeout=int(os.getenv('JOB_TIMEOUT_SECONDS','60')) try: broker.xgroup_create(stream,group,id='0',mkstream=True) except redis.ResponseError as e: if 'BUSYGROUP' not in str(e): raise def publish_outbox(): # DB intent remains pending until XADD succeeds; crash in the gap yields a safe duplicate. with db.connection() as c: rows=c.execute("SELECT job_id,attachment_id,pipeline_version,attempt,created_at FROM attachment_jobs WHERE state IN ('pending','published') AND available_at<=clock_timestamp() ORDER BY created_at LIMIT 100 FOR UPDATE SKIP LOCKED").fetchall() for r in rows: msg={'schema_version':1,'job_id':str(r['job_id']),'type':'attachment.process.v1','attachment_id':str(r['attachment_id']),'pipeline_version':r['pipeline_version'],'attempt':max(1,r['attempt']+1),'created_at':r['created_at'].isoformat().replace('+00:00','Z')} broker.xadd(stream,{'job':json.dumps(msg,separators=(',',':'))}) c.execute("UPDATE attachment_jobs SET state='published',available_at=clock_timestamp()+interval '15 seconds' WHERE job_id=%s AND state IN ('pending','published')",(r['job_id'],)) def resize(raw): im=Image.open(io.BytesIO(raw)); im.load(); im=ImageOps.exif_transpose(im) if im.width*im.height>40_000_000: raise ValueError('oversized_image') if im.mode in ('RGBA','LA') or 'transparency' in im.info: rgba=im.convert('RGBA'); bg=Image.new('RGBA',rgba.size,'white'); bg.alpha_composite(rgba); im=bg.convert('RGB') else: im=im.convert('RGB') thumb=im.copy(); thumb.thumbnail((256,256),Image.Resampling.LANCZOS) out=io.BytesIO(); thumb.save(out,format='JPEG',quality=85,optimize=True) crop=ImageOps.fit(im,(128,128),method=Image.Resampling.LANCZOS,centering=(.5,.5)); cropout=io.BytesIO(); crop.save(cropout,format='JPEG',quality=85,optimize=True) opt=im.copy(); opt.thumbnail((1280,1280),Image.Resampling.LANCZOS); webp=io.BytesIO(); opt.save(webp,format='WEBP',quality=82,method=4,exif=b'') return [('thumbnail','image/jpeg',thumb.size,out.getvalue()),('crop','image/jpeg',crop.size,cropout.getvalue()),('optimized','image/webp',opt.size,webp.getvalue())] def process(job): jid=uuid.UUID(job['job_id']); aid=uuid.UUID(job['attachment_id']) with db.connection() as c: c.execute('BEGIN') r=c.execute('SELECT j.*,a.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j,a',(jid,)).fetchone() if not r: c.rollback(); return if r['state']=='done' or r['state']=='failed' or r['state']=='processing' and r['lease_until'] and r['lease_until'].timestamp()>time.time(): c.rollback(); return c.execute("UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_until=clock_timestamp()+(%s * interval '1 second') WHERE job_id=%s",(timeout,jid)) c.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'",(aid,)) c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'processing')",(jid,)); c.commit() try: raw=s3.get_object(Bucket=bucket,Key=r['staging_key'])['Body'].read(10_485_761) if len(raw)>10_485_760 or hashlib.sha256(raw).hexdigest()!=r['sha256'].strip(): raise ValueError('invalid_image') variants=resize(raw) # Each object key is deterministic, so a crash and redelivery overwrites the same effect. s3.put_object(Bucket=bucket,Key=r['object_key'],Body=raw,ContentType=r['content_type'],Metadata={'sha256':r['sha256'].strip()}) meta=[] for name,mime,size,data in variants: key=f"variants/v1/{aid}/{name}" s3.put_object(Bucket=bucket,Key=key,Body=data,ContentType=mime,Metadata={'pipeline-version':'1'}) meta.append({'name':name,'content_type':mime,'size_bytes':len(data),'sha256':hashlib.sha256(data).hexdigest(),'width':size[0],'height':size[1]}) if os.getenv('FAULT_AFTER_S3_WRITE_ONCE','0')=='1': with db.connection() as c: seen=c.execute("SELECT 1 FROM attachment_job_audit WHERE job_id=%s AND event='fault_injected'",(jid,)).fetchone() if not seen: c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'fault_injected','crash after deterministic S3 writes, before ready commit')",(jid,)) c.commit() os._exit(72) with db.connection() as c: c.execute("UPDATE attachments SET status='ready',variants=%s::jsonb,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=1 AND status<>'ready'",(json.dumps(meta),aid)) c.execute("UPDATE attachment_jobs SET state='done',lease_until=NULL,last_error=NULL WHERE job_id=%s",(jid,)) c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'ready')",(jid,)) s3.delete_object(Bucket=bucket,Key=r['staging_key']) except (UnidentifiedImageError,ValueError) as e: retry_or_fail(jid,aid,'invalid_image',True) except Exception as e: retry_or_fail(jid,aid,'processing_failed',False) def retry_or_fail(jid,aid,code,permanent): with db.connection() as c: r=c.execute('SELECT attempt,state FROM attachment_jobs WHERE job_id=%s',(jid,)).fetchone() if not r or r['state']=='done': return n=r['attempt'] if permanent or n>=attempts: c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error=%s WHERE job_id=%s",(code,jid)) c.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s",('invalid_image' if permanent else 'retry_exhausted',aid)) c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed',%s)",(jid,code)) broker.xadd('attachment-jobs-dlq',{'job_id':str(jid),'attachment_id':str(aid),'error_code':code}) else: delay=min(60,2**n)*(0.8+random.random()*.4) c.execute("UPDATE attachment_jobs SET state='pending',available_at=clock_timestamp()+(%s * interval '1 second'),lease_until=NULL,last_error=%s WHERE job_id=%s",(delay,code,jid)) c.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s",(aid,)) def recover_leases(): with db.connection() as c: expired=c.execute("SELECT job_id,attachment_id,attempt FROM attachment_jobs WHERE state='processing' AND lease_until=attempts: c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error='retry_exhausted' WHERE job_id=%s",(row['job_id'],)) c.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['attachment_id'],)) c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed','lease attempts exhausted')",(row['job_id'],)) broker.xadd('attachment-jobs-dlq',{'job_id':str(row['job_id']),'attachment_id':str(row['attachment_id']),'error_code':'retry_exhausted'}) else: c.execute("UPDATE attachment_jobs SET state='pending',lease_until=NULL,available_at=clock_timestamp() WHERE job_id=%s",(row['job_id'],)) c.execute("UPDATE attachments a SET status='queued',updated_at=clock_timestamp() FROM attachment_jobs j WHERE j.attachment_id=a.id AND j.state='pending' AND a.status='processing'") def main(): consumer=f"worker-{uuid.uuid4()}" while True: try: recover_leases(); publish_outbox() if os.getenv('WORKER_PAUSE_CONSUME','0')=='1': time.sleep(.2); continue msgs=broker.xreadgroup(group,consumer,{stream:'>'},count=concurrency,block=1000) # Reclaim jobs left pending by a worker crash; a ready job becomes a no-op then ack. claimed=broker.xautoclaim(stream,group,consumer,min_idle_time=timeout*1000,start_id='0-0',count=concurrency) if claimed and claimed[1]: msgs.append((stream,claimed[1])) for _,entries in msgs: with ThreadPoolExecutor(max_workers=concurrency) as executor: futures=[] for mid,fields in entries: try: job=json.loads(fields['job']) except Exception: broker.xadd('attachment-jobs-dlq',{'message_id':mid,'error_code':'invalid_job'}); broker.xack(stream,group,mid); continue futures.append((mid,executor.submit(process,job))) for mid,future in futures: try: future.result(); broker.xack(stream,group,mid) except Exception: pass except Exception: time.sleep(1) if __name__=='__main__': main()