Files
mtusi2026lab5/app/worker.py
T

132 lines
10 KiB
Python

"""Durable outbox relay and idempotent Redis Streams image worker."""
import io, json, os, time, uuid, hashlib, random, shutil
from concurrent.futures import ThreadPoolExecutor
import redis, boto3, psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from botocore.config import Config
from PIL import Image, ImageOps, UnidentifiedImageError
from app.migrate import setting
def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name]
db=ConnectionPool(setting('DATABASE_URL'),min_size=1,max_size=8,kwargs={'row_factory':dict_row,'connect_timeout':3})
broker=redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=1,socket_timeout=2,decode_responses=True)
s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2}))
bucket=os.getenv('S3_BUCKET','messenger-attachments'); stream='attachment-jobs'; group='image-workers'; attempts=int(os.getenv('JOB_MAX_ATTEMPTS','5')); concurrency=int(os.getenv('WORKER_CONCURRENCY','2')); timeout=int(os.getenv('JOB_TIMEOUT_SECONDS','60'))
try: broker.xgroup_create(stream,group,id='0',mkstream=True)
except redis.ResponseError as e:
if 'BUSYGROUP' not in str(e): raise
def publish_outbox():
# DB intent remains pending until XADD succeeds; crash in the gap yields a safe duplicate.
with db.connection() as c:
rows=c.execute("SELECT job_id,attachment_id,pipeline_version,attempt,created_at FROM attachment_jobs WHERE state IN ('pending','published') AND available_at<=clock_timestamp() ORDER BY created_at LIMIT 100 FOR UPDATE SKIP LOCKED").fetchall()
for r in rows:
msg={'schema_version':1,'job_id':str(r['job_id']),'type':'attachment.process.v1','attachment_id':str(r['attachment_id']),'pipeline_version':r['pipeline_version'],'attempt':max(1,r['attempt']+1),'created_at':r['created_at'].isoformat().replace('+00:00','Z')}
broker.xadd(stream,{'job':json.dumps(msg,separators=(',',':'))})
c.execute("UPDATE attachment_jobs SET state='published',available_at=clock_timestamp()+interval '15 seconds' WHERE job_id=%s AND state IN ('pending','published')",(r['job_id'],))
def resize(raw):
im=Image.open(io.BytesIO(raw)); im.load(); im=ImageOps.exif_transpose(im)
if im.width*im.height>40_000_000: raise ValueError('oversized_image')
if im.mode in ('RGBA','LA') or 'transparency' in im.info:
rgba=im.convert('RGBA'); bg=Image.new('RGBA',rgba.size,'white'); bg.alpha_composite(rgba); im=bg.convert('RGB')
else: im=im.convert('RGB')
thumb=im.copy(); thumb.thumbnail((256,256),Image.Resampling.LANCZOS)
out=io.BytesIO(); thumb.save(out,format='JPEG',quality=85,optimize=True)
crop=ImageOps.fit(im,(128,128),method=Image.Resampling.LANCZOS,centering=(.5,.5)); cropout=io.BytesIO(); crop.save(cropout,format='JPEG',quality=85,optimize=True)
opt=im.copy(); opt.thumbnail((1280,1280),Image.Resampling.LANCZOS); webp=io.BytesIO(); opt.save(webp,format='WEBP',quality=82,method=4,exif=b'')
return [('thumbnail','image/jpeg',thumb.size,out.getvalue()),('crop','image/jpeg',crop.size,cropout.getvalue()),('optimized','image/webp',opt.size,webp.getvalue())]
def process(job):
jid=uuid.UUID(job['job_id']); aid=uuid.UUID(job['attachment_id'])
with db.connection() as c:
c.execute('BEGIN')
r=c.execute('SELECT j.*,a.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j,a',(jid,)).fetchone()
if not r: c.rollback(); return
if r['state']=='done' or r['state']=='failed' or r['state']=='processing' and r['lease_until'] and r['lease_until'].timestamp()>time.time(): c.rollback(); return
c.execute("UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_until=clock_timestamp()+(%s * interval '1 second') WHERE job_id=%s",(timeout,jid))
c.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'",(aid,))
c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'processing')",(jid,)); c.commit()
try:
raw=s3.get_object(Bucket=bucket,Key=r['staging_key'])['Body'].read(10_485_761)
if len(raw)>10_485_760 or hashlib.sha256(raw).hexdigest()!=r['sha256'].strip(): raise ValueError('invalid_image')
variants=resize(raw)
# Each object key is deterministic, so a crash and redelivery overwrites the same effect.
s3.put_object(Bucket=bucket,Key=r['object_key'],Body=raw,ContentType=r['content_type'],Metadata={'sha256':r['sha256'].strip()})
meta=[]
for name,mime,size,data in variants:
key=f"variants/v1/{aid}/{name}"
s3.put_object(Bucket=bucket,Key=key,Body=data,ContentType=mime,Metadata={'pipeline-version':'1'})
meta.append({'name':name,'content_type':mime,'size_bytes':len(data),'sha256':hashlib.sha256(data).hexdigest(),'width':size[0],'height':size[1]})
if os.getenv('FAULT_AFTER_S3_WRITE_ONCE','0')=='1':
with db.connection() as c:
seen=c.execute("SELECT 1 FROM attachment_job_audit WHERE job_id=%s AND event='fault_injected'",(jid,)).fetchone()
if not seen:
c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'fault_injected','crash after deterministic S3 writes, before ready commit')",(jid,))
c.commit()
os._exit(72)
with db.connection() as c:
c.execute("UPDATE attachments SET status='ready',variants=%s::jsonb,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=1 AND status<>'ready'",(json.dumps(meta),aid))
c.execute("UPDATE attachment_jobs SET state='done',lease_until=NULL,last_error=NULL WHERE job_id=%s",(jid,))
c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'ready')",(jid,))
s3.delete_object(Bucket=bucket,Key=r['staging_key'])
except (UnidentifiedImageError,ValueError) as e:
retry_or_fail(jid,aid,'invalid_image',True)
except Exception as e:
retry_or_fail(jid,aid,'processing_failed',False)
def retry_or_fail(jid,aid,code,permanent):
with db.connection() as c:
r=c.execute('SELECT attempt,state FROM attachment_jobs WHERE job_id=%s',(jid,)).fetchone()
if not r or r['state']=='done': return
n=r['attempt']
if permanent or n>=attempts:
c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error=%s WHERE job_id=%s",(code,jid))
c.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s",('invalid_image' if permanent else 'retry_exhausted',aid))
c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed',%s)",(jid,code))
broker.xadd('attachment-jobs-dlq',{'job_id':str(jid),'attachment_id':str(aid),'error_code':code})
else:
delay=min(60,2**n)*(0.8+random.random()*.4)
c.execute("UPDATE attachment_jobs SET state='pending',available_at=clock_timestamp()+(%s * interval '1 second'),lease_until=NULL,last_error=%s WHERE job_id=%s",(delay,code,jid))
c.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s",(aid,))
def recover_leases():
with db.connection() as c:
expired=c.execute("SELECT job_id,attachment_id,attempt FROM attachment_jobs WHERE state='processing' AND lease_until<clock_timestamp() FOR UPDATE SKIP LOCKED").fetchall()
for row in expired:
if row['attempt']>=attempts:
c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error='retry_exhausted' WHERE job_id=%s",(row['job_id'],))
c.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['attachment_id'],))
c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed','lease attempts exhausted')",(row['job_id'],))
broker.xadd('attachment-jobs-dlq',{'job_id':str(row['job_id']),'attachment_id':str(row['attachment_id']),'error_code':'retry_exhausted'})
else:
c.execute("UPDATE attachment_jobs SET state='pending',lease_until=NULL,available_at=clock_timestamp() WHERE job_id=%s",(row['job_id'],))
c.execute("UPDATE attachments a SET status='queued',updated_at=clock_timestamp() FROM attachment_jobs j WHERE j.attachment_id=a.id AND j.state='pending' AND a.status='processing'")
def main():
consumer=f"worker-{uuid.uuid4()}"
while True:
try:
recover_leases(); publish_outbox()
if os.getenv('WORKER_PAUSE_CONSUME','0')=='1':
time.sleep(.2); continue
msgs=broker.xreadgroup(group,consumer,{stream:'>'},count=concurrency,block=1000)
# Reclaim jobs left pending by a worker crash; a ready job becomes a no-op then ack.
claimed=broker.xautoclaim(stream,group,consumer,min_idle_time=timeout*1000,start_id='0-0',count=concurrency)
if claimed and claimed[1]: msgs.append((stream,claimed[1]))
for _,entries in msgs:
with ThreadPoolExecutor(max_workers=concurrency) as executor:
futures=[]
for mid,fields in entries:
try: job=json.loads(fields['job'])
except Exception:
broker.xadd('attachment-jobs-dlq',{'message_id':mid,'error_code':'invalid_job'}); broker.xack(stream,group,mid); continue
futures.append((mid,executor.submit(process,job)))
for mid,future in futures:
try: future.result(); broker.xack(stream,group,mid)
except Exception: pass
except Exception: time.sleep(1)
if __name__=='__main__': main()