forked from mtusi2026/mtusi2026lab5
132 lines
10 KiB
Python
132 lines
10 KiB
Python
"""Durable outbox relay and idempotent Redis Streams image worker."""
|
|
import io, json, os, time, uuid, hashlib, random, shutil
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import redis, boto3, psycopg
|
|
from psycopg.rows import dict_row
|
|
from psycopg_pool import ConnectionPool
|
|
from botocore.config import Config
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
from app.migrate import setting
|
|
|
|
def envsetting(name): return setting(name) if os.getenv(name+'_FILE') else os.environ[name]
|
|
db=ConnectionPool(setting('DATABASE_URL'),min_size=1,max_size=8,kwargs={'row_factory':dict_row,'connect_timeout':3})
|
|
broker=redis.Redis.from_url(setting('BROKER_URL'),socket_connect_timeout=1,socket_timeout=2,decode_responses=True)
|
|
s3=boto3.client('s3',endpoint_url=os.getenv('S3_ENDPOINT','http://rustfs:9000'),region_name=os.getenv('S3_REGION','us-east-1'),aws_access_key_id=envsetting('S3_ACCESS_KEY_ID'),aws_secret_access_key=envsetting('S3_SECRET_ACCESS_KEY'),config=Config(connect_timeout=3,read_timeout=10,retries={'max_attempts':2}))
|
|
bucket=os.getenv('S3_BUCKET','messenger-attachments'); stream='attachment-jobs'; group='image-workers'; attempts=int(os.getenv('JOB_MAX_ATTEMPTS','5')); concurrency=int(os.getenv('WORKER_CONCURRENCY','2')); timeout=int(os.getenv('JOB_TIMEOUT_SECONDS','60'))
|
|
try: broker.xgroup_create(stream,group,id='0',mkstream=True)
|
|
except redis.ResponseError as e:
|
|
if 'BUSYGROUP' not in str(e): raise
|
|
|
|
def publish_outbox():
|
|
# DB intent remains pending until XADD succeeds; crash in the gap yields a safe duplicate.
|
|
with db.connection() as c:
|
|
rows=c.execute("SELECT job_id,attachment_id,pipeline_version,attempt,created_at FROM attachment_jobs WHERE state IN ('pending','published') AND available_at<=clock_timestamp() ORDER BY created_at LIMIT 100 FOR UPDATE SKIP LOCKED").fetchall()
|
|
for r in rows:
|
|
msg={'schema_version':1,'job_id':str(r['job_id']),'type':'attachment.process.v1','attachment_id':str(r['attachment_id']),'pipeline_version':r['pipeline_version'],'attempt':max(1,r['attempt']+1),'created_at':r['created_at'].isoformat().replace('+00:00','Z')}
|
|
broker.xadd(stream,{'job':json.dumps(msg,separators=(',',':'))})
|
|
c.execute("UPDATE attachment_jobs SET state='published',available_at=clock_timestamp()+interval '15 seconds' WHERE job_id=%s AND state IN ('pending','published')",(r['job_id'],))
|
|
|
|
def resize(raw):
|
|
im=Image.open(io.BytesIO(raw)); im.load(); im=ImageOps.exif_transpose(im)
|
|
if im.width*im.height>40_000_000: raise ValueError('oversized_image')
|
|
if im.mode in ('RGBA','LA') or 'transparency' in im.info:
|
|
rgba=im.convert('RGBA'); bg=Image.new('RGBA',rgba.size,'white'); bg.alpha_composite(rgba); im=bg.convert('RGB')
|
|
else: im=im.convert('RGB')
|
|
thumb=im.copy(); thumb.thumbnail((256,256),Image.Resampling.LANCZOS)
|
|
out=io.BytesIO(); thumb.save(out,format='JPEG',quality=85,optimize=True)
|
|
crop=ImageOps.fit(im,(128,128),method=Image.Resampling.LANCZOS,centering=(.5,.5)); cropout=io.BytesIO(); crop.save(cropout,format='JPEG',quality=85,optimize=True)
|
|
opt=im.copy(); opt.thumbnail((1280,1280),Image.Resampling.LANCZOS); webp=io.BytesIO(); opt.save(webp,format='WEBP',quality=82,method=4,exif=b'')
|
|
return [('thumbnail','image/jpeg',thumb.size,out.getvalue()),('crop','image/jpeg',crop.size,cropout.getvalue()),('optimized','image/webp',opt.size,webp.getvalue())]
|
|
|
|
def process(job):
|
|
jid=uuid.UUID(job['job_id']); aid=uuid.UUID(job['attachment_id'])
|
|
with db.connection() as c:
|
|
c.execute('BEGIN')
|
|
r=c.execute('SELECT j.*,a.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s FOR UPDATE OF j,a',(jid,)).fetchone()
|
|
if not r: c.rollback(); return
|
|
if r['state']=='done' or r['state']=='failed' or r['state']=='processing' and r['lease_until'] and r['lease_until'].timestamp()>time.time(): c.rollback(); return
|
|
c.execute("UPDATE attachment_jobs SET state='processing',attempt=attempt+1,lease_until=clock_timestamp()+(%s * interval '1 second') WHERE job_id=%s",(timeout,jid))
|
|
c.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'",(aid,))
|
|
c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'processing')",(jid,)); c.commit()
|
|
try:
|
|
raw=s3.get_object(Bucket=bucket,Key=r['staging_key'])['Body'].read(10_485_761)
|
|
if len(raw)>10_485_760 or hashlib.sha256(raw).hexdigest()!=r['sha256'].strip(): raise ValueError('invalid_image')
|
|
variants=resize(raw)
|
|
# Each object key is deterministic, so a crash and redelivery overwrites the same effect.
|
|
s3.put_object(Bucket=bucket,Key=r['object_key'],Body=raw,ContentType=r['content_type'],Metadata={'sha256':r['sha256'].strip()})
|
|
meta=[]
|
|
for name,mime,size,data in variants:
|
|
key=f"variants/v1/{aid}/{name}"
|
|
s3.put_object(Bucket=bucket,Key=key,Body=data,ContentType=mime,Metadata={'pipeline-version':'1'})
|
|
meta.append({'name':name,'content_type':mime,'size_bytes':len(data),'sha256':hashlib.sha256(data).hexdigest(),'width':size[0],'height':size[1]})
|
|
if os.getenv('FAULT_AFTER_S3_WRITE_ONCE','0')=='1':
|
|
with db.connection() as c:
|
|
seen=c.execute("SELECT 1 FROM attachment_job_audit WHERE job_id=%s AND event='fault_injected'",(jid,)).fetchone()
|
|
if not seen:
|
|
c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'fault_injected','crash after deterministic S3 writes, before ready commit')",(jid,))
|
|
c.commit()
|
|
os._exit(72)
|
|
with db.connection() as c:
|
|
c.execute("UPDATE attachments SET status='ready',variants=%s::jsonb,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=1 AND status<>'ready'",(json.dumps(meta),aid))
|
|
c.execute("UPDATE attachment_jobs SET state='done',lease_until=NULL,last_error=NULL WHERE job_id=%s",(jid,))
|
|
c.execute("INSERT INTO attachment_job_audit(job_id,event) VALUES (%s,'ready')",(jid,))
|
|
s3.delete_object(Bucket=bucket,Key=r['staging_key'])
|
|
except (UnidentifiedImageError,ValueError) as e:
|
|
retry_or_fail(jid,aid,'invalid_image',True)
|
|
except Exception as e:
|
|
retry_or_fail(jid,aid,'processing_failed',False)
|
|
|
|
def retry_or_fail(jid,aid,code,permanent):
|
|
with db.connection() as c:
|
|
r=c.execute('SELECT attempt,state FROM attachment_jobs WHERE job_id=%s',(jid,)).fetchone()
|
|
if not r or r['state']=='done': return
|
|
n=r['attempt']
|
|
if permanent or n>=attempts:
|
|
c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error=%s WHERE job_id=%s",(code,jid))
|
|
c.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s",('invalid_image' if permanent else 'retry_exhausted',aid))
|
|
c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed',%s)",(jid,code))
|
|
broker.xadd('attachment-jobs-dlq',{'job_id':str(jid),'attachment_id':str(aid),'error_code':code})
|
|
else:
|
|
delay=min(60,2**n)*(0.8+random.random()*.4)
|
|
c.execute("UPDATE attachment_jobs SET state='pending',available_at=clock_timestamp()+(%s * interval '1 second'),lease_until=NULL,last_error=%s WHERE job_id=%s",(delay,code,jid))
|
|
c.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id=%s",(aid,))
|
|
|
|
def recover_leases():
|
|
with db.connection() as c:
|
|
expired=c.execute("SELECT job_id,attachment_id,attempt FROM attachment_jobs WHERE state='processing' AND lease_until<clock_timestamp() FOR UPDATE SKIP LOCKED").fetchall()
|
|
for row in expired:
|
|
if row['attempt']>=attempts:
|
|
c.execute("UPDATE attachment_jobs SET state='failed',lease_until=NULL,last_error='retry_exhausted' WHERE job_id=%s",(row['job_id'],))
|
|
c.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['attachment_id'],))
|
|
c.execute("INSERT INTO attachment_job_audit(job_id,event,detail) VALUES (%s,'failed','lease attempts exhausted')",(row['job_id'],))
|
|
broker.xadd('attachment-jobs-dlq',{'job_id':str(row['job_id']),'attachment_id':str(row['attachment_id']),'error_code':'retry_exhausted'})
|
|
else:
|
|
c.execute("UPDATE attachment_jobs SET state='pending',lease_until=NULL,available_at=clock_timestamp() WHERE job_id=%s",(row['job_id'],))
|
|
c.execute("UPDATE attachments a SET status='queued',updated_at=clock_timestamp() FROM attachment_jobs j WHERE j.attachment_id=a.id AND j.state='pending' AND a.status='processing'")
|
|
|
|
def main():
|
|
consumer=f"worker-{uuid.uuid4()}"
|
|
while True:
|
|
try:
|
|
recover_leases(); publish_outbox()
|
|
if os.getenv('WORKER_PAUSE_CONSUME','0')=='1':
|
|
time.sleep(.2); continue
|
|
msgs=broker.xreadgroup(group,consumer,{stream:'>'},count=concurrency,block=1000)
|
|
# Reclaim jobs left pending by a worker crash; a ready job becomes a no-op then ack.
|
|
claimed=broker.xautoclaim(stream,group,consumer,min_idle_time=timeout*1000,start_id='0-0',count=concurrency)
|
|
if claimed and claimed[1]: msgs.append((stream,claimed[1]))
|
|
for _,entries in msgs:
|
|
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
|
futures=[]
|
|
for mid,fields in entries:
|
|
try: job=json.loads(fields['job'])
|
|
except Exception:
|
|
broker.xadd('attachment-jobs-dlq',{'message_id':mid,'error_code':'invalid_job'}); broker.xack(stream,group,mid); continue
|
|
futures.append((mid,executor.submit(process,job)))
|
|
for mid,future in futures:
|
|
try: future.result(); broker.xack(stream,group,mid)
|
|
except Exception: pass
|
|
except Exception: time.sleep(1)
|
|
|
|
if __name__=='__main__': main()
|