forked from mtusi2026/mtusi2026lab5
73 lines
4.2 KiB
Python
73 lines
4.2 KiB
Python
import hashlib
|
|
import io
|
|
import json
|
|
import random
|
|
import time
|
|
from PIL import Image, ImageOps
|
|
from src.db import make_pool
|
|
from src.storage import BUCKET, s3
|
|
from src.queue import ensure_group, client as broker, STREAM, GROUP
|
|
|
|
MAX_ATTEMPTS = 5
|
|
|
|
def make_variant(image, name):
|
|
image = ImageOps.exif_transpose(image)
|
|
if name == 'thumbnail':
|
|
image.thumbnail((256, 256), Image.Resampling.LANCZOS); target = image
|
|
elif name == 'crop':
|
|
target = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS)
|
|
else:
|
|
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS); target = image
|
|
if target.mode not in ('RGB', 'L'):
|
|
bg = Image.new('RGB', target.size, 'white')
|
|
bg.paste(target, mask=target.getchannel('A') if 'A' in target.getbands() else None)
|
|
target = bg
|
|
out = io.BytesIO()
|
|
content_type = 'image/webp' if name == 'optimized' else 'image/jpeg'
|
|
target.convert('RGB').save(out, format='WEBP' if name == 'optimized' else 'JPEG', quality=82, exif=b'')
|
|
raw = out.getvalue()
|
|
return raw, content_type, target.width, target.height
|
|
|
|
def process(row, pool):
|
|
raw = s3.get_object(Bucket=BUCKET, Key=row['object_key'])['Body'].read()
|
|
with Image.open(io.BytesIO(raw)) as image:
|
|
variants=[]
|
|
for name in ('thumbnail','crop','optimized'):
|
|
data, content_type, width, height = make_variant(image.copy(), name)
|
|
s3.put_object(Bucket=BUCKET, Key=row['object_key']+'.'+name, Body=data, ContentType=content_type)
|
|
variants.append({'name':name,'content_type':content_type,'size_bytes':len(data),'width':width,'height':height,'sha256':hashlib.sha256(data).hexdigest()})
|
|
with pool.connection() as conn:
|
|
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'", (json.dumps(variants),row['id']))
|
|
conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),locked_until=NULL WHERE attachment_id=%s',(row['id'],))
|
|
|
|
def loop():
|
|
pool=make_pool()
|
|
ensure_group()
|
|
while True:
|
|
# Repair the transactional-outbox -> stream gap after a crash between commit and XADD.
|
|
with pool.connection() as conn:
|
|
pending = conn.execute("SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) LIMIT 50").fetchall()
|
|
for pending_row in pending:
|
|
broker.xadd(STREAM, {'attachment_id': str(pending_row['attachment_id'])}, maxlen=10000, approximate=True)
|
|
messages = broker.xreadgroup(GROUP, 'worker-1', {STREAM:'>'}, count=1, block=1000)
|
|
if messages:
|
|
for _, entries in messages:
|
|
for message_id, fields in entries:
|
|
broker.xack(STREAM, GROUP, message_id)
|
|
with pool.connection() as conn:
|
|
job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *").fetchone()
|
|
row=conn.execute('SELECT * FROM attachments WHERE id=%s',(job['attachment_id'],)).fetchone() if job else None
|
|
if not row:
|
|
time.sleep(.25); continue
|
|
try: process(row,pool)
|
|
except Exception as error:
|
|
with pool.connection() as conn:
|
|
if job['attempts'] >= MAX_ATTEMPTS:
|
|
conn.execute("UPDATE attachments SET status='failed',error_code='processing_failed',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
|
conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),last_error=%s WHERE attachment_id=%s',(type(error).__name__,row['id']))
|
|
else:
|
|
delay=min(60,2**job['attempts'])+random.random()
|
|
conn.execute("UPDATE attachment_jobs SET available_at=clock_timestamp()+(%s*interval '1 second'),locked_until=NULL,last_error=%s WHERE attachment_id=%s",(delay,type(error).__name__,row['id']))
|
|
|
|
if __name__ == '__main__': loop()
|