Implement lab 5 durable image processing worker

This commit is contained in:
2026-11-20 19:08:09 +03:00
parent 276dbdad8a
commit d8332e17f6
28 changed files with 1330 additions and 24 deletions
+60
View File
@@ -0,0 +1,60 @@
import hashlib
import io
import json
import random
import time
from PIL import Image, ImageOps
from src.db import make_pool
from src.storage import BUCKET, s3
MAX_ATTEMPTS = 5
def make_variant(image, name):
image = ImageOps.exif_transpose(image)
if name == 'thumbnail':
image.thumbnail((256, 256), Image.Resampling.LANCZOS); target = image
elif name == 'crop':
target = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS)
else:
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS); target = image
if target.mode not in ('RGB', 'L'):
bg = Image.new('RGB', target.size, 'white')
bg.paste(target, mask=target.getchannel('A') if 'A' in target.getbands() else None)
target = bg
out = io.BytesIO()
content_type = 'image/webp' if name == 'optimized' else 'image/jpeg'
target.convert('RGB').save(out, format='WEBP' if name == 'optimized' else 'JPEG', quality=82, exif=b'')
raw = out.getvalue()
return raw, content_type, target.width, target.height
def process(row, pool):
raw = s3.get_object(Bucket=BUCKET, Key=row['object_key'])['Body'].read()
with Image.open(io.BytesIO(raw)) as image:
variants=[]
for name in ('thumbnail','crop','optimized'):
data, content_type, width, height = make_variant(image.copy(), name)
s3.put_object(Bucket=BUCKET, Key=row['object_key']+'.'+name, Body=data, ContentType=content_type)
variants.append({'name':name,'content_type':content_type,'size_bytes':len(data),'width':width,'height':height,'sha256':hashlib.sha256(data).hexdigest()})
with pool.connection() as conn:
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'", (json.dumps(variants),row['id']))
conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),locked_until=NULL WHERE attachment_id=%s',(row['id'],))
def loop():
pool=make_pool()
while True:
with pool.connection() as conn:
job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until<clock_timestamp()) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *").fetchone()
row=conn.execute('SELECT * FROM attachments WHERE id=%s',(job['attachment_id'],)).fetchone() if job else None
if not row:
time.sleep(.25); continue
try: process(row,pool)
except Exception as error:
with pool.connection() as conn:
if job['attempts'] >= MAX_ATTEMPTS:
conn.execute("UPDATE attachments SET status='failed',error_code='processing_failed',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),last_error=%s WHERE attachment_id=%s',(type(error).__name__,row['id']))
else:
delay=min(60,2**job['attempts'])+random.random()
conn.execute("UPDATE attachment_jobs SET available_at=clock_timestamp()+(%s*interval '1 second'),locked_until=NULL,last_error=%s WHERE attachment_id=%s",(delay,type(error).__name__,row['id']))
if __name__ == '__main__': loop()