Files
mtusi2026lab5/app/worker.py
T

112 lines
5.4 KiB
Python

"""At-least-once consumer; PostgreSQL session lock serializes each logical effect."""
import hashlib
import json
import os
import time
from io import BytesIO
from uuid import UUID
from PIL import Image, ImageOps, UnidentifiedImageError
from psycopg.types.json import Jsonb
from app.broker import connect, QUEUE
from app.s3 import s3_client, bucket
from app.storage import db
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
Image.MAX_IMAGE_PIXELS = 40_000_000
def render(raw):
with Image.open(BytesIO(raw)) as source:
if source.width * source.height > 40_000_000:
raise ValueError('pixel limit')
image = ImageOps.exif_transpose(source).convert('RGBA')
scale = min(1, 256/image.width, 256/image.height)
dimensions = (max(1, int(image.width*scale)), max(1, int(image.height*scale)))
image = image.resize(dimensions, Image.Resampling.LANCZOS)
white = Image.new('RGB', dimensions, 'white')
white.paste(image, mask=image.getchannel('A'))
output = BytesIO()
white.save(output, format='JPEG', quality=85)
data = output.getvalue()
return data, dict(name='thumbnail', content_type='image/jpeg', size_bytes=len(data),
width=dimensions[0], height=dimensions[1], sha256=hashlib.sha256(data).hexdigest())
def process(message):
job_id = UUID(message['job_id'])
# Stable session advisory lock, released automatically on worker death.
lock = int.from_bytes(hashlib.sha256(job_id.bytes).digest()[:8], 'big', signed=True)
with db() as conn:
if not conn.execute('SELECT pg_try_advisory_lock(%s) AS locked', (lock,)).fetchone()['locked']:
return # active owner or DB delivery intent will finish/recover this job
conn.commit()
row = conn.execute('''SELECT a.*, j.attempts, j.pipeline_version FROM attachment_jobs j
JOIN attachments a ON a.id=j.attachment_id WHERE j.job_id=%s''', (job_id,)).fetchone()
if not row or row['status'] in ('ready', 'failed'):
return
if str(row['id']) != message['attachment_id'] or message['pipeline_version'] != row['pipeline_version']:
return
if row['attempts'] >= MAX_ATTEMPTS:
conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s", (row['id'],))
return
conn.execute('UPDATE attachment_jobs SET attempts=attempts+1 WHERE job_id=%s', (job_id,))
conn.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (row['id'],))
conn.commit()
try:
client = s3_client()
obj = client.get_object(Bucket=bucket(), Key=row['object_key'])
try:
raw = obj['Body'].read(10*1024*1024+1)
finally:
obj['Body'].close()
if len(raw) != row['size_bytes'] or hashlib.sha256(raw).hexdigest() != row['sha256']:
raise ValueError('checksum mismatch')
thumbnail, metadata = render(raw)
prefix = f"results/{row['id']}/v1/"
client.put_object(Bucket=bucket(), Key=prefix+'original', Body=raw, ContentType=row['content_type'])
client.put_object(Bucket=bucket(), Key=prefix+'thumbnail', Body=thumbnail, ContentType='image/jpeg')
conn.execute("""UPDATE attachments SET status='ready',object_key=%s,variants=%s,
error_code=NULL,updated_at=now() WHERE id=%s""",
(prefix+'original', Jsonb([metadata]), row['id']))
conn.commit() # consumer ack happens only AFTER this durable result
print(f"ready attachment={row['id']} attempt={row['attempts']+1}", flush=True)
except Exception as exc:
conn.rollback()
permanent = isinstance(exc, (ValueError, UnidentifiedImageError, Image.DecompressionBombError))
exhausted = row['attempts']+1 >= MAX_ATTEMPTS
status = 'failed' if permanent or exhausted else 'queued'
code = ('invalid_image' if permanent else 'retry_exhausted') if status=='failed' else None
conn.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=now() WHERE id=%s',
(status, code, row['id']))
conn.execute("UPDATE attachment_jobs SET next_publish=now()+interval '5 seconds' WHERE job_id=%s", (job_id,))
print(f"{status} attachment={row['id']} reason={type(exc).__name__}", flush=True)
def consume(channel, method, properties, body):
try:
message = json.loads(body)
UUID(message['job_id'])
UUID(message['attachment_id'])
if message['schema_version'] != 1 or message['type'] != 'attachment.process.v1':
raise ValueError('unsupported job')
except (ValueError, KeyError, TypeError):
channel.basic_reject(method.delivery_tag, requeue=False)
return
process(message)
channel.basic_ack(method.delivery_tag)
def main():
while True:
connection = None
try:
connection, channel = connect()
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
channel.start_consuming()
except Exception as exc:
print('worker reconnect:', type(exc).__name__, flush=True)
time.sleep(2)
finally:
if connection and connection.is_open:
connection.close()
if __name__ == '__main__':
main()