Files
mtusi2026lab5/scripts/worker.py
T

271 lines
11 KiB
Python

"""Redis Streams worker for durable attachment processing."""
import hashlib
import io
import os
import random
import time
import uuid
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
import redis
from botocore.config import Config
from PIL import Image, ImageOps, UnidentifiedImageError
from psycopg.rows import dict_row
DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
BROKER_URL = os.getenv('BROKER_URL', 'redis://broker:6379/0')
S3_ENDPOINT = os.getenv('S3_ENDPOINT', 'http://minio:9000')
S3_BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID', 'messenger-api')
S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123')
MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
MAX_JOB_ATTEMPTS = int(os.getenv('MAX_JOB_ATTEMPTS', '5'))
PROCESSING_LEASE_SECONDS = int(os.getenv('PROCESSING_LEASE_SECONDS', '30'))
ACK_DELAY_SECONDS = int(os.getenv('ACK_DELAY_SECONDS', '0'))
STREAM = 'attachment-jobs'
GROUP = 'attachment-workers'
CONSUMER = os.getenv('WORKER_ID', 'worker-' + uuid.uuid4().hex[:8])
broker = redis.Redis.from_url(
BROKER_URL, decode_responses=True, socket_connect_timeout=2,
socket_timeout=3, retry_on_timeout=False,
)
s3 = boto3.client(
's3', endpoint_url=S3_ENDPOINT, region_name='us-east-1',
aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY,
config=Config(connect_timeout=2, read_timeout=10, retries={'max_attempts': 1}),
)
def now():
return datetime.now(timezone.utc)
def ensure_group():
try:
broker.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):
raise
def recover_expired_leases():
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
rows = connection.execute(
"SELECT id FROM attachments WHERE status='processing' AND lease_until < now()"
).fetchall()
for item in rows:
attachment_id = item['id']
connection.execute(
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
"updated_at=now(), error_code=NULL WHERE id=%s",
(attachment_id,),
)
connection.execute(
"UPDATE attachment_jobs SET published_at=NULL, next_attempt_at=now(), "
"last_error='worker_lease_expired' WHERE attachment_id=%s",
(attachment_id,),
)
def publish_outbox():
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
row = connection.execute(
"SELECT job_id, attachment_id, pipeline_version, attempt "
"FROM attachment_jobs WHERE published_at IS NULL AND next_attempt_at <= now() "
"ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED"
).fetchone()
if row is None:
return False
payload = {
'schema_version': '1', 'job_id': str(row['job_id']),
'type': 'attachment.process.v1', 'attachment_id': str(row['attachment_id']),
'pipeline_version': str(row['pipeline_version']), 'attempt': str(row['attempt']),
'created_at': now().isoformat().replace('+00:00', 'Z'),
}
broker.xadd(STREAM, payload, maxlen=10000, approximate=True)
connection.execute(
'UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s', (row['job_id'],)
)
connection.commit()
return True
def white_background(image):
image = ImageOps.exif_transpose(image)
if image.mode in ('RGBA', 'LA') or 'transparency' in image.info:
background = Image.new('RGB', image.size, 'white')
alpha = image.convert('RGBA')
background.paste(alpha, mask=alpha.getchannel('A'))
return background
return image.convert('RGB')
def jpeg_bytes(image, size, crop=False):
image = image.copy()
if crop:
side = min(image.size)
left = (image.width - side) // 2
top = (image.height - side) // 2
image = image.crop((left, top, left + side, top + side))
if crop:
image = image.resize((size, size), Image.Resampling.LANCZOS)
else:
image.thumbnail((size, size), Image.Resampling.LANCZOS)
output = io.BytesIO()
image.save(output, format='JPEG', quality=85, optimize=True)
return output.getvalue(), image.size
def webp_bytes(image):
image = image.copy()
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
output = io.BytesIO()
image.save(output, format='WEBP', quality=82, method=6)
return output.getvalue(), image.size
def get_attachment(attachment_id):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
return connection.execute('SELECT * FROM attachments WHERE id=%s', (attachment_id,)).fetchone()
def claim(attachment_id, message_attempt):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
row = connection.execute(
'SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)
).fetchone()
if row is None or row['status'] in ('ready', 'failed'):
return row, False
if row['status'] == 'processing' and row['lease_until'] and row['lease_until'] > now():
return row, False
attempt = max(row['attempt'] + 1, message_attempt)
connection.execute(
"UPDATE attachments SET status='processing', attempt=%s, lease_until=%s, "
"updated_at=now(), error_code=NULL WHERE id=%s",
(attempt, now() + timedelta(seconds=PROCESSING_LEASE_SECONDS), attachment_id),
)
row['attempt'] = attempt
row['status'] = 'processing'
return row, True
def mark_retry(attachment_id, attempt, error_code, permanent=False):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
if permanent or attempt >= MAX_JOB_ATTEMPTS:
connection.execute(
"UPDATE attachments SET status='failed', lease_until=NULL, error_code=%s, updated_at=now() WHERE id=%s",
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
)
connection.execute(
"UPDATE attachment_jobs SET published_at=now(), last_error=%s WHERE attachment_id=%s",
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
)
else:
delay = min(30, 2 ** max(0, attempt - 1)) + random.uniform(0, 1)
connection.execute(
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
"error_code=%s, updated_at=now() WHERE id=%s",
(error_code, attachment_id),
)
connection.execute(
"UPDATE attachment_jobs SET attempt=%s, published_at=NULL, next_attempt_at=now()+%s * interval '1 second', last_error=%s "
"WHERE attachment_id=%s",
(attempt + 1, delay, error_code, attachment_id),
)
def process(attachment_id, message_attempt):
row, owned = claim(attachment_id, message_attempt)
if not owned:
return
try:
value = s3.get_object(Bucket=S3_BUCKET, Key=row['staging_key'])
original = value['Body'].read()
with Image.open(io.BytesIO(original)) as source:
if source.format not in ('PNG', 'JPEG') or source.width * source.height > MAX_IMAGE_PIXELS:
raise ValueError('invalid image')
image = white_background(source)
thumbnail, thumb_size = jpeg_bytes(image, 256)
crop, crop_size = jpeg_bytes(image, 128, crop=True)
optimized, optimized_size = webp_bytes(image)
base = 'attachments/' + str(attachment_id) + '/'
objects = [
('original', original, row['content_type'], image.size),
('thumbnail', thumbnail, 'image/jpeg', thumb_size),
('crop', crop, 'image/jpeg', crop_size),
('optimized', optimized, 'image/webp', optimized_size),
]
for name, data, content_type, _size in objects:
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp' if content_type == 'image/webp' else '')
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=data, ContentType=content_type, ContentLength=len(data))
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
connection.execute(
"UPDATE attachments SET status='ready', lease_until=NULL, error_code=NULL, updated_at=now() WHERE id=%s",
(attachment_id,),
)
for name, data, content_type, dimensions in objects[1:]:
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp')
connection.execute(
'INSERT INTO attachment_variants '
'(attachment_id,name,content_type,object_key,size_bytes,width,height,sha256) '
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) '
'ON CONFLICT (attachment_id,name) DO UPDATE SET '
'content_type=EXCLUDED.content_type,object_key=EXCLUDED.object_key,size_bytes=EXCLUDED.size_bytes,'
'width=EXCLUDED.width,height=EXCLUDED.height,sha256=EXCLUDED.sha256',
(attachment_id, name, content_type, key, len(data), dimensions[0], dimensions[1], hashlib.sha256(data).hexdigest()),
)
try:
s3.delete_object(Bucket=S3_BUCKET, Key=row['staging_key'])
except Exception:
pass
if ACK_DELAY_SECONDS:
time.sleep(ACK_DELAY_SECONDS)
except (UnidentifiedImageError, ValueError, OSError):
mark_retry(attachment_id, row['attempt'], 'invalid_image', permanent=True)
except Exception:
mark_retry(attachment_id, row['attempt'], 'processing_failed')
def handle_message(message_id, fields):
process(uuid.UUID(fields['attachment_id']), int(fields.get('attempt', '1')))
broker.xack(STREAM, GROUP, message_id)
def reclaim_pending():
try:
result = broker.xautoclaim(STREAM, GROUP, CONSUMER, min_idle_time=1000, start_id='0-0', count=10)
for message_id, fields in result[1]:
handle_message(message_id, fields)
except (redis.ResponseError, AttributeError):
pass
def main():
ensure_group()
print('worker started', CONSUMER, flush=True)
while True:
try:
recover_expired_leases()
publish_outbox()
reclaim_pending()
messages = broker.xreadgroup(GROUP, CONSUMER, {STREAM: '>'}, count=1, block=1000)
for _stream, items in messages:
for message_id, fields in items:
handle_message(message_id, fields)
except Exception as exc:
print('worker loop error:', type(exc).__name__, flush=True)
time.sleep(1)
if __name__ == '__main__':
main()