271 lines
11 KiB
Python
271 lines
11 KiB
Python
"""Redis Streams worker for durable attachment processing."""
|
|
import hashlib
|
|
import io
|
|
import os
|
|
import random
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import boto3
|
|
import psycopg
|
|
import redis
|
|
from botocore.config import Config
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
from psycopg.rows import dict_row
|
|
|
|
DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
|
|
BROKER_URL = os.getenv('BROKER_URL', 'redis://broker:6379/0')
|
|
S3_ENDPOINT = os.getenv('S3_ENDPOINT', 'http://minio:9000')
|
|
S3_BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
|
S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID', 'messenger-api')
|
|
S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123')
|
|
MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
|
|
MAX_JOB_ATTEMPTS = int(os.getenv('MAX_JOB_ATTEMPTS', '5'))
|
|
PROCESSING_LEASE_SECONDS = int(os.getenv('PROCESSING_LEASE_SECONDS', '30'))
|
|
ACK_DELAY_SECONDS = int(os.getenv('ACK_DELAY_SECONDS', '0'))
|
|
STREAM = 'attachment-jobs'
|
|
GROUP = 'attachment-workers'
|
|
CONSUMER = os.getenv('WORKER_ID', 'worker-' + uuid.uuid4().hex[:8])
|
|
|
|
broker = redis.Redis.from_url(
|
|
BROKER_URL, decode_responses=True, socket_connect_timeout=2,
|
|
socket_timeout=3, retry_on_timeout=False,
|
|
)
|
|
s3 = boto3.client(
|
|
's3', endpoint_url=S3_ENDPOINT, region_name='us-east-1',
|
|
aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
|
config=Config(connect_timeout=2, read_timeout=10, retries={'max_attempts': 1}),
|
|
)
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def ensure_group():
|
|
try:
|
|
broker.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if 'BUSYGROUP' not in str(exc):
|
|
raise
|
|
|
|
|
|
def recover_expired_leases():
|
|
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
|
with connection.transaction():
|
|
rows = connection.execute(
|
|
"SELECT id FROM attachments WHERE status='processing' AND lease_until < now()"
|
|
).fetchall()
|
|
for item in rows:
|
|
attachment_id = item['id']
|
|
connection.execute(
|
|
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
|
|
"updated_at=now(), error_code=NULL WHERE id=%s",
|
|
(attachment_id,),
|
|
)
|
|
connection.execute(
|
|
"UPDATE attachment_jobs SET published_at=NULL, next_attempt_at=now(), "
|
|
"last_error='worker_lease_expired' WHERE attachment_id=%s",
|
|
(attachment_id,),
|
|
)
|
|
|
|
|
|
def publish_outbox():
|
|
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
|
row = connection.execute(
|
|
"SELECT job_id, attachment_id, pipeline_version, attempt "
|
|
"FROM attachment_jobs WHERE published_at IS NULL AND next_attempt_at <= now() "
|
|
"ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED"
|
|
).fetchone()
|
|
if row is None:
|
|
return False
|
|
payload = {
|
|
'schema_version': '1', 'job_id': str(row['job_id']),
|
|
'type': 'attachment.process.v1', 'attachment_id': str(row['attachment_id']),
|
|
'pipeline_version': str(row['pipeline_version']), 'attempt': str(row['attempt']),
|
|
'created_at': now().isoformat().replace('+00:00', 'Z'),
|
|
}
|
|
broker.xadd(STREAM, payload, maxlen=10000, approximate=True)
|
|
connection.execute(
|
|
'UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s', (row['job_id'],)
|
|
)
|
|
connection.commit()
|
|
return True
|
|
|
|
|
|
def white_background(image):
|
|
image = ImageOps.exif_transpose(image)
|
|
if image.mode in ('RGBA', 'LA') or 'transparency' in image.info:
|
|
background = Image.new('RGB', image.size, 'white')
|
|
alpha = image.convert('RGBA')
|
|
background.paste(alpha, mask=alpha.getchannel('A'))
|
|
return background
|
|
return image.convert('RGB')
|
|
|
|
|
|
def jpeg_bytes(image, size, crop=False):
|
|
image = image.copy()
|
|
if crop:
|
|
side = min(image.size)
|
|
left = (image.width - side) // 2
|
|
top = (image.height - side) // 2
|
|
image = image.crop((left, top, left + side, top + side))
|
|
if crop:
|
|
image = image.resize((size, size), Image.Resampling.LANCZOS)
|
|
else:
|
|
image.thumbnail((size, size), Image.Resampling.LANCZOS)
|
|
output = io.BytesIO()
|
|
image.save(output, format='JPEG', quality=85, optimize=True)
|
|
return output.getvalue(), image.size
|
|
|
|
|
|
def webp_bytes(image):
|
|
image = image.copy()
|
|
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
|
output = io.BytesIO()
|
|
image.save(output, format='WEBP', quality=82, method=6)
|
|
return output.getvalue(), image.size
|
|
|
|
|
|
def get_attachment(attachment_id):
|
|
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
|
return connection.execute('SELECT * FROM attachments WHERE id=%s', (attachment_id,)).fetchone()
|
|
|
|
|
|
def claim(attachment_id, message_attempt):
|
|
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
|
with connection.transaction():
|
|
row = connection.execute(
|
|
'SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)
|
|
).fetchone()
|
|
if row is None or row['status'] in ('ready', 'failed'):
|
|
return row, False
|
|
if row['status'] == 'processing' and row['lease_until'] and row['lease_until'] > now():
|
|
return row, False
|
|
attempt = max(row['attempt'] + 1, message_attempt)
|
|
connection.execute(
|
|
"UPDATE attachments SET status='processing', attempt=%s, lease_until=%s, "
|
|
"updated_at=now(), error_code=NULL WHERE id=%s",
|
|
(attempt, now() + timedelta(seconds=PROCESSING_LEASE_SECONDS), attachment_id),
|
|
)
|
|
row['attempt'] = attempt
|
|
row['status'] = 'processing'
|
|
return row, True
|
|
|
|
|
|
def mark_retry(attachment_id, attempt, error_code, permanent=False):
|
|
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
|
with connection.transaction():
|
|
if permanent or attempt >= MAX_JOB_ATTEMPTS:
|
|
connection.execute(
|
|
"UPDATE attachments SET status='failed', lease_until=NULL, error_code=%s, updated_at=now() WHERE id=%s",
|
|
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
|
|
)
|
|
connection.execute(
|
|
"UPDATE attachment_jobs SET published_at=now(), last_error=%s WHERE attachment_id=%s",
|
|
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
|
|
)
|
|
else:
|
|
delay = min(30, 2 ** max(0, attempt - 1)) + random.uniform(0, 1)
|
|
connection.execute(
|
|
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
|
|
"error_code=%s, updated_at=now() WHERE id=%s",
|
|
(error_code, attachment_id),
|
|
)
|
|
connection.execute(
|
|
"UPDATE attachment_jobs SET attempt=%s, published_at=NULL, next_attempt_at=now()+%s * interval '1 second', last_error=%s "
|
|
"WHERE attachment_id=%s",
|
|
(attempt + 1, delay, error_code, attachment_id),
|
|
)
|
|
|
|
|
|
def process(attachment_id, message_attempt):
|
|
row, owned = claim(attachment_id, message_attempt)
|
|
if not owned:
|
|
return
|
|
try:
|
|
value = s3.get_object(Bucket=S3_BUCKET, Key=row['staging_key'])
|
|
original = value['Body'].read()
|
|
with Image.open(io.BytesIO(original)) as source:
|
|
if source.format not in ('PNG', 'JPEG') or source.width * source.height > MAX_IMAGE_PIXELS:
|
|
raise ValueError('invalid image')
|
|
image = white_background(source)
|
|
thumbnail, thumb_size = jpeg_bytes(image, 256)
|
|
crop, crop_size = jpeg_bytes(image, 128, crop=True)
|
|
optimized, optimized_size = webp_bytes(image)
|
|
|
|
base = 'attachments/' + str(attachment_id) + '/'
|
|
objects = [
|
|
('original', original, row['content_type'], image.size),
|
|
('thumbnail', thumbnail, 'image/jpeg', thumb_size),
|
|
('crop', crop, 'image/jpeg', crop_size),
|
|
('optimized', optimized, 'image/webp', optimized_size),
|
|
]
|
|
for name, data, content_type, _size in objects:
|
|
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp' if content_type == 'image/webp' else '')
|
|
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=data, ContentType=content_type, ContentLength=len(data))
|
|
|
|
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
|
with connection.transaction():
|
|
connection.execute(
|
|
"UPDATE attachments SET status='ready', lease_until=NULL, error_code=NULL, updated_at=now() WHERE id=%s",
|
|
(attachment_id,),
|
|
)
|
|
for name, data, content_type, dimensions in objects[1:]:
|
|
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp')
|
|
connection.execute(
|
|
'INSERT INTO attachment_variants '
|
|
'(attachment_id,name,content_type,object_key,size_bytes,width,height,sha256) '
|
|
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) '
|
|
'ON CONFLICT (attachment_id,name) DO UPDATE SET '
|
|
'content_type=EXCLUDED.content_type,object_key=EXCLUDED.object_key,size_bytes=EXCLUDED.size_bytes,'
|
|
'width=EXCLUDED.width,height=EXCLUDED.height,sha256=EXCLUDED.sha256',
|
|
(attachment_id, name, content_type, key, len(data), dimensions[0], dimensions[1], hashlib.sha256(data).hexdigest()),
|
|
)
|
|
try:
|
|
s3.delete_object(Bucket=S3_BUCKET, Key=row['staging_key'])
|
|
except Exception:
|
|
pass
|
|
if ACK_DELAY_SECONDS:
|
|
time.sleep(ACK_DELAY_SECONDS)
|
|
except (UnidentifiedImageError, ValueError, OSError):
|
|
mark_retry(attachment_id, row['attempt'], 'invalid_image', permanent=True)
|
|
except Exception:
|
|
mark_retry(attachment_id, row['attempt'], 'processing_failed')
|
|
|
|
|
|
def handle_message(message_id, fields):
|
|
process(uuid.UUID(fields['attachment_id']), int(fields.get('attempt', '1')))
|
|
broker.xack(STREAM, GROUP, message_id)
|
|
|
|
|
|
def reclaim_pending():
|
|
try:
|
|
result = broker.xautoclaim(STREAM, GROUP, CONSUMER, min_idle_time=1000, start_id='0-0', count=10)
|
|
for message_id, fields in result[1]:
|
|
handle_message(message_id, fields)
|
|
except (redis.ResponseError, AttributeError):
|
|
pass
|
|
|
|
|
|
def main():
|
|
ensure_group()
|
|
print('worker started', CONSUMER, flush=True)
|
|
while True:
|
|
try:
|
|
recover_expired_leases()
|
|
publish_outbox()
|
|
reclaim_pending()
|
|
messages = broker.xreadgroup(GROUP, CONSUMER, {STREAM: '>'}, count=1, block=1000)
|
|
for _stream, items in messages:
|
|
for message_id, fields in items:
|
|
handle_message(message_id, fields)
|
|
except Exception as exc:
|
|
print('worker loop error:', type(exc).__name__, flush=True)
|
|
time.sleep(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|