Реализовать асинхронную обработку изображений
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
"""Redis Streams worker for durable attachment processing."""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import boto3
|
||||
import psycopg
|
||||
import redis
|
||||
from botocore.config import Config
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
|
||||
BROKER_URL = os.getenv('BROKER_URL', 'redis://broker:6379/0')
|
||||
S3_ENDPOINT = os.getenv('S3_ENDPOINT', 'http://minio:9000')
|
||||
S3_BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID', 'messenger-api')
|
||||
S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123')
|
||||
MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
|
||||
MAX_JOB_ATTEMPTS = int(os.getenv('MAX_JOB_ATTEMPTS', '5'))
|
||||
PROCESSING_LEASE_SECONDS = int(os.getenv('PROCESSING_LEASE_SECONDS', '30'))
|
||||
ACK_DELAY_SECONDS = int(os.getenv('ACK_DELAY_SECONDS', '0'))
|
||||
STREAM = 'attachment-jobs'
|
||||
GROUP = 'attachment-workers'
|
||||
CONSUMER = os.getenv('WORKER_ID', 'worker-' + uuid.uuid4().hex[:8])
|
||||
|
||||
broker = redis.Redis.from_url(
|
||||
BROKER_URL, decode_responses=True, socket_connect_timeout=2,
|
||||
socket_timeout=3, retry_on_timeout=False,
|
||||
)
|
||||
s3 = boto3.client(
|
||||
's3', endpoint_url=S3_ENDPOINT, region_name='us-east-1',
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
config=Config(connect_timeout=2, read_timeout=10, retries={'max_attempts': 1}),
|
||||
)
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def ensure_group():
|
||||
try:
|
||||
broker.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True)
|
||||
except redis.ResponseError as exc:
|
||||
if 'BUSYGROUP' not in str(exc):
|
||||
raise
|
||||
|
||||
|
||||
def recover_expired_leases():
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
||||
with connection.transaction():
|
||||
rows = connection.execute(
|
||||
"SELECT id FROM attachments WHERE status='processing' AND lease_until < now()"
|
||||
).fetchall()
|
||||
for item in rows:
|
||||
attachment_id = item['id']
|
||||
connection.execute(
|
||||
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
|
||||
"updated_at=now(), error_code=NULL WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE attachment_jobs SET published_at=NULL, next_attempt_at=now(), "
|
||||
"last_error='worker_lease_expired' WHERE attachment_id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
|
||||
|
||||
def publish_outbox():
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
||||
row = connection.execute(
|
||||
"SELECT job_id, attachment_id, pipeline_version, attempt "
|
||||
"FROM attachment_jobs WHERE published_at IS NULL AND next_attempt_at <= now() "
|
||||
"ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False
|
||||
payload = {
|
||||
'schema_version': '1', 'job_id': str(row['job_id']),
|
||||
'type': 'attachment.process.v1', 'attachment_id': str(row['attachment_id']),
|
||||
'pipeline_version': str(row['pipeline_version']), 'attempt': str(row['attempt']),
|
||||
'created_at': now().isoformat().replace('+00:00', 'Z'),
|
||||
}
|
||||
broker.xadd(STREAM, payload, maxlen=10000, approximate=True)
|
||||
connection.execute(
|
||||
'UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s', (row['job_id'],)
|
||||
)
|
||||
connection.commit()
|
||||
return True
|
||||
|
||||
|
||||
def white_background(image):
|
||||
image = ImageOps.exif_transpose(image)
|
||||
if image.mode in ('RGBA', 'LA') or 'transparency' in image.info:
|
||||
background = Image.new('RGB', image.size, 'white')
|
||||
alpha = image.convert('RGBA')
|
||||
background.paste(alpha, mask=alpha.getchannel('A'))
|
||||
return background
|
||||
return image.convert('RGB')
|
||||
|
||||
|
||||
def jpeg_bytes(image, size, crop=False):
|
||||
image = image.copy()
|
||||
if crop:
|
||||
side = min(image.size)
|
||||
left = (image.width - side) // 2
|
||||
top = (image.height - side) // 2
|
||||
image = image.crop((left, top, left + side, top + side))
|
||||
if crop:
|
||||
image = image.resize((size, size), Image.Resampling.LANCZOS)
|
||||
else:
|
||||
image.thumbnail((size, size), Image.Resampling.LANCZOS)
|
||||
output = io.BytesIO()
|
||||
image.save(output, format='JPEG', quality=85, optimize=True)
|
||||
return output.getvalue(), image.size
|
||||
|
||||
|
||||
def webp_bytes(image):
|
||||
image = image.copy()
|
||||
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
output = io.BytesIO()
|
||||
image.save(output, format='WEBP', quality=82, method=6)
|
||||
return output.getvalue(), image.size
|
||||
|
||||
|
||||
def get_attachment(attachment_id):
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
||||
return connection.execute('SELECT * FROM attachments WHERE id=%s', (attachment_id,)).fetchone()
|
||||
|
||||
|
||||
def claim(attachment_id, message_attempt):
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
||||
with connection.transaction():
|
||||
row = connection.execute(
|
||||
'SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)
|
||||
).fetchone()
|
||||
if row is None or row['status'] in ('ready', 'failed'):
|
||||
return row, False
|
||||
if row['status'] == 'processing' and row['lease_until'] and row['lease_until'] > now():
|
||||
return row, False
|
||||
attempt = max(row['attempt'] + 1, message_attempt)
|
||||
connection.execute(
|
||||
"UPDATE attachments SET status='processing', attempt=%s, lease_until=%s, "
|
||||
"updated_at=now(), error_code=NULL WHERE id=%s",
|
||||
(attempt, now() + timedelta(seconds=PROCESSING_LEASE_SECONDS), attachment_id),
|
||||
)
|
||||
row['attempt'] = attempt
|
||||
row['status'] = 'processing'
|
||||
return row, True
|
||||
|
||||
|
||||
def mark_retry(attachment_id, attempt, error_code, permanent=False):
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
||||
with connection.transaction():
|
||||
if permanent or attempt >= MAX_JOB_ATTEMPTS:
|
||||
connection.execute(
|
||||
"UPDATE attachments SET status='failed', lease_until=NULL, error_code=%s, updated_at=now() WHERE id=%s",
|
||||
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE attachment_jobs SET published_at=now(), last_error=%s WHERE attachment_id=%s",
|
||||
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
|
||||
)
|
||||
else:
|
||||
delay = min(30, 2 ** max(0, attempt - 1)) + random.uniform(0, 1)
|
||||
connection.execute(
|
||||
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
|
||||
"error_code=%s, updated_at=now() WHERE id=%s",
|
||||
(error_code, attachment_id),
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE attachment_jobs SET attempt=%s, published_at=NULL, next_attempt_at=now()+%s * interval '1 second', last_error=%s "
|
||||
"WHERE attachment_id=%s",
|
||||
(attempt + 1, delay, error_code, attachment_id),
|
||||
)
|
||||
|
||||
|
||||
def process(attachment_id, message_attempt):
|
||||
row, owned = claim(attachment_id, message_attempt)
|
||||
if not owned:
|
||||
return
|
||||
try:
|
||||
value = s3.get_object(Bucket=S3_BUCKET, Key=row['staging_key'])
|
||||
original = value['Body'].read()
|
||||
with Image.open(io.BytesIO(original)) as source:
|
||||
if source.format not in ('PNG', 'JPEG') or source.width * source.height > MAX_IMAGE_PIXELS:
|
||||
raise ValueError('invalid image')
|
||||
image = white_background(source)
|
||||
thumbnail, thumb_size = jpeg_bytes(image, 256)
|
||||
crop, crop_size = jpeg_bytes(image, 128, crop=True)
|
||||
optimized, optimized_size = webp_bytes(image)
|
||||
|
||||
base = 'attachments/' + str(attachment_id) + '/'
|
||||
objects = [
|
||||
('original', original, row['content_type'], image.size),
|
||||
('thumbnail', thumbnail, 'image/jpeg', thumb_size),
|
||||
('crop', crop, 'image/jpeg', crop_size),
|
||||
('optimized', optimized, 'image/webp', optimized_size),
|
||||
]
|
||||
for name, data, content_type, _size in objects:
|
||||
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp' if content_type == 'image/webp' else '')
|
||||
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=data, ContentType=content_type, ContentLength=len(data))
|
||||
|
||||
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
|
||||
with connection.transaction():
|
||||
connection.execute(
|
||||
"UPDATE attachments SET status='ready', lease_until=NULL, error_code=NULL, updated_at=now() WHERE id=%s",
|
||||
(attachment_id,),
|
||||
)
|
||||
for name, data, content_type, dimensions in objects[1:]:
|
||||
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp')
|
||||
connection.execute(
|
||||
'INSERT INTO attachment_variants '
|
||||
'(attachment_id,name,content_type,object_key,size_bytes,width,height,sha256) '
|
||||
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) '
|
||||
'ON CONFLICT (attachment_id,name) DO UPDATE SET '
|
||||
'content_type=EXCLUDED.content_type,object_key=EXCLUDED.object_key,size_bytes=EXCLUDED.size_bytes,'
|
||||
'width=EXCLUDED.width,height=EXCLUDED.height,sha256=EXCLUDED.sha256',
|
||||
(attachment_id, name, content_type, key, len(data), dimensions[0], dimensions[1], hashlib.sha256(data).hexdigest()),
|
||||
)
|
||||
try:
|
||||
s3.delete_object(Bucket=S3_BUCKET, Key=row['staging_key'])
|
||||
except Exception:
|
||||
pass
|
||||
if ACK_DELAY_SECONDS:
|
||||
time.sleep(ACK_DELAY_SECONDS)
|
||||
except (UnidentifiedImageError, ValueError, OSError):
|
||||
mark_retry(attachment_id, row['attempt'], 'invalid_image', permanent=True)
|
||||
except Exception:
|
||||
mark_retry(attachment_id, row['attempt'], 'processing_failed')
|
||||
|
||||
|
||||
def handle_message(message_id, fields):
|
||||
process(uuid.UUID(fields['attachment_id']), int(fields.get('attempt', '1')))
|
||||
broker.xack(STREAM, GROUP, message_id)
|
||||
|
||||
|
||||
def reclaim_pending():
|
||||
try:
|
||||
result = broker.xautoclaim(STREAM, GROUP, CONSUMER, min_idle_time=1000, start_id='0-0', count=10)
|
||||
for message_id, fields in result[1]:
|
||||
handle_message(message_id, fields)
|
||||
except (redis.ResponseError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
ensure_group()
|
||||
print('worker started', CONSUMER, flush=True)
|
||||
while True:
|
||||
try:
|
||||
recover_expired_leases()
|
||||
publish_outbox()
|
||||
reclaim_pending()
|
||||
messages = broker.xreadgroup(GROUP, CONSUMER, {STREAM: '>'}, count=1, block=1000)
|
||||
for _stream, items in messages:
|
||||
for message_id, fields in items:
|
||||
handle_message(message_id, fields)
|
||||
except Exception as exc:
|
||||
print('worker loop error:', type(exc).__name__, flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user