Реализовать асинхронную обработку изображений

This commit is contained in:
Nik
2026-11-05 20:34:10 +03:00
parent 276dbdad8a
commit 35371114cd
28 changed files with 1968 additions and 45 deletions
+10
View File
@@ -0,0 +1,10 @@
import os
import psycopg
from pathlib import Path
url = os.getenv("DATABASE_URL", "postgresql://messenger:messenger@postgres:5432/messenger")
with psycopg.connect(url) as conn:
for path in sorted(Path("migrations").glob("*.sql")):
conn.execute(path.read_text())
conn.commit()
print("migrations applied")
+44
View File
@@ -0,0 +1,44 @@
"""Explicit operator replay for one failed attachment job."""
import argparse
import json
import os
import uuid
import psycopg
def main():
parser = argparse.ArgumentParser()
parser.add_argument('attachment_id', type=uuid.UUID)
args = parser.parse_args()
database_url = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
with psycopg.connect(database_url) as connection:
with connection.transaction():
row = connection.execute(
"SELECT a.status, j.job_id FROM attachments a "
"JOIN attachment_jobs j ON j.attachment_id=a.id "
"WHERE a.id=%s FOR UPDATE",
(args.attachment_id,),
).fetchone()
if row is None:
raise SystemExit('attachment not found')
if row[0] != 'failed':
raise SystemExit('attachment is not failed')
replay_id = uuid.uuid4()
connection.execute(
'INSERT INTO attachment_replays (replay_id,job_id) VALUES (%s,%s)',
(replay_id, row[1]),
)
connection.execute(
"UPDATE attachments SET status='queued', attempt=0, error_code=NULL, lease_until=NULL, updated_at=now() WHERE id=%s",
(args.attachment_id,),
)
connection.execute(
"UPDATE attachment_jobs SET attempt=1, published_at=NULL, next_attempt_at=now(), last_error=NULL WHERE job_id=%s",
(row[1],),
)
print(json.dumps({'attachment_id': str(args.attachment_id), 'job_id': str(row[1]), 'replay_id': str(replay_id)}))
if __name__ == '__main__':
main()
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
# Перезапускаем стенд, не удаляя volumes.
# Балансировщик запускаем после API, чтобы DNS имён реплик уже был доступен.
docker compose up -d minio postgres redis broker minio-init migrate api-1 api-2 api-3 api-4 worker
docker compose restart minio postgres redis broker api-1 api-2 api-3 api-4 worker
docker compose up -d --force-recreate lb
docker compose ps
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
TARGET_INSTANCE_ID=${TARGET_INSTANCE_ID:-api-1}
case "$TARGET_INSTANCE_ID" in
api-1) service=api-1 ;;
api-2) service=api-2 ;;
api-3) service=api-3 ;;
api-4) service=api-4 ;;
*) echo "unknown TARGET_INSTANCE_ID: $TARGET_INSTANCE_ID" >&2; exit 2 ;;
esac
docker compose stop "$service"
+270
View File
@@ -0,0 +1,270 @@
"""Redis Streams worker for durable attachment processing."""
import hashlib
import io
import os
import random
import time
import uuid
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
import redis
from botocore.config import Config
from PIL import Image, ImageOps, UnidentifiedImageError
from psycopg.rows import dict_row
DATABASE_URL = os.getenv('DATABASE_URL', 'postgresql://messenger:messenger@postgres:5432/messenger')
BROKER_URL = os.getenv('BROKER_URL', 'redis://broker:6379/0')
S3_ENDPOINT = os.getenv('S3_ENDPOINT', 'http://minio:9000')
S3_BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID', 'messenger-api')
S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY', 'messenger-api-secret-123')
MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
MAX_JOB_ATTEMPTS = int(os.getenv('MAX_JOB_ATTEMPTS', '5'))
PROCESSING_LEASE_SECONDS = int(os.getenv('PROCESSING_LEASE_SECONDS', '30'))
ACK_DELAY_SECONDS = int(os.getenv('ACK_DELAY_SECONDS', '0'))
STREAM = 'attachment-jobs'
GROUP = 'attachment-workers'
CONSUMER = os.getenv('WORKER_ID', 'worker-' + uuid.uuid4().hex[:8])
broker = redis.Redis.from_url(
BROKER_URL, decode_responses=True, socket_connect_timeout=2,
socket_timeout=3, retry_on_timeout=False,
)
s3 = boto3.client(
's3', endpoint_url=S3_ENDPOINT, region_name='us-east-1',
aws_access_key_id=S3_ACCESS_KEY_ID, aws_secret_access_key=S3_SECRET_ACCESS_KEY,
config=Config(connect_timeout=2, read_timeout=10, retries={'max_attempts': 1}),
)
def now():
return datetime.now(timezone.utc)
def ensure_group():
try:
broker.xgroup_create(STREAM, GROUP, id='0-0', mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):
raise
def recover_expired_leases():
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
rows = connection.execute(
"SELECT id FROM attachments WHERE status='processing' AND lease_until < now()"
).fetchall()
for item in rows:
attachment_id = item['id']
connection.execute(
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
"updated_at=now(), error_code=NULL WHERE id=%s",
(attachment_id,),
)
connection.execute(
"UPDATE attachment_jobs SET published_at=NULL, next_attempt_at=now(), "
"last_error='worker_lease_expired' WHERE attachment_id=%s",
(attachment_id,),
)
def publish_outbox():
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
row = connection.execute(
"SELECT job_id, attachment_id, pipeline_version, attempt "
"FROM attachment_jobs WHERE published_at IS NULL AND next_attempt_at <= now() "
"ORDER BY created_at LIMIT 1 FOR UPDATE SKIP LOCKED"
).fetchone()
if row is None:
return False
payload = {
'schema_version': '1', 'job_id': str(row['job_id']),
'type': 'attachment.process.v1', 'attachment_id': str(row['attachment_id']),
'pipeline_version': str(row['pipeline_version']), 'attempt': str(row['attempt']),
'created_at': now().isoformat().replace('+00:00', 'Z'),
}
broker.xadd(STREAM, payload, maxlen=10000, approximate=True)
connection.execute(
'UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s', (row['job_id'],)
)
connection.commit()
return True
def white_background(image):
image = ImageOps.exif_transpose(image)
if image.mode in ('RGBA', 'LA') or 'transparency' in image.info:
background = Image.new('RGB', image.size, 'white')
alpha = image.convert('RGBA')
background.paste(alpha, mask=alpha.getchannel('A'))
return background
return image.convert('RGB')
def jpeg_bytes(image, size, crop=False):
image = image.copy()
if crop:
side = min(image.size)
left = (image.width - side) // 2
top = (image.height - side) // 2
image = image.crop((left, top, left + side, top + side))
if crop:
image = image.resize((size, size), Image.Resampling.LANCZOS)
else:
image.thumbnail((size, size), Image.Resampling.LANCZOS)
output = io.BytesIO()
image.save(output, format='JPEG', quality=85, optimize=True)
return output.getvalue(), image.size
def webp_bytes(image):
image = image.copy()
image.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
output = io.BytesIO()
image.save(output, format='WEBP', quality=82, method=6)
return output.getvalue(), image.size
def get_attachment(attachment_id):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
return connection.execute('SELECT * FROM attachments WHERE id=%s', (attachment_id,)).fetchone()
def claim(attachment_id, message_attempt):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
row = connection.execute(
'SELECT * FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)
).fetchone()
if row is None or row['status'] in ('ready', 'failed'):
return row, False
if row['status'] == 'processing' and row['lease_until'] and row['lease_until'] > now():
return row, False
attempt = max(row['attempt'] + 1, message_attempt)
connection.execute(
"UPDATE attachments SET status='processing', attempt=%s, lease_until=%s, "
"updated_at=now(), error_code=NULL WHERE id=%s",
(attempt, now() + timedelta(seconds=PROCESSING_LEASE_SECONDS), attachment_id),
)
row['attempt'] = attempt
row['status'] = 'processing'
return row, True
def mark_retry(attachment_id, attempt, error_code, permanent=False):
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
if permanent or attempt >= MAX_JOB_ATTEMPTS:
connection.execute(
"UPDATE attachments SET status='failed', lease_until=NULL, error_code=%s, updated_at=now() WHERE id=%s",
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
)
connection.execute(
"UPDATE attachment_jobs SET published_at=now(), last_error=%s WHERE attachment_id=%s",
('invalid_image' if permanent else 'retry_exhausted', attachment_id),
)
else:
delay = min(30, 2 ** max(0, attempt - 1)) + random.uniform(0, 1)
connection.execute(
"UPDATE attachments SET status='queued', lease_until=NULL, next_attempt_at=now(), "
"error_code=%s, updated_at=now() WHERE id=%s",
(error_code, attachment_id),
)
connection.execute(
"UPDATE attachment_jobs SET attempt=%s, published_at=NULL, next_attempt_at=now()+%s * interval '1 second', last_error=%s "
"WHERE attachment_id=%s",
(attempt + 1, delay, error_code, attachment_id),
)
def process(attachment_id, message_attempt):
row, owned = claim(attachment_id, message_attempt)
if not owned:
return
try:
value = s3.get_object(Bucket=S3_BUCKET, Key=row['staging_key'])
original = value['Body'].read()
with Image.open(io.BytesIO(original)) as source:
if source.format not in ('PNG', 'JPEG') or source.width * source.height > MAX_IMAGE_PIXELS:
raise ValueError('invalid image')
image = white_background(source)
thumbnail, thumb_size = jpeg_bytes(image, 256)
crop, crop_size = jpeg_bytes(image, 128, crop=True)
optimized, optimized_size = webp_bytes(image)
base = 'attachments/' + str(attachment_id) + '/'
objects = [
('original', original, row['content_type'], image.size),
('thumbnail', thumbnail, 'image/jpeg', thumb_size),
('crop', crop, 'image/jpeg', crop_size),
('optimized', optimized, 'image/webp', optimized_size),
]
for name, data, content_type, _size in objects:
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp' if content_type == 'image/webp' else '')
s3.put_object(Bucket=S3_BUCKET, Key=key, Body=data, ContentType=content_type, ContentLength=len(data))
with psycopg.connect(DATABASE_URL, row_factory=dict_row) as connection:
with connection.transaction():
connection.execute(
"UPDATE attachments SET status='ready', lease_until=NULL, error_code=NULL, updated_at=now() WHERE id=%s",
(attachment_id,),
)
for name, data, content_type, dimensions in objects[1:]:
key = base + name + ('.jpg' if content_type == 'image/jpeg' else '.webp')
connection.execute(
'INSERT INTO attachment_variants '
'(attachment_id,name,content_type,object_key,size_bytes,width,height,sha256) '
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) '
'ON CONFLICT (attachment_id,name) DO UPDATE SET '
'content_type=EXCLUDED.content_type,object_key=EXCLUDED.object_key,size_bytes=EXCLUDED.size_bytes,'
'width=EXCLUDED.width,height=EXCLUDED.height,sha256=EXCLUDED.sha256',
(attachment_id, name, content_type, key, len(data), dimensions[0], dimensions[1], hashlib.sha256(data).hexdigest()),
)
try:
s3.delete_object(Bucket=S3_BUCKET, Key=row['staging_key'])
except Exception:
pass
if ACK_DELAY_SECONDS:
time.sleep(ACK_DELAY_SECONDS)
except (UnidentifiedImageError, ValueError, OSError):
mark_retry(attachment_id, row['attempt'], 'invalid_image', permanent=True)
except Exception:
mark_retry(attachment_id, row['attempt'], 'processing_failed')
def handle_message(message_id, fields):
process(uuid.UUID(fields['attachment_id']), int(fields.get('attempt', '1')))
broker.xack(STREAM, GROUP, message_id)
def reclaim_pending():
try:
result = broker.xautoclaim(STREAM, GROUP, CONSUMER, min_idle_time=1000, start_id='0-0', count=10)
for message_id, fields in result[1]:
handle_message(message_id, fields)
except (redis.ResponseError, AttributeError):
pass
def main():
ensure_group()
print('worker started', CONSUMER, flush=True)
while True:
try:
recover_expired_leases()
publish_outbox()
reclaim_pending()
messages = broker.xreadgroup(GROUP, CONSUMER, {STREAM: '>'}, count=1, block=1000)
for _stream, items in messages:
for message_id, fields in items:
handle_message(message_id, fields)
except Exception as exc:
print('worker loop error:', type(exc).__name__, flush=True)
time.sleep(1)
if __name__ == '__main__':
main()