From 910252c277a88ff0f4cc0f4601cdcb7e81762d87 Mon Sep 17 00:00:00 2001 From: casablanca Date: Thu, 12 Nov 2026 00:28:00 +0300 Subject: [PATCH] =?UTF-8?q?=D0=BB=D0=B0=D0=B1=D0=B0=20=D0=BD=D0=B0=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../versions/a6b12026a002_job_retries.py | 42 ++++ .../a7b12026a003_failed_publication.py | 33 +++ .../versions/b8b12026a004_job_generation.py | 24 ++ scripts/replay_job.py | 82 +++++++ src/dispatcher.py | 74 +++++- src/main.py | 21 +- src/models.py | 1 + src/queue.py | 17 ++ src/worker.py | 222 ++++++++++++++++-- 9 files changed, 474 insertions(+), 42 deletions(-) create mode 100644 migrations/versions/a6b12026a002_job_retries.py create mode 100644 migrations/versions/a7b12026a003_failed_publication.py create mode 100644 migrations/versions/b8b12026a004_job_generation.py create mode 100644 scripts/replay_job.py diff --git a/migrations/versions/a6b12026a002_job_retries.py b/migrations/versions/a6b12026a002_job_retries.py new file mode 100644 index 00000000..3fde2807 --- /dev/null +++ b/migrations/versions/a6b12026a002_job_retries.py @@ -0,0 +1,42 @@ +from alembic import op +import sqlalchemy as sa + + +revision = "a6b12026a002" +down_revision = "e5a12026a001" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "attachment_jobs", + sa.Column("next_attempt_at", sa.DateTime(timezone=True)), + ) + + op.add_column( + "attachment_jobs", + sa.Column("last_error_code", sa.String(64)), + ) + + op.add_column( + "attachment_jobs", + sa.Column("finished_at", sa.DateTime(timezone=True)), + ) + + op.create_index( + "ix_attachment_jobs_retry_schedule", + "attachment_jobs", + ["status", "next_attempt_at"], + ) + + +def downgrade(): + op.drop_index( + "ix_attachment_jobs_retry_schedule", + table_name="attachment_jobs", + ) + + op.drop_column("attachment_jobs", "finished_at") + op.drop_column("attachment_jobs", "last_error_code") + op.drop_column("attachment_jobs", "next_attempt_at") \ No newline at end of file diff --git a/migrations/versions/a7b12026a003_failed_publication.py b/migrations/versions/a7b12026a003_failed_publication.py new file mode 100644 index 00000000..b82e944e --- /dev/null +++ b/migrations/versions/a7b12026a003_failed_publication.py @@ -0,0 +1,33 @@ +from alembic import op +import sqlalchemy as sa + + +revision = "a7b12026a003" +down_revision = "a6b12026a002" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "attachment_jobs", + sa.Column("failed_published_at", sa.DateTime(timezone=True)), + ) + + op.create_index( + "ix_attachment_jobs_failed_unpublished", + "attachment_jobs", + ["created_at"], + postgresql_where=sa.text( + "status = 'failed' AND failed_published_at IS NULL" + ), + ) + + +def downgrade(): + op.drop_index( + "ix_attachment_jobs_failed_unpublished", + table_name="attachment_jobs", + ) + + op.drop_column("attachment_jobs", "failed_published_at") \ No newline at end of file diff --git a/migrations/versions/b8b12026a004_job_generation.py b/migrations/versions/b8b12026a004_job_generation.py new file mode 100644 index 00000000..d9666d30 --- /dev/null +++ b/migrations/versions/b8b12026a004_job_generation.py @@ -0,0 +1,24 @@ +from alembic import op +import sqlalchemy as sa + + +revision = "b8b12026a004" +down_revision = "a7b12026a003" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "attachment_jobs", + sa.Column( + "generation", + sa.Integer(), + nullable=False, + server_default="1", + ), + ) + + +def downgrade(): + op.drop_column("attachment_jobs", "generation") \ No newline at end of file diff --git a/scripts/replay_job.py b/scripts/replay_job.py new file mode 100644 index 00000000..e24b2066 --- /dev/null +++ b/scripts/replay_job.py @@ -0,0 +1,82 @@ + +import argparse +import asyncio +from uuid import UUID + +from sqlalchemy import text + +from src.database import SessionFactory, engine + + +async def replay(job_id: UUID): + async with SessionFactory() as session: + async with session.begin(): + result = await session.execute( + text(""" + SELECT j.id, j.attachment_id, j.status, + j.generation, a.status AS attachment_status + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE j.id = :job_id + FOR UPDATE OF j, a + """), + {"job_id": job_id}, + ) + + row = result.one_or_none() + + if row is None: + print("REJECTED: job not found") + return + + if row.status != "failed" or row.attachment_status != "failed": + print("REJECTED: job and attachment must be failed") + return + + await session.execute( + text(""" + UPDATE attachment_jobs + SET status = 'queued', + generation = generation + 1, + attempt = 1, + published_at = NULL, + started_at = NULL, + next_attempt_at = NULL, + last_error_code = NULL, + finished_at = NULL, + failed_published_at = NULL + WHERE id = :job_id + """), + {"job_id": job_id}, + ) + + await session.execute( + text(""" + UPDATE attachments + SET status = 'queued', + error_code = NULL, + updated_at = now() + WHERE id = :attachment_id + """), + {"attachment_id": row.attachment_id}, + ) + + print( + f"REPLAY ACCEPTED: job={job_id}, " + f"generation={row.generation + 1}" + ) + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("job_id", type=UUID) + args = parser.parse_args() + + try: + await replay(args.job_id) + finally: + await engine.dispose() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/dispatcher.py b/src/dispatcher.py index 6a581cad..744ab069 100644 --- a/src/dispatcher.py +++ b/src/dispatcher.py @@ -4,7 +4,7 @@ import logging from sqlalchemy import text from src.database import SessionFactory, engine -from src.queue import connect, publish +from src.queue import connect, publish, publish_failed logging.basicConfig(level=logging.INFO) @@ -39,14 +39,18 @@ async def dispatch(channel): result = await session.execute( text(""" - SELECT id, attachment_id, attempt, created_at - FROM attachment_jobs - WHERE published_at IS NULL - AND status = 'queued' - ORDER BY created_at - LIMIT 25 - FOR UPDATE SKIP LOCKED - """) + SELECT id, attachment_id, attempt, generation, created_at + FROM attachment_jobs + WHERE published_at IS NULL + AND status = 'queued' + AND ( + next_attempt_at IS NULL + OR next_attempt_at <= now() + ) + ORDER BY created_at LIMIT 25 + FOR + UPDATE SKIP LOCKED + """) ) rows = result.all() @@ -66,6 +70,55 @@ async def dispatch(channel): return len(rows) +async def dispatch_failed(channel): + async with SessionFactory() as session: + async with session.begin(): + result = await session.execute( + text(""" + SELECT id, attachment_id, attempt, generation, + last_error_code, created_at, finished_at + FROM attachment_jobs + WHERE status = 'failed' + AND failed_published_at IS NULL + ORDER BY created_at + LIMIT 25 + FOR UPDATE SKIP LOCKED + """) + ) + + rows = result.all() + + for row in rows: + payload = { + "schema_version": 1, + "type": "attachment.failed.v1", + "job_id": str(row.id), + "attachment_id": str(row.attachment_id), + "attempt": row.attempt, + "generation": row.generation, + "error_code": row.last_error_code, + "created_at": row.created_at.isoformat(), + "finished_at": ( + row.finished_at.isoformat() + if row.finished_at + else None + ), + } + + publish_failed(channel, payload) + + await session.execute( + text(""" + UPDATE attachment_jobs + SET failed_published_at = now() + WHERE id = :id + """), + {"id": row.id}, + ) + + return len(rows) + + async def main(): while True: connection = None @@ -76,7 +129,8 @@ async def main(): while True: count = await dispatch(channel) - await asyncio.sleep(0.2 if count else 1) + failed_count = await dispatch_failed(channel) + await asyncio.sleep(0.2 if count or failed_count else 1) except Exception: logging.exception("Dispatcher reconnect") diff --git a/src/main.py b/src/main.py index aee19592..1537c003 100644 --- a/src/main.py +++ b/src/main.py @@ -724,9 +724,12 @@ async def get_attachment_content_endpoint( message="Attachment is not ready", ) - if variant == "thumbnail": + if variant == "original": + media_type = attachment.content_type + + elif variant in {"thumbnail", "crop", "optimized"}: selected = next( - (item for item in attachment.variants if item.name == "thumbnail"), + (item for item in attachment.variants if item.name == variant), None, ) @@ -737,14 +740,18 @@ async def get_attachment_content_endpoint( message="Variant not found", ) - s3_key = ( - f"variants/{attachment.chat_id}/{attachment_id}/thumbnail.jpg" + extension = { + "thumbnail": "jpg", + "crop": "jpg", + "optimized": "webp", + }[variant] + + s3_key = selected.s3_key or ( + f"variants/{attachment.chat_id}/{attachment_id}/" + f"{variant}.{extension}" ) media_type = selected.content_type - elif variant == "original": - media_type = attachment.content_type - else: raise APIError( status_code=404, diff --git a/src/models.py b/src/models.py index f24e5ecb..d52102bf 100644 --- a/src/models.py +++ b/src/models.py @@ -133,6 +133,7 @@ class Variant(BaseModel): width: int = Field(ge=1) height: int = Field(ge=1) sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + s3_key: str | None = Field(default=None, exclude=True) class Attachment(BaseModel): diff --git a/src/queue.py b/src/queue.py index 65d5d775..1b9fe326 100644 --- a/src/queue.py +++ b/src/queue.py @@ -6,6 +6,7 @@ from pathlib import Path import pika QUEUE = "attachment.process.v1" +DLQ = "attachment.failed.v1" def connect(): @@ -31,6 +32,7 @@ def connect(): channel = connection.channel() channel.queue_declare(queue=QUEUE, durable=True) + channel.queue_declare(queue=DLQ, durable=True) return connection, channel @@ -43,6 +45,7 @@ def publish(channel, row): "attachment_id": str(row.attachment_id), "pipeline_version": 1, "attempt": row.attempt, + "generation": row.generation, "created_at": row.created_at.astimezone( timezone.utc ).isoformat().replace("+00:00", "Z"), @@ -58,4 +61,18 @@ def publish(channel, row): message_id=str(row.id), ), mandatory=True, + ) + + +def publish_failed(channel, payload: dict): + channel.basic_publish( + exchange="", + routing_key=DLQ, + body=json.dumps(payload).encode(), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + message_id=payload["job_id"], + ), + mandatory=True, ) \ No newline at end of file diff --git a/src/worker.py b/src/worker.py index 81190b46..63910c2d 100644 --- a/src/worker.py +++ b/src/worker.py @@ -1,20 +1,54 @@ import asyncio import hashlib import io +import os import json import logging import time +import random from uuid import UUID -from PIL import Image, ImageOps +from PIL import Image, ImageOps, UnidentifiedImageError from sqlalchemy import text from src.database import SessionFactory from src.queue import QUEUE, connect from src.storage import get_object, put_object +from src.images import MAX_IMAGE_PIXELS, MAX_UPLOAD_BYTES logging.basicConfig(level=logging.INFO) +MAX_PROCESSING_SECONDS = float( + os.getenv("MAX_PROCESSING_SECONDS", "30") +) + + +class PermanentProcessingError(Exception): + pass + + +def retry_delay(attempt: int) -> float: + base = min(60.0, 2.0 ** attempt) + return base * random.uniform(0.75, 1.25) + + +def validate_worker_image(body: bytes): + if not body or len(body) > MAX_UPLOAD_BYTES: + raise PermanentProcessingError("invalid_image_size") + + try: + with Image.open(io.BytesIO(body)) as image: + if image.format not in {"PNG", "JPEG"}: + raise PermanentProcessingError("unsupported_image_format") + + if image.width * image.height > MAX_IMAGE_PIXELS: + raise PermanentProcessingError("image_pixel_limit") + + image.verify() + + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as exc: + raise PermanentProcessingError("invalid_image") from exc + def make_thumbnail(body: bytes): with Image.open(io.BytesIO(body)) as source: @@ -34,15 +68,44 @@ def make_thumbnail(body: bytes): return output.getvalue(), image.width, image.height +def make_crop(body: bytes): + with Image.open(io.BytesIO(body)) as source: + source.load() + image = ImageOps.exif_transpose(source).convert("RGB") + image = ImageOps.fit( + image, + (128, 128), + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5), + ) + + output = io.BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + + return output.getvalue(), image.width, image.height + + +def make_optimized(body: bytes): + with Image.open(io.BytesIO(body)) as source: + source.load() + image = ImageOps.exif_transpose(source).convert("RGB") + + output = io.BytesIO() + image.save(output, format="WEBP", quality=80, method=4) + + return output.getvalue(), image.width, image.height + + async def process(payload: dict): job_id = UUID(payload["job_id"]) attachment_id = UUID(payload["attachment_id"]) + generation = payload.get("generation", 1) async with SessionFactory() as session: async with session.begin(): result = await session.execute( text(""" - SELECT j.status, j.attempt, + SELECT j.status, j.attempt, j.generation, a.s3_key, a.content_type, a.chat_id FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id @@ -61,6 +124,9 @@ async def process(payload: dict): if row.attempt != payload["attempt"]: return + if row.generation != generation: + return + if row.status == "processing": return @@ -84,22 +150,41 @@ async def process(payload: dict): try: body = await get_object(row.s3_key) + validate_worker_image(body) thumbnail, width, height = await asyncio.to_thread( make_thumbnail, body ) + crop, crop_width, crop_height = await asyncio.to_thread( + make_crop, body + ) + + optimized, optimized_width, optimized_height = await asyncio.to_thread( + make_optimized, body + ) + original_key = ( - f"attachments/{row.chat_id}/{attachment_id}/original" + f"attachments/{row.chat_id}/{attachment_id}/g{generation}/original" ) thumbnail_key = ( - f"variants/{row.chat_id}/{attachment_id}/thumbnail.jpg" + f"variants/{row.chat_id}/{attachment_id}/g{generation}/thumbnail.jpg" ) await put_object(original_key, body, row.content_type) await put_object(thumbnail_key, thumbnail, "image/jpeg") + crop_key = ( + f"variants/{row.chat_id}/{attachment_id}/g{generation}/crop.jpg" + ) + optimized_key = ( + f"variants/{row.chat_id}/{attachment_id}/g{generation}/optimized.webp" + ) + + await put_object(crop_key, crop, "image/jpeg") + await put_object(optimized_key, optimized, "image/webp") variant = { "name": "thumbnail", + "s3_key": thumbnail_key, "content_type": "image/jpeg", "size_bytes": len(thumbnail), "width": width, @@ -109,6 +194,25 @@ async def process(payload: dict): async with SessionFactory() as session: async with session.begin(): + result = await session.execute( + text(""" + SELECT status, attempt, generation + FROM attachment_jobs + WHERE id = :id + FOR UPDATE + """), + {"id": job_id}, + ) + + current = result.one_or_none() + + if ( + current is None + or current.status != "processing" + or current.attempt != row.attempt + or current.generation != generation + ): + return await session.execute( text(""" UPDATE attachments @@ -122,43 +226,105 @@ async def process(payload: dict): { "id": attachment_id, "key": original_key, - "variants": json.dumps([variant]), + "variants": json.dumps([ + variant, + { + "name": "crop", + "s3_key": crop_key, + "content_type": "image/jpeg", + "size_bytes": len(crop), + "width": crop_width, + "height": crop_height, + "sha256": hashlib.sha256(crop).hexdigest(), + }, + { + "name": "optimized", + "s3_key": optimized_key, + "content_type": "image/webp", + "size_bytes": len(optimized), + "width": optimized_width, + "height": optimized_height, + "sha256": hashlib.sha256(optimized).hexdigest(), + }, + ]), }, ) await session.execute( text(""" - UPDATE attachment_jobs - SET status = 'ready' - WHERE id = :id - """), + UPDATE attachment_jobs + SET status = 'ready', + next_attempt_at = NULL, + last_error_code = NULL, + finished_at = now() + WHERE id = :id + """), {"id": job_id}, ) - except Exception: + + except Exception as exc: + logging.exception("Processing failed: %s", attachment_id) + permanent = isinstance(exc, PermanentProcessingError) + + error_code = ( + + str(exc)[:64] + + if permanent + + else "temporary_processing_error" + + ) + async with SessionFactory() as session: async with session.begin(): - if row.attempt >= 5: + result = await session.execute( + text(""" + SELECT status, attempt, generation + FROM attachment_jobs + WHERE id = :id + FOR UPDATE + """), + {"id": job_id}, + ) + + current = result.one_or_none() + + if ( + current is None + or current.status != "processing" + or current.attempt != row.attempt + or current.generation != generation + ): + return + if permanent or row.attempt >= 5: await session.execute( text(""" - UPDATE attachment_jobs - SET status = 'failed' - WHERE id = :id - """), - {"id": job_id}, + UPDATE attachment_jobs + SET status = 'failed', + next_attempt_at = NULL, + last_error_code = :error_code, + finished_at = now() + WHERE id = :id + """), + {"id": job_id, "error_code": error_code}, ) await session.execute( text(""" - UPDATE attachments - SET status = 'failed', - error_code = 'retry_exhausted', - updated_at = now() - WHERE id = :id - """), - {"id": attachment_id}, + UPDATE attachments + SET status = 'failed', + error_code = :error_code, + updated_at = now() + WHERE id = :id + """), + { + "id": attachment_id, + "error_code": error_code, + }, ) else: await session.execute( @@ -167,10 +333,16 @@ async def process(payload: dict): SET status = 'queued', attempt = attempt + 1, published_at = NULL, - started_at = NULL + started_at = NULL, + next_attempt_at = now() + (:delay * interval '1 second'), + last_error_code = :error_code WHERE id = :id """), - {"id": job_id}, + { + "id": job_id, + "delay": retry_delay(row.attempt), + "error_code": error_code, + } ) await session.execute(