лаба на 4
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "a6b12026a002"
|
||||
down_revision = "e5a12026a001"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column("last_error_code", sa.String(64)),
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
|
||||
op.create_index(
|
||||
"ix_attachment_jobs_retry_schedule",
|
||||
"attachment_jobs",
|
||||
["status", "next_attempt_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index(
|
||||
"ix_attachment_jobs_retry_schedule",
|
||||
table_name="attachment_jobs",
|
||||
)
|
||||
|
||||
op.drop_column("attachment_jobs", "finished_at")
|
||||
op.drop_column("attachment_jobs", "last_error_code")
|
||||
op.drop_column("attachment_jobs", "next_attempt_at")
|
||||
@@ -0,0 +1,33 @@
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "a7b12026a003"
|
||||
down_revision = "a6b12026a002"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column("failed_published_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
|
||||
op.create_index(
|
||||
"ix_attachment_jobs_failed_unpublished",
|
||||
"attachment_jobs",
|
||||
["created_at"],
|
||||
postgresql_where=sa.text(
|
||||
"status = 'failed' AND failed_published_at IS NULL"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index(
|
||||
"ix_attachment_jobs_failed_unpublished",
|
||||
table_name="attachment_jobs",
|
||||
)
|
||||
|
||||
op.drop_column("attachment_jobs", "failed_published_at")
|
||||
@@ -0,0 +1,24 @@
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = "b8b12026a004"
|
||||
down_revision = "a7b12026a003"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column(
|
||||
"attachment_jobs",
|
||||
sa.Column(
|
||||
"generation",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default="1",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("attachment_jobs", "generation")
|
||||
@@ -0,0 +1,82 @@
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.database import SessionFactory, engine
|
||||
|
||||
|
||||
async def replay(job_id: UUID):
|
||||
async with SessionFactory() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT j.id, j.attachment_id, j.status,
|
||||
j.generation, a.status AS attachment_status
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.id = :job_id
|
||||
FOR UPDATE OF j, a
|
||||
"""),
|
||||
{"job_id": job_id},
|
||||
)
|
||||
|
||||
row = result.one_or_none()
|
||||
|
||||
if row is None:
|
||||
print("REJECTED: job not found")
|
||||
return
|
||||
|
||||
if row.status != "failed" or row.attachment_status != "failed":
|
||||
print("REJECTED: job and attachment must be failed")
|
||||
return
|
||||
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'queued',
|
||||
generation = generation + 1,
|
||||
attempt = 1,
|
||||
published_at = NULL,
|
||||
started_at = NULL,
|
||||
next_attempt_at = NULL,
|
||||
last_error_code = NULL,
|
||||
finished_at = NULL,
|
||||
failed_published_at = NULL
|
||||
WHERE id = :job_id
|
||||
"""),
|
||||
{"job_id": job_id},
|
||||
)
|
||||
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachments
|
||||
SET status = 'queued',
|
||||
error_code = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = :attachment_id
|
||||
"""),
|
||||
{"attachment_id": row.attachment_id},
|
||||
)
|
||||
|
||||
print(
|
||||
f"REPLAY ACCEPTED: job={job_id}, "
|
||||
f"generation={row.generation + 1}"
|
||||
)
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("job_id", type=UUID)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
await replay(args.job_id)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
+64
-10
@@ -4,7 +4,7 @@ import logging
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.database import SessionFactory, engine
|
||||
from src.queue import connect, publish
|
||||
from src.queue import connect, publish, publish_failed
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
@@ -39,14 +39,18 @@ async def dispatch(channel):
|
||||
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT id, attachment_id, attempt, created_at
|
||||
FROM attachment_jobs
|
||||
WHERE published_at IS NULL
|
||||
AND status = 'queued'
|
||||
ORDER BY created_at
|
||||
LIMIT 25
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""")
|
||||
SELECT id, attachment_id, attempt, generation, created_at
|
||||
FROM attachment_jobs
|
||||
WHERE published_at IS NULL
|
||||
AND status = 'queued'
|
||||
AND (
|
||||
next_attempt_at IS NULL
|
||||
OR next_attempt_at <= now()
|
||||
)
|
||||
ORDER BY created_at LIMIT 25
|
||||
FOR
|
||||
UPDATE SKIP LOCKED
|
||||
""")
|
||||
)
|
||||
|
||||
rows = result.all()
|
||||
@@ -66,6 +70,55 @@ async def dispatch(channel):
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def dispatch_failed(channel):
|
||||
async with SessionFactory() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT id, attachment_id, attempt, generation,
|
||||
last_error_code, created_at, finished_at
|
||||
FROM attachment_jobs
|
||||
WHERE status = 'failed'
|
||||
AND failed_published_at IS NULL
|
||||
ORDER BY created_at
|
||||
LIMIT 25
|
||||
FOR UPDATE SKIP LOCKED
|
||||
""")
|
||||
)
|
||||
|
||||
rows = result.all()
|
||||
|
||||
for row in rows:
|
||||
payload = {
|
||||
"schema_version": 1,
|
||||
"type": "attachment.failed.v1",
|
||||
"job_id": str(row.id),
|
||||
"attachment_id": str(row.attachment_id),
|
||||
"attempt": row.attempt,
|
||||
"generation": row.generation,
|
||||
"error_code": row.last_error_code,
|
||||
"created_at": row.created_at.isoformat(),
|
||||
"finished_at": (
|
||||
row.finished_at.isoformat()
|
||||
if row.finished_at
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
publish_failed(channel, payload)
|
||||
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachment_jobs
|
||||
SET failed_published_at = now()
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": row.id},
|
||||
)
|
||||
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def main():
|
||||
while True:
|
||||
connection = None
|
||||
@@ -76,7 +129,8 @@ async def main():
|
||||
|
||||
while True:
|
||||
count = await dispatch(channel)
|
||||
await asyncio.sleep(0.2 if count else 1)
|
||||
failed_count = await dispatch_failed(channel)
|
||||
await asyncio.sleep(0.2 if count or failed_count else 1)
|
||||
|
||||
except Exception:
|
||||
logging.exception("Dispatcher reconnect")
|
||||
|
||||
+14
-7
@@ -724,9 +724,12 @@ async def get_attachment_content_endpoint(
|
||||
message="Attachment is not ready",
|
||||
)
|
||||
|
||||
if variant == "thumbnail":
|
||||
if variant == "original":
|
||||
media_type = attachment.content_type
|
||||
|
||||
elif variant in {"thumbnail", "crop", "optimized"}:
|
||||
selected = next(
|
||||
(item for item in attachment.variants if item.name == "thumbnail"),
|
||||
(item for item in attachment.variants if item.name == variant),
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -737,14 +740,18 @@ async def get_attachment_content_endpoint(
|
||||
message="Variant not found",
|
||||
)
|
||||
|
||||
s3_key = (
|
||||
f"variants/{attachment.chat_id}/{attachment_id}/thumbnail.jpg"
|
||||
extension = {
|
||||
"thumbnail": "jpg",
|
||||
"crop": "jpg",
|
||||
"optimized": "webp",
|
||||
}[variant]
|
||||
|
||||
s3_key = selected.s3_key or (
|
||||
f"variants/{attachment.chat_id}/{attachment_id}/"
|
||||
f"{variant}.{extension}"
|
||||
)
|
||||
media_type = selected.content_type
|
||||
|
||||
elif variant == "original":
|
||||
media_type = attachment.content_type
|
||||
|
||||
else:
|
||||
raise APIError(
|
||||
status_code=404,
|
||||
|
||||
@@ -133,6 +133,7 @@ class Variant(BaseModel):
|
||||
width: int = Field(ge=1)
|
||||
height: int = Field(ge=1)
|
||||
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
s3_key: str | None = Field(default=None, exclude=True)
|
||||
|
||||
|
||||
class Attachment(BaseModel):
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
import pika
|
||||
|
||||
QUEUE = "attachment.process.v1"
|
||||
DLQ = "attachment.failed.v1"
|
||||
|
||||
|
||||
def connect():
|
||||
@@ -31,6 +32,7 @@ def connect():
|
||||
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.queue_declare(queue=DLQ, durable=True)
|
||||
|
||||
return connection, channel
|
||||
|
||||
@@ -43,6 +45,7 @@ def publish(channel, row):
|
||||
"attachment_id": str(row.attachment_id),
|
||||
"pipeline_version": 1,
|
||||
"attempt": row.attempt,
|
||||
"generation": row.generation,
|
||||
"created_at": row.created_at.astimezone(
|
||||
timezone.utc
|
||||
).isoformat().replace("+00:00", "Z"),
|
||||
@@ -58,4 +61,18 @@ def publish(channel, row):
|
||||
message_id=str(row.id),
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
|
||||
|
||||
def publish_failed(channel, payload: dict):
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=DLQ,
|
||||
body=json.dumps(payload).encode(),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
message_id=payload["job_id"],
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
+197
-25
@@ -1,20 +1,54 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import random
|
||||
from uuid import UUID
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.database import SessionFactory
|
||||
from src.queue import QUEUE, connect
|
||||
from src.storage import get_object, put_object
|
||||
from src.images import MAX_IMAGE_PIXELS, MAX_UPLOAD_BYTES
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
MAX_PROCESSING_SECONDS = float(
|
||||
os.getenv("MAX_PROCESSING_SECONDS", "30")
|
||||
)
|
||||
|
||||
|
||||
class PermanentProcessingError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def retry_delay(attempt: int) -> float:
|
||||
base = min(60.0, 2.0 ** attempt)
|
||||
return base * random.uniform(0.75, 1.25)
|
||||
|
||||
|
||||
def validate_worker_image(body: bytes):
|
||||
if not body or len(body) > MAX_UPLOAD_BYTES:
|
||||
raise PermanentProcessingError("invalid_image_size")
|
||||
|
||||
try:
|
||||
with Image.open(io.BytesIO(body)) as image:
|
||||
if image.format not in {"PNG", "JPEG"}:
|
||||
raise PermanentProcessingError("unsupported_image_format")
|
||||
|
||||
if image.width * image.height > MAX_IMAGE_PIXELS:
|
||||
raise PermanentProcessingError("image_pixel_limit")
|
||||
|
||||
image.verify()
|
||||
|
||||
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as exc:
|
||||
raise PermanentProcessingError("invalid_image") from exc
|
||||
|
||||
|
||||
def make_thumbnail(body: bytes):
|
||||
with Image.open(io.BytesIO(body)) as source:
|
||||
@@ -34,15 +68,44 @@ def make_thumbnail(body: bytes):
|
||||
return output.getvalue(), image.width, image.height
|
||||
|
||||
|
||||
def make_crop(body: bytes):
|
||||
with Image.open(io.BytesIO(body)) as source:
|
||||
source.load()
|
||||
image = ImageOps.exif_transpose(source).convert("RGB")
|
||||
image = ImageOps.fit(
|
||||
image,
|
||||
(128, 128),
|
||||
method=Image.Resampling.LANCZOS,
|
||||
centering=(0.5, 0.5),
|
||||
)
|
||||
|
||||
output = io.BytesIO()
|
||||
image.save(output, format="JPEG", quality=85, optimize=True)
|
||||
|
||||
return output.getvalue(), image.width, image.height
|
||||
|
||||
|
||||
def make_optimized(body: bytes):
|
||||
with Image.open(io.BytesIO(body)) as source:
|
||||
source.load()
|
||||
image = ImageOps.exif_transpose(source).convert("RGB")
|
||||
|
||||
output = io.BytesIO()
|
||||
image.save(output, format="WEBP", quality=80, method=4)
|
||||
|
||||
return output.getvalue(), image.width, image.height
|
||||
|
||||
|
||||
async def process(payload: dict):
|
||||
job_id = UUID(payload["job_id"])
|
||||
attachment_id = UUID(payload["attachment_id"])
|
||||
generation = payload.get("generation", 1)
|
||||
|
||||
async with SessionFactory() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT j.status, j.attempt,
|
||||
SELECT j.status, j.attempt, j.generation,
|
||||
a.s3_key, a.content_type, a.chat_id
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id
|
||||
@@ -61,6 +124,9 @@ async def process(payload: dict):
|
||||
if row.attempt != payload["attempt"]:
|
||||
return
|
||||
|
||||
if row.generation != generation:
|
||||
return
|
||||
|
||||
if row.status == "processing":
|
||||
return
|
||||
|
||||
@@ -84,22 +150,41 @@ async def process(payload: dict):
|
||||
|
||||
try:
|
||||
body = await get_object(row.s3_key)
|
||||
validate_worker_image(body)
|
||||
thumbnail, width, height = await asyncio.to_thread(
|
||||
make_thumbnail, body
|
||||
)
|
||||
|
||||
crop, crop_width, crop_height = await asyncio.to_thread(
|
||||
make_crop, body
|
||||
)
|
||||
|
||||
optimized, optimized_width, optimized_height = await asyncio.to_thread(
|
||||
make_optimized, body
|
||||
)
|
||||
|
||||
original_key = (
|
||||
f"attachments/{row.chat_id}/{attachment_id}/original"
|
||||
f"attachments/{row.chat_id}/{attachment_id}/g{generation}/original"
|
||||
)
|
||||
thumbnail_key = (
|
||||
f"variants/{row.chat_id}/{attachment_id}/thumbnail.jpg"
|
||||
f"variants/{row.chat_id}/{attachment_id}/g{generation}/thumbnail.jpg"
|
||||
)
|
||||
|
||||
await put_object(original_key, body, row.content_type)
|
||||
await put_object(thumbnail_key, thumbnail, "image/jpeg")
|
||||
crop_key = (
|
||||
f"variants/{row.chat_id}/{attachment_id}/g{generation}/crop.jpg"
|
||||
)
|
||||
optimized_key = (
|
||||
f"variants/{row.chat_id}/{attachment_id}/g{generation}/optimized.webp"
|
||||
)
|
||||
|
||||
await put_object(crop_key, crop, "image/jpeg")
|
||||
await put_object(optimized_key, optimized, "image/webp")
|
||||
|
||||
variant = {
|
||||
"name": "thumbnail",
|
||||
"s3_key": thumbnail_key,
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(thumbnail),
|
||||
"width": width,
|
||||
@@ -109,6 +194,25 @@ async def process(payload: dict):
|
||||
|
||||
async with SessionFactory() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT status, attempt, generation
|
||||
FROM attachment_jobs
|
||||
WHERE id = :id
|
||||
FOR UPDATE
|
||||
"""),
|
||||
{"id": job_id},
|
||||
)
|
||||
|
||||
current = result.one_or_none()
|
||||
|
||||
if (
|
||||
current is None
|
||||
or current.status != "processing"
|
||||
or current.attempt != row.attempt
|
||||
or current.generation != generation
|
||||
):
|
||||
return
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachments
|
||||
@@ -122,43 +226,105 @@ async def process(payload: dict):
|
||||
{
|
||||
"id": attachment_id,
|
||||
"key": original_key,
|
||||
"variants": json.dumps([variant]),
|
||||
"variants": json.dumps([
|
||||
variant,
|
||||
{
|
||||
"name": "crop",
|
||||
"s3_key": crop_key,
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(crop),
|
||||
"width": crop_width,
|
||||
"height": crop_height,
|
||||
"sha256": hashlib.sha256(crop).hexdigest(),
|
||||
},
|
||||
{
|
||||
"name": "optimized",
|
||||
"s3_key": optimized_key,
|
||||
"content_type": "image/webp",
|
||||
"size_bytes": len(optimized),
|
||||
"width": optimized_width,
|
||||
"height": optimized_height,
|
||||
"sha256": hashlib.sha256(optimized).hexdigest(),
|
||||
},
|
||||
]),
|
||||
},
|
||||
)
|
||||
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready'
|
||||
WHERE id = :id
|
||||
"""),
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'ready',
|
||||
next_attempt_at = NULL,
|
||||
last_error_code = NULL,
|
||||
finished_at = now()
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": job_id},
|
||||
)
|
||||
|
||||
except Exception:
|
||||
|
||||
except Exception as exc:
|
||||
|
||||
logging.exception("Processing failed: %s", attachment_id)
|
||||
|
||||
permanent = isinstance(exc, PermanentProcessingError)
|
||||
|
||||
error_code = (
|
||||
|
||||
str(exc)[:64]
|
||||
|
||||
if permanent
|
||||
|
||||
else "temporary_processing_error"
|
||||
|
||||
)
|
||||
|
||||
async with SessionFactory() as session:
|
||||
async with session.begin():
|
||||
if row.attempt >= 5:
|
||||
result = await session.execute(
|
||||
text("""
|
||||
SELECT status, attempt, generation
|
||||
FROM attachment_jobs
|
||||
WHERE id = :id
|
||||
FOR UPDATE
|
||||
"""),
|
||||
{"id": job_id},
|
||||
)
|
||||
|
||||
current = result.one_or_none()
|
||||
|
||||
if (
|
||||
current is None
|
||||
or current.status != "processing"
|
||||
or current.attempt != row.attempt
|
||||
or current.generation != generation
|
||||
):
|
||||
return
|
||||
if permanent or row.attempt >= 5:
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed'
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": job_id},
|
||||
UPDATE attachment_jobs
|
||||
SET status = 'failed',
|
||||
next_attempt_at = NULL,
|
||||
last_error_code = :error_code,
|
||||
finished_at = now()
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": job_id, "error_code": error_code},
|
||||
)
|
||||
|
||||
await session.execute(
|
||||
text("""
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = 'retry_exhausted',
|
||||
updated_at = now()
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": attachment_id},
|
||||
UPDATE attachments
|
||||
SET status = 'failed',
|
||||
error_code = :error_code,
|
||||
updated_at = now()
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{
|
||||
"id": attachment_id,
|
||||
"error_code": error_code,
|
||||
},
|
||||
)
|
||||
else:
|
||||
await session.execute(
|
||||
@@ -167,10 +333,16 @@ async def process(payload: dict):
|
||||
SET status = 'queued',
|
||||
attempt = attempt + 1,
|
||||
published_at = NULL,
|
||||
started_at = NULL
|
||||
started_at = NULL,
|
||||
next_attempt_at = now() + (:delay * interval '1 second'),
|
||||
last_error_code = :error_code
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": job_id},
|
||||
{
|
||||
"id": job_id,
|
||||
"delay": retry_delay(row.attempt),
|
||||
"error_code": error_code,
|
||||
}
|
||||
)
|
||||
|
||||
await session.execute(
|
||||
|
||||
Reference in New Issue
Block a user