лаба на 4

This commit is contained in:
2026-11-12 00:28:00 +03:00
parent 4602b83074
commit 910252c277
9 changed files with 474 additions and 42 deletions
@@ -0,0 +1,42 @@
from alembic import op
import sqlalchemy as sa
revision = "a6b12026a002"
down_revision = "e5a12026a001"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
"attachment_jobs",
sa.Column("next_attempt_at", sa.DateTime(timezone=True)),
)
op.add_column(
"attachment_jobs",
sa.Column("last_error_code", sa.String(64)),
)
op.add_column(
"attachment_jobs",
sa.Column("finished_at", sa.DateTime(timezone=True)),
)
op.create_index(
"ix_attachment_jobs_retry_schedule",
"attachment_jobs",
["status", "next_attempt_at"],
)
def downgrade():
op.drop_index(
"ix_attachment_jobs_retry_schedule",
table_name="attachment_jobs",
)
op.drop_column("attachment_jobs", "finished_at")
op.drop_column("attachment_jobs", "last_error_code")
op.drop_column("attachment_jobs", "next_attempt_at")
@@ -0,0 +1,33 @@
from alembic import op
import sqlalchemy as sa
revision = "a7b12026a003"
down_revision = "a6b12026a002"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
"attachment_jobs",
sa.Column("failed_published_at", sa.DateTime(timezone=True)),
)
op.create_index(
"ix_attachment_jobs_failed_unpublished",
"attachment_jobs",
["created_at"],
postgresql_where=sa.text(
"status = 'failed' AND failed_published_at IS NULL"
),
)
def downgrade():
op.drop_index(
"ix_attachment_jobs_failed_unpublished",
table_name="attachment_jobs",
)
op.drop_column("attachment_jobs", "failed_published_at")
@@ -0,0 +1,24 @@
from alembic import op
import sqlalchemy as sa
revision = "b8b12026a004"
down_revision = "a7b12026a003"
branch_labels = None
depends_on = None
def upgrade():
op.add_column(
"attachment_jobs",
sa.Column(
"generation",
sa.Integer(),
nullable=False,
server_default="1",
),
)
def downgrade():
op.drop_column("attachment_jobs", "generation")
+82
View File
@@ -0,0 +1,82 @@
import argparse
import asyncio
from uuid import UUID
from sqlalchemy import text
from src.database import SessionFactory, engine
async def replay(job_id: UUID):
async with SessionFactory() as session:
async with session.begin():
result = await session.execute(
text("""
SELECT j.id, j.attachment_id, j.status,
j.generation, a.status AS attachment_status
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id
WHERE j.id = :job_id
FOR UPDATE OF j, a
"""),
{"job_id": job_id},
)
row = result.one_or_none()
if row is None:
print("REJECTED: job not found")
return
if row.status != "failed" or row.attachment_status != "failed":
print("REJECTED: job and attachment must be failed")
return
await session.execute(
text("""
UPDATE attachment_jobs
SET status = 'queued',
generation = generation + 1,
attempt = 1,
published_at = NULL,
started_at = NULL,
next_attempt_at = NULL,
last_error_code = NULL,
finished_at = NULL,
failed_published_at = NULL
WHERE id = :job_id
"""),
{"job_id": job_id},
)
await session.execute(
text("""
UPDATE attachments
SET status = 'queued',
error_code = NULL,
updated_at = now()
WHERE id = :attachment_id
"""),
{"attachment_id": row.attachment_id},
)
print(
f"REPLAY ACCEPTED: job={job_id}, "
f"generation={row.generation + 1}"
)
async def main():
parser = argparse.ArgumentParser()
parser.add_argument("job_id", type=UUID)
args = parser.parse_args()
try:
await replay(args.job_id)
finally:
await engine.dispose()
if __name__ == "__main__":
asyncio.run(main())
+64 -10
View File
@@ -4,7 +4,7 @@ import logging
from sqlalchemy import text
from src.database import SessionFactory, engine
from src.queue import connect, publish
from src.queue import connect, publish, publish_failed
logging.basicConfig(level=logging.INFO)
@@ -39,14 +39,18 @@ async def dispatch(channel):
result = await session.execute(
text("""
SELECT id, attachment_id, attempt, created_at
FROM attachment_jobs
WHERE published_at IS NULL
AND status = 'queued'
ORDER BY created_at
LIMIT 25
FOR UPDATE SKIP LOCKED
""")
SELECT id, attachment_id, attempt, generation, created_at
FROM attachment_jobs
WHERE published_at IS NULL
AND status = 'queued'
AND (
next_attempt_at IS NULL
OR next_attempt_at <= now()
)
ORDER BY created_at LIMIT 25
FOR
UPDATE SKIP LOCKED
""")
)
rows = result.all()
@@ -66,6 +70,55 @@ async def dispatch(channel):
return len(rows)
async def dispatch_failed(channel):
async with SessionFactory() as session:
async with session.begin():
result = await session.execute(
text("""
SELECT id, attachment_id, attempt, generation,
last_error_code, created_at, finished_at
FROM attachment_jobs
WHERE status = 'failed'
AND failed_published_at IS NULL
ORDER BY created_at
LIMIT 25
FOR UPDATE SKIP LOCKED
""")
)
rows = result.all()
for row in rows:
payload = {
"schema_version": 1,
"type": "attachment.failed.v1",
"job_id": str(row.id),
"attachment_id": str(row.attachment_id),
"attempt": row.attempt,
"generation": row.generation,
"error_code": row.last_error_code,
"created_at": row.created_at.isoformat(),
"finished_at": (
row.finished_at.isoformat()
if row.finished_at
else None
),
}
publish_failed(channel, payload)
await session.execute(
text("""
UPDATE attachment_jobs
SET failed_published_at = now()
WHERE id = :id
"""),
{"id": row.id},
)
return len(rows)
async def main():
while True:
connection = None
@@ -76,7 +129,8 @@ async def main():
while True:
count = await dispatch(channel)
await asyncio.sleep(0.2 if count else 1)
failed_count = await dispatch_failed(channel)
await asyncio.sleep(0.2 if count or failed_count else 1)
except Exception:
logging.exception("Dispatcher reconnect")
+14 -7
View File
@@ -724,9 +724,12 @@ async def get_attachment_content_endpoint(
message="Attachment is not ready",
)
if variant == "thumbnail":
if variant == "original":
media_type = attachment.content_type
elif variant in {"thumbnail", "crop", "optimized"}:
selected = next(
(item for item in attachment.variants if item.name == "thumbnail"),
(item for item in attachment.variants if item.name == variant),
None,
)
@@ -737,14 +740,18 @@ async def get_attachment_content_endpoint(
message="Variant not found",
)
s3_key = (
f"variants/{attachment.chat_id}/{attachment_id}/thumbnail.jpg"
extension = {
"thumbnail": "jpg",
"crop": "jpg",
"optimized": "webp",
}[variant]
s3_key = selected.s3_key or (
f"variants/{attachment.chat_id}/{attachment_id}/"
f"{variant}.{extension}"
)
media_type = selected.content_type
elif variant == "original":
media_type = attachment.content_type
else:
raise APIError(
status_code=404,
+1
View File
@@ -133,6 +133,7 @@ class Variant(BaseModel):
width: int = Field(ge=1)
height: int = Field(ge=1)
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
s3_key: str | None = Field(default=None, exclude=True)
class Attachment(BaseModel):
+17
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import pika
QUEUE = "attachment.process.v1"
DLQ = "attachment.failed.v1"
def connect():
@@ -31,6 +32,7 @@ def connect():
channel = connection.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.queue_declare(queue=DLQ, durable=True)
return connection, channel
@@ -43,6 +45,7 @@ def publish(channel, row):
"attachment_id": str(row.attachment_id),
"pipeline_version": 1,
"attempt": row.attempt,
"generation": row.generation,
"created_at": row.created_at.astimezone(
timezone.utc
).isoformat().replace("+00:00", "Z"),
@@ -58,4 +61,18 @@ def publish(channel, row):
message_id=str(row.id),
),
mandatory=True,
)
def publish_failed(channel, payload: dict):
channel.basic_publish(
exchange="",
routing_key=DLQ,
body=json.dumps(payload).encode(),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
message_id=payload["job_id"],
),
mandatory=True,
)
+197 -25
View File
@@ -1,20 +1,54 @@
import asyncio
import hashlib
import io
import os
import json
import logging
import time
import random
from uuid import UUID
from PIL import Image, ImageOps
from PIL import Image, ImageOps, UnidentifiedImageError
from sqlalchemy import text
from src.database import SessionFactory
from src.queue import QUEUE, connect
from src.storage import get_object, put_object
from src.images import MAX_IMAGE_PIXELS, MAX_UPLOAD_BYTES
logging.basicConfig(level=logging.INFO)
MAX_PROCESSING_SECONDS = float(
os.getenv("MAX_PROCESSING_SECONDS", "30")
)
class PermanentProcessingError(Exception):
pass
def retry_delay(attempt: int) -> float:
base = min(60.0, 2.0 ** attempt)
return base * random.uniform(0.75, 1.25)
def validate_worker_image(body: bytes):
if not body or len(body) > MAX_UPLOAD_BYTES:
raise PermanentProcessingError("invalid_image_size")
try:
with Image.open(io.BytesIO(body)) as image:
if image.format not in {"PNG", "JPEG"}:
raise PermanentProcessingError("unsupported_image_format")
if image.width * image.height > MAX_IMAGE_PIXELS:
raise PermanentProcessingError("image_pixel_limit")
image.verify()
except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as exc:
raise PermanentProcessingError("invalid_image") from exc
def make_thumbnail(body: bytes):
with Image.open(io.BytesIO(body)) as source:
@@ -34,15 +68,44 @@ def make_thumbnail(body: bytes):
return output.getvalue(), image.width, image.height
def make_crop(body: bytes):
with Image.open(io.BytesIO(body)) as source:
source.load()
image = ImageOps.exif_transpose(source).convert("RGB")
image = ImageOps.fit(
image,
(128, 128),
method=Image.Resampling.LANCZOS,
centering=(0.5, 0.5),
)
output = io.BytesIO()
image.save(output, format="JPEG", quality=85, optimize=True)
return output.getvalue(), image.width, image.height
def make_optimized(body: bytes):
with Image.open(io.BytesIO(body)) as source:
source.load()
image = ImageOps.exif_transpose(source).convert("RGB")
output = io.BytesIO()
image.save(output, format="WEBP", quality=80, method=4)
return output.getvalue(), image.width, image.height
async def process(payload: dict):
job_id = UUID(payload["job_id"])
attachment_id = UUID(payload["attachment_id"])
generation = payload.get("generation", 1)
async with SessionFactory() as session:
async with session.begin():
result = await session.execute(
text("""
SELECT j.status, j.attempt,
SELECT j.status, j.attempt, j.generation,
a.s3_key, a.content_type, a.chat_id
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id
@@ -61,6 +124,9 @@ async def process(payload: dict):
if row.attempt != payload["attempt"]:
return
if row.generation != generation:
return
if row.status == "processing":
return
@@ -84,22 +150,41 @@ async def process(payload: dict):
try:
body = await get_object(row.s3_key)
validate_worker_image(body)
thumbnail, width, height = await asyncio.to_thread(
make_thumbnail, body
)
crop, crop_width, crop_height = await asyncio.to_thread(
make_crop, body
)
optimized, optimized_width, optimized_height = await asyncio.to_thread(
make_optimized, body
)
original_key = (
f"attachments/{row.chat_id}/{attachment_id}/original"
f"attachments/{row.chat_id}/{attachment_id}/g{generation}/original"
)
thumbnail_key = (
f"variants/{row.chat_id}/{attachment_id}/thumbnail.jpg"
f"variants/{row.chat_id}/{attachment_id}/g{generation}/thumbnail.jpg"
)
await put_object(original_key, body, row.content_type)
await put_object(thumbnail_key, thumbnail, "image/jpeg")
crop_key = (
f"variants/{row.chat_id}/{attachment_id}/g{generation}/crop.jpg"
)
optimized_key = (
f"variants/{row.chat_id}/{attachment_id}/g{generation}/optimized.webp"
)
await put_object(crop_key, crop, "image/jpeg")
await put_object(optimized_key, optimized, "image/webp")
variant = {
"name": "thumbnail",
"s3_key": thumbnail_key,
"content_type": "image/jpeg",
"size_bytes": len(thumbnail),
"width": width,
@@ -109,6 +194,25 @@ async def process(payload: dict):
async with SessionFactory() as session:
async with session.begin():
result = await session.execute(
text("""
SELECT status, attempt, generation
FROM attachment_jobs
WHERE id = :id
FOR UPDATE
"""),
{"id": job_id},
)
current = result.one_or_none()
if (
current is None
or current.status != "processing"
or current.attempt != row.attempt
or current.generation != generation
):
return
await session.execute(
text("""
UPDATE attachments
@@ -122,43 +226,105 @@ async def process(payload: dict):
{
"id": attachment_id,
"key": original_key,
"variants": json.dumps([variant]),
"variants": json.dumps([
variant,
{
"name": "crop",
"s3_key": crop_key,
"content_type": "image/jpeg",
"size_bytes": len(crop),
"width": crop_width,
"height": crop_height,
"sha256": hashlib.sha256(crop).hexdigest(),
},
{
"name": "optimized",
"s3_key": optimized_key,
"content_type": "image/webp",
"size_bytes": len(optimized),
"width": optimized_width,
"height": optimized_height,
"sha256": hashlib.sha256(optimized).hexdigest(),
},
]),
},
)
await session.execute(
text("""
UPDATE attachment_jobs
SET status = 'ready'
WHERE id = :id
"""),
UPDATE attachment_jobs
SET status = 'ready',
next_attempt_at = NULL,
last_error_code = NULL,
finished_at = now()
WHERE id = :id
"""),
{"id": job_id},
)
except Exception:
except Exception as exc:
logging.exception("Processing failed: %s", attachment_id)
permanent = isinstance(exc, PermanentProcessingError)
error_code = (
str(exc)[:64]
if permanent
else "temporary_processing_error"
)
async with SessionFactory() as session:
async with session.begin():
if row.attempt >= 5:
result = await session.execute(
text("""
SELECT status, attempt, generation
FROM attachment_jobs
WHERE id = :id
FOR UPDATE
"""),
{"id": job_id},
)
current = result.one_or_none()
if (
current is None
or current.status != "processing"
or current.attempt != row.attempt
or current.generation != generation
):
return
if permanent or row.attempt >= 5:
await session.execute(
text("""
UPDATE attachment_jobs
SET status = 'failed'
WHERE id = :id
"""),
{"id": job_id},
UPDATE attachment_jobs
SET status = 'failed',
next_attempt_at = NULL,
last_error_code = :error_code,
finished_at = now()
WHERE id = :id
"""),
{"id": job_id, "error_code": error_code},
)
await session.execute(
text("""
UPDATE attachments
SET status = 'failed',
error_code = 'retry_exhausted',
updated_at = now()
WHERE id = :id
"""),
{"id": attachment_id},
UPDATE attachments
SET status = 'failed',
error_code = :error_code,
updated_at = now()
WHERE id = :id
"""),
{
"id": attachment_id,
"error_code": error_code,
},
)
else:
await session.execute(
@@ -167,10 +333,16 @@ async def process(payload: dict):
SET status = 'queued',
attempt = attempt + 1,
published_at = NULL,
started_at = NULL
started_at = NULL,
next_attempt_at = now() + (:delay * interval '1 second'),
last_error_code = :error_code
WHERE id = :id
"""),
{"id": job_id},
{
"id": job_id,
"delay": retry_delay(row.attempt),
"error_code": error_code,
}
)
await session.execute(