add level 4
This commit is contained in:
@@ -11,7 +11,9 @@ Python 3.12, PostgreSQL 17.6, SeaweedFS 4.17, RabbitMQ 4.1 и HAProxy 3.0 зап
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
PostgreSQL хранит метаданные и состояние, SeaweedFS — private staging/final objects, RabbitMQ — durable сообщения, worker — отдельный процесс. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job, затем ждёт RabbitMQ publisher confirm. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат.
|
||||
PostgreSQL хранит метаданные, jobs и transactional outbox; SeaweedFS — private staging/final objects; RabbitMQ — durable сообщения; worker и outbox-dispatcher — отдельные процессы. До HTTP 202 API полностью принимает и валидирует bytes, сохраняет staging-object и в одной DB-транзакции метаданные + job + outbox event. Dispatcher публикует событие с publisher confirm и отмечает event после confirm, поэтому crash между COMMIT и publish восстанавливается без ручной job. Worker ack делает только после durable результата либо после финального failed. Обработка идемпотентна по attachment/pipeline/variant: повтор ready-job не меняет результат. Lease ограничен `JOB_TIMEOUT_SECONDS`; истёкшая processing-job возвращается в очередь dispatcher-ом. Retry ограничен `JOB_MAX_ATTEMPTS`, использует exponential backoff с jitter; исчерпанная job остаётся failed и её служебное сообщение попадает в DLQ. Явный replay: `docker compose run --rm outbox /app/.venv/bin/python -m src.replay <attachment_uuid>`.
|
||||
|
||||
На уровне 4 worker создаёт: thumbnail JPEG 256×256 без увеличения; crop JPEG 128×128, центрированный квадрат; optimized WebP до 1280×1280 без увеличения. JPEG quality=85, WebP quality=80; входные EXIF/metadata не сохраняются.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ x-api: &api
|
||||
UPLOAD_CONCURRENCY: "1"
|
||||
UPLOAD_TIMEOUT_SECONDS: "30"
|
||||
UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0}
|
||||
OUTBOX_FAULT_AFTER_COMMIT: ${OUTBOX_FAULT_AFTER_COMMIT:-0}
|
||||
secrets:
|
||||
- database_url
|
||||
- cursor_secret
|
||||
@@ -197,6 +198,8 @@ services:
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
secrets:
|
||||
- database_url
|
||||
- cursor_secret
|
||||
@@ -212,6 +215,27 @@ services:
|
||||
condition: service_healthy
|
||||
stop_grace_period: 30s
|
||||
|
||||
outbox:
|
||||
build:
|
||||
context: .
|
||||
command: ["/app/.venv/bin/python", "-m", "src.outbox"]
|
||||
environment:
|
||||
APP_ENV: lab5
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
OUTBOX_DISPATCH_INTERVAL_SECONDS: "1"
|
||||
secrets:
|
||||
- database_url
|
||||
- cursor_secret
|
||||
- broker_url
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
image: haproxy:3.0.11-alpine
|
||||
ports:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(),
|
||||
ADD COLUMN lease_until timestamptz,
|
||||
ADD COLUMN replayed_at timestamptz;
|
||||
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(status, next_attempt_at);
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(status, lease_until);
|
||||
|
||||
CREATE TABLE attachment_outbox (
|
||||
id uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(job_id) ON DELETE CASCADE,
|
||||
payload jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
attempts integer NOT NULL DEFAULT 0,
|
||||
UNIQUE (job_id, published_at)
|
||||
);
|
||||
CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(created_at) WHERE published_at IS NULL;
|
||||
+3
-7
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import os
|
||||
from tempfile import TemporaryFile
|
||||
@@ -16,7 +15,6 @@ from .errors import ApiError
|
||||
from .models import Attachment
|
||||
from .routes import get_current_user, get_store
|
||||
from .storage.postgres import PostgresStore
|
||||
from .broker import publish
|
||||
|
||||
router = APIRouter()
|
||||
Image.MAX_IMAGE_PIXELS = 40000000
|
||||
@@ -93,11 +91,9 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...),
|
||||
attachment_id, chat_id, user["id"], filename, content_type,
|
||||
size, digest.hexdigest(), key, job_id, connection=connection,
|
||||
)
|
||||
await publish({
|
||||
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1,
|
||||
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
})
|
||||
# Fault-injection for level-4 proof: durable outbox must recover this commit.
|
||||
if os.environ.get("OUTBOX_FAULT_AFTER_COMMIT") == "1":
|
||||
os._exit(70)
|
||||
attachment["variants"] = []
|
||||
return attachment
|
||||
except ClientDisconnect:
|
||||
|
||||
+23
-1
@@ -9,6 +9,7 @@ from starlette.concurrency import run_in_threadpool
|
||||
from .errors import ApiError
|
||||
|
||||
QUEUE = "attachment.process.v1"
|
||||
DLQ = "attachment.process.v1.dlq"
|
||||
|
||||
|
||||
def broker_url() -> str:
|
||||
@@ -21,7 +22,9 @@ def broker_url() -> str:
|
||||
|
||||
def _connection() -> pika.BlockingConnection:
|
||||
connection = pika.BlockingConnection(pika.URLParameters(broker_url()))
|
||||
connection.channel().queue_declare(queue=QUEUE, durable=True)
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=QUEUE, durable=True)
|
||||
channel.queue_declare(queue=DLQ, durable=True)
|
||||
return connection
|
||||
|
||||
|
||||
@@ -47,6 +50,25 @@ async def publish(payload: dict) -> None:
|
||||
raise ApiError(503, "unavailable", "Брокер недоступен") from error
|
||||
|
||||
|
||||
async def publish_dlq(payload: dict) -> None:
|
||||
payload = {**payload, "failure": "retry_exhausted"}
|
||||
try:
|
||||
await run_in_threadpool(_publish_to, DLQ, payload)
|
||||
except (pika.exceptions.AMQPError, OSError, ValueError) as error:
|
||||
raise RuntimeError("DLQ is unavailable") from error
|
||||
|
||||
|
||||
def _publish_to(queue: str, payload: dict) -> None:
|
||||
connection = _connection()
|
||||
try:
|
||||
connection.channel().basic_publish(
|
||||
exchange="", routing_key=queue, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"),
|
||||
properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent, content_type="application/json"),
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
async def ready() -> None:
|
||||
connection = None
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Transactional-outbox dispatcher; safe to run as one or more replicas."""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
|
||||
from .broker import publish
|
||||
from .config import load_settings
|
||||
from .storage.postgres import PostgresStore
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s")
|
||||
|
||||
|
||||
async def run() -> None:
|
||||
settings = load_settings()
|
||||
store = PostgresStore(settings.database_url)
|
||||
await store.connect()
|
||||
interval = max(0.1, float(os.environ.get("OUTBOX_DISPATCH_INTERVAL_SECONDS", "1")))
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
await store.schedule_due_jobs()
|
||||
for event in await store.due_outbox_events():
|
||||
await publish(event["payload"])
|
||||
await store.mark_outbox_published(event["id"])
|
||||
except Exception:
|
||||
logging.exception("outbox dispatch failed; durable event remains pending")
|
||||
await asyncio.sleep(interval)
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run())
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Operator-only safe replay for a failed attachment job."""
|
||||
import argparse
|
||||
import asyncio
|
||||
from uuid import UUID
|
||||
|
||||
from .config import load_settings
|
||||
from .storage.postgres import PostgresStore
|
||||
|
||||
|
||||
async def run(attachment_id: UUID) -> None:
|
||||
store = PostgresStore(load_settings().database_url)
|
||||
await store.connect()
|
||||
try:
|
||||
if not await store.replay_failed_attachment(attachment_id):
|
||||
raise SystemExit("attachment is not in failed state")
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("attachment_id", type=UUID)
|
||||
args = parser.parse_args()
|
||||
asyncio.run(run(args.attachment_id))
|
||||
+83
-15
@@ -1,4 +1,4 @@
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from contextlib import asynccontextmanager
|
||||
from hashlib import sha256
|
||||
import json
|
||||
@@ -330,6 +330,18 @@ class PostgresStore:
|
||||
await (connection or self.pool).execute(
|
||||
"INSERT INTO attachment_jobs (job_id, attachment_id) VALUES ($1, $2)", job_id, attachment_id
|
||||
)
|
||||
payload = {
|
||||
"schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1",
|
||||
"attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1,
|
||||
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
await (connection or self.pool).execute(
|
||||
"INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)",
|
||||
uuid4(), job_id, json.dumps(payload, separators=(",", ":")),
|
||||
)
|
||||
await (connection or self.pool).execute(
|
||||
"UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", job_id
|
||||
)
|
||||
except DB_ERRORS as error:
|
||||
raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error
|
||||
return dict(row)
|
||||
@@ -356,37 +368,44 @@ class PostgresStore:
|
||||
result["variants"] = [dict(item) for item in variants]
|
||||
return result
|
||||
|
||||
async def claim_attachment_job(self, attachment_id: UUID) -> dict | None:
|
||||
async def claim_attachment_job(self, attachment_id: UUID, lease_seconds: int) -> dict | None:
|
||||
try:
|
||||
async with self.pool.acquire() as connection, connection.transaction():
|
||||
row = await connection.fetchrow("SELECT * FROM attachments WHERE id = $1 FOR UPDATE", attachment_id)
|
||||
row = await connection.fetchrow(
|
||||
"SELECT a.*, j.job_id, j.attempt, j.lease_until FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
|
||||
"WHERE a.id = $1 FOR UPDATE", attachment_id
|
||||
)
|
||||
if row is None or row["status"] in {"ready", "failed"}:
|
||||
return None
|
||||
if row["status"] == "processing" and row["lease_until"] is not None and row["lease_until"] > datetime.now(timezone.utc):
|
||||
return None
|
||||
await connection.execute(
|
||||
"UPDATE attachments SET status = 'processing', processing_started_at = now(), updated_at = now() WHERE id = $1",
|
||||
attachment_id,
|
||||
)
|
||||
await connection.execute(
|
||||
"UPDATE attachment_jobs SET status = 'processing', updated_at = now() WHERE attachment_id = $1", attachment_id
|
||||
"UPDATE attachment_jobs SET status = 'processing', lease_until = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1",
|
||||
attachment_id, lease_seconds,
|
||||
)
|
||||
return dict(row)
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variant: dict) -> None:
|
||||
async def complete_attachment_job(self, attachment_id: UUID, original_key: str, variants: list[dict]) -> None:
|
||||
try:
|
||||
async with self.pool.acquire() as connection, connection.transaction():
|
||||
await connection.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) "
|
||||
"VALUES ($1, 1, 'thumbnail', $2, $3, $4, $5, $6, $7) "
|
||||
"ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256",
|
||||
attachment_id, variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"],
|
||||
)
|
||||
for variant in variants:
|
||||
await connection.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) "
|
||||
"VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) "
|
||||
"ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type, size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height, sha256 = EXCLUDED.sha256",
|
||||
attachment_id, variant["name"], variant["object_key"], variant["content_type"], variant["size_bytes"], variant["width"], variant["height"], variant["sha256"],
|
||||
)
|
||||
await connection.execute(
|
||||
"UPDATE attachments SET status = 'ready', original_key = $2, error_code = NULL, updated_at = now() WHERE id = $1",
|
||||
attachment_id, original_key,
|
||||
)
|
||||
await connection.execute("UPDATE attachment_jobs SET status = 'ready', updated_at = now() WHERE attachment_id = $1", attachment_id)
|
||||
await connection.execute("UPDATE attachment_jobs SET status = 'ready', lease_until = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id)
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
@@ -401,7 +420,7 @@ class PostgresStore:
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int) -> bool:
|
||||
async def retry_attachment_job(self, attachment_id: UUID, max_attempts: int, delay_seconds: float) -> bool:
|
||||
"""Return true only when a bounded at-least-once retry was recorded."""
|
||||
try:
|
||||
async with self.pool.acquire() as connection, connection.transaction():
|
||||
@@ -410,8 +429,8 @@ class PostgresStore:
|
||||
)
|
||||
if job is not None and job["attempt"] < max_attempts:
|
||||
await connection.execute(
|
||||
"UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', updated_at = now() WHERE attachment_id = $1",
|
||||
attachment_id,
|
||||
"UPDATE attachment_jobs SET attempt = attempt + 1, status = 'queued', last_error = 'processing_failed', lease_until = NULL, next_attempt_at = now() + $2 * interval '1 second', updated_at = now() WHERE attachment_id = $1",
|
||||
attachment_id, delay_seconds,
|
||||
)
|
||||
await connection.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", attachment_id
|
||||
@@ -422,6 +441,55 @@ class PostgresStore:
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
async def due_outbox_events(self, limit: int = 20) -> list[dict]:
|
||||
try:
|
||||
return [dict(row) for row in await self.pool.fetch(
|
||||
"SELECT id, payload FROM attachment_outbox WHERE published_at IS NULL ORDER BY created_at LIMIT $1", limit
|
||||
)]
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
async def mark_outbox_published(self, event_id: UUID) -> None:
|
||||
await self.pool.execute("UPDATE attachment_outbox SET published_at = now(), attempts = attempts + 1 WHERE id = $1", event_id)
|
||||
|
||||
async def schedule_due_jobs(self) -> None:
|
||||
"""Outbox dispatcher also restores expired leases and scheduled retries."""
|
||||
try:
|
||||
async with self.pool.acquire() as connection, connection.transaction():
|
||||
rows = await connection.fetch(
|
||||
"SELECT job_id, attachment_id, pipeline_version, attempt, status FROM attachment_jobs "
|
||||
"WHERE (status = 'queued' AND next_attempt_at <= now()) OR (status = 'processing' AND lease_until < now()) "
|
||||
"FOR UPDATE SKIP LOCKED"
|
||||
)
|
||||
for row in rows:
|
||||
if row["status"] == "processing":
|
||||
await connection.execute(
|
||||
"UPDATE attachment_jobs SET status = 'queued', lease_until = NULL, next_attempt_at = now(), last_error = 'lease_expired', updated_at = now() WHERE job_id = $1", row["job_id"]
|
||||
)
|
||||
await connection.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = $1", row["attachment_id"])
|
||||
payload = {"schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}
|
||||
await connection.execute("INSERT INTO attachment_outbox (id, job_id, payload) VALUES ($1, $2, $3::jsonb)", uuid4(), row["job_id"], json.dumps(payload, separators=(",", ":")))
|
||||
await connection.execute("UPDATE attachment_jobs SET next_attempt_at = 'infinity' WHERE job_id = $1", row["job_id"])
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
async def replay_failed_attachment(self, attachment_id: UUID) -> bool:
|
||||
"""Explicit operator-only replay; never requeues an already successful job."""
|
||||
try:
|
||||
async with self.pool.acquire() as connection, connection.transaction():
|
||||
job = await connection.fetchrow(
|
||||
"SELECT job_id, status FROM attachment_jobs WHERE attachment_id = $1 FOR UPDATE", attachment_id
|
||||
)
|
||||
if job is None or job["status"] != "failed":
|
||||
return False
|
||||
await connection.execute(
|
||||
"UPDATE attachment_jobs SET status = 'queued', attempt = 1, next_attempt_at = now(), lease_until = NULL, replayed_at = now(), last_error = NULL, updated_at = now() WHERE attachment_id = $1", attachment_id
|
||||
)
|
||||
await connection.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = $1", attachment_id)
|
||||
return True
|
||||
except DB_ERRORS as error:
|
||||
raise RuntimeError("PostgreSQL is unavailable") from error
|
||||
|
||||
async def attachment_keys(self) -> dict[str, UUID]:
|
||||
"""Inventory used only by the maintenance reconciler."""
|
||||
try:
|
||||
|
||||
+55
-16
@@ -5,11 +5,12 @@ from io import BytesIO
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from uuid import UUID
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from .broker import QUEUE, _connection
|
||||
from .broker import QUEUE, _connection, publish_dlq
|
||||
from .config import load_settings
|
||||
from .storage.postgres import PostgresStore
|
||||
from .storage.s3 import S3Settings, S3Store
|
||||
@@ -32,21 +33,55 @@ def thumbnail(data: bytes) -> tuple[bytes, int, int]:
|
||||
return output.getvalue(), image.width, image.height
|
||||
|
||||
|
||||
async def process(store: PostgresStore, s3: S3Store, payload: dict) -> None:
|
||||
def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]:
|
||||
"""Render required variants without preserving EXIF or other input metadata."""
|
||||
with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source:
|
||||
if source.width * source.height > max_pixels:
|
||||
raise ValueError("image exceeds configured pixel limit")
|
||||
image = ImageOps.exif_transpose(source).convert("RGBA")
|
||||
white = Image.new("RGBA", image.size, "white")
|
||||
white.alpha_composite(image)
|
||||
rgb = white.convert("RGB")
|
||||
|
||||
thumb = rgb.copy()
|
||||
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
||||
thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True)
|
||||
|
||||
side = min(rgb.width, rgb.height)
|
||||
left, top = (rgb.width - side) // 2, (rgb.height - side) // 2
|
||||
crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
|
||||
crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True)
|
||||
|
||||
optimized = rgb.copy()
|
||||
optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4)
|
||||
return [
|
||||
("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height),
|
||||
("crop", crop_out.getvalue(), "image/jpeg", 128, 128),
|
||||
("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height),
|
||||
]
|
||||
|
||||
|
||||
async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> None:
|
||||
attachment_id = UUID(payload["attachment_id"])
|
||||
attachment = await store.claim_attachment_job(attachment_id)
|
||||
attachment = await store.claim_attachment_job(attachment_id, lease_seconds)
|
||||
if attachment is None:
|
||||
return
|
||||
original_key = f"originals/{attachment_id}"
|
||||
variant_key = f"variants/{attachment_id}/thumbnail.jpg"
|
||||
await s3.copy(attachment["staging_key"], original_key, attachment["content_type"])
|
||||
original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"])
|
||||
rendered, width, height = await asyncio.to_thread(thumbnail, original)
|
||||
await s3.put(variant_key, BytesIO(rendered), len(rendered), "image/jpeg")
|
||||
await store.complete_attachment_job(attachment_id, original_key, {
|
||||
"object_key": variant_key, "content_type": "image/jpeg", "size_bytes": len(rendered),
|
||||
"width": width, "height": height, "sha256": sha256(rendered).hexdigest(),
|
||||
})
|
||||
rendered = await asyncio.wait_for(
|
||||
asyncio.to_thread(variants, original, s3.settings.max_pixels), timeout=lease_seconds
|
||||
)
|
||||
metadata = []
|
||||
for name, content, content_type, width, height in rendered:
|
||||
extension = "webp" if name == "optimized" else "jpg"
|
||||
object_key = f"variants/{attachment_id}/{name}.{extension}"
|
||||
await s3.put(object_key, BytesIO(content), len(content), content_type)
|
||||
metadata.append({"name": name, "object_key": object_key, "content_type": content_type,
|
||||
"size_bytes": len(content), "width": width, "height": height,
|
||||
"sha256": sha256(content).hexdigest()})
|
||||
await store.complete_attachment_job(attachment_id, original_key, metadata)
|
||||
await s3.delete(attachment["staging_key"])
|
||||
|
||||
|
||||
@@ -65,20 +100,24 @@ def main() -> None:
|
||||
payload = json.loads(body)
|
||||
if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1:
|
||||
raise ValueError("unexpected broker payload")
|
||||
asyncio.run(process(store, s3, payload))
|
||||
asyncio.run(process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")))))
|
||||
except Exception:
|
||||
logging.exception("job failed")
|
||||
try:
|
||||
attachment_id = UUID(payload["attachment_id"])
|
||||
attempt = int(payload.get("attempt", 1))
|
||||
delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5)
|
||||
retry = asyncio.run(store.retry_attachment_job(
|
||||
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5")))
|
||||
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay
|
||||
))
|
||||
except Exception:
|
||||
retry = False
|
||||
if retry:
|
||||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||||
else:
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
if not retry:
|
||||
try:
|
||||
asyncio.run(publish_dlq(payload))
|
||||
except Exception:
|
||||
logging.exception("could not publish failed job to DLQ")
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user