Process versioned image variants with leased worker (U3)
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
"""Authorize metadata and stream versioned variants from the private bucket."""
|
||||
|
||||
import logging
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from .attachments import BUCKET, attachment_view, storage
|
||||
from .store import connection
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def metadata(actor_id: str, attachment_id: UUID) -> dict:
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.* FROM attachments a "
|
||||
"JOIN memberships m ON m.chat_id=a.chat_id "
|
||||
"WHERE a.id=%s AND m.user_id=%s",
|
||||
(attachment_id, UUID(actor_id)),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise HTTPException(404)
|
||||
return row
|
||||
|
||||
|
||||
def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse:
|
||||
row = metadata(actor_id, attachment_id)
|
||||
if row["status"] != "ready":
|
||||
raise HTTPException(409)
|
||||
with connection() as conn:
|
||||
selected = conn.execute(
|
||||
"SELECT object_key,content_type,size_bytes FROM attachment_variants "
|
||||
"WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
|
||||
(attachment_id, row["pipeline_version"], variant),
|
||||
).fetchone()
|
||||
if selected is None:
|
||||
raise HTTPException(404)
|
||||
try:
|
||||
obj = storage().get_object(BUCKET, selected["object_key"])
|
||||
except Exception as exc:
|
||||
logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__)
|
||||
raise HTTPException(503) from None
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
yield from obj.stream(64 * 1024)
|
||||
finally:
|
||||
obj.close()
|
||||
obj.release_conn()
|
||||
|
||||
return StreamingResponse(
|
||||
chunks(), media_type=selected["content_type"],
|
||||
headers={"Content-Length": str(selected["size_bytes"])}
|
||||
)
|
||||
|
||||
|
||||
def get_view(actor_id: str, attachment_id: UUID) -> dict:
|
||||
row = metadata(actor_id, attachment_id)
|
||||
view = attachment_view(row)
|
||||
if row["status"] == "ready":
|
||||
with connection() as conn:
|
||||
variants = conn.execute(
|
||||
"SELECT name,content_type,size_bytes,width,height,sha256 "
|
||||
"FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s "
|
||||
"AND name <> 'original' ORDER BY name",
|
||||
(attachment_id, row["pipeline_version"]),
|
||||
).fetchall()
|
||||
view["variants"] = [dict(variant) for variant in variants]
|
||||
return view
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Deterministic, bounded image variants for one accepted PNG or JPEG."""
|
||||
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
|
||||
|
||||
class InvalidImage(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def _fit(image: Image.Image, bound: int) -> Image.Image:
|
||||
factor = min(1, bound / max(image.size))
|
||||
size = (max(1, int(image.width * factor)), max(1, int(image.height * factor)))
|
||||
return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image
|
||||
|
||||
|
||||
def _jpeg(image: Image.Image) -> bytes:
|
||||
output = BytesIO()
|
||||
image.save(output, format="JPEG", quality=85, optimize=True)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def _variant(data: bytes, content_type: str, image: Image.Image) -> dict:
|
||||
return {
|
||||
"data": data,
|
||||
"content_type": content_type,
|
||||
"size_bytes": len(data),
|
||||
"width": image.width,
|
||||
"height": image.height,
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def render(data: bytes, content_type: str) -> dict[str, dict]:
|
||||
"""Preserve original bytes; orient, flatten alpha, strip metadata on derivatives."""
|
||||
if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}:
|
||||
raise InvalidImage("invalid original")
|
||||
try:
|
||||
with Image.open(BytesIO(data)) as source:
|
||||
if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type:
|
||||
raise InvalidImage("wrong image type")
|
||||
if source.width * source.height > MAX_PIXELS:
|
||||
raise InvalidImage("too many pixels")
|
||||
source.load()
|
||||
original_size = source.size
|
||||
oriented = ImageOps.exif_transpose(source)
|
||||
if oriented.mode in {"RGBA", "LA", "P"}:
|
||||
rgba = oriented.convert("RGBA")
|
||||
image = Image.new("RGB", rgba.size, "white")
|
||||
image.paste(rgba, mask=rgba.getchannel("A"))
|
||||
else:
|
||||
image = oriented.convert("RGB")
|
||||
|
||||
thumbnail = _fit(image, 256)
|
||||
side = min(image.size)
|
||||
left = (image.width - side) // 2
|
||||
top = (image.height - side) // 2
|
||||
crop = image.resize(
|
||||
(128, 128), Image.Resampling.LANCZOS,
|
||||
box=(left, top, left + side, top + side),
|
||||
)
|
||||
optimized = _fit(image, 1280)
|
||||
webp = BytesIO()
|
||||
optimized.save(webp, format="WEBP", quality=82, method=4)
|
||||
return {
|
||||
"original": {
|
||||
"data": data, "content_type": content_type,
|
||||
"size_bytes": len(data), "width": original_size[0], "height": original_size[1],
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
},
|
||||
"thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail),
|
||||
"crop": _variant(_jpeg(crop), "image/jpeg", crop),
|
||||
"optimized": _variant(webp.getvalue(), "image/webp", optimized),
|
||||
}
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc:
|
||||
raise InvalidImage("image decoding failed") from exc
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import signal
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from minio.error import S3Error
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .attachments import BUCKET, storage
|
||||
from .db import pool
|
||||
from .image_variants import InvalidImage, MAX_BYTES, render
|
||||
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
||||
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
||||
LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
|
||||
|
||||
|
||||
def parse_message(body: bytes) -> dict:
|
||||
if len(body) > 4096:
|
||||
raise ValueError("oversized job message")
|
||||
item = json.loads(body)
|
||||
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
if (not isinstance(item, dict) or not required <= item.keys() or
|
||||
item.keys() - required - {"traceparent", "tracestate"} or
|
||||
type(item["schema_version"]) is not int or item["schema_version"] != 1 or
|
||||
item["type"] != "attachment.process.v1" or
|
||||
type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or
|
||||
type(item["attempt"]) is not int or item["attempt"] < 1 or
|
||||
not isinstance(item["job_id"], str) or
|
||||
not isinstance(item["attachment_id"], str) or
|
||||
not isinstance(item["created_at"], str) or
|
||||
("traceparent" in item and
|
||||
(not isinstance(item["traceparent"], str) or
|
||||
re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or
|
||||
("tracestate" in item and
|
||||
(not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))):
|
||||
raise ValueError("invalid job message")
|
||||
UUID(item["job_id"])
|
||||
UUID(item["attachment_id"])
|
||||
created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00"))
|
||||
if created_at.utcoffset() is None:
|
||||
raise ValueError("job timestamp lacks timezone")
|
||||
return item
|
||||
|
||||
|
||||
def _source_bytes(job) -> bytes:
|
||||
if not 0 < job["size_bytes"] <= MAX_BYTES:
|
||||
raise InvalidImage("invalid accepted size")
|
||||
# Reprocessing uses the byte-identical final original, so ready staging
|
||||
# can be reclaimed without making a future pipeline version impossible.
|
||||
key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"]
|
||||
response = storage().get_object(BUCKET, key)
|
||||
try:
|
||||
chunks = []
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
for chunk in response.stream(64 * 1024):
|
||||
size += len(chunk)
|
||||
if size > job["size_bytes"]:
|
||||
raise InvalidImage("source exceeds accepted size")
|
||||
chunks.append(chunk)
|
||||
digest.update(chunk)
|
||||
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
|
||||
raise InvalidImage("source checksum mismatch")
|
||||
return b"".join(chunks)
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
|
||||
def _write_variants(job, variants: dict) -> dict:
|
||||
extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
|
||||
stored = {}
|
||||
for name, variant in variants.items():
|
||||
key = (job["object_key"] if name == "original" else
|
||||
f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}")
|
||||
data = variant["data"]
|
||||
storage().put_object(
|
||||
BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"]
|
||||
)
|
||||
stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"}
|
||||
stored[name]["object_key"] = key
|
||||
return stored
|
||||
|
||||
|
||||
def _delay(attempt: int) -> float:
|
||||
base = min(30, 2 ** min(attempt - 1, 5))
|
||||
return base + random.uniform(0, base / 2)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _deadline(seconds: int):
|
||||
def expired(_signal, _frame):
|
||||
raise TimeoutError("image job exceeded worker deadline")
|
||||
|
||||
previous = signal.signal(signal.SIGALRM, expired)
|
||||
signal.setitimer(signal.ITIMER_REAL, seconds)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
signal.signal(signal.SIGALRM, previous)
|
||||
|
||||
|
||||
def process_message(item: dict) -> str:
|
||||
with pool().connection() as conn:
|
||||
job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS)
|
||||
if job is None:
|
||||
return "duplicate_or_stale"
|
||||
token = job["lease_token"]
|
||||
try:
|
||||
with _deadline(LEASE_SECONDS):
|
||||
data = _source_bytes(job)
|
||||
variants = render(data, job["content_type"])
|
||||
stored = _write_variants(job, variants)
|
||||
with pool().connection() as conn:
|
||||
current = complete_job(conn, job["id"], token, stored)
|
||||
if current and job["staging_key"]:
|
||||
try:
|
||||
storage().remove_object(BUCKET, job["staging_key"])
|
||||
except Exception as exc:
|
||||
# Periodic cleanup recovers this post-COMMIT gap.
|
||||
logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__)
|
||||
return "ready"
|
||||
except psycopg.Error:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if isinstance(exc, InvalidImage):
|
||||
code, permanent = "invalid_image", True
|
||||
elif isinstance(exc, TimeoutError):
|
||||
code, permanent = "worker_timeout", False
|
||||
elif isinstance(exc, S3Error) or isinstance(exc, OSError):
|
||||
code, permanent = "storage_unavailable", False
|
||||
elif isinstance(exc, ValueError):
|
||||
code, permanent = "processing_failed", True
|
||||
else:
|
||||
code, permanent = "processing_failed", False
|
||||
logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code)
|
||||
with pool().connection() as conn:
|
||||
try:
|
||||
return finish_attempt(
|
||||
conn, job["id"], token, max_attempts=MAX_ATTEMPTS,
|
||||
delay_seconds=_delay(job["attempt"]), error_code=code,
|
||||
permanent=permanent,
|
||||
)
|
||||
except ValueError:
|
||||
# A lease recovery already owns the next attempt.
|
||||
return "stale_lease"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
|
||||
raise SystemExit("invalid worker retry settings")
|
||||
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
|
||||
params = pika.URLParameters(url)
|
||||
params.heartbeat = max(120, LEASE_SECONDS * 2)
|
||||
params.socket_timeout = 5
|
||||
params.blocked_connection_timeout = 10
|
||||
while True:
|
||||
broker = None
|
||||
try:
|
||||
broker = pika.BlockingConnection(params)
|
||||
channel = broker.channel()
|
||||
channel.queue_declare(queue=QUEUE, passive=True)
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
last_recovery = 0.0
|
||||
while broker.is_open:
|
||||
if time.monotonic() - last_recovery >= 5:
|
||||
with pool().connection() as conn:
|
||||
recover_expired(conn, max_attempts=MAX_ATTEMPTS)
|
||||
last_recovery = time.monotonic()
|
||||
method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False)
|
||||
if method is None:
|
||||
broker.process_data_events(time_limit=1)
|
||||
continue
|
||||
try:
|
||||
item = parse_message(body)
|
||||
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||
logger.warning("Rejecting invalid job message")
|
||||
channel.basic_reject(method.delivery_tag, requeue=False)
|
||||
continue
|
||||
try:
|
||||
result = process_message(item)
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
logger.info("Job %s: %s", item["job_id"], result)
|
||||
except (psycopg.Error, PoolTimeout, OSError) as exc:
|
||||
logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__)
|
||||
channel.basic_nack(method.delivery_tag, requeue=True)
|
||||
time.sleep(2)
|
||||
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
|
||||
logger.warning("Worker reconnecting after %s", type(exc).__name__)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
if broker is not None and broker.is_open:
|
||||
broker.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Small checks for the worker's deterministic image and payload boundary."""
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from src.image_variants import InvalidImage, render
|
||||
from src.worker import parse_message
|
||||
|
||||
|
||||
def encoded(image, format="PNG", **options):
|
||||
stream = BytesIO()
|
||||
image.save(stream, format=format, **options)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
class ImagePipelineTest(unittest.TestCase):
|
||||
def test_contract_sizes_original_and_content_types(self):
|
||||
original = encoded(Image.new("RGB", (320, 200), "red"))
|
||||
result = render(original, "image/png")
|
||||
self.assertEqual(result["original"]["data"], original)
|
||||
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160))
|
||||
self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128))
|
||||
self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200))
|
||||
self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")],
|
||||
["image/jpeg", "image/jpeg", "image/webp"])
|
||||
|
||||
def test_alpha_is_white_and_small_image_is_not_upscaled(self):
|
||||
original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0)))
|
||||
result = render(original, "image/png")
|
||||
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12))
|
||||
with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb:
|
||||
self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0))))
|
||||
|
||||
def test_exif_orientation_and_center_crop(self):
|
||||
image = Image.new("RGB", (30, 10), "green")
|
||||
image.paste("red", (0, 0, 10, 10))
|
||||
image.paste("blue", (20, 0, 30, 10))
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
original = encoded(image, "JPEG", exif=exif)
|
||||
result = render(original, "image/jpeg")
|
||||
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30))
|
||||
with Image.open(BytesIO(result["crop"]["data"])) as crop:
|
||||
red, green, blue = crop.getpixel((64, 64))
|
||||
self.assertGreater(green, red)
|
||||
self.assertGreater(green, blue)
|
||||
|
||||
def test_rejects_wrong_type_and_corrupt_file(self):
|
||||
original = encoded(Image.new("RGB", (2, 2), "black"))
|
||||
with self.assertRaises(InvalidImage):
|
||||
render(original, "image/jpeg")
|
||||
with self.assertRaises(InvalidImage):
|
||||
render(b"broken", "image/png")
|
||||
|
||||
def test_payload_contains_only_versioned_job_identifiers(self):
|
||||
body = {
|
||||
"schema_version": 1,
|
||||
"job_id": "11111111-1111-4111-8111-111111111111",
|
||||
"type": "attachment.process.v1",
|
||||
"attachment_id": "22222222-2222-4222-8222-222222222222",
|
||||
"pipeline_version": 1,
|
||||
"attempt": 1,
|
||||
"created_at": "2026-09-01T12:00:00Z",
|
||||
}
|
||||
self.assertEqual(parse_message(json.dumps(body).encode()), body)
|
||||
with self.assertRaises(ValueError):
|
||||
parse_message(json.dumps({**body, "staging_key": "private"}).encode())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check every public variant against committed metadata and original bytes."""
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Lab5 ready variants")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
ready = client.wait_ready(bob, attachment)
|
||||
variants = {item["name"]: item for item in ready["variants"]}
|
||||
check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set")
|
||||
expected = {
|
||||
"thumbnail": ("image/jpeg", 256, 160),
|
||||
"crop": ("image/jpeg", 128, 128),
|
||||
"optimized": ("image/webp", 320, 200),
|
||||
}
|
||||
for name, (content_type, width, height) in expected.items():
|
||||
variant = variants[name]
|
||||
check((variant["content_type"], variant["width"], variant["height"]) ==
|
||||
(content_type, width, height), f"wrong {name} metadata")
|
||||
response = client.call(
|
||||
"GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob
|
||||
)
|
||||
check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME")
|
||||
check(len(response["bytes"]) == variant["size_bytes"] and
|
||||
hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"],
|
||||
f"wrong {name} bytes")
|
||||
client.call(
|
||||
"GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}",
|
||||
404, actor=eve,
|
||||
)
|
||||
downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)
|
||||
check(downloaded["bytes"] == original, "original bytes changed")
|
||||
print("PASS: all three variants match metadata; original preserved; outsider denied")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user