Process versioned image variants with leased worker (U3)

This commit is contained in:
Самохвалов Андрей Ремович
2026-10-02 14:39:08 +03:00
parent b23ecc111f
commit 03899d790b
5 changed files with 492 additions and 0 deletions
+72
View File
@@ -0,0 +1,72 @@
"""Authorize metadata and stream versioned variants from the private bucket."""
import logging
from uuid import UUID
from fastapi import HTTPException
from fastapi.responses import StreamingResponse
from .attachments import BUCKET, attachment_view, storage
from .store import connection
logger = logging.getLogger(__name__)
def metadata(actor_id: str, attachment_id: UUID) -> dict:
with connection() as conn:
row = conn.execute(
"SELECT a.* FROM attachments a "
"JOIN memberships m ON m.chat_id=a.chat_id "
"WHERE a.id=%s AND m.user_id=%s",
(attachment_id, UUID(actor_id)),
).fetchone()
if row is None:
raise HTTPException(404)
return row
def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse:
row = metadata(actor_id, attachment_id)
if row["status"] != "ready":
raise HTTPException(409)
with connection() as conn:
selected = conn.execute(
"SELECT object_key,content_type,size_bytes FROM attachment_variants "
"WHERE attachment_id=%s AND pipeline_version=%s AND name=%s",
(attachment_id, row["pipeline_version"], variant),
).fetchone()
if selected is None:
raise HTTPException(404)
try:
obj = storage().get_object(BUCKET, selected["object_key"])
except Exception as exc:
logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__)
raise HTTPException(503) from None
def chunks():
try:
yield from obj.stream(64 * 1024)
finally:
obj.close()
obj.release_conn()
return StreamingResponse(
chunks(), media_type=selected["content_type"],
headers={"Content-Length": str(selected["size_bytes"])}
)
def get_view(actor_id: str, attachment_id: UUID) -> dict:
row = metadata(actor_id, attachment_id)
view = attachment_view(row)
if row["status"] == "ready":
with connection() as conn:
variants = conn.execute(
"SELECT name,content_type,size_bytes,width,height,sha256 "
"FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s "
"AND name <> 'original' ORDER BY name",
(attachment_id, row["pipeline_version"]),
).fetchall()
view["variants"] = [dict(variant) for variant in variants]
return view
+83
View File
@@ -0,0 +1,83 @@
"""Deterministic, bounded image variants for one accepted PNG or JPEG."""
import hashlib
from io import BytesIO
from PIL import Image, ImageOps, UnidentifiedImageError
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
class InvalidImage(ValueError):
pass
def _fit(image: Image.Image, bound: int) -> Image.Image:
factor = min(1, bound / max(image.size))
size = (max(1, int(image.width * factor)), max(1, int(image.height * factor)))
return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image
def _jpeg(image: Image.Image) -> bytes:
output = BytesIO()
image.save(output, format="JPEG", quality=85, optimize=True)
return output.getvalue()
def _variant(data: bytes, content_type: str, image: Image.Image) -> dict:
return {
"data": data,
"content_type": content_type,
"size_bytes": len(data),
"width": image.width,
"height": image.height,
"sha256": hashlib.sha256(data).hexdigest(),
}
def render(data: bytes, content_type: str) -> dict[str, dict]:
"""Preserve original bytes; orient, flatten alpha, strip metadata on derivatives."""
if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}:
raise InvalidImage("invalid original")
try:
with Image.open(BytesIO(data)) as source:
if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type:
raise InvalidImage("wrong image type")
if source.width * source.height > MAX_PIXELS:
raise InvalidImage("too many pixels")
source.load()
original_size = source.size
oriented = ImageOps.exif_transpose(source)
if oriented.mode in {"RGBA", "LA", "P"}:
rgba = oriented.convert("RGBA")
image = Image.new("RGB", rgba.size, "white")
image.paste(rgba, mask=rgba.getchannel("A"))
else:
image = oriented.convert("RGB")
thumbnail = _fit(image, 256)
side = min(image.size)
left = (image.width - side) // 2
top = (image.height - side) // 2
crop = image.resize(
(128, 128), Image.Resampling.LANCZOS,
box=(left, top, left + side, top + side),
)
optimized = _fit(image, 1280)
webp = BytesIO()
optimized.save(webp, format="WEBP", quality=82, method=4)
return {
"original": {
"data": data, "content_type": content_type,
"size_bytes": len(data), "width": original_size[0], "height": original_size[1],
"sha256": hashlib.sha256(data).hexdigest(),
},
"thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail),
"crop": _variant(_jpeg(crop), "image/jpeg", crop),
"optimized": _variant(webp.getvalue(), "image/webp", optimized),
}
except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc:
raise InvalidImage("image decoding failed") from exc
+216
View File
@@ -0,0 +1,216 @@
"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects."""
import hashlib
import json
import logging
import os
import random
import re
import signal
import time
from contextlib import contextmanager
from datetime import datetime
from io import BytesIO
from pathlib import Path
from uuid import UUID
import pika
import psycopg
from minio.error import S3Error
from psycopg_pool import PoolTimeout
from .attachments import BUCKET, storage
from .db import pool
from .image_variants import InvalidImage, MAX_BYTES, render
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
logger = logging.getLogger(__name__)
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
def parse_message(body: bytes) -> dict:
if len(body) > 4096:
raise ValueError("oversized job message")
item = json.loads(body)
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
if (not isinstance(item, dict) or not required <= item.keys() or
item.keys() - required - {"traceparent", "tracestate"} or
type(item["schema_version"]) is not int or item["schema_version"] != 1 or
item["type"] != "attachment.process.v1" or
type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or
type(item["attempt"]) is not int or item["attempt"] < 1 or
not isinstance(item["job_id"], str) or
not isinstance(item["attachment_id"], str) or
not isinstance(item["created_at"], str) or
("traceparent" in item and
(not isinstance(item["traceparent"], str) or
re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or
("tracestate" in item and
(not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))):
raise ValueError("invalid job message")
UUID(item["job_id"])
UUID(item["attachment_id"])
created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00"))
if created_at.utcoffset() is None:
raise ValueError("job timestamp lacks timezone")
return item
def _source_bytes(job) -> bytes:
if not 0 < job["size_bytes"] <= MAX_BYTES:
raise InvalidImage("invalid accepted size")
# Reprocessing uses the byte-identical final original, so ready staging
# can be reclaimed without making a future pipeline version impossible.
key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"]
response = storage().get_object(BUCKET, key)
try:
chunks = []
size = 0
digest = hashlib.sha256()
for chunk in response.stream(64 * 1024):
size += len(chunk)
if size > job["size_bytes"]:
raise InvalidImage("source exceeds accepted size")
chunks.append(chunk)
digest.update(chunk)
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
raise InvalidImage("source checksum mismatch")
return b"".join(chunks)
finally:
response.close()
response.release_conn()
def _write_variants(job, variants: dict) -> dict:
extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
stored = {}
for name, variant in variants.items():
key = (job["object_key"] if name == "original" else
f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}")
data = variant["data"]
storage().put_object(
BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"]
)
stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"}
stored[name]["object_key"] = key
return stored
def _delay(attempt: int) -> float:
base = min(30, 2 ** min(attempt - 1, 5))
return base + random.uniform(0, base / 2)
@contextmanager
def _deadline(seconds: int):
def expired(_signal, _frame):
raise TimeoutError("image job exceeded worker deadline")
previous = signal.signal(signal.SIGALRM, expired)
signal.setitimer(signal.ITIMER_REAL, seconds)
try:
yield
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
signal.signal(signal.SIGALRM, previous)
def process_message(item: dict) -> str:
with pool().connection() as conn:
job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS)
if job is None:
return "duplicate_or_stale"
token = job["lease_token"]
try:
with _deadline(LEASE_SECONDS):
data = _source_bytes(job)
variants = render(data, job["content_type"])
stored = _write_variants(job, variants)
with pool().connection() as conn:
current = complete_job(conn, job["id"], token, stored)
if current and job["staging_key"]:
try:
storage().remove_object(BUCKET, job["staging_key"])
except Exception as exc:
# Periodic cleanup recovers this post-COMMIT gap.
logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__)
return "ready"
except psycopg.Error:
raise
except Exception as exc:
if isinstance(exc, InvalidImage):
code, permanent = "invalid_image", True
elif isinstance(exc, TimeoutError):
code, permanent = "worker_timeout", False
elif isinstance(exc, S3Error) or isinstance(exc, OSError):
code, permanent = "storage_unavailable", False
elif isinstance(exc, ValueError):
code, permanent = "processing_failed", True
else:
code, permanent = "processing_failed", False
logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code)
with pool().connection() as conn:
try:
return finish_attempt(
conn, job["id"], token, max_attempts=MAX_ATTEMPTS,
delay_seconds=_delay(job["attempt"]), error_code=code,
permanent=permanent,
)
except ValueError:
# A lease recovery already owns the next attempt.
return "stale_lease"
def main() -> None:
logging.basicConfig(level=logging.INFO)
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
raise SystemExit("invalid worker retry settings")
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
params = pika.URLParameters(url)
params.heartbeat = max(120, LEASE_SECONDS * 2)
params.socket_timeout = 5
params.blocked_connection_timeout = 10
while True:
broker = None
try:
broker = pika.BlockingConnection(params)
channel = broker.channel()
channel.queue_declare(queue=QUEUE, passive=True)
channel.basic_qos(prefetch_count=1)
last_recovery = 0.0
while broker.is_open:
if time.monotonic() - last_recovery >= 5:
with pool().connection() as conn:
recover_expired(conn, max_attempts=MAX_ATTEMPTS)
last_recovery = time.monotonic()
method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False)
if method is None:
broker.process_data_events(time_limit=1)
continue
try:
item = parse_message(body)
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
logger.warning("Rejecting invalid job message")
channel.basic_reject(method.delivery_tag, requeue=False)
continue
try:
result = process_message(item)
channel.basic_ack(method.delivery_tag)
logger.info("Job %s: %s", item["job_id"], result)
except (psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__)
channel.basic_nack(method.delivery_tag, requeue=True)
time.sleep(2)
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Worker reconnecting after %s", type(exc).__name__)
time.sleep(2)
finally:
if broker is not None and broker.is_open:
broker.close()
if __name__ == "__main__":
main()
+74
View File
@@ -0,0 +1,74 @@
"""Small checks for the worker's deterministic image and payload boundary."""
import json
import unittest
from io import BytesIO
from PIL import Image
from src.image_variants import InvalidImage, render
from src.worker import parse_message
def encoded(image, format="PNG", **options):
stream = BytesIO()
image.save(stream, format=format, **options)
return stream.getvalue()
class ImagePipelineTest(unittest.TestCase):
def test_contract_sizes_original_and_content_types(self):
original = encoded(Image.new("RGB", (320, 200), "red"))
result = render(original, "image/png")
self.assertEqual(result["original"]["data"], original)
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160))
self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128))
self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200))
self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")],
["image/jpeg", "image/jpeg", "image/webp"])
def test_alpha_is_white_and_small_image_is_not_upscaled(self):
original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0)))
result = render(original, "image/png")
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12))
with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb:
self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0))))
def test_exif_orientation_and_center_crop(self):
image = Image.new("RGB", (30, 10), "green")
image.paste("red", (0, 0, 10, 10))
image.paste("blue", (20, 0, 30, 10))
exif = Image.Exif()
exif[274] = 6
original = encoded(image, "JPEG", exif=exif)
result = render(original, "image/jpeg")
self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30))
with Image.open(BytesIO(result["crop"]["data"])) as crop:
red, green, blue = crop.getpixel((64, 64))
self.assertGreater(green, red)
self.assertGreater(green, blue)
def test_rejects_wrong_type_and_corrupt_file(self):
original = encoded(Image.new("RGB", (2, 2), "black"))
with self.assertRaises(InvalidImage):
render(original, "image/jpeg")
with self.assertRaises(InvalidImage):
render(b"broken", "image/png")
def test_payload_contains_only_versioned_job_identifiers(self):
body = {
"schema_version": 1,
"job_id": "11111111-1111-4111-8111-111111111111",
"type": "attachment.process.v1",
"attachment_id": "22222222-2222-4222-8222-222222222222",
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-09-01T12:00:00Z",
}
self.assertEqual(parse_message(json.dumps(body).encode()), body)
with self.assertRaises(ValueError):
parse_message(json.dumps({**body, "staging_key": "private"}).encode())
if __name__ == "__main__":
unittest.main()
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Check every public variant against committed metadata and original bytes."""
import hashlib
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, arguments, check, run # noqa: E402
def main():
args = arguments("Lab5 ready variants")
client = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
ready = client.wait_ready(bob, attachment)
variants = {item["name"]: item for item in ready["variants"]}
check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set")
expected = {
"thumbnail": ("image/jpeg", 256, 160),
"crop": ("image/jpeg", 128, 128),
"optimized": ("image/webp", 320, 200),
}
for name, (content_type, width, height) in expected.items():
variant = variants[name]
check((variant["content_type"], variant["width"], variant["height"]) ==
(content_type, width, height), f"wrong {name} metadata")
response = client.call(
"GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob
)
check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME")
check(len(response["bytes"]) == variant["size_bytes"] and
hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"],
f"wrong {name} bytes")
client.call(
"GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}",
404, actor=eve,
)
downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)
check(downloaded["bytes"] == original, "original bytes changed")
print("PASS: all three variants match metadata; original preserved; outsider denied")
if __name__ == "__main__":
run(main)