From 03899d790bb8e7047d2f2fccc6bfa10957b0208a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:39:08 +0300 Subject: [PATCH] Process versioned image variants with leased worker (U3) --- src/attachment_read.py | 72 +++++++++ src/image_variants.py | 83 ++++++++++ src/worker.py | 216 +++++++++++++++++++++++++++ student-tests/test_image_pipeline.py | 74 +++++++++ student-tests/test_variants_api.py | 47 ++++++ 5 files changed, 492 insertions(+) create mode 100644 src/attachment_read.py create mode 100644 src/image_variants.py create mode 100644 src/worker.py create mode 100644 student-tests/test_image_pipeline.py create mode 100644 student-tests/test_variants_api.py diff --git a/src/attachment_read.py b/src/attachment_read.py new file mode 100644 index 00000000..34f91a69 --- /dev/null +++ b/src/attachment_read.py @@ -0,0 +1,72 @@ +"""Authorize metadata and stream versioned variants from the private bucket.""" + +import logging +from uuid import UUID + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from .attachments import BUCKET, attachment_view, storage +from .store import connection + + +logger = logging.getLogger(__name__) + + +def metadata(actor_id: str, attachment_id: UUID) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a " + "JOIN memberships m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (attachment_id, UUID(actor_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return row + + +def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse: + row = metadata(actor_id, attachment_id) + if row["status"] != "ready": + raise HTTPException(409) + with connection() as conn: + selected = conn.execute( + "SELECT object_key,content_type,size_bytes FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + if selected is None: + raise HTTPException(404) + try: + obj = storage().get_object(BUCKET, selected["object_key"]) + except Exception as exc: + logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + + def chunks(): + try: + yield from obj.stream(64 * 1024) + finally: + obj.close() + obj.release_conn() + + return StreamingResponse( + chunks(), media_type=selected["content_type"], + headers={"Content-Length": str(selected["size_bytes"])} + ) + + +def get_view(actor_id: str, attachment_id: UUID) -> dict: + row = metadata(actor_id, attachment_id) + view = attachment_view(row) + if row["status"] == "ready": + with connection() as conn: + variants = conn.execute( + "SELECT name,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s " + "AND name <> 'original' ORDER BY name", + (attachment_id, row["pipeline_version"]), + ).fetchall() + view["variants"] = [dict(variant) for variant in variants] + return view diff --git a/src/image_variants.py b/src/image_variants.py new file mode 100644 index 00000000..72a8cd33 --- /dev/null +++ b/src/image_variants.py @@ -0,0 +1,83 @@ +"""Deterministic, bounded image variants for one accepted PNG or JPEG.""" + +import hashlib +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class InvalidImage(ValueError): + pass + + +def _fit(image: Image.Image, bound: int) -> Image.Image: + factor = min(1, bound / max(image.size)) + size = (max(1, int(image.width * factor)), max(1, int(image.height * factor))) + return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image + + +def _jpeg(image: Image.Image) -> bytes: + output = BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue() + + +def _variant(data: bytes, content_type: str, image: Image.Image) -> dict: + return { + "data": data, + "content_type": content_type, + "size_bytes": len(data), + "width": image.width, + "height": image.height, + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def render(data: bytes, content_type: str) -> dict[str, dict]: + """Preserve original bytes; orient, flatten alpha, strip metadata on derivatives.""" + if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}: + raise InvalidImage("invalid original") + try: + with Image.open(BytesIO(data)) as source: + if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type: + raise InvalidImage("wrong image type") + if source.width * source.height > MAX_PIXELS: + raise InvalidImage("too many pixels") + source.load() + original_size = source.size + oriented = ImageOps.exif_transpose(source) + if oriented.mode in {"RGBA", "LA", "P"}: + rgba = oriented.convert("RGBA") + image = Image.new("RGB", rgba.size, "white") + image.paste(rgba, mask=rgba.getchannel("A")) + else: + image = oriented.convert("RGB") + + thumbnail = _fit(image, 256) + side = min(image.size) + left = (image.width - side) // 2 + top = (image.height - side) // 2 + crop = image.resize( + (128, 128), Image.Resampling.LANCZOS, + box=(left, top, left + side, top + side), + ) + optimized = _fit(image, 1280) + webp = BytesIO() + optimized.save(webp, format="WEBP", quality=82, method=4) + return { + "original": { + "data": data, "content_type": content_type, + "size_bytes": len(data), "width": original_size[0], "height": original_size[1], + "sha256": hashlib.sha256(data).hexdigest(), + }, + "thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail), + "crop": _variant(_jpeg(crop), "image/jpeg", crop), + "optimized": _variant(webp.getvalue(), "image/webp", optimized), + } + except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc: + raise InvalidImage("image decoding failed") from exc diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..b0c2f7f3 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,216 @@ +"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects.""" + +import hashlib +import json +import logging +import os +import random +import re +import signal +import time +from contextlib import contextmanager +from datetime import datetime +from io import BytesIO +from pathlib import Path +from uuid import UUID + +import pika +import psycopg +from minio.error import S3Error +from psycopg_pool import PoolTimeout + +from .attachments import BUCKET, storage +from .db import pool +from .image_variants import InvalidImage, MAX_BYTES, render +from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired + + +logger = logging.getLogger(__name__) +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) + + +def parse_message(body: bytes) -> dict: + if len(body) > 4096: + raise ValueError("oversized job message") + item = json.loads(body) + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if (not isinstance(item, dict) or not required <= item.keys() or + item.keys() - required - {"traceparent", "tracestate"} or + type(item["schema_version"]) is not int or item["schema_version"] != 1 or + item["type"] != "attachment.process.v1" or + type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or + type(item["attempt"]) is not int or item["attempt"] < 1 or + not isinstance(item["job_id"], str) or + not isinstance(item["attachment_id"], str) or + not isinstance(item["created_at"], str) or + ("traceparent" in item and + (not isinstance(item["traceparent"], str) or + re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or + ("tracestate" in item and + (not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))): + raise ValueError("invalid job message") + UUID(item["job_id"]) + UUID(item["attachment_id"]) + created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00")) + if created_at.utcoffset() is None: + raise ValueError("job timestamp lacks timezone") + return item + + +def _source_bytes(job) -> bytes: + if not 0 < job["size_bytes"] <= MAX_BYTES: + raise InvalidImage("invalid accepted size") + # Reprocessing uses the byte-identical final original, so ready staging + # can be reclaimed without making a future pipeline version impossible. + key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"] + response = storage().get_object(BUCKET, key) + try: + chunks = [] + size = 0 + digest = hashlib.sha256() + for chunk in response.stream(64 * 1024): + size += len(chunk) + if size > job["size_bytes"]: + raise InvalidImage("source exceeds accepted size") + chunks.append(chunk) + digest.update(chunk) + if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]: + raise InvalidImage("source checksum mismatch") + return b"".join(chunks) + finally: + response.close() + response.release_conn() + + +def _write_variants(job, variants: dict) -> dict: + extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + stored = {} + for name, variant in variants.items(): + key = (job["object_key"] if name == "original" else + f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}") + data = variant["data"] + storage().put_object( + BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"] + ) + stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"} + stored[name]["object_key"] = key + return stored + + +def _delay(attempt: int) -> float: + base = min(30, 2 ** min(attempt - 1, 5)) + return base + random.uniform(0, base / 2) + + +@contextmanager +def _deadline(seconds: int): + def expired(_signal, _frame): + raise TimeoutError("image job exceeded worker deadline") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def process_message(item: dict) -> str: + with pool().connection() as conn: + job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS) + if job is None: + return "duplicate_or_stale" + token = job["lease_token"] + try: + with _deadline(LEASE_SECONDS): + data = _source_bytes(job) + variants = render(data, job["content_type"]) + stored = _write_variants(job, variants) + with pool().connection() as conn: + current = complete_job(conn, job["id"], token, stored) + if current and job["staging_key"]: + try: + storage().remove_object(BUCKET, job["staging_key"]) + except Exception as exc: + # Periodic cleanup recovers this post-COMMIT gap. + logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__) + return "ready" + except psycopg.Error: + raise + except Exception as exc: + if isinstance(exc, InvalidImage): + code, permanent = "invalid_image", True + elif isinstance(exc, TimeoutError): + code, permanent = "worker_timeout", False + elif isinstance(exc, S3Error) or isinstance(exc, OSError): + code, permanent = "storage_unavailable", False + elif isinstance(exc, ValueError): + code, permanent = "processing_failed", True + else: + code, permanent = "processing_failed", False + logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code) + with pool().connection() as conn: + try: + return finish_attempt( + conn, job["id"], token, max_attempts=MAX_ATTEMPTS, + delay_seconds=_delay(job["attempt"]), error_code=code, + permanent=permanent, + ) + except ValueError: + # A lease recovery already owns the next attempt. + return "stale_lease" + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: + raise SystemExit("invalid worker retry settings") + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = max(120, LEASE_SECONDS * 2) + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.basic_qos(prefetch_count=1) + last_recovery = 0.0 + while broker.is_open: + if time.monotonic() - last_recovery >= 5: + with pool().connection() as conn: + recover_expired(conn, max_attempts=MAX_ATTEMPTS) + last_recovery = time.monotonic() + method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False) + if method is None: + broker.process_data_events(time_limit=1) + continue + try: + item = parse_message(body) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + logger.warning("Rejecting invalid job message") + channel.basic_reject(method.delivery_tag, requeue=False) + continue + try: + result = process_message(item) + channel.basic_ack(method.delivery_tag) + logger.info("Job %s: %s", item["job_id"], result) + except (psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__) + channel.basic_nack(method.delivery_tag, requeue=True) + time.sleep(2) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Worker reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + broker.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 00000000..404b8633 --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,74 @@ +"""Small checks for the worker's deterministic image and payload boundary.""" + +import json +import unittest +from io import BytesIO + +from PIL import Image + +from src.image_variants import InvalidImage, render +from src.worker import parse_message + + +def encoded(image, format="PNG", **options): + stream = BytesIO() + image.save(stream, format=format, **options) + return stream.getvalue() + + +class ImagePipelineTest(unittest.TestCase): + def test_contract_sizes_original_and_content_types(self): + original = encoded(Image.new("RGB", (320, 200), "red")) + result = render(original, "image/png") + self.assertEqual(result["original"]["data"], original) + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160)) + self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128)) + self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200)) + self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")], + ["image/jpeg", "image/jpeg", "image/webp"]) + + def test_alpha_is_white_and_small_image_is_not_upscaled(self): + original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0))) + result = render(original, "image/png") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12)) + with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb: + self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0)))) + + def test_exif_orientation_and_center_crop(self): + image = Image.new("RGB", (30, 10), "green") + image.paste("red", (0, 0, 10, 10)) + image.paste("blue", (20, 0, 30, 10)) + exif = Image.Exif() + exif[274] = 6 + original = encoded(image, "JPEG", exif=exif) + result = render(original, "image/jpeg") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30)) + with Image.open(BytesIO(result["crop"]["data"])) as crop: + red, green, blue = crop.getpixel((64, 64)) + self.assertGreater(green, red) + self.assertGreater(green, blue) + + def test_rejects_wrong_type_and_corrupt_file(self): + original = encoded(Image.new("RGB", (2, 2), "black")) + with self.assertRaises(InvalidImage): + render(original, "image/jpeg") + with self.assertRaises(InvalidImage): + render(b"broken", "image/png") + + def test_payload_contains_only_versioned_job_identifiers(self): + body = { + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-09-01T12:00:00Z", + } + self.assertEqual(parse_message(json.dumps(body).encode()), body) + with self.assertRaises(ValueError): + parse_message(json.dumps({**body, "staging_key": "private"}).encode()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_variants_api.py b/student-tests/test_variants_api.py new file mode 100644 index 00000000..fd4aee87 --- /dev/null +++ b/student-tests/test_variants_api.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Check every public variant against committed metadata and original bytes.""" + +import hashlib +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 ready variants") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready = client.wait_ready(bob, attachment) + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set") + expected = { + "thumbnail": ("image/jpeg", 256, 160), + "crop": ("image/jpeg", 128, 128), + "optimized": ("image/webp", 320, 200), + } + for name, (content_type, width, height) in expected.items(): + variant = variants[name] + check((variant["content_type"], variant["width"], variant["height"]) == + (content_type, width, height), f"wrong {name} metadata") + response = client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob + ) + check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME") + check(len(response["bytes"]) == variant["size_bytes"] and + hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"], + f"wrong {name} bytes") + client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", + 404, actor=eve, + ) + downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(downloaded["bytes"] == original, "original bytes changed") + print("PASS: all three variants match metadata; original preserved; outsider denied") + + +if __name__ == "__main__": + run(main)