Files
mtusi2026lab5/app/imaging.py
T

225 lines
8.5 KiB
Python

"""Versioned image pipeline, executed in an isolated child process with time and memory limits.
Pipeline v1 (the main course version):
thumbnail JPEG q=85, fit into 256x256, no upscaling, floor rounding (min 1 px), EXIF orientation applied,
alpha composited on white -> 320x200 gives 256x160
crop JPEG q=85, centred square 128x128 (small images are upscaled), same orientation/alpha rules
optimized WebP q=80 method=4, fit into 1280x1280 without upscaling, alpha kept
Metadata: no EXIF/XMP/ICC/comments are written (Pillow writes them only when passed explicitly), the
orientation is baked into the pixels instead. Encoding is deterministic for a given Pillow build, so a retry of
the same (attachment, version) produces byte-identical objects at the same keys.
Pipeline v2 exists to demonstrate versioning (track 5): different quality settings, separate S3 keys.
"""
from __future__ import annotations
import hashlib
import io
import multiprocessing
import os
import resource
import time
import warnings
from typing import Dict, List, Tuple
from PIL import Image, ImageOps
from app import config
Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS
Image.LOAD_TRUNCATED_IMAGES = False
PIPELINES: Dict[int, Dict[str, dict]] = {
1: {
"thumbnail": {"box": 256, "format": "JPEG", "quality": 85},
"crop": {"size": 128, "format": "JPEG", "quality": 85},
"optimized": {"box": 1280, "format": "WEBP", "quality": 80, "method": 4},
},
2: {
"thumbnail": {"box": 256, "format": "JPEG", "quality": 90},
"crop": {"size": 128, "format": "JPEG", "quality": 90},
"optimized": {"box": 1280, "format": "WEBP", "quality": 72, "method": 6},
},
}
MIME = {"JPEG": "image/jpeg", "WEBP": "image/webp", "PNG": "image/png"}
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
class ImageRejected(Exception):
"""Permanent: retrying the same bytes cannot succeed."""
def __init__(self, code: str):
super().__init__(code)
self.code = code
class TransientFailure(Exception):
def __init__(self, code: str):
super().__init__(code)
self.code = code
def fit_size(width: int, height: int, box: int) -> Tuple[int, int]:
"""Fit into box x box keeping proportions, never upscale, round down, at least 1 px."""
if width <= box and height <= box:
return width, height
if width >= height:
return box, max(1, height * box // width)
return max(1, width * box // height), box
def _flatten(im: Image.Image) -> Image.Image:
"""RGB for JPEG: alpha (RGBA/LA/P+transparency) composited on white."""
if im.mode in ("I", "I;16", "I;16B", "I;16L", "F"):
im = im.convert("I").point(lambda v: v * (1 / 256)).convert("L")
if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info):
rgba = im.convert("RGBA")
out = Image.new("RGB", rgba.size, (255, 255, 255))
out.paste(rgba, mask=rgba.getchannel("A"))
return out
return im.convert("RGB")
def _with_alpha(im: Image.Image) -> Image.Image:
if im.mode in ("I", "I;16", "I;16B", "I;16L", "F"):
im = im.convert("I").point(lambda v: v * (1 / 256)).convert("L")
if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info):
return im.convert("RGBA")
return im.convert("RGB")
def _encode(im: Image.Image, spec: dict) -> bytes:
buf = io.BytesIO()
if spec["format"] == "JPEG":
im.save(buf, "JPEG", quality=spec["quality"], optimize=True, progressive=False, subsampling="4:2:0")
else:
im.save(buf, "WEBP", quality=spec["quality"], method=spec["method"], exact=False)
return buf.getvalue()
def transform(src_path: str, content_type: str, out_dir: str, version: int) -> List[dict]:
"""Decode once, produce every variant of `version`. Runs inside the limited child process."""
if version not in PIPELINES:
raise ImageRejected("unsupported_pipeline")
spec = PIPELINES[version]
fmt = FORMATS.get(content_type)
if fmt is None:
raise ImageRejected("invalid_image")
try:
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(src_path, formats=[fmt]) as im:
width, height = im.size
if width < 1 or height < 1:
raise ImageRejected("invalid_image")
if width * height > config.MAX_IMAGE_PIXELS:
raise ImageRejected("image_too_large")
orientation = im.getexif().get(0x0112, 1)
if orientation in (5, 6, 7, 8):
width, height = height, width
largest = spec["optimized"]["box"]
if fmt == "JPEG":
# DCT-domain downscale while decoding: memory ~ pixels/4..1/64 for big photos.
im.draft("RGB", (min(width, largest), min(height, largest)))
im.load()
im = ImageOps.exif_transpose(im)
except ImageRejected:
raise
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise ImageRejected("image_too_large") from None
except MemoryError:
raise ImageRejected("image_too_large") from None
except (OSError, SyntaxError, ValueError, EOFError, TypeError, IndexError):
raise ImageRejected("invalid_image") from None
flat = _flatten(im)
outputs = []
t = spec["thumbnail"]
tw, th = fit_size(width, height, t["box"])
outputs.append(("thumbnail", flat.resize((tw, th), Image.LANCZOS, reducing_gap=3.0), t))
c = spec["crop"]
outputs.append(("crop", ImageOps.fit(flat, (c["size"], c["size"]), Image.LANCZOS, centering=(0.5, 0.5)), c))
o = spec["optimized"]
ow, oh = fit_size(width, height, o["box"])
rich = _with_alpha(im)
outputs.append(("optimized", rich if rich.size == (ow, oh) else rich.resize((ow, oh), Image.LANCZOS, reducing_gap=3.0), o))
result = []
for name, img, s in outputs:
data = _encode(img, s)
path = os.path.join(out_dir, name)
with open(path, "wb") as fh:
fh.write(data)
result.append({
"name": name,
"content_type": MIME[s["format"]],
"path": path,
"size_bytes": len(data),
"width": img.size[0],
"height": img.size[1],
"sha256": hashlib.sha256(data).hexdigest(),
})
return result
def _child(conn, src_path: str, content_type: str, out_dir: str, version: int, memory_mb: int) -> None:
limit = memory_mb * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
try:
conn.send(("ok", transform(src_path, content_type, out_dir, version)))
except ImageRejected as exc:
conn.send(("rejected", exc.code))
except MemoryError:
conn.send(("rejected", "image_too_large"))
except Exception as exc: # unexpected: transient, bounded by max attempts
conn.send(("transient", "processing_failed:" + type(exc).__name__))
finally:
conn.close()
_ctx = None
def _context():
global _ctx
if _ctx is None:
# forkserver: children are forked from a clean single-threaded server (not from the asyncio/boto3 parent).
_ctx = multiprocessing.get_context("forkserver")
_ctx.set_forkserver_preload(["app.imaging"])
return _ctx
def run_isolated(src_path: str, content_type: str, out_dir: str, version: int,
timeout: float, memory_mb: int) -> List[dict]:
"""Blocking; call from a thread. Kills the child on timeout (hung decoder) and reports how it ended."""
ctx = _context()
parent, child = ctx.Pipe(duplex=False)
proc = ctx.Process(target=_child, args=(child, src_path, content_type, out_dir, version, memory_mb), daemon=True)
proc.start()
child.close()
try:
deadline = time.monotonic() + timeout
if not parent.poll(max(0.0, deadline - time.monotonic())):
proc.kill()
proc.join(5)
raise TransientFailure("processing_timeout")
try:
kind, value = parent.recv()
except EOFError:
proc.join(5)
# killed by the kernel/cgroup (e.g. OOM) or crashed in native code
raise TransientFailure("worker_process_died:%s" % proc.exitcode) from None
proc.join(5)
finally:
parent.close()
if proc.is_alive():
proc.kill()
if kind == "ok":
return value
if kind == "rejected":
raise ImageRejected(value)
raise TransientFailure(value)