forked from mtusi2026/mtusi2026lab5
225 lines
8.5 KiB
Python
225 lines
8.5 KiB
Python
"""Versioned image pipeline, executed in an isolated child process with time and memory limits.
|
|
|
|
Pipeline v1 (the main course version):
|
|
thumbnail JPEG q=85, fit into 256x256, no upscaling, floor rounding (min 1 px), EXIF orientation applied,
|
|
alpha composited on white -> 320x200 gives 256x160
|
|
crop JPEG q=85, centred square 128x128 (small images are upscaled), same orientation/alpha rules
|
|
optimized WebP q=80 method=4, fit into 1280x1280 without upscaling, alpha kept
|
|
Metadata: no EXIF/XMP/ICC/comments are written (Pillow writes them only when passed explicitly), the
|
|
orientation is baked into the pixels instead. Encoding is deterministic for a given Pillow build, so a retry of
|
|
the same (attachment, version) produces byte-identical objects at the same keys.
|
|
|
|
Pipeline v2 exists to demonstrate versioning (track 5): different quality settings, separate S3 keys.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import io
|
|
import multiprocessing
|
|
import os
|
|
import resource
|
|
import time
|
|
import warnings
|
|
from typing import Dict, List, Tuple
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from app import config
|
|
|
|
Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS
|
|
Image.LOAD_TRUNCATED_IMAGES = False
|
|
|
|
PIPELINES: Dict[int, Dict[str, dict]] = {
|
|
1: {
|
|
"thumbnail": {"box": 256, "format": "JPEG", "quality": 85},
|
|
"crop": {"size": 128, "format": "JPEG", "quality": 85},
|
|
"optimized": {"box": 1280, "format": "WEBP", "quality": 80, "method": 4},
|
|
},
|
|
2: {
|
|
"thumbnail": {"box": 256, "format": "JPEG", "quality": 90},
|
|
"crop": {"size": 128, "format": "JPEG", "quality": 90},
|
|
"optimized": {"box": 1280, "format": "WEBP", "quality": 72, "method": 6},
|
|
},
|
|
}
|
|
MIME = {"JPEG": "image/jpeg", "WEBP": "image/webp", "PNG": "image/png"}
|
|
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
|
|
|
|
|
class ImageRejected(Exception):
|
|
"""Permanent: retrying the same bytes cannot succeed."""
|
|
|
|
def __init__(self, code: str):
|
|
super().__init__(code)
|
|
self.code = code
|
|
|
|
|
|
class TransientFailure(Exception):
|
|
def __init__(self, code: str):
|
|
super().__init__(code)
|
|
self.code = code
|
|
|
|
|
|
def fit_size(width: int, height: int, box: int) -> Tuple[int, int]:
|
|
"""Fit into box x box keeping proportions, never upscale, round down, at least 1 px."""
|
|
if width <= box and height <= box:
|
|
return width, height
|
|
if width >= height:
|
|
return box, max(1, height * box // width)
|
|
return max(1, width * box // height), box
|
|
|
|
|
|
def _flatten(im: Image.Image) -> Image.Image:
|
|
"""RGB for JPEG: alpha (RGBA/LA/P+transparency) composited on white."""
|
|
if im.mode in ("I", "I;16", "I;16B", "I;16L", "F"):
|
|
im = im.convert("I").point(lambda v: v * (1 / 256)).convert("L")
|
|
if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info):
|
|
rgba = im.convert("RGBA")
|
|
out = Image.new("RGB", rgba.size, (255, 255, 255))
|
|
out.paste(rgba, mask=rgba.getchannel("A"))
|
|
return out
|
|
return im.convert("RGB")
|
|
|
|
|
|
def _with_alpha(im: Image.Image) -> Image.Image:
|
|
if im.mode in ("I", "I;16", "I;16B", "I;16L", "F"):
|
|
im = im.convert("I").point(lambda v: v * (1 / 256)).convert("L")
|
|
if im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info):
|
|
return im.convert("RGBA")
|
|
return im.convert("RGB")
|
|
|
|
|
|
def _encode(im: Image.Image, spec: dict) -> bytes:
|
|
buf = io.BytesIO()
|
|
if spec["format"] == "JPEG":
|
|
im.save(buf, "JPEG", quality=spec["quality"], optimize=True, progressive=False, subsampling="4:2:0")
|
|
else:
|
|
im.save(buf, "WEBP", quality=spec["quality"], method=spec["method"], exact=False)
|
|
return buf.getvalue()
|
|
|
|
|
|
def transform(src_path: str, content_type: str, out_dir: str, version: int) -> List[dict]:
|
|
"""Decode once, produce every variant of `version`. Runs inside the limited child process."""
|
|
if version not in PIPELINES:
|
|
raise ImageRejected("unsupported_pipeline")
|
|
spec = PIPELINES[version]
|
|
fmt = FORMATS.get(content_type)
|
|
if fmt is None:
|
|
raise ImageRejected("invalid_image")
|
|
try:
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
|
with Image.open(src_path, formats=[fmt]) as im:
|
|
width, height = im.size
|
|
if width < 1 or height < 1:
|
|
raise ImageRejected("invalid_image")
|
|
if width * height > config.MAX_IMAGE_PIXELS:
|
|
raise ImageRejected("image_too_large")
|
|
orientation = im.getexif().get(0x0112, 1)
|
|
if orientation in (5, 6, 7, 8):
|
|
width, height = height, width
|
|
largest = spec["optimized"]["box"]
|
|
if fmt == "JPEG":
|
|
# DCT-domain downscale while decoding: memory ~ pixels/4..1/64 for big photos.
|
|
im.draft("RGB", (min(width, largest), min(height, largest)))
|
|
im.load()
|
|
im = ImageOps.exif_transpose(im)
|
|
except ImageRejected:
|
|
raise
|
|
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
|
raise ImageRejected("image_too_large") from None
|
|
except MemoryError:
|
|
raise ImageRejected("image_too_large") from None
|
|
except (OSError, SyntaxError, ValueError, EOFError, TypeError, IndexError):
|
|
raise ImageRejected("invalid_image") from None
|
|
|
|
flat = _flatten(im)
|
|
outputs = []
|
|
|
|
t = spec["thumbnail"]
|
|
tw, th = fit_size(width, height, t["box"])
|
|
outputs.append(("thumbnail", flat.resize((tw, th), Image.LANCZOS, reducing_gap=3.0), t))
|
|
|
|
c = spec["crop"]
|
|
outputs.append(("crop", ImageOps.fit(flat, (c["size"], c["size"]), Image.LANCZOS, centering=(0.5, 0.5)), c))
|
|
|
|
o = spec["optimized"]
|
|
ow, oh = fit_size(width, height, o["box"])
|
|
rich = _with_alpha(im)
|
|
outputs.append(("optimized", rich if rich.size == (ow, oh) else rich.resize((ow, oh), Image.LANCZOS, reducing_gap=3.0), o))
|
|
|
|
result = []
|
|
for name, img, s in outputs:
|
|
data = _encode(img, s)
|
|
path = os.path.join(out_dir, name)
|
|
with open(path, "wb") as fh:
|
|
fh.write(data)
|
|
result.append({
|
|
"name": name,
|
|
"content_type": MIME[s["format"]],
|
|
"path": path,
|
|
"size_bytes": len(data),
|
|
"width": img.size[0],
|
|
"height": img.size[1],
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
})
|
|
return result
|
|
|
|
|
|
def _child(conn, src_path: str, content_type: str, out_dir: str, version: int, memory_mb: int) -> None:
|
|
limit = memory_mb * 1024 * 1024
|
|
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
|
try:
|
|
conn.send(("ok", transform(src_path, content_type, out_dir, version)))
|
|
except ImageRejected as exc:
|
|
conn.send(("rejected", exc.code))
|
|
except MemoryError:
|
|
conn.send(("rejected", "image_too_large"))
|
|
except Exception as exc: # unexpected: transient, bounded by max attempts
|
|
conn.send(("transient", "processing_failed:" + type(exc).__name__))
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
_ctx = None
|
|
|
|
|
|
def _context():
|
|
global _ctx
|
|
if _ctx is None:
|
|
# forkserver: children are forked from a clean single-threaded server (not from the asyncio/boto3 parent).
|
|
_ctx = multiprocessing.get_context("forkserver")
|
|
_ctx.set_forkserver_preload(["app.imaging"])
|
|
return _ctx
|
|
|
|
|
|
def run_isolated(src_path: str, content_type: str, out_dir: str, version: int,
|
|
timeout: float, memory_mb: int) -> List[dict]:
|
|
"""Blocking; call from a thread. Kills the child on timeout (hung decoder) and reports how it ended."""
|
|
ctx = _context()
|
|
parent, child = ctx.Pipe(duplex=False)
|
|
proc = ctx.Process(target=_child, args=(child, src_path, content_type, out_dir, version, memory_mb), daemon=True)
|
|
proc.start()
|
|
child.close()
|
|
try:
|
|
deadline = time.monotonic() + timeout
|
|
if not parent.poll(max(0.0, deadline - time.monotonic())):
|
|
proc.kill()
|
|
proc.join(5)
|
|
raise TransientFailure("processing_timeout")
|
|
try:
|
|
kind, value = parent.recv()
|
|
except EOFError:
|
|
proc.join(5)
|
|
# killed by the kernel/cgroup (e.g. OOM) or crashed in native code
|
|
raise TransientFailure("worker_process_died:%s" % proc.exitcode) from None
|
|
proc.join(5)
|
|
finally:
|
|
parent.close()
|
|
if proc.is_alive():
|
|
proc.kill()
|
|
if kind == "ok":
|
|
return value
|
|
if kind == "rejected":
|
|
raise ImageRejected(value)
|
|
raise TransientFailure(value)
|