forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ, published_version, jobs с lease и токеном попытки, job_attempts с заранее известными ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs, post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник. pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения. Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
209 lines
9.0 KiB
Python
209 lines
9.0 KiB
Python
"""Variants of ONE image, built in a separate child process with hard limits (lab 5 worker).
|
|
|
|
parent (worker): await render(data, content_type, version, ...) -> list[Rendered]
|
|
child: python -m messenger.imaging --type image/png --pipeline 1 ...
|
|
stdin: original bytes; stdout: one JSON header line + variant bytes
|
|
|
|
Why a child process per image:
|
|
* memory: RLIMIT_AS (IMAGE_MAX_MEMORY_MB) caps the address space of the decoder; a picture that
|
|
needs more fails with MemoryError -> `image_too_large` (permanent), the worker keeps running;
|
|
* time: the parent kills the child at JOB_TIMEOUT_SECONDS (C code of Pillow cannot be
|
|
interrupted inside a thread) -> `processing_timeout` (transient, bounded by JOB_MAX_ATTEMPTS);
|
|
* pixels: the size is read from the header and checked against MAX_IMAGE_PIXELS BEFORE the
|
|
pixels are decoded (`Image.open` is lazy).
|
|
A crash of the decoder (signal) is a transient `processing_killed`: the next attempt may run
|
|
with less memory pressure; JOB_MAX_ATTEMPTS bounds it.
|
|
|
|
Geometry and encoder settings: pipeline.py. Alpha is handled in premultiplied form (RGBa) while
|
|
resizing, so transparent pixels do not bleed their colour into the edges; for JPEG the result is
|
|
then composited onto white.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import warnings
|
|
from dataclasses import dataclass
|
|
|
|
from .pipeline import CONTENT_TYPES, CROP_SIDE, OPTIMIZED_BOX, PIPELINES, THUMBNAIL_BOX, center_square, fit
|
|
|
|
EXIT_INVALID = 3 # the bytes are not a decodable PNG/JPEG
|
|
EXIT_TOO_LARGE = 4 # pixel limit or memory limit
|
|
HEADER_LIMIT = 4096
|
|
|
|
|
|
class ProcessingError(Exception):
|
|
"""Outcome of an attempt that did not produce variants. `permanent`: another attempt with the
|
|
same input cannot succeed (job -> failed at once); otherwise retry with backoff."""
|
|
|
|
def __init__(self, code: str, permanent: bool):
|
|
super().__init__(code)
|
|
self.code = code
|
|
self.permanent = permanent
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Rendered:
|
|
name: str
|
|
content_type: str
|
|
width: int
|
|
height: int
|
|
data: bytes
|
|
|
|
@property
|
|
def sha256(self) -> bytes:
|
|
return hashlib.sha256(self.data).digest()
|
|
|
|
|
|
# ------------------------------------------------------------------ parent side
|
|
async def render(data: bytes, content_type: str, pipeline_version: int, *, max_pixels: int, timeout: float,
|
|
memory_mb: int) -> list[Rendered]:
|
|
process = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-m", "messenger.imaging", "--type", content_type, "--pipeline", str(pipeline_version),
|
|
"--max-pixels", str(max_pixels), "--memory-mb", str(memory_mb),
|
|
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
|
try:
|
|
out, _ = await asyncio.wait_for(process.communicate(data), timeout)
|
|
except asyncio.TimeoutError:
|
|
process.kill()
|
|
await process.wait()
|
|
raise ProcessingError("processing_timeout", permanent=False) from None
|
|
except asyncio.CancelledError:
|
|
process.kill()
|
|
await process.wait()
|
|
raise
|
|
code = process.returncode
|
|
if code == EXIT_INVALID:
|
|
raise ProcessingError("invalid_image", permanent=True)
|
|
if code == EXIT_TOO_LARGE:
|
|
raise ProcessingError("image_too_large", permanent=True)
|
|
if code is None or code < 0:
|
|
raise ProcessingError("processing_killed", permanent=False) # OOM killer or another signal
|
|
if code != 0:
|
|
raise ProcessingError("processing_failed", permanent=False)
|
|
return parse_output(out)
|
|
|
|
|
|
def parse_output(out: bytes) -> list[Rendered]:
|
|
newline = out.find(b"\n", 0, HEADER_LIMIT)
|
|
if newline < 0:
|
|
raise ProcessingError("processing_failed", permanent=False)
|
|
header = json.loads(out[:newline])
|
|
pos = newline + 1
|
|
result = []
|
|
for item in header["variants"]:
|
|
size = int(item["size"])
|
|
result.append(Rendered(item["name"], item["content_type"], int(item["width"]), int(item["height"]),
|
|
out[pos:pos + size]))
|
|
pos += size
|
|
if pos != len(out) or [r.name for r in result] != list(CONTENT_TYPES):
|
|
raise ProcessingError("processing_failed", permanent=False)
|
|
return result
|
|
|
|
|
|
# ------------------------------------------------------------------ child side
|
|
def build(data: bytes, content_type: str, pipeline_version: int, max_pixels: int) -> list[Rendered]:
|
|
"""Decode once, build every variant. Raises the Pillow/OS errors of a bad file."""
|
|
from PIL import Image, ImageOps
|
|
|
|
pipeline = PIPELINES[pipeline_version]
|
|
warnings.simplefilter("ignore", Image.DecompressionBombWarning) # our own pixel limit below
|
|
Image.MAX_IMAGE_PIXELS = max_pixels
|
|
expected = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type]
|
|
img = Image.open(io.BytesIO(data), formats=[expected]) # header only: nothing decoded yet
|
|
width, height = img.size
|
|
if width * height > max_pixels:
|
|
raise TooLarge()
|
|
if expected == "JPEG":
|
|
# DCT scaling: decode a big JPEG directly at 1/2..1/8 of its size, never below what the
|
|
# largest variant needs (the box stays square, so EXIF rotation does not matter).
|
|
img.draft("RGB", (OPTIMIZED_BOX, OPTIMIZED_BOX))
|
|
img.load()
|
|
img = ImageOps.exif_transpose(img) # orientation goes into the pixels, the tag is dropped
|
|
img = _normalize(img)
|
|
alpha = img.mode == "RGBA"
|
|
base = img.convert("RGBa") if alpha else img # premultiplied alpha for resampling
|
|
del img
|
|
|
|
def finish(im, *, flatten: bool):
|
|
if alpha:
|
|
im = im.convert("RGBA")
|
|
if flatten:
|
|
white = Image.new("RGB", im.size, (255, 255, 255))
|
|
white.paste(im, mask=im.getchannel("A"))
|
|
im = white
|
|
im.info = {} # no EXIF/XMP/ICC/comments of the source in a variant
|
|
return im
|
|
|
|
def resized(im, size):
|
|
return im if im.size == size else im.resize(size, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
|
|
|
thumbnail = finish(resized(base, fit(*base.size, THUMBNAIL_BOX)), flatten=True)
|
|
crop = finish(resized(base.crop(center_square(*base.size)), (CROP_SIDE, CROP_SIDE)), flatten=True)
|
|
optimized = finish(resized(base, fit(*base.size, OPTIMIZED_BOX)), flatten=False)
|
|
out = []
|
|
for name, im in (("thumbnail", thumbnail), ("crop", crop), ("optimized", optimized)):
|
|
buf = io.BytesIO()
|
|
if CONTENT_TYPES[name] == "image/jpeg":
|
|
im.save(buf, "JPEG", quality=pipeline.jpeg_quality, optimize=True, subsampling=2, progressive=False)
|
|
else:
|
|
im.save(buf, "WEBP", quality=pipeline.webp_quality, method=pipeline.webp_method, exif=b"",
|
|
icc_profile="", xmp=b"")
|
|
out.append(Rendered(name, CONTENT_TYPES[name], im.width, im.height, buf.getvalue()))
|
|
return out
|
|
|
|
|
|
class TooLarge(Exception):
|
|
pass
|
|
|
|
|
|
def _normalize(img):
|
|
"""Any PNG/JPEG mode -> RGB, or RGBA when the image has transparency."""
|
|
if img.mode.startswith("I;16") or img.mode == "I":
|
|
# 16-bit grey: scale 0..65535 to 0..255 (a plain convert("L") would clip to white).
|
|
img = img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
|
|
has_alpha = img.mode in ("RGBA", "LA", "PA", "La", "RGBa") or "transparency" in img.info
|
|
return img.convert("RGBA" if has_alpha else "RGB") if img.mode != ("RGBA" if has_alpha else "RGB") else img
|
|
|
|
|
|
def child_main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--type", required=True, choices=["image/png", "image/jpeg"])
|
|
parser.add_argument("--pipeline", type=int, required=True, choices=sorted(PIPELINES))
|
|
parser.add_argument("--max-pixels", type=int, required=True)
|
|
parser.add_argument("--memory-mb", type=int, required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
import resource
|
|
limit = args.memory_mb * 1024 * 1024
|
|
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
|
except (ImportError, ValueError, OSError):
|
|
pass # not Linux (unit tests on Windows): the container memory limit still applies
|
|
data = sys.stdin.buffer.read()
|
|
try:
|
|
variants = build(data, args.type, args.pipeline, args.max_pixels)
|
|
except (TooLarge, MemoryError):
|
|
return EXIT_TOO_LARGE
|
|
except Exception as exc: # noqa: BLE001 - every decoder error of the CONTENT is "invalid image"
|
|
from PIL import Image
|
|
if isinstance(exc, Image.DecompressionBombError):
|
|
return EXIT_TOO_LARGE
|
|
if isinstance(exc, (OSError, SyntaxError, ValueError, EOFError, KeyError, IndexError, TypeError)):
|
|
return EXIT_INVALID
|
|
raise
|
|
header = {"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, "height": v.height,
|
|
"size": len(v.data)} for v in variants]}
|
|
sys.stdout.buffer.write(json.dumps(header).encode() + b"\n")
|
|
for v in variants:
|
|
sys.stdout.buffer.write(v.data)
|
|
sys.stdout.buffer.flush()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(child_main())
|