Files
mtusi2026lab5/src/messenger/imaging.py
T
Alexey Bodanyuk a633949919 lab5: схема 0005 (jobs, outbox, попытки, варианты, аудит) и модули pipeline
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Миграция только добавляет: статусы pending/queued/processing/ready/failed, staging-ключ,
published_version, jobs с lease и токеном попытки, job_attempts с заранее известными
ключами, attachment_variants (PK = ключ эффекта), outbox, job_events, view failed_jobs,
post_message_v3. CHECK attachments_active_has_source: у queued/processing всегда есть источник.
pipeline.py — версии 1/2/3 и геометрия по контракту; imaging.py — варианты в дочернем
процессе (RLIMIT_AS, таймаут, лимит пикселей); jobs.py — все переходы заданий одной
транзакцией с fencing по токену; broker.py — топология и строгая схема сообщения.
Зависимости: aio-pika 9.6.2, pillow 12.3.0 (lock с хешами).
2026-10-31 16:12:51 +03:00

209 lines
9.0 KiB
Python

"""Variants of ONE image, built in a separate child process with hard limits (lab 5 worker).
parent (worker): await render(data, content_type, version, ...) -> list[Rendered]
child: python -m messenger.imaging --type image/png --pipeline 1 ...
stdin: original bytes; stdout: one JSON header line + variant bytes
Why a child process per image:
* memory: RLIMIT_AS (IMAGE_MAX_MEMORY_MB) caps the address space of the decoder; a picture that
needs more fails with MemoryError -> `image_too_large` (permanent), the worker keeps running;
* time: the parent kills the child at JOB_TIMEOUT_SECONDS (C code of Pillow cannot be
interrupted inside a thread) -> `processing_timeout` (transient, bounded by JOB_MAX_ATTEMPTS);
* pixels: the size is read from the header and checked against MAX_IMAGE_PIXELS BEFORE the
pixels are decoded (`Image.open` is lazy).
A crash of the decoder (signal) is a transient `processing_killed`: the next attempt may run
with less memory pressure; JOB_MAX_ATTEMPTS bounds it.
Geometry and encoder settings: pipeline.py. Alpha is handled in premultiplied form (RGBa) while
resizing, so transparent pixels do not bleed their colour into the edges; for JPEG the result is
then composited onto white.
"""
from __future__ import annotations
import argparse
import asyncio
import hashlib
import io
import json
import sys
import warnings
from dataclasses import dataclass
from .pipeline import CONTENT_TYPES, CROP_SIDE, OPTIMIZED_BOX, PIPELINES, THUMBNAIL_BOX, center_square, fit
EXIT_INVALID = 3 # the bytes are not a decodable PNG/JPEG
EXIT_TOO_LARGE = 4 # pixel limit or memory limit
HEADER_LIMIT = 4096
class ProcessingError(Exception):
"""Outcome of an attempt that did not produce variants. `permanent`: another attempt with the
same input cannot succeed (job -> failed at once); otherwise retry with backoff."""
def __init__(self, code: str, permanent: bool):
super().__init__(code)
self.code = code
self.permanent = permanent
@dataclass(frozen=True, slots=True)
class Rendered:
name: str
content_type: str
width: int
height: int
data: bytes
@property
def sha256(self) -> bytes:
return hashlib.sha256(self.data).digest()
# ------------------------------------------------------------------ parent side
async def render(data: bytes, content_type: str, pipeline_version: int, *, max_pixels: int, timeout: float,
memory_mb: int) -> list[Rendered]:
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "messenger.imaging", "--type", content_type, "--pipeline", str(pipeline_version),
"--max-pixels", str(max_pixels), "--memory-mb", str(memory_mb),
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
try:
out, _ = await asyncio.wait_for(process.communicate(data), timeout)
except asyncio.TimeoutError:
process.kill()
await process.wait()
raise ProcessingError("processing_timeout", permanent=False) from None
except asyncio.CancelledError:
process.kill()
await process.wait()
raise
code = process.returncode
if code == EXIT_INVALID:
raise ProcessingError("invalid_image", permanent=True)
if code == EXIT_TOO_LARGE:
raise ProcessingError("image_too_large", permanent=True)
if code is None or code < 0:
raise ProcessingError("processing_killed", permanent=False) # OOM killer or another signal
if code != 0:
raise ProcessingError("processing_failed", permanent=False)
return parse_output(out)
def parse_output(out: bytes) -> list[Rendered]:
newline = out.find(b"\n", 0, HEADER_LIMIT)
if newline < 0:
raise ProcessingError("processing_failed", permanent=False)
header = json.loads(out[:newline])
pos = newline + 1
result = []
for item in header["variants"]:
size = int(item["size"])
result.append(Rendered(item["name"], item["content_type"], int(item["width"]), int(item["height"]),
out[pos:pos + size]))
pos += size
if pos != len(out) or [r.name for r in result] != list(CONTENT_TYPES):
raise ProcessingError("processing_failed", permanent=False)
return result
# ------------------------------------------------------------------ child side
def build(data: bytes, content_type: str, pipeline_version: int, max_pixels: int) -> list[Rendered]:
"""Decode once, build every variant. Raises the Pillow/OS errors of a bad file."""
from PIL import Image, ImageOps
pipeline = PIPELINES[pipeline_version]
warnings.simplefilter("ignore", Image.DecompressionBombWarning) # our own pixel limit below
Image.MAX_IMAGE_PIXELS = max_pixels
expected = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type]
img = Image.open(io.BytesIO(data), formats=[expected]) # header only: nothing decoded yet
width, height = img.size
if width * height > max_pixels:
raise TooLarge()
if expected == "JPEG":
# DCT scaling: decode a big JPEG directly at 1/2..1/8 of its size, never below what the
# largest variant needs (the box stays square, so EXIF rotation does not matter).
img.draft("RGB", (OPTIMIZED_BOX, OPTIMIZED_BOX))
img.load()
img = ImageOps.exif_transpose(img) # orientation goes into the pixels, the tag is dropped
img = _normalize(img)
alpha = img.mode == "RGBA"
base = img.convert("RGBa") if alpha else img # premultiplied alpha for resampling
del img
def finish(im, *, flatten: bool):
if alpha:
im = im.convert("RGBA")
if flatten:
white = Image.new("RGB", im.size, (255, 255, 255))
white.paste(im, mask=im.getchannel("A"))
im = white
im.info = {} # no EXIF/XMP/ICC/comments of the source in a variant
return im
def resized(im, size):
return im if im.size == size else im.resize(size, Image.Resampling.LANCZOS, reducing_gap=3.0)
thumbnail = finish(resized(base, fit(*base.size, THUMBNAIL_BOX)), flatten=True)
crop = finish(resized(base.crop(center_square(*base.size)), (CROP_SIDE, CROP_SIDE)), flatten=True)
optimized = finish(resized(base, fit(*base.size, OPTIMIZED_BOX)), flatten=False)
out = []
for name, im in (("thumbnail", thumbnail), ("crop", crop), ("optimized", optimized)):
buf = io.BytesIO()
if CONTENT_TYPES[name] == "image/jpeg":
im.save(buf, "JPEG", quality=pipeline.jpeg_quality, optimize=True, subsampling=2, progressive=False)
else:
im.save(buf, "WEBP", quality=pipeline.webp_quality, method=pipeline.webp_method, exif=b"",
icc_profile="", xmp=b"")
out.append(Rendered(name, CONTENT_TYPES[name], im.width, im.height, buf.getvalue()))
return out
class TooLarge(Exception):
pass
def _normalize(img):
"""Any PNG/JPEG mode -> RGB, or RGBA when the image has transparency."""
if img.mode.startswith("I;16") or img.mode == "I":
# 16-bit grey: scale 0..65535 to 0..255 (a plain convert("L") would clip to white).
img = img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
has_alpha = img.mode in ("RGBA", "LA", "PA", "La", "RGBa") or "transparency" in img.info
return img.convert("RGBA" if has_alpha else "RGB") if img.mode != ("RGBA" if has_alpha else "RGB") else img
def child_main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--type", required=True, choices=["image/png", "image/jpeg"])
parser.add_argument("--pipeline", type=int, required=True, choices=sorted(PIPELINES))
parser.add_argument("--max-pixels", type=int, required=True)
parser.add_argument("--memory-mb", type=int, required=True)
args = parser.parse_args(argv)
try:
import resource
limit = args.memory_mb * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
except (ImportError, ValueError, OSError):
pass # not Linux (unit tests on Windows): the container memory limit still applies
data = sys.stdin.buffer.read()
try:
variants = build(data, args.type, args.pipeline, args.max_pixels)
except (TooLarge, MemoryError):
return EXIT_TOO_LARGE
except Exception as exc: # noqa: BLE001 - every decoder error of the CONTENT is "invalid image"
from PIL import Image
if isinstance(exc, Image.DecompressionBombError):
return EXIT_TOO_LARGE
if isinstance(exc, (OSError, SyntaxError, ValueError, EOFError, KeyError, IndexError, TypeError)):
return EXIT_INVALID
raise
header = {"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, "height": v.height,
"size": len(v.data)} for v in variants]}
sys.stdout.buffer.write(json.dumps(header).encode() + b"\n")
for v in variants:
sys.stdout.buffer.write(v.data)
sys.stdout.buffer.flush()
return 0
if __name__ == "__main__":
sys.exit(child_main())