Files
mtusi2026lab5/src/messenger/images.py
T
Alexey BodanyukandClaude Opus 5.5 5112be83cc перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 15:19:00 +03:00

329 lines
14 KiB
Python

"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
Two stages, both with bounded memory and both BEFORE anything is stored:
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
(413) before the rest of the body is read and long before any pixel is decoded.
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
warning is fatal, so a truncated file or a file without scan data after SOS fails.
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
nothing here may turn user input into a 500.
"""
from __future__ import annotations
import asyncio
import os
import shutil
import struct
import zlib
from dataclasses import dataclass
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SOI = b"\xff\xd8\xff"
PNG = "image/png"
JPEG = "image/jpeg"
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
# Adam7 passes: (x start, y start, x step, y step).
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
# Standalone markers without a length field.
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
MAX_DIMENSION = 2**31 - 1
class ImageError(Exception):
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
def __init__(self, status: int, message: str):
super().__init__(message)
self.status = status
self.message = message
class VerifierUnavailable(Exception):
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
return ImageError(415, message)
@dataclass(frozen=True, slots=True)
class ImageInfo:
content_type: str
width: int
height: int
def detect(head: bytes) -> str | None:
"""Format by signature; None while fewer bytes than a signature have arrived."""
if head[:8] == PNG_SIGNATURE:
return PNG
if head[:3] == JPEG_SOI:
return JPEG
probe = min(len(head), 8)
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
return None
raise unsupported()
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
"""Format and size from the beginning of the file, or None if more bytes are needed.
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
kind = detect(head)
if kind is None:
return None
info = _png_header(head) if kind == PNG else _jpeg_header(head)
if info is None:
return None
if info.width * info.height > max_pixels:
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
return info
def _png_header(head: bytes) -> ImageInfo | None:
if len(head) < 33:
return None
length, ctype = struct.unpack(">I4s", head[8:16])
if ctype != b"IHDR" or length != 13:
raise unsupported()
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
raise unsupported()
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
raise unsupported()
return ImageInfo(PNG, width, height)
def _jpeg_header(head: bytes) -> ImageInfo | None:
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
i = 2
n = len(head)
while True:
if i >= n:
return None
if head[i] != 0xFF:
raise unsupported()
while i < n and head[i] == 0xFF: # fill bytes
i += 1
if i >= n:
return None
marker = head[i]
i += 1
if marker in JPEG_STANDALONE:
continue
if marker in (0xD8, 0xD9, 0xDA, 0x00):
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
raise unsupported()
if i + 2 > n:
return None
length = struct.unpack(">H", head[i:i + 2])[0]
if length < 2:
raise unsupported()
if marker in JPEG_SOF:
if i + 7 > n:
return None
if length < 8:
raise unsupported()
height, width = struct.unpack(">HH", head[i + 3:i + 7])
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
raise unsupported()
return ImageInfo(JPEG, width, height)
i += length
# ------------------------------------------------------------------ full verification
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
jpeg_max_memory_mb: int = 256) -> None:
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
if info.content_type == PNG:
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
else:
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
bpp = PNG_CHANNELS[color] * depth
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
segments = []
for xs, ys, dx, dy in passes:
pw = (width - xs + dx - 1) // dx if width > xs else 0
ph = (height - ys + dy - 1) // dy if height > ys else 0
if pw and ph:
stride = 1 + (pw * bpp + 7) // 8
segments.append((ph * stride, stride))
return segments
class _RawChecker:
"""Counts inflated bytes against the expected size and checks the filter byte of every row
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
VALID_FILTERS = bytes(range(5))
def __init__(self, segments: list[tuple[int, int]]):
self.segments = segments
self.index = 0 # current pass
self.offset = 0 # bytes already seen inside the current pass
self.expected = sum(size for size, _ in segments)
self.total = 0
def feed(self, piece: bytes) -> None:
self.total += len(piece)
if self.total > self.expected:
raise unsupported()
pos = 0
while pos < len(piece):
size, stride = self.segments[self.index]
take = min(size - self.offset, len(piece) - pos)
part = piece[pos:pos + take]
first = (-self.offset) % stride
if part[first::stride].translate(None, self.VALID_FILTERS):
raise unsupported()
self.offset += take
pos += take
if self.offset == size:
self.index += 1
self.offset = 0
def verify_png(data: bytes | bytearray | memoryview) -> None:
view = memoryview(data)
if bytes(view[:8]) != PNG_SIGNATURE:
raise unsupported()
pos = 8
n = len(view)
checker: _RawChecker | None = None
inflater = zlib.decompressobj()
color = None
seen_plte = seen_idat = idat_done = False
first = True
while True:
if pos + 12 > n:
raise unsupported() # truncated: no room for another chunk header + CRC
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
if length > n - pos - 12:
raise unsupported()
body = view[pos + 8:pos + 8 + length]
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
raise unsupported()
pos += 12 + length
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
raise unsupported()
if first:
if ctype != b"IHDR" or length != 13:
raise unsupported()
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
raise unsupported()
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
first = False
continue
assert checker is not None
if ctype == b"IHDR":
raise unsupported()
if ctype == b"PLTE":
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise unsupported()
seen_plte = True
elif ctype == b"IDAT":
if idat_done:
raise unsupported() # IDAT chunks must be consecutive
if color == 3 and not seen_plte:
raise unsupported()
seen_idat = True
_inflate(inflater, body, checker)
elif ctype == b"IEND":
if length != 0 or not seen_idat:
raise unsupported()
if pos != n:
raise unsupported() # bytes after IEND (polyglot files) are not accepted
break
elif not 97 <= ctype[0] <= 122:
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
if ctype != b"IDAT" and seen_idat:
idat_done = True
# The zlib stream must end exactly with the last pixel row, without trailing data.
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
raise unsupported()
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
if inflater.eof:
if len(body):
raise unsupported() # data after the end of the zlib stream
return
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
try:
for start in range(0, len(body), INPUT_STEP):
pending: bytes | memoryview = body[start:start + INPUT_STEP]
while pending:
if inflater.eof:
raise unsupported() # data after the end of the zlib stream
checker.feed(inflater.decompress(pending, INFLATE_STEP))
pending = inflater.unconsumed_tail
if inflater.eof and inflater.unused_data:
raise unsupported()
except zlib.error:
raise unsupported() from None
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
buffers of progressive files); the separate process is killed at the deadline.
"""
process = await asyncio.create_subprocess_exec(
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
stderr = b""
try:
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
except (asyncio.TimeoutError, asyncio.CancelledError):
process.kill()
await process.wait()
raise
except (BrokenPipeError, ConnectionResetError):
await process.wait() # djpeg stopped reading: the file is already known to be broken
if process.returncode is not None and process.returncode < 0:
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
# SERVER, not of the file - never report it as 415.
raise VerifierUnavailable()
if process.returncode != 0:
if b"Insufficient memory" in (stderr or b""):
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
raise ImageError(413, "Изображение слишком велико для обработки")
raise unsupported()