forked from mtusi2026/mtusi2026lab5
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались. Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker добавляются в следующих коммитах). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
329 lines
14 KiB
Python
329 lines
14 KiB
Python
"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
|
|
|
|
Two stages, both with bounded memory and both BEFORE anything is stored:
|
|
|
|
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
|
|
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
|
|
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
|
|
(413) before the rest of the body is read and long before any pixel is decoded.
|
|
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
|
|
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
|
|
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
|
|
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
|
|
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
|
|
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
|
|
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
|
|
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
|
|
warning is fatal, so a truncated file or a file without scan data after SOS fails.
|
|
|
|
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
|
|
nothing here may turn user input into a 500.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import os
|
|
import shutil
|
|
import struct
|
|
import zlib
|
|
from dataclasses import dataclass
|
|
|
|
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
|
JPEG_SOI = b"\xff\xd8\xff"
|
|
|
|
PNG = "image/png"
|
|
JPEG = "image/jpeg"
|
|
|
|
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
|
|
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
|
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
|
# Adam7 passes: (x start, y start, x step, y step).
|
|
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
|
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
|
|
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
|
# Standalone markers without a length field.
|
|
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
|
|
|
|
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
|
|
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
|
|
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
|
|
MAX_DIMENSION = 2**31 - 1
|
|
|
|
|
|
class ImageError(Exception):
|
|
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
|
|
|
|
def __init__(self, status: int, message: str):
|
|
super().__init__(message)
|
|
self.status = status
|
|
self.message = message
|
|
|
|
|
|
class VerifierUnavailable(Exception):
|
|
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
|
|
|
|
|
|
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
|
|
return ImageError(415, message)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ImageInfo:
|
|
content_type: str
|
|
width: int
|
|
height: int
|
|
|
|
|
|
def detect(head: bytes) -> str | None:
|
|
"""Format by signature; None while fewer bytes than a signature have arrived."""
|
|
if head[:8] == PNG_SIGNATURE:
|
|
return PNG
|
|
if head[:3] == JPEG_SOI:
|
|
return JPEG
|
|
probe = min(len(head), 8)
|
|
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
|
|
return None
|
|
raise unsupported()
|
|
|
|
|
|
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
|
|
"""Format and size from the beginning of the file, or None if more bytes are needed.
|
|
|
|
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
|
|
kind = detect(head)
|
|
if kind is None:
|
|
return None
|
|
info = _png_header(head) if kind == PNG else _jpeg_header(head)
|
|
if info is None:
|
|
return None
|
|
if info.width * info.height > max_pixels:
|
|
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
|
|
return info
|
|
|
|
|
|
def _png_header(head: bytes) -> ImageInfo | None:
|
|
if len(head) < 33:
|
|
return None
|
|
length, ctype = struct.unpack(">I4s", head[8:16])
|
|
if ctype != b"IHDR" or length != 13:
|
|
raise unsupported()
|
|
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
|
|
raise unsupported()
|
|
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
|
|
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
|
|
raise unsupported()
|
|
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
|
|
raise unsupported()
|
|
return ImageInfo(PNG, width, height)
|
|
|
|
|
|
def _jpeg_header(head: bytes) -> ImageInfo | None:
|
|
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
|
|
i = 2
|
|
n = len(head)
|
|
while True:
|
|
if i >= n:
|
|
return None
|
|
if head[i] != 0xFF:
|
|
raise unsupported()
|
|
while i < n and head[i] == 0xFF: # fill bytes
|
|
i += 1
|
|
if i >= n:
|
|
return None
|
|
marker = head[i]
|
|
i += 1
|
|
if marker in JPEG_STANDALONE:
|
|
continue
|
|
if marker in (0xD8, 0xD9, 0xDA, 0x00):
|
|
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
|
|
raise unsupported()
|
|
if i + 2 > n:
|
|
return None
|
|
length = struct.unpack(">H", head[i:i + 2])[0]
|
|
if length < 2:
|
|
raise unsupported()
|
|
if marker in JPEG_SOF:
|
|
if i + 7 > n:
|
|
return None
|
|
if length < 8:
|
|
raise unsupported()
|
|
height, width = struct.unpack(">HH", head[i + 3:i + 7])
|
|
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
|
|
raise unsupported()
|
|
return ImageInfo(JPEG, width, height)
|
|
i += length
|
|
|
|
|
|
# ------------------------------------------------------------------ full verification
|
|
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
|
|
jpeg_max_memory_mb: int = 256) -> None:
|
|
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
|
|
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
|
|
if info.content_type == PNG:
|
|
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
|
|
else:
|
|
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
|
|
|
|
|
|
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
|
|
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
|
|
bpp = PNG_CHANNELS[color] * depth
|
|
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
|
|
segments = []
|
|
for xs, ys, dx, dy in passes:
|
|
pw = (width - xs + dx - 1) // dx if width > xs else 0
|
|
ph = (height - ys + dy - 1) // dy if height > ys else 0
|
|
if pw and ph:
|
|
stride = 1 + (pw * bpp + 7) // 8
|
|
segments.append((ph * stride, stride))
|
|
return segments
|
|
|
|
|
|
class _RawChecker:
|
|
"""Counts inflated bytes against the expected size and checks the filter byte of every row
|
|
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
|
|
|
|
VALID_FILTERS = bytes(range(5))
|
|
|
|
def __init__(self, segments: list[tuple[int, int]]):
|
|
self.segments = segments
|
|
self.index = 0 # current pass
|
|
self.offset = 0 # bytes already seen inside the current pass
|
|
self.expected = sum(size for size, _ in segments)
|
|
self.total = 0
|
|
|
|
def feed(self, piece: bytes) -> None:
|
|
self.total += len(piece)
|
|
if self.total > self.expected:
|
|
raise unsupported()
|
|
pos = 0
|
|
while pos < len(piece):
|
|
size, stride = self.segments[self.index]
|
|
take = min(size - self.offset, len(piece) - pos)
|
|
part = piece[pos:pos + take]
|
|
first = (-self.offset) % stride
|
|
if part[first::stride].translate(None, self.VALID_FILTERS):
|
|
raise unsupported()
|
|
self.offset += take
|
|
pos += take
|
|
if self.offset == size:
|
|
self.index += 1
|
|
self.offset = 0
|
|
|
|
|
|
def verify_png(data: bytes | bytearray | memoryview) -> None:
|
|
view = memoryview(data)
|
|
if bytes(view[:8]) != PNG_SIGNATURE:
|
|
raise unsupported()
|
|
pos = 8
|
|
n = len(view)
|
|
checker: _RawChecker | None = None
|
|
inflater = zlib.decompressobj()
|
|
color = None
|
|
seen_plte = seen_idat = idat_done = False
|
|
first = True
|
|
while True:
|
|
if pos + 12 > n:
|
|
raise unsupported() # truncated: no room for another chunk header + CRC
|
|
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
|
|
if length > n - pos - 12:
|
|
raise unsupported()
|
|
body = view[pos + 8:pos + 8 + length]
|
|
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
|
|
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
|
|
raise unsupported()
|
|
pos += 12 + length
|
|
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
|
|
raise unsupported()
|
|
if first:
|
|
if ctype != b"IHDR" or length != 13:
|
|
raise unsupported()
|
|
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
|
|
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
|
|
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
|
|
raise unsupported()
|
|
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
|
|
first = False
|
|
continue
|
|
assert checker is not None
|
|
if ctype == b"IHDR":
|
|
raise unsupported()
|
|
if ctype == b"PLTE":
|
|
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
|
raise unsupported()
|
|
seen_plte = True
|
|
elif ctype == b"IDAT":
|
|
if idat_done:
|
|
raise unsupported() # IDAT chunks must be consecutive
|
|
if color == 3 and not seen_plte:
|
|
raise unsupported()
|
|
seen_idat = True
|
|
_inflate(inflater, body, checker)
|
|
elif ctype == b"IEND":
|
|
if length != 0 or not seen_idat:
|
|
raise unsupported()
|
|
if pos != n:
|
|
raise unsupported() # bytes after IEND (polyglot files) are not accepted
|
|
break
|
|
elif not 97 <= ctype[0] <= 122:
|
|
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
|
|
if ctype != b"IDAT" and seen_idat:
|
|
idat_done = True
|
|
# The zlib stream must end exactly with the last pixel row, without trailing data.
|
|
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
|
|
raise unsupported()
|
|
|
|
|
|
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
|
|
if inflater.eof:
|
|
if len(body):
|
|
raise unsupported() # data after the end of the zlib stream
|
|
return
|
|
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
|
|
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
|
|
try:
|
|
for start in range(0, len(body), INPUT_STEP):
|
|
pending: bytes | memoryview = body[start:start + INPUT_STEP]
|
|
while pending:
|
|
if inflater.eof:
|
|
raise unsupported() # data after the end of the zlib stream
|
|
checker.feed(inflater.decompress(pending, INFLATE_STEP))
|
|
pending = inflater.unconsumed_tail
|
|
if inflater.eof and inflater.unused_data:
|
|
raise unsupported()
|
|
except zlib.error:
|
|
raise unsupported() from None
|
|
|
|
|
|
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
|
|
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
|
|
|
|
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
|
|
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
|
|
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
|
|
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
|
|
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
|
|
buffers of progressive files); the separate process is killed at the deadline.
|
|
"""
|
|
process = await asyncio.create_subprocess_exec(
|
|
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
|
|
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
|
|
stderr = b""
|
|
try:
|
|
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
|
|
except (asyncio.TimeoutError, asyncio.CancelledError):
|
|
process.kill()
|
|
await process.wait()
|
|
raise
|
|
except (BrokenPipeError, ConnectionResetError):
|
|
await process.wait() # djpeg stopped reading: the file is already known to be broken
|
|
if process.returncode is not None and process.returncode < 0:
|
|
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
|
|
# SERVER, not of the file - never report it as 415.
|
|
raise VerifierUnavailable()
|
|
if process.returncode != 0:
|
|
if b"Insufficient memory" in (stderr or b""):
|
|
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
|
|
raise ImageError(413, "Изображение слишком велико для обработки")
|
|
raise unsupported()
|