"""Приём и проверка загружаемого изображения с ограниченной памятью. 1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. 2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. """ import asyncio import hashlib import logging import struct import tempfile import threading import time import unicodedata import zlib from dataclasses import dataclass from typing import BinaryIO from PIL import Image from starlette.requests import ClientDisconnect, Request import settings from errors import bad_request, payload_too_large, unsupported_media_type log = logging.getLogger("messenger.uploads") PNG = "image/png" JPEG = "image/jpeg" ALLOWED_TYPES = (PNG, JPEG) PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" JPEG_SIGNATURE = b"\xff\xd8\xff" SPOOL_IN_MEMORY = 256 * 1024 INFLATE_CHUNK = 64 * 1024 CRC_CHUNK = 64 * 1024 # Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS # Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. _decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) @dataclass class Upload: file: BinaryIO size: int sha256: str @dataclass class ImageInfo: content_type: str width: int height: int class Damaged(Exception): pass class TooManyPixels(Exception): pass # ---------- имя файла и заголовки ---------- def validate_filename(value: str | None) -> str: """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" if value is None: raise bad_request("Query parameter filename is required") if not 1 <= len(value) <= 255 or not value.strip(): raise bad_request("filename must be 1-255 characters") if value in (".", ".."): raise bad_request("filename is not allowed") for char in value: if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): raise bad_request("filename contains forbidden characters") return value def declared_type(header: str | None) -> str: base = (header or "").split(";", 1)[0].strip().lower() if base not in ALLOWED_TYPES: raise unsupported_media_type("Content-Type must be image/png or image/jpeg") return base def declared_length(header: str | None) -> int | None: if header is None: return None if not header.isdigit(): raise bad_request("Invalid Content-Length") length = int(header) if length > settings.MAX_UPLOAD_BYTES: raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") return length # ---------- приём тела ---------- async def receive(request: Request) -> Upload: """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) digest = hashlib.sha256() size = 0 deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS stream = request.stream().__aiter__() try: while True: wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) if wait <= 0: raise bad_request("Upload took too long") try: chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) except StopAsyncIteration: break except asyncio.TimeoutError: raise bad_request("Upload stalled or took too long") from None if not chunk: continue size += len(chunk) if size > settings.MAX_UPLOAD_BYTES: raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") digest.update(chunk) spool.write(chunk) except ClientDisconnect: spool.close() log.info("Upload aborted by the client after %s bytes, nothing stored", size) raise bad_request("Upload was interrupted") from None except BaseException: spool.close() raise if size == 0: spool.close() raise bad_request("Empty body") return Upload(spool, size, digest.hexdigest()) async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. """ read = 0 deadline = time.monotonic() + seconds try: stream = request.stream().__aiter__() while read <= limit: wait = deadline - time.monotonic() if wait <= 0: return try: chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) except (StopAsyncIteration, asyncio.TimeoutError): return read += len(chunk) except (ClientDisconnect, RuntimeError): return # ---------- проверка содержимого ---------- def sniff(head: bytes) -> str | None: if head.startswith(PNG_SIGNATURE): return PNG if head.startswith(JPEG_SIGNATURE): return JPEG return None def inspect(upload: Upload, declared: str) -> ImageInfo: upload.file.seek(0) detected = sniff(upload.file.read(16)) if detected is None: raise unsupported_media_type("File is not a PNG or JPEG image") if detected != declared: raise unsupported_media_type("Content-Type does not match the file content") upload.file.seek(0) try: with _decode_slots: width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file) except TooManyPixels: raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None except Damaged as exc: log.info("Rejected damaged %s: %s", detected, exc) raise unsupported_media_type("Image is damaged or not supported") from None except Image.DecompressionBombError: raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: log.info("Rejected damaged %s: %s", detected, type(exc).__name__) raise unsupported_media_type("Image is damaged or not supported") from None finally: upload.file.seek(0) return ImageInfo(detected, width, height) def check_pixels(width: int, height: int) -> None: if width < 1 or height < 1: raise Damaged("zero dimension") if width * height > settings.MAX_IMAGE_PIXELS: raise TooManyPixels() def check_jpeg(file: BinaryIO) -> tuple[int, int]: with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS width, height = image.size check_pixels(width, height) # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. image.draft("RGB", (max(1, width // 8), max(1, height // 8))) image.load() return width, height # PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) FILTER_BYTES = bytes(range(5)) class PngRows: """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" def __init__(self, width, height, depth, color, interlace): bits = depth * PNG_CHANNELS[color] def row_bytes(w): return (w * bits + 7) // 8 passes = [(width, height)] if interlace == 0 else [ ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] # (длина строки с байтом фильтра, байт в проходе) self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] self.expected = sum(total for _, total in self.passes) self.index, self.pos = 0, 0 self.total = 0 def feed(self, data: bytes) -> None: view = memoryview(data) self.total += len(data) if self.total > self.expected: raise Damaged("more pixel data than the header says") offset = 0 while offset < len(view): stride, size = self.passes[self.index] take = min(len(view) - offset, size - self.pos) first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): raise Damaged("invalid filter type") self.pos += take offset += take if self.pos == size: self.index, self.pos = self.index + 1, 0 def done(self) -> bool: return self.total == self.expected def check_png(file: BinaryIO) -> tuple[int, int]: if file.read(8) != PNG_SIGNATURE: raise Damaged("signature") rows = inflater = None width = height = 0 seen_idat = idat_finished = have_plte = False color = None first = True while True: header = file.read(8) if len(header) < 8: raise Damaged("truncated: no IEND") length, kind = struct.unpack(">I4s", header) if length > 0x7FFFFFFF: raise Damaged("chunk length") if first and kind != b"IHDR": raise Damaged("IHDR must be first") crc = zlib.crc32(kind) keep = kind in (b"IHDR",) kept = b"" left = length while left: piece = file.read(min(CRC_CHUNK, left)) if not piece: raise Damaged("truncated chunk") left -= len(piece) crc = zlib.crc32(piece, crc) if keep: kept += piece elif kind == b"IDAT": if rows is None or idat_finished: raise Damaged("IDAT out of order") feed_inflate(inflater, rows, piece) stored = file.read(4) if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: raise Damaged(f"CRC mismatch in {kind!r}") if kind == b"IHDR": if not first or length != 13: raise Damaged("IHDR") width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) if width > 0x7FFFFFFF or height > 0x7FFFFFFF: raise Damaged("dimensions") check_pixels(width, height) # до распаковки if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: raise Damaged("color type / bit depth") if compression != 0 or filter_method != 0 or interlace not in (0, 1): raise Damaged("IHDR methods") rows = PngRows(width, height, depth, color, interlace) inflater = zlib.decompressobj() elif kind == b"PLTE": have_plte = True elif kind == b"IDAT": seen_idat = True elif kind == b"IEND": break if seen_idat and kind != b"IDAT": idat_finished = True first = False if file.read(1): raise Damaged("data after IEND") if color == 3 and not have_plte: raise Damaged("palette image without PLTE") if not seen_idat: raise Damaged("no IDAT") tail = inflater.flush(INFLATE_CHUNK) if tail: rows.feed(tail) if not inflater.eof or inflater.unused_data or not rows.done(): raise Damaged("pixel data is incomplete") return width, height def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" if inflater.eof: if data: raise Damaged("data after the end of zlib stream") return out = inflater.decompress(data, INFLATE_CHUNK) rows.feed(out) while inflater.unconsumed_tail and not inflater.eof: out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) rows.feed(out) if inflater.eof and inflater.unused_data: raise Damaged("data after the end of zlib stream")