352 lines
14 KiB
Python
352 lines
14 KiB
Python
"""Приём и проверка загружаемого изображения с ограниченной памятью.
|
|
|
|
1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше
|
|
MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном
|
|
файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса.
|
|
2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях
|
|
читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково
|
|
(CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма),
|
|
JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение.
|
|
"""
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import struct
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unicodedata
|
|
import zlib
|
|
from dataclasses import dataclass
|
|
from typing import BinaryIO
|
|
|
|
from PIL import Image
|
|
from starlette.requests import ClientDisconnect, Request
|
|
|
|
import settings
|
|
from errors import bad_request, payload_too_large, unsupported_media_type
|
|
|
|
log = logging.getLogger("messenger.uploads")
|
|
|
|
PNG = "image/png"
|
|
JPEG = "image/jpeg"
|
|
ALLOWED_TYPES = (PNG, JPEG)
|
|
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
|
JPEG_SIGNATURE = b"\xff\xd8\xff"
|
|
|
|
SPOOL_IN_MEMORY = 256 * 1024
|
|
INFLATE_CHUNK = 64 * 1024
|
|
CRC_CHUNK = 64 * 1024
|
|
|
|
# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке).
|
|
Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS
|
|
|
|
# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY.
|
|
_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY)
|
|
|
|
|
|
@dataclass
|
|
class Upload:
|
|
file: BinaryIO
|
|
size: int
|
|
sha256: str
|
|
|
|
|
|
@dataclass
|
|
class ImageInfo:
|
|
content_type: str
|
|
width: int
|
|
height: int
|
|
|
|
|
|
class Damaged(Exception):
|
|
pass
|
|
|
|
|
|
class TooManyPixels(Exception):
|
|
pass
|
|
|
|
|
|
# ---------- имя файла и заголовки ----------
|
|
|
|
def validate_filename(value: str | None) -> str:
|
|
"""filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов."""
|
|
if value is None:
|
|
raise bad_request("Query parameter filename is required")
|
|
if not 1 <= len(value) <= 255 or not value.strip():
|
|
raise bad_request("filename must be 1-255 characters")
|
|
if value in (".", ".."):
|
|
raise bad_request("filename is not allowed")
|
|
for char in value:
|
|
if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"):
|
|
raise bad_request("filename contains forbidden characters")
|
|
return value
|
|
|
|
|
|
def declared_type(header: str | None) -> str:
|
|
base = (header or "").split(";", 1)[0].strip().lower()
|
|
if base not in ALLOWED_TYPES:
|
|
raise unsupported_media_type("Content-Type must be image/png or image/jpeg")
|
|
return base
|
|
|
|
|
|
def declared_length(header: str | None) -> int | None:
|
|
if header is None:
|
|
return None
|
|
if not header.isdigit():
|
|
raise bad_request("Invalid Content-Length")
|
|
length = int(header)
|
|
if length > settings.MAX_UPLOAD_BYTES:
|
|
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
|
|
return length
|
|
|
|
|
|
# ---------- приём тела ----------
|
|
|
|
async def receive(request: Request) -> Upload:
|
|
"""Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется."""
|
|
spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR)
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS
|
|
stream = request.stream().__aiter__()
|
|
try:
|
|
while True:
|
|
wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic())
|
|
if wait <= 0:
|
|
raise bad_request("Upload took too long")
|
|
try:
|
|
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
|
|
except StopAsyncIteration:
|
|
break
|
|
except asyncio.TimeoutError:
|
|
raise bad_request("Upload stalled or took too long") from None
|
|
if not chunk:
|
|
continue
|
|
size += len(chunk)
|
|
if size > settings.MAX_UPLOAD_BYTES:
|
|
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
except ClientDisconnect:
|
|
spool.close()
|
|
log.info("Upload aborted by the client after %s bytes, nothing stored", size)
|
|
raise bad_request("Upload was interrupted") from None
|
|
except BaseException:
|
|
spool.close()
|
|
raise
|
|
if size == 0:
|
|
spool.close()
|
|
raise bad_request("Empty body")
|
|
return Upload(spool, size, digest.hexdigest())
|
|
|
|
|
|
async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None:
|
|
"""Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв.
|
|
|
|
Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается.
|
|
"""
|
|
read = 0
|
|
deadline = time.monotonic() + seconds
|
|
try:
|
|
stream = request.stream().__aiter__()
|
|
while read <= limit:
|
|
wait = deadline - time.monotonic()
|
|
if wait <= 0:
|
|
return
|
|
try:
|
|
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
|
|
except (StopAsyncIteration, asyncio.TimeoutError):
|
|
return
|
|
read += len(chunk)
|
|
except (ClientDisconnect, RuntimeError):
|
|
return
|
|
|
|
|
|
# ---------- проверка содержимого ----------
|
|
|
|
def sniff(head: bytes) -> str | None:
|
|
if head.startswith(PNG_SIGNATURE):
|
|
return PNG
|
|
if head.startswith(JPEG_SIGNATURE):
|
|
return JPEG
|
|
return None
|
|
|
|
|
|
def inspect(upload: Upload, declared: str) -> ImageInfo:
|
|
upload.file.seek(0)
|
|
detected = sniff(upload.file.read(16))
|
|
if detected is None:
|
|
raise unsupported_media_type("File is not a PNG or JPEG image")
|
|
if detected != declared:
|
|
raise unsupported_media_type("Content-Type does not match the file content")
|
|
upload.file.seek(0)
|
|
try:
|
|
with _decode_slots:
|
|
width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file)
|
|
except TooManyPixels:
|
|
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
|
|
except Damaged as exc:
|
|
log.info("Rejected damaged %s: %s", detected, exc)
|
|
raise unsupported_media_type("Image is damaged or not supported") from None
|
|
except Image.DecompressionBombError:
|
|
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
|
|
except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc:
|
|
log.info("Rejected damaged %s: %s", detected, type(exc).__name__)
|
|
raise unsupported_media_type("Image is damaged or not supported") from None
|
|
finally:
|
|
upload.file.seek(0)
|
|
return ImageInfo(detected, width, height)
|
|
|
|
|
|
def check_pixels(width: int, height: int) -> None:
|
|
if width < 1 or height < 1:
|
|
raise Damaged("zero dimension")
|
|
if width * height > settings.MAX_IMAGE_PIXELS:
|
|
raise TooManyPixels()
|
|
|
|
|
|
def check_jpeg(file: BinaryIO) -> tuple[int, int]:
|
|
with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS
|
|
width, height = image.size
|
|
check_pixels(width, height)
|
|
# Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8:
|
|
# 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB.
|
|
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
|
image.load()
|
|
return width, height
|
|
|
|
|
|
# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR).
|
|
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
|
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
|
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
|
FILTER_BYTES = bytes(range(5))
|
|
|
|
|
|
class PngRows:
|
|
"""Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки."""
|
|
|
|
def __init__(self, width, height, depth, color, interlace):
|
|
bits = depth * PNG_CHANNELS[color]
|
|
|
|
def row_bytes(w):
|
|
return (w * bits + 7) // 8
|
|
|
|
passes = [(width, height)] if interlace == 0 else [
|
|
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
|
|
# (длина строки с байтом фильтра, байт в проходе)
|
|
self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0]
|
|
self.expected = sum(total for _, total in self.passes)
|
|
self.index, self.pos = 0, 0
|
|
self.total = 0
|
|
|
|
def feed(self, data: bytes) -> None:
|
|
view = memoryview(data)
|
|
self.total += len(data)
|
|
if self.total > self.expected:
|
|
raise Damaged("more pixel data than the header says")
|
|
offset = 0
|
|
while offset < len(view):
|
|
stride, size = self.passes[self.index]
|
|
take = min(len(view) - offset, size - self.pos)
|
|
first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride
|
|
if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES):
|
|
raise Damaged("invalid filter type")
|
|
self.pos += take
|
|
offset += take
|
|
if self.pos == size:
|
|
self.index, self.pos = self.index + 1, 0
|
|
|
|
def done(self) -> bool:
|
|
return self.total == self.expected
|
|
|
|
|
|
def check_png(file: BinaryIO) -> tuple[int, int]:
|
|
if file.read(8) != PNG_SIGNATURE:
|
|
raise Damaged("signature")
|
|
rows = inflater = None
|
|
width = height = 0
|
|
seen_idat = idat_finished = have_plte = False
|
|
color = None
|
|
first = True
|
|
while True:
|
|
header = file.read(8)
|
|
if len(header) < 8:
|
|
raise Damaged("truncated: no IEND")
|
|
length, kind = struct.unpack(">I4s", header)
|
|
if length > 0x7FFFFFFF:
|
|
raise Damaged("chunk length")
|
|
if first and kind != b"IHDR":
|
|
raise Damaged("IHDR must be first")
|
|
crc = zlib.crc32(kind)
|
|
keep = kind in (b"IHDR",)
|
|
kept = b""
|
|
left = length
|
|
while left:
|
|
piece = file.read(min(CRC_CHUNK, left))
|
|
if not piece:
|
|
raise Damaged("truncated chunk")
|
|
left -= len(piece)
|
|
crc = zlib.crc32(piece, crc)
|
|
if keep:
|
|
kept += piece
|
|
elif kind == b"IDAT":
|
|
if rows is None or idat_finished:
|
|
raise Damaged("IDAT out of order")
|
|
feed_inflate(inflater, rows, piece)
|
|
stored = file.read(4)
|
|
if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF:
|
|
raise Damaged(f"CRC mismatch in {kind!r}")
|
|
|
|
if kind == b"IHDR":
|
|
if not first or length != 13:
|
|
raise Damaged("IHDR")
|
|
width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept)
|
|
if width > 0x7FFFFFFF or height > 0x7FFFFFFF:
|
|
raise Damaged("dimensions")
|
|
check_pixels(width, height) # до распаковки
|
|
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]:
|
|
raise Damaged("color type / bit depth")
|
|
if compression != 0 or filter_method != 0 or interlace not in (0, 1):
|
|
raise Damaged("IHDR methods")
|
|
rows = PngRows(width, height, depth, color, interlace)
|
|
inflater = zlib.decompressobj()
|
|
elif kind == b"PLTE":
|
|
have_plte = True
|
|
elif kind == b"IDAT":
|
|
seen_idat = True
|
|
elif kind == b"IEND":
|
|
break
|
|
if seen_idat and kind != b"IDAT":
|
|
idat_finished = True
|
|
first = False
|
|
|
|
if file.read(1):
|
|
raise Damaged("data after IEND")
|
|
if color == 3 and not have_plte:
|
|
raise Damaged("palette image without PLTE")
|
|
if not seen_idat:
|
|
raise Damaged("no IDAT")
|
|
tail = inflater.flush(INFLATE_CHUNK)
|
|
if tail:
|
|
rows.feed(tail)
|
|
if not inflater.eof or inflater.unused_data or not rows.done():
|
|
raise Damaged("pixel data is incomplete")
|
|
return width, height
|
|
|
|
|
|
def feed_inflate(inflater, rows: PngRows, data: bytes) -> None:
|
|
"""Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память."""
|
|
if inflater.eof:
|
|
if data:
|
|
raise Damaged("data after the end of zlib stream")
|
|
return
|
|
out = inflater.decompress(data, INFLATE_CHUNK)
|
|
rows.feed(out)
|
|
while inflater.unconsumed_tail and not inflater.eof:
|
|
out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK)
|
|
rows.feed(out)
|
|
if inflater.eof and inflater.unused_data:
|
|
raise Damaged("data after the end of zlib stream")
|