Files
mtusi2026lab4/uploads.py
T

352 lines
14 KiB
Python

"""Приём и проверка загружаемого изображения с ограниченной памятью.
1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше
MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном
файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса.
2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях
читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково
(CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма),
JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение.
"""
import asyncio
import hashlib
import logging
import struct
import tempfile
import threading
import time
import unicodedata
import zlib
from dataclasses import dataclass
from typing import BinaryIO
from PIL import Image
from starlette.requests import ClientDisconnect, Request
import settings
from errors import bad_request, payload_too_large, unsupported_media_type
log = logging.getLogger("messenger.uploads")
PNG = "image/png"
JPEG = "image/jpeg"
ALLOWED_TYPES = (PNG, JPEG)
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
JPEG_SIGNATURE = b"\xff\xd8\xff"
SPOOL_IN_MEMORY = 256 * 1024
INFLATE_CHUNK = 64 * 1024
CRC_CHUNK = 64 * 1024
# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке).
Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS
# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY.
_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY)
@dataclass
class Upload:
file: BinaryIO
size: int
sha256: str
@dataclass
class ImageInfo:
content_type: str
width: int
height: int
class Damaged(Exception):
pass
class TooManyPixels(Exception):
pass
# ---------- имя файла и заголовки ----------
def validate_filename(value: str | None) -> str:
"""filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов."""
if value is None:
raise bad_request("Query parameter filename is required")
if not 1 <= len(value) <= 255 or not value.strip():
raise bad_request("filename must be 1-255 characters")
if value in (".", ".."):
raise bad_request("filename is not allowed")
for char in value:
if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"):
raise bad_request("filename contains forbidden characters")
return value
def declared_type(header: str | None) -> str:
base = (header or "").split(";", 1)[0].strip().lower()
if base not in ALLOWED_TYPES:
raise unsupported_media_type("Content-Type must be image/png or image/jpeg")
return base
def declared_length(header: str | None) -> int | None:
if header is None:
return None
if not header.isdigit():
raise bad_request("Invalid Content-Length")
length = int(header)
if length > settings.MAX_UPLOAD_BYTES:
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
return length
# ---------- приём тела ----------
async def receive(request: Request) -> Upload:
"""Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется."""
spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR)
digest = hashlib.sha256()
size = 0
deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS
stream = request.stream().__aiter__()
try:
while True:
wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic())
if wait <= 0:
raise bad_request("Upload took too long")
try:
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
except StopAsyncIteration:
break
except asyncio.TimeoutError:
raise bad_request("Upload stalled or took too long") from None
if not chunk:
continue
size += len(chunk)
if size > settings.MAX_UPLOAD_BYTES:
raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes")
digest.update(chunk)
spool.write(chunk)
except ClientDisconnect:
spool.close()
log.info("Upload aborted by the client after %s bytes, nothing stored", size)
raise bad_request("Upload was interrupted") from None
except BaseException:
spool.close()
raise
if size == 0:
spool.close()
raise bad_request("Empty body")
return Upload(spool, size, digest.hexdigest())
async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None:
"""Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв.
Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается.
"""
read = 0
deadline = time.monotonic() + seconds
try:
stream = request.stream().__aiter__()
while read <= limit:
wait = deadline - time.monotonic()
if wait <= 0:
return
try:
chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait)
except (StopAsyncIteration, asyncio.TimeoutError):
return
read += len(chunk)
except (ClientDisconnect, RuntimeError):
return
# ---------- проверка содержимого ----------
def sniff(head: bytes) -> str | None:
if head.startswith(PNG_SIGNATURE):
return PNG
if head.startswith(JPEG_SIGNATURE):
return JPEG
return None
def inspect(upload: Upload, declared: str) -> ImageInfo:
upload.file.seek(0)
detected = sniff(upload.file.read(16))
if detected is None:
raise unsupported_media_type("File is not a PNG or JPEG image")
if detected != declared:
raise unsupported_media_type("Content-Type does not match the file content")
upload.file.seek(0)
try:
with _decode_slots:
width, height = check_png(upload.file) if detected == PNG else check_jpeg(upload.file)
except TooManyPixels:
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
except Damaged as exc:
log.info("Rejected damaged %s: %s", detected, exc)
raise unsupported_media_type("Image is damaged or not supported") from None
except Image.DecompressionBombError:
raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None
except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc:
log.info("Rejected damaged %s: %s", detected, type(exc).__name__)
raise unsupported_media_type("Image is damaged or not supported") from None
finally:
upload.file.seek(0)
return ImageInfo(detected, width, height)
def check_pixels(width: int, height: int) -> None:
if width < 1 or height < 1:
raise Damaged("zero dimension")
if width * height > settings.MAX_IMAGE_PIXELS:
raise TooManyPixels()
def check_jpeg(file: BinaryIO) -> tuple[int, int]:
with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS
width, height = image.size
check_pixels(width, height)
# Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8:
# 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB.
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
image.load()
return width, height
# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR).
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
FILTER_BYTES = bytes(range(5))
class PngRows:
"""Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки."""
def __init__(self, width, height, depth, color, interlace):
bits = depth * PNG_CHANNELS[color]
def row_bytes(w):
return (w * bits + 7) // 8
passes = [(width, height)] if interlace == 0 else [
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
# (длина строки с байтом фильтра, байт в проходе)
self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0]
self.expected = sum(total for _, total in self.passes)
self.index, self.pos = 0, 0
self.total = 0
def feed(self, data: bytes) -> None:
view = memoryview(data)
self.total += len(data)
if self.total > self.expected:
raise Damaged("more pixel data than the header says")
offset = 0
while offset < len(view):
stride, size = self.passes[self.index]
take = min(len(view) - offset, size - self.pos)
first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride
if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES):
raise Damaged("invalid filter type")
self.pos += take
offset += take
if self.pos == size:
self.index, self.pos = self.index + 1, 0
def done(self) -> bool:
return self.total == self.expected
def check_png(file: BinaryIO) -> tuple[int, int]:
if file.read(8) != PNG_SIGNATURE:
raise Damaged("signature")
rows = inflater = None
width = height = 0
seen_idat = idat_finished = have_plte = False
color = None
first = True
while True:
header = file.read(8)
if len(header) < 8:
raise Damaged("truncated: no IEND")
length, kind = struct.unpack(">I4s", header)
if length > 0x7FFFFFFF:
raise Damaged("chunk length")
if first and kind != b"IHDR":
raise Damaged("IHDR must be first")
crc = zlib.crc32(kind)
keep = kind in (b"IHDR",)
kept = b""
left = length
while left:
piece = file.read(min(CRC_CHUNK, left))
if not piece:
raise Damaged("truncated chunk")
left -= len(piece)
crc = zlib.crc32(piece, crc)
if keep:
kept += piece
elif kind == b"IDAT":
if rows is None or idat_finished:
raise Damaged("IDAT out of order")
feed_inflate(inflater, rows, piece)
stored = file.read(4)
if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF:
raise Damaged(f"CRC mismatch in {kind!r}")
if kind == b"IHDR":
if not first or length != 13:
raise Damaged("IHDR")
width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept)
if width > 0x7FFFFFFF or height > 0x7FFFFFFF:
raise Damaged("dimensions")
check_pixels(width, height) # до распаковки
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]:
raise Damaged("color type / bit depth")
if compression != 0 or filter_method != 0 or interlace not in (0, 1):
raise Damaged("IHDR methods")
rows = PngRows(width, height, depth, color, interlace)
inflater = zlib.decompressobj()
elif kind == b"PLTE":
have_plte = True
elif kind == b"IDAT":
seen_idat = True
elif kind == b"IEND":
break
if seen_idat and kind != b"IDAT":
idat_finished = True
first = False
if file.read(1):
raise Damaged("data after IEND")
if color == 3 and not have_plte:
raise Damaged("palette image without PLTE")
if not seen_idat:
raise Damaged("no IDAT")
tail = inflater.flush(INFLATE_CHUNK)
if tail:
rows.feed(tail)
if not inflater.eof or inflater.unused_data or not rows.done():
raise Damaged("pixel data is incomplete")
return width, height
def feed_inflate(inflater, rows: PngRows, data: bytes) -> None:
"""Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память."""
if inflater.eof:
if data:
raise Damaged("data after the end of zlib stream")
return
out = inflater.decompress(data, INFLATE_CHUNK)
rows.feed(out)
while inflater.unconsumed_tail and not inflater.eof:
out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK)
rows.feed(out)
if inflater.eof and inflater.unused_data:
raise Damaged("data after the end of zlib stream")