Reject corrupt PNG pixels with bounded stream validation

This commit is contained in:
Artyom
2026-11-06 12:57:32 +03:00
parent f0521fa1e0
commit 9387c3d82a
9 changed files with 420 additions and 6 deletions
@@ -14,6 +14,7 @@
2. После CRC проверять PNG через stdlib zlib: IDAT составляет один поток, включая Adler-32 и признак завершения. Ограничить и входные порции, и выдачу распаковщика 64 KiB; не собирать распакованный поток или строку целиком. Проверять точную длину проходов и байты фильтров 0–4, включая все семь проходов Adam7, малые размеры и глубины 1/2/4/8/16. Любой набор остальных байтов восстанавливается обратимыми PNG-фильтрами, поэтому пиксельный буфер для проверки не нужен. Источник: [PNG, разделы 7–11](https://www.w3.org/TR/png-3/).
3. Полный `Image.load()` для PNG отвергнут: 40 MP потребуют до 160 MB пиксельного буфера на проверку. Собственный ограниченный разбор добавляет код, зато сохраняет O(64 KiB) дополнительной памяти независимо от ширины строки. Число распакованных байтов ограничено IHDR; превышение немедленно отвергается. Новых зависимостей нет.
4. Существующие проверки размера тела и пикселей выполняются первыми. Проверка идёт в существующем потоке под `image_check_concurrency`, тело ограничено `max_upload_bytes`, одновременные загрузки — `upload_concurrency`. Исключения разбора отображаются существующим `UnsupportedImage` в `415`; записей PostgreSQL/S3 до проверки нет.
5. Неиспользованные байты после завершённого zlib-потока игнорируются согласно PNG §11.2.3. Распаковщик после EOF больше не вызывается, поэтому `unused_data` не накапливается. Превышение числа распакованных пиксельных байтов по-прежнему отклоняется; разделённые посторонним чанком IDAT запрещены спецификацией PNG.
## Risks / Trade-offs
@@ -1,11 +1,11 @@
## 1. Исправление уровня «3»
- [ ] 1.1 Добавить регрессию PNG с корректными CRC и битым IDAT, зафиксировать исходный отказ теста offline.
- [ ] 1.2 Дополнить адаптер ImageInspector ограниченной проверкой потока PNG до записи в хранилища.
- [ ] 1.3 Проверить offline повреждённые потоки, фильтры, длины строк и корректные варианты PNG/JPEG; подтвердить отсутствие записей в handler и `415` через ASGI с подменёнными портами.
- [x] 1.1 Добавить регрессию PNG с корректными CRC и битым IDAT, зафиксировать исходный отказ теста offline.
- [x] 1.2 Дополнить адаптер ImageInspector ограниченной проверкой потока PNG до записи в хранилища.
- [x] 1.3 Проверить offline повреждённые потоки, фильтры, длины строк и корректные варианты PNG/JPEG; подтвердить отсутствие записей в handler и `415` через ASGI с подменёнными портами.
## 2. Ограничения уровня «4» и передача evidence
- [ ] 2.1 Проверить ограниченность распаковки, предел пикселей до неё и добавить регрессию через LB для финального этапа, не запуская Docker.
- [x] 2.1 Проверить ограниченность распаковки, предел пикселей до неё и добавить регрессию через LB для финального этапа, не запуская Docker.
- [ ] 2.2 Выполнить make check, make validate, make lint и относящиеся offline-тесты; сохранить команды, результаты и перечень обязательных динамических проверок в evidence.
- [ ] 2.3 Синхронизировать спецификацию, архивировать change и выполнить строгую OpenSpec-валидацию.
@@ -6,6 +6,7 @@ from PIL import Image
from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage
from messenger.attachments.domain.values import ImageInfo, ImageKind
from messenger.attachments.infrastructure.png import verify_png_pixels
KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG}
BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning)
@@ -44,6 +45,7 @@ class PillowImageInspector:
image.load()
else:
image.verify()
verify_png_pixels(data)
except Exception as error:
raise UnsupportedImage("изображение повреждено или обрезано") from error
return ImageInfo(kind, width, height)
@@ -0,0 +1,97 @@
import struct
import zlib
from collections.abc import Iterator
BLOCK_BYTES = 65_536
MAX_FILTER = 4
PIXELS = {
0: (1, (1, 2, 4, 8, 16)),
2: (3, (8, 16)),
3: (1, (1, 2, 4, 8)),
4: (2, (8, 16)),
6: (4, (8, 16)),
}
ADAM7 = (
(0, 0, 8, 8),
(4, 0, 8, 8),
(0, 4, 4, 8),
(2, 0, 4, 4),
(0, 2, 2, 4),
(1, 0, 2, 2),
(0, 1, 1, 2),
)
def _scanlines(data: bytes) -> Iterator[tuple[int, int]]:
width, height, depth, color, compression, filtering, interlace = struct.unpack_from(
">IIBBBBB", data, 16
)
samples, depths = PIXELS[color]
if depth not in depths or compression or filtering or interlace not in (0, 1):
raise ValueError("invalid PNG header")
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
for x, y, dx, dy in passes:
columns, rows = (width - x + dx - 1) // dx, (height - y + dy - 1) // dy
if columns > 0 and rows > 0:
stride = 1 + (columns * samples * depth + 7) // 8
yield stride, stride * rows
def _idat(data: bytes) -> Iterator[memoryview]:
view = memoryview(data)
offset = 8
started = ended = False
while offset < len(data):
size = struct.unpack_from(">I", data, offset)[0]
kind = data[offset + 4 : offset + 8]
start, end = offset + 8, offset + 8 + size
if kind == b"IDAT":
if ended:
raise ValueError("nonconsecutive PNG image data")
started = True
for position in range(start, end, BLOCK_BYTES):
yield view[position : min(position + BLOCK_BYTES, end)]
elif started:
ended = True
if kind == b"IEND":
break
offset = end + 4
def _inflate(data: bytes) -> Iterator[bytes]:
decoder = zlib.decompressobj()
for block in _idat(data):
if decoder.eof:
continue
pending = block
while True:
decoded = decoder.decompress(pending, BLOCK_BYTES)
yield decoded
pending = decoder.unconsumed_tail
if decoder.eof or (not pending and len(decoded) < BLOCK_BYTES):
break
if not decoder.eof:
raise ValueError("incomplete PNG compressed data")
def verify_png_pixels(data: bytes) -> None:
passes = _scanlines(data)
stride, remaining = next(passes)
offset = 0
for decoded in _inflate(data):
cursor = 0
while cursor < len(decoded):
if not remaining:
stride, remaining = next(passes, (0, 0))
offset = 0
if not remaining:
raise ValueError("extra PNG pixel data")
size = min(remaining, len(decoded) - cursor)
filters = decoded[cursor + (-offset % stride) : cursor + size : stride]
if max(filters, default=0) > MAX_FILTER:
raise ValueError("invalid PNG filter")
cursor += size
offset += size
remaining -= size
if remaining or next(passes, None) is not None:
raise ValueError("incomplete PNG pixel data")
+2 -1
View File
@@ -10,7 +10,7 @@ from httpx import AsyncClient, Response
from conftest import actor, make_chat, register, resolve, running_app, scalar
from fakes import MemoryObjectStorage
from messenger.attachments.application.ports import ObjectStorage
from pictures import GIF, JPEG, SAMPLE, WEBP, png
from pictures import BROKEN_PNG, GIF, JPEG, SAMPLE, WEBP, png
LIMIT = 10_485_760
PNG = {"Content-Type": "image/png"}
@@ -181,6 +181,7 @@ async def test_pixel_limit_is_read_from_header(
@pytest.mark.parametrize(
("content", "header"),
[
(BROKEN_PNG, "image/png"),
(JPEG, "image/png"),
(GIF, "image/png"),
(WEBP, "image/png"),
+22
View File
@@ -14,6 +14,28 @@ def chunk(kind: bytes, data: bytes) -> bytes:
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
def png_from_idat(
data: bytes,
width: int = 1,
height: int = 1,
*,
depth: int = 8,
color: int = 2,
interlace: int = 0,
chunk_size: int = 65_536,
) -> bytes:
header = struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)
palette = chunk(b"PLTE", bytes(range(256)) * 3) if color == 3 else b""
blocks = [
chunk(b"IDAT", data[start : start + chunk_size])
for start in range(0, len(data), chunk_size)
]
return SIGNATURE + chunk(b"IHDR", header) + palette + b"".join(blocks) + chunk(b"IEND", b"")
BROKEN_PNG = png_from_idat(b"not a zlib stream")
def png(width: int, height: int, *, size: int | None = None, rows: bool = True) -> bytes:
header = struct.pack(">IIBBBBB", width, height, 1, 0, 0, 0, 0)
raw = (b"\x00" * (1 + (width + 7) // 8)) * height if rows else b"\x00"
@@ -15,7 +15,7 @@ import httpx
import pytest
from httpx import AsyncClient, Response
from pictures import GIF, JPEG, SAMPLE, WEBP, png
from pictures import BROKEN_PNG, GIF, JPEG, SAMPLE, WEBP, png
from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited
pytestmark = pytest.mark.stack
@@ -310,6 +310,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None:
)
expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413)
for label, content, kind in (
("PNG с корректными CRC и битым IDAT", BROKEN_PNG, "image/png"),
("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"),
("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"),
("JPEG как image/png", JPEG, "image/png"),
+213
View File
@@ -0,0 +1,213 @@
import tracemalloc
import zlib
from io import BytesIO
import pytest
from PIL import Image, ImageFile
from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage
from messenger.attachments.domain.values import ImageInfo, ImageKind
from messenger.attachments.infrastructure import png as png_decoder
from messenger.attachments.infrastructure.images import PillowImageInspector
from pictures import BROKEN_PNG, JPEG, SAMPLE, chunk, png, png_from_idat
FORMATS = [(0, d, 1) for d in (1, 2, 4, 8, 16)] + [
(color, depth, samples)
for color, depths, samples in (
(2, (8, 16), 3),
(3, (1, 2, 4, 8), 1),
(4, (8, 16), 2),
(6, (8, 16), 4),
)
for depth in depths
]
ADAM7_GRID = (
"16462646",
"77777777",
"56565656",
"77777777",
"36463646",
"77777777",
"56565656",
"77777777",
)
PIXEL = zlib.compress(b"\x00\x11\x22\x33")
BAD_STREAMS = [
pytest.param(b"not a zlib stream", id="invalid-zlib"),
pytest.param(PIXEL[:-1], id="truncated-zlib-footer"),
pytest.param(PIXEL[:-1] + bytes([PIXEL[-1] ^ 1]), id="bad-adler32"),
pytest.param(zlib.compress(b"\x00\x11\x22"), id="short-row"),
pytest.param(zlib.compress(b"\x00\x11\x22\x33\x44"), id="long-row"),
pytest.param(zlib.compress(b"\x05\x11\x22\x33"), id="invalid-filter"),
]
def scanlines(width: int, height: int, bits: int, interlace: int, filtering: int = 0) -> bytes:
rows = []
for number in range(1, 8) if interlace else (0,):
for y in range(height):
columns = (
sum(int(ADAM7_GRID[y % 8][x % 8]) == number for x in range(width))
if interlace
else width
)
if columns:
rows.append(bytes([filtering]) + b"\x00" * ((columns * bits + 7) // 8))
return b"".join(rows)
async def test_png_with_valid_crc_and_invalid_zlib_is_rejected() -> None:
assert len(BROKEN_PNG) == 74
with Image.open(BytesIO(BROKEN_PNG)) as image:
image.verify()
with Image.open(BytesIO(BROKEN_PNG)) as image, pytest.raises(OSError):
image.load()
with pytest.raises(UnsupportedImage):
await PillowImageInspector(40_000_000, 1).inspect(BROKEN_PNG, ImageKind.PNG)
@pytest.mark.parametrize("stream", BAD_STREAMS)
async def test_corrupt_pixels_are_rejected_even_with_valid_chunk_crc(stream: bytes) -> None:
data = png_from_idat(stream, chunk_size=1)
with Image.open(BytesIO(data)) as image:
image.verify()
with pytest.raises(UnsupportedImage):
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
async def test_idat_chunks_must_be_consecutive() -> None:
data = png_from_idat(PIXEL)
data = (
data[:33]
+ chunk(b"IDAT", PIXEL[:3])
+ chunk(b"tEXt", b"name\x00value")
+ chunk(b"IDAT", PIXEL[3:])
+ data[-12:]
)
with Image.open(BytesIO(data)) as image:
image.verify()
with pytest.raises(UnsupportedImage):
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
@pytest.mark.parametrize("padding", [b"\x00" * 131_072, b"unused buffer", zlib.compress(b"")])
async def test_unused_idat_trailing_bytes_are_ignored(padding: bytes) -> None:
data = png_from_idat(PIXEL + padding, chunk_size=1_000_000)
with Image.open(BytesIO(data)) as image:
image.load()
assert await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) == ImageInfo(
ImageKind.PNG, 1, 1
)
@pytest.mark.parametrize(("color", "depth", "samples"), FORMATS)
@pytest.mark.parametrize(("width", "height"), [(1, 1), (1, 9), (9, 1), (5, 5), (17, 13)])
@pytest.mark.parametrize("interlace", [0, 1])
async def test_supported_png_formats_match_pillow_decoding(
*, color: int, depth: int, samples: int, width: int, height: int, interlace: int
) -> None:
raw = scanlines(width, height, depth * samples, interlace)
data = png_from_idat(
zlib.compress(raw),
width,
height,
depth=depth,
color=color,
interlace=interlace,
chunk_size=3,
)
with Image.open(BytesIO(data)) as image:
image.load()
assert image.size == (width, height)
assert await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) == ImageInfo(
ImageKind.PNG, width, height
)
@pytest.mark.parametrize("filtering", range(5))
@pytest.mark.parametrize("interlace", [0, 1])
@pytest.mark.parametrize("block_size", [3, 7, 31])
async def test_filters_and_passes_cross_decompression_blocks(
filtering: int, interlace: int, block_size: int, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(png_decoder, "BLOCK_BYTES", block_size)
raw = scanlines(17, 13, 32, interlace, filtering)
data = png_from_idat(zlib.compress(raw), 17, 13, color=6, interlace=interlace)
with Image.open(BytesIO(data)) as image:
image.load()
assert (await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)).width == 17
@pytest.mark.parametrize("interlace", [0, 1])
@pytest.mark.parametrize("damage", ["filter", "short", "long"])
async def test_corruption_in_later_rows_or_passes(
interlace: int, damage: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(png_decoder, "BLOCK_BYTES", 7)
raw = bytearray(scanlines(17, 13, 32, interlace))
if damage == "filter":
raw[-69] = 255
elif damage == "short":
raw.pop()
else:
raw.append(0)
data = png_from_idat(zlib.compress(raw), 17, 13, color=6, interlace=interlace, chunk_size=5)
with pytest.raises(UnsupportedImage):
await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)
@pytest.mark.parametrize(("width", "height"), [(1_000_000, 1), (1000, 1000)])
def test_png_validation_does_not_allocate_pixels_or_whole_rows(
width: int, height: int, monkeypatch: pytest.MonkeyPatch
) -> None:
data = png_from_idat(
zlib.compress(b"\x00" * ((width * 8 + 1) * height)), width, height, depth=16, color=6
)
def forbidden_load(*args: object, **kwargs: object) -> None:
pytest.fail("PNG validation must not allocate a Pillow pixel buffer")
monkeypatch.setattr(ImageFile.ImageFile, "load", forbidden_load)
inspector = PillowImageInspector(40_000_000, 1)
tracemalloc.start()
try:
assert inspector._inspect(data, ImageKind.PNG) == ImageInfo(ImageKind.PNG, width, height)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert peak < 2_000_000
def test_excess_inflated_data_is_rejected_with_bounded_memory() -> None:
data = png_from_idat(zlib.compress(b"\x00" * 8_000_000))
inspector = PillowImageInspector(40_000_000, 1)
tracemalloc.start()
try:
with pytest.raises(UnsupportedImage):
inspector._inspect(data, ImageKind.PNG)
_, peak = tracemalloc.get_traced_memory()
finally:
tracemalloc.stop()
assert peak < 2_000_000
async def test_pixel_limit_precedes_pixel_validation(monkeypatch: pytest.MonkeyPatch) -> None:
def forbidden_validation(data: bytes) -> None:
pytest.fail("oversized PNG must be rejected before decompression")
monkeypatch.setattr(
"messenger.attachments.infrastructure.images.verify_png_pixels", forbidden_validation
)
with pytest.raises(PayloadTooLarge):
await PillowImageInspector(40_000_000, 1).inspect(
png(8001, 5000, rows=False), ImageKind.PNG
)
async def test_exact_pixel_limit_and_existing_fixtures_still_work() -> None:
inspector = PillowImageInspector(40_000_000, 1)
assert await inspector.inspect(png(8000, 5000), ImageKind.PNG) == ImageInfo(
ImageKind.PNG, 8000, 5000
)
assert (await inspector.inspect(SAMPLE, ImageKind.PNG)).kind is ImageKind.PNG
assert (await inspector.inspect(JPEG, ImageKind.JPEG)).kind is ImageKind.JPEG
+77
View File
@@ -0,0 +1,77 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from uuid import uuid4
from fastapi import FastAPI
from httpx import ASGITransport, AsyncClient
from starlette.types import Scope
from fakes import NullLogger, RecordingPublisher
from messenger.attachments.application.bus import AttachmentsCommandBus
from messenger.attachments.application.commands import UploadAttachment
from messenger.attachments.application.handlers import UploadAttachmentHandler, UploadSlots
from messenger.attachments.application.ports import (
AttachmentRepository,
AttachmentsUnitOfWork,
ChatMembership,
ObjectStorage,
)
from messenger.attachments.entrypoints.errors import ERROR_CODES
from messenger.attachments.entrypoints.http import uploads_router
from messenger.attachments.infrastructure.images import PillowImageInspector
from messenger.shared.application.ports import Clock, IdGenerator
from messenger.shared.entrypoints.errors import install
from messenger.shared.entrypoints.pipeline import RequestPipeline
from pictures import BROKEN_PNG
async def test_broken_png_returns_415_before_any_storage_write() -> None:
uow = Mock(spec=AttachmentsUnitOfWork)
uow.attachments = AsyncMock(spec=AttachmentRepository)
uow.collect_new_events.return_value = []
storage = AsyncMock(spec=ObjectStorage)
membership = AsyncMock(spec=ChatMembership)
membership.is_member.return_value = True
logger = NullLogger()
publisher = RecordingPublisher()
handler = UploadAttachmentHandler(
uow,
membership,
PillowImageInspector(40_000_000, 1),
storage,
clock=Mock(spec=Clock),
ids=Mock(spec=IdGenerator),
limit=10_485_760,
slots=UploadSlots(1),
logger=logger,
)
bus = AttachmentsCommandBus({UploadAttachment: handler}, uow, publisher)
container = SimpleNamespace(get=AsyncMock(return_value=bus))
owner, chat = uuid4(), uuid4()
async def guard(scope: Scope) -> None:
scope["state"].update(actor=owner, dishka_container=container)
app = FastAPI()
app.include_router(uploads_router)
install(app, logger, ERROR_CODES, lambda _: "Bearer", "offline-test")
transport = ASGITransport(
app=RequestPipeline(app, logger, 10_485_760, 5, guard, "offline-test")
)
async with AsyncClient(transport=transport, base_url="http://testserver") as http:
response = await http.post(
f"/api/v1/chats/{chat}/attachments?filename=broken.png",
headers={"Content-Type": "image/png", "X-Request-Id": "png-regression"},
content=BROKEN_PNG,
)
assert response.status_code == 415, response.text
assert response.json()["error"]["code"] == "unsupported_media_type"
assert response.json()["error"]["request_id"] == "png-regression"
assert response.headers["X-Request-Id"] == "png-regression"
assert response.headers["X-Instance-Id"] == "offline-test"
membership.is_member.assert_awaited_once_with(chat, owner)
uow.transaction.assert_not_called()
uow.commit.assert_not_called()
assert uow.attachments.mock_calls == []
assert storage.mock_calls == []
assert publisher.events == []