diff --git a/openspec/changes/fix-png-pixel-validation/design.md b/openspec/changes/fix-png-pixel-validation/design.md index 047b5f35..4bdccf1c 100644 --- a/openspec/changes/fix-png-pixel-validation/design.md +++ b/openspec/changes/fix-png-pixel-validation/design.md @@ -14,6 +14,7 @@ 2. После CRC проверять PNG через stdlib zlib: IDAT составляет один поток, включая Adler-32 и признак завершения. Ограничить и входные порции, и выдачу распаковщика 64 KiB; не собирать распакованный поток или строку целиком. Проверять точную длину проходов и байты фильтров 0–4, включая все семь проходов Adam7, малые размеры и глубины 1/2/4/8/16. Любой набор остальных байтов восстанавливается обратимыми PNG-фильтрами, поэтому пиксельный буфер для проверки не нужен. Источник: [PNG, разделы 7–11](https://www.w3.org/TR/png-3/). 3. Полный `Image.load()` для PNG отвергнут: 40 MP потребуют до 160 MB пиксельного буфера на проверку. Собственный ограниченный разбор добавляет код, зато сохраняет O(64 KiB) дополнительной памяти независимо от ширины строки. Число распакованных байтов ограничено IHDR; превышение немедленно отвергается. Новых зависимостей нет. 4. Существующие проверки размера тела и пикселей выполняются первыми. Проверка идёт в существующем потоке под `image_check_concurrency`, тело ограничено `max_upload_bytes`, одновременные загрузки — `upload_concurrency`. Исключения разбора отображаются существующим `UnsupportedImage` в `415`; записей PostgreSQL/S3 до проверки нет. +5. Неиспользованные байты после завершённого zlib-потока игнорируются согласно PNG §11.2.3. Распаковщик после EOF больше не вызывается, поэтому `unused_data` не накапливается. Превышение числа распакованных пиксельных байтов по-прежнему отклоняется; разделённые посторонним чанком IDAT запрещены спецификацией PNG. ## Risks / Trade-offs diff --git a/openspec/changes/fix-png-pixel-validation/tasks.md b/openspec/changes/fix-png-pixel-validation/tasks.md index 12688faf..8946a89d 100644 --- a/openspec/changes/fix-png-pixel-validation/tasks.md +++ b/openspec/changes/fix-png-pixel-validation/tasks.md @@ -1,11 +1,11 @@ ## 1. Исправление уровня «3» -- [ ] 1.1 Добавить регрессию PNG с корректными CRC и битым IDAT, зафиксировать исходный отказ теста offline. -- [ ] 1.2 Дополнить адаптер ImageInspector ограниченной проверкой потока PNG до записи в хранилища. -- [ ] 1.3 Проверить offline повреждённые потоки, фильтры, длины строк и корректные варианты PNG/JPEG; подтвердить отсутствие записей в handler и `415` через ASGI с подменёнными портами. +- [x] 1.1 Добавить регрессию PNG с корректными CRC и битым IDAT, зафиксировать исходный отказ теста offline. +- [x] 1.2 Дополнить адаптер ImageInspector ограниченной проверкой потока PNG до записи в хранилища. +- [x] 1.3 Проверить offline повреждённые потоки, фильтры, длины строк и корректные варианты PNG/JPEG; подтвердить отсутствие записей в handler и `415` через ASGI с подменёнными портами. ## 2. Ограничения уровня «4» и передача evidence -- [ ] 2.1 Проверить ограниченность распаковки, предел пикселей до неё и добавить регрессию через LB для финального этапа, не запуская Docker. +- [x] 2.1 Проверить ограниченность распаковки, предел пикселей до неё и добавить регрессию через LB для финального этапа, не запуская Docker. - [ ] 2.2 Выполнить make check, make validate, make lint и относящиеся offline-тесты; сохранить команды, результаты и перечень обязательных динамических проверок в evidence. - [ ] 2.3 Синхронизировать спецификацию, архивировать change и выполнить строгую OpenSpec-валидацию. diff --git a/src/messenger/attachments/infrastructure/images.py b/src/messenger/attachments/infrastructure/images.py index 6d63f670..9f9dae5e 100644 --- a/src/messenger/attachments/infrastructure/images.py +++ b/src/messenger/attachments/infrastructure/images.py @@ -6,6 +6,7 @@ from PIL import Image from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.infrastructure.png import verify_png_pixels KINDS = {"PNG": ImageKind.PNG, "JPEG": ImageKind.JPEG} BOMBS = (Image.DecompressionBombError, Image.DecompressionBombWarning) @@ -44,6 +45,7 @@ class PillowImageInspector: image.load() else: image.verify() + verify_png_pixels(data) except Exception as error: raise UnsupportedImage("изображение повреждено или обрезано") from error return ImageInfo(kind, width, height) diff --git a/src/messenger/attachments/infrastructure/png.py b/src/messenger/attachments/infrastructure/png.py new file mode 100644 index 00000000..4e3aa4f9 --- /dev/null +++ b/src/messenger/attachments/infrastructure/png.py @@ -0,0 +1,97 @@ +import struct +import zlib +from collections.abc import Iterator + +BLOCK_BYTES = 65_536 +MAX_FILTER = 4 +PIXELS = { + 0: (1, (1, 2, 4, 8, 16)), + 2: (3, (8, 16)), + 3: (1, (1, 2, 4, 8)), + 4: (2, (8, 16)), + 6: (4, (8, 16)), +} +ADAM7 = ( + (0, 0, 8, 8), + (4, 0, 8, 8), + (0, 4, 4, 8), + (2, 0, 4, 4), + (0, 2, 2, 4), + (1, 0, 2, 2), + (0, 1, 1, 2), +) + + +def _scanlines(data: bytes) -> Iterator[tuple[int, int]]: + width, height, depth, color, compression, filtering, interlace = struct.unpack_from( + ">IIBBBBB", data, 16 + ) + samples, depths = PIXELS[color] + if depth not in depths or compression or filtering or interlace not in (0, 1): + raise ValueError("invalid PNG header") + passes = ADAM7 if interlace else ((0, 0, 1, 1),) + for x, y, dx, dy in passes: + columns, rows = (width - x + dx - 1) // dx, (height - y + dy - 1) // dy + if columns > 0 and rows > 0: + stride = 1 + (columns * samples * depth + 7) // 8 + yield stride, stride * rows + + +def _idat(data: bytes) -> Iterator[memoryview]: + view = memoryview(data) + offset = 8 + started = ended = False + while offset < len(data): + size = struct.unpack_from(">I", data, offset)[0] + kind = data[offset + 4 : offset + 8] + start, end = offset + 8, offset + 8 + size + if kind == b"IDAT": + if ended: + raise ValueError("nonconsecutive PNG image data") + started = True + for position in range(start, end, BLOCK_BYTES): + yield view[position : min(position + BLOCK_BYTES, end)] + elif started: + ended = True + if kind == b"IEND": + break + offset = end + 4 + + +def _inflate(data: bytes) -> Iterator[bytes]: + decoder = zlib.decompressobj() + for block in _idat(data): + if decoder.eof: + continue + pending = block + while True: + decoded = decoder.decompress(pending, BLOCK_BYTES) + yield decoded + pending = decoder.unconsumed_tail + if decoder.eof or (not pending and len(decoded) < BLOCK_BYTES): + break + if not decoder.eof: + raise ValueError("incomplete PNG compressed data") + + +def verify_png_pixels(data: bytes) -> None: + passes = _scanlines(data) + stride, remaining = next(passes) + offset = 0 + for decoded in _inflate(data): + cursor = 0 + while cursor < len(decoded): + if not remaining: + stride, remaining = next(passes, (0, 0)) + offset = 0 + if not remaining: + raise ValueError("extra PNG pixel data") + size = min(remaining, len(decoded) - cursor) + filters = decoded[cursor + (-offset % stride) : cursor + size : stride] + if max(filters, default=0) > MAX_FILTER: + raise ValueError("invalid PNG filter") + cursor += size + offset += size + remaining -= size + if remaining or next(passes, None) is not None: + raise ValueError("incomplete PNG pixel data") diff --git a/student-tests/api/test_attachments.py b/student-tests/api/test_attachments.py index b29bd54e..055eccdc 100644 --- a/student-tests/api/test_attachments.py +++ b/student-tests/api/test_attachments.py @@ -10,7 +10,7 @@ from httpx import AsyncClient, Response from conftest import actor, make_chat, register, resolve, running_app, scalar from fakes import MemoryObjectStorage from messenger.attachments.application.ports import ObjectStorage -from pictures import GIF, JPEG, SAMPLE, WEBP, png +from pictures import BROKEN_PNG, GIF, JPEG, SAMPLE, WEBP, png LIMIT = 10_485_760 PNG = {"Content-Type": "image/png"} @@ -181,6 +181,7 @@ async def test_pixel_limit_is_read_from_header( @pytest.mark.parametrize( ("content", "header"), [ + (BROKEN_PNG, "image/png"), (JPEG, "image/png"), (GIF, "image/png"), (WEBP, "image/png"), diff --git a/student-tests/pictures.py b/student-tests/pictures.py index f53a5296..9b97353f 100644 --- a/student-tests/pictures.py +++ b/student-tests/pictures.py @@ -14,6 +14,28 @@ def chunk(kind: bytes, data: bytes) -> bytes: return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc) +def png_from_idat( + data: bytes, + width: int = 1, + height: int = 1, + *, + depth: int = 8, + color: int = 2, + interlace: int = 0, + chunk_size: int = 65_536, +) -> bytes: + header = struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + palette = chunk(b"PLTE", bytes(range(256)) * 3) if color == 3 else b"" + blocks = [ + chunk(b"IDAT", data[start : start + chunk_size]) + for start in range(0, len(data), chunk_size) + ] + return SIGNATURE + chunk(b"IHDR", header) + palette + b"".join(blocks) + chunk(b"IEND", b"") + + +BROKEN_PNG = png_from_idat(b"not a zlib stream") + + def png(width: int, height: int, *, size: int | None = None, rows: bool = True) -> bytes: header = struct.pack(">IIBBBBB", width, height, 1, 0, 0, 0, 0) raw = (b"\x00" * (1 + (width + 7) // 8)) * height if rows else b"\x00" diff --git a/student-tests/stack/test_attachment_faults.py b/student-tests/stack/test_attachment_faults.py index 66af9fe0..dcc70ef0 100644 --- a/student-tests/stack/test_attachment_faults.py +++ b/student-tests/stack/test_attachment_faults.py @@ -15,7 +15,7 @@ import httpx import pytest from httpx import AsyncClient, Response -from pictures import GIF, JPEG, SAMPLE, WEBP, png +from pictures import BROKEN_PNG, GIF, JPEG, SAMPLE, WEBP, png from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited pytestmark = pytest.mark.stack @@ -310,6 +310,7 @@ async def test_user_errors_never_become_500(http: AsyncClient) -> None: ) expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413) for label, content, kind in ( + ("PNG с корректными CRC и битым IDAT", BROKEN_PNG, "image/png"), ("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"), ("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"), ("JPEG как image/png", JPEG, "image/png"), diff --git a/student-tests/unit/test_image_inspection.py b/student-tests/unit/test_image_inspection.py new file mode 100644 index 00000000..d1498d75 --- /dev/null +++ b/student-tests/unit/test_image_inspection.py @@ -0,0 +1,213 @@ +import tracemalloc +import zlib +from io import BytesIO + +import pytest +from PIL import Image, ImageFile + +from messenger.attachments.domain.errors import PayloadTooLarge, UnsupportedImage +from messenger.attachments.domain.values import ImageInfo, ImageKind +from messenger.attachments.infrastructure import png as png_decoder +from messenger.attachments.infrastructure.images import PillowImageInspector +from pictures import BROKEN_PNG, JPEG, SAMPLE, chunk, png, png_from_idat + +FORMATS = [(0, d, 1) for d in (1, 2, 4, 8, 16)] + [ + (color, depth, samples) + for color, depths, samples in ( + (2, (8, 16), 3), + (3, (1, 2, 4, 8), 1), + (4, (8, 16), 2), + (6, (8, 16), 4), + ) + for depth in depths +] +ADAM7_GRID = ( + "16462646", + "77777777", + "56565656", + "77777777", + "36463646", + "77777777", + "56565656", + "77777777", +) +PIXEL = zlib.compress(b"\x00\x11\x22\x33") +BAD_STREAMS = [ + pytest.param(b"not a zlib stream", id="invalid-zlib"), + pytest.param(PIXEL[:-1], id="truncated-zlib-footer"), + pytest.param(PIXEL[:-1] + bytes([PIXEL[-1] ^ 1]), id="bad-adler32"), + pytest.param(zlib.compress(b"\x00\x11\x22"), id="short-row"), + pytest.param(zlib.compress(b"\x00\x11\x22\x33\x44"), id="long-row"), + pytest.param(zlib.compress(b"\x05\x11\x22\x33"), id="invalid-filter"), +] + + +def scanlines(width: int, height: int, bits: int, interlace: int, filtering: int = 0) -> bytes: + rows = [] + for number in range(1, 8) if interlace else (0,): + for y in range(height): + columns = ( + sum(int(ADAM7_GRID[y % 8][x % 8]) == number for x in range(width)) + if interlace + else width + ) + if columns: + rows.append(bytes([filtering]) + b"\x00" * ((columns * bits + 7) // 8)) + return b"".join(rows) + + +async def test_png_with_valid_crc_and_invalid_zlib_is_rejected() -> None: + assert len(BROKEN_PNG) == 74 + with Image.open(BytesIO(BROKEN_PNG)) as image: + image.verify() + with Image.open(BytesIO(BROKEN_PNG)) as image, pytest.raises(OSError): + image.load() + with pytest.raises(UnsupportedImage): + await PillowImageInspector(40_000_000, 1).inspect(BROKEN_PNG, ImageKind.PNG) + + +@pytest.mark.parametrize("stream", BAD_STREAMS) +async def test_corrupt_pixels_are_rejected_even_with_valid_chunk_crc(stream: bytes) -> None: + data = png_from_idat(stream, chunk_size=1) + with Image.open(BytesIO(data)) as image: + image.verify() + with pytest.raises(UnsupportedImage): + await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) + + +async def test_idat_chunks_must_be_consecutive() -> None: + data = png_from_idat(PIXEL) + data = ( + data[:33] + + chunk(b"IDAT", PIXEL[:3]) + + chunk(b"tEXt", b"name\x00value") + + chunk(b"IDAT", PIXEL[3:]) + + data[-12:] + ) + with Image.open(BytesIO(data)) as image: + image.verify() + with pytest.raises(UnsupportedImage): + await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) + + +@pytest.mark.parametrize("padding", [b"\x00" * 131_072, b"unused buffer", zlib.compress(b"")]) +async def test_unused_idat_trailing_bytes_are_ignored(padding: bytes) -> None: + data = png_from_idat(PIXEL + padding, chunk_size=1_000_000) + with Image.open(BytesIO(data)) as image: + image.load() + assert await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) == ImageInfo( + ImageKind.PNG, 1, 1 + ) + + +@pytest.mark.parametrize(("color", "depth", "samples"), FORMATS) +@pytest.mark.parametrize(("width", "height"), [(1, 1), (1, 9), (9, 1), (5, 5), (17, 13)]) +@pytest.mark.parametrize("interlace", [0, 1]) +async def test_supported_png_formats_match_pillow_decoding( + *, color: int, depth: int, samples: int, width: int, height: int, interlace: int +) -> None: + raw = scanlines(width, height, depth * samples, interlace) + data = png_from_idat( + zlib.compress(raw), + width, + height, + depth=depth, + color=color, + interlace=interlace, + chunk_size=3, + ) + with Image.open(BytesIO(data)) as image: + image.load() + assert image.size == (width, height) + assert await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) == ImageInfo( + ImageKind.PNG, width, height + ) + + +@pytest.mark.parametrize("filtering", range(5)) +@pytest.mark.parametrize("interlace", [0, 1]) +@pytest.mark.parametrize("block_size", [3, 7, 31]) +async def test_filters_and_passes_cross_decompression_blocks( + filtering: int, interlace: int, block_size: int, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(png_decoder, "BLOCK_BYTES", block_size) + raw = scanlines(17, 13, 32, interlace, filtering) + data = png_from_idat(zlib.compress(raw), 17, 13, color=6, interlace=interlace) + with Image.open(BytesIO(data)) as image: + image.load() + assert (await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG)).width == 17 + + +@pytest.mark.parametrize("interlace", [0, 1]) +@pytest.mark.parametrize("damage", ["filter", "short", "long"]) +async def test_corruption_in_later_rows_or_passes( + interlace: int, damage: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(png_decoder, "BLOCK_BYTES", 7) + raw = bytearray(scanlines(17, 13, 32, interlace)) + if damage == "filter": + raw[-69] = 255 + elif damage == "short": + raw.pop() + else: + raw.append(0) + data = png_from_idat(zlib.compress(raw), 17, 13, color=6, interlace=interlace, chunk_size=5) + with pytest.raises(UnsupportedImage): + await PillowImageInspector(40_000_000, 1).inspect(data, ImageKind.PNG) + + +@pytest.mark.parametrize(("width", "height"), [(1_000_000, 1), (1000, 1000)]) +def test_png_validation_does_not_allocate_pixels_or_whole_rows( + width: int, height: int, monkeypatch: pytest.MonkeyPatch +) -> None: + data = png_from_idat( + zlib.compress(b"\x00" * ((width * 8 + 1) * height)), width, height, depth=16, color=6 + ) + + def forbidden_load(*args: object, **kwargs: object) -> None: + pytest.fail("PNG validation must not allocate a Pillow pixel buffer") + + monkeypatch.setattr(ImageFile.ImageFile, "load", forbidden_load) + inspector = PillowImageInspector(40_000_000, 1) + tracemalloc.start() + try: + assert inspector._inspect(data, ImageKind.PNG) == ImageInfo(ImageKind.PNG, width, height) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < 2_000_000 + + +def test_excess_inflated_data_is_rejected_with_bounded_memory() -> None: + data = png_from_idat(zlib.compress(b"\x00" * 8_000_000)) + inspector = PillowImageInspector(40_000_000, 1) + tracemalloc.start() + try: + with pytest.raises(UnsupportedImage): + inspector._inspect(data, ImageKind.PNG) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert peak < 2_000_000 + + +async def test_pixel_limit_precedes_pixel_validation(monkeypatch: pytest.MonkeyPatch) -> None: + def forbidden_validation(data: bytes) -> None: + pytest.fail("oversized PNG must be rejected before decompression") + + monkeypatch.setattr( + "messenger.attachments.infrastructure.images.verify_png_pixels", forbidden_validation + ) + with pytest.raises(PayloadTooLarge): + await PillowImageInspector(40_000_000, 1).inspect( + png(8001, 5000, rows=False), ImageKind.PNG + ) + + +async def test_exact_pixel_limit_and_existing_fixtures_still_work() -> None: + inspector = PillowImageInspector(40_000_000, 1) + assert await inspector.inspect(png(8000, 5000), ImageKind.PNG) == ImageInfo( + ImageKind.PNG, 8000, 5000 + ) + assert (await inspector.inspect(SAMPLE, ImageKind.PNG)).kind is ImageKind.PNG + assert (await inspector.inspect(JPEG, ImageKind.JPEG)).kind is ImageKind.JPEG diff --git a/student-tests/unit/test_png_upload.py b/student-tests/unit/test_png_upload.py new file mode 100644 index 00000000..b55e9b2a --- /dev/null +++ b/student-tests/unit/test_png_upload.py @@ -0,0 +1,77 @@ +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock +from uuid import uuid4 + +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient +from starlette.types import Scope + +from fakes import NullLogger, RecordingPublisher +from messenger.attachments.application.bus import AttachmentsCommandBus +from messenger.attachments.application.commands import UploadAttachment +from messenger.attachments.application.handlers import UploadAttachmentHandler, UploadSlots +from messenger.attachments.application.ports import ( + AttachmentRepository, + AttachmentsUnitOfWork, + ChatMembership, + ObjectStorage, +) +from messenger.attachments.entrypoints.errors import ERROR_CODES +from messenger.attachments.entrypoints.http import uploads_router +from messenger.attachments.infrastructure.images import PillowImageInspector +from messenger.shared.application.ports import Clock, IdGenerator +from messenger.shared.entrypoints.errors import install +from messenger.shared.entrypoints.pipeline import RequestPipeline +from pictures import BROKEN_PNG + + +async def test_broken_png_returns_415_before_any_storage_write() -> None: + uow = Mock(spec=AttachmentsUnitOfWork) + uow.attachments = AsyncMock(spec=AttachmentRepository) + uow.collect_new_events.return_value = [] + storage = AsyncMock(spec=ObjectStorage) + membership = AsyncMock(spec=ChatMembership) + membership.is_member.return_value = True + logger = NullLogger() + publisher = RecordingPublisher() + handler = UploadAttachmentHandler( + uow, + membership, + PillowImageInspector(40_000_000, 1), + storage, + clock=Mock(spec=Clock), + ids=Mock(spec=IdGenerator), + limit=10_485_760, + slots=UploadSlots(1), + logger=logger, + ) + bus = AttachmentsCommandBus({UploadAttachment: handler}, uow, publisher) + container = SimpleNamespace(get=AsyncMock(return_value=bus)) + owner, chat = uuid4(), uuid4() + + async def guard(scope: Scope) -> None: + scope["state"].update(actor=owner, dishka_container=container) + + app = FastAPI() + app.include_router(uploads_router) + install(app, logger, ERROR_CODES, lambda _: "Bearer", "offline-test") + transport = ASGITransport( + app=RequestPipeline(app, logger, 10_485_760, 5, guard, "offline-test") + ) + async with AsyncClient(transport=transport, base_url="http://testserver") as http: + response = await http.post( + f"/api/v1/chats/{chat}/attachments?filename=broken.png", + headers={"Content-Type": "image/png", "X-Request-Id": "png-regression"}, + content=BROKEN_PNG, + ) + assert response.status_code == 415, response.text + assert response.json()["error"]["code"] == "unsupported_media_type" + assert response.json()["error"]["request_id"] == "png-regression" + assert response.headers["X-Request-Id"] == "png-regression" + assert response.headers["X-Instance-Id"] == "offline-test" + membership.is_member.assert_awaited_once_with(chat, owner) + uow.transaction.assert_not_called() + uow.commit.assert_not_called() + assert uow.attachments.mock_calls == [] + assert storage.mock_calls == [] + assert publisher.events == []