Files
mtusi2026lab4/app/imagecheck.py
T
alenchha 078a8341b5 ЛР 4: вложения в S3 (MinIO), reconciler БД<->S3, отчёт и evidence
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API
с потоковой проверкой целостности, метаданные в PostgreSQL и байты в
приватном bucket, отдельный S3-пользователь с ограниченными правами,
идемпотентный bootstrap bucket, компенсация и reconciler (трек 2),
тесты отказов, эксперименты по памяти и согласованности.
2026-11-02 16:59:17 +03:00

193 lines
7.1 KiB
Python

"""Integrity checks for uploaded PNG/JPEG with bounded memory.
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: libjpeg decodes the whole entropy
stream through Pillow in 1/8 scale, so memory stays at ~1/64 of the image. Both finish before anything is stored.
"""
import struct
import time
import zlib
from PIL import Image
from app import config
from app.errors import ApiError
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
Image.MAX_IMAGE_PIXELS = None
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
FULL_DECODE_MAX_PIXELS = 4000000
SCAN_SECONDS = 10.0
OUT_STEP = 1 << 20
class Invalid(Exception):
pass
def invalid() -> ApiError:
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
class RowPlan:
def __init__(self, segments):
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
self.seg = 0
self.done = 0
self.pos = 0
def feed(self, data: bytes) -> None:
i, n = 0, len(data)
while i < n:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
if self.seg >= len(self.segments):
raise Invalid("data after the last row")
length, rows = self.segments[self.seg]
if self.pos > 0:
take = min(length - self.pos, n - i)
i += take
self.pos += take
if self.pos == length:
self.pos = 0
self.done += 1
continue
full = min((n - i) // length, rows - self.done)
if full > 0:
if max(data[i:i + full * length:length]) > 4:
raise Invalid("bad filter type")
i += full * length
self.done += full
else:
if data[i] > 4:
raise Invalid("bad filter type")
self.pos = n - i
i = n
def complete(self) -> bool:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
return self.seg >= len(self.segments) and self.pos == 0
def png_plan(width, height, depth, color, interlace):
bits = CHANNELS[color] * depth
if not interlace:
return RowPlan([(1 + (width * bits + 7) // 8, height)])
segments = []
for x0, y0, dx, dy in ADAM7:
w = (width - x0 + dx - 1) // dx if width > x0 else 0
h = (height - y0 + dy - 1) // dy if height > y0 else 0
if w and h:
segments.append((1 + (w * bits + 7) // 8, h))
return RowPlan(segments)
def scan_png(fileobj) -> None:
deadline = time.monotonic() + SCAN_SECONDS
fileobj.seek(0)
if fileobj.read(8) != PNG_SIGNATURE:
raise Invalid("signature")
plan = decomp = None
palette = seen_idat = idat_closed = False
color = None
first = True
while True:
if time.monotonic() > deadline:
raise Invalid("scan too slow")
head = fileobj.read(8)
if len(head) < 8:
raise Invalid("truncated before IEND")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise Invalid("chunk length")
if first and kind != b"IHDR":
raise Invalid("IHDR must be first")
crc = zlib.crc32(kind)
if kind == b"IHDR":
body = fileobj.read(length)
if not first or length != 13 or len(body) != 13:
raise Invalid("IHDR")
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
raise Invalid("IHDR fields")
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
raise Invalid("color type / depth")
plan = png_plan(width, height, depth, color, interlace)
decomp = zlib.decompressobj()
crc = zlib.crc32(body, crc)
else:
remaining = length
while remaining:
buf = fileobj.read(min(remaining, 65536))
if not buf:
raise Invalid("truncated chunk")
remaining -= len(buf)
crc = zlib.crc32(buf, crc)
if kind == b"IDAT":
if idat_closed or (color == 3 and not palette):
raise Invalid("IDAT order")
seen_idat = True
out = decomp.decompress(buf, OUT_STEP)
while True:
plan.feed(out)
if not decomp.unconsumed_tail:
break
if time.monotonic() > deadline:
raise Invalid("scan too slow")
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
if kind == b"PLTE":
palette = True
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise Invalid("PLTE")
elif kind != b"IDAT" and seen_idat:
idat_closed = True
tail = fileobj.read(4)
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
raise Invalid("crc")
first = False
if kind == b"IEND":
if length or not seen_idat:
raise Invalid("IEND")
break
if not decomp.eof or decomp.unused_data or not plan.complete():
raise Invalid("pixel data does not match IHDR")
def inspect_image(fileobj, size: int, content_type: str) -> None:
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
try:
fileobj.seek(0)
img = Image.open(fileobj)
width, height = img.size
if img.format != FORMATS[content_type]:
raise invalid()
if width * height > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
small = width * height <= FULL_DECODE_MAX_PIXELS
if content_type == "image/png":
scan_png(fileobj)
else:
fileobj.seek(max(0, size - 1024))
if b"\xff\xd9" not in fileobj.read():
raise invalid()
if not small:
fileobj.seek(0)
scaled = Image.open(fileobj)
scaled.draft(scaled.mode, (1, 1))
scaled.load()
if small:
fileobj.seek(0)
Image.open(fileobj).load()
except ApiError:
raise
except Exception:
raise invalid() from None
finally:
fileobj.seek(0)