forked from mtusi2026/mtusi2026lab4
Перенесён минимум lab3. Добавлены загрузка/скачивание PNG/JPEG через API с потоковой проверкой целостности, метаданные в PostgreSQL и байты в приватном bucket, отдельный S3-пользователь с ограниченными правами, идемпотентный bootstrap bucket, компенсация и reconciler (трек 2), тесты отказов, эксперименты по памяти и согласованности.
193 lines
7.1 KiB
Python
193 lines
7.1 KiB
Python
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
|
|
|
|
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
|
|
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: libjpeg decodes the whole entropy
|
|
stream through Pillow in 1/8 scale, so memory stays at ~1/64 of the image. Both finish before anything is stored.
|
|
"""
|
|
import struct
|
|
import time
|
|
import zlib
|
|
|
|
from PIL import Image
|
|
|
|
from app import config
|
|
from app.errors import ApiError
|
|
|
|
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
|
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
|
|
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
|
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
|
Image.MAX_IMAGE_PIXELS = None
|
|
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
|
FULL_DECODE_MAX_PIXELS = 4000000
|
|
SCAN_SECONDS = 10.0
|
|
OUT_STEP = 1 << 20
|
|
|
|
|
|
class Invalid(Exception):
|
|
pass
|
|
|
|
|
|
def invalid() -> ApiError:
|
|
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
|
|
|
|
|
|
class RowPlan:
|
|
def __init__(self, segments):
|
|
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
|
|
self.seg = 0
|
|
self.done = 0
|
|
self.pos = 0
|
|
|
|
def feed(self, data: bytes) -> None:
|
|
i, n = 0, len(data)
|
|
while i < n:
|
|
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
|
self.seg += 1
|
|
self.done = 0
|
|
if self.seg >= len(self.segments):
|
|
raise Invalid("data after the last row")
|
|
length, rows = self.segments[self.seg]
|
|
if self.pos > 0:
|
|
take = min(length - self.pos, n - i)
|
|
i += take
|
|
self.pos += take
|
|
if self.pos == length:
|
|
self.pos = 0
|
|
self.done += 1
|
|
continue
|
|
full = min((n - i) // length, rows - self.done)
|
|
if full > 0:
|
|
if max(data[i:i + full * length:length]) > 4:
|
|
raise Invalid("bad filter type")
|
|
i += full * length
|
|
self.done += full
|
|
else:
|
|
if data[i] > 4:
|
|
raise Invalid("bad filter type")
|
|
self.pos = n - i
|
|
i = n
|
|
|
|
def complete(self) -> bool:
|
|
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
|
self.seg += 1
|
|
self.done = 0
|
|
return self.seg >= len(self.segments) and self.pos == 0
|
|
|
|
|
|
def png_plan(width, height, depth, color, interlace):
|
|
bits = CHANNELS[color] * depth
|
|
if not interlace:
|
|
return RowPlan([(1 + (width * bits + 7) // 8, height)])
|
|
segments = []
|
|
for x0, y0, dx, dy in ADAM7:
|
|
w = (width - x0 + dx - 1) // dx if width > x0 else 0
|
|
h = (height - y0 + dy - 1) // dy if height > y0 else 0
|
|
if w and h:
|
|
segments.append((1 + (w * bits + 7) // 8, h))
|
|
return RowPlan(segments)
|
|
|
|
|
|
def scan_png(fileobj) -> None:
|
|
deadline = time.monotonic() + SCAN_SECONDS
|
|
fileobj.seek(0)
|
|
if fileobj.read(8) != PNG_SIGNATURE:
|
|
raise Invalid("signature")
|
|
plan = decomp = None
|
|
palette = seen_idat = idat_closed = False
|
|
color = None
|
|
first = True
|
|
while True:
|
|
if time.monotonic() > deadline:
|
|
raise Invalid("scan too slow")
|
|
head = fileobj.read(8)
|
|
if len(head) < 8:
|
|
raise Invalid("truncated before IEND")
|
|
length, kind = struct.unpack(">I4s", head)
|
|
if length > 0x7FFFFFFF:
|
|
raise Invalid("chunk length")
|
|
if first and kind != b"IHDR":
|
|
raise Invalid("IHDR must be first")
|
|
crc = zlib.crc32(kind)
|
|
if kind == b"IHDR":
|
|
body = fileobj.read(length)
|
|
if not first or length != 13 or len(body) != 13:
|
|
raise Invalid("IHDR")
|
|
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
|
|
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
|
|
raise Invalid("IHDR fields")
|
|
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
|
|
raise Invalid("color type / depth")
|
|
plan = png_plan(width, height, depth, color, interlace)
|
|
decomp = zlib.decompressobj()
|
|
crc = zlib.crc32(body, crc)
|
|
else:
|
|
remaining = length
|
|
while remaining:
|
|
buf = fileobj.read(min(remaining, 65536))
|
|
if not buf:
|
|
raise Invalid("truncated chunk")
|
|
remaining -= len(buf)
|
|
crc = zlib.crc32(buf, crc)
|
|
if kind == b"IDAT":
|
|
if idat_closed or (color == 3 and not palette):
|
|
raise Invalid("IDAT order")
|
|
seen_idat = True
|
|
out = decomp.decompress(buf, OUT_STEP)
|
|
while True:
|
|
plan.feed(out)
|
|
if not decomp.unconsumed_tail:
|
|
break
|
|
if time.monotonic() > deadline:
|
|
raise Invalid("scan too slow")
|
|
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
|
|
if kind == b"PLTE":
|
|
palette = True
|
|
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
|
raise Invalid("PLTE")
|
|
elif kind != b"IDAT" and seen_idat:
|
|
idat_closed = True
|
|
tail = fileobj.read(4)
|
|
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
|
|
raise Invalid("crc")
|
|
first = False
|
|
if kind == b"IEND":
|
|
if length or not seen_idat:
|
|
raise Invalid("IEND")
|
|
break
|
|
if not decomp.eof or decomp.unused_data or not plan.complete():
|
|
raise Invalid("pixel data does not match IHDR")
|
|
|
|
|
|
def inspect_image(fileobj, size: int, content_type: str) -> None:
|
|
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
|
|
try:
|
|
fileobj.seek(0)
|
|
img = Image.open(fileobj)
|
|
width, height = img.size
|
|
if img.format != FORMATS[content_type]:
|
|
raise invalid()
|
|
if width * height > config.MAX_IMAGE_PIXELS:
|
|
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
|
|
small = width * height <= FULL_DECODE_MAX_PIXELS
|
|
if content_type == "image/png":
|
|
scan_png(fileobj)
|
|
else:
|
|
fileobj.seek(max(0, size - 1024))
|
|
if b"\xff\xd9" not in fileobj.read():
|
|
raise invalid()
|
|
if not small:
|
|
fileobj.seek(0)
|
|
scaled = Image.open(fileobj)
|
|
scaled.draft(scaled.mode, (1, 1))
|
|
scaled.load()
|
|
if small:
|
|
fileobj.seek(0)
|
|
Image.open(fileobj).load()
|
|
except ApiError:
|
|
raise
|
|
except Exception:
|
|
raise invalid() from None
|
|
finally:
|
|
fileobj.seek(0)
|