forked from mtusi2026/mtusi2026lab4
lab4: память загрузок и S3 под лимитом, документация защиты
- тело загрузки: один буфер ровно Content-Length, без копий до S3 и djpeg; - PNG: zlib получает вход кусками по 64 KiB (unconsumed_tail копировал остаток IDAT); - djpeg, убитый OOM-killer'ом, -> 503 (ресурсы сервера), а не 415; - api: memswap_limit = memory (жёсткий лимит без swap); S3: GOMEMLIMIT и лимиты одновременных загрузок SeaweedFS, память 1g; - docs: CONSISTENCY.md, DEFENSE.md, WINDOWS.md, DATA_MODEL.md под lab4; scripts/demo_attachments.py для сценария защиты; CI и collect_evidence под lab4; - bench/reconcile_faults.py: отчёты трека 2 (ошибки с обеих сторон, прерванный apply). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a2d867bec6
commit
4ea6924db1
@@ -382,14 +382,14 @@ async def upload_attachment(request: Request) -> JSONResponse:
|
||||
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
|
||||
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
|
||||
declared = attachments.declared_type(request)
|
||||
attachments.declared_length(request, settings.max_upload_bytes)
|
||||
length = attachments.declared_length(request, settings.max_upload_bytes)
|
||||
slots: asyncio.Semaphore = request.app.state.upload_slots
|
||||
try:
|
||||
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
|
||||
try:
|
||||
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots)
|
||||
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length)
|
||||
finally:
|
||||
slots.release()
|
||||
except ApiError:
|
||||
|
||||
@@ -164,7 +164,7 @@ class Busy(Unavailable):
|
||||
|
||||
@dataclass
|
||||
class Upload:
|
||||
data: bytes
|
||||
data: bytearray
|
||||
info: images.ImageInfo
|
||||
sha256: bytes
|
||||
|
||||
@@ -227,14 +227,19 @@ def declared_length(request: Request, limit: int) -> int | None:
|
||||
return length
|
||||
|
||||
|
||||
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore) -> Upload:
|
||||
"""Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format
|
||||
and the pixel count are known after the first bytes, so a picture that is too big or not a
|
||||
PNG/JPEG is rejected without reading/decoding the rest."""
|
||||
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
|
||||
length: int | None) -> Upload:
|
||||
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
|
||||
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
|
||||
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
|
||||
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
|
||||
reading/decoding the rest."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + settings.upload_read_timeout_seconds
|
||||
limit = settings.max_upload_bytes
|
||||
buffer = bytearray()
|
||||
buffer = bytearray(length) if length else bytearray()
|
||||
view = memoryview(buffer) if length else None
|
||||
filled = 0
|
||||
digest = hashlib.sha256()
|
||||
info: images.ImageInfo | None = None
|
||||
while True:
|
||||
@@ -247,27 +252,36 @@ async def read_image(reader: BodyReader, settings: Settings, declared: str, veri
|
||||
raise InvalidRequest("Тело запроса не получено вовремя") from None
|
||||
if chunk is None:
|
||||
break
|
||||
if len(buffer) + len(chunk) > limit:
|
||||
if filled + len(chunk) > (length or limit):
|
||||
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
||||
buffer += chunk
|
||||
if view is not None:
|
||||
view[filled:filled + len(chunk)] = chunk
|
||||
else:
|
||||
buffer += chunk
|
||||
filled += len(chunk)
|
||||
digest.update(chunk)
|
||||
if info is None and buffer:
|
||||
info = _sniff(buffer, settings, declared)
|
||||
if not buffer:
|
||||
if info is None and filled:
|
||||
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
|
||||
if view is not None:
|
||||
view.release()
|
||||
if not filled:
|
||||
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
||||
if length and filled != length:
|
||||
raise InvalidRequest("Тело запроса короче Content-Length")
|
||||
if info is None:
|
||||
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
|
||||
data = bytes(buffer)
|
||||
del buffer
|
||||
try:
|
||||
async with verify_slots:
|
||||
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
|
||||
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
|
||||
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
|
||||
except images.ImageError as exc:
|
||||
raise _image_error(exc) from None
|
||||
except asyncio.TimeoutError:
|
||||
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
|
||||
return Upload(data, info, digest.digest())
|
||||
except (images.VerifierUnavailable, MemoryError):
|
||||
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
|
||||
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
|
||||
return Upload(buffer, info, digest.digest())
|
||||
|
||||
|
||||
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
|
||||
|
||||
+22
-11
@@ -45,6 +45,7 @@ JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0x
|
||||
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
|
||||
|
||||
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
|
||||
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
|
||||
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
|
||||
MAX_DIMENSION = 2**31 - 1
|
||||
|
||||
@@ -58,7 +59,11 @@ class ImageError(Exception):
|
||||
self.message = message
|
||||
|
||||
|
||||
def unsupported(message: str = "Файл не является корректным изображением PNG или JPEG") -> ImageError:
|
||||
class VerifierUnavailable(Exception):
|
||||
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
|
||||
|
||||
|
||||
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
|
||||
return ImageError(415, message)
|
||||
|
||||
|
||||
@@ -274,16 +279,18 @@ def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
|
||||
if len(body):
|
||||
raise unsupported() # data after the end of the zlib stream
|
||||
return
|
||||
pending: bytes | memoryview = body
|
||||
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
|
||||
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
|
||||
try:
|
||||
while pending:
|
||||
piece = inflater.decompress(pending, INFLATE_STEP)
|
||||
checker.feed(piece)
|
||||
pending = inflater.unconsumed_tail
|
||||
if inflater.eof:
|
||||
if pending or inflater.unused_data:
|
||||
raise unsupported()
|
||||
break
|
||||
for start in range(0, len(body), INPUT_STEP):
|
||||
pending: bytes | memoryview = body[start:start + INPUT_STEP]
|
||||
while pending:
|
||||
if inflater.eof:
|
||||
raise unsupported() # data after the end of the zlib stream
|
||||
checker.feed(inflater.decompress(pending, INFLATE_STEP))
|
||||
pending = inflater.unconsumed_tail
|
||||
if inflater.eof and inflater.unused_data:
|
||||
raise unsupported()
|
||||
except zlib.error:
|
||||
raise unsupported() from None
|
||||
|
||||
@@ -303,13 +310,17 @@ async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_
|
||||
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
|
||||
stderr = b""
|
||||
try:
|
||||
_, stderr = await asyncio.wait_for(process.communicate(bytes(data)), timeout)
|
||||
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
await process.wait() # djpeg stopped reading: the file is already known to be broken
|
||||
if process.returncode is not None and process.returncode < 0:
|
||||
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
|
||||
# SERVER, not of the file - never report it as 415.
|
||||
raise VerifierUnavailable()
|
||||
if process.returncode != 0:
|
||||
if b"Insufficient memory" in (stderr or b""):
|
||||
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
|
||||
|
||||
@@ -111,7 +111,7 @@ class Storage:
|
||||
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
|
||||
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
|
||||
await self._call("put", self.client.put_object(
|
||||
Bucket=self.bucket, Key=key, Body=bytes(data), ContentType=content_type, ContentLength=len(data),
|
||||
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
|
||||
ContentMD5=md5, Metadata=metadata), deadline)
|
||||
|
||||
async def head(self, key: str) -> ObjectHead:
|
||||
|
||||
Reference in New Issue
Block a user