lab4: память загрузок и S3 под лимитом, документация защиты

- тело загрузки: один буфер ровно Content-Length, без копий до S3 и djpeg;
- PNG: zlib получает вход кусками по 64 KiB (unconsumed_tail копировал остаток IDAT);
- djpeg, убитый OOM-killer'ом, -> 503 (ресурсы сервера), а не 415;
- api: memswap_limit = memory (жёсткий лимит без swap); S3: GOMEMLIMIT и лимиты
  одновременных загрузок SeaweedFS, память 1g;
- docs: CONSISTENCY.md, DEFENSE.md, WINDOWS.md, DATA_MODEL.md под lab4;
  scripts/demo_attachments.py для сценария защиты; CI и collect_evidence под lab4;
- bench/reconcile_faults.py: отчёты трека 2 (ошибки с обеих сторон, прерванный apply).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-25 14:21:43 +03:00
co-authored by Claude Opus 5.5
parent a2d867bec6
commit 4ea6924db1
18 changed files with 701 additions and 120 deletions
+2 -2
View File
@@ -382,14 +382,14 @@ async def upload_attachment(request: Request) -> JSONResponse:
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
declared = attachments.declared_type(request)
attachments.declared_length(request, settings.max_upload_bytes)
length = attachments.declared_length(request, settings.max_upload_bytes)
slots: asyncio.Semaphore = request.app.state.upload_slots
try:
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
except asyncio.TimeoutError:
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
try:
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots)
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length)
finally:
slots.release()
except ApiError:
+29 -15
View File
@@ -164,7 +164,7 @@ class Busy(Unavailable):
@dataclass
class Upload:
data: bytes
data: bytearray
info: images.ImageInfo
sha256: bytes
@@ -227,14 +227,19 @@ def declared_length(request: Request, limit: int) -> int | None:
return length
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore) -> Upload:
"""Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format
and the pixel count are known after the first bytes, so a picture that is too big or not a
PNG/JPEG is rejected without reading/decoding the rest."""
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
length: int | None) -> Upload:
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
reading/decoding the rest."""
loop = asyncio.get_running_loop()
deadline = loop.time() + settings.upload_read_timeout_seconds
limit = settings.max_upload_bytes
buffer = bytearray()
buffer = bytearray(length) if length else bytearray()
view = memoryview(buffer) if length else None
filled = 0
digest = hashlib.sha256()
info: images.ImageInfo | None = None
while True:
@@ -247,27 +252,36 @@ async def read_image(reader: BodyReader, settings: Settings, declared: str, veri
raise InvalidRequest("Тело запроса не получено вовремя") from None
if chunk is None:
break
if len(buffer) + len(chunk) > limit:
if filled + len(chunk) > (length or limit):
raise PayloadTooLarge(f"Файл больше {limit} байт")
buffer += chunk
if view is not None:
view[filled:filled + len(chunk)] = chunk
else:
buffer += chunk
filled += len(chunk)
digest.update(chunk)
if info is None and buffer:
info = _sniff(buffer, settings, declared)
if not buffer:
if info is None and filled:
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
if view is not None:
view.release()
if not filled:
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
if length and filled != length:
raise InvalidRequest("Тело запроса короче Content-Length")
if info is None:
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
data = bytes(buffer)
del buffer
try:
async with verify_slots:
await images.verify(data, info, timeout=settings.image_verify_timeout_seconds,
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
except images.ImageError as exc:
raise _image_error(exc) from None
except asyncio.TimeoutError:
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
return Upload(data, info, digest.digest())
except (images.VerifierUnavailable, MemoryError):
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
return Upload(buffer, info, digest.digest())
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
+22 -11
View File
@@ -45,6 +45,7 @@ JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0x
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
MAX_DIMENSION = 2**31 - 1
@@ -58,7 +59,11 @@ class ImageError(Exception):
self.message = message
def unsupported(message: str = "Файл не является корректным изображением PNG или JPEG") -> ImageError:
class VerifierUnavailable(Exception):
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
return ImageError(415, message)
@@ -274,16 +279,18 @@ def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
if len(body):
raise unsupported() # data after the end of the zlib stream
return
pending: bytes | memoryview = body
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
try:
while pending:
piece = inflater.decompress(pending, INFLATE_STEP)
checker.feed(piece)
pending = inflater.unconsumed_tail
if inflater.eof:
if pending or inflater.unused_data:
raise unsupported()
break
for start in range(0, len(body), INPUT_STEP):
pending: bytes | memoryview = body[start:start + INPUT_STEP]
while pending:
if inflater.eof:
raise unsupported() # data after the end of the zlib stream
checker.feed(inflater.decompress(pending, INFLATE_STEP))
pending = inflater.unconsumed_tail
if inflater.eof and inflater.unused_data:
raise unsupported()
except zlib.error:
raise unsupported() from None
@@ -303,13 +310,17 @@ async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
stderr = b""
try:
_, stderr = await asyncio.wait_for(process.communicate(bytes(data)), timeout)
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
except (asyncio.TimeoutError, asyncio.CancelledError):
process.kill()
await process.wait()
raise
except (BrokenPipeError, ConnectionResetError):
await process.wait() # djpeg stopped reading: the file is already known to be broken
if process.returncode is not None and process.returncode < 0:
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
# SERVER, not of the file - never report it as 415.
raise VerifierUnavailable()
if process.returncode != 0:
if b"Insufficient memory" in (stderr or b""):
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
+1 -1
View File
@@ -111,7 +111,7 @@ class Storage:
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
await self._call("put", self.client.put_object(
Bucket=self.bucket, Key=key, Body=bytes(data), ContentType=content_type, ContentLength=len(data),
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
ContentMD5=md5, Metadata=metadata), deadline)
async def head(self, key: str) -> ObjectHead: