JPEG проверяется через djpeg -strict: обрезанный поток с дописанным EOI теперь даёт 415. Память nginx больше не зависит от тел файлов: загрузка идёт потоком до API, скачивание без временных файлов, воркеров 2. API не закрывает соединение при недочитанном теле, что давало HTML 502; nginx сам отклоняет тела больше лимитов API. Лимит памяти API поднят до 512 MiB под проверку CMYK-JPEG на 40 Мпикс. Добавлен стресс-тест через LB.
92 lines
3.4 KiB
Python
92 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""JPEG integrity matrix and peak memory of the checker.
|
|
docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py"""
|
|
import io
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
sys.path.insert(0, "/app")
|
|
|
|
from PIL import Image # noqa: E402
|
|
|
|
from app.errors import ApiError # noqa: E402
|
|
from app.imagecheck import DJPEG, inspect_image # noqa: E402
|
|
|
|
|
|
def encode(img, **kw):
|
|
out = io.BytesIO()
|
|
img.save(out, "JPEG", **kw)
|
|
return out.getvalue()
|
|
|
|
|
|
def verdict(data):
|
|
with tempfile.SpooledTemporaryFile(max_size=1048576) as spool:
|
|
spool.write(data)
|
|
try:
|
|
inspect_image(spool, len(data), "image/jpeg")
|
|
return "accepted"
|
|
except ApiError as exc:
|
|
return "rejected %d" % exc.status
|
|
|
|
|
|
def child_peak_kib(data):
|
|
with tempfile.NamedTemporaryFile() as handle:
|
|
handle.write(data)
|
|
handle.flush()
|
|
handle.seek(0)
|
|
proc = subprocess.Popen(DJPEG, stdin=handle)
|
|
peak = 0
|
|
while proc.poll() is None:
|
|
try:
|
|
with open("/proc/%d/status" % proc.pid) as status:
|
|
for line in status:
|
|
if line.startswith("VmHWM"):
|
|
peak = max(peak, int(line.split()[1]))
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.002)
|
|
return peak
|
|
|
|
|
|
noise = Image.frombytes("RGB", (1000, 750), os.urandom(1000 * 750 * 3))
|
|
base = encode(noise, quality=90)
|
|
progressive = encode(noise, quality=90, progressive=True)
|
|
flat = Image.new("RGB", (8000, 5000), (120, 30, 200))
|
|
big_base = encode(flat, quality=90)
|
|
big_progressive = encode(flat, quality=90, progressive=True)
|
|
big_444 = encode(flat, quality=90, progressive=True, subsampling=0)
|
|
big_cmyk = encode(flat.convert("CMYK"), quality=90, progressive=True)
|
|
del flat
|
|
sos = base.index(b"\xff\xda")
|
|
cases = [
|
|
("valid baseline", base, "accepted"),
|
|
("valid progressive", progressive, "accepted"),
|
|
("valid grayscale", encode(noise.convert("L")), "accepted"),
|
|
("valid CMYK", encode(noise.convert("CMYK")), "accepted"),
|
|
("valid, junk after EOI", base + b"trailing", "accepted"),
|
|
("valid 40 Mpx baseline", big_base, "accepted"),
|
|
("valid 40 Mpx progressive", big_progressive, "accepted"),
|
|
("valid 40 Mpx progressive 444", big_444, "accepted"),
|
|
("valid 40 Mpx progressive CMYK", big_cmyk, "accepted"),
|
|
("27-byte stub", bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "rejected 415"),
|
|
("first half + EOI", base[: len(base) // 2] + b"\xff\xd9", "rejected 415"),
|
|
("first half, no EOI", base[: len(base) // 2], "rejected 415"),
|
|
("progressive half + EOI", progressive[: len(progressive) // 2] + b"\xff\xd9", "rejected 415"),
|
|
("headers only + EOI", base[:sos] + b"\xff\xd9", "rejected 415"),
|
|
]
|
|
failed = 0
|
|
for name, data, expected in cases:
|
|
started = time.monotonic()
|
|
got = verdict(data)
|
|
ok = got == expected
|
|
failed += not ok
|
|
print("%-28s %9d bytes %-13s %.2fs %s" % (name, len(data), got, time.monotonic() - started, "ok" if ok else "FAIL"))
|
|
for name, data in (("40 Mpx baseline", big_base), ("40 Mpx progressive", big_progressive),
|
|
("40 Mpx progressive 444", big_444), ("40 Mpx progressive CMYK", big_cmyk)):
|
|
print("checker peak RSS, %-24s %6.1f MiB" % (name, child_peak_kib(data) / 1024))
|
|
print("RESULT: %d/%d" % (len(cases) - failed, len(cases)))
|
|
sys.exit(1 if failed else 0)
|