Исправить замечания ревью: битый JPEG, память nginx, 502 вместо 413
JPEG проверяется через djpeg -strict: обрезанный поток с дописанным EOI теперь даёт 415. Память nginx больше не зависит от тел файлов: загрузка идёт потоком до API, скачивание без временных файлов, воркеров 2. API не закрывает соединение при недочитанном теле, что давало HTML 502; nginx сам отклоняет тела больше лимитов API. Лимит памяти API поднят до 512 MiB под проверку CMYK-JPEG на 40 Мпикс. Добавлен стресс-тест через LB.
This commit is contained in:
@@ -1,33 +1,91 @@
|
||||
import io, os, sys, tempfile, time
|
||||
#!/usr/bin/env python3
|
||||
"""JPEG integrity matrix and peak memory of the checker.
|
||||
docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py"""
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
from PIL import Image
|
||||
from app.errors import ApiError
|
||||
from app.imagecheck import inspect_image
|
||||
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
from app.errors import ApiError # noqa: E402
|
||||
from app.imagecheck import DJPEG, inspect_image # noqa: E402
|
||||
|
||||
|
||||
def encode(img, **kw):
|
||||
out = io.BytesIO()
|
||||
img.save(out, "JPEG", **kw)
|
||||
return out.getvalue()
|
||||
|
||||
def jpeg(w, h, progressive=False):
|
||||
img = Image.frombytes("RGB", (w, h), os.urandom(w * h * 3))
|
||||
b = io.BytesIO(); img.save(b, "JPEG", quality=80, progressive=progressive); return b.getvalue()
|
||||
|
||||
def verdict(data):
|
||||
f = tempfile.SpooledTemporaryFile(); f.write(data)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
inspect_image(f, len(data), "image/jpeg"); r = "accepted"
|
||||
except ApiError as e:
|
||||
r = "rejected %d" % e.status
|
||||
return r, time.monotonic() - started
|
||||
with tempfile.SpooledTemporaryFile(max_size=1048576) as spool:
|
||||
spool.write(data)
|
||||
try:
|
||||
inspect_image(spool, len(data), "image/jpeg")
|
||||
return "accepted"
|
||||
except ApiError as exc:
|
||||
return "rejected %d" % exc.status
|
||||
|
||||
ok = True
|
||||
for name, data, expect in [
|
||||
("valid 3000x2000 baseline", jpeg(3000, 2000), "accepted"),
|
||||
("valid 3000x2000 progressive", jpeg(3000, 2000, True), "accepted"),
|
||||
|
||||
def child_peak_kib(data):
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
proc = subprocess.Popen(DJPEG, stdin=handle)
|
||||
peak = 0
|
||||
while proc.poll() is None:
|
||||
try:
|
||||
with open("/proc/%d/status" % proc.pid) as status:
|
||||
for line in status:
|
||||
if line.startswith("VmHWM"):
|
||||
peak = max(peak, int(line.split()[1]))
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.002)
|
||||
return peak
|
||||
|
||||
|
||||
noise = Image.frombytes("RGB", (1000, 750), os.urandom(1000 * 750 * 3))
|
||||
base = encode(noise, quality=90)
|
||||
progressive = encode(noise, quality=90, progressive=True)
|
||||
flat = Image.new("RGB", (8000, 5000), (120, 30, 200))
|
||||
big_base = encode(flat, quality=90)
|
||||
big_progressive = encode(flat, quality=90, progressive=True)
|
||||
big_444 = encode(flat, quality=90, progressive=True, subsampling=0)
|
||||
big_cmyk = encode(flat.convert("CMYK"), quality=90, progressive=True)
|
||||
del flat
|
||||
sos = base.index(b"\xff\xda")
|
||||
cases = [
|
||||
("valid baseline", base, "accepted"),
|
||||
("valid progressive", progressive, "accepted"),
|
||||
("valid grayscale", encode(noise.convert("L")), "accepted"),
|
||||
("valid CMYK", encode(noise.convert("CMYK")), "accepted"),
|
||||
("valid, junk after EOI", base + b"trailing", "accepted"),
|
||||
("valid 40 Mpx baseline", big_base, "accepted"),
|
||||
("valid 40 Mpx progressive", big_progressive, "accepted"),
|
||||
("valid 40 Mpx progressive 444", big_444, "accepted"),
|
||||
("valid 40 Mpx progressive CMYK", big_cmyk, "accepted"),
|
||||
("27-byte stub", bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "rejected 415"),
|
||||
]:
|
||||
r, t = verdict(data); ok &= r == expect; print("%-34s %-13s %.2fs %s" % (name, r, t, "ok" if r == expect else "FAIL"))
|
||||
big = jpeg(3000, 2000)
|
||||
sos = big.index(b"\xff\xda")
|
||||
cut = big[: len(big) // 2]
|
||||
r, t = verdict(cut); ok &= r == "rejected 415"; print("%-34s %-13s %.2fs %s" % ("scan cut in half, no EOI", r, t, "ok" if r == "rejected 415" else "FAIL"))
|
||||
noscan = big[:sos] + b"\xff\xd9"
|
||||
r, t = verdict(noscan); ok &= r == "rejected 415"; print("%-34s %-13s %.2fs %s" % ("headers only + EOI", r, t, "ok" if r == "rejected 415" else "FAIL"))
|
||||
sys.exit(0 if ok else 1)
|
||||
("first half + EOI", base[: len(base) // 2] + b"\xff\xd9", "rejected 415"),
|
||||
("first half, no EOI", base[: len(base) // 2], "rejected 415"),
|
||||
("progressive half + EOI", progressive[: len(progressive) // 2] + b"\xff\xd9", "rejected 415"),
|
||||
("headers only + EOI", base[:sos] + b"\xff\xd9", "rejected 415"),
|
||||
]
|
||||
failed = 0
|
||||
for name, data, expected in cases:
|
||||
started = time.monotonic()
|
||||
got = verdict(data)
|
||||
ok = got == expected
|
||||
failed += not ok
|
||||
print("%-28s %9d bytes %-13s %.2fs %s" % (name, len(data), got, time.monotonic() - started, "ok" if ok else "FAIL"))
|
||||
for name, data in (("40 Mpx baseline", big_base), ("40 Mpx progressive", big_progressive),
|
||||
("40 Mpx progressive 444", big_444), ("40 Mpx progressive CMYK", big_cmyk)):
|
||||
print("checker peak RSS, %-24s %6.1f MiB" % (name, child_peak_kib(data) / 1024))
|
||||
print("RESULT: %d/%d" % (len(cases) - failed, len(cases)))
|
||||
sys.exit(1 if failed else 0)
|
||||
|
||||
Reference in New Issue
Block a user