Close the source image after converting to RGBA so PNGs at the pixel limit fit the render memory limit. Strengthen the fencing test so that it fails without the attempt check, and measure worker and relay stops under load. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
532 lines
17 KiB
Python
532 lines
17 KiB
Python
"""Спек `attachment-processing`: то, что видно по HTTP на любом стенде с worker.
|
||
|
||
Thumbnail, скачивание варианта, готовность и `failed` от недекодируемого
|
||
изображения. Сценарии с остановкой worker, брокера и S3 — проверки отказов
|
||
(`test_failures.py`).
|
||
"""
|
||
|
||
import time
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from PIL import Image
|
||
|
||
import images
|
||
from api_client import Actor, Api
|
||
from contract import (
|
||
assert_error,
|
||
assert_instance_id,
|
||
assert_ok,
|
||
assert_replicas,
|
||
assert_variant_content,
|
||
assert_variant_image,
|
||
)
|
||
|
||
# сколько раз читать метаданные через LB, чтобы увидеть другую реплику
|
||
REPLICA_ATTEMPTS = 20
|
||
# допуск к цвету пикселя: JPEG теряет, но красный, синий и белый различимы
|
||
COLOR_TOLERANCE = 40
|
||
RED = (255, 0, 0)
|
||
BLUE = (0, 0, 255)
|
||
WHITE = (255, 255, 255)
|
||
BLACK = (0, 0, 0)
|
||
VARIANT_TYPES = {
|
||
"thumbnail": "image/jpeg",
|
||
"crop": "image/jpeg",
|
||
"optimized": "image/webp",
|
||
}
|
||
VARIANT_NAMES = tuple(VARIANT_TYPES)
|
||
LARGE_BODY_CLIENT_TIMEOUT_S = 30
|
||
|
||
|
||
def thumbnail_of(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
) -> Image.Image:
|
||
"""Вариант `thumbnail` готового вложения: скачан, сверен и открыт как
|
||
настоящий JPEG; настоящие размеры равны полям варианта."""
|
||
return variant_of(api, actor, attachment, "thumbnail")
|
||
|
||
|
||
def variant_of(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
name: str,
|
||
) -> Image.Image:
|
||
"""Вариант из метаданных: у вложения по одному варианту каждого имени, байты
|
||
скачаны и сверены с SHA-256 и размером, файл — настоящее изображение своего
|
||
типа и размера."""
|
||
assert sorted(item["name"] for item in attachment["variants"]) == sorted(
|
||
VARIANT_NAMES,
|
||
)
|
||
variant = next(
|
||
item for item in attachment["variants"] if item["name"] == name
|
||
)
|
||
assert variant["content_type"] == VARIANT_TYPES[name]
|
||
data = assert_variant_content(
|
||
api.attachment_content(actor, attachment["id"], variant=name),
|
||
variant,
|
||
)
|
||
return assert_variant_image(data, variant)
|
||
|
||
|
||
def assert_color(
|
||
image: Image.Image,
|
||
position: tuple[int, int],
|
||
color: tuple[int, int, int],
|
||
) -> None:
|
||
got = image.convert("RGB").getpixel(position)
|
||
assert all(
|
||
abs(a - b) <= COLOR_TOLERANCE for a, b in zip(got, color, strict=True)
|
||
), f"pixel {position}: {got}, expected about {color}"
|
||
|
||
|
||
def thumbnail_size(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat_id: str,
|
||
data: bytes,
|
||
) -> tuple[int, int]:
|
||
attachment = api.upload_ready(actor, chat_id, data)
|
||
return thumbnail_of(api, actor, attachment).size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Вложение становится готовым")
|
||
def test_attachment_becomes_ready(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
accepted = api.upload(alice, chat["id"], data, filename="sample.png")
|
||
started = time.monotonic()
|
||
ready = api.wait_attachment(alice, accepted["id"])
|
||
print(f"ready in {time.monotonic() - started:.2f} s")
|
||
assert ready["error_code"] is None
|
||
for name in VARIANT_NAMES:
|
||
variant_of(api, alice, ready, name)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Fixture 320 × 200")
|
||
def test_thumbnail_fixture(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
assert thumbnail_of(api, alice, attachment).size == (256, 160)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Маленькое изображение не увеличивается",
|
||
)
|
||
def test_thumbnail_not_enlarged(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size in ((100, 50), (256, 256)):
|
||
data = images.png(*size)
|
||
assert thumbnail_size(api, alice, chat["id"], data) == size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Округление вниз и минимум")
|
||
def test_thumbnail_rounding(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size, expected in (
|
||
((1000, 601), (256, 153)),
|
||
((601, 1000), (153, 256)),
|
||
((5000, 10), (256, 1)),
|
||
((10, 5000), (1, 256)),
|
||
):
|
||
data = images.png(*size)
|
||
assert thumbnail_size(api, alice, chat["id"], data) == expected, size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "EXIF orientation")
|
||
def test_thumbnail_exif_orientation(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert image.size == (128, 256)
|
||
assert_color(image, (64, 32), RED)
|
||
assert_color(image, (64, 224), BLUE)
|
||
|
||
|
||
def test_thumbnail_every_orientation(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `attachment-processing`, «EXIF orientation»: все восемь значений —
|
||
где оказывается красная половина показанного изображения."""
|
||
# значение → размер и точки красного и синего: левая половина файла
|
||
# красная, правая синяя
|
||
wide, tall = (256, 128), (128, 256)
|
||
left, right = (32, 64), (224, 64)
|
||
top, bottom = (64, 32), (64, 224)
|
||
expected = {
|
||
1: (wide, left, right),
|
||
2: (wide, right, left),
|
||
3: (wide, right, left),
|
||
4: (wide, left, right),
|
||
5: (tall, top, bottom),
|
||
6: (tall, top, bottom),
|
||
7: (tall, bottom, top),
|
||
8: (tall, bottom, top),
|
||
}
|
||
for orientation, (size, red, blue) in expected.items():
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=orientation),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert image.size == size, orientation
|
||
assert_color(image, red, RED)
|
||
assert_color(image, blue, BLUE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Alpha на белый")
|
||
def test_thumbnail_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for data in (
|
||
images.png_half_transparent(),
|
||
images.png_palette_transparent(),
|
||
):
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert_color(image, (20, image.height // 2), BLACK)
|
||
assert_color(image, (image.width - 20, image.height // 2), WHITE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Разновидности форматов")
|
||
def test_thumbnail_format_variants(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
kinds = [
|
||
("image/png", name, data)
|
||
for name, data in images.png_variants().items()
|
||
] + [
|
||
("image/jpeg", name, data)
|
||
for name, data in images.jpeg_variants().items()
|
||
]
|
||
for content_type, name, data in kinds:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
data,
|
||
content_type=content_type,
|
||
filename=f"{name}.img",
|
||
)
|
||
for variant in VARIANT_NAMES:
|
||
variant_of(api, alice, attachment, variant)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Изображение в лимите пикселей")
|
||
@pytest.mark.parametrize(
|
||
("color", "depth", "transparent"),
|
||
[
|
||
pytest.param(4, 8, False, id="gray-alpha-8"),
|
||
pytest.param(2, 8, True, id="rgb-8-trns"),
|
||
pytest.param(2, 16, True, id="rgb-16-trns"),
|
||
pytest.param(3, 8, True, id="palette-trns"),
|
||
pytest.param(0, 16, False, id="gray-16"),
|
||
],
|
||
)
|
||
def test_pixel_limit_png_modes(
|
||
base_url: str,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
color: int,
|
||
depth: int,
|
||
transparent: bool,
|
||
) -> None:
|
||
"""Контракт разрешает 40 млн пикселей в любом типе PNG. Эти типы
|
||
переводятся в другой режим при подготовке: держать исходник на время
|
||
перевода и масштаба — пик выше лимита памяти процесса обработки."""
|
||
data = images.png_at_pixel_limit(color, depth, transparent=transparent)
|
||
# срок клиента: тело на 40 млн пикселей проверяется дольше 5 с по умолчанию
|
||
with Api(base_url, timeout=LARGE_BODY_CLIENT_TIMEOUT_S) as slow:
|
||
attachment = slow.upload_ready(alice, chat["id"], data)
|
||
sizes = {
|
||
"thumbnail": (256, 160),
|
||
"crop": (128, 128),
|
||
"optimized": (1280, 800),
|
||
}
|
||
for name, size in sizes.items():
|
||
assert variant_of(slow, alice, attachment, name).size == size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Crop большого изображения")
|
||
def test_crop_center(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.png_stripes())
|
||
image = variant_of(api, alice, attachment, "crop")
|
||
assert image.size == (128, 128)
|
||
for position in ((64, 64), (3, 3), (124, 124), (3, 124), (124, 3)):
|
||
assert_color(image, position, BLUE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Crop маленького изображения")
|
||
def test_crop_small(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.png_solid(50, 20))
|
||
assert variant_of(api, alice, attachment, "crop").size == (128, 128)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Crop с ориентацией и прозрачностью",
|
||
)
|
||
def test_crop_orientation_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
rotated = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = variant_of(api, alice, rotated, "crop")
|
||
assert_color(image, (64, 16), RED)
|
||
assert_color(image, (64, 112), BLUE)
|
||
transparent = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.png_half_transparent(),
|
||
)
|
||
image = variant_of(api, alice, transparent, "crop")
|
||
assert_color(image, (16, 64), BLACK)
|
||
assert_color(image, (112, 64), WHITE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized большого изображения")
|
||
def test_optimized_fit(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size, expected in (
|
||
((2000, 1000), (1280, 640)),
|
||
((5000, 3001), (1280, 768)),
|
||
((5000, 10), (1280, 2)),
|
||
):
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.png_solid(*size),
|
||
)
|
||
image = variant_of(api, alice, attachment, "optimized")
|
||
assert image.size == expected, size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized маленького изображения")
|
||
def test_optimized_small(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.png_solid(100, 50))
|
||
assert variant_of(api, alice, attachment, "optimized").size == (100, 50)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized без метаданных")
|
||
def test_optimized_no_metadata(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_with_metadata(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = variant_of(api, alice, attachment, "optimized")
|
||
assert image.size == (200, 400)
|
||
assert_color(image, (100, 50), RED)
|
||
assert_color(image, (100, 350), BLUE)
|
||
data = api.attachment_content(
|
||
alice,
|
||
attachment["id"],
|
||
variant="optimized",
|
||
).content
|
||
for chunk in (b"EXIF", b"XMP ", b"ICCP", images.JPEG_MAKER.encode()):
|
||
assert chunk not in data, chunk
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized сохраняет прозрачность")
|
||
def test_optimized_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.png_half_transparent(),
|
||
)
|
||
image = variant_of(api, alice, attachment, "optimized")
|
||
assert image.convert("RGBA").getpixel((190, 50))[3] == 0
|
||
assert image.convert("RGBA").getpixel((10, 50))[3] == 255
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание crop и optimized")
|
||
def test_download_crop_optimized(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for name, content_type, size in (
|
||
("crop", "image/jpeg", (128, 128)),
|
||
("optimized", "image/webp", (320, 200)),
|
||
):
|
||
resp = api.attachment_content(bob, attachment["id"], variant=name)
|
||
assert not resp.is_redirect
|
||
assert resp.headers["Content-Type"] == content_type
|
||
assert variant_of(api, bob, attachment, name).size == size
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание thumbnail")
|
||
def test_download_thumbnail(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
resp = api.attachment_content(
|
||
bob,
|
||
attachment["id"],
|
||
variant="thumbnail",
|
||
)
|
||
assert not resp.is_redirect
|
||
thumbnail = next(
|
||
item for item in attachment["variants"] if item["name"] == "thumbnail"
|
||
)
|
||
assert_variant_content(resp, thumbnail)
|
||
assert resp.headers["Content-Type"] == "image/jpeg"
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Вариант для постороннего")
|
||
def test_variant_outsider(
|
||
api: Api,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
outsider = assert_error(
|
||
api.attachment_content(eve, attachment["id"], variant="thumbnail"),
|
||
404,
|
||
)
|
||
unknown = assert_error(
|
||
api.attachment_content(
|
||
alice,
|
||
"00000000-0000-4000-8000-000000000000",
|
||
variant="thumbnail",
|
||
),
|
||
404,
|
||
)
|
||
assert outsider["message"] == unknown["message"]
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение failed")
|
||
def test_message_with_failed_attachment(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
)
|
||
failed = api.wait_attachment(alice, accepted["id"], status="failed")
|
||
assert failed["error_code"] == "invalid_image"
|
||
assert failed["variants"] == []
|
||
key = "failed-attachment-key"
|
||
assert_error(
|
||
api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "x", "attachment_ids": [failed["id"]]},
|
||
key=key,
|
||
),
|
||
409,
|
||
)
|
||
assert_error(
|
||
api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={
|
||
"text": "x",
|
||
"attachment_ids": [
|
||
failed["id"],
|
||
"00000000-0000-4000-8000-000000000000",
|
||
],
|
||
},
|
||
key=key,
|
||
),
|
||
404,
|
||
)
|
||
assert api.all_messages(alice, chat["id"]) == []
|
||
# откат освободил ключ: то же значение годится для другого тела
|
||
sent = api.send_message(alice, chat["id"], "без вложения", key=key)
|
||
assert sent["attachment_ids"] == []
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Принятое вложение видно сразу")
|
||
def test_accepted_attachment_visible_on_other_replica(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
replicas: int,
|
||
) -> None:
|
||
upload = api.post_attachment(alice, chat["id"], images.sample_png())
|
||
assert upload.status_code == 202, upload.text
|
||
accepted = upload.json()
|
||
uploaded_via = assert_instance_id(upload)
|
||
seen = {uploaded_via}
|
||
for _ in range(REPLICA_ATTEMPTS):
|
||
resp = api.attachment_meta(bob, accepted["id"])
|
||
meta = assert_ok(resp, 200, "Attachment")
|
||
assert meta["id"] == accepted["id"]
|
||
seen.add(assert_instance_id(resp))
|
||
if len(seen) > 1:
|
||
break
|
||
assert_replicas(seen, replicas)
|