Move publishing out of the upload path: a relay process publishes queued jobs with confirms. Add lease and attempt fencing, backoff with jitter, a per-job render process with CPU and memory limits, crop and optimized variants, the failed registry with audited replay, and staging cleanup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
496 lines
15 KiB
Python
496 lines
15 KiB
Python
"""Спек `attachment-processing`: то, что видно по HTTP на любом стенде с worker.
|
||
|
||
Thumbnail, скачивание варианта, готовность и `failed` от недекодируемого
|
||
изображения. Сценарии с остановкой worker, брокера и S3 — проверки отказов
|
||
(`test_failures.py`).
|
||
"""
|
||
|
||
import time
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from PIL import Image
|
||
|
||
import images
|
||
from api_client import Actor, Api
|
||
from contract import (
|
||
assert_error,
|
||
assert_instance_id,
|
||
assert_ok,
|
||
assert_replicas,
|
||
assert_variant_content,
|
||
assert_variant_image,
|
||
)
|
||
|
||
# сколько раз читать метаданные через LB, чтобы увидеть другую реплику
|
||
REPLICA_ATTEMPTS = 20
|
||
# допуск к цвету пикселя: JPEG теряет, но красный, синий и белый различимы
|
||
COLOR_TOLERANCE = 40
|
||
RED = (255, 0, 0)
|
||
BLUE = (0, 0, 255)
|
||
WHITE = (255, 255, 255)
|
||
BLACK = (0, 0, 0)
|
||
VARIANT_TYPES = {
|
||
"thumbnail": "image/jpeg",
|
||
"crop": "image/jpeg",
|
||
"optimized": "image/webp",
|
||
}
|
||
VARIANT_NAMES = tuple(VARIANT_TYPES)
|
||
|
||
|
||
def thumbnail_of(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
) -> Image.Image:
|
||
"""Вариант `thumbnail` готового вложения: скачан, сверен и открыт как
|
||
настоящий JPEG; настоящие размеры равны полям варианта."""
|
||
return variant_of(api, actor, attachment, "thumbnail")
|
||
|
||
|
||
def variant_of(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
name: str,
|
||
) -> Image.Image:
|
||
"""Вариант из метаданных: у вложения по одному варианту каждого имени, байты
|
||
скачаны и сверены с SHA-256 и размером, файл — настоящее изображение своего
|
||
типа и размера."""
|
||
assert sorted(item["name"] for item in attachment["variants"]) == sorted(
|
||
VARIANT_NAMES,
|
||
)
|
||
variant = next(
|
||
item for item in attachment["variants"] if item["name"] == name
|
||
)
|
||
assert variant["content_type"] == VARIANT_TYPES[name]
|
||
data = assert_variant_content(
|
||
api.attachment_content(actor, attachment["id"], variant=name),
|
||
variant,
|
||
)
|
||
return assert_variant_image(data, variant)
|
||
|
||
|
||
def assert_color(
|
||
image: Image.Image,
|
||
position: tuple[int, int],
|
||
color: tuple[int, int, int],
|
||
) -> None:
|
||
got = image.convert("RGB").getpixel(position)
|
||
assert all(
|
||
abs(a - b) <= COLOR_TOLERANCE for a, b in zip(got, color, strict=True)
|
||
), f"pixel {position}: {got}, expected about {color}"
|
||
|
||
|
||
def thumbnail_size(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat_id: str,
|
||
data: bytes,
|
||
) -> tuple[int, int]:
|
||
attachment = api.upload_ready(actor, chat_id, data)
|
||
return thumbnail_of(api, actor, attachment).size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Вложение становится готовым")
|
||
def test_attachment_becomes_ready(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
accepted = api.upload(alice, chat["id"], data, filename="sample.png")
|
||
started = time.monotonic()
|
||
ready = api.wait_attachment(alice, accepted["id"])
|
||
print(f"ready in {time.monotonic() - started:.2f} s")
|
||
assert ready["error_code"] is None
|
||
for name in VARIANT_NAMES:
|
||
variant_of(api, alice, ready, name)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Fixture 320 × 200")
|
||
def test_thumbnail_fixture(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
assert thumbnail_of(api, alice, attachment).size == (256, 160)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Маленькое изображение не увеличивается",
|
||
)
|
||
def test_thumbnail_not_enlarged(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size in ((100, 50), (256, 256)):
|
||
data = images.png(*size)
|
||
assert thumbnail_size(api, alice, chat["id"], data) == size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Округление вниз и минимум")
|
||
def test_thumbnail_rounding(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size, expected in (
|
||
((1000, 601), (256, 153)),
|
||
((601, 1000), (153, 256)),
|
||
((5000, 10), (256, 1)),
|
||
((10, 5000), (1, 256)),
|
||
):
|
||
data = images.png(*size)
|
||
assert thumbnail_size(api, alice, chat["id"], data) == expected, size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "EXIF orientation")
|
||
def test_thumbnail_exif_orientation(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert image.size == (128, 256)
|
||
assert_color(image, (64, 32), RED)
|
||
assert_color(image, (64, 224), BLUE)
|
||
|
||
|
||
def test_thumbnail_every_orientation(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `attachment-processing`, «EXIF orientation»: все восемь значений —
|
||
где оказывается красная половина показанного изображения."""
|
||
# значение → размер и точки красного и синего: левая половина файла
|
||
# красная, правая синяя
|
||
wide, tall = (256, 128), (128, 256)
|
||
left, right = (32, 64), (224, 64)
|
||
top, bottom = (64, 32), (64, 224)
|
||
expected = {
|
||
1: (wide, left, right),
|
||
2: (wide, right, left),
|
||
3: (wide, right, left),
|
||
4: (wide, left, right),
|
||
5: (tall, top, bottom),
|
||
6: (tall, top, bottom),
|
||
7: (tall, bottom, top),
|
||
8: (tall, bottom, top),
|
||
}
|
||
for orientation, (size, red, blue) in expected.items():
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=orientation),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert image.size == size, orientation
|
||
assert_color(image, red, RED)
|
||
assert_color(image, blue, BLUE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Alpha на белый")
|
||
def test_thumbnail_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for data in (
|
||
images.png_half_transparent(),
|
||
images.png_palette_transparent(),
|
||
):
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert_color(image, (20, image.height // 2), BLACK)
|
||
assert_color(image, (image.width - 20, image.height // 2), WHITE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Разновидности форматов")
|
||
def test_thumbnail_format_variants(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
kinds = [
|
||
("image/png", name, data)
|
||
for name, data in images.png_variants().items()
|
||
] + [
|
||
("image/jpeg", name, data)
|
||
for name, data in images.jpeg_variants().items()
|
||
]
|
||
for content_type, name, data in kinds:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
data,
|
||
content_type=content_type,
|
||
filename=f"{name}.img",
|
||
)
|
||
for variant in VARIANT_NAMES:
|
||
variant_of(api, alice, attachment, variant)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Crop большого изображения")
|
||
def test_crop_center(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.png_stripes())
|
||
image = variant_of(api, alice, attachment, "crop")
|
||
assert image.size == (128, 128)
|
||
for position in ((64, 64), (3, 3), (124, 124), (3, 124), (124, 3)):
|
||
assert_color(image, position, BLUE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Crop маленького изображения")
|
||
def test_crop_small(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.png_solid(50, 20))
|
||
assert variant_of(api, alice, attachment, "crop").size == (128, 128)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Crop с ориентацией и прозрачностью",
|
||
)
|
||
def test_crop_orientation_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
rotated = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = variant_of(api, alice, rotated, "crop")
|
||
assert_color(image, (64, 16), RED)
|
||
assert_color(image, (64, 112), BLUE)
|
||
transparent = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.png_half_transparent(),
|
||
)
|
||
image = variant_of(api, alice, transparent, "crop")
|
||
assert_color(image, (16, 64), BLACK)
|
||
assert_color(image, (112, 64), WHITE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized большого изображения")
|
||
def test_optimized_fit(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size, expected in (
|
||
((2000, 1000), (1280, 640)),
|
||
((5000, 3001), (1280, 768)),
|
||
((5000, 10), (1280, 2)),
|
||
):
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.png_solid(*size),
|
||
)
|
||
image = variant_of(api, alice, attachment, "optimized")
|
||
assert image.size == expected, size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized маленького изображения")
|
||
def test_optimized_small(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.png_solid(100, 50))
|
||
assert variant_of(api, alice, attachment, "optimized").size == (100, 50)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized без метаданных")
|
||
def test_optimized_no_metadata(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_with_metadata(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = variant_of(api, alice, attachment, "optimized")
|
||
assert image.size == (200, 400)
|
||
assert_color(image, (100, 50), RED)
|
||
assert_color(image, (100, 350), BLUE)
|
||
data = api.attachment_content(
|
||
alice,
|
||
attachment["id"],
|
||
variant="optimized",
|
||
).content
|
||
for chunk in (b"EXIF", b"XMP ", b"ICCP", images.JPEG_MAKER.encode()):
|
||
assert chunk not in data, chunk
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Optimized сохраняет прозрачность")
|
||
def test_optimized_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.png_half_transparent(),
|
||
)
|
||
image = variant_of(api, alice, attachment, "optimized")
|
||
assert image.convert("RGBA").getpixel((190, 50))[3] == 0
|
||
assert image.convert("RGBA").getpixel((10, 50))[3] == 255
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание crop и optimized")
|
||
def test_download_crop_optimized(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for name, content_type, size in (
|
||
("crop", "image/jpeg", (128, 128)),
|
||
("optimized", "image/webp", (320, 200)),
|
||
):
|
||
resp = api.attachment_content(bob, attachment["id"], variant=name)
|
||
assert not resp.is_redirect
|
||
assert resp.headers["Content-Type"] == content_type
|
||
assert variant_of(api, bob, attachment, name).size == size
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание thumbnail")
|
||
def test_download_thumbnail(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
resp = api.attachment_content(
|
||
bob,
|
||
attachment["id"],
|
||
variant="thumbnail",
|
||
)
|
||
assert not resp.is_redirect
|
||
thumbnail = next(
|
||
item for item in attachment["variants"] if item["name"] == "thumbnail"
|
||
)
|
||
assert_variant_content(resp, thumbnail)
|
||
assert resp.headers["Content-Type"] == "image/jpeg"
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Вариант для постороннего")
|
||
def test_variant_outsider(
|
||
api: Api,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
outsider = assert_error(
|
||
api.attachment_content(eve, attachment["id"], variant="thumbnail"),
|
||
404,
|
||
)
|
||
unknown = assert_error(
|
||
api.attachment_content(
|
||
alice,
|
||
"00000000-0000-4000-8000-000000000000",
|
||
variant="thumbnail",
|
||
),
|
||
404,
|
||
)
|
||
assert outsider["message"] == unknown["message"]
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение failed")
|
||
def test_message_with_failed_attachment(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
)
|
||
failed = api.wait_attachment(alice, accepted["id"], status="failed")
|
||
assert failed["error_code"] == "invalid_image"
|
||
assert failed["variants"] == []
|
||
key = "failed-attachment-key"
|
||
assert_error(
|
||
api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "x", "attachment_ids": [failed["id"]]},
|
||
key=key,
|
||
),
|
||
409,
|
||
)
|
||
assert_error(
|
||
api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={
|
||
"text": "x",
|
||
"attachment_ids": [
|
||
failed["id"],
|
||
"00000000-0000-4000-8000-000000000000",
|
||
],
|
||
},
|
||
key=key,
|
||
),
|
||
404,
|
||
)
|
||
assert api.all_messages(alice, chat["id"]) == []
|
||
# откат освободил ключ: то же значение годится для другого тела
|
||
sent = api.send_message(alice, chat["id"], "без вложения", key=key)
|
||
assert sent["attachment_ids"] == []
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Принятое вложение видно сразу")
|
||
def test_accepted_attachment_visible_on_other_replica(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
replicas: int,
|
||
) -> None:
|
||
upload = api.post_attachment(alice, chat["id"], images.sample_png())
|
||
assert upload.status_code == 202, upload.text
|
||
accepted = upload.json()
|
||
uploaded_via = assert_instance_id(upload)
|
||
seen = {uploaded_via}
|
||
for _ in range(REPLICA_ATTEMPTS):
|
||
resp = api.attachment_meta(bob, accepted["id"])
|
||
meta = assert_ok(resp, 200, "Attachment")
|
||
assert meta["id"] == accepted["id"]
|
||
seen.add(assert_instance_id(resp))
|
||
if len(seen) > 1:
|
||
break
|
||
assert_replicas(seen, replicas)
|