Upload answers 202 after staging, a queued job in PostgreSQL and a broker confirm; a separate worker publishes the original and a thumbnail with bounded retries. Adds the job and variant tables, the broker service, the worker S3 user, cleanup and reconcile support for staging and variants, the failure stand with broker and worker, bench and driver updates. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
346 lines
11 KiB
Python
346 lines
11 KiB
Python
"""Спек `attachment-processing`: то, что видно по HTTP на любом стенде с worker.
|
||
|
||
Thumbnail, скачивание варианта, готовность и `failed` от недекодируемого
|
||
изображения. Сценарии с остановкой worker, брокера и S3 — проверки отказов
|
||
(`test_failures.py`).
|
||
"""
|
||
|
||
import io
|
||
import time
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from PIL import Image
|
||
|
||
import images
|
||
from api_client import Actor, Api
|
||
from contract import (
|
||
assert_error,
|
||
assert_instance_id,
|
||
assert_ok,
|
||
assert_replicas,
|
||
assert_variant_content,
|
||
)
|
||
|
||
# сколько раз читать метаданные через LB, чтобы увидеть другую реплику
|
||
REPLICA_ATTEMPTS = 20
|
||
# допуск к цвету пикселя: JPEG теряет, но красный, синий и белый различимы
|
||
COLOR_TOLERANCE = 40
|
||
RED = (255, 0, 0)
|
||
BLUE = (0, 0, 255)
|
||
WHITE = (255, 255, 255)
|
||
BLACK = (0, 0, 0)
|
||
|
||
|
||
def thumbnail_of(
|
||
api: Api,
|
||
actor: Actor,
|
||
attachment: dict[str, Any],
|
||
) -> Image.Image:
|
||
"""Вариант из метаданных готового вложения: скачан, сверен и открыт
|
||
как настоящий JPEG; настоящие размеры равны полям варианта."""
|
||
assert [item["name"] for item in attachment["variants"]] == ["thumbnail"]
|
||
variant = attachment["variants"][0]
|
||
data = assert_variant_content(
|
||
api.attachment_content(actor, attachment["id"], variant="thumbnail"),
|
||
variant,
|
||
)
|
||
image = Image.open(io.BytesIO(data))
|
||
assert image.format == "JPEG"
|
||
assert image.size == (variant["width"], variant["height"])
|
||
assert variant["content_type"] == "image/jpeg"
|
||
return image
|
||
|
||
|
||
def assert_color(
|
||
image: Image.Image,
|
||
position: tuple[int, int],
|
||
color: tuple[int, int, int],
|
||
) -> None:
|
||
got = image.convert("RGB").getpixel(position)
|
||
assert all(
|
||
abs(a - b) <= COLOR_TOLERANCE for a, b in zip(got, color, strict=True)
|
||
), f"pixel {position}: {got}, expected about {color}"
|
||
|
||
|
||
def thumbnail_size(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat_id: str,
|
||
data: bytes,
|
||
) -> tuple[int, int]:
|
||
attachment = api.upload_ready(actor, chat_id, data)
|
||
return thumbnail_of(api, actor, attachment).size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Вложение становится готовым")
|
||
def test_attachment_becomes_ready(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
accepted = api.upload(alice, chat["id"], data, filename="sample.png")
|
||
started = time.monotonic()
|
||
ready = api.wait_attachment(alice, accepted["id"])
|
||
print(f"ready in {time.monotonic() - started:.2f} s")
|
||
assert ready["error_code"] is None
|
||
assert [item["name"] for item in ready["variants"]] == ["thumbnail"]
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Fixture 320 × 200")
|
||
def test_thumbnail_fixture(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
assert thumbnail_of(api, alice, attachment).size == (256, 160)
|
||
|
||
|
||
@pytest.mark.spec(
|
||
"attachment-processing",
|
||
"Маленькое изображение не увеличивается",
|
||
)
|
||
def test_thumbnail_not_enlarged(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size in ((100, 50), (256, 256)):
|
||
data = images.png(*size)
|
||
assert thumbnail_size(api, alice, chat["id"], data) == size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Округление вниз и минимум")
|
||
def test_thumbnail_rounding(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for size, expected in (
|
||
((1000, 601), (256, 153)),
|
||
((601, 1000), (153, 256)),
|
||
((5000, 10), (256, 1)),
|
||
((10, 5000), (1, 256)),
|
||
):
|
||
data = images.png(*size)
|
||
assert thumbnail_size(api, alice, chat["id"], data) == expected, size
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "EXIF orientation")
|
||
def test_thumbnail_exif_orientation(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=6),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert image.size == (128, 256)
|
||
assert_color(image, (64, 32), RED)
|
||
assert_color(image, (64, 224), BLUE)
|
||
|
||
|
||
def test_thumbnail_every_orientation(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `attachment-processing`, «EXIF orientation»: все восемь значений —
|
||
где оказывается красная половина показанного изображения."""
|
||
# значение → размер и точки красного и синего: левая половина файла
|
||
# красная, правая синяя
|
||
wide, tall = (256, 128), (128, 256)
|
||
left, right = (32, 64), (224, 64)
|
||
top, bottom = (64, 32), (64, 224)
|
||
expected = {
|
||
1: (wide, left, right),
|
||
2: (wide, right, left),
|
||
3: (wide, right, left),
|
||
4: (wide, left, right),
|
||
5: (tall, top, bottom),
|
||
6: (tall, top, bottom),
|
||
7: (tall, bottom, top),
|
||
8: (tall, bottom, top),
|
||
}
|
||
for orientation, (size, red, blue) in expected.items():
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_halves(orientation=orientation),
|
||
content_type="image/jpeg",
|
||
)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert image.size == size, orientation
|
||
assert_color(image, red, RED)
|
||
assert_color(image, blue, BLUE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Alpha на белый")
|
||
def test_thumbnail_alpha(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
for data in (
|
||
images.png_half_transparent(),
|
||
images.png_palette_transparent(),
|
||
):
|
||
attachment = api.upload_ready(alice, chat["id"], data)
|
||
image = thumbnail_of(api, alice, attachment)
|
||
assert_color(image, (20, image.height // 2), BLACK)
|
||
assert_color(image, (image.width - 20, image.height // 2), WHITE)
|
||
|
||
|
||
@pytest.mark.spec("attachment-processing", "Разновидности форматов")
|
||
def test_thumbnail_format_variants(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
kinds = [
|
||
("image/png", name, data)
|
||
for name, data in images.png_variants().items()
|
||
] + [
|
||
("image/jpeg", name, data)
|
||
for name, data in images.jpeg_variants().items()
|
||
]
|
||
for content_type, name, data in kinds:
|
||
attachment = api.upload_ready(
|
||
alice,
|
||
chat["id"],
|
||
data,
|
||
content_type=content_type,
|
||
filename=f"{name}.img",
|
||
)
|
||
thumbnail_of(api, alice, attachment)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Скачивание thumbnail")
|
||
def test_download_thumbnail(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
resp = api.attachment_content(
|
||
bob,
|
||
attachment["id"],
|
||
variant="thumbnail",
|
||
)
|
||
assert not resp.is_redirect
|
||
assert_variant_content(resp, attachment["variants"][0])
|
||
assert resp.headers["Content-Type"] == "image/jpeg"
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отсутствующий вариант")
|
||
def test_missing_variant(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
for variant in ("crop", "optimized"):
|
||
assert_error(
|
||
api.attachment_content(alice, attachment["id"], variant=variant),
|
||
404,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Вариант для постороннего")
|
||
def test_variant_outsider(
|
||
api: Api,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
attachment = api.upload_ready(alice, chat["id"], images.sample_png())
|
||
outsider = assert_error(
|
||
api.attachment_content(eve, attachment["id"], variant="thumbnail"),
|
||
404,
|
||
)
|
||
unknown = assert_error(
|
||
api.attachment_content(
|
||
alice,
|
||
"00000000-0000-4000-8000-000000000000",
|
||
variant="thumbnail",
|
||
),
|
||
404,
|
||
)
|
||
assert outsider["message"] == unknown["message"]
|
||
|
||
|
||
@pytest.mark.spec("messages", "Вложение failed")
|
||
def test_message_with_failed_attachment(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
accepted = api.upload(
|
||
alice,
|
||
chat["id"],
|
||
images.jpeg_precision_12(),
|
||
content_type="image/jpeg",
|
||
)
|
||
failed = api.wait_attachment(alice, accepted["id"], status="failed")
|
||
assert failed["error_code"] == "invalid_image"
|
||
assert failed["variants"] == []
|
||
key = "failed-attachment-key"
|
||
assert_error(
|
||
api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "x", "attachment_ids": [failed["id"]]},
|
||
key=key,
|
||
),
|
||
409,
|
||
)
|
||
assert_error(
|
||
api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={
|
||
"text": "x",
|
||
"attachment_ids": [
|
||
failed["id"],
|
||
"00000000-0000-4000-8000-000000000000",
|
||
],
|
||
},
|
||
key=key,
|
||
),
|
||
404,
|
||
)
|
||
assert api.all_messages(alice, chat["id"]) == []
|
||
# откат освободил ключ: то же значение годится для другого тела
|
||
sent = api.send_message(alice, chat["id"], "без вложения", key=key)
|
||
assert sent["attachment_ids"] == []
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Принятое вложение видно сразу")
|
||
def test_accepted_attachment_visible_on_other_replica(
|
||
api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
replicas: int,
|
||
) -> None:
|
||
upload = api.post_attachment(alice, chat["id"], images.sample_png())
|
||
assert upload.status_code == 202, upload.text
|
||
accepted = upload.json()
|
||
uploaded_via = assert_instance_id(upload)
|
||
seen = {uploaded_via}
|
||
for _ in range(REPLICA_ATTEMPTS):
|
||
resp = api.attachment_meta(bob, accepted["id"])
|
||
meta = assert_ok(resp, 200, "Attachment")
|
||
assert meta["id"] == accepted["id"]
|
||
seen.add(assert_instance_id(resp))
|
||
if len(seen) > 1:
|
||
break
|
||
assert_replicas(seen, replicas)
|