"""Спек `attachments`: загрузка, лимиты, тип и целостность, имя файла, доступ, метаданные, скачивание, повтор, реплики. Проверки отказов S3 — `test_failures.py`; сценарии, которым нужен стенд (остановка реплики, рестарт, взгляд в БД и bucket), — исключения сверки покрытия (`spec_coverage.py`). """ import hashlib import threading import time import urllib.parse import uuid from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor from typing import Any import pytest import images from api_client import API, Actor, Api from contract import ( BEARER_CHALLENGE, assert_content, assert_error, assert_errors, assert_instance_id, assert_no_storage_traces, assert_ok, assert_replicas, assert_unauthorized, ) # больше лимита в 3 раза: 30 MiB BIG_BYTES = 30 * 1024 * 1024 # сколько скачиваний ждать ответа другой реплики (round-robin LB) REPLICA_ATTEMPTS = 20 # одновременные загрузки сценария «Загрузки в пределах лимита» CONCURRENT_UPLOADS = 10 # срок клиента загрузки: больше ожидания места (5 с) и сроков шагов CONCURRENT_CLIENT_TIMEOUT_S = 30 def chunks(data: bytes, size: int = 1024 * 1024) -> Iterator[bytes]: """Тело частями: httpx шлёт его без `Content-Length`.""" for offset in range(0, len(data), size): yield data[offset : offset + size] def big_body() -> bytes: """30 MiB, которые начинаются как PNG.""" return images.PNG_SIGNATURE + bytes(BIG_BYTES - len(images.PNG_SIGNATURE)) @pytest.mark.spec("attachments", "Загрузка PNG") def test_upload_png(api: Api, alice: Actor, chat: dict[str, Any]) -> None: data = images.sample_png() attachment = api.upload(alice, chat["id"], data, filename="sample.png") assert attachment["status"] == "ready" assert attachment["variants"] == [] assert attachment["error_code"] is None assert attachment["content_type"] == "image/png" assert attachment["size_bytes"] == 111817 assert attachment["sha256"] == ( "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" ) @pytest.mark.spec("attachments", "Загрузка JPEG") def test_upload_jpeg(api: Api, bob: Actor, chat: dict[str, Any]) -> None: data = images.jpeg(size=(320, 200)) attachment = api.upload( bob, chat["id"], data, content_type="image/jpeg", filename="photo.jpg", ) assert attachment["content_type"] == "image/jpeg" @pytest.mark.spec("attachments", "Разновидности PNG и JPEG") def test_image_kinds(api: Api, alice: Actor, chat: dict[str, Any]) -> None: kinds = [ ("image/png", name, data) for name, data in images.png_variants().items() ] + [ ("image/jpeg", name, data) for name, data in images.jpeg_variants().items() ] for content_type, name, data in kinds: attachment = api.upload( alice, chat["id"], data, content_type=content_type, filename=f"{name}.img", ) assert attachment["content_type"] == content_type, name @pytest.mark.spec("attachments", "Граница размера") def test_size_boundary(api: Api, alice: Actor, chat: dict[str, Any]) -> None: exact = images.png_of_size(images.MAX_BYTES) api.upload(alice, chat["id"], exact) over = images.png_of_size(images.MAX_BYTES + 1) assert_error(api.post_attachment(alice, chat["id"], over), 413) @pytest.mark.spec("attachments", "Заявленный размер больше лимита") def test_declared_size_over_limit( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: resp = api.post_attachment(alice, chat["id"], big_body()) assert resp.request.headers["Content-Length"] == str(BIG_BYTES) assert_error(resp, 413) @pytest.mark.spec("attachments", "Большое тело без Content-Length") def test_chunked_body_over_limit( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: resp = api.post_attachment(alice, chat["id"], chunks(big_body())) assert "Content-Length" not in resp.request.headers assert_error(resp, 413) @pytest.mark.spec("attachments", "Огромные размеры в маленьком файле") def test_huge_dimensions(api: Api, alice: Actor, chat: dict[str, Any]) -> None: assert_errors( { "png 8000x5001": api.post_attachment( alice, chat["id"], images.png_huge(8000, 5001), ), "png 1x40000001": api.post_attachment( alice, chat["id"], images.png_huge(1, 40_000_001), ), "jpeg 10000x4001": api.post_attachment( alice, chat["id"], images.jpeg_huge(10000, 4001), content_type="image/jpeg", ), "jpeg 65535x65535": api.post_attachment( alice, chat["id"], images.jpeg_huge(65535, 65535), content_type="image/jpeg", ), }, 413, ) @pytest.mark.spec("attachments", "Ровно 40 млн пикселей") def test_exactly_max_pixels( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: data = images.png(8000, 5000, color=0, depth=1) api.upload(alice, chat["id"], data) @pytest.mark.spec("attachments", "Данные длиннее размеров") def test_decompression_bomb( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: assert_error( api.post_attachment(alice, chat["id"], images.png_bomb()), 415, ) @pytest.mark.spec("attachments", "Не изображение") def test_not_an_image(api: Api, alice: Actor, chat: dict[str, Any]) -> None: assert_errors( { name: api.post_attachment(alice, chat["id"], data) for name, data in ( ("text", "просто текст".encode()), ("gif", b"GIF89a" + bytes(64)), ("empty", b""), ) }, 415, ) @pytest.mark.spec("attachments", "Подмена MIME") def test_mime_spoofing(api: Api, alice: Actor, chat: dict[str, Any]) -> None: assert_errors( { "jpeg as png": api.post_attachment( alice, chat["id"], images.jpeg(), content_type="image/png", ), "png as jpeg": api.post_attachment( alice, chat["id"], images.sample_png(), content_type="image/jpeg", ), }, 415, ) @pytest.mark.spec("attachments", "Неподходящий Content-Type") def test_unsuitable_content_type( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() assert_errors( { str(content_type): api.post_attachment( alice, chat["id"], data, content_type=content_type, ) for content_type in ( "application/octet-stream", "text/plain", None, ) }, 415, ) resp = api.post_attachment( alice, chat["id"], data, content_type="IMAGE/PNG; charset=binary", ) assert assert_ok(resp, 201, "Attachment")["content_type"] == "image/png" @pytest.mark.spec("attachments", "Обрезанный файл") def test_truncated_files(api: Api, alice: Actor, chat: dict[str, Any]) -> None: png, jpeg = images.sample_png(), images.jpeg(size=(320, 200)) assert_errors( { "png half": api.post_attachment( alice, chat["id"], images.truncated(png), ), "png -1 byte": api.post_attachment(alice, chat["id"], png[:-1]), "jpeg half": api.post_attachment( alice, chat["id"], images.truncated(jpeg), content_type="image/jpeg", ), "jpeg -2 bytes": api.post_attachment( alice, chat["id"], jpeg[:-2], content_type="image/jpeg", ), }, 415, ) @pytest.mark.spec("attachments", "Испорченные данные PNG") def test_damaged_png_data( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: garbage = images.PNG_SIGNATURE + uuid.uuid4().bytes * 256 assert_errors( { "idat byte": api.post_attachment( alice, chat["id"], images.png_idat_byte_changed(), ), "garbage": api.post_attachment(alice, chat["id"], garbage), }, 415, ) @pytest.mark.spec("attachments", "Хвост после конца изображения") def test_tail_after_end( api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: for content_type, data in ( ("image/png", images.with_tail(images.sample_png())), ("image/jpeg", images.with_tail(images.jpeg())), ): attachment = api.upload( alice, chat["id"], data, content_type=content_type, ) assert_content( api.attachment_content(bob, attachment["id"]), data, content_type, ) @pytest.mark.spec("attachments", "Опасное имя") def test_dangerous_filename( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() names = { "dot-dot": "../x.png", "slash": "a/b.png", "backslash": "a\\b.png", "nul": "a\x00b.png", "newline": "a\nb.png", "del": "a\x7fb.png", "c1 nel": "a\x85b.png", "empty": "", "spaces": " ", "256": "я" * 256, } assert_errors( { name: api.post_attachment(alice, chat["id"], data, filename=value) for name, value in names.items() } | { "missing": api.post_attachment( alice, chat["id"], data, filename=None, ), }, 400, ) @pytest.mark.spec("attachments", "Имя не в UTF-8") def test_filename_not_utf8( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: """Процент-коды `filename` идут в строке запроса как есть: httpx их не перекодирует. Без строгого разбора `a%FFb.png` давал 201 с U+FFFD в имени вместо байта.""" data = images.sample_png() path = f"{API}/chats/{chat['id']}/attachments" def upload(query: str) -> Any: return api.request( "POST", f"{path}?{query}", actor=alice, headers={"Content-Type": "image/png"}, content=data, ) assert_errors( { "invalid byte": upload("filename=a%FFb.png"), "truncated sequence": upload("filename=a%C3.png"), "encoded surrogate": upload("filename=a%ED%A0%80b.png"), }, 400, ) # верный UTF-8 в процент-кодах — то же имя, что прислано attachment = assert_ok( upload("filename=%C3%A9t%C3%A9.png"), 201, "Attachment", ) assert attachment["filename"] == "été.png" def assert_name_kept( api: Api, actor: Actor, chat_id: str, filename: str, ) -> None: attachment = api.upload( actor, chat_id, images.sample_png(), filename=filename, ) meta = assert_ok( api.attachment_meta(actor, attachment["id"]), 200, "Attachment", ) assert meta["filename"] == attachment["filename"] == filename @pytest.mark.spec("attachments", "Имя в Unicode") def test_unicode_filename( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: family = "👨‍👩‍👧" core = f"фото {family} отпуска" name = " " + core + "ё" * (253 - len(core)) + " " assert len(name) == 255 assert_name_kept(api, alice, chat["id"], name) @pytest.mark.spec("attachments", "Символы форматирования в имени") def test_format_characters_in_filename( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: for name in ( "‮gnp.exe", "⁦שלום⁩.png", "строка
вторая.png", "пере­нос.png", ): assert_name_kept(api, alice, chat["id"], name) @pytest.mark.spec("attachments", "Посторонний с большим файлом") def test_outsider_with_big_file( api: Api, eve: Actor, chat: dict[str, Any], ) -> None: assert_error(api.post_attachment(eve, chat["id"], big_body()), 404) @pytest.mark.spec("attachments", "Без сессии с опасным именем") def test_no_session_with_dangerous_name( api: Api, chat: dict[str, Any], ) -> None: for filename in ("a/b.png", "ok.png"): resp = api.post_attachment( None, chat["id"], images.sample_png(), filename=filename, ) assert_unauthorized(resp, BEARER_CHALLENGE) @pytest.mark.spec("attachments", "Неверный тип и большой размер") def test_wrong_type_and_big_size( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: resp = api.post_attachment( alice, chat["id"], big_body(), content_type="text/plain", ) assert_error(resp, 415) @pytest.mark.spec("attachments", "Другой участник") def test_other_member_reads( api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() attachment = api.upload(alice, chat["id"], data) assert_ok(api.attachment_meta(bob, attachment["id"]), 200, "Attachment") assert_content( api.attachment_content(bob, attachment["id"]), data, "image/png", ) @pytest.mark.spec("attachments", "Посторонний и неизвестный id") def test_outsider_and_unknown_id( api: Api, alice: Actor, eve: Actor, chat: dict[str, Any], ) -> None: attachment = api.upload(alice, chat["id"], images.sample_png()) unknown = str(uuid.uuid4()) responses = { "eve meta": api.attachment_meta(eve, attachment["id"]), "eve content": api.attachment_content(eve, attachment["id"]), "unknown meta": api.attachment_meta(alice, unknown), "unknown content": api.attachment_content(alice, unknown), } assert_errors(responses, 404) messages = {resp.json()["error"]["message"] for resp in responses.values()} assert len(messages) == 1, messages @pytest.mark.spec("attachments", "Неверный id вложения") def test_bad_attachment_id(api: Api, alice: Actor) -> None: assert_errors( { f"{kind} {value}": call(alice, value) for value in ("123", "abc") for kind, call in ( ("meta", api.attachment_meta), ("content", api.attachment_content), ) }, 400, ) @pytest.mark.spec("attachments", "Чтение без сессии") def test_read_without_session( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: attachment = api.upload(alice, chat["id"], images.sample_png()) for resp in ( api.attachment_meta(None, attachment["id"]), api.attachment_content(None, attachment["id"]), ): assert_unauthorized(resp, BEARER_CHALLENGE) @pytest.mark.spec("attachments", "Метаданные равны ответу загрузки") def test_meta_equals_upload( api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: attachment = api.upload(alice, chat["id"], images.sample_png()) meta = assert_ok( api.attachment_meta(bob, attachment["id"]), 200, "Attachment", ) assert meta == attachment @pytest.mark.spec("attachments", "Побайтный оригинал") def test_original_bytes( api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: for content_type, data in ( ("image/png", images.sample_png()), ("image/jpeg", images.jpeg(size=(320, 200))), ): attachment = api.upload( alice, chat["id"], data, content_type=content_type, ) resp = api.attachment_content(bob, attachment["id"]) assert not resp.is_redirect assert_content(resp, data, content_type) assert hashlib.sha256(resp.content).hexdigest() == attachment["sha256"] @pytest.mark.spec("attachments", "Вариант original") def test_variant_original( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() attachment = api.upload(alice, chat["id"], data) assert_content( api.attachment_content(alice, attachment["id"], variant="original"), data, "image/png", ) assert_errors( { variant or "empty": api.attachment_content( alice, attachment["id"], variant=variant, ) for variant in ("thumbnail", "") }, 400, ) @pytest.mark.spec("attachments", "Без следов хранилища в ответе") def test_no_storage_traces( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() upload = api.post_attachment(alice, chat["id"], data) # assert_ok сверяет тело со схемой контракта: лишних полей нет attachment = assert_ok(upload, 201, "Attachment") meta = api.attachment_meta(alice, attachment["id"]) assert_ok(meta, 200, "Attachment") content = api.attachment_content(alice, attachment["id"]) assert_content(content, data, "image/png") for resp in (upload, meta, content): assert_no_storage_traces(resp) @pytest.mark.spec("attachments", "Повторная загрузка") def test_repeated_upload( api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() first = api.upload(alice, chat["id"], data) second = api.upload(alice, chat["id"], data) assert first["id"] != second["id"] assert first["sha256"] == second["sha256"] for attachment in (first, second): assert_content( api.attachment_content(bob, attachment["id"]), data, "image/png", ) @pytest.mark.spec("attachments", "Загрузки в пределах лимита") def test_concurrent_uploads(api: Api, base_url: str) -> None: """10 участников одного чата загружают по PNG ~1 MiB одновременно: при параметрах загрузок по умолчанию (4 места и 16 в очереди на процесс) все помещаются даже в один процесс (design change `upload-memory`, решение 11).""" actors = [ api.register(display_name=f"Uploader {number}") for number in range(CONCURRENT_UPLOADS) ] chat = api.create_chat(actors[0], *actors[1:], title="Uploads") bodies = [ images.png_noise(seed=number) for number in range(CONCURRENT_UPLOADS) ] barrier = threading.Barrier(CONCURRENT_UPLOADS) def upload(number: int) -> tuple[dict[str, Any], float, float]: with Api(base_url, timeout=CONCURRENT_CLIENT_TIMEOUT_S) as own: # соединение открыто до барьера: после него сразу загрузка own.request("GET", "/health/live") barrier.wait() started = time.monotonic() attachment = own.upload( actors[number], chat["id"], bodies[number], filename=f"noise-{number}.png", ) return attachment, started, time.monotonic() with ThreadPoolExecutor(max_workers=CONCURRENT_UPLOADS) as pool: results = list(pool.map(upload, range(CONCURRENT_UPLOADS))) attachments = [attachment for attachment, _, _ in results] assert len({item["id"] for item in attachments}) == CONCURRENT_UPLOADS # загрузки правда шли одновременно, а не по очереди клиента spans = sorted((started, ended) for _, started, ended in results) overlapping = sum(started < spans[0][1] for started, _ in spans) assert overlapping >= 2, f"uploads did not overlap: {spans}" for number, attachment in enumerate(attachments): assert_content( api.attachment_content(actors[0], attachment["id"]), bodies[number], "image/png", ) @pytest.mark.spec("attachments", "Скачивание через другую реплику") def test_download_through_other_replica( api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], replicas: int, ) -> None: data = images.sample_png() upload = api.post_attachment(alice, chat["id"], data) attachment = assert_ok(upload, 201, "Attachment") uploaded_via = assert_instance_id(upload) seen = {uploaded_via} for _ in range(REPLICA_ATTEMPTS): resp = api.attachment_content(bob, attachment["id"]) assert_content(resp, data, "image/png") seen.add(assert_instance_id(resp)) if assert_instance_id(resp) != uploaded_via: break assert_replicas(seen, replicas) print(f"uploaded via {uploaded_via}, downloaded via {sorted(seen)}") def test_upload_path_with_quoted_chat_id( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: """Спек `api-conventions`, «UUID на входе»: `chat_id` загрузки в другой записи — 400, как у остальных ручек чата.""" data = images.sample_png() value = chat["id"] assert_errors( { form: api.post_attachment( alice, urllib.parse.quote(raw, safe=""), data, ) for form, raw in ( ("no hyphens", value.replace("-", "")), ("urn", f"urn:uuid:{value}"), ("braces", f"{{{value}}}"), ("number", "123"), ) }, 400, )