"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`). Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres, Valkey и S3, уборщик незавершённых загрузок (grace и интервал — `FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API (`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении), «тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного соединения), «узкий по загрузкам» (`FAULT_UPLOADS_BASE_URL`: одно место загрузки, одна в очереди, ожидание 1 с, срок тела 3 с — change `upload-memory`) и «с коротким сроком S3» (`FAULT_S3_BASE_URL`: срок вызова S3 1 с — change `fix-upload-put-flake`), брокер и worker (4 попытки, пауза 5 с, срок попытки 10 с, срок остановки 2 с — `FAULT_WORKER_*`, change `async-attachments`). На хранилища и API тесты действуют только через фикстуру `stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в рабочее состояние. Решение 4 design change `failure-checks`, решение 10 design change `upload-consistency`, решение 11 design change `track2-reconciler` (сверка bucket и БД — в конце модуля: её тесты оставляют помеченные потерянными вложения и объекты staging без строки, прежние тесты таких не ждут), решение 3 design change `fix-upload-put-flake`, решение 15 design change `async-attachments`. """ import hashlib import json import math import os import random import statistics import threading import time import uuid from collections.abc import Callable, Collection, Iterator, Sequence from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Any import httpx import pytest from jsonschema import Draft202012Validator, FormatChecker import images from api_client import API, Actor, Api, new_key from contract import ( BEARER_CHALLENGE, assert_accepted, assert_content, assert_error, assert_ok, assert_unauthorized, assert_variant_content, assert_variant_image, describe, ) pytestmark = pytest.mark.fault type Stand = Callable[..., str] RESTORE = {"stop": "start", "pause": "unpause", "kill": "start"} # запас к границам времени: планировщик Docker Desktop SLACK_S = 0.3 # [cache] timeout_seconds и retry_after_seconds по умолчанию CACHE_TIMEOUT_S = 0.2 CACHE_RETRY_AFTER_S = 5 # окружение «тесного» API (TIGHT_ENV в run_failures.py) SESSION_TTL_S = 2 IDEMPOTENCY_TTL_S = 2 POOL_TIMEOUT_S = 1 # срок проверки Postgres в /health/ready READINESS_TIMEOUT_S = 2 # health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек # health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с OBJECT_STORAGE_HEALTH_S = 1 # [s3] timeout_seconds по умолчанию: срок одного вызова S3 S3_TIMEOUT_S = 5 # сколько секунд hook держит слоты max_connections; clear-connections # освобождает их раньше FILL_HOLD_S = 30 ME = f"{API}/users/me" @contextmanager def outage(stand: Stand, action: str, service: str) -> Iterator[None]: """`stop`, `pause` или `kill` сервиса; в `finally` — обратно.""" stand(action, service) try: yield finally: stand(RESTORE[action], service) def timed( call: Callable[[], httpx.Response], ) -> tuple[httpx.Response, float]: started = time.monotonic() resp = call() return resp, time.monotonic() - started def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response: return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor) def wait_cache_ready( api: Api, stand: Stand, actor: Actor, chat: dict[str, Any], ) -> None: """Дождаться, пока чтение снова пишет карточку в кэш. После отказа в прошлом тесте API пропускает кэш ещё до 5 с. """ deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2 stand("flush-cache") while time.monotonic() < deadline: assert_ok(read_chat(api, actor, chat), 200, "Chat") if stand("cache-has-chat", chat["id"]) == "1": return time.sleep(0.5) raise AssertionError("cache did not come back") def chat_titles(api: Api, actor: Actor) -> set[str]: return { chat["title"] for page in api.chat_pages(actor) for chat in page["items"] } # --------------------------------------------------------------------------- # Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном # --------------------------------------------------------------------------- @pytest.mark.spec("sessions", "Истёкший токен") def test_expired_token(tight_api: Api) -> None: actor = tight_api.register() assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User") time.sleep(SESSION_TTL_S + 1) assert_unauthorized( tight_api.request("GET", ME, actor=actor), BEARER_CHALLENGE, ) @pytest.mark.spec("sessions", "Срок не продлевается") def test_use_does_not_extend_session(tight_api: Api) -> None: actor = tight_api.register() logged_in = time.monotonic() successes = 0 while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S: assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User") successes += 1 time.sleep(0.5) assert successes >= 3, "too few requests before expiry" time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic())) assert_unauthorized( tight_api.request("GET", ME, actor=actor), BEARER_CHALLENGE, ) @pytest.mark.spec("sessions", "БД остановлена при запросе с токеном") def test_token_while_database_stopped(api: Api, stand: Stand) -> None: actor = api.register() with outage(stand, "stop", "postgres"): resp = api.request("GET", ME, actor=actor) assert_error(resp, 503) # недоступность хранилища — не отказ в аутентификации assert "WWW-Authenticate" not in resp.headers assert_ok(api.request("GET", ME, actor=actor), 200, "User") # --------------------------------------------------------------------------- # Хранение: откат, остановленная БД, исчерпанный пул # --------------------------------------------------------------------------- @pytest.mark.spec("data-storage", "Сбой при добавлении участников") def test_members_failure_rolls_back_chat( api: Api, stand: Stand, alice: Actor, bob: Actor, ) -> None: title = f"fault-{uuid.uuid4().hex}" stand("fail-members", title) try: resp = api.request( "POST", f"{API}/chats", actor=alice, json={"title": title, "member_ids": [alice.id, bob.id]}, ) assert_error(resp, 500) finally: stand("clear-fail-members") for actor in (alice, bob): assert title not in chat_titles(api, actor) assert stand("count-chats", title) == "0" @pytest.mark.spec("data-storage", "БД остановлена") @pytest.mark.spec("data-storage", "БД вернулась") def test_database_stopped_and_back( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with outage(stand, "stop", "postgres"): resp, took = timed( lambda: api.post_message( alice, chat["id"], json={"text": "во время отказа"}, ), ) error = assert_error(resp, 503) assert took < 2, f"503 took {took:.2f}s" for word in ("postgres", "sql", "connect"): assert word not in error["message"].lower() message = api.send_message(alice, chat["id"], "после возврата") # запись во время отказа не зафиксирована, новая — видна assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("data-storage", "Соединения исчерпаны") def test_pool_exhausted(tight_api: Api, stand: Stand) -> None: actor = tight_api.register() holder = Api(str(tight_api.client.base_url), timeout=15) first: list[int] = [] def hold_connection() -> None: first.append(holder.request("GET", ME, actor=actor).status_code) with outage(stand, "pause", "postgres"): thread = threading.Thread(target=hold_connection) thread.start() time.sleep(SLACK_S) resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor)) assert_error(resp, 503) assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s" thread.join() holder.close() assert first == [503] # срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом fresh = tight_api.register() assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User") @pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан") def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None: actor = tight_api.register() # hook оборвал соединение пула: при выдаче pool_pre_ping # переподключается и получает «too many clients» (SQLSTATE 53300) stand("fill-connections", str(FILL_HOLD_S)) try: resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor)) error = assert_error(resp, 503) # отказ PostgreSQL мгновенный: ответ не ждёт срока пула assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s" # недоступность хранилища — не отказ в аутентификации assert "WWW-Authenticate" not in resp.headers for word in ("postgres", "sql", "client", "connect"): assert word not in error["message"].lower() finally: stand("clear-connections") # слоты свободны: сервис снова обслуживает запросы без рестарта; срок # сессии 2 с за время заполнения истёк — новый вход fresh = tight_api.register() assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User") # пачка одновременных запросов больше пула обычного API (10) BURST_THREADS = 48 BURST_AFTER_BARRIER = 4 # PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас STATS_FLUSH_S = 11 def burst(api: Api, actor: Actor) -> None: """Пачка: у каждого потока свой клиент (keep-alive), один запрос до барьера и BURST_AFTER_BARRIER после — около BURST_THREADS одновременно. Все ответы — 200.""" barrier = threading.Barrier(BURST_THREADS) clients = [ Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS) ] responses: list[httpx.Response] = [] errors: list[BaseException] = [] lock = threading.Lock() def worker(client: Api) -> None: try: got = [client.request("GET", ME, actor=actor)] barrier.wait() got += [ client.request("GET", ME, actor=actor) for _ in range(BURST_AFTER_BARRIER) ] except BaseException as e: # noqa: BLE001 — поднимется в тесте barrier.abort() with lock: errors.append(e) return with lock: responses.extend(got) threads = [ threading.Thread(target=worker, args=(client,)) for client in clients ] for thread in threads: thread.start() for thread in threads: thread.join() for client in clients: client.close() assert not errors, f"burst failed: {errors[0]!r}" assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER) for resp in responses: assert_ok(resp, 200, "User") @pytest.mark.spec("data-storage", "Соединения пула не переоткрываются") def test_pool_keeps_connections( api: Api, tight_api: Api, stand: Stand, ) -> None: actor = api.register() # соединение «тесного» API могли оборвать прежние тесты # (fill-connections): его переподключение не должно попасть в окно tight = tight_api.register() assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User") # прогрев: пул обычного API набирает свой размер for _ in range(2): burst(api, actor) # новый backend учитывает себя в sessions не сразу: его счётчики уходят # в общую статистику без ожидания блокировки и при занятой блокировке # откладываются. Простаивающий backend сбрасывает их принудительно по # таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева # попадали в окно по одному time.sleep(STATS_FLUSH_S) # калибровка: второй вызов видит сессию первого вызова hook, и только её s1 = int(stand("db-sessions")) s2 = int(stand("db-sessions")) assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls" for _ in range(3): burst(api, actor) s3 = int(stand("db-sessions")) opened = s3 - s2 - 1 assert opened == 0, ( f"PostgreSQL accepted {opened} new sessions during " f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests" ) # --------------------------------------------------------------------------- # Readiness # --------------------------------------------------------------------------- @pytest.mark.spec("health", "Хранилище не готово") def test_readiness_without_database(api: Api, stand: Stand) -> None: with outage(stand, "stop", "postgres"): resp, took = timed(lambda: api.request("GET", "/health/ready")) assert_error(resp, 503) assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s" live = api.request("GET", "/health/live") assert live.status_code == 200 assert api.request("GET", "/health/ready").status_code == 200 @pytest.mark.spec("health", "Кэш недоступен") def test_readiness_without_cache(api: Api, stand: Stand) -> None: with outage(stand, "stop", "valkey"): assert api.request("GET", "/health/ready").status_code == 200 @pytest.mark.spec("health", "Хранилище объектов недоступно") @pytest.mark.parametrize("action", ["stop", "pause"]) def test_readiness_without_object_storage( api: Api, stand: Stand, action: str, ) -> None: # pause: соединение с S3 есть, ответа нет — проверка, которая всё же # ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения with outage(stand, action, "s3"): for path in ("/health/ready", "/health/live"): resp, took = timed(lambda path=path: api.request("GET", path)) assert assert_ok(resp, 200, "Health") == {"status": "ok"} assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s" # --------------------------------------------------------------------------- # Кэш карточки чата # --------------------------------------------------------------------------- @pytest.mark.spec("chat-cache", "Очистка кэша") def test_flush_cache( api: Api, stand: Stand, bob: Actor, chat: dict[str, Any], ) -> None: wait_cache_ready(api, stand, bob, chat) stand("flush-cache") assert stand("cache-has-chat", chat["id"]) == "0" assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat assert stand("cache-has-chat", chat["id"]) == "1" @pytest.mark.spec("chat-cache", "Valkey остановлен") def test_valkey_stopped( api: Api, stand: Stand, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: sent = [api.send_message(alice, chat["id"], "до отказа")] with outage(stand, "stop", "valkey"): resp, took = timed(lambda: read_chat(api, bob, chat)) assert assert_ok(resp, 200, "Chat") == chat assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s" resp, took = timed( lambda: api.post_message( alice, chat["id"], json={"text": "во время отказа"}, ), ) sent.append(assert_ok(resp, 201, "Message")) assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s" assert api.all_messages(bob, chat["id"]) == sent @pytest.mark.spec("chat-cache", "Valkey завис") def test_valkey_paused( api: Api, stand: Stand, bob: Actor, chat: dict[str, Any], ) -> None: wait_cache_ready(api, stand, bob, chat) baseline = statistics.median( timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5) ) with outage(stand, "pause", "valkey"): resp, took = timed(lambda: read_chat(api, bob, chat)) assert assert_ok(resp, 200, "Chat") == chat assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, ( f"read took {took:.2f}s, baseline {baseline:.3f}s" ) # окно пропуска: следующий запрос таймаута не ждёт resp, took = timed(lambda: read_chat(api, bob, chat)) assert assert_ok(resp, 200, "Chat") == chat assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s" @pytest.mark.spec("chat-cache", "Valkey вернулся") def test_valkey_back( api: Api, stand: Stand, bob: Actor, chat: dict[str, Any], ) -> None: with outage(stand, "stop", "valkey"): assert_ok(read_chat(api, bob, chat), 200, "Chat") # API не перезапускался; кэш снова в работе после окна пропуска time.sleep(CACHE_RETRY_AFTER_S + 0.5) assert_ok(read_chat(api, bob, chat), 200, "Chat") assert stand("cache-has-chat", chat["id"]) == "1" hits = int(stand("cache-hits")) assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat assert int(stand("cache-hits")) > hits # --------------------------------------------------------------------------- # Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API # --------------------------------------------------------------------------- @pytest.mark.spec("idempotency", "Сбой при записи сообщения") def test_message_failure_keeps_key_free( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: # ключ вставляется первым: триггер роняет транзакцию уже после него text = f"fault-{uuid.uuid4().hex}" key = new_key() stand("fail-messages", text) try: resp = api.post_message( alice, chat["id"], key=key, json={"text": text}, ) assert_error(resp, 500) finally: stand("clear-fail-messages") # ключ не занят: повтор выполняется заново, а не 500 и не 409 message = api.send_message(alice, chat["id"], text, key=key) assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("idempotency", "Отказ БД при отправке") def test_database_outage_keeps_key_free( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: text = "при отказе БД" key = new_key() with outage(stand, "stop", "postgres"): resp = api.post_message( alice, chat["id"], key=key, json={"text": text}, ) assert_error(resp, 503) message = api.send_message(alice, chat["id"], text, key=key) assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("idempotency", "Ключ после срока") def test_key_free_after_ttl(tight_api: Api) -> None: alice = tight_api.register() chat = tight_api.create_chat(alice) key = new_key() first = tight_api.send_message(alice, chat["id"], "до срока", key=key) # в пределах срока ключ занят: другое тело — 409 resp = tight_api.post_message( alice, chat["id"], key=key, json={"text": "после срока"}, ) assert_error(resp, 409) time.sleep(IDEMPOTENCY_TTL_S + SLACK_S) # сессия «тесного» API тоже истекла: новый вход, ключ тот же alice = tight_api.login(alice.user, alice.password) second = tight_api.send_message(alice, chat["id"], "после срока", key=key) assert second["id"] != first["id"] history = tight_api.all_messages(alice, chat["id"]) assert [item["id"] for item in history] == [first["id"], second["id"]] # --------------------------------------------------------------------------- # Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки # и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает # строки прежних тестов параллельно (design change upload-consistency, # решение 10) # --------------------------------------------------------------------------- # grace и интервал уборщика стенда отказов (run_failures.py, # failures.compose.yml) CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20")) CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1")) # запас поверх grace и интервала: проход, вызов hook, планировщик Docker # Desktop CLEANUP_SLACK_S = 5 # задержка перевода в queued триггером stall и в ready триггером ready-stall # (fault_hook.py) STALL_S = 4 # срок клиента загрузок в потоке THREAD_CLIENT_TIMEOUT_S = 30 # имя базы стенда: ключи объектов — `<база>/attachments//original` и # `<база>/staging//original` FAULT_DATABASE = os.environ.get("FAULT_DATABASE", "messenger") def attachment_rows(stand: Stand) -> dict[str, str]: """Вложения стенда: `id` → статус.""" return dict(line.split() for line in stand("attachment-ids").splitlines()) def stored_ids(stand: Stand) -> set[str]: """`id` вложений с объектом в конечном префиксе bucket стенда.""" return set(stand("object-ids").splitlines()) def staged_ids(stand: Stand) -> set[str]: """`id` вложений с объектом в staging.""" return set(stand("staging-ids").splitlines()) def object_ids(stand: Stand) -> set[str]: """`id` вложений с объектом в любом из двух префиксов.""" return stored_ids(stand) | staged_ids(stand) def orphan_objects(stand: Stand) -> set[str]: """Объекты вложений без строки. Сначала строки, потом объекты: уборщик удаляет объект раньше строки, и убранная между вызовами строка не даёт ложного «объекта без строки».""" rows = attachment_rows(stand) return object_ids(stand) - set(rows) def accepted_without_staging(stand: Stand) -> set[str]: """Принятые вложения без своего объекта: в работе — staging, готовые — конечного оригинала (staging у них убирает уборщик). У `failed` его может не быть по самой причине отказа.""" rows = attachment_rows(stand) working = { item for item, status in rows.items() if status in ("queued", "processing") } return (working - staged_ids(stand)) | ( ready_ids(rows) - stored_ids(stand) ) def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]: return set(after) - set(before) def objects_and_rows(stand: Stand) -> tuple[dict[str, str], set[str]]: """Снимок: строки и объекты вложений обоих префиксов.""" return attachment_rows(stand), object_ids(stand) def new_objects( before: tuple[dict[str, str], set[str]], stand: Stand, ) -> set[str]: """Объекты, которых не было в снимке, у `id`, которых в нём тоже не было. Worker параллельно доводит задания прежних тестов: объект конечного ключа у прежнего `id` — не след новой загрузки.""" return object_ids(stand) - before[1] - set(before[0]) def job_row(stand: Stand, attachment_id: str) -> dict[str, Any] | None: """Задание вложения; нет — `None`. Два задания дали бы две строки и ошибку разбора: задание у вложения одно.""" output = stand("job", attachment_id) return json.loads(output) if output else None def ready_ids(rows: dict[str, str]) -> set[str]: return {item for item, status in rows.items() if status == "ready"} def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None: """Строка и объект вложения убраны не позже grace + интервал + запас от `since` (момент до загрузки: `created_at` строки не раньше).""" deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S while True: rows = attachment_rows(stand) stored = object_ids(stand) if attachment_id not in rows and attachment_id not in stored: return if time.monotonic() > deadline: raise AssertionError( f"attachment {attachment_id} is not cleaned " f"{time.monotonic() - since:.1f}s after upload: " f"row {rows.get(attachment_id)}, " f"object {attachment_id in stored}", ) time.sleep(0.5) def sleep_until(moment: float) -> None: time.sleep(max(0.0, moment - time.monotonic())) def fault_filename() -> str: """Уникальное имя загрузки, на которое срабатывает триггер hook.""" return f"fault-{uuid.uuid4().hex}.png" def upload_raw( api: Api, actor: Actor, chat: dict[str, Any], data: Any, **kwargs: Any, ) -> httpx.Response: return api.post_attachment(actor, chat["id"], data, **kwargs) def assert_invisible( api: Api, actor: Actor, chat: dict[str, Any], attachment_id: str, ) -> None: """Незавершённая загрузка: метаданные, байты и ссылка в сообщении — 404 с тем же `message`, что у неизвестного `id`.""" def calls(item: str) -> tuple[httpx.Response, ...]: return ( api.attachment_meta(actor, item), api.attachment_content(actor, item), api.post_message( actor, chat["id"], json={"text": "ссылка", "attachment_ids": [item]}, ), ) unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))] hidden = [assert_error(resp, 404) for resp in calls(attachment_id)] assert [error["message"] for error in hidden] == [ error["message"] for error in unknown ] @pytest.mark.spec("attachments", "Запись в S3 не удалась") @pytest.mark.spec("attachments", "Незавершённая загрузка не видна") def test_upload_with_object_storage_stopped( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() # хотя бы одно готовое вложение: контроль ниже не пуст случайно ready = api.upload_ready(alice, chat["id"], data) before = attachment_rows(stand) assert ready["id"] in ready_ids(before) with outage(stand, "stop", "s3"): resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data)) assert_error(resp, 503) assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s" after = attachment_rows(stand) # строка-намерение осталась; её уберёт уборщик # (test_cleaner_survives_outages) (pending,) = new_ids(before, after) assert after[pending] == "pending" # ready не откатывается; число готовых может расти: worker # параллельно доводит задания прежних тестов assert ready_ids(before) <= ready_ids(after) # заданий и принятых не прибавилось: новая строка одна и pending assert job_row(stand, pending) is None # замечание S6 verify attachments: id последней pending — от hook assert stand("last-pending") == pending assert_invisible(api, alice, chat, pending) api.send_message(alice, chat["id"], "текст без S3") api.upload(alice, chat["id"], data) @pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет") def test_rejected_uploads_leave_nothing( api: Api, stand: Stand, alice: Actor, eve: Actor, chat: dict[str, Any], ) -> None: big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2) def chunked() -> Iterator[bytes]: for offset in range(0, len(big), 1024 * 1024): yield big[offset : offset + 1024 * 1024] # хотя бы одно готовое вложение: списки hook не пусты случайно ready = api.upload_ready(alice, chat["id"], images.sample_png()) before = objects_and_rows(stand) assert before[0].get(ready["id"]) == "ready", before[0] assert ready["id"] in before[1], before[1] rejected = ( (404, upload_raw(api, eve, chat, images.sample_png())), (413, upload_raw(api, alice, chat, big)), (413, upload_raw(api, alice, chat, chunked())), (413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))), ( 415, upload_raw( api, alice, chat, images.sample_png(), content_type="text/plain", ), ), (415, upload_raw(api, alice, chat, images.png_idat_byte_changed())), ) for status, resp in rejected: assert_error(resp, status) assert not new_ids(before[0], attachment_rows(stand)) assert not new_objects(before, stand) @pytest.mark.spec("attachments", "Сбой записи метаданных") def test_metadata_insert_failure( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: filename = fault_filename() before = objects_and_rows(stand) stand("fail-attachments", "insert", filename) try: resp = upload_raw( api, alice, chat, images.sample_png(), filename=filename, ) assert_error(resp, 503) finally: stand("clear-fail-attachments") # строки нет — объект не записывался assert not new_ids(before[0], attachment_rows(stand)) assert not new_objects(before, stand) class BodyAborted(Exception): """Клиент обрывает передачу тела загрузки.""" @pytest.mark.spec("attachments", "Обрыв передачи тела") def test_interrupted_body( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = objects_and_rows(stand) def aborted_body() -> Iterator[bytes]: yield images.PNG_SIGNATURE + bytes(1024 * 1024) raise BodyAborted # свой клиент: его соединение обрывается вместе с телом with ( Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own, pytest.raises((BodyAborted, httpx.TransportError)), ): upload_raw(own, alice, chat, aborted_body()) # API замечает обрыв не мгновенно; строки до конца тела нет в любом # случае time.sleep(1) assert not new_ids(before[0], attachment_rows(stand)) assert not new_objects(before, stand) api.upload(alice, chat["id"], images.sample_png()) @pytest.mark.spec("attachments", "Фиксация задания не удалась") @pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута") @pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана") def test_queued_failure_compensated( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "queued", filename) try: started = time.monotonic() resp = upload_raw( api, alice, chat, images.sample_png(), filename=filename, ) assert_error(resp, 503) finally: stand("clear-fail-attachments") after = attachment_rows(stand) (pending,) = new_ids(before, after) assert after[pending] == "pending" assert stand("last-pending") == pending # компенсация: объект удалён до ответа 503, задания нет assert pending not in object_ids(stand) assert job_row(stand, pending) is None assert_invisible(api, alice, chat, pending) # моложе grace строку уборщик не трогает: к половине grace он сделал # около G / 2 / I проходов sleep_until(started + CLEANER_GRACE_S / 2) assert attachment_rows(stand).get(pending) == "pending" wait_cleaned(stand, pending, since=started) assert not orphan_objects(stand) @pytest.mark.spec("storage-consistency", "Объект без фиксации убран") def test_killed_between_put_and_ready( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: filename = fault_filename() before = attachment_rows(stand) outcome: dict[str, Any] = {} client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) def upload() -> None: try: outcome["status"] = upload_raw( client, alice, chat, images.sample_png(), filename=filename, ).status_code except httpx.TransportError as e: outcome["error"] = e stand("fail-attachments", "stall", filename) killed = False try: started = time.monotonic() thread = threading.Thread(target=upload) thread.start() # строка pending и объект staging — до конца задержки перевода в # queued deadline = started + STALL_S pending = "" while time.monotonic() < deadline: pending = stand("last-pending") if ( pending and pending not in before and pending in staged_ids(stand) ): break time.sleep(0.05) else: raise AssertionError("no pending row with object during stall") stand("kill-api") killed = True thread.join(THREAD_CLIENT_TIMEOUT_S) assert not thread.is_alive() # 202 до клиента не дошёл: перевод в queued ждал в триггере assert outcome.get("status") != 202, outcome assert attachment_rows(stand).get(pending) == "pending" assert pending in staged_ids(stand) assert job_row(stand, pending) is None finally: client.close() stand("clear-fail-attachments") if killed: stand("start-api") wait_cleaned(stand, pending, since=started) assert not orphan_objects(stand) api.upload(alice, chat["id"], images.sample_png()) @pytest.mark.spec("attachments", "Исход фиксации неизвестен") def test_queued_commit_outcome_unknown( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Перевод в queued падает на самом COMMIT (отложенный триггер): API не знает, зафиксировано ли вложение, и объект не трогает (verify W2).""" filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "commit", filename) try: started = time.monotonic() resp = upload_raw( api, alice, chat, images.sample_png(), filename=filename, ) assert_error(resp, 503) finally: stand("clear-fail-attachments") after = attachment_rows(stand) (pending,) = new_ids(before, after) assert after[pending] == "pending" # компенсации нет: вложение могло стать queued assert pending in staged_ids(stand) assert_invisible(api, alice, chat, pending) wait_cleaned(stand, pending, since=started) assert not orphan_objects(stand) @pytest.mark.spec("storage-consistency", "Загрузка дольше grace") def test_upload_outlives_grace( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Строка вставлена «на час старше», перевод в queued ждёт в триггере, пока её не уберёт уборщик стенда: загрузка, пережившая grace, без заморозки процесса (verify W3).""" filename = fault_filename() before = objects_and_rows(stand) stand("fail-attachments", "outlive", filename) # ожидание в триггере до 4,5 с: срок клиента больше slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) try: with slow: resp = upload_raw( slow, alice, chat, images.sample_png(), filename=filename, ) # не 202 (принято без объекта) и не 500 (дефект ветки) assert_error(resp, 503) finally: stand("clear-fail-attachments") rows = attachment_rows(stand) assert not new_ids(before[0], rows), rows assert not new_objects(before, stand) assert not orphan_objects(stand) @pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта") def test_cleaner_failure_after_object_delete( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Уборщик удалил объект, а удаление строки упало (08006), пока загрузка ждёт перевода в queued: строка помечена до удаления объекта и queued уже не станет (verify W1).""" filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "undeletable", filename) slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) try: with slow: resp = upload_raw( slow, alice, chat, images.sample_png(), filename=filename, ) assert_error(resp, 503) (pending,) = new_ids(before, attachment_rows(stand)) assert attachment_rows(stand).get(pending) == "pending" assert job_row(stand, pending) is None assert pending not in staged_ids(stand) assert not accepted_without_staging(stand) assert_invisible(api, alice, chat, pending) finally: stand("clear-fail-attachments") # строка старше grace: следующий проход доводит уборку returned = time.monotonic() wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S) assert not orphan_objects(stand) assert not accepted_without_staging(stand) @pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки") @pytest.mark.spec("storage-consistency", "БД недоступна во время уборки") @pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты") def test_cleaner_survives_outages( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() ready = api.upload_ready(alice, chat["id"], data) with outage(stand, "stop", "s3"): before = attachment_rows(stand) started = time.monotonic() assert_error(upload_raw(api, alice, chat, data), 503) (pending,) = new_ids(before, attachment_rows(stand)) # PostgreSQL недоступен, пока загрузка моложе grace with outage(stand, "stop", "postgres"): time.sleep(3) # grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S) assert attachment_rows(stand).get(pending) == "pending" # S3 вернулся: уборщик без перезапуска убирает строку за проход returned = time.monotonic() deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S while pending in attachment_rows(stand): assert time.monotonic() < deadline, ( f"pending {pending} is still there " f"{time.monotonic() - returned:.1f}s after S3 came back" ) time.sleep(0.5) assert pending not in object_ids(stand) assert not orphan_objects(stand) # готовое вложение старше grace на месте meta = assert_ok( api.attachment_meta(alice, ready["id"]), 200, "Attachment", ) assert meta == ready assert meta["status"] == "ready" assert_content( api.attachment_content(alice, ready["id"]), data, "image/png", ) @pytest.mark.spec("attachments", "S3 остановлен") def test_object_storage_stopped( api: Api, stand: Stand, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() attachment = api.upload_ready(alice, chat["id"], data) with outage(stand, "stop", "s3"): resp, elapsed = timed( lambda: api.attachment_content(bob, attachment["id"]), ) assert_error(resp, 503) assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s" meta = assert_ok( api.attachment_meta(bob, attachment["id"]), 200, "Attachment", ) assert meta == attachment message = api.send_message( bob, chat["id"], "", attachment_ids=[attachment["id"]], ) # история с attachment_ids — из PostgreSQL, S3 ей не нужен history = api.all_messages(alice, chat["id"]) assert [item["id"] for item in history] == [message["id"]] assert history[0]["attachment_ids"] == [attachment["id"]] assert_ok(api.request("GET", "/health/ready"), 200, "Health") assert_content( api.attachment_content(bob, attachment["id"]), data, "image/png", ) def test_environment_does_not_change_limits( api: Api, alice: Actor, chat: dict[str, Any], ) -> None: """Спек `service-configuration`, «Окружение не меняет лимит» (вне сверки покрытия — сценарий в докстринге): обычный API стенда отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` (`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт, 64 000 пикселей) всё равно принимается: лимиты — константы.""" limits = dict( item.split("=", 1) for item in os.environ.get("FAULT_LIMITS_ENV", "").split() ) # без этих переменных на стенде тест ничего не доказывает assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, ( f"stand started without limit variables: {limits}" ) api.upload(alice, chat["id"], images.sample_png(), filename="sample.png") @pytest.mark.spec("attachments", "S3 завис") @pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана") def test_object_storage_paused( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() attachment = api.upload_ready(alice, chat["id"], data) before = attachment_rows(stand) started = time.monotonic() # срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S) with slow, outage(stand, "pause", "s3"): for call in ( lambda: upload_raw(slow, alice, chat, data), lambda: slow.attachment_content(alice, attachment["id"]), ): resp, elapsed = timed(call) assert_error(resp, 503) # срок вызова S3 и немного на шаги с БД, а не зависание assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, ( f"503 after {elapsed:.2f}s" ) assert_content( api.attachment_content(alice, attachment["id"]), data, "image/png", ) # загрузка во время заморозки оставила строку pending; объект после # разморозки не дописался, строку уборщик убирает после grace (pending,) = new_ids(before, attachment_rows(stand)) wait_cleaned(stand, pending, since=started) assert not orphan_objects(stand) # --------------------------------------------------------------------------- # Места загрузки и срок тела (change upload-memory, design, решение 11): # «узкий по загрузкам» API — одно место, одна загрузка в очереди, ожидание # места и срок тела из FAULT_UPLOADS_ENV (run_failures.py, UPLOADS_ENV). # Место держит загрузка, тело которой стоит на threading.Event теста # --------------------------------------------------------------------------- UPLOADS_ENV = dict( item.split("=", 1) for item in os.environ.get("FAULT_UPLOADS_ENV", "").split() ) # окружение, с которым тесты ниже ждут «узкий» API EXPECTED_UPLOADS_ENV = { "UPLOADS__MAX_CONCURRENT": "1", "UPLOADS__MAX_WAITING": "1", "UPLOADS__WAIT_TIMEOUT_SECONDS": "1", "UPLOADS__BODY_TIMEOUT_SECONDS": "3", } WAIT_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__WAIT_TIMEOUT_SECONDS", "1")) BODY_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__BODY_TIMEOUT_SECONDS", "3")) # отказ без ожидания места: вход, членство и ответ IMMEDIATE_S = 0.5 # пауза, за которую запрос в соседнем потоке доходит до места или очереди SETTLE_S = 0.2 # сколько держатель места ждёт отпускания, если тест упал раньше HOLD_MAX_S = 10 def assert_uploads_stand() -> None: """Без этого окружения «узкого» API тесты мест ничего не доказывают.""" assert UPLOADS_ENV == EXPECTED_UPLOADS_ENV, ( f"uploads stand started with {UPLOADS_ENV}, " f"expected {EXPECTED_UPLOADS_ENV}" ) class Upload: """Загрузка в своём потоке и своём соединении; ответ — `result()`.""" def __init__(self, api: Api, actor: Actor, chat: dict[str, Any]) -> None: self._base_url = str(api.client.base_url) self._actor = actor self._chat = chat self._outcome: dict[str, Any] = {} self._thread: threading.Thread | None = None def start(self, data: Any, **kwargs: Any) -> "Upload": def run() -> None: with Api(self._base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own: started = time.monotonic() try: self._outcome["resp"] = upload_raw( own, self._actor, self._chat, data, **kwargs, ) except (BodyAborted, httpx.TransportError) as e: self._outcome["error"] = e self._outcome["elapsed"] = time.monotonic() - started self._outcome["ended"] = time.monotonic() self._thread = threading.Thread(target=run) self._thread.start() return self def result(self) -> tuple[httpx.Response, float, float]: """Ответ, длительность и момент ответа (`time.monotonic`).""" assert self._thread is not None self._thread.join(THREAD_CLIENT_TIMEOUT_S) assert not self._thread.is_alive(), "upload thread hangs" assert "resp" in self._outcome, self._outcome return ( self._outcome["resp"], self._outcome["elapsed"], self._outcome["ended"], ) @contextmanager def held_slot( api: Api, actor: Actor, chat: dict[str, Any], data: bytes, ) -> Iterator[Callable[[], httpx.Response]]: """Загрузка A держит место: отдаёт половину тела и ждёт отпускания. Внутри — после того, как A заняла место; вызов отдаваемой функции отпускает A и возвращает её ответ. В `finally` A отпускается всегда. """ sent = threading.Event() release = threading.Event() def body() -> Iterator[bytes]: yield data[: len(data) // 2] sent.set() if not release.wait(HOLD_MAX_S): raise BodyAborted yield data[len(data) // 2 :] holder = Upload(api, actor, chat).start(body()) try: assert sent.wait(HOLD_MAX_S), "holder did not start its body" # вход и членство — короткие шаги БД, затем место time.sleep(SETTLE_S) def finish() -> httpx.Response: release.set() return holder.result()[0] yield finish finally: release.set() def slow_chunks(data: bytes, *, parts: int, seconds: float) -> Iterator[bytes]: """Тело `parts` частями, растянутое на `seconds`.""" size = -(-len(data) // parts) for offset in range(0, len(data), size): if offset: time.sleep(seconds / (parts - 1)) yield data[offset : offset + size] @pytest.mark.spec("attachments", "Все места заняты") @pytest.mark.spec("attachments", "Очередь ожидания полна") @pytest.mark.spec("api-conventions", "Лимит одновременных загрузок") def test_upload_slots_busy( uploads_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_uploads_stand() data = images.sample_png() before = objects_and_rows(stand) with held_slot(uploads_api, alice, chat, data) as finish_holder: # B ждёт место в очереди, C застаёт полную очередь waiting = Upload(uploads_api, alice, chat).start(data) time.sleep(SETTLE_S) busy, busy_s = timed( lambda: upload_raw(uploads_api, alice, chat, data), ) waited, waited_s, _ = waiting.result() # A отпускается сразу: её тело ограничено сроком тела стенда holder = finish_holder() assert_error(busy, 429) assert busy_s < IMMEDIATE_S, f"429 of a full queue after {busy_s:.2f}s" assert_error(waited, 429) assert waited_s >= WAIT_TIMEOUT_S, f"429 after {waited_s:.2f}s" for resp in (busy, waited): assert int(resp.headers["Retry-After"]) >= 1 attachment = assert_accepted(holder) # из трёх загрузок осталась только принятая: у 429 ни строки, ни объекта assert new_ids(before[0], attachment_rows(stand)) == {attachment["id"]} assert new_objects(before, stand) == {attachment["id"]} @pytest.mark.spec("attachments", "Отказ раньше очереди") def test_rejections_before_queue( uploads_api: Api, alice: Actor, eve: Actor, chat: dict[str, Any], ) -> None: assert_uploads_stand() data = images.sample_png() # 30 MiB, которые начинаются как PNG: Content-Length 31 457 280 big = images.PNG_SIGNATURE + bytes( 3 * images.MAX_BYTES - len(images.PNG_SIGNATURE), ) with held_slot(uploads_api, alice, chat, data) as finish_holder: waiting = Upload(uploads_api, alice, chat).start(data) time.sleep(SETTLE_S) early = [ (status, *timed(call)) for status, call in ( (404, lambda: upload_raw(uploads_api, eve, chat, data)), (413, lambda: upload_raw(uploads_api, alice, chat, big)), ( 415, lambda: upload_raw( uploads_api, alice, chat, data, content_type="text/plain", ), ), ) ] early_done = time.monotonic() waited, _, waited_at = waiting.result() holder = finish_holder() # очередь была полна всё время ранних отказов assert_error(waited, 429) assert early_done < waited_at for status, resp, elapsed in early: assert_error(resp, status) # без ожидания места: ожидающий получил бы его не раньше срока assert elapsed < WAIT_TIMEOUT_S, f"{status} after {elapsed:.2f}s" assert early[1][1].request.headers["Content-Length"] == "31457280" assert_accepted(holder) @pytest.mark.spec("attachments", "Ожидание места") @pytest.mark.spec( "storage-consistency", "Ожидание места не входит в возраст загрузки", ) def test_upload_waits_for_slot( uploads_api: Api, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: assert_uploads_stand() first = images.sample_png() second = images.png(80, 60) with held_slot(uploads_api, alice, chat, first) as finish_holder: waiting = Upload(uploads_api, bob, chat).start(second) time.sleep(SETTLE_S) # A заканчивается раньше срока ожидания B holder = finish_holder() held = assert_accepted(holder) waited_resp, waited_s, _ = waiting.result() waited = assert_accepted(waited_resp) assert waited_s < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, waited_s # метаданные ожидавшей записаны после перевода в queued державшей место: # ожидание прошло до строки pending (граница B уборщика прежняя) assert waited["created_at"] > held["updated_at"], (held, waited) for attachment, data in ((held, first), (waited, second)): uploads_api.wait_attachment(alice, attachment["id"]) assert_content( uploads_api.attachment_content(alice, attachment["id"]), data, "image/png", ) @pytest.mark.spec("attachments", "Место освобождается") def test_slot_released_after_failures( uploads_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_uploads_stand() over = images.png_of_size(images.MAX_BYTES + 1) def aborted_body() -> Iterator[bytes]: yield images.PNG_SIGNATURE + bytes(1024 * 1024) raise BodyAborted def next_upload_gets_slot(after: str) -> None: resp, elapsed = timed( lambda: upload_raw(uploads_api, alice, chat, images.sample_png()), ) assert_accepted(resp) assert elapsed < WAIT_TIMEOUT_S, ( f"upload after {after} waited {elapsed:.2f}s for a slot" ) assert_error( upload_raw( uploads_api, alice, chat, slow_chunks(over, parts=10, seconds=0), ), 413, ) next_upload_gets_slot("413 by counter") assert_error(upload_raw(uploads_api, alice, chat, b"not an image"), 415) next_upload_gets_slot("415 by content") slow = slow_chunks( images.sample_png(), parts=4, seconds=BODY_TIMEOUT_S + 1, ) assert_error(upload_raw(uploads_api, alice, chat, slow), 400) next_upload_gets_slot("400 by body timeout") with ( Api( str(uploads_api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S, ) as own, pytest.raises((BodyAborted, httpx.TransportError)), ): upload_raw(own, alice, chat, aborted_body()) # API замечает обрыв не мгновенно time.sleep(SETTLE_S) next_upload_gets_slot("interrupted body") with outage(stand, "stop", "s3"): assert_error( upload_raw(uploads_api, alice, chat, images.sample_png()), 503, ) next_upload_gets_slot("503 with S3 stopped") @pytest.mark.spec("attachments", "Медленный клиент") @pytest.mark.spec("attachments", "Медленная передача в срок") def test_slow_body( uploads_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_uploads_stand() data = images.sample_png() before = objects_and_rows(stand) late = upload_raw( uploads_api, alice, chat, slow_chunks(data, parts=6, seconds=BODY_TIMEOUT_S + 1), ) error = assert_error(late, 400) assert error["message"] == "Upload body was not received in time" assert not new_ids(before[0], attachment_rows(stand)) assert not new_objects(before, stand) # мелкие части с паузами, но в срок: тело принято как есть in_time = slow_chunks(data, parts=20, seconds=BODY_TIMEOUT_S / 2) resp, elapsed = timed( lambda: upload_raw(uploads_api, alice, chat, in_time), ) attachment = assert_accepted(resp) assert elapsed < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, elapsed assert attachment["size_bytes"] == len(data) assert attachment["sha256"] == hashlib.sha256(data).hexdigest() uploads_api.wait_attachment(alice, attachment["id"]) assert_content( uploads_api.attachment_content(alice, attachment["id"]), data, "image/png", ) # --------------------------------------------------------------------------- # Простой S3 после скачиваний (change fix-upload-put-flake, design, решение # 3): API «с коротким сроком S3» — срок вызова S3 из FAULT_S3_ENV # (run_failures.py, S3_ENV), других тестов на нём нет # --------------------------------------------------------------------------- S3_ENV = dict( item.split("=", 1) for item in os.environ.get("FAULT_S3_ENV", "").split() ) # окружение, с которым тест ниже ждёт API с коротким сроком S3 EXPECTED_S3_ENV = {"S3__TIMEOUT_SECONDS": "1"} SHORT_S3_TIMEOUT_S = float(S3_ENV.get("S3__TIMEOUT_SECONDS", "1")) # раунды: скачивания, простой дольше срока вызова S3, загрузки IDLE_ROUNDS = 8 IDLE_DOWNLOADS = 10 IDLE_UPLOADS = 12 IDLE_EXTRA_S = 0.5 # медленный клиент: пауза после каждого блока 64 KiB IDLE_READ_PAUSE_S = 0.002 def assert_s3_stand() -> None: """Без этого окружения API с коротким сроком S3 тест ничего не доказывает: простой 1,5 с был бы короче срока вызова.""" assert S3_ENV == EXPECTED_S3_ENV, ( f"short S3 timeout stand started with {S3_ENV}, " f"expected {EXPECTED_S3_ENV}" ) @pytest.mark.spec("attachments", "Загрузка после простоя S3") def test_upload_after_idle_s3( s3_api: Api, alice: Actor, chat: dict[str, Any], ) -> None: """Скачивания большого оригинала, простой S3 дольше срока вызова, затем загрузки — каждая 202, ни одной 503. Раунд: 10 одновременных медленных скачиваний (своё соединение у каждого клиента, барьер, 2 мс на блок 64 KiB) PNG ~4 MiB, простой `S3__TIMEOUT_SECONDS + 0,5` с без запросов к этому API, 12 загрузок `sample.png` по одной: пул клиента S3 отдаёт соединения по порядку возврата, и 12 вызовов проходят через все ≤ 10 его соединений. До правки клиента (срок чтения aiohttp, переживший ответ) соединение портится в раунде с p ≈ 0,28 (design, «Как подтверждено», п. 8): промах раунда — 0,72^10 ≈ 0,037, восьми раундов — ≈ 4e-12. После правки 8 раундов — ~16 с. """ assert_s3_stand() base_url = str(s3_api.client.base_url) original = images.png_noise(1024, 1365, seed=4) attachment = s3_api.upload_ready( alice, chat["id"], original, filename="idle-original.png", ) sample = images.sample_png() def download(barrier: threading.Barrier) -> httpx.Response: with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own: # соединение открыто до барьера: после него сразу скачивание own.request("GET", "/health/live") barrier.wait() return own.attachment_content_slowly( alice, attachment["id"], pause_s=IDLE_READ_PAUSE_S, ) for round_number in range(1, IDLE_ROUNDS + 1): barrier = threading.Barrier(IDLE_DOWNLOADS) with ThreadPoolExecutor(max_workers=IDLE_DOWNLOADS) as pool: downloads = list( pool.map(download, [barrier] * IDLE_DOWNLOADS), ) for resp in downloads: assert_content(resp, original, "image/png") time.sleep(SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S) for upload_number in range(1, IDLE_UPLOADS + 1): resp = upload_raw(s3_api, alice, chat, sample) assert resp.status_code == 202, describe( resp, f"round {round_number}, upload {upload_number} after " f"{SHORT_S3_TIMEOUT_S + IDLE_EXTRA_S:.1f}s of S3 idle", ) assert_accepted(resp) # --------------------------------------------------------------------------- # Фоновая обработка, брокер и worker (change async-attachments, design, решение # 15). Worker стенда: 4 попытки, пауза 5 с, срок попытки 10 с, срок остановки # 2 с (FAULT_WORKER_*). Перед остановкой worker очередь доводится до пустой: # задания прежних тестов не считаются в «одно сообщение». Worker и триггеры # возвращаются в finally, упавшая проверка не ломает следующие # --------------------------------------------------------------------------- WORKER_MAX_ATTEMPTS = int(os.environ.get("FAULT_WORKER_MAX_ATTEMPTS", "4")) WORKER_RETRY_BASE_S = float(os.environ.get("FAULT_WORKER_RETRY_BASE_S", "4")) WORKER_RETRY_MAX_S = float(os.environ.get("FAULT_WORKER_RETRY_MAX_S", "16")) WORKER_JOB_TIMEOUT_S = float( os.environ.get("FAULT_WORKER_JOB_TIMEOUT_S", "10"), ) WORKER_LEASE_MARGIN_S = float( os.environ.get("FAULT_WORKER_LEASE_MARGIN_S", "2"), ) WORKER_LEASE_S = WORKER_JOB_TIMEOUT_S + WORKER_LEASE_MARGIN_S RELAY_REPUBLISH_S = float(os.environ.get("FAULT_RELAY_REPUBLISH_S", "5")) STAGING_RETENTION_S = float( os.environ.get("FAULT_CLEANER_STAGING_RETENTION_S", "60"), ) # [worker] concurrency по умолчанию: стенд отказов её не меняет WORKER_CONCURRENCY = 2 # ответ на загрузку не ждёт брокера и relay (спек job-outbox) ACCEPT_MAX_S = 1 # отказ брокера не держит health: быстрее секунды, как у S3 BROKER_HEALTH_S = 1 # срок ожидания итога задания: все попытки и паузы с запасом JOB_WAIT_S = 60 # relay публикует намерение за проход; срок ожидания сообщения с отказом брокера PUBLISH_WAIT_S = 15 JOBS_SCHEMA_PATH = ( Path(__file__).resolve().parents[1] / "contracts" / "jobs.schema.json" ) # срок «Stopped worker» по контракту остановки контейнера WORKER_STOP_MAX_S = 10 ALL_VARIANTS = (None, "original", "thumbnail", "crop", "optimized") VARIANT_NAMES = {"thumbnail", "crop", "optimized"} # предел памяти брокера в конфигурации стенда: его возвращает проба сдерживания BROKER_MEMORY_LIMIT = "400MiB" def queue_state(stand: Stand) -> dict[str, Any]: state = json.loads(stand("queue-state")) assert state is not None, "job queue does not exist" return state def wait_queue_messages( stand: Stand, count: int, *, timeout: float = PUBLISH_WAIT_S, ) -> dict[str, Any]: """В очереди столько сообщений: relay публикует намерения за проход.""" deadline = time.monotonic() + timeout while True: state = json.loads(stand("queue-state")) if state is not None and state["messages"] == count: return state assert time.monotonic() < deadline, ( f"queue has not {count} messages: {state}" ) time.sleep(0.2) @contextmanager def relay_stopped(stand: Stand) -> Iterator[None]: """Relay остановлен; в `finally` запускается.""" with outage(stand, "stop", "relay"): yield def relay_records(stand: Stand) -> list[dict[str, Any]]: return [ json.loads(line) for line in stand("relay-log").splitlines() if line.startswith("{") ] def wait_relay_record( stand: Stand, event: str, *, timeout: float = 30, skip: int = 0, **fields: Any, ) -> dict[str, Any]: """Запись журнала relay с этими полями после первых `skip` записей.""" deadline = time.monotonic() + timeout while True: for record in relay_records(stand)[skip:]: if record.get("event") == event and all( record.get(name) == value for name, value in fields.items() ): return record assert time.monotonic() < deadline, ( f"no relay record {event!r} with {fields} in {timeout} s" ) time.sleep(0.3) def extra_records(stand: Stand, name: str = "worker") -> list[dict[str, Any]]: """Записи журнала дополнительного процесса hook `run-worker`.""" return [ json.loads(line) for line in stand("extra-log", name).splitlines() if line.startswith("{") ] def wait_extra_record( stand: Stand, event: str, *, timeout: float = 30, extra: str = "worker", **fields: Any, ) -> dict[str, Any]: deadline = time.monotonic() + timeout while True: for record in extra_records(stand, extra): if record.get("event") == event and all( record.get(name) == value for name, value in fields.items() ): return record assert time.monotonic() < deadline, ( f"no {extra} record {event!r} with {fields} in {timeout} s" ) time.sleep(0.3) def wait_queue_empty(stand: Stand, *, timeout: float = 90) -> None: """В очереди не осталось сообщений, в том числе неподтверждённых.""" deadline = time.monotonic() + timeout while True: state = queue_state(stand) if state["messages"] == 0: return assert time.monotonic() < deadline, f"queue is not empty: {state}" time.sleep(0.5) @contextmanager def worker_stopped(stand: Stand) -> Iterator[None]: """Worker остановлен при пустой очереди; в `finally` запускается.""" wait_queue_empty(stand) with outage(stand, "stop", "worker"): yield def worker_records(stand: Stand) -> list[dict[str, Any]]: """Записи журнала worker за все его запуски.""" return [ json.loads(line) for line in stand("worker-log").splitlines() if line.startswith("{") ] def wait_worker_record( stand: Stand, event: str, *, timeout: float = 30, skip: int = 0, **fields: Any, ) -> dict[str, Any]: """Запись журнала worker с этими полями после первых `skip` записей.""" deadline = time.monotonic() + timeout while True: for record in worker_records(stand)[skip:]: if record.get("event") == event and all( record.get(name) == value for name, value in fields.items() ): return record assert time.monotonic() < deadline, ( f"no worker record {event!r} with {fields} in {timeout} s" ) time.sleep(0.3) def job_body( job_id: str, attachment_id: str, *, attempt: int = 1, ) -> bytes: """Сообщение задания по `contracts/jobs.schema.json`.""" return json.dumps( { "schema_version": 1, "job_id": job_id, "type": "attachment.process.v1", "attachment_id": attachment_id, "pipeline_version": 1, "attempt": attempt, "created_at": "2026-10-31T12:00:00.000000Z", }, ).encode() def wait_job( stand: Stand, attachment_id: str, *, state: str, attempt: int, timeout: float = 20, ) -> dict[str, Any]: """Задание в этом состоянии с этим или большим числом попыток.""" deadline = time.monotonic() + timeout while True: job = job_row(stand, attachment_id) if job and job["state"] == state and job["attempt"] >= attempt: return job assert time.monotonic() < deadline, ( f"job of {attachment_id} is not {state} with attempt >= " f"{attempt}: {job}" ) time.sleep(0.2) def assert_variants( api: Api, stand: Stand, actor: Actor, meta: dict[str, Any], ) -> dict[str, dict[str, Any]]: """По одному варианту каждого имени в метаданных и по объекту на каждый, с его SHA-256; настоящие размеры и тип. Возвращает варианты по имени.""" assert meta["status"] == "ready", meta variants = {item["name"]: item for item in meta["variants"]} assert len(meta["variants"]) == len(VARIANT_NAMES), meta["variants"] assert set(variants) == VARIANT_NAMES, variants assert stand("variant-object-count", meta["id"]) == str(len(VARIANT_NAMES)) for name, variant in variants.items(): data = assert_variant_content( api.attachment_content(actor, meta["id"], variant=name), variant, ) assert_variant_image(data, variant) return variants def assert_not_ready_content( api: Api, actor: Actor, attachment_id: str, ) -> None: """Любое скачивание вложения не в `ready` — 409.""" for variant in ALL_VARIANTS: assert_error( api.attachment_content(actor, attachment_id, variant=variant), 409, ) def staging_key(attachment_id: str) -> str: return f"{FAULT_DATABASE}/staging/{attachment_id}/original" def wait_staging_gone(stand: Stand, attachment_id: str) -> None: """Объект staging и ключ в метаданных убраны: не позже следующего прохода уборщика с запасом.""" deadline = time.monotonic() + CLEANER_INTERVAL_S + CLEANUP_SLACK_S while True: row = json.loads(stand("attachment-row", attachment_id)) if row["staging_key"] is None and attachment_id not in staged_ids( stand, ): return assert time.monotonic() < deadline, (attachment_id, row) time.sleep(0.3) @pytest.mark.spec("attachment-processing", "Принятое вложение в очереди") def test_queued_while_worker_stopped( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) time.sleep(3) meta = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert meta == accepted api.wait_attachment(alice, accepted["id"]) @pytest.mark.spec("attachment-processing", "Обработка видна в API") def test_processing_visible( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() filename = fault_filename() stand("fail-attachments", "ready-stall", filename) try: accepted = api.upload(alice, chat["id"], data, filename=filename) api.wait_attachment( alice, accepted["id"], status="processing", timeout=JOB_WAIT_S, ) assert_error(api.attachment_content(alice, accepted["id"]), 409) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) finally: stand("clear-fail-attachments") assert_content( api.attachment_content(alice, accepted["id"]), data, "image/png", ) @pytest.mark.spec("attachment-processing", "Готовое вложение не откатывается") @pytest.mark.spec("attachment-processing", "Повторная доставка готового") def test_redelivery_of_ready_is_noop( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: meta = api.upload_ready(alice, chat["id"], images.sample_png()) job = job_row(stand, meta["id"]) assert job is not None wait_queue_empty(stand) seen = len(worker_records(stand)) stand("publish-job", stdin=job_body(job["id"], meta["id"])) wait_worker_record( stand, "Skipped job message", skip=seen, job_id=job["id"], reason="finished", ) wait_queue_empty(stand) assert ( assert_ok(api.attachment_meta(alice, meta["id"]), 200, "Attachment") == meta ) assert job_row(stand, meta["id"]) == job assert_variants(api, stand, alice, meta) @pytest.mark.spec("attachment-processing", "Что сохранено к ответу 202") def test_durable_state_at_202( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): with relay_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) attachment_id = accepted["id"] assert attachment_id in staged_ids(stand) assert attachment_id not in stored_ids(stand) assert attachment_rows(stand)[attachment_id] == "queued" job = job_row(stand, attachment_id) assert job is not None assert (job["state"], job["attempt"]) == ("queued", 0), job assert job["published_at"] is None, job assert queue_state(stand)["messages"] == 0 wait_queue_messages(stand, 1) published = job_row(stand, attachment_id) assert published is not None assert published["published_at"] is not None, published api.wait_attachment(alice, attachment_id) @pytest.mark.spec("attachment-processing", "Временная ошибка") @pytest.mark.spec("attachment-processing", "Повторы под одним заданием") def test_transient_error_retried( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Сначала S3 остановлен и возвращается за ~8 с: попыток больше одной. Затем S3 заморожен на первую попытку и размораживается в паузе между попытками: ровно две, задание то же.""" data = images.sample_png() with worker_stopped(stand): stopped = api.upload(alice, chat["id"], data) first = job_row(stand, stopped["id"]) with outage(stand, "stop", "s3"): stand("start", "worker") wait_job(stand, stopped["id"], state="queued", attempt=1) meta = api.wait_attachment(alice, stopped["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) job = job_row(stand, stopped["id"]) assert job is not None and first is not None assert job["id"] == first["id"], (job, first) assert job["attempt"] > 1, job assert job["state"] == "succeeded", job with worker_stopped(stand): paused = api.upload(alice, chat["id"], data) wait_queue_messages(stand, 1) first = job_row(stand, paused["id"]) with relay_stopped(stand), outage(stand, "pause", "s3"): stand("start", "worker") # срок вызова S3 5 с: первая попытка заканчивается отказом S3. # Срок следующей отодвинут, пока S3 размораживается: случайная # пауза могла бы её начать раньше, relay стоит до этого wait_job(stand, paused["id"], state="queued", attempt=1) stand("set-next-attempt", paused["id"], "6") meta = api.wait_attachment(alice, paused["id"], timeout=JOB_WAIT_S) job = job_row(stand, paused["id"]) assert job is not None and first is not None assert job["id"] == first["id"], (job, first) assert job["attempt"] == 2, job assert_variants(api, stand, alice, meta) @pytest.mark.spec("attachment-processing", "Сообщение по схеме") def test_job_message_matches_schema( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: schema = json.loads(JOBS_SCHEMA_PATH.read_text(encoding="utf-8")) filename = fault_filename() with worker_stopped(stand): accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) wait_queue_messages(stand, 1) peeked = json.loads(stand("peek-job")) text = peeked["body"] body = json.loads(text) errors = list( Draft202012Validator( schema, format_checker=FormatChecker(), ).iter_errors(body), ) assert not errors, [error.message for error in errors] datetime.fromisoformat(body["created_at"]) job = job_row(stand, accepted["id"]) assert job is not None assert body["attachment_id"] == accepted["id"] assert body["job_id"] == job["id"] assert body["attempt"] == 1 # ни имени файла, ни ключей, ни адресов, ни учётных данных lowered = text.lower() for word in ( filename.lower(), os.environ.get("FAULT_S3_BUCKET", "attachments").lower(), f"{FAULT_DATABASE}/", "staging", "original", "http", "password", "secret", ): assert word not in lowered, f"{word!r} in the job message" api.wait_attachment(alice, accepted["id"]) @pytest.mark.spec("message-broker", "Очередь и сообщения устойчивы") def test_queue_durable( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) state = wait_queue_messages(stand, 1) assert state["durable"] is True, state peeked = json.loads(stand("peek-job")) assert peeked["delivery_mode"] == 2, peeked # прочитанное без подтверждения вернулось в очередь state = queue_state(stand) assert (state["messages"], state["messages_ready"]) == (1, 1), state api.wait_attachment(alice, accepted["id"]) @pytest.mark.spec("attachment-processing", "Сообщение не по схеме") def test_malformed_message_dropped( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: marker = uuid.uuid4().hex wait_queue_empty(stand) seen = len(worker_records(stand)) stand( "publish-job", stdin=json.dumps({"schema_version": 1, "note": marker}).encode(), ) record = wait_worker_record( stand, "Dropped malformed job message", skip=seen, ) assert record["level"] == "error", record # содержимое сообщения в журнал не попало assert marker not in stand("worker-log") wait_queue_empty(stand) api.upload_ready(alice, chat["id"], images.sample_png()) @pytest.mark.spec("attachment-processing", "Неизвестное задание") def test_unknown_job_acked(stand: Stand) -> None: job_id, attachment_id = str(uuid.uuid4()), str(uuid.uuid4()) wait_queue_empty(stand) before = objects_and_rows(stand) seen = len(worker_records(stand)) stand("publish-job", stdin=job_body(job_id, attachment_id)) wait_worker_record( stand, "Skipped job message", skip=seen, job_id=job_id, reason="unknown", ) wait_queue_empty(stand) assert not new_ids(before[0], attachment_rows(stand)) assert not new_objects(before, stand) @pytest.mark.spec("attachment-processing", "Worker убит посреди обработки") def test_worker_killed_during_processing( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: filename = fault_filename() wait_queue_empty(stand) stand("fail-attachments", "ready-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) api.wait_attachment( alice, accepted["id"], status="processing", timeout=JOB_WAIT_S, ) stand("kill", "worker") finally: stand("clear-fail-attachments") stand("start", "worker") # сообщение вернулось, но аренда жива: новую попытку начнёт её истечение meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 2, job wait_queue_empty(stand) @pytest.mark.spec("attachment-processing", "Дубль во время обработки") def test_duplicate_during_processing( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Две ветки дубля. S3 заморожен короче срока вызова, попытка ждёт его без блокировок БД: дубли видят живую аренду («busy») и подтверждаются, а после разморозки та же попытка доходит до `ready`. Фиксация ждёт в триггере и держит строку задания: дубль дожидается её и видит итог («finished»).""" with worker_stopped(stand): live = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) job = job_row(stand, live["id"]) assert job is not None stand("publish-job", stdin=job_body(job["id"], live["id"])) stand("publish-job", stdin=job_body(job["id"], live["id"])) wait_queue_messages(stand, 3) seen = len(worker_records(stand)) with outage(stand, "pause", "s3"): stand("start", "worker") deadline = time.monotonic() + S3_TIMEOUT_S while True: busy = [ item for item in worker_records(stand)[seen:] if item.get("event") == "Skipped job message" and item.get("job_id") == job["id"] and item.get("reason") == "busy" ] if len(busy) == 2: break assert time.monotonic() < deadline, busy time.sleep(0.1) meta = api.wait_attachment(alice, live["id"], timeout=JOB_WAIT_S) wait_queue_empty(stand) assert_variants(api, stand, alice, meta) done = job_row(stand, live["id"]) assert done is not None assert done["attempt"] == 1, done filename = fault_filename() seen = len(worker_records(stand)) stand("fail-attachments", "ready-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) api.wait_attachment( alice, accepted["id"], status="processing", timeout=JOB_WAIT_S, ) job = job_row(stand, accepted["id"]) assert job is not None stand("publish-job", stdin=job_body(job["id"], accepted["id"])) wait_worker_record( stand, "Skipped job message", skip=seen, job_id=job["id"], reason="finished", ) meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) finally: stand("clear-fail-attachments") wait_queue_empty(stand) assert_variants(api, stand, alice, meta) done = job_row(stand, accepted["id"]) assert done is not None assert done["attempt"] == 1, done @pytest.mark.spec("attachment-processing", "Оригинал неизменен") def test_original_published_unchanged( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.png_noise(64, 48, seed=23) meta = api.upload_ready(alice, chat["id"], data) assert_content( api.attachment_content(alice, meta["id"]), data, "image/png", ) assert meta["sha256"] == hashlib.sha256(data).hexdigest() assert meta["id"] in stored_ids(stand) wait_staging_gone(stand, meta["id"]) @pytest.mark.spec("attachment-processing", "Испорченный staging") def test_staging_corrupted( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Те же размер и другие байты — и другой размер: обе ветки сверки.""" data = images.png_noise(64, 48, seed=29) with worker_stopped(stand): same_size = api.upload(alice, chat["id"], data) other_size = api.upload(alice, chat["id"], data) put_object( stand, staging_key(same_size["id"]), same_size_bytes(data, seed=29), ) put_object(stand, staging_key(other_size["id"]), data + b"\0") for accepted in (same_size, other_size): meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "processing_failed", meta assert meta["variants"] == [], meta job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 1, job assert job["last_error_code"] == "source_corrupted", job assert accepted["id"] not in stored_ids(stand) @pytest.mark.spec("attachment-processing", "Нет объекта staging") def test_staging_missing( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) stand("delete-object", staging_key(accepted["id"])) meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "processing_failed", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 1, job assert job["last_error_code"] == "source_missing", job @pytest.mark.spec("attachment-processing", "Изображение в лимите пикселей") def test_max_pixels_fit_worker_memory( base_url: str, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Столько PNG RGBA 8000 × 5000, сколько заданий worker обрабатывает одновременно: они берутся разом, и контейнер не завершён из-за памяти.""" data = images.png_rgba_large() before = json.loads(stand("worker-state")) # срок клиента: тело на 40 млн пикселей проверяется дольше 5 с по умолчанию with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow: with worker_stopped(stand): accepted = [ slow.upload(alice, chat["id"], data) for _ in range(WORKER_CONCURRENCY) ] sizes = { "thumbnail": (256, 160), "crop": (128, 128), "optimized": (1280, 800), } for item in accepted: meta = slow.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) variants = assert_variants(slow, stand, alice, meta) for name, size in sizes.items(): variant = variants[name] assert (variant["width"], variant["height"]) == size, variant state = json.loads(stand("worker-state")) assert state["running"] is True, state assert state["oom_killed"] is False, state assert state["restart_count"] == before["restart_count"], state @pytest.mark.spec("attachment-processing", "Повтор после записи объектов") @pytest.mark.spec("attachments", "Фиксация ready не удалась") def test_ready_failure_retried( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Объекты записаны, фиксация падает отказом БД (08006): сообщение уходит в очередь; пока триггер стоит, вложение не готово. Аренда жива: следующую попытку начинает её истечение, а триггер снимается раньше.""" filename = fault_filename() wait_queue_empty(stand) seen = len(worker_records(stand)) stand("fail-attachments", "ready-fail", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) job = job_row(stand, accepted["id"]) assert job is not None wait_worker_record( stand, "Job waits for database", skip=seen, job_id=job["id"], ) assert stand("variant-object-count", accepted["id"]) == str( len(VARIANT_NAMES), ) assert accepted["id"] in stored_ids(stand) meta = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert meta["status"] in ("queued", "processing"), meta assert_not_ready_content(api, alice, accepted["id"]) finally: stand("clear-fail-attachments") meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) done = job_row(stand, accepted["id"]) assert done is not None assert done["id"] == job["id"], done assert done["attempt"] >= 2, done @pytest.mark.spec("attachment-processing", "Состояние между попытками") def test_queued_between_attempts( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) with outage(stand, "stop", "s3"): stand("start", "worker") job = wait_job(stand, accepted["id"], state="queued", attempt=1) assert job["last_error_code"] == "storage_unavailable", job assert job["next_attempt_at"] is not None, job meta = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert meta["status"] == "queued", meta assert meta["error_code"] is None, meta api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("attachment-processing", "Попытки исчерпаны") def test_attempts_exhausted( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) with outage(stand, "stop", "s3"): stand("start", "worker") meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "retry_exhausted", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == WORKER_MAX_ATTEMPTS, job assert job["state"] == "failed", job wait_queue_empty(stand) # S3 вернулся: итог окончателен time.sleep(WORKER_RETRY_BASE_S) after = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert after == meta assert job_row(stand, accepted["id"]) == job @pytest.mark.spec( "attachment-processing", "PostgreSQL недоступен при обработке", ) def test_database_outage_during_processing( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = json.loads(stand("worker-state")) with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) job = job_row(stand, accepted["id"]) assert job is not None seen = len(worker_records(stand)) with outage(stand, "stop", "postgres"): stand("start", "worker") # журнал называет отказ, задание ждёт в очереди wait_worker_record(stand, "Job waits for database", skip=seen) meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) wait_queue_empty(stand) done = job_row(stand, accepted["id"]) assert done is not None # попытки на отказе БД не считаются assert done["attempt"] == 1, done state = json.loads(stand("worker-state")) assert state["running"] is True, state assert state["restart_count"] == before["restart_count"], state @pytest.mark.spec("attachment-processing", "Изображение не декодируется") def test_undecodable_becomes_failed( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: accepted = api.upload( alice, chat["id"], images.jpeg_precision_12(), content_type="image/jpeg", filename="image.jpg", ) meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "invalid_image", meta assert meta["variants"] == [], meta job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 1, job assert job["last_error_code"] == "undecodable", job assert_error(api.attachment_content(alice, accepted["id"]), 409) @pytest.mark.spec("attachment-processing", "Worker недоступен") def test_worker_down_api_serves( api: Api, stand: Stand, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) assert accepted["status"] == "queued" message = api.send_message(alice, chat["id"], "текст без worker") history = api.all_messages(bob, chat["id"]) assert [item["id"] for item in history] == [message["id"]] assert_ok(api.request("GET", "/health/ready"), 200, "Health") api.wait_attachment(alice, accepted["id"]) @pytest.mark.spec("attachment-processing", "Ограниченная конкуренция") def test_concurrency_bounded( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Фиксация каждого задания ждёт 4 с в триггере: заданий больше, чем мест, и видно, сколько worker держит разом.""" filename = fault_filename() total = WORKER_CONCURRENCY + 3 stand("fail-attachments", "ready-stall", filename) try: with worker_stopped(stand): ids = [ api.upload( alice, chat["id"], images.sample_png(), filename=filename, )["id"] for _ in range(total) ] api.wait_attachment( alice, ids[0], status="processing", timeout=JOB_WAIT_S, ) most = 0 for _ in range(4): rows = attachment_rows(stand) statuses = [rows[item] for item in ids] busy = statuses.count("processing") most = max(most, busy) assert busy <= WORKER_CONCURRENCY, statuses assert ( statuses.count("queued") + busy + statuses.count( "ready", ) == total ), statuses state = queue_state(stand) assert state["messages_unacknowledged"] <= WORKER_CONCURRENCY, ( state ) time.sleep(0.3) assert most == WORKER_CONCURRENCY, most for item in ids: api.wait_attachment(alice, item, timeout=JOB_WAIT_S) finally: stand("clear-fail-attachments") wait_queue_empty(stand) @pytest.mark.spec("attachment-processing", "Реплика API убита после ответа") def test_api_killed_after_202( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: accepted = api.upload(alice, chat["id"], images.sample_png()) stand("kill-api") try: # реплики нет, а работа идёт: итог — от worker wait_job( stand, accepted["id"], state="succeeded", attempt=1, timeout=JOB_WAIT_S, ) finally: stand("start-api") meta = api.wait_attachment(alice, accepted["id"]) assert_variants(api, stand, alice, meta) @pytest.mark.spec("attachment-processing", "Рестарт брокера с volume") def test_broker_restart_keeps_job( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) with outage(stand, "stop", "rabbitmq"): pass assert queue_state(stand)["messages"] == 1 api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("attachment-processing", "Принудительное завершение брокера") @pytest.mark.spec("message-broker", "Подтверждённые сообщения после KILL") def test_broker_kill_keeps_confirmed( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: confirmed = 5 with worker_stopped(stand): accepted = [ api.upload(alice, chat["id"], images.sample_png()) for _ in range(confirmed) ] wait_queue_messages(stand, confirmed) with outage(stand, "kill", "rabbitmq"): pass assert queue_state(stand)["messages"] == confirmed for item in accepted: api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("attachment-processing", "Сценарий защиты") def test_defense_scenario( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() with worker_stopped(stand): accepted = api.upload(alice, chat["id"], data) attachment_id = accepted["id"] message = api.send_message( alice, chat["id"], "", attachment_ids=[attachment_id], ) assert_error(api.attachment_content(alice, attachment_id), 409) text = api.send_message(alice, chat["id"], "текст при очереди") wait_queue_messages(stand, 1) with outage(stand, "stop", "rabbitmq"): stand("kill-api") stand("start-api") meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S) assert meta["id"] == attachment_id assert_content( api.attachment_content(alice, attachment_id), data, "image/png", ) assert_variants(api, stand, alice, meta) history = { item["id"]: item for item in api.all_messages(alice, chat["id"]) } assert history[message["id"]]["attachment_ids"] == [attachment_id] assert text["id"] in history @pytest.mark.spec("message-broker", "Брокер остановлен при загрузке") @pytest.mark.spec("message-broker", "Брокер вернулся") @pytest.mark.spec("attachments", "Брокер не подтвердил задание") @pytest.mark.spec("job-outbox", "Загрузка при остановленном брокере") def test_upload_with_broker_stopped( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() ready = api.upload_ready(alice, chat["id"], data) with outage(stand, "stop", "rabbitmq"): resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data)) accepted = assert_accepted(resp) assert elapsed < ACCEPT_MAX_S, elapsed meta = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert meta["status"] == "queued", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["published_at"] is None, job assert accepted["id"] in staged_ids(stand) # обязательный путь без брокера работает api.send_message(alice, chat["id"], "текст без брокера") assert ( assert_ok( api.attachment_meta(alice, ready["id"]), 200, "Attachment", ) == ready ) assert_content( api.attachment_content(alice, ready["id"]), data, "image/png", ) # relay и worker не перезапускались: принятое без брокера доходит до ready, # и следующая загрузка тоже done = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, done) api.upload_ready(alice, chat["id"], data) @pytest.mark.spec("message-broker", "Брокер завис") @pytest.mark.spec("job-outbox", "Загрузка при замороженном брокере") def test_upload_with_broker_paused( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Соединение издателя живо: публикация relay уходит в заморозку и кончается сроком, исход неизвестен. Приём этого не видит, а дубль после разморозки worker подтверждает без эффекта.""" data = images.sample_png() api.upload_ready(alice, chat["id"], data) wait_queue_empty(stand) with outage(stand, "pause", "rabbitmq"): resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data)) accepted = assert_accepted(resp) assert elapsed < ACCEPT_MAX_S, elapsed meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) wait_queue_empty(stand) job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 1, job @pytest.mark.spec("message-broker", "Очередь удалена") def test_queue_deleted( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() # соединение издателя живо: его канал очередь больше не найдёт, а новое # соединение объявит её заново api.upload_ready(alice, chat["id"], data) with worker_stopped(stand): stand("delete-queue") assert json.loads(stand("queue-state")) is None accepted = [api.upload(alice, chat["id"], data) for _ in range(2)] state = wait_queue_messages(stand, 2) assert state["durable"] is True, state for item in accepted: api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("message-broker", "Пересоздание контейнера") def test_broker_recreated( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: confirmed = 2 with worker_stopped(stand): accepted = [ api.upload(alice, chat["id"], images.sample_png()) for _ in range(confirmed) ] wait_queue_messages(stand, confirmed) stand("recreate", "rabbitmq") state = queue_state(stand) assert (state["durable"], state["messages"]) == (True, confirmed), ( state ) for item in accepted: api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("message-broker", "Старт без брокера") def test_start_without_broker( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with ( worker_stopped(stand), relay_stopped(stand), outage(stand, "stop", "rabbitmq"), ): seen = len(worker_records(stand)) seen_relay = len(relay_records(stand)) stand("kill-api") stand("start-api") stand("start", "worker") stand("start", "relay") # реплика готова и принимает файл assert_ok(api.request("GET", "/health/ready"), 200, "Health") api.send_message(alice, chat["id"], "текст при старте без брокера") accepted = api.upload(alice, chat["id"], images.sample_png()) # worker и relay называют отказ и не завершаются wait_worker_record(stand, "Broker is unavailable", skip=seen) wait_relay_record(stand, "Broker is unavailable", skip=seen_relay) state = json.loads(stand("worker-state")) assert state["running"] is True, state state = json.loads(stand("relay-state")) assert state["running"] is True, state # брокер запущен, ни реплика, ни worker не перезапускались api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("message-broker", "Рестарт брокера при работающем worker") def test_worker_survives_broker_restart( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = json.loads(stand("worker-state")) assert before["running"] is True, before with outage(stand, "stop", "rabbitmq"): pass api.upload_ready(alice, chat["id"], images.sample_png()) state = json.loads(stand("worker-state")) assert state["running"] is True, state assert state["restart_count"] == before["restart_count"], state @pytest.mark.spec("attachments", "Скачивание до готовности") def test_content_before_ready( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Каждое скачивание вложения в `queued`, `processing` и `failed` — 409.""" filename = fault_filename() stand("fail-attachments", "ready-stall", filename) try: with worker_stopped(stand): accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) assert_not_ready_content(api, alice, accepted["id"]) api.wait_attachment( alice, accepted["id"], status="processing", timeout=JOB_WAIT_S, ) assert_not_ready_content(api, alice, accepted["id"]) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) finally: stand("clear-fail-attachments") failed = api.upload( alice, chat["id"], images.jpeg_precision_12(), content_type="image/jpeg", filename="image.jpg", ) api.wait_attachment( alice, failed["id"], status="failed", timeout=JOB_WAIT_S, ) assert_not_ready_content(api, alice, failed["id"]) @pytest.mark.spec("messages", "Вложение в обработке") def test_message_with_queued_attachment( api: Api, stand: Stand, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) message = api.send_message( alice, chat["id"], "с вложением в очереди", attachment_ids=[accepted["id"]], ) meta = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert meta["status"] == "queued", meta assert api.all_messages(bob, chat["id"]) == [message] api.wait_attachment(alice, accepted["id"]) @pytest.mark.spec("messages", "Вложение стало failed после отправки") def test_attachment_failed_after_message( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload( alice, chat["id"], images.jpeg_precision_12(), content_type="image/jpeg", filename="image.jpg", ) message = api.send_message( alice, chat["id"], "", attachment_ids=[accepted["id"]], ) meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "invalid_image", meta assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("idempotency", "Повтор после failed вложения") def test_replay_after_attachment_failed( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: key = new_key() with worker_stopped(stand): accepted = api.upload( alice, chat["id"], images.jpeg_precision_12(), content_type="image/jpeg", filename="image.jpg", ) body = {"text": "повтор", "attachment_ids": [accepted["id"]]} message = assert_ok( api.post_message(alice, chat["id"], key=key, json=body), 201, "Message", ) api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) replay = assert_ok( api.post_message(alice, chat["id"], key=key, json=body), 201, "Message", ) assert replay == message assert_error(api.post_message(alice, chat["id"], json=body), 409) assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("health", "Брокер и worker недоступны") @pytest.mark.parametrize("action", ["stop", "pause"]) def test_readiness_without_broker_and_worker( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], action: str, ) -> None: with ( worker_stopped(stand), relay_stopped(stand), outage(stand, action, "rabbitmq"), ): for path in ("/health/ready", "/health/live"): resp, took = timed(lambda path=path: api.request("GET", path)) assert assert_ok(resp, 200, "Health") == {"status": "ok"} assert took < BROKER_HEALTH_S, f"{path} took {took:.2f}s" # загрузка от них не зависит accepted = api.upload(alice, chat["id"], images.sample_png()) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) def test_worker_stop_during_processing( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Спек `graceful-shutdown`, «docker stop во время обработки» (вне сверки покрытия — сценарий в докстринге): фиксация ждёт 4 с, а срок остановки worker стенда 2 с. `stop` быстрее 10 с, код 0, сообщение снова в очереди, а после запуска worker вложение становится `ready`.""" filename = fault_filename() wait_queue_empty(stand) stand("fail-attachments", "ready-stall", filename) stopped = False try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) api.wait_attachment( alice, accepted["id"], status="processing", timeout=JOB_WAIT_S, ) seen = len(worker_records(stand)) started = time.monotonic() stand("stop", "worker") stopped = True assert time.monotonic() - started < WORKER_STOP_MAX_S state = json.loads(stand("worker-state")) assert (state["running"], state["exit_code"]) == (False, 0), state assert queue_state(stand)["messages"] == 1 records = worker_records(stand)[seen:] assert any(item.get("event") == "Stopped worker" for item in records) finally: stand("clear-fail-attachments") if not stopped: stand("stop", "worker") stand("start", "worker") meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) wait_queue_empty(stand) def test_worker_stop_during_render( base_url: str, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Спек `graceful-shutdown`, «docker stop во время декодирования» (вне сверки покрытия — сценарий в докстринге): дорогое изображение декодируется дольше срока остановки worker стенда (2 с). `stop` быстрее 10 с, код 0, процессов обработки нет, а задание получает следующую попытку по аренде.""" wait_queue_empty(stand) with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow: accepted = slow.upload( alice, chat["id"], images.jpeg_scan_bomb(), content_type="image/jpeg", filename="bomb.jpg", ) wait_render_process(stand) started = time.monotonic() stand("stop", "worker") try: assert time.monotonic() - started < WORKER_STOP_MAX_S state = json.loads(stand("worker-state")) assert (state["running"], state["exit_code"]) == (False, 0), state finally: stand("start", "worker") # контейнер снова запущен: процессов обработки прежней попытки нет, # попытка 2 дойдёт до лимита CPU meta = slow.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 2, job assert meta["error_code"] == "invalid_image", meta assert job["last_error_code"] == "limit_cpu", job def wait_render_process(stand: Stand, *args: str, timeout: float = 15) -> None: """В контейнере worker идёт процесс обработки.""" deadline = time.monotonic() + timeout while stand("render-count", *args) == "0": assert time.monotonic() < deadline, "no render process in time" time.sleep(0.05) def wait_variant_objects(stand: Stand, attachment_id: str) -> None: """Объекты всех вариантов записаны: пока фиксация ждёт в триггере.""" deadline = time.monotonic() + STALL_S while stand("variant-object-count", attachment_id) != str( len(VARIANT_NAMES), ): assert time.monotonic() < deadline, "no variant objects in time" time.sleep(0.1) @contextmanager def current_work( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> Iterator[dict[str, dict[str, Any]]]: """Worker убит посреди задания и лежит, relay остановлен: иначе он вернул бы P по истечении аренды. Вложения F в `failed` (конечный объект у него есть) — старше grace, но моложе срока хранения staging, P в `processing` (объекты записаны, фиксация не дошла) и Q в `queued` — старше grace и срока хранения по строке. Выход — worker и relay запущены, P и Q становятся `ready`.""" wait_queue_empty(stand) failed = api.upload( alice, chat["id"], images.jpeg_precision_12(), content_type="image/jpeg", filename="image.jpg", ) api.wait_attachment( alice, failed["id"], status="failed", timeout=JOB_WAIT_S, ) filename = fault_filename() stand("fail-attachments", "ready-stall", filename) work: dict[str, dict[str, Any]] = {"failed": failed} try: processing = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) api.wait_attachment( alice, processing["id"], status="processing", timeout=JOB_WAIT_S, ) work["processing"] = processing # конечный объект записан до фиксации; ждём и варианты wait_variant_objects(stand, processing["id"]) stand("kill", "worker") stand("stop", "relay") stand("clear-fail-attachments") work["queued"] = api.upload(alice, chat["id"], images.sample_png()) for name, item in work.items(): age = FAILED_AGED_S if name == "failed" else AGED_S stand("age-row", item["id"], str(age)) yield work finally: stand("clear-fail-attachments") stand("start", "relay") stand("start", "worker") for name in ("processing", "queued"): api.wait_attachment(alice, work[name]["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("storage-consistency", "Текущая работа не тронута") def test_cleaner_keeps_current_work( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with current_work(api, stand, alice, chat) as work: # проходов уборщика за это время — несколько time.sleep(3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S) rows = attachment_rows(stand) staged = staged_ids(stand) for name, item in work.items(): assert rows.get(item["id"]) == name, (name, rows.get(item["id"])) assert item["id"] in staged, name # --------------------------------------------------------------------------- # Сверка bucket и БД (change track2-reconciler, design, решение 11). Hook # вносит расхождения с обеих сторон и запускает разовую сверку; плановая # сверка уборщика стенда выключена. Записи сверки фильтруются по своим id и # ключам: остатки прежних тестов (строки pending) могут попасть в отчёт, и # код выхода сверяется с записями того же прохода, а не с числом. Объекты # состарить нельзя (LastModified ставит S3): тесты объектов без строки ждут # G + 1 с, строки состаривает hook (age-row) # --------------------------------------------------------------------------- # на сколько hook состаривает строку: заведомо больше grace AGED_S = CLEANER_GRACE_S + 60 # строка failed: старше grace, а срок хранения staging (60 с) тест не перейдёт FAILED_AGED_S = CLEANER_GRACE_S + 5 # объекты без строки старше grace: grace и секунда точности LastModified OBJECT_AGE_WAIT_S = CLEANER_GRACE_S + 1 # grace разового прохода, когда он сам дольше grace стенда; меньше AGED_S FRESH_GRACE_S = int(AGED_S) - 20 FOUND = "Found storage discrepancy" FIXED = "Fixed storage discrepancy" FINISHED = "Finished reconcile pass" LIMIT_EXCEEDED = "Reconcile changes exceed limit" # сверка под нагрузкой: потоки загрузок, период сверки, пауза потока LOAD_THREADS = 4 LOAD_EXTRA_S = 15 LOAD_RECONCILE_PERIOD_S = 3 LOAD_PAUSE_S = 0.1 # плановая сверка отдельного уборщика: столько секунд он работает SCHEDULED_RUN_S = 60 @dataclass(frozen=True) class Reconcile: """Один разовый проход сверки: записи журнала процесса и код выхода.""" records: list[dict[str, Any]] exit_code: int def events(self, event: str) -> list[dict[str, Any]]: return [item for item in self.records if item.get("event") == event] @property def finished(self) -> dict[str, Any]: (record,) = self.events(FINISHED) return record def about( self, event: str, *, ids: Collection[str] = (), keys: Collection[str] = (), ) -> list[dict[str, Any]]: """Записи `event` о своих `id` вложений и ключах объектов.""" return [ item for item in self.events(event) if item.get("attachment_id") in ids or item.get("object_key") in keys ] def kinds( self, event: str, *, ids: Collection[str] = (), keys: Collection[str] = (), ) -> dict[str, str]: """`id` или ключ → класс расхождения в записях `event`.""" found: dict[str, str] = {} for item in self.about(event, ids=ids, keys=keys): name = ( item["attachment_id"] if item.get("attachment_id") in ids else item["object_key"] ) found[name] = item["kind"] return found def assert_code_matches(self) -> None: """Код выхода соответствует записям прохода: 3 — остались расхождения (запись `Found`) или отказ по пределу, иначе 0.""" finished = self.finished assert finished["interrupted_by"] is None, finished expected = 3 if self.events(FOUND) or finished["refused"] else 0 assert self.exit_code == expected, (self.exit_code, finished) def run_reconcile( stand: Stand, mode: str, limit: int | None = None, *, options: Sequence[str] = (), ) -> Reconcile: """Разовая сверка hook: `dry-run` или `apply`, с пределом изменений; `options` — стратегия, глубина и предел скорости (change `track2-reconcile-cost`) как аргументы команды.""" args = [mode] if limit is None else [mode, str(limit)] lines = stand("reconcile", *args, *options).splitlines() assert lines and lines[-1].startswith("exit_code="), lines[-5:] records = [] for line in lines[:-1]: if line.startswith("{"): records.append(json.loads(line)) return Reconcile(records=records, exit_code=int(lines[-1].split("=")[1])) def object_key(attachment_id: str) -> str: return f"{FAULT_DATABASE}/attachments/{attachment_id}/original" def variant_key(attachment_id: str, name: str = "thumbnail") -> str: return f"{FAULT_DATABASE}/attachments/{attachment_id}/v1-{name}" def lost_ids(stand: Stand) -> set[str]: return set(stand("lost-ids").splitlines()) def put_object(stand: Stand, key: str, data: bytes) -> None: stand("put-object", key, stdin=data) def lose_original(stand: Stand, attachment: dict[str, Any]) -> None: """Готовое вложение старше grace без объекта: строка состарена, объект удалён мимо API.""" stand("age-row", attachment["id"], str(AGED_S)) stand("delete-object", object_key(attachment["id"])) def assert_lost_content( api: Api, actor: Actor, attachment: dict[str, Any], *, aged: bool = False, ) -> None: """Помеченное вложение: байты — 404, метаданные — как при загрузке. `aged` — строку состарил hook (`age-row`): её `created_at` и `updated_at` сдвинуты им, а не сверкой, и сравниваются остальные поля. """ assert_error(api.attachment_content(actor, attachment["id"]), 404) meta = assert_ok( api.attachment_meta(actor, attachment["id"]), 200, "Attachment", ) if aged: times = ("created_at", "updated_at") meta = {key: value for key, value in meta.items() if key not in times} attachment = { key: value for key, value in attachment.items() if key not in times } assert meta == attachment @pytest.mark.spec("storage-consistency", "Расхождения с обеих сторон найдены") @pytest.mark.spec("storage-consistency", "Исправление расхождений") @pytest.mark.spec("storage-consistency", "Повторная сверка") @pytest.mark.spec("attachments", "Потерянный оригинал") def test_reconcile_finds_and_fixes( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() # уборщик стенда стоит: зависшую загрузку P должна убрать сверка with outage(stand, "stop", "cleaner"): a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3)) api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]]) orphan = str(uuid.uuid4()) keys = { "o": object_key(orphan), "b": object_key(b["id"]), "bv": variant_key(b["id"]), } put_object(stand, keys["o"], data) stand("delete-object", object_key(a["id"])) stand("delete-row", b["id"]) stand("set-size", c["id"], str(len(data) + 1)) filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "commit", filename) try: resp = upload_raw(api, alice, chat, data, filename=filename) assert_error(resp, 503) finally: stand("clear-fail-attachments") (pending,) = new_ids(before, attachment_rows(stand)) assert pending in staged_ids(stand) # последний объект записан только что: после G + 1 всё старше grace time.sleep(OBJECT_AGE_WAIT_S) ids = {a["id"], c["id"], pending} snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand)) dry = run_reconcile(stand, "dry-run") assert dry.kinds(FOUND, ids=ids, keys=keys.values()) == { keys["o"]: "orphan_object", keys["b"]: "orphan_object", keys["bv"]: "orphan_object", a["id"]: "missing_object", c["id"]: "size_mismatch", pending: "stale_pending", } for record in dry.about(FOUND, ids=ids, keys=keys.values()): assert record["age_seconds"] >= CLEANER_GRACE_S, record assert dry.exit_code == 3 # dry-run ничего не меняет: ни строк, ни объектов, ни пометок after = (attachment_rows(stand), object_ids(stand), lost_ids(stand)) assert after == snapshot assert_error(api.attachment_content(alice, a["id"]), 500) assert_ok(api.attachment_meta(alice, a["id"]), 200, "Attachment") fixed = run_reconcile(stand, "apply") actions = { record.get("attachment_id") if record.get("attachment_id") in ids else record["object_key"]: record["action"] for record in fixed.about(FIXED, ids=ids, keys=keys.values()) } assert actions == { keys["o"]: "deleted_object", keys["b"]: "deleted_object", keys["bv"]: "deleted_object", a["id"]: "marked_lost", c["id"]: "marked_lost", pending: "removed_pending", } assert not fixed.about(FOUND, ids=ids, keys=keys.values()) fixed.assert_code_matches() rows, stored = attachment_rows(stand), stored_ids(stand) assert not {orphan, b["id"], pending} & stored assert stand("variant-object-count", b["id"]) == "0" assert pending not in staged_ids(stand) assert pending not in rows assert {a["id"], c["id"]} <= lost_ids(stand) assert rows[a["id"]] == rows[c["id"]] == "ready" assert_lost_content(api, alice, a) # у C размер в метаданных изменил hook (set-size), а не сверка assert_lost_content(api, alice, c | {"size_bytes": len(data) + 1}) again = run_reconcile(stand, "dry-run") assert not again.about(FOUND, ids=ids, keys=keys.values()) assert not again.about(FIXED, ids=ids, keys=keys.values()) again.assert_code_matches() @pytest.mark.spec("storage-consistency", "Свежие не тронуты") @pytest.mark.spec("storage-consistency", "Используемое вложение не удаляется") @pytest.mark.spec("storage-consistency", "Чужой объект не удаляется") def test_reconcile_keeps_fresh_used_and_unknown( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() used = api.upload_ready(alice, chat["id"], data) message = api.send_message( alice, chat["id"], "", attachment_ids=[used["id"]], ) lose_original(stand, used) # ключ, которого API не пишет: под префиксом вложений, но не /original unknown = f"{FAULT_DATABASE}/attachments/x-{uuid.uuid4()}" put_object(stand, unknown, data) time.sleep(OBJECT_AGE_WAIT_S) # свежие: готовое вложение без объекта и объект без строки fresh = api.upload_ready(alice, chat["id"], data) stand("delete-object", object_key(fresh["id"])) fresh_object = str(uuid.uuid4()) put_object(stand, object_key(fresh_object), data) ready_before = ready_ids(attachment_rows(stand)) try: run = run_reconcile(stand, "apply") keys = {unknown, object_key(fresh_object)} ids = {used["id"], fresh["id"], fresh_object} assert run.kinds(FOUND, ids=ids, keys=keys) == { unknown: "unknown_object", } assert run.kinds(FIXED, ids=ids, keys=keys) == { used["id"]: "missing_object", } assert run.exit_code == 3 assert run.finished["fresh_objects"] >= 1, run.finished assert run.finished["fresh_rows"] >= 1, run.finished # свежий объект на месте, свежее вложение не помечено assert fresh_object in stored_ids(stand) lost = lost_ids(stand) assert fresh["id"] not in lost # используемое: помечено, метаданные и ссылка в истории на месте assert used["id"] in lost assert_lost_content(api, alice, used, aged=True) history = api.all_messages(alice, chat["id"]) (sent,) = (item for item in history if item["id"] == message["id"]) assert sent["attachment_ids"] == [used["id"]] assert ready_before <= ready_ids(attachment_rows(stand)) # чужой объект не тронут: следующий проход снова его видит again = run_reconcile(stand, "dry-run") assert again.kinds(FOUND, keys={unknown}) == { unknown: "unknown_object", } finally: # стенд — без чужого и лишнего объекта, у свежего — его байты stand("delete-object", unknown) stand("delete-object", object_key(fresh_object)) put_object(stand, object_key(fresh["id"]), data) @pytest.mark.spec("storage-consistency", "Оригинал вернулся") def test_reconcile_restored_original( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.png_noise(64, 48, seed=7) restored = api.upload_ready(alice, chat["id"], data) lose_original(stand, restored) marked = run_reconcile(stand, "apply") assert marked.kinds(FIXED, ids={restored["id"]}) == { restored["id"]: "missing_object", } assert restored["id"] in lost_ids(stand) assert_lost_content(api, alice, restored, aged=True) # те же байты снова под ключом вложения put_object(stand, object_key(restored["id"]), data) cleared = run_reconcile(stand, "apply") (record,) = cleared.about(FIXED, ids={restored["id"]}) assert (record["kind"], record["action"]) == ( "restored_object", "cleared_lost", ) assert restored["id"] not in lost_ids(stand) assert_content( api.attachment_content(alice, restored["id"]), data, "image/png", ) again = run_reconcile(stand, "dry-run") assert not again.about(FOUND, ids={restored["id"]}) @pytest.mark.spec("storage-consistency", "Предел изменений") def test_reconcile_change_limit( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: limit = 2 batch = [ api.upload_ready(alice, chat["id"], images.sample_png()) for _ in range(3) ] ids = {item["id"] for item in batch} for item in batch: lose_original(stand, item) refused = run_reconcile(stand, "apply", limit) assert refused.exit_code == 3 (error,) = refused.events(LIMIT_EXCEEDED) assert error["level"] == "error" assert error["changes"] >= len(batch), error assert error["max_changes"] == limit assert refused.finished["refused"] is True assert refused.finished["level"] == "warning" assert not refused.about(FIXED, ids=ids) assert not ids & lost_ids(stand) assert all( record["kind"] == "missing_object" for record in refused.about(FOUND, ids=ids) ) # предел не меньше числа изменений: исправлено всё applied = run_reconcile(stand, "apply", 100) assert set(applied.kinds(FIXED, ids=ids)) == ids assert ids <= lost_ids(stand) @pytest.mark.spec( "storage-consistency", "Объект, на который ссылается строка, не удаляется", ) def test_reconcile_keeps_referenced_objects( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Замечание W1 verify: объект `/original`, у строки которого другой ключ, и объект `/original`, на который ссылается строка Y другого `id`, — чужие, а не объекты без метаданных.""" data = images.sample_png() x, y = (api.upload_ready(alice, chat["id"], data) for _ in range(2)) z = str(uuid.uuid4()) keys = { "x": object_key(x["id"]), "y": object_key(y["id"]), "z": object_key(z), # ключ X: под префиксом вложений, объекта по нему нет "other": f"{FAULT_DATABASE}/attachments/x-other-{uuid.uuid4()}", } put_object(stand, keys["z"], data) try: stand("set-key", x["id"], keys["other"]) stand("set-key", y["id"], keys["z"]) for item in (x, y): stand("age-row", item["id"], str(AGED_S)) # объект Z записан только что: после G + 1 все объекты старше grace time.sleep(OBJECT_AGE_WAIT_S) unknown = {keys[name]: "unknown_object" for name in ("x", "y", "z")} dry = run_reconcile(stand, "dry-run") assert dry.kinds(FOUND, keys=keys.values()) == unknown | { keys["other"]: "missing_object", } assert dry.exit_code == 3 applied = run_reconcile(stand, "apply") assert applied.kinds(FIXED, keys=keys.values()) == { keys["other"]: "missing_object", } assert applied.kinds(FOUND, keys=keys.values()) == unknown assert applied.exit_code == 3 # ни один из трёх объектов не удалён, строки на месте assert {x["id"], y["id"], z} <= stored_ids(stand) rows = attachment_rows(stand) assert rows[x["id"]] == rows[y["id"]] == "ready" # X без объекта по своему ключу помечено, у Y объект по ключу есть lost = lost_ids(stand) assert x["id"] in lost assert y["id"] not in lost finally: # прежние ключи; объект Z, на который больше никто не ссылается, — # прочь, чтобы следующие проходы не удаляли его как свой stand("set-key", x["id"], keys["x"]) stand("set-key", y["id"], keys["y"]) stand("delete-object", keys["z"]) @pytest.mark.spec( "storage-consistency", "Предел не останавливает уборку зависших", ) def test_reconcile_limit_keeps_cleanup( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Замечание W1 verify: отказ по пределу не мешает уборке зависшей загрузки — её уборщик делает и без сверки.""" data = images.sample_png() limit = 2 # уборщик стенда стоит: зависшую загрузку P должна убрать сверка with outage(stand, "stop", "cleaner"): filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "commit", filename) try: assert_error( upload_raw(api, alice, chat, data, filename=filename), 503, ) finally: stand("clear-fail-attachments") (pending,) = new_ids(before, attachment_rows(stand)) assert pending in staged_ids(stand) stand("age-row", pending, str(AGED_S)) batch = [api.upload_ready(alice, chat["id"], data) for _ in range(3)] ids = {item["id"] for item in batch} for item in batch: lose_original(stand, item) refused = run_reconcile(stand, "apply", limit) assert refused.exit_code == 3 (error,) = refused.events(LIMIT_EXCEEDED) assert error["changes"] >= len(batch), error assert refused.finished["refused"] is True assert refused.kinds(FIXED, ids={pending}) == { pending: "stale_pending", } (record,) = refused.about(FIXED, ids={pending}) assert record["action"] == "removed_pending" assert pending not in attachment_rows(stand) assert pending not in staged_ids(stand) # изменений под пределом нет: ни одной пометки assert not refused.about(FIXED, ids=ids) assert not ids & lost_ids(stand) # стенд — без внесённых потерь для следующих тестов applied = run_reconcile(stand, "apply", 100) assert set(applied.kinds(FIXED, ids=ids)) == ids @pytest.mark.spec( "storage-consistency", "Ключ вне префикса вложений не трогается", ) def test_reconcile_foreign_key_untouched( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Замечание W2 verify: строка с ключом вне `<база>/attachments/` — готовая и зависшая загрузка — только в отчёте как чужой объект: сверка не делает HEAD чужого префикса, не помечает и не убирает строку.""" data = images.sample_png() # уборщик стенда стоит: иначе зависшую загрузку убрал бы он with outage(stand, "stop", "cleaner"): ready = api.upload_ready(alice, chat["id"], data) filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "commit", filename) try: assert_error( upload_raw(api, alice, chat, data, filename=filename), 503, ) finally: stand("clear-fail-attachments") (pending,) = new_ids(before, attachment_rows(stand)) ids = {ready["id"], pending} try: for item in ids: # префикс своей базы, но не вложений: список его не видит stand("set-key", item, f"{FAULT_DATABASE}/elsewhere/{item}") stand("age-row", item, str(AGED_S)) unknown = dict.fromkeys(ids, "unknown_object") dry = run_reconcile(stand, "dry-run") assert dry.kinds(FOUND, ids=ids) == unknown assert dry.exit_code == 3 applied = run_reconcile(stand, "apply") assert applied.kinds(FOUND, ids=ids) == unknown assert not applied.about(FIXED, ids=ids) assert applied.exit_code == 3 # ни пометки, ни уборки: строки и объекты под префиксом на месте rows = attachment_rows(stand) assert (rows.get(ready["id"]), rows.get(pending)) == ( "ready", "pending", ) assert ready["id"] not in lost_ids(stand) assert ready["id"] in stored_ids(stand) assert pending in staged_ids(stand) finally: # прежние ключи у тех строк, что остались: зависшую загрузку # после теста уберёт уборщик стенда вместе с её объектом rows = attachment_rows(stand) for item in ids & set(rows): stand("set-key", item, object_key(item)) @pytest.mark.spec("storage-consistency", "S3 недоступен во время сверки") def test_reconcile_object_storage_stopped( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: missing = api.upload_ready(alice, chat["id"], images.sample_png()) lose_original(stand, missing) with outage(stand, "stop", "s3"): interrupted = run_reconcile(stand, "apply") assert interrupted.exit_code == 1 assert interrupted.finished["interrupted_by"] == "object_storage" assert not interrupted.events(FIXED) assert missing["id"] not in lost_ids(stand) applied = run_reconcile(stand, "apply") assert applied.kinds(FIXED, ids={missing["id"]}) == { missing["id"]: "missing_object", } assert missing["id"] in lost_ids(stand) @pytest.mark.spec( "storage-consistency", "Прерванная сверка доводится повтором", ) def test_reconcile_interrupted_by_database( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: filename = fault_filename() failing = api.upload_ready( alice, chat["id"], images.sample_png(), filename=filename, ) other = api.upload_ready(alice, chat["id"], images.sample_png()) ids = {failing["id"], other["id"]} for item in (failing, other): lose_original(stand, item) stand("fail-attachments", "mark", filename) try: interrupted = run_reconcile(stand, "apply") assert interrupted.exit_code == 1 assert interrupted.finished["interrupted_by"] == "database" assert failing["id"] not in lost_ids(stand) finally: stand("clear-fail-attachments") applied = run_reconcile(stand, "apply") assert ids <= lost_ids(stand) assert not applied.about(FOUND, ids=ids) again = run_reconcile(stand, "dry-run") assert not again.about(FOUND, ids=ids) @pytest.mark.spec("storage-consistency", "Сверка под нагрузкой") def test_reconcile_under_load( api: Api, stand: Stand, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: """Загрузки и отправки дольше grace, сверка с `--apply` раз в 3 с: внесённое исправлено, ни одно подтверждённое вложение не потеряно.""" data = images.sample_png() orphans = {str(uuid.uuid4()) for _ in range(2)} for item in orphans: put_object(stand, object_key(item), data) injected = [api.upload_ready(alice, chat["id"], data) for _ in range(2)] for item in injected: lose_original(stand, item) lost_before = lost_ids(stand) deadline = time.monotonic() + CLEANER_GRACE_S + LOAD_EXTRA_S lock = threading.Lock() uploads: dict[str, str] = {} messages: dict[str, list[str]] = {} failures: list[str] = [] passes: list[Reconcile] = [] def load(worker: int) -> None: actor = alice if worker % 2 == 0 else bob with Api( str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S, ) as client: number = 0 while time.monotonic() < deadline: number += 1 body = images.png_noise(200, 170, seed=worker * 10**6 + number) resp = client.post_attachment(actor, chat["id"], body) if resp.status_code != 202: with lock: failures.append(f"upload {resp.status_code}") continue attachment = assert_accepted(resp) # загрузка подтверждена: она обязана стать ready и скачиваться with lock: uploads[attachment["id"]] = hashlib.sha256( body, ).hexdigest() message = client.send_message( actor, chat["id"], "", attachment_ids=[attachment["id"]], ) with lock: messages[message["id"]] = [attachment["id"]] time.sleep(LOAD_PAUSE_S) def reconcile() -> None: while time.monotonic() < deadline: started = time.monotonic() passes.append(run_reconcile(stand, "apply")) sleep_until(started + LOAD_RECONCILE_PERIOD_S) def guarded(target: Callable[..., None], *args: Any) -> None: # исключение потока — в список, а не молча в stderr try: target(*args) except Exception as e: with lock: failures.append(repr(e)) threads = [ threading.Thread(target=guarded, args=(load, worker)) for worker in range(LOAD_THREADS) ] threads.append(threading.Thread(target=guarded, args=(reconcile,))) for thread in threads: thread.start() for thread in threads: thread.join() assert not failures, failures[:5] assert uploads, "no uploads under load" assert len(passes) >= 3, len(passes) assert all(item.exit_code in (0, 3) for item in passes), [ item.exit_code for item in passes ] # внесённые исправлены: объекты без строки удалены, потери помечены assert not orphans & stored_ids(stand) assert lost_ids(stand) - lost_before == {item["id"] for item in injected} # ни одно подтверждённое вложение не потеряно for attachment_id, digest in uploads.items(): api.wait_attachment(alice, attachment_id) resp = api.attachment_content(alice, attachment_id) assert resp.status_code == 200, (attachment_id, resp.status_code) assert hashlib.sha256(resp.content).hexdigest() == digest history = { item["id"]: item["attachment_ids"] for item in api.all_messages(alice, chat["id"]) } for message_id, attachment_ids in messages.items(): assert history[message_id] == attachment_ids @pytest.mark.spec("storage-consistency", "Плановая сверка без команды") def test_scheduled_reconcile( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: scheduled = api.upload_ready(alice, chat["id"], images.sample_png()) lose_original(stand, scheduled) log = stand("run-cleaner", str(SCHEDULED_RUN_S)) records = [ json.loads(line) for line in log.splitlines() if line.startswith("{") ] passes = [item for item in records if item.get("event") == FINISHED] assert passes, log[-2000:] assert any( item["apply"] and item["interrupted_by"] is None for item in passes ), passes assert scheduled["id"] in lost_ids(stand) # --------------------------------------------------------------------------- # Стоимость сверки (change track2-reconcile-cost, design, решение 14): # стратегия HEAD, глубокая сверка и класс «другой SHA-256», снятие пометки # только при том же SHA-256, предел скорости, отказ по пределу без лишних # перепроверок. Аргументы сверки hook передаёт команде как есть # --------------------------------------------------------------------------- HEAD_STRATEGY = ("--strategy", "head") DEEP = ("--deep",) NO_RATE_LIMIT = ("--max-objects-per-second", "0") # сколько вложений загружает тест предела скорости: объектов в bucket # заведомо хватает на несколько секунд прохода RATE_UPLOADS = 8 # во сколько раз проход с пределом длиннее числа объектов на предел # (допуск на округление `duration_ms`) RATE_SLACK = 0.95 def same_size_bytes(data: bytes, *, seed: int) -> bytes: """Другие байты того же размера, от зерна теста.""" other = random.Random(seed).randbytes(len(data)) assert other != data return other def findings(run: Reconcile) -> set[tuple[str, str]]: """Все расхождения прохода: класс и ключ объекта.""" return {(item["kind"], item["object_key"]) for item in run.events(FOUND)} @pytest.mark.spec("storage-consistency", "Стратегия HEAD находит то же") def test_reconcile_head_strategy_same_findings( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Те же расхождения, что в `test_reconcile_finds_and_fixes`: dry-run стратегией по умолчанию и `--strategy head` называют одно и то же, а HEAD-проход дороже по запросам к S3 и сеансам PostgreSQL.""" data = images.sample_png() # уборщик стенда стоит: зависшую загрузку P не уберут между проходами with outage(stand, "stop", "cleaner"): a, b, c = (api.upload_ready(alice, chat["id"], data) for _ in range(3)) api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]]) orphan = str(uuid.uuid4()) keys = { "o": object_key(orphan), "b": object_key(b["id"]), "bv": variant_key(b["id"]), } put_object(stand, keys["o"], data) stand("delete-object", object_key(a["id"])) stand("delete-row", b["id"]) stand("set-size", c["id"], str(len(data) + 1)) filename = fault_filename() before = attachment_rows(stand) stand("fail-attachments", "commit", filename) try: resp = upload_raw(api, alice, chat, data, filename=filename) assert_error(resp, 503) finally: stand("clear-fail-attachments") (pending,) = new_ids(before, attachment_rows(stand)) time.sleep(OBJECT_AGE_WAIT_S) ids = {a["id"], c["id"], pending} expected = { keys["o"]: "orphan_object", keys["b"]: "orphan_object", keys["bv"]: "orphan_object", a["id"]: "missing_object", c["id"]: "size_mismatch", pending: "stale_pending", } snapshot = (attachment_rows(stand), object_ids(stand), lost_ids(stand)) batch = run_reconcile(stand, "dry-run") head = run_reconcile(stand, "dry-run", options=HEAD_STRATEGY) assert batch.kinds(FOUND, ids=ids, keys=keys.values()) == expected assert head.kinds(FOUND, ids=ids, keys=keys.values()) == expected # не только свои: весь отчёт и итог по классам одинаковы assert findings(head) == findings(batch) for field in ("found", "listed_objects", "checked_rows"): assert head.finished[field] == batch.finished[field], field assert head.exit_code == batch.exit_code == 3 assert (batch.finished["strategy"], head.finished["strategy"]) == ( "batch", "head", ) assert ( head.finished["s3_requests"]["head"] > batch.finished["s3_requests"]["head"] ), (head.finished, batch.finished) assert head.finished["db_sessions"] > batch.finished["db_sessions"] after = (attachment_rows(stand), object_ids(stand), lost_ids(stand)) assert after == snapshot # стенд — без внесённых расхождений для следующих тестов fixed = run_reconcile(stand, "apply") assert set(fixed.kinds(FIXED, ids=ids, keys=keys.values())) == set( expected, ) @pytest.mark.spec( "storage-consistency", "Другой SHA-256 находит глубокая сверка", ) def test_reconcile_deep_checksum( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.png_noise(64, 48, seed=11) a = api.upload_ready(alice, chat["id"], data) api.send_message(alice, chat["id"], "", attachment_ids=[a["id"]]) other = same_size_bytes(data, seed=11) # объект перезаписан мимо API байтами того же размера put_object(stand, object_key(a["id"]), other) stand("age-row", a["id"], str(AGED_S)) time.sleep(OBJECT_AGE_WAIT_S) snapshot = (attachment_rows(stand), stored_ids(stand), lost_ids(stand)) shallow = run_reconcile(stand, "dry-run") assert not shallow.about(FOUND, ids={a["id"]}) assert shallow.finished["deep"] is False deep = run_reconcile(stand, "dry-run", options=DEEP) assert deep.kinds(FOUND, ids={a["id"]}) == {a["id"]: "checksum_mismatch"} assert deep.finished["deep"] is True assert deep.finished["read_bytes"] >= len(data), deep.finished assert deep.finished["s3_requests"]["get"] >= 1, deep.finished assert deep.exit_code == 3 after = (attachment_rows(stand), stored_ids(stand), lost_ids(stand)) assert after == snapshot # SHA-256 объектов в журнале нет journal = json.dumps(deep.records) for body in (data, other): assert hashlib.sha256(body).hexdigest() not in journal applied = run_reconcile(stand, "apply", options=DEEP) (record,) = applied.about(FIXED, ids={a["id"]}) assert (record["kind"], record["action"]) == ( "checksum_mismatch", "marked_lost", ) assert a["id"] in lost_ids(stand) assert_lost_content(api, alice, a, aged=True) @pytest.mark.spec( "storage-consistency", "Вернулся другой объект того же размера", ) @pytest.mark.spec("storage-consistency", "Оригинал вернулся") def test_reconcile_restore_needs_same_bytes( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.png_noise(64, 48, seed=13) restored = api.upload_ready(alice, chat["id"], data) ids = {restored["id"]} lose_original(stand, restored) marked = run_reconcile(stand, "apply") assert marked.kinds(FIXED, ids=ids) == {restored["id"]: "missing_object"} assert restored["id"] in lost_ids(stand) # под ключом — другие байты того же размера: пометка остаётся put_object( stand, object_key(restored["id"]), same_size_bytes(data, seed=13), ) time.sleep(OBJECT_AGE_WAIT_S) wrong = run_reconcile(stand, "apply") assert not wrong.about(FIXED, ids=ids) assert all( record["kind"] != "restored_object" for record in wrong.about(FOUND, ids=ids) ) assert restored["id"] in lost_ids(stand) assert_lost_content(api, alice, restored, aged=True) # те же байты, что были загружены: пометка снята put_object(stand, object_key(restored["id"]), data) time.sleep(OBJECT_AGE_WAIT_S) cleared = run_reconcile(stand, "apply") (record,) = cleared.about(FIXED, ids=ids) assert (record["kind"], record["action"]) == ( "restored_object", "cleared_lost", ) assert restored["id"] not in lost_ids(stand) assert_content( api.attachment_content(alice, restored["id"]), data, "image/png", ) again = run_reconcile(stand, "dry-run") assert not again.about(FOUND, ids=ids) @pytest.mark.spec("storage-consistency", "Скорость прохода ограничена") @pytest.mark.spec("storage-consistency", "Без ограничения скорости") def test_reconcile_rate_limit( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Предел на четверть объектов прохода: проход длится не меньше объектов на предел, находки и код — как без предела.""" data = images.sample_png() for _ in range(RATE_UPLOADS): api.upload_ready(alice, chat["id"], data) lost = api.upload_ready(alice, chat["id"], data) ids = {lost["id"]} lose_original(stand, lost) # уборщик стенда стоит: между проходами стенд не меняется with outage(stand, "stop", "cleaner"): free = run_reconcile(stand, "dry-run", options=NO_RATE_LIMIT) rate = max(5, math.ceil(free.finished["storage_objects"] / 4)) limited = run_reconcile( stand, "dry-run", options=("--max-objects-per-second", str(rate)), ) assert free.finished["max_objects_per_second"] == 0 assert limited.finished["max_objects_per_second"] == rate objects = limited.finished["storage_objects"] assert objects >= RATE_UPLOADS, limited.finished assert ( limited.finished["duration_ms"] >= 1000 * objects / rate * RATE_SLACK ), limited.finished assert free.finished["duration_ms"] < limited.finished["duration_ms"] expected = {lost["id"]: "missing_object"} assert free.kinds(FOUND, ids=ids) == expected assert limited.kinds(FOUND, ids=ids) == expected assert free.exit_code == limited.exit_code == 3 # стенд — без внесённой потери applied = run_reconcile(stand, "apply") assert applied.kinds(FIXED, ids=ids) == expected @pytest.mark.spec( "storage-consistency", "Отказ по пределу без лишних перепроверок", ) def test_reconcile_limit_stops_rechecks( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: limit = 2 batch = [ api.upload_ready(alice, chat["id"], images.sample_png()) for _ in range(6) ] ids = {item["id"] for item in batch} for item in batch: lose_original(stand, item) refused = run_reconcile(stand, "apply", limit) assert refused.exit_code == 3 (error,) = refused.events(LIMIT_EXCEEDED) assert error["level"] == "error" # отказ решён на третьем подтверждённом: остальные не перепроверены assert error["changes"] == limit + 1, error assert error["max_changes"] == limit assert error["unchecked"] >= len(batch) - (limit + 1), error assert refused.finished["unchecked"] == error["unchecked"] assert refused.finished["refused"] is True assert not refused.about(FIXED, ids=ids) assert not ids & lost_ids(stand) # dry-run перепроверяет каждого и называет все шесть dry = run_reconcile(stand, "dry-run") assert dry.kinds(FOUND, ids=ids) == dict.fromkeys(ids, "missing_object") assert dry.finished["unchecked"] == 0 applied = run_reconcile(stand, "apply", 100) assert set(applied.kinds(FIXED, ids=ids)) == ids assert ids <= lost_ids(stand) def own_findings( run: Reconcile, event: str, keys: Collection[str], ) -> set[tuple[str, str, str | None]]: """Записи `event` о своих ключах: класс, ключ и `id` вложения — у строки, ссылающейся на чужой ключ, `id` и ключ из разных вложений.""" return { (item["kind"], item["object_key"], item.get("attachment_id")) for item in run.events(event) if item.get("object_key") in keys } @pytest.mark.spec( "storage-consistency", "Строка ссылается на объект другого id", ) def test_reconcile_row_with_other_id_key( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Замечание W2 verify `track2-reconcile-cost`: строка Y ссылается на `/original` чужого `id`. Обе стратегии сравнивают Y с объектом по её ключу: другой размер и, с `--deep`, другой SHA-256 — расхождения Y; объекты Z и `/original` — чужие и на месте.""" data = images.png_noise(64, 48, seed=17) y1, y2 = (api.upload_ready(alice, chat["id"], data) for _ in range(2)) z1, z2 = (str(uuid.uuid4()) for _ in range(2)) keys = { "y1": object_key(y1["id"]), "y2": object_key(y2["id"]), "z1": object_key(z1), "z2": object_key(z2), } # Z1 — другого размера, Z2 — того же размера с другими байтами put_object(stand, keys["z1"], data + b"\0") put_object(stand, keys["z2"], same_size_bytes(data, seed=17)) try: stand("set-key", y1["id"], keys["z1"]) stand("set-key", y2["id"], keys["z2"]) for item in (y1, y2): stand("age-row", item["id"], str(AGED_S)) # объекты Z записаны только что: после G + 1 все старше grace time.sleep(OBJECT_AGE_WAIT_S) expected = { ("size_mismatch", keys["z1"], y1["id"]), ("checksum_mismatch", keys["z2"], y2["id"]), ("unknown_object", keys["z1"], z1), ("unknown_object", keys["z2"], z2), ("unknown_object", keys["y1"], y1["id"]), ("unknown_object", keys["y2"], y2["id"]), } batch = run_reconcile(stand, "dry-run", options=DEEP) head = run_reconcile( stand, "dry-run", options=(*DEEP, *HEAD_STRATEGY), ) assert own_findings(batch, FOUND, keys.values()) == expected assert own_findings(head, FOUND, keys.values()) == expected assert batch.exit_code == head.exit_code == 3 # поверхностная сверка видит только другой размер shallow = run_reconcile(stand, "dry-run") assert own_findings(shallow, FOUND, keys.values()) == expected - { ("checksum_mismatch", keys["z2"], y2["id"]), } applied = run_reconcile(stand, "apply", options=DEEP) assert own_findings(applied, FIXED, keys.values()) == { ("size_mismatch", keys["z1"], y1["id"]), ("checksum_mismatch", keys["z2"], y2["id"]), } assert applied.exit_code == 3 # ни один объект не удалён, обе строки — ready и помечены assert {y1["id"], y2["id"], z1, z2} <= stored_ids(stand) rows = attachment_rows(stand) assert rows[y1["id"]] == rows[y2["id"]] == "ready" assert {y1["id"], y2["id"]} <= lost_ids(stand) finally: # прежние ключи; объекты Z, на которые больше никто не ссылается, # — прочь, чтобы следующие проходы не удаляли их как свои stand("set-key", y1["id"], keys["y1"]) stand("set-key", y2["id"], keys["y2"]) stand("delete-object", keys["z1"]) stand("delete-object", keys["z2"]) @pytest.mark.spec( "storage-consistency", "Недавно перезаписанный объект не тронут", ) def test_reconcile_fresh_overwrite_untouched( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Замечание S1 verify `track2-reconcile-cost`: объект готового вложения старше grace перезаписан мимо API меньше grace назад — другим размером (A) и теми же размером с другими байтами (B). Сверка с `--deep --apply` и HEAD-стратегией считает оба объекта свежими; после grace — расхождения. Глубокий проход читает весь bucket стенда и к концу набора длится дольше G: окно «свежий» — grace разового прохода `FRESH_GRACE_S`, он меньше возраста строк.""" data = images.png_noise(64, 48, seed=19) a, b = (api.upload_ready(alice, chat["id"], data) for _ in range(2)) ids = {a["id"], b["id"]} for item in (a, b): stand("age-row", item["id"], str(AGED_S)) def overwrite() -> float: """Объекты A и B заново перезаписаны: моложе grace с этого момента.""" put_object(stand, object_key(a["id"]), data + b"\0") put_object(stand, object_key(b["id"]), same_size_bytes(data, seed=19)) return time.monotonic() grace = ("--grace", str(FRESH_GRACE_S)) try: # перед каждым проходом объекты перезаписываются заново: глубокий # проход читает весь bucket стенда, и два подряд дольше grace for options, mode in ( ((*DEEP, *grace), "apply"), ((*DEEP, *HEAD_STRATEGY, *grace), "dry-run"), ): written = overwrite() run = run_reconcile(stand, mode, options=options) # проход — пока объекты моложе grace, иначе он ничего не # доказывает assert time.monotonic() - written < FRESH_GRACE_S, ( "reconcile run took longer than its grace period" ) assert not run.about(FOUND, ids=ids), run.finished assert not run.about(FIXED, ids=ids), run.finished assert run.finished["fresh_objects"] >= len(ids), run.finished assert not ids & lost_ids(stand) assert_error(api.attachment_content(alice, a["id"]), 500) time.sleep(max(0.0, OBJECT_AGE_WAIT_S - (time.monotonic() - written))) aged = run_reconcile(stand, "dry-run", options=DEEP) assert aged.kinds(FOUND, ids=ids) == { a["id"]: "size_mismatch", b["id"]: "checksum_mismatch", } finally: # стенд — с исходными байтами: расхождений не остаётся for item in (a, b): put_object(stand, object_key(item["id"]), data) # --------------------------------------------------------------------------- # Сверка и фоновая обработка (change async-attachments, design, решение 11): # объекты вариантов своих вложений и текущая работа — не расхождения # --------------------------------------------------------------------------- @pytest.mark.spec("storage-consistency", "Объекты вариантов своих вложений") def test_reconcile_variant_objects( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() batch = [api.upload_ready(alice, chat["id"], data) for _ in range(2)] ids = {item["id"] for item in batch} keys = {variant_key(item["id"]) for item in batch} for item in batch: assert stand("variant-object-count", item["id"]) == str( len(VARIANT_NAMES), ) stand("age-row", item["id"], str(AGED_S)) # объекты вариантов записаны только что: после G + 1 старше grace time.sleep(OBJECT_AGE_WAIT_S) for options in ((), HEAD_STRATEGY): run = run_reconcile(stand, "dry-run", options=options) assert not run.about(FOUND, ids=ids, keys=keys), run.finished run.assert_code_matches() @pytest.mark.spec("storage-consistency", "Объект варианта без метаданных") def test_reconcile_orphan_variant( stand: Stand, ) -> None: data = images.sample_png() orphan = str(uuid.uuid4()) key = variant_key(orphan) put_object(stand, key, data) assert stand("variant-object-count", orphan) == "1" time.sleep(OBJECT_AGE_WAIT_S) run = run_reconcile(stand, "apply") (record,) = run.about(FIXED, keys={key}) assert (record["kind"], record["action"]) == ( "orphan_object", "deleted_object", ) assert not run.about(FOUND, keys={key}) assert stand("variant-object-count", orphan) == "0" @pytest.mark.spec( "storage-consistency", "Текущая работа не считается расхождением", ) def test_reconcile_keeps_current_work( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: # уборщик стоит: срок хранения `failed` на стенде 60 с, а проход к концу набора # длинный, и уборщик убрал бы staging строки `failed` раньше проверки with ( outage(stand, "stop", "cleaner"), current_work(api, stand, alice, chat) as work, ): # конечные объекты F и P и варианты P старше grace time.sleep(OBJECT_AGE_WAIT_S) ids = {item["id"] for item in work.values()} keys = {staging_key(item) for item in ids} | { object_key(item) for item in ids } run = run_reconcile(stand, "apply") assert not run.about(FOUND, ids=ids, keys=keys), run.finished assert not run.about(FIXED, ids=ids, keys=keys), run.finished run.assert_code_matches() assert not ids & lost_ids(stand) rows = attachment_rows(stand) staged = staged_ids(stand) for name, item in work.items(): assert rows.get(item["id"]) == name, (name, rows.get(item["id"])) assert item["id"] in staged, name stored = stored_ids(stand) # конечный объект у задания, дошедшего до него, на месте assert work["failed"]["id"] in stored assert work["processing"]["id"] in stored # --------------------------------------------------------------------------- # Надёжность конвейера (change pipeline-reliability, design, решения 2–9, 14, # 15): relay, окна сбоя без точек в коде, аренда и fencing, лимиты процесса # обработки, backoff, реестр failed и replay, prefetch, очистка staging. Worker # и relay запускаются в finally, триггеры и дополнительные контейнеры снимаются # в finally # --------------------------------------------------------------------------- @contextmanager def extra_worker(stand: Stand, *env: str) -> Iterator[None]: """Основной worker остановлен, работает второй с этим окружением.""" wait_queue_empty(stand) with outage(stand, "stop", "worker"): stand("run-worker", *env) try: yield finally: stand("stop-extra") @pytest.mark.spec("job-outbox", "Ответ не ждёт публикации") @pytest.mark.spec("job-outbox", "Crash между фиксацией и публикацией") @pytest.mark.spec("job-outbox", "Задание без отметки от прежней версии") def test_relay_killed_before_publish( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Relay убит: загрузка 202, сообщения нет, метаданные `queued`, задание без отметки — то же, что строка прежней версии. После запуска — `ready` с тем же заданием.""" wait_queue_empty(stand) with outage(stand, "kill", "relay"): resp, elapsed = timed( lambda: upload_raw(api, alice, chat, images.sample_png()), ) accepted = assert_accepted(resp) assert elapsed < ACCEPT_MAX_S, elapsed assert queue_state(stand)["messages"] == 0 meta = assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) assert meta["status"] == "queued", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["published_at"] is None, job meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) done = job_row(stand, accepted["id"]) assert done is not None assert done["id"] == job["id"], done assert done["published_at"] is not None, done assert done["attempt"] == 1, done def crash_relay_after_confirm( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Worker остановлен, отметка задержана триггером, брокер сообщение уже подтвердил: relay убит, транзакция отметки откатилась. После запуска relay публикует ещё раз; дубль — no-op.""" filename = fault_filename() with worker_stopped(stand): stand("fail-attachments", "relay-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) wait_queue_messages(stand, 1) job = job_row(stand, accepted["id"]) assert job is not None assert job["published_at"] is None, job stand("kill", "relay") finally: stand("clear-fail-attachments") stand("start", "relay") # та же работа опубликована ещё раз: отметки не было wait_queue_messages(stand, 2) meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, meta) wait_queue_empty(stand) done = job_row(stand, accepted["id"]) assert done is not None assert done["attempt"] == 1, done @pytest.mark.spec("job-outbox", "Crash после подтверждения до отметки") def test_relay_killed_after_confirm( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: crash_relay_after_confirm(api, stand, alice, chat) @pytest.mark.spec("job-outbox", "Отказ брокера во время работы relay") def test_relay_survives_broker_outage( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = json.loads(stand("relay-state")) seen = len(relay_records(stand)) with outage(stand, "stop", "rabbitmq"): accepted = api.upload(alice, chat["id"], images.sample_png()) wait_relay_record(stand, "Broker is unavailable", skip=seen) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) after = json.loads(stand("relay-state")) assert after["running"] is True, after assert after["started_at"] == before["started_at"], after assert after["restart_count"] == before["restart_count"], after @pytest.mark.spec("job-outbox", "Отказ PostgreSQL во время работы relay") def test_relay_survives_database_outage( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = json.loads(stand("relay-state")) seen = len(relay_records(stand)) with outage(stand, "stop", "postgres"): wait_relay_record(stand, "Relay pass was interrupted", skip=seen) accepted = api.upload(alice, chat["id"], images.sample_png()) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) after = json.loads(stand("relay-state")) assert after["running"] is True, after assert after["started_at"] == before["started_at"], after assert after["restart_count"] == before["restart_count"], after @pytest.mark.spec("job-outbox", "Два relay") def test_two_relays( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: files = 6 with worker_stopped(stand): stand("run-relay", "RELAY__POLL_INTERVAL_SECONDS=0.05") try: accepted = [ api.upload(alice, chat["id"], images.sample_png()) for _ in range(files) ] wait_queue_messages(stand, files) time.sleep(1) assert queue_state(stand)["messages"] == files finally: stand("stop-extra") for item in accepted: api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) wait_queue_empty(stand) @pytest.mark.spec("job-outbox", "Следующая попытка не раньше срока") def test_relay_respects_next_attempt( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: delay = 6 with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) # relay стоит, пока срок отодвигается: случайная пауза короче хода теста with relay_stopped(stand), outage(stand, "stop", "s3"): stand("start", "worker") wait_job(stand, accepted["id"], state="queued", attempt=1) stand("stop", "worker") stand("set-next-attempt", accepted["id"], str(delay)) due = time.monotonic() + delay assert queue_state(stand)["messages"] == 0 wait_queue_messages(stand, 1, timeout=delay + PUBLISH_WAIT_S) assert time.monotonic() >= due - SLACK_S peeked = json.loads(stand("peek-job")) assert json.loads(peeked["body"])["attempt"] == 2, peeked api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("job-outbox", "Очередь удалена вместе с сообщением") def test_lost_message_republished( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) stand("delete-queue") # срок прошёл: relay объявляет очередь и публикует снова state = wait_queue_messages( stand, 1, timeout=RELAY_REPUBLISH_S + PUBLISH_WAIT_S, ) assert state["durable"] is True, state api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("job-outbox", "Очередь с сообщениями не обрастает копиями") def test_no_copies_while_queue_has_messages( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: files = 2 with worker_stopped(stand): accepted = [ api.upload(alice, chat["id"], images.sample_png()) for _ in range(files) ] wait_queue_messages(stand, files) time.sleep(RELAY_REPUBLISH_S + 2) assert queue_state(stand)["messages"] == files for item in accepted: api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("job-outbox", "Очередь ограничена пределом") def test_published_backlog_limit( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: limit, files = 2, 5 with worker_stopped(stand), relay_stopped(stand): stand("run-relay", f"RELAY__MAX_PUBLISHED_BACKLOG={limit}") try: accepted = [ api.upload(alice, chat["id"], images.sample_png()) for _ in range(files) ] wait_queue_messages(stand, limit) time.sleep(1) assert queue_state(stand)["messages"] == limit jobs = [job_row(stand, item["id"]) for item in accepted] marked = [job for job in jobs if job and job["published_at"]] assert len(marked) == limit, jobs finally: stand("stop-extra") for item in accepted: api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) wait_queue_empty(stand) @pytest.mark.spec("job-outbox", "Брокер сдерживает публикацию") def test_broker_memory_alarm( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): seen = len(relay_records(stand)) stand("broker-memory-limit", "1MiB") try: resp, elapsed = timed( lambda: upload_raw(api, alice, chat, images.sample_png()), ) accepted = assert_accepted(resp) assert elapsed < ACCEPT_MAX_S, elapsed wait_relay_record(stand, "Broker is unavailable", skip=seen) job = job_row(stand, accepted["id"]) assert job is not None assert job["published_at"] is None, job finally: stand("broker-memory-limit", BROKER_MEMORY_LIMIT) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) wait_queue_empty(stand) def crash_after_objects( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Все объекты записаны, фиксация ждёт в триггере: KILL. Новая попытка — по аренде — пишет те же ключи теми же байтами.""" filename = fault_filename() wait_queue_empty(stand) stand("fail-attachments", "ready-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) attachment_id = accepted["id"] api.wait_attachment( alice, attachment_id, status="processing", timeout=JOB_WAIT_S, ) wait_variant_objects(stand, attachment_id) # заморозка: окно триггера короче запроса к брокеру, а замёрзший клиент # COMMIT не отправит stand("freeze", "worker") # ack — только после фиксации: задание восстановила бы и аренда, поэтому # ранний ack виден только здесь held = queue_state(stand) assert held["messages_unacknowledged"] == 1, held stand("kill", "worker") before = { name: stand("object-sha256", variant_key(attachment_id, name)) for name in VARIANT_NAMES } assert all(before.values()), before finally: stand("clear-fail-attachments") stand("start", "worker") meta = api.wait_attachment(alice, attachment_id, timeout=JOB_WAIT_S) variants = assert_variants(api, stand, alice, meta) for name, digest in before.items(): assert variants[name]["sha256"] == digest, name done = job_row(stand, attachment_id) assert done is not None assert done["attempt"] == 2, done wait_queue_empty(stand) @pytest.mark.spec( "attachment-processing", "Crash после записи объектов до фиксации", ) def test_crash_after_objects_before_commit( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: crash_after_objects(api, stand, alice, chat) def crash_after_commit( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """COMMIT фиксации ждёт в отложенном триггере, worker заморожен (контейнер `running`, не `paused`, поэтому KILL проходит), строка `succeeded` при замороженном клиенте, KILL: ack не ушёл. Следующий запуск получает сообщение снова и подтверждает его без эффекта.""" filename = fault_filename() wait_queue_empty(stand) stand("fail-attachments", "ready-commit-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) attachment_id = accepted["id"] stand("wait-commit-stall") stand("freeze", "worker") wait_job(stand, attachment_id, state="succeeded", attempt=1) stand("kill", "worker") job = job_row(stand, attachment_id) assert job is not None ready = assert_ok( api.attachment_meta(alice, attachment_id), 200, "Attachment", ) assert ready["status"] == "ready", ready seen = len(worker_records(stand)) finally: stand("clear-fail-attachments") stand("start", "worker") record = wait_worker_record( stand, "Skipped job message", skip=seen, job_id=job["id"], reason="finished", ) assert record["level"] == "info", record wait_queue_empty(stand) assert ( assert_ok(api.attachment_meta(alice, attachment_id), 200, "Attachment") == ready ) assert job_row(stand, attachment_id) == job assert job["attempt"] == 1, job assert_variants(api, stand, alice, ready) @pytest.mark.spec( "attachment-processing", "Crash после фиксации до подтверждения", ) def test_crash_after_commit_before_ack( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: crash_after_commit(api, stand, alice, chat) @pytest.mark.spec( "attachment-processing", "Пауза растёт со случайной добавкой", ) def test_retry_backoff_with_jitter( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Формула решения 4: пауза попытки N — из `[d / 2, d]`, `d = min(max, base × 2^(N − 1))`, у двух заданий паузы разные.""" seen = len(worker_records(stand)) with worker_stopped(stand): first = api.upload(alice, chat["id"], images.sample_png()) second = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 2) with outage(stand, "stop", "s3"): stand("start", "worker") for item in (first, second): api.wait_attachment( alice, item["id"], status="failed", timeout=JOB_WAIT_S, ) delays: dict[str, dict[int, float]] = {first["id"]: {}, second["id"]: {}} for record in worker_records(stand)[seen:]: if record.get("event") == "Retrying job": delays[record["attachment_id"]][record["attempt"]] = record[ "delay_seconds" ] for attachment_id, pauses in delays.items(): assert sorted(pauses) == list(range(1, WORKER_MAX_ATTEMPTS)), pauses for attempt, delay in pauses.items(): top = min( WORKER_RETRY_MAX_S, WORKER_RETRY_BASE_S * 2 ** (attempt - 1), ) assert top / 2 <= delay <= top, (attachment_id, attempt, delay) for attempt in range(1, WORKER_MAX_ATTEMPTS): assert delays[first["id"]][attempt] != delays[second["id"]][attempt] @pytest.mark.spec("attachment-processing", "Попытка не раньше срока") @pytest.mark.spec( "attachment-processing", "Пауза не занимает место конкуренции", ) def test_retry_waits_outside_worker( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Задание A ждёт следующую попытку: дубль его сообщения и перезапуск worker попытку не начинают, а другое вложение становится `ready` раньше срока.""" wait = 20 with worker_stopped(stand): waiting = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) with relay_stopped(stand), outage(stand, "stop", "s3"): stand("start", "worker") wait_job(stand, waiting["id"], state="queued", attempt=1) stand("set-next-attempt", waiting["id"], str(wait)) due = time.monotonic() + wait job = job_row(stand, waiting["id"]) assert job is not None next_attempt_at = datetime.fromisoformat(job["next_attempt_at"]) stand("stop", "worker") stand( "publish-job", stdin=job_body(job["id"], waiting["id"], attempt=2), ) seen = len(worker_records(stand)) stand("start", "worker") wait_worker_record( stand, "Skipped job message", skip=seen, job_id=job["id"], reason="not_due", ) wait_queue_empty(stand) same = job_row(stand, waiting["id"]) assert same is not None assert (same["attempt"], same["state"]) == (1, "queued"), same other = api.upload_ready(alice, chat["id"], images.sample_png()) assert time.monotonic() < due, "other attachment was not ready in time" assert other["status"] == "ready" meta = assert_ok( api.attachment_meta(alice, waiting["id"]), 200, "Attachment", ) assert meta["status"] == "queued", meta done = api.wait_attachment(alice, waiting["id"], timeout=JOB_WAIT_S) assert done["status"] == "ready" final = job_row(stand, waiting["id"]) assert final is not None assert final["attempt"] == 2, final started = wait_worker_record( stand, "Started job attempt", skip=seen, job_id=job["id"], attempt=2, ) assert datetime.fromisoformat(started["timestamp"]) >= next_attempt_at @pytest.mark.spec("attachment-processing", "Пикселей больше лимита") def test_limit_pixels( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: huge = images.png_huge(10000, 5000) with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) put_object(stand, staging_key(accepted["id"]), huge) stand("set-size", accepted["id"], str(len(huge))) stand("set-sha256", accepted["id"], hashlib.sha256(huge).hexdigest()) meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "invalid_image", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["last_error_code"] == "limit_pixels", job assert job["attempt"] == 1, job @pytest.mark.spec("attachment-processing", "Объект больше лимита не читается") def test_limit_oversized_staging( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) put_object( stand, staging_key(accepted["id"]), bytes(images.MAX_BYTES + 1), ) meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "processing_failed", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["last_error_code"] == "source_corrupted", job assert job["attempt"] == 1, job @pytest.mark.spec("attachment-processing", "Дорогое изображение") def test_limit_cpu_scan_bomb( base_url: str, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = json.loads(stand("worker-state")) with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow: accepted = slow.upload( alice, chat["id"], images.jpeg_scan_bomb(), content_type="image/jpeg", filename="bomb.jpg", ) meta = slow.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "invalid_image", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["last_error_code"] == "limit_cpu", job assert job["attempt"] == 1, job state = json.loads(stand("worker-state")) assert state["running"] is True, state assert state["restart_count"] == before["restart_count"], state # следующий файл обрабатывается slow.upload_ready(alice, chat["id"], images.sample_png()) @pytest.mark.spec("attachment-processing", "Предел памяти процесса обработки") def test_limit_memory( base_url: str, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with ( Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow, extra_worker(stand, "WORKER__RENDER_MEMORY_LIMIT_MIB=200"), ): accepted = slow.upload(alice, chat["id"], images.png_rgba_large()) meta = slow.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "invalid_image", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["last_error_code"] == "limit_memory", job assert job["attempt"] == 1, job # процесс worker работает: следующий файл обрабатывается slow.upload_ready(alice, chat["id"], images.sample_png()) @pytest.mark.spec("attachment-processing", "Срок процесса обработки") def test_limit_render_timeout( base_url: str, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with ( Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow, extra_worker( stand, "WORKER__RENDER_TIMEOUT_SECONDS=1", "WORKER__RENDER_CPU_LIMIT_SECONDS=30", ), ): accepted = slow.upload( alice, chat["id"], images.jpeg_scan_bomb(), content_type="image/jpeg", filename="bomb.jpg", ) meta = slow.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "retry_exhausted", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["last_error_code"] == "timeout", job assert job["attempt"] == WORKER_MAX_ATTEMPTS, job assert stand("render-count", "extra") == "0" @pytest.mark.spec("attachment-processing", "Процесс обработки убит") @pytest.mark.spec("attachment-processing", "Секретов нет у процесса обработки") def test_render_killed( base_url: str, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: before = json.loads(stand("worker-state")) wait_queue_empty(stand) with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as slow: accepted = slow.upload( alice, chat["id"], images.jpeg_scan_bomb(), content_type="image/jpeg", filename="bomb.jpg", ) wait_render_process(stand) names = stand("render-env").split() assert names, "no environment of the render process" for name in names: assert not name.startswith( ("DATABASE__", "S3__", "BROKER__", "CACHE__"), ), names assert stand("kill-render") != "0" job = wait_job(stand, accepted["id"], state="queued", attempt=1) assert job["last_error_code"] == "render_killed", job state = json.loads(stand("worker-state")) assert state["running"] is True, state assert state["restart_count"] == before["restart_count"], state # следующая попытка упирается в лимит CPU: итог постоянный meta = slow.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "invalid_image", meta done = job_row(stand, accepted["id"]) assert done is not None assert done["attempt"] == 2, done @pytest.mark.spec("attachment-processing", "Прежняя попытка при живой новой") def test_hung_worker_lease_fencing( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Первая попытка размораживается, когда вторая жива: задание в `processing`, блокировки строки нет. Решает только номер попытки: по `state` фиксация первой прошла бы. Оба worker — со сроками больше времени теста, иначе при разморозке их прервал бы таймер, а не fencing. S3 заморожен, пока вторая не взята: она ждёт его и не успеет зафиксировать результат раньше своей заморозки. Итог даёт только вторая. Аренду делает истекшей hook: при таком сроке её не дождаться.""" second = "worker-second" timers = ( "WORKER__JOB_TIMEOUT_SECONDS=120", "WORKER__RENDER_TIMEOUT_SECONDS=60", "S3__TIMEOUT_SECONDS=60", ) seen_relay = len(relay_records(stand)) try: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) attachment_id = accepted["id"] wait_queue_messages(stand, 1) with outage(stand, "pause", "s3"): stand("run-worker", *timers) wait_job(stand, attachment_id, state="processing", attempt=1) wait_extra_record( stand, "Started job attempt", attachment_id=attachment_id, attempt=1, ) stand("pause-extra", "worker") stand("expire-lease", attachment_id) wait_relay_record( stand, "Recovered expired lease", skip=seen_relay, attachment_id=attachment_id, ) stand("run-worker-second", *timers) wait_job(stand, attachment_id, state="processing", attempt=2) wait_extra_record( stand, "Started job attempt", extra=second, attachment_id=attachment_id, attempt=2, ) stand("pause-extra", second) # вторая взята и заморожена в вызове S3: транзакции нет assert stand("attachment-free", attachment_id) taken = job_row(stand, attachment_id) assert taken is not None assert taken["state"] == "processing", taken assert taken["attempt"] == 2, taken stand("unpause-extra", "worker") wait_extra_record( stand, "Skipped job message", attachment_id=attachment_id, attempt=1, reason="superseded", ) assert job_row(stand, attachment_id) == taken assert ( assert_ok( api.attachment_meta(alice, attachment_id), 200, "Attachment", )["status"] == "processing" ) stand("unpause-extra", second) meta = api.wait_attachment( alice, attachment_id, timeout=JOB_WAIT_S, ) wait_extra_record( stand, "Attachment became ready", extra=second, attachment_id=attachment_id, attempt=2, ) job = job_row(stand, attachment_id) assert job is not None assert job["state"] == "succeeded", job assert job["attempt"] == 2, job assert not [ record for record in extra_records(stand) if record.get("event") == "Attachment became ready" and record.get("attachment_id") == attachment_id ] finally: stand("stop-extra") assert_variants(api, stand, alice, meta) @pytest.mark.spec("attachment-processing", "Зависший worker") def test_hung_worker_thawed_after_ready( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Вторая попытка уже дала `ready`, первая разморожена: итог её не меняет. Первый worker — со сроками больше времени теста: разморозка застаёт его таймеры живыми, и он доходит до фиксации (с обычными сроками его прервал бы таймер, а не fencing). Здесь решает и `state`; номер попытки — `test_hung_worker_lease_fencing`. Второй процесс — основной worker стенда.""" seen_relay = len(relay_records(stand)) with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) with outage(stand, "pause", "s3"): stand( "run-worker", "WORKER__JOB_TIMEOUT_SECONDS=120", "WORKER__RENDER_TIMEOUT_SECONDS=60", "S3__TIMEOUT_SECONDS=60", ) try: wait_job(stand, accepted["id"], state="processing", attempt=1) # попытка уже ждёт S3 wait_extra_record( stand, "Started job attempt", attachment_id=accepted["id"], ) stand("pause-extra", "worker") except BaseException: stand("stop-extra") raise try: stand("expire-lease", accepted["id"]) wait_relay_record( stand, "Recovered expired lease", skip=seen_relay, attachment_id=accepted["id"], ) meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 2, job stand("unpause-extra", "worker") wait_extra_record( stand, "Skipped job message", attachment_id=accepted["id"], reason="superseded", ) assert ( assert_ok( api.attachment_meta(alice, accepted["id"]), 200, "Attachment", ) == meta ) assert job_row(stand, accepted["id"]) == job finally: stand("stop-extra") assert_variants(api, stand, alice, meta) @pytest.mark.spec("attachment-processing", "Долгая живая попытка") def test_long_attempt_keeps_lease( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Фиксация ждёт в триггере, попытка укладывается в срок: relay её не возвращает.""" filename = fault_filename() seen = len(relay_records(stand)) stand("fail-attachments", "ready-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) meta = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) finally: stand("clear-fail-attachments") job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 1, job assert meta["status"] == "ready" assert not [ record for record in relay_records(stand)[seen:] if record.get("event") == "Recovered expired lease" and record.get("attachment_id") == accepted["id"] ] @pytest.mark.spec("attachment-processing", "Последняя попытка потеряна") def test_last_attempt_lost( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: filename = fault_filename() with extra_worker(stand, "WORKER__MAX_ATTEMPTS=1"): stand("fail-attachments", "ready-stall", filename) try: accepted = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) api.wait_attachment( alice, accepted["id"], status="processing", timeout=JOB_WAIT_S, ) stand("kill-extra", "worker") finally: stand("clear-fail-attachments") stand("run-worker", "WORKER__MAX_ATTEMPTS=1") meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "retry_exhausted", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["attempt"] == 1, job @dataclass class Replay: """Один вызов команды реестра: записи и код выхода.""" records: list[dict[str, Any]] exit_code: int def about(self, event: str, attachment_id: str) -> list[dict[str, Any]]: return [ item for item in self.records if item.get("event") == event and item.get("attachment_id") == attachment_id ] @property def finished(self) -> dict[str, Any]: (record,) = [ item for item in self.records if item.get("event") == "Finished replay pass" ] return record def run_replay(stand: Stand, *args: str) -> Replay: lines = stand("replay", *args).splitlines() assert lines and lines[-1].startswith("exit_code="), lines[-5:] records = [json.loads(line) for line in lines[:-1] if line.startswith("{")] return Replay(records=records, exit_code=int(lines[-1].split("=")[1])) def replay_id(stand: Stand, attachment_id: str) -> Replay: return run_replay( stand, "--attachment-id", attachment_id, "--apply", "--operator", "student-tests", ) def replays(stand: Stand, attachment_id: str) -> list[dict[str, Any]]: return [ json.loads(line) for line in stand("replays", attachment_id).splitlines() ] def failed_invalid( api: Api, alice: Actor, chat: dict[str, Any], ) -> dict[str, Any]: accepted = api.upload( alice, chat["id"], images.jpeg_precision_12(), content_type="image/jpeg", filename="image.jpg", ) return api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) def failed_exhausted( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], *, files: int = 1, ) -> list[dict[str, Any]]: """Вложения `failed` с `retry_exhausted`: S3 остановлен дольше всех попыток. S3 при выходе возвращён.""" with worker_stopped(stand): accepted = [ api.upload(alice, chat["id"], images.sample_png()) for _ in range(files) ] wait_queue_messages(stand, files) with outage(stand, "stop", "s3"): stand("start", "worker") done = [ api.wait_attachment( alice, item["id"], status="failed", timeout=JOB_WAIT_S, ) for item in accepted ] for meta in done: assert meta["error_code"] == "retry_exhausted", meta return done @pytest.mark.spec("attachment-processing", "Реестр показан без изменений") def test_replay_registry_dry_run( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: failed = failed_invalid(api, alice, chat) job = job_row(stand, failed["id"]) assert job is not None run = run_replay(stand, "--attachment-id", failed["id"]) assert run.exit_code == 0, run.finished (record,) = run.about("Found failed job", failed["id"]) assert record["job_id"] == job["id"], record assert record["error_code"] == "invalid_image", record assert record["reason"] == "undecodable", record assert ( assert_ok(api.attachment_meta(alice, failed["id"]), 200, "Attachment") == failed ) assert job_row(stand, failed["id"]) == job assert replays(stand, failed["id"]) == [] def replay_returns_work( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Объект staging заменён — `failed` `processing_failed` — реестр — объект возвращён — replay — `ready` с тем же заданием и аудитом.""" data = images.png_noise(64, 48, seed=37) with worker_stopped(stand): accepted = api.upload(alice, chat["id"], data) put_object( stand, staging_key(accepted["id"]), same_size_bytes(data, seed=37), ) meta = api.wait_attachment( alice, accepted["id"], status="failed", timeout=JOB_WAIT_S, ) assert meta["error_code"] == "processing_failed", meta job = job_row(stand, accepted["id"]) assert job is not None assert job["last_error_code"] == "source_corrupted", job registry = run_replay(stand, "--error-code", "processing_failed") assert registry.about("Found failed job", accepted["id"]), ( registry.finished ) put_object(stand, staging_key(accepted["id"]), data) run = replay_id(stand, accepted["id"]) assert run.exit_code == 0, run.finished (record,) = run.about("Replayed job", accepted["id"]) assert record["operator"] == "student-tests", record assert record["error_code"] == "processing_failed", record assert record["reason"] == "source_corrupted", record ready = api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) assert_content( api.attachment_content(alice, accepted["id"]), data, "image/png", ) assert ready["sha256"] == hashlib.sha256(data).hexdigest() assert_variants(api, stand, alice, ready) done = job_row(stand, accepted["id"]) assert done is not None assert done["id"] == job["id"], done assert done["attempt"] == 2, done (audit,) = replays(stand, accepted["id"]) assert audit["error_code"] == "processing_failed", audit assert audit["reason"] == "source_corrupted", audit assert audit["operator"] == "student-tests", audit @pytest.mark.spec("attachment-processing", "Replay возвращает работу") def test_replay_returns_work( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: replay_returns_work(api, stand, alice, chat) def replay_ready_refused( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Replay готовой работы — отказ, код 3, ничего не изменилось.""" ready = api.upload_ready(alice, chat["id"], images.sample_png()) job = job_row(stand, ready["id"]) run = replay_id(stand, ready["id"]) assert run.exit_code == 3, run.finished (record,) = run.about("Refused job replay", ready["id"]) assert record["refusal"] == "already_ready", record assert not run.about("Replayed job", ready["id"]) assert ( assert_ok(api.attachment_meta(alice, ready["id"]), 200, "Attachment") == ready ) assert job_row(stand, ready["id"]) == job assert_variants(api, stand, alice, ready) assert replays(stand, ready["id"]) == [] @pytest.mark.spec("attachment-processing", "Replay готового не создаёт дубль") def test_replay_ready_refused( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: replay_ready_refused(api, stand, alice, chat) @pytest.mark.spec("attachment-processing", "Replay идущей работы") def test_replay_in_progress_refused( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): queued = api.upload(alice, chat["id"], images.sample_png()) wait_queue_messages(stand, 1) run = replay_id(stand, queued["id"]) assert run.exit_code == 3, run.finished (record,) = run.about("Refused job replay", queued["id"]) assert record["refusal"] == "not_failed", record assert queue_state(stand)["messages"] == 1 job = job_row(stand, queued["id"]) assert job is not None assert job["attempt"] == 0, job api.wait_attachment(alice, queued["id"], timeout=JOB_WAIT_S) filename = fault_filename() stand("fail-attachments", "ready-stall", filename) try: busy = api.upload( alice, chat["id"], images.sample_png(), filename=filename, ) api.wait_attachment( alice, busy["id"], status="processing", timeout=JOB_WAIT_S, ) run = replay_id(stand, busy["id"]) assert run.exit_code == 3, run.finished (record,) = run.about("Refused job replay", busy["id"]) # фиксация могла успеть, пока шла команда: и то, и другое — отказ assert record["refusal"] in ("not_failed", "already_ready"), record meta = api.wait_attachment(alice, busy["id"], timeout=JOB_WAIT_S) finally: stand("clear-fail-attachments") assert meta["status"] == "ready" done = job_row(stand, busy["id"]) assert done is not None assert done["attempt"] == 1, done assert replays(stand, busy["id"]) == [] @pytest.mark.spec("attachment-processing", "Replay без исходника") def test_replay_without_source_refused( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: failed = failed_invalid(api, alice, chat) stand("age-row", failed["id"], str(STAGING_RETENTION_S + 5)) wait_staging_gone(stand, failed["id"]) run = replay_id(stand, failed["id"]) assert run.exit_code == 3, run.finished (record,) = run.about("Refused job replay", failed["id"]) assert record["refusal"] == "source_gone", record meta = assert_ok( api.attachment_meta(alice, failed["id"]), 200, "Attachment", ) assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image") @pytest.mark.spec("attachment-processing", "Новый бюджет попыток") def test_replay_new_budget( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: (failed,) = failed_exhausted(api, stand, alice, chat) run = replay_id(stand, failed["id"]) assert run.exit_code == 0, run.finished ready = api.wait_attachment(alice, failed["id"], timeout=JOB_WAIT_S) assert_variants(api, stand, alice, ready) job = job_row(stand, failed["id"]) assert job is not None assert job["attempt"] > WORKER_MAX_ATTEMPTS, job assert job["attempt_base"] == WORKER_MAX_ATTEMPTS, job @pytest.mark.spec("attachment-processing", "Replay по причине") def test_replay_by_code( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: exhausted = failed_exhausted(api, stand, alice, chat, files=2) invalid = failed_invalid(api, alice, chat) run = run_replay( stand, "--error-code", "retry_exhausted", "--apply", "--operator", "student-tests", ) for item in exhausted: assert run.about("Replayed job", item["id"]), run.finished api.wait_attachment(alice, item["id"], timeout=JOB_WAIT_S) assert not run.about("Replayed job", invalid["id"]) meta = assert_ok( api.attachment_meta(alice, invalid["id"]), 200, "Attachment", ) assert meta == invalid @pytest.mark.spec("attachment-processing", "Prefetch больше конкуренции") def test_prefetch_bounded( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: prefetch, files = 3, 5 filename = fault_filename() stand("fail-attachments", "ready-stall", filename) try: wait_queue_empty(stand) with outage(stand, "stop", "worker"): ids = [ api.upload( alice, chat["id"], images.sample_png(), filename=filename, )["id"] for _ in range(files) ] wait_queue_messages(stand, files) stand( "run-worker", "WORKER__CONCURRENCY=1", f"WORKER__PREFETCH={prefetch}", ) try: busy_seen = 0 unacked_seen = 0 deadline = time.monotonic() + JOB_WAIT_S while True: rows = attachment_rows(stand) statuses = [rows[item] for item in ids] busy = statuses.count("processing") assert busy <= 1, statuses busy_seen = max(busy_seen, busy) unacked = queue_state(stand)["messages_unacknowledged"] assert unacked <= prefetch, unacked unacked_seen = max(unacked_seen, unacked) if statuses.count("ready") == files: break assert time.monotonic() < deadline, statuses time.sleep(0.3) assert busy_seen == 1 assert unacked_seen > 1, unacked_seen finally: stand("stop-extra") finally: stand("clear-fail-attachments") wait_queue_empty(stand) @pytest.mark.spec("attachments", "Отсутствующий вариант") def test_missing_variant( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: ready = api.upload_ready(alice, chat["id"], images.sample_png()) stand("delete-variant", ready["id"], "crop") assert_error( api.attachment_content(alice, ready["id"], variant="crop"), 404, ) assert_content( api.attachment_content(alice, ready["id"]), images.sample_png(), "image/png", ) @pytest.mark.spec( "storage-consistency", "Загрузка без подтверждения брокера убрана", ) def test_cancelled_upload_of_previous_version( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Реплика прежней версии при отказе брокера отменяла задание: строка `pending`, помеченная к уборке. Её убирает уборщик вместе с объектом.""" started = time.monotonic() with worker_stopped(stand), relay_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) stand("cancel-to-pending", accepted["id"]) assert job_row(stand, accepted["id"]) is None assert_invisible(api, alice, chat, accepted["id"]) wait_cleaned(stand, accepted["id"], since=started) assert job_row(stand, accepted["id"]) is None @pytest.mark.spec("storage-consistency", "Staging готового вложения убран") def test_staging_removed_after_ready( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() ready = api.upload_ready(alice, chat["id"], data) wait_staging_gone(stand, ready["id"]) assert_content( api.attachment_content(alice, ready["id"]), data, "image/png", ) assert_variants(api, stand, alice, ready) @pytest.mark.spec( "storage-consistency", "Staging неудачного вложения хранится до срока", ) @pytest.mark.spec("storage-consistency", "Срок хранения прошёл") def test_staging_retention( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: failed = failed_invalid(api, alice, chat) attachment_id = failed["id"] # конечный оригинал записан до декодирования; варианты прежней попытки for name in VARIANT_NAMES: put_object(stand, variant_key(attachment_id, name), b"variant") time.sleep(3 * CLEANER_INTERVAL_S + 1) assert attachment_id in staged_ids(stand) assert attachment_id in stored_ids(stand) dry = run_replay(stand, "--attachment-id", attachment_id) assert dry.about("Found failed job", attachment_id), dry.finished stand("age-row", attachment_id, str(STAGING_RETENTION_S + 5)) wait_staging_gone(stand, attachment_id) assert attachment_id not in stored_ids(stand) assert stand("variant-object-count", attachment_id) == "0" meta = assert_ok( api.attachment_meta(alice, attachment_id), 200, "Attachment", ) assert (meta["status"], meta["error_code"]) == ("failed", "invalid_image") @pytest.mark.spec("storage-consistency", "Текущая работа не убирается") def test_staging_keeps_current_work( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with worker_stopped(stand): accepted = api.upload(alice, chat["id"], images.sample_png()) stand("age-row", accepted["id"], str(STAGING_RETENTION_S + 5)) time.sleep(3 * CLEANER_INTERVAL_S + 1) assert accepted["id"] in staged_ids(stand) api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) def wait_row_held(stand: Stand, attachment_id: str) -> None: """Строку вложения держит транзакция уборщика. Старт уборщика при 0,25 CPU — до нескольких секунд: без ожидания запрос прошёл бы раньше блокировки.""" deadline = time.monotonic() + 3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S + 10 while stand("attachment-free", attachment_id): assert time.monotonic() < deadline, "cleaner does not hold the row" time.sleep(0.1) @pytest.mark.spec("storage-consistency", "Очистка при отказе S3") def test_staging_cleanup_survives_outage( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Уборщик держит строку и ждёт DELETE в замороженном S3: отправка сообщения со ссылкой на вложение берёт `FOR KEY SHARE` и не ждёт его. Очередь уборщика сначала пуста: кандидаты идут от старых, и при замороженном S3 проход каждый раз прерывался бы на строке прежнего теста, не дойдя до этой.""" drained = api.upload_ready(alice, chat["id"], images.sample_png()) wait_staging_gone(stand, drained["id"]) with outage(stand, "stop", "cleaner"): ready = api.upload_ready(alice, chat["id"], images.sample_png()) assert ready["id"] in staged_ids(stand) stand("pause", "s3") try: for number in range(3): wait_row_held(stand, ready["id"]) resp, took = timed( lambda number=number: api.post_message( alice, chat["id"], json={ "text": f"пока S3 заморожен {number}", "attachment_ids": [ready["id"]], }, ), ) assert_ok(resp, 201, "Message") assert took < ACCEPT_MAX_S, took row = json.loads(stand("attachment-row", ready["id"])) assert row["staging_key"] is not None, row finally: stand("unpause", "s3") wait_staging_gone(stand, ready["id"]) @pytest.mark.spec( "storage-consistency", "Ключ staging другого вида не удаляется", ) def test_staging_foreign_key_kept( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: data = images.sample_png() foreign = f"{FAULT_DATABASE}/attachments/{uuid.uuid4()}/foreign" with outage(stand, "stop", "cleaner"): ready = api.upload_ready(alice, chat["id"], data) put_object(stand, foreign, data) stand("set-staging-key", ready["id"], foreign) try: deadline = time.monotonic() + 3 * CLEANER_INTERVAL_S + CLEANUP_SLACK_S while True: named = [ record for line in stand("cleaner-log").splitlines() if line.startswith("{") for record in [json.loads(line)] if record.get("event") == "Staging key is not own" and record.get("object_key") == foreign ] if named: break assert time.monotonic() < deadline, "cleaner did not name the key" time.sleep(0.3) assert named[0]["level"] == "warning", named[0] digest = hashlib.sha256(data).hexdigest() assert stand("object-sha256", foreign) == digest row = json.loads(stand("attachment-row", ready["id"])) assert row["staging_key"] == foreign, row finally: # следующий проход уберёт свой ключ, журнал стенда не засоряется stand("set-staging-key", ready["id"], staging_key(ready["id"])) stand("delete-object", foreign) @pytest.mark.spec("storage-consistency", "Объект staging без метаданных") def test_reconcile_orphan_staging(stand: Stand) -> None: data = images.sample_png() orphan = str(uuid.uuid4()) key = staging_key(orphan) other = f"{FAULT_DATABASE}/staging/{orphan}/other" put_object(stand, key, data) put_object(stand, other, data) young = run_reconcile(stand, "dry-run") assert not young.about(FOUND, keys={key, other}), young.finished assert young.finished["listed_staging"] >= 2, young.finished # старше срока хранения: срок прохода — окружением команды time.sleep(2) retention = ("--staging-retention", "1") for options in (retention, (*HEAD_STRATEGY, *retention)): old = run_reconcile(stand, "dry-run", options=options) assert old.kinds(FOUND, keys={key, other}) == { key: "orphan_staging", other: "unknown_object", }, old.finished assert old.exit_code == 3, old.finished assert stand("object-sha256", key) fixed = run_reconcile(stand, "apply", options=retention) (record,) = fixed.about(FIXED, keys={key}) assert (record["kind"], record["action"]) == ( "orphan_staging", "deleted_object", ) assert stand("object-sha256", key) == "" # чужой ключ под префиксом: назван, не тронут assert stand("object-sha256", other) stand("delete-object", other) def test_defense_faults( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: """Спек `student-tests`, «Сценарий защиты автоматизирован», шаг 4 (вне сверки покрытия — сценарий в докстринге), тем же порядком, что `driver.py defense-faults`: crash после записи объектов и после фиксации до ack, relay убит до публикации и после подтверждения до отметки, replay неудачного и отказ replay готового.""" crash_after_objects(api, stand, alice, chat) crash_after_commit(api, stand, alice, chat) with outage(stand, "kill", "relay"): accepted = api.upload(alice, chat["id"], images.sample_png()) assert queue_state(stand)["messages"] == 0 api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) crash_relay_after_confirm(api, stand, alice, chat) replay_returns_work(api, stand, alice, chat) replay_ready_refused(api, stand, alice, chat) # Предел незавершённых заданий (design change `track1-queue-overload`, решение # 13). Работает пятый API стенда: у прочих предел выключен, и они копят задания, # пока worker стоит у чужих проверок. BACKLOG_ENV = dict( item.split("=", 1) for item in os.environ.get("FAULT_BACKLOG_ENV", "").split() ) BACKLOG_LIMIT = int(BACKLOG_ENV.get("UPLOADS__MAX_UNFINISHED_JOBS", "3")) EXPECTED_BACKLOG_ENV = {"UPLOADS__MAX_UNFINISHED_JOBS": "3"} BACKLOG_RETRY_AFTER_S = "5" # обе остальные проверки загрузки на пределе — без тела и без места: быстрее BACKLOG_REJECT_MAX_S = 1 # прежние проверки могли оставить задание на следующую попытку или аренду UNFINISHED_WAIT_S = 120 # загрузки в полёте реплики: настройки по умолчанию `[uploads]` пятого API BACKLOG_IN_FLIGHT = 4 + 16 BACKLOG_CLIENTS = 40 def assert_backlog_stand() -> None: """Без малого предела пятого API тесты предела ничего не доказывают.""" assert BACKLOG_ENV == EXPECTED_BACKLOG_ENV, ( f"backlog stand started with {BACKLOG_ENV}, " f"expected {EXPECTED_BACKLOG_ENV}" ) def unfinished_jobs(stand: Stand) -> int: return int(stand("unfinished-jobs")) def wait_no_unfinished( stand: Stand, *, timeout: float = UNFINISHED_WAIT_S, ) -> None: """Незавершённых заданий нет: worker и relay работают.""" deadline = time.monotonic() + timeout while True: count = unfinished_jobs(stand) if count == 0: return assert time.monotonic() < deadline, f"{count} unfinished jobs left" time.sleep(0.5) def assert_backlog_rejected(resp: httpx.Response, elapsed: float) -> None: assert_error(resp, 429) assert resp.headers["Retry-After"] == BACKLOG_RETRY_AFTER_S assert elapsed < BACKLOG_REJECT_MAX_S, f"429 after {elapsed:.2f}s" def backlog_records(stand: Stand) -> list[dict[str, Any]]: return [ json.loads(line) for line in stand("backlog-api-log").splitlines() if line.startswith("{") ] @pytest.mark.spec("queue-overload", "Предел достигнут") @pytest.mark.spec("queue-overload", "Порядок проверок у предела") @pytest.mark.spec("queue-overload", "Остальной API у предела") @pytest.mark.spec("queue-overload", "Загрузки снова принимаются") def test_backlog_limit( backlog_api: Api, stand: Stand, alice: Actor, eve: Actor, chat: dict[str, Any], ) -> None: assert_backlog_stand() data = images.sample_png() finished = backlog_api.upload_ready(alice, chat["id"], data) wait_no_unfinished(stand) # 30 MiB, которые начинаются как PNG: Content-Length 31 457 280 big = images.PNG_SIGNATURE + bytes( 3 * images.MAX_BYTES - len(images.PNG_SIGNATURE), ) with worker_stopped(stand): accepted = [ backlog_api.upload(alice, chat["id"], data) for _ in range(BACKLOG_LIMIT) ] assert unfinished_jobs(stand) == BACKLOG_LIMIT rows, objects = objects_and_rows(stand) resp, elapsed = timed( lambda: upload_raw(backlog_api, alice, chat, data), ) assert_backlog_rejected(resp, elapsed) assert resp.headers["X-Request-Id"] assert resp.headers["X-Instance-Id"] after_rows, after_objects = objects_and_rows(stand) assert not new_ids(rows, after_rows) assert not after_objects - objects assert unfinished_jobs(stand) == BACKLOG_LIMIT # порядок проверок: сначала дешёвые, и они раньше предела assert_error(upload_raw(backlog_api, eve, chat, data), 404) too_big = upload_raw(backlog_api, alice, chat, big) assert_error(too_big, 413) assert too_big.request.headers["Content-Length"] == "31457280" assert_error( upload_raw( backlog_api, alice, chat, data, content_type="text/plain", ), 415, ) # остальной API у предела backlog_api.send_message(alice, chat["id"], "text at the limit") assert_ok( backlog_api.messages_page(alice, chat["id"]), 200, "MessagePage", ) assert ( backlog_api.attachment_meta(alice, finished["id"]).status_code == 200 ) assert_content( backlog_api.attachment_content(alice, finished["id"]), data, "image/png", ) backlog_api.send_message( alice, chat["id"], "", attachment_ids=[accepted[0]["id"]], ) ready = backlog_api.request("GET", "/health/ready") assert ready.status_code == 200, describe(ready) wait_no_unfinished(stand) more = backlog_api.upload(alice, chat["id"], data) for attachment in (*accepted, more): backlog_api.wait_attachment( alice, attachment["id"], timeout=JOB_WAIT_S, ) records = backlog_records(stand) started = next(r for r in records if r.get("event") == "Started API") assert started["uploads_max_unfinished_jobs"] == BACKLOG_LIMIT rejected = [ record for record in records if record.get("event") == "Rejected attachment" and record.get("reason") == "backlog" ] assert rejected, "no backlog rejection in the replica log" assert rejected[-1]["limit"] == BACKLOG_LIMIT assert rejected[-1]["unfinished"] >= BACKLOG_LIMIT @pytest.mark.spec("queue-overload", "Завершённые задания не считаются") def test_backlog_ignores_finished( backlog_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_backlog_stand() wait_no_unfinished(stand) data = images.sample_png() undecodable = images.jpeg_precision_12() for _ in range(BACKLOG_LIMIT): backlog_api.upload_ready(alice, chat["id"], data) for _ in range(BACKLOG_LIMIT): failed = backlog_api.upload( alice, chat["id"], undecodable, content_type="image/jpeg", ) backlog_api.wait_attachment( alice, failed["id"], status="failed", timeout=JOB_WAIT_S, ) wait_no_unfinished(stand) accepted = backlog_api.upload(alice, chat["id"], data) backlog_api.wait_attachment(alice, accepted["id"], timeout=JOB_WAIT_S) @pytest.mark.spec("queue-overload", "Ожидание следующей попытки считается") def test_backlog_counts_waiting_retry( backlog_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_backlog_stand() wait_no_unfinished(stand) data = images.sample_png() with worker_stopped(stand), relay_stopped(stand): accepted = [ backlog_api.upload(alice, chat["id"], data) for _ in range(BACKLOG_LIMIT) ] # срок следующей попытки далеко: задание ждёт, а не работает stand("set-next-attempt", accepted[0]["id"], "600") job = job_row(stand, accepted[0]["id"]) assert job is not None and job["state"] == "queued", job assert job["next_attempt_at"] is not None, job resp, elapsed = timed( lambda: upload_raw(backlog_api, alice, chat, data), ) assert_backlog_rejected(resp, elapsed) stand("set-next-attempt", accepted[0]["id"], "0") for attachment in accepted: backlog_api.wait_attachment( alice, attachment["id"], timeout=JOB_WAIT_S, ) @pytest.mark.spec("queue-overload", "Предел при остановленных брокере и relay") def test_backlog_without_broker( backlog_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_backlog_stand() wait_no_unfinished(stand) data = images.sample_png() accepted = [] with ( worker_stopped(stand), relay_stopped(stand), outage(stand, "stop", "rabbitmq"), ): for _ in range(BACKLOG_LIMIT): resp, elapsed = timed( lambda: upload_raw(backlog_api, alice, chat, data), ) accepted.append(assert_accepted(resp)) assert elapsed < BACKLOG_REJECT_MAX_S, f"202 after {elapsed:.2f}s" resp, elapsed = timed( lambda: upload_raw(backlog_api, alice, chat, data), ) assert_backlog_rejected(resp, elapsed) for attachment in accepted: backlog_api.wait_attachment( alice, attachment["id"], timeout=JOB_WAIT_S, ) @pytest.mark.spec("queue-overload", "Превышение ограничено") def test_backlog_overshoot_bounded( backlog_api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_backlog_stand() wait_no_unfinished(stand) base_url = str(backlog_api.client.base_url) data = images.sample_png() barrier = threading.Barrier(BACKLOG_CLIENTS) def upload_at_once(_: int) -> httpx.Response: with Api(base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as client: barrier.wait(timeout=30) return upload_raw(client, alice, chat, data) with worker_stopped(stand), ThreadPoolExecutor(BACKLOG_CLIENTS) as pool: answers = list(pool.map(upload_at_once, range(BACKLOG_CLIENTS))) accepted = [resp.json() for resp in answers if resp.status_code == 202] statuses = sorted({resp.status_code for resp in answers}) assert set(statuses) <= {202, 429}, statuses assert ( BACKLOG_LIMIT <= len(accepted) <= BACKLOG_LIMIT + BACKLOG_IN_FLIGHT ), f"accepted {len(accepted)}, statuses {statuses}" for resp in answers: if resp.status_code == 429: assert_error(resp, 429) for attachment in accepted: backlog_api.wait_attachment( alice, attachment["id"], timeout=JOB_WAIT_S, ) @pytest.mark.spec("queue-overload", "Предел выключен") def test_backlog_off( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: assert_backlog_stand() wait_no_unfinished(stand) data = images.sample_png() with worker_stopped(stand): accepted = [ api.upload(alice, chat["id"], data) for _ in range(BACKLOG_LIMIT + 2) ] assert unfinished_jobs(stand) == BACKLOG_LIMIT + 2 for attachment in accepted: api.wait_attachment(alice, attachment["id"], timeout=JOB_WAIT_S)