1481 lines
57 KiB
Python
1481 lines
57 KiB
Python
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
|
||
|
||
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
|
||
Valkey и S3, уборщик незавершённых загрузок (grace и интервал —
|
||
`FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API
|
||
(`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении),
|
||
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
|
||
соединения) и «узкий по загрузкам» (`FAULT_UPLOADS_BASE_URL`: одно место
|
||
загрузки, одна в очереди, ожидание 1 с, срок тела 2 с — change
|
||
`upload-memory`). На хранилища и API тесты действуют только через фикстуру
|
||
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
|
||
рабочее состояние. Решение 4 design change `failure-checks`, решение 10
|
||
design change `upload-consistency`.
|
||
"""
|
||
|
||
import hashlib
|
||
import os
|
||
import statistics
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable, Iterator
|
||
from contextlib import contextmanager
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
import images
|
||
from api_client import API, Actor, Api, new_key
|
||
from contract import (
|
||
BEARER_CHALLENGE,
|
||
assert_content,
|
||
assert_error,
|
||
assert_ok,
|
||
assert_unauthorized,
|
||
)
|
||
|
||
pytestmark = pytest.mark.fault
|
||
|
||
type Stand = Callable[..., str]
|
||
|
||
RESTORE = {"stop": "start", "pause": "unpause"}
|
||
# запас к границам времени: планировщик Docker Desktop
|
||
SLACK_S = 0.3
|
||
# [cache] timeout_seconds и retry_after_seconds по умолчанию
|
||
CACHE_TIMEOUT_S = 0.2
|
||
CACHE_RETRY_AFTER_S = 5
|
||
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
|
||
SESSION_TTL_S = 2
|
||
IDEMPOTENCY_TTL_S = 2
|
||
POOL_TIMEOUT_S = 1
|
||
# срок проверки Postgres в /health/ready
|
||
READINESS_TIMEOUT_S = 2
|
||
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
|
||
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
|
||
OBJECT_STORAGE_HEALTH_S = 1
|
||
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
|
||
S3_TIMEOUT_S = 5
|
||
# сколько секунд hook держит слоты max_connections; clear-connections
|
||
# освобождает их раньше
|
||
FILL_HOLD_S = 30
|
||
ME = f"{API}/users/me"
|
||
|
||
|
||
@contextmanager
|
||
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
||
"""`stop` или `pause` хранилища; в `finally` — обратно."""
|
||
stand(action, service)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand(RESTORE[action], service)
|
||
|
||
|
||
def timed(
|
||
call: Callable[[], httpx.Response],
|
||
) -> tuple[httpx.Response, float]:
|
||
started = time.monotonic()
|
||
resp = call()
|
||
return resp, time.monotonic() - started
|
||
|
||
|
||
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
|
||
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
|
||
|
||
|
||
def wait_cache_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
||
|
||
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
||
"""
|
||
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
|
||
stand("flush-cache")
|
||
while time.monotonic() < deadline:
|
||
assert_ok(read_chat(api, actor, chat), 200, "Chat")
|
||
if stand("cache-has-chat", chat["id"]) == "1":
|
||
return
|
||
time.sleep(0.5)
|
||
raise AssertionError("cache did not come back")
|
||
|
||
|
||
def chat_titles(api: Api, actor: Actor) -> set[str]:
|
||
return {
|
||
chat["title"]
|
||
for page in api.chat_pages(actor)
|
||
for chat in page["items"]
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Истёкший токен")
|
||
def test_expired_token(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
time.sleep(SESSION_TTL_S + 1)
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Срок не продлевается")
|
||
def test_use_does_not_extend_session(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
logged_in = time.monotonic()
|
||
successes = 0
|
||
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
successes += 1
|
||
time.sleep(0.5)
|
||
assert successes >= 3, "too few requests before expiry"
|
||
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
|
||
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
|
||
actor = api.register()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.request("GET", ME, actor=actor)
|
||
assert_error(resp, 503)
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Хранение: откат, остановленная БД, исчерпанный пул
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
|
||
def test_members_failure_rolls_back_chat(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
) -> None:
|
||
title = f"fault-{uuid.uuid4().hex}"
|
||
stand("fail-members", title)
|
||
try:
|
||
resp = api.request(
|
||
"POST",
|
||
f"{API}/chats",
|
||
actor=alice,
|
||
json={"title": title, "member_ids": [alice.id, bob.id]},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-members")
|
||
for actor in (alice, bob):
|
||
assert title not in chat_titles(api, actor)
|
||
assert stand("count-chats", title) == "0"
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "БД остановлена")
|
||
@pytest.mark.spec("data-storage", "БД вернулась")
|
||
def test_database_stopped_and_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
error = assert_error(resp, 503)
|
||
assert took < 2, f"503 took {took:.2f}s"
|
||
for word in ("postgres", "sql", "connect"):
|
||
assert word not in error["message"].lower()
|
||
message = api.send_message(alice, chat["id"], "после возврата")
|
||
# запись во время отказа не зафиксирована, новая — видна
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
|
||
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
holder = Api(str(tight_api.client.base_url), timeout=15)
|
||
first: list[int] = []
|
||
|
||
def hold_connection() -> None:
|
||
first.append(holder.request("GET", ME, actor=actor).status_code)
|
||
|
||
with outage(stand, "pause", "postgres"):
|
||
thread = threading.Thread(target=hold_connection)
|
||
thread.start()
|
||
time.sleep(SLACK_S)
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
assert_error(resp, 503)
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
thread.join()
|
||
holder.close()
|
||
assert first == [503]
|
||
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
|
||
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
# hook оборвал соединение пула: при выдаче pool_pre_ping
|
||
# переподключается и получает «too many clients» (SQLSTATE 53300)
|
||
stand("fill-connections", str(FILL_HOLD_S))
|
||
try:
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
error = assert_error(resp, 503)
|
||
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
for word in ("postgres", "sql", "client", "connect"):
|
||
assert word not in error["message"].lower()
|
||
finally:
|
||
stand("clear-connections")
|
||
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
|
||
# сессии 2 с за время заполнения истёк — новый вход
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
# пачка одновременных запросов больше пула обычного API (10)
|
||
BURST_THREADS = 48
|
||
BURST_AFTER_BARRIER = 4
|
||
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
|
||
STATS_FLUSH_S = 11
|
||
|
||
|
||
def burst(api: Api, actor: Actor) -> None:
|
||
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
|
||
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
|
||
одновременно. Все ответы — 200."""
|
||
barrier = threading.Barrier(BURST_THREADS)
|
||
clients = [
|
||
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
|
||
]
|
||
responses: list[httpx.Response] = []
|
||
errors: list[BaseException] = []
|
||
lock = threading.Lock()
|
||
|
||
def worker(client: Api) -> None:
|
||
try:
|
||
got = [client.request("GET", ME, actor=actor)]
|
||
barrier.wait()
|
||
got += [
|
||
client.request("GET", ME, actor=actor)
|
||
for _ in range(BURST_AFTER_BARRIER)
|
||
]
|
||
except BaseException as e: # noqa: BLE001 — поднимется в тесте
|
||
barrier.abort()
|
||
with lock:
|
||
errors.append(e)
|
||
return
|
||
with lock:
|
||
responses.extend(got)
|
||
|
||
threads = [
|
||
threading.Thread(target=worker, args=(client,)) for client in clients
|
||
]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
for client in clients:
|
||
client.close()
|
||
assert not errors, f"burst failed: {errors[0]!r}"
|
||
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
|
||
for resp in responses:
|
||
assert_ok(resp, 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
|
||
def test_pool_keeps_connections(
|
||
api: Api,
|
||
tight_api: Api,
|
||
stand: Stand,
|
||
) -> None:
|
||
actor = api.register()
|
||
# соединение «тесного» API могли оборвать прежние тесты
|
||
# (fill-connections): его переподключение не должно попасть в окно
|
||
tight = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
|
||
# прогрев: пул обычного API набирает свой размер
|
||
for _ in range(2):
|
||
burst(api, actor)
|
||
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
|
||
# в общую статистику без ожидания блокировки и при занятой блокировке
|
||
# откладываются. Простаивающий backend сбрасывает их принудительно по
|
||
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
|
||
# попадали в окно по одному
|
||
time.sleep(STATS_FLUSH_S)
|
||
# калибровка: второй вызов видит сессию первого вызова hook, и только её
|
||
s1 = int(stand("db-sessions"))
|
||
s2 = int(stand("db-sessions"))
|
||
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
|
||
for _ in range(3):
|
||
burst(api, actor)
|
||
s3 = int(stand("db-sessions"))
|
||
opened = s3 - s2 - 1
|
||
assert opened == 0, (
|
||
f"PostgreSQL accepted {opened} new sessions during "
|
||
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Readiness
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище не готово")
|
||
def test_readiness_without_database(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(lambda: api.request("GET", "/health/ready"))
|
||
assert_error(resp, 503)
|
||
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
|
||
live = api.request("GET", "/health/live")
|
||
assert live.status_code == 200
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Кэш недоступен")
|
||
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище объектов недоступно")
|
||
@pytest.mark.parametrize("action", ["stop", "pause"])
|
||
def test_readiness_without_object_storage(
|
||
api: Api,
|
||
stand: Stand,
|
||
action: str,
|
||
) -> None:
|
||
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
|
||
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
|
||
with outage(stand, action, "s3"):
|
||
for path in ("/health/ready", "/health/live"):
|
||
resp, took = timed(lambda path=path: api.request("GET", path))
|
||
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
|
||
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Кэш карточки чата
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Очистка кэша")
|
||
def test_flush_cache(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
stand("flush-cache")
|
||
assert stand("cache-has-chat", chat["id"]) == "0"
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey остановлен")
|
||
def test_valkey_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
sent = [api.send_message(alice, chat["id"], "до отказа")]
|
||
with outage(stand, "stop", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
sent.append(assert_ok(resp, 201, "Message"))
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
|
||
assert api.all_messages(bob, chat["id"]) == sent
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey завис")
|
||
def test_valkey_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
baseline = statistics.median(
|
||
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
|
||
)
|
||
with outage(stand, "pause", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
|
||
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
||
)
|
||
# окно пропуска: следующий запрос таймаута не ждёт
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey вернулся")
|
||
def test_valkey_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
# API не перезапускался; кэш снова в работе после окна пропуска
|
||
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
hits = int(stand("cache-hits"))
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert int(stand("cache-hits")) > hits
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
|
||
def test_message_failure_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
# ключ вставляется первым: триггер роняет транзакцию уже после него
|
||
text = f"fault-{uuid.uuid4().hex}"
|
||
key = new_key()
|
||
stand("fail-messages", text)
|
||
try:
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-messages")
|
||
# ключ не занят: повтор выполняется заново, а не 500 и не 409
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
|
||
def test_database_outage_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
text = "при отказе БД"
|
||
key = new_key()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 503)
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Ключ после срока")
|
||
def test_key_free_after_ttl(tight_api: Api) -> None:
|
||
alice = tight_api.register()
|
||
chat = tight_api.create_chat(alice)
|
||
key = new_key()
|
||
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
|
||
# в пределах срока ключ занят: другое тело — 409
|
||
resp = tight_api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": "после срока"},
|
||
)
|
||
assert_error(resp, 409)
|
||
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
|
||
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
|
||
alice = tight_api.login(alice.user, alice.password)
|
||
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
|
||
assert second["id"] != first["id"]
|
||
history = tight_api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [first["id"], second["id"]]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки
|
||
# и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает
|
||
# строки прежних тестов параллельно (design change upload-consistency,
|
||
# решение 10)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# grace и интервал уборщика стенда отказов (run_failures.py,
|
||
# failures.compose.yml)
|
||
CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20"))
|
||
CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1"))
|
||
# запас поверх grace и интервала: проход, вызов hook, планировщик Docker
|
||
# Desktop
|
||
CLEANUP_SLACK_S = 5
|
||
# задержка перевода в ready триггером stall (fault_hook.py)
|
||
STALL_S = 4
|
||
# срок клиента загрузок в потоке
|
||
THREAD_CLIENT_TIMEOUT_S = 30
|
||
|
||
|
||
def attachment_rows(stand: Stand) -> dict[str, str]:
|
||
"""Вложения стенда: `id` → статус."""
|
||
return dict(line.split() for line in stand("attachment-ids").splitlines())
|
||
|
||
|
||
def stored_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений, у которых есть объект в bucket стенда."""
|
||
return set(stand("object-ids").splitlines())
|
||
|
||
|
||
def orphan_objects(stand: Stand) -> set[str]:
|
||
"""Объекты вложений без строки. Сначала строки, потом объекты: уборщик
|
||
удаляет объект раньше строки, и убранная между вызовами строка не
|
||
даёт ложного «объекта без строки»."""
|
||
rows = attachment_rows(stand)
|
||
return stored_ids(stand) - set(rows)
|
||
|
||
|
||
def ready_without_object(stand: Stand) -> set[str]:
|
||
"""Готовые вложения без объекта: строки `ready` уборщик не трогает,
|
||
поэтому порядок вызовов здесь не важен."""
|
||
return ready_ids(attachment_rows(stand)) - stored_ids(stand)
|
||
|
||
|
||
def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]:
|
||
return set(after) - set(before)
|
||
|
||
|
||
def ready_ids(rows: dict[str, str]) -> set[str]:
|
||
return {item for item, status in rows.items() if status == "ready"}
|
||
|
||
|
||
def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None:
|
||
"""Строка и объект вложения убраны не позже grace + интервал + запас
|
||
от `since` (момент до загрузки: `created_at` строки не раньше)."""
|
||
deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while True:
|
||
rows = attachment_rows(stand)
|
||
stored = stored_ids(stand)
|
||
if attachment_id not in rows and attachment_id not in stored:
|
||
return
|
||
if time.monotonic() > deadline:
|
||
raise AssertionError(
|
||
f"attachment {attachment_id} is not cleaned "
|
||
f"{time.monotonic() - since:.1f}s after upload: "
|
||
f"row {rows.get(attachment_id)}, "
|
||
f"object {attachment_id in stored}",
|
||
)
|
||
time.sleep(0.5)
|
||
|
||
|
||
def sleep_until(moment: float) -> None:
|
||
time.sleep(max(0.0, moment - time.monotonic()))
|
||
|
||
|
||
def fault_filename() -> str:
|
||
"""Уникальное имя загрузки, на которое срабатывает триггер hook."""
|
||
return f"fault-{uuid.uuid4().hex}.png"
|
||
|
||
|
||
def upload_raw(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: Any,
|
||
**kwargs: Any,
|
||
) -> httpx.Response:
|
||
return api.post_attachment(actor, chat["id"], data, **kwargs)
|
||
|
||
|
||
def assert_invisible(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
attachment_id: str,
|
||
) -> None:
|
||
"""Вложение не в статусе ready: метаданные, байты и ссылка в сообщении
|
||
— 404 с тем же `message`, что у неизвестного `id`."""
|
||
|
||
def calls(item: str) -> tuple[httpx.Response, ...]:
|
||
return (
|
||
api.attachment_meta(actor, item),
|
||
api.attachment_content(actor, item),
|
||
api.post_message(
|
||
actor,
|
||
chat["id"],
|
||
json={"text": "ссылка", "attachment_ids": [item]},
|
||
),
|
||
)
|
||
|
||
unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))]
|
||
hidden = [assert_error(resp, 404) for resp in calls(attachment_id)]
|
||
assert [error["message"] for error in hidden] == [
|
||
error["message"] for error in unknown
|
||
]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
|
||
@pytest.mark.spec("attachments", "Незавершённая загрузка не видна")
|
||
def test_upload_with_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
before = attachment_rows(stand)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
after = attachment_rows(stand)
|
||
# строка-намерение осталась, готовых не прибавилось; её уберёт
|
||
# уборщик (test_cleaner_survives_outages)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert ready_ids(after) == ready_ids(before)
|
||
# замечание S6 verify attachments: id последней pending — от hook
|
||
assert stand("last-pending") == pending
|
||
assert_invisible(api, alice, chat, pending)
|
||
api.send_message(alice, chat["id"], "текст без S3")
|
||
api.upload(alice, chat["id"], data)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
|
||
def test_rejected_uploads_leave_nothing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
|
||
|
||
def chunked() -> Iterator[bytes]:
|
||
for offset in range(0, len(big), 1024 * 1024):
|
||
yield big[offset : offset + 1024 * 1024]
|
||
|
||
# хотя бы одно готовое вложение: списки hook не пусты случайно
|
||
ready = api.upload(alice, chat["id"], images.sample_png())
|
||
before = (attachment_rows(stand), stored_ids(stand))
|
||
assert before[0].get(ready["id"]) == "ready", before[0]
|
||
assert ready["id"] in before[1], before[1]
|
||
rejected = (
|
||
(404, upload_raw(api, eve, chat, images.sample_png())),
|
||
(413, upload_raw(api, alice, chat, big)),
|
||
(413, upload_raw(api, alice, chat, chunked())),
|
||
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
|
||
(
|
||
415,
|
||
upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
|
||
)
|
||
for status, resp in rejected:
|
||
assert_error(resp, status)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not stored_ids(stand) - before[1]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Сбой записи метаданных")
|
||
def test_metadata_insert_failure(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = (attachment_rows(stand), stored_ids(stand))
|
||
stand("fail-attachments", "insert", filename)
|
||
try:
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строки нет — объект не записывался
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not stored_ids(stand) - before[1]
|
||
|
||
|
||
class BodyAborted(Exception):
|
||
"""Клиент обрывает передачу тела загрузки."""
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Обрыв передачи тела")
|
||
def test_interrupted_body(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = (attachment_rows(stand), stored_ids(stand))
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
# свой клиент: его соединение обрывается вместе с телом
|
||
with (
|
||
Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно; строки до конца тела нет в любом
|
||
# случае
|
||
time.sleep(1)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not stored_ids(stand) - before[1]
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Фиксация ready не удалась")
|
||
@pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_ready_failure_compensated(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "ready", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert stand("last-pending") == pending
|
||
# компенсация: объект удалён до ответа 503
|
||
assert pending not in stored_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
# моложе grace строку уборщик не трогает: к половине grace он сделал
|
||
# около G / 2 / I проходов
|
||
sleep_until(started + CLEANER_GRACE_S / 2)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект без фиксации убран")
|
||
def test_killed_between_put_and_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
outcome: dict[str, Any] = {}
|
||
client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
|
||
def upload() -> None:
|
||
try:
|
||
outcome["status"] = upload_raw(
|
||
client,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
).status_code
|
||
except httpx.TransportError as e:
|
||
outcome["error"] = e
|
||
|
||
stand("fail-attachments", "stall", filename)
|
||
killed = False
|
||
try:
|
||
started = time.monotonic()
|
||
thread = threading.Thread(target=upload)
|
||
thread.start()
|
||
# строка pending и объект — до конца задержки перевода в ready
|
||
deadline = started + STALL_S
|
||
pending = ""
|
||
while time.monotonic() < deadline:
|
||
pending = stand("last-pending")
|
||
if (
|
||
pending
|
||
and pending not in before
|
||
and pending in stored_ids(stand)
|
||
):
|
||
break
|
||
time.sleep(0.05)
|
||
else:
|
||
raise AssertionError("no pending row with object during stall")
|
||
stand("kill-api")
|
||
killed = True
|
||
thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not thread.is_alive()
|
||
# 201 до клиента не дошёл: перевод в ready ждал в триггере
|
||
assert outcome.get("status") != 201, outcome
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert pending in stored_ids(stand)
|
||
finally:
|
||
client.close()
|
||
stand("clear-fail-attachments")
|
||
if killed:
|
||
stand("start-api")
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Исход фиксации неизвестен")
|
||
def test_ready_commit_outcome_unknown(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Перевод в ready падает на самом COMMIT (отложенный триггер): API не
|
||
знает, зафиксировано ли вложение, и объект не трогает (verify W2)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
# компенсации нет: вложение могло стать ready
|
||
assert pending in stored_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Загрузка дольше grace")
|
||
def test_upload_outlives_grace(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Строка вставлена «на час старше», перевод в ready ждёт в триггере,
|
||
пока её не уберёт уборщик стенда: загрузка, пережившая grace, без
|
||
заморозки процесса (verify W3)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "outlive", filename)
|
||
# ожидание в триггере до 4,5 с: срок клиента больше
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
# не 201 (ready без объекта) и не 500 (дефект ветки)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
rows = attachment_rows(stand)
|
||
assert not new_ids(before, rows), rows
|
||
assert not stored_ids(stand) - set(before)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта")
|
||
def test_cleaner_failure_after_object_delete(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Уборщик удалил объект, а удаление строки упало (08006), пока загрузка
|
||
ждёт перевода в ready: строка помечена до удаления объекта и ready уже
|
||
не станет (verify W1)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "undeletable", filename)
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert not ready_without_object(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строка старше grace: следующий проход доводит уборку
|
||
returned = time.monotonic()
|
||
wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S)
|
||
assert not orphan_objects(stand)
|
||
assert not ready_without_object(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "БД недоступна во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты")
|
||
def test_cleaner_survives_outages(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
assert_error(upload_raw(api, alice, chat, data), 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
# PostgreSQL недоступен, пока загрузка моложе grace
|
||
with outage(stand, "stop", "postgres"):
|
||
time.sleep(3)
|
||
# grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте
|
||
sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
# S3 вернулся: уборщик без перезапуска убирает строку за проход
|
||
returned = time.monotonic()
|
||
deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while pending in attachment_rows(stand):
|
||
assert time.monotonic() < deadline, (
|
||
f"pending {pending} is still there "
|
||
f"{time.monotonic() - returned:.1f}s after S3 came back"
|
||
)
|
||
time.sleep(0.5)
|
||
assert pending not in stored_ids(stand)
|
||
assert not orphan_objects(stand)
|
||
# готовое вложение старше grace на месте
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, ready["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == ready
|
||
assert meta["status"] == "ready"
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 остановлен")
|
||
def test_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(
|
||
lambda: api.attachment_content(bob, attachment["id"]),
|
||
)
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
meta = assert_ok(
|
||
api.attachment_meta(bob, attachment["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == attachment
|
||
message = api.send_message(
|
||
bob,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment["id"]],
|
||
)
|
||
# история с attachment_ids — из PostgreSQL, S3 ей не нужен
|
||
history = api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [message["id"]]
|
||
assert history[0]["attachment_ids"] == [attachment["id"]]
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
assert_content(
|
||
api.attachment_content(bob, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
def test_environment_does_not_change_limits(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `service-configuration`, «Окружение не меняет лимит» (вне
|
||
сверки покрытия — сценарий в докстринге): обычный API стенда
|
||
отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1`
|
||
(`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт,
|
||
64 000 пикселей) всё равно принимается: лимиты — константы."""
|
||
limits = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_LIMITS_ENV", "").split()
|
||
)
|
||
# без этих переменных на стенде тест ничего не доказывает
|
||
assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, (
|
||
f"stand started without limit variables: {limits}"
|
||
)
|
||
api.upload(alice, chat["id"], images.sample_png(), filename="sample.png")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 завис")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_object_storage_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload(alice, chat["id"], data)
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
|
||
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
|
||
with slow, outage(stand, "pause", "s3"):
|
||
for call in (
|
||
lambda: upload_raw(slow, alice, chat, data),
|
||
lambda: slow.attachment_content(alice, attachment["id"]),
|
||
):
|
||
resp, elapsed = timed(call)
|
||
assert_error(resp, 503)
|
||
# срок вызова S3 и немного на шаги с БД, а не зависание
|
||
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
|
||
f"503 after {elapsed:.2f}s"
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
# загрузка во время заморозки оставила строку pending; объект после
|
||
# разморозки не дописался, строку уборщик убирает после grace
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Места загрузки и срок тела (change upload-memory, design, решение 11):
|
||
# «узкий по загрузкам» API — одно место, одна загрузка в очереди, ожидание
|
||
# места и срок тела из FAULT_UPLOADS_ENV (run_failures.py, UPLOADS_ENV).
|
||
# Место держит загрузка, тело которой стоит на threading.Event теста
|
||
# ---------------------------------------------------------------------------
|
||
|
||
UPLOADS_ENV = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_UPLOADS_ENV", "").split()
|
||
)
|
||
# окружение, с которым тесты ниже ждут «узкий» API
|
||
EXPECTED_UPLOADS_ENV = {
|
||
"UPLOADS__MAX_CONCURRENT": "1",
|
||
"UPLOADS__MAX_WAITING": "1",
|
||
"UPLOADS__WAIT_TIMEOUT_SECONDS": "1",
|
||
"UPLOADS__BODY_TIMEOUT_SECONDS": "2",
|
||
}
|
||
WAIT_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__WAIT_TIMEOUT_SECONDS", "1"))
|
||
BODY_TIMEOUT_S = float(UPLOADS_ENV.get("UPLOADS__BODY_TIMEOUT_SECONDS", "2"))
|
||
# отказ без ожидания места: вход, членство и ответ
|
||
IMMEDIATE_S = 0.5
|
||
# пауза, за которую запрос в соседнем потоке доходит до места или очереди
|
||
SETTLE_S = 0.2
|
||
# сколько держатель места ждёт отпускания, если тест упал раньше
|
||
HOLD_MAX_S = 10
|
||
|
||
|
||
def assert_uploads_stand() -> None:
|
||
"""Без этого окружения «узкого» API тесты мест ничего не доказывают."""
|
||
assert UPLOADS_ENV == EXPECTED_UPLOADS_ENV, (
|
||
f"uploads stand started with {UPLOADS_ENV}, "
|
||
f"expected {EXPECTED_UPLOADS_ENV}"
|
||
)
|
||
|
||
|
||
class Upload:
|
||
"""Загрузка в своём потоке и своём соединении; ответ — `result()`."""
|
||
|
||
def __init__(self, api: Api, actor: Actor, chat: dict[str, Any]) -> None:
|
||
self._base_url = str(api.client.base_url)
|
||
self._actor = actor
|
||
self._chat = chat
|
||
self._outcome: dict[str, Any] = {}
|
||
self._thread: threading.Thread | None = None
|
||
|
||
def start(self, data: Any, **kwargs: Any) -> "Upload":
|
||
def run() -> None:
|
||
with Api(self._base_url, timeout=THREAD_CLIENT_TIMEOUT_S) as own:
|
||
started = time.monotonic()
|
||
try:
|
||
self._outcome["resp"] = upload_raw(
|
||
own,
|
||
self._actor,
|
||
self._chat,
|
||
data,
|
||
**kwargs,
|
||
)
|
||
except (BodyAborted, httpx.TransportError) as e:
|
||
self._outcome["error"] = e
|
||
self._outcome["elapsed"] = time.monotonic() - started
|
||
self._outcome["ended"] = time.monotonic()
|
||
|
||
self._thread = threading.Thread(target=run)
|
||
self._thread.start()
|
||
return self
|
||
|
||
def result(self) -> tuple[httpx.Response, float, float]:
|
||
"""Ответ, длительность и момент ответа (`time.monotonic`)."""
|
||
assert self._thread is not None
|
||
self._thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not self._thread.is_alive(), "upload thread hangs"
|
||
assert "resp" in self._outcome, self._outcome
|
||
return (
|
||
self._outcome["resp"],
|
||
self._outcome["elapsed"],
|
||
self._outcome["ended"],
|
||
)
|
||
|
||
|
||
@contextmanager
|
||
def held_slot(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: bytes,
|
||
) -> Iterator[Callable[[], httpx.Response]]:
|
||
"""Загрузка A держит место: отдаёт половину тела и ждёт отпускания.
|
||
|
||
Внутри — после того, как A заняла место; вызов отдаваемой функции
|
||
отпускает A и возвращает её ответ. В `finally` A отпускается всегда.
|
||
"""
|
||
sent = threading.Event()
|
||
release = threading.Event()
|
||
|
||
def body() -> Iterator[bytes]:
|
||
yield data[: len(data) // 2]
|
||
sent.set()
|
||
if not release.wait(HOLD_MAX_S):
|
||
raise BodyAborted
|
||
yield data[len(data) // 2 :]
|
||
|
||
holder = Upload(api, actor, chat).start(body())
|
||
try:
|
||
assert sent.wait(HOLD_MAX_S), "holder did not start its body"
|
||
# вход и членство — короткие шаги БД, затем место
|
||
time.sleep(SETTLE_S)
|
||
|
||
def finish() -> httpx.Response:
|
||
release.set()
|
||
return holder.result()[0]
|
||
|
||
yield finish
|
||
finally:
|
||
release.set()
|
||
|
||
|
||
def slow_chunks(data: bytes, *, parts: int, seconds: float) -> Iterator[bytes]:
|
||
"""Тело `parts` частями, растянутое на `seconds`."""
|
||
size = -(-len(data) // parts)
|
||
for offset in range(0, len(data), size):
|
||
if offset:
|
||
time.sleep(seconds / (parts - 1))
|
||
yield data[offset : offset + size]
|
||
|
||
|
||
def objects_and_rows(stand: Stand) -> tuple[dict[str, str], set[str]]:
|
||
return attachment_rows(stand), stored_ids(stand)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Все места заняты")
|
||
@pytest.mark.spec("attachments", "Очередь ожидания полна")
|
||
@pytest.mark.spec("api-conventions", "Лимит одновременных загрузок")
|
||
def test_upload_slots_busy(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
before = objects_and_rows(stand)
|
||
with held_slot(uploads_api, alice, chat, data) as finish_holder:
|
||
# B ждёт место в очереди, C застаёт полную очередь
|
||
waiting = Upload(uploads_api, alice, chat).start(data)
|
||
time.sleep(SETTLE_S)
|
||
busy, busy_s = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, data),
|
||
)
|
||
waited, waited_s, _ = waiting.result()
|
||
# A отпускается сразу: её тело ограничено сроком тела стенда
|
||
holder = finish_holder()
|
||
assert_error(busy, 429)
|
||
assert busy_s < IMMEDIATE_S, f"429 of a full queue after {busy_s:.2f}s"
|
||
assert_error(waited, 429)
|
||
assert waited_s >= WAIT_TIMEOUT_S, f"429 after {waited_s:.2f}s"
|
||
for resp in (busy, waited):
|
||
assert int(resp.headers["Retry-After"]) >= 1
|
||
attachment = assert_ok(holder, 201, "Attachment")
|
||
# из трёх загрузок осталась только принятая: у 429 ни строки, ни объекта
|
||
rows, stored = objects_and_rows(stand)
|
||
assert new_ids(before[0], rows) == {attachment["id"]}
|
||
assert stored - before[1] == {attachment["id"]}
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отказ раньше очереди")
|
||
def test_rejections_before_queue(
|
||
uploads_api: Api,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
# 30 MiB, которые начинаются как PNG: Content-Length 31 457 280
|
||
big = images.PNG_SIGNATURE + bytes(
|
||
3 * images.MAX_BYTES - len(images.PNG_SIGNATURE),
|
||
)
|
||
with held_slot(uploads_api, alice, chat, data) as finish_holder:
|
||
waiting = Upload(uploads_api, alice, chat).start(data)
|
||
time.sleep(SETTLE_S)
|
||
early = [
|
||
(status, *timed(call))
|
||
for status, call in (
|
||
(404, lambda: upload_raw(uploads_api, eve, chat, data)),
|
||
(413, lambda: upload_raw(uploads_api, alice, chat, big)),
|
||
(
|
||
415,
|
||
lambda: upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
data,
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
)
|
||
]
|
||
early_done = time.monotonic()
|
||
waited, _, waited_at = waiting.result()
|
||
holder = finish_holder()
|
||
# очередь была полна всё время ранних отказов
|
||
assert_error(waited, 429)
|
||
assert early_done < waited_at
|
||
for status, resp, elapsed in early:
|
||
assert_error(resp, status)
|
||
# без ожидания места: ожидающий получил бы его не раньше срока
|
||
assert elapsed < WAIT_TIMEOUT_S, f"{status} after {elapsed:.2f}s"
|
||
assert early[1][1].request.headers["Content-Length"] == "31457280"
|
||
assert_ok(holder, 201, "Attachment")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Ожидание места")
|
||
@pytest.mark.spec(
|
||
"storage-consistency",
|
||
"Ожидание места не входит в возраст загрузки",
|
||
)
|
||
def test_upload_waits_for_slot(
|
||
uploads_api: Api,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
first = images.sample_png()
|
||
second = images.png(80, 60)
|
||
with held_slot(uploads_api, alice, chat, first) as finish_holder:
|
||
waiting = Upload(uploads_api, bob, chat).start(second)
|
||
time.sleep(SETTLE_S)
|
||
# A заканчивается раньше срока ожидания B
|
||
holder = finish_holder()
|
||
held = assert_ok(holder, 201, "Attachment")
|
||
waited_resp, waited_s, _ = waiting.result()
|
||
waited = assert_ok(waited_resp, 201, "Attachment")
|
||
assert waited_s < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, waited_s
|
||
# метаданные ожидавшей записаны после перевода в ready державшей место:
|
||
# ожидание прошло до строки pending (граница B уборщика прежняя)
|
||
assert waited["created_at"] > held["updated_at"], (held, waited)
|
||
for attachment, data in ((held, first), (waited, second)):
|
||
assert_content(
|
||
uploads_api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Место освобождается")
|
||
def test_slot_released_after_failures(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
over = images.png_of_size(images.MAX_BYTES + 1)
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
def next_upload_gets_slot(after: str) -> None:
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, images.sample_png()),
|
||
)
|
||
assert_ok(resp, 201, "Attachment")
|
||
assert elapsed < WAIT_TIMEOUT_S, (
|
||
f"upload after {after} waited {elapsed:.2f}s for a slot"
|
||
)
|
||
|
||
assert_error(
|
||
upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
slow_chunks(over, parts=10, seconds=0),
|
||
),
|
||
413,
|
||
)
|
||
next_upload_gets_slot("413 by counter")
|
||
assert_error(upload_raw(uploads_api, alice, chat, b"not an image"), 415)
|
||
next_upload_gets_slot("415 by content")
|
||
slow = slow_chunks(
|
||
images.sample_png(),
|
||
parts=4,
|
||
seconds=BODY_TIMEOUT_S + 1,
|
||
)
|
||
assert_error(upload_raw(uploads_api, alice, chat, slow), 400)
|
||
next_upload_gets_slot("400 by body timeout")
|
||
with (
|
||
Api(
|
||
str(uploads_api.client.base_url),
|
||
timeout=THREAD_CLIENT_TIMEOUT_S,
|
||
) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно
|
||
time.sleep(SETTLE_S)
|
||
next_upload_gets_slot("interrupted body")
|
||
with outage(stand, "stop", "s3"):
|
||
assert_error(
|
||
upload_raw(uploads_api, alice, chat, images.sample_png()),
|
||
503,
|
||
)
|
||
next_upload_gets_slot("503 with S3 stopped")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Медленный клиент")
|
||
@pytest.mark.spec("attachments", "Медленная передача в срок")
|
||
def test_slow_body(
|
||
uploads_api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
assert_uploads_stand()
|
||
data = images.sample_png()
|
||
before = objects_and_rows(stand)
|
||
late = upload_raw(
|
||
uploads_api,
|
||
alice,
|
||
chat,
|
||
slow_chunks(data, parts=6, seconds=BODY_TIMEOUT_S + 1),
|
||
)
|
||
error = assert_error(late, 400)
|
||
assert error["message"] == "Upload body was not received in time"
|
||
rows, stored = objects_and_rows(stand)
|
||
assert not new_ids(before[0], rows)
|
||
assert not stored - before[1]
|
||
# мелкие части с паузами, но в срок: тело принято как есть
|
||
in_time = slow_chunks(data, parts=20, seconds=BODY_TIMEOUT_S / 2)
|
||
resp, elapsed = timed(
|
||
lambda: upload_raw(uploads_api, alice, chat, in_time),
|
||
)
|
||
attachment = assert_ok(resp, 201, "Attachment")
|
||
assert elapsed < WAIT_TIMEOUT_S + BODY_TIMEOUT_S, elapsed
|
||
assert attachment["size_bytes"] == len(data)
|
||
assert attachment["sha256"] == hashlib.sha256(data).hexdigest()
|
||
assert_content(
|
||
uploads_api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|