Mark a stale pending row in its own committed transaction before the cleaner deletes the object, and let mark_ready promote only unmarked rows, so a cleaner rollback after the object delete can no longer leave a ready attachment without its object. Add fault checks for an unknown COMMIT outcome, an upload outliving the grace period and a cleaner failure after the object delete; fix the cleaner entrypoint in rules. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1133 lines
43 KiB
Python
1133 lines
43 KiB
Python
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
|
||
|
||
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
|
||
Valkey и S3, уборщик незавершённых загрузок (grace и интервал —
|
||
`FAULT_CLEANER_GRACE_S`, `FAULT_CLEANER_INTERVAL_S`), обычный API
|
||
(`BASE_URL`, с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1` в окружении) и
|
||
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
|
||
соединения). На хранилища и API тесты действуют только через фикстуру
|
||
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
|
||
рабочее состояние. Решение 4 design change `failure-checks`, решение 10
|
||
design change `upload-consistency`.
|
||
"""
|
||
|
||
import os
|
||
import statistics
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable, Iterator
|
||
from contextlib import contextmanager
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
import images
|
||
from api_client import API, Actor, Api, new_key
|
||
from contract import (
|
||
BEARER_CHALLENGE,
|
||
assert_content,
|
||
assert_error,
|
||
assert_ok,
|
||
assert_unauthorized,
|
||
)
|
||
|
||
pytestmark = pytest.mark.fault
|
||
|
||
type Stand = Callable[..., str]
|
||
|
||
RESTORE = {"stop": "start", "pause": "unpause"}
|
||
# запас к границам времени: планировщик Docker Desktop
|
||
SLACK_S = 0.3
|
||
# [cache] timeout_seconds и retry_after_seconds по умолчанию
|
||
CACHE_TIMEOUT_S = 0.2
|
||
CACHE_RETRY_AFTER_S = 5
|
||
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
|
||
SESSION_TTL_S = 2
|
||
IDEMPOTENCY_TTL_S = 2
|
||
POOL_TIMEOUT_S = 1
|
||
# срок проверки Postgres в /health/ready
|
||
READINESS_TIMEOUT_S = 2
|
||
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
|
||
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
|
||
OBJECT_STORAGE_HEALTH_S = 1
|
||
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
|
||
S3_TIMEOUT_S = 5
|
||
# сколько секунд hook держит слоты max_connections; clear-connections
|
||
# освобождает их раньше
|
||
FILL_HOLD_S = 30
|
||
ME = f"{API}/users/me"
|
||
|
||
|
||
@contextmanager
|
||
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
||
"""`stop` или `pause` хранилища; в `finally` — обратно."""
|
||
stand(action, service)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand(RESTORE[action], service)
|
||
|
||
|
||
def timed(
|
||
call: Callable[[], httpx.Response],
|
||
) -> tuple[httpx.Response, float]:
|
||
started = time.monotonic()
|
||
resp = call()
|
||
return resp, time.monotonic() - started
|
||
|
||
|
||
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
|
||
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
|
||
|
||
|
||
def wait_cache_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
||
|
||
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
||
"""
|
||
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
|
||
stand("flush-cache")
|
||
while time.monotonic() < deadline:
|
||
assert_ok(read_chat(api, actor, chat), 200, "Chat")
|
||
if stand("cache-has-chat", chat["id"]) == "1":
|
||
return
|
||
time.sleep(0.5)
|
||
raise AssertionError("cache did not come back")
|
||
|
||
|
||
def chat_titles(api: Api, actor: Actor) -> set[str]:
|
||
return {
|
||
chat["title"]
|
||
for page in api.chat_pages(actor)
|
||
for chat in page["items"]
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Истёкший токен")
|
||
def test_expired_token(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
time.sleep(SESSION_TTL_S + 1)
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Срок не продлевается")
|
||
def test_use_does_not_extend_session(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
logged_in = time.monotonic()
|
||
successes = 0
|
||
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
successes += 1
|
||
time.sleep(0.5)
|
||
assert successes >= 3, "too few requests before expiry"
|
||
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
|
||
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
|
||
actor = api.register()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.request("GET", ME, actor=actor)
|
||
assert_error(resp, 503)
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Хранение: откат, остановленная БД, исчерпанный пул
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
|
||
def test_members_failure_rolls_back_chat(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
) -> None:
|
||
title = f"fault-{uuid.uuid4().hex}"
|
||
stand("fail-members", title)
|
||
try:
|
||
resp = api.request(
|
||
"POST",
|
||
f"{API}/chats",
|
||
actor=alice,
|
||
json={"title": title, "member_ids": [alice.id, bob.id]},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-members")
|
||
for actor in (alice, bob):
|
||
assert title not in chat_titles(api, actor)
|
||
assert stand("count-chats", title) == "0"
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "БД остановлена")
|
||
@pytest.mark.spec("data-storage", "БД вернулась")
|
||
def test_database_stopped_and_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
error = assert_error(resp, 503)
|
||
assert took < 2, f"503 took {took:.2f}s"
|
||
for word in ("postgres", "sql", "connect"):
|
||
assert word not in error["message"].lower()
|
||
message = api.send_message(alice, chat["id"], "после возврата")
|
||
# запись во время отказа не зафиксирована, новая — видна
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
|
||
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
holder = Api(str(tight_api.client.base_url), timeout=15)
|
||
first: list[int] = []
|
||
|
||
def hold_connection() -> None:
|
||
first.append(holder.request("GET", ME, actor=actor).status_code)
|
||
|
||
with outage(stand, "pause", "postgres"):
|
||
thread = threading.Thread(target=hold_connection)
|
||
thread.start()
|
||
time.sleep(SLACK_S)
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
assert_error(resp, 503)
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
thread.join()
|
||
holder.close()
|
||
assert first == [503]
|
||
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
|
||
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
# hook оборвал соединение пула: при выдаче pool_pre_ping
|
||
# переподключается и получает «too many clients» (SQLSTATE 53300)
|
||
stand("fill-connections", str(FILL_HOLD_S))
|
||
try:
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
error = assert_error(resp, 503)
|
||
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
for word in ("postgres", "sql", "client", "connect"):
|
||
assert word not in error["message"].lower()
|
||
finally:
|
||
stand("clear-connections")
|
||
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
|
||
# сессии 2 с за время заполнения истёк — новый вход
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
# пачка одновременных запросов больше пула обычного API (10)
|
||
BURST_THREADS = 48
|
||
BURST_AFTER_BARRIER = 4
|
||
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
|
||
STATS_FLUSH_S = 11
|
||
|
||
|
||
def burst(api: Api, actor: Actor) -> None:
|
||
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
|
||
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
|
||
одновременно. Все ответы — 200."""
|
||
barrier = threading.Barrier(BURST_THREADS)
|
||
clients = [
|
||
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
|
||
]
|
||
responses: list[httpx.Response] = []
|
||
errors: list[BaseException] = []
|
||
lock = threading.Lock()
|
||
|
||
def worker(client: Api) -> None:
|
||
try:
|
||
got = [client.request("GET", ME, actor=actor)]
|
||
barrier.wait()
|
||
got += [
|
||
client.request("GET", ME, actor=actor)
|
||
for _ in range(BURST_AFTER_BARRIER)
|
||
]
|
||
except BaseException as e: # noqa: BLE001 — поднимется в тесте
|
||
barrier.abort()
|
||
with lock:
|
||
errors.append(e)
|
||
return
|
||
with lock:
|
||
responses.extend(got)
|
||
|
||
threads = [
|
||
threading.Thread(target=worker, args=(client,)) for client in clients
|
||
]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
for client in clients:
|
||
client.close()
|
||
assert not errors, f"burst failed: {errors[0]!r}"
|
||
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
|
||
for resp in responses:
|
||
assert_ok(resp, 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
|
||
def test_pool_keeps_connections(
|
||
api: Api,
|
||
tight_api: Api,
|
||
stand: Stand,
|
||
) -> None:
|
||
actor = api.register()
|
||
# соединение «тесного» API могли оборвать прежние тесты
|
||
# (fill-connections): его переподключение не должно попасть в окно
|
||
tight = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
|
||
# прогрев: пул обычного API набирает свой размер
|
||
for _ in range(2):
|
||
burst(api, actor)
|
||
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
|
||
# в общую статистику без ожидания блокировки и при занятой блокировке
|
||
# откладываются. Простаивающий backend сбрасывает их принудительно по
|
||
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
|
||
# попадали в окно по одному
|
||
time.sleep(STATS_FLUSH_S)
|
||
# калибровка: второй вызов видит сессию первого вызова hook, и только её
|
||
s1 = int(stand("db-sessions"))
|
||
s2 = int(stand("db-sessions"))
|
||
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
|
||
for _ in range(3):
|
||
burst(api, actor)
|
||
s3 = int(stand("db-sessions"))
|
||
opened = s3 - s2 - 1
|
||
assert opened == 0, (
|
||
f"PostgreSQL accepted {opened} new sessions during "
|
||
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Readiness
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище не готово")
|
||
def test_readiness_without_database(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(lambda: api.request("GET", "/health/ready"))
|
||
assert_error(resp, 503)
|
||
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
|
||
live = api.request("GET", "/health/live")
|
||
assert live.status_code == 200
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Кэш недоступен")
|
||
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище объектов недоступно")
|
||
@pytest.mark.parametrize("action", ["stop", "pause"])
|
||
def test_readiness_without_object_storage(
|
||
api: Api,
|
||
stand: Stand,
|
||
action: str,
|
||
) -> None:
|
||
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
|
||
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
|
||
with outage(stand, action, "s3"):
|
||
for path in ("/health/ready", "/health/live"):
|
||
resp, took = timed(lambda path=path: api.request("GET", path))
|
||
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
|
||
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Кэш карточки чата
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Очистка кэша")
|
||
def test_flush_cache(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
stand("flush-cache")
|
||
assert stand("cache-has-chat", chat["id"]) == "0"
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey остановлен")
|
||
def test_valkey_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
sent = [api.send_message(alice, chat["id"], "до отказа")]
|
||
with outage(stand, "stop", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
sent.append(assert_ok(resp, 201, "Message"))
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
|
||
assert api.all_messages(bob, chat["id"]) == sent
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey завис")
|
||
def test_valkey_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
baseline = statistics.median(
|
||
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
|
||
)
|
||
with outage(stand, "pause", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
|
||
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
||
)
|
||
# окно пропуска: следующий запрос таймаута не ждёт
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey вернулся")
|
||
def test_valkey_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
# API не перезапускался; кэш снова в работе после окна пропуска
|
||
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
hits = int(stand("cache-hits"))
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert int(stand("cache-hits")) > hits
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
|
||
def test_message_failure_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
# ключ вставляется первым: триггер роняет транзакцию уже после него
|
||
text = f"fault-{uuid.uuid4().hex}"
|
||
key = new_key()
|
||
stand("fail-messages", text)
|
||
try:
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-messages")
|
||
# ключ не занят: повтор выполняется заново, а не 500 и не 409
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
|
||
def test_database_outage_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
text = "при отказе БД"
|
||
key = new_key()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 503)
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Ключ после срока")
|
||
def test_key_free_after_ttl(tight_api: Api) -> None:
|
||
alice = tight_api.register()
|
||
chat = tight_api.create_chat(alice)
|
||
key = new_key()
|
||
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
|
||
# в пределах срока ключ занят: другое тело — 409
|
||
resp = tight_api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": "после срока"},
|
||
)
|
||
assert_error(resp, 409)
|
||
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
|
||
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
|
||
alice = tight_api.login(alice.user, alice.password)
|
||
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
|
||
assert second["id"] != first["id"]
|
||
history = tight_api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [first["id"], second["id"]]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Вложения: S3 остановлен или заморожен, сбои записи строки, уборщик. Строки
|
||
# и объекты — списками id от hook, а не счётчиками: уборщик стенда убирает
|
||
# строки прежних тестов параллельно (design change upload-consistency,
|
||
# решение 10)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# grace и интервал уборщика стенда отказов (run_failures.py,
|
||
# failures.compose.yml)
|
||
CLEANER_GRACE_S = float(os.environ.get("FAULT_CLEANER_GRACE_S", "20"))
|
||
CLEANER_INTERVAL_S = float(os.environ.get("FAULT_CLEANER_INTERVAL_S", "1"))
|
||
# запас поверх grace и интервала: проход, вызов hook, планировщик Docker
|
||
# Desktop
|
||
CLEANUP_SLACK_S = 5
|
||
# задержка перевода в ready триггером stall (fault_hook.py)
|
||
STALL_S = 4
|
||
# срок клиента загрузок в потоке
|
||
THREAD_CLIENT_TIMEOUT_S = 30
|
||
|
||
|
||
def attachment_rows(stand: Stand) -> dict[str, str]:
|
||
"""Вложения стенда: `id` → статус."""
|
||
return dict(line.split() for line in stand("attachment-ids").splitlines())
|
||
|
||
|
||
def stored_ids(stand: Stand) -> set[str]:
|
||
"""`id` вложений, у которых есть объект в bucket стенда."""
|
||
return set(stand("object-ids").splitlines())
|
||
|
||
|
||
def orphan_objects(stand: Stand) -> set[str]:
|
||
"""Объекты вложений без строки. Сначала строки, потом объекты: уборщик
|
||
удаляет объект раньше строки, и убранная между вызовами строка не
|
||
даёт ложного «объекта без строки»."""
|
||
rows = attachment_rows(stand)
|
||
return stored_ids(stand) - set(rows)
|
||
|
||
|
||
def ready_without_object(stand: Stand) -> set[str]:
|
||
"""Готовые вложения без объекта: строки `ready` уборщик не трогает,
|
||
поэтому порядок вызовов здесь не важен."""
|
||
return ready_ids(attachment_rows(stand)) - stored_ids(stand)
|
||
|
||
|
||
def new_ids(before: dict[str, str], after: dict[str, str]) -> set[str]:
|
||
return set(after) - set(before)
|
||
|
||
|
||
def ready_ids(rows: dict[str, str]) -> set[str]:
|
||
return {item for item, status in rows.items() if status == "ready"}
|
||
|
||
|
||
def wait_cleaned(stand: Stand, attachment_id: str, *, since: float) -> None:
|
||
"""Строка и объект вложения убраны не позже grace + интервал + запас
|
||
от `since` (момент до загрузки: `created_at` строки не раньше)."""
|
||
deadline = since + CLEANER_GRACE_S + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while True:
|
||
rows = attachment_rows(stand)
|
||
stored = stored_ids(stand)
|
||
if attachment_id not in rows and attachment_id not in stored:
|
||
return
|
||
if time.monotonic() > deadline:
|
||
raise AssertionError(
|
||
f"attachment {attachment_id} is not cleaned "
|
||
f"{time.monotonic() - since:.1f}s after upload: "
|
||
f"row {rows.get(attachment_id)}, "
|
||
f"object {attachment_id in stored}",
|
||
)
|
||
time.sleep(0.5)
|
||
|
||
|
||
def sleep_until(moment: float) -> None:
|
||
time.sleep(max(0.0, moment - time.monotonic()))
|
||
|
||
|
||
def fault_filename() -> str:
|
||
"""Уникальное имя загрузки, на которое срабатывает триггер hook."""
|
||
return f"fault-{uuid.uuid4().hex}.png"
|
||
|
||
|
||
def upload_raw(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
data: Any,
|
||
**kwargs: Any,
|
||
) -> httpx.Response:
|
||
return api.post_attachment(actor, chat["id"], data, **kwargs)
|
||
|
||
|
||
def assert_invisible(
|
||
api: Api,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
attachment_id: str,
|
||
) -> None:
|
||
"""Вложение не в статусе ready: метаданные, байты и ссылка в сообщении
|
||
— 404 с тем же `message`, что у неизвестного `id`."""
|
||
|
||
def calls(item: str) -> tuple[httpx.Response, ...]:
|
||
return (
|
||
api.attachment_meta(actor, item),
|
||
api.attachment_content(actor, item),
|
||
api.post_message(
|
||
actor,
|
||
chat["id"],
|
||
json={"text": "ссылка", "attachment_ids": [item]},
|
||
),
|
||
)
|
||
|
||
unknown = [assert_error(resp, 404) for resp in calls(str(uuid.uuid4()))]
|
||
hidden = [assert_error(resp, 404) for resp in calls(attachment_id)]
|
||
assert [error["message"] for error in hidden] == [
|
||
error["message"] for error in unknown
|
||
]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
|
||
@pytest.mark.spec("attachments", "Незавершённая загрузка не видна")
|
||
def test_upload_with_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
before = attachment_rows(stand)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
after = attachment_rows(stand)
|
||
# строка-намерение осталась, готовых не прибавилось; её уберёт
|
||
# уборщик (test_cleaner_survives_outages)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert ready_ids(after) == ready_ids(before)
|
||
# замечание S6 verify attachments: id последней pending — от hook
|
||
assert stand("last-pending") == pending
|
||
assert_invisible(api, alice, chat, pending)
|
||
api.send_message(alice, chat["id"], "текст без S3")
|
||
api.upload(alice, chat["id"], data)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
|
||
def test_rejected_uploads_leave_nothing(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
eve: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
|
||
|
||
def chunked() -> Iterator[bytes]:
|
||
for offset in range(0, len(big), 1024 * 1024):
|
||
yield big[offset : offset + 1024 * 1024]
|
||
|
||
# хотя бы одно готовое вложение: списки hook не пусты случайно
|
||
ready = api.upload(alice, chat["id"], images.sample_png())
|
||
before = (attachment_rows(stand), stored_ids(stand))
|
||
assert before[0].get(ready["id"]) == "ready", before[0]
|
||
assert ready["id"] in before[1], before[1]
|
||
rejected = (
|
||
(404, upload_raw(api, eve, chat, images.sample_png())),
|
||
(413, upload_raw(api, alice, chat, big)),
|
||
(413, upload_raw(api, alice, chat, chunked())),
|
||
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
|
||
(
|
||
415,
|
||
upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
content_type="text/plain",
|
||
),
|
||
),
|
||
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
|
||
)
|
||
for status, resp in rejected:
|
||
assert_error(resp, status)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not stored_ids(stand) - before[1]
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Сбой записи метаданных")
|
||
def test_metadata_insert_failure(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = (attachment_rows(stand), stored_ids(stand))
|
||
stand("fail-attachments", "insert", filename)
|
||
try:
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строки нет — объект не записывался
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not stored_ids(stand) - before[1]
|
||
|
||
|
||
class BodyAborted(Exception):
|
||
"""Клиент обрывает передачу тела загрузки."""
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Обрыв передачи тела")
|
||
def test_interrupted_body(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
before = (attachment_rows(stand), stored_ids(stand))
|
||
|
||
def aborted_body() -> Iterator[bytes]:
|
||
yield images.PNG_SIGNATURE + bytes(1024 * 1024)
|
||
raise BodyAborted
|
||
|
||
# свой клиент: его соединение обрывается вместе с телом
|
||
with (
|
||
Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S) as own,
|
||
pytest.raises((BodyAborted, httpx.TransportError)),
|
||
):
|
||
upload_raw(own, alice, chat, aborted_body())
|
||
# API замечает обрыв не мгновенно; строки до конца тела нет в любом
|
||
# случае
|
||
time.sleep(1)
|
||
assert not new_ids(before[0], attachment_rows(stand))
|
||
assert not stored_ids(stand) - before[1]
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Фиксация ready не удалась")
|
||
@pytest.mark.spec("storage-consistency", "Свежая загрузка не тронута")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_ready_failure_compensated(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "ready", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
assert stand("last-pending") == pending
|
||
# компенсация: объект удалён до ответа 503
|
||
assert pending not in stored_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
# моложе grace строку уборщик не трогает: к половине grace он сделал
|
||
# около G / 2 / I проходов
|
||
sleep_until(started + CLEANER_GRACE_S / 2)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Объект без фиксации убран")
|
||
def test_killed_between_put_and_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
outcome: dict[str, Any] = {}
|
||
client = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
|
||
def upload() -> None:
|
||
try:
|
||
outcome["status"] = upload_raw(
|
||
client,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
).status_code
|
||
except httpx.TransportError as e:
|
||
outcome["error"] = e
|
||
|
||
stand("fail-attachments", "stall", filename)
|
||
killed = False
|
||
try:
|
||
started = time.monotonic()
|
||
thread = threading.Thread(target=upload)
|
||
thread.start()
|
||
# строка pending и объект — до конца задержки перевода в ready
|
||
deadline = started + STALL_S
|
||
pending = ""
|
||
while time.monotonic() < deadline:
|
||
pending = stand("last-pending")
|
||
if (
|
||
pending
|
||
and pending not in before
|
||
and pending in stored_ids(stand)
|
||
):
|
||
break
|
||
time.sleep(0.05)
|
||
else:
|
||
raise AssertionError("no pending row with object during stall")
|
||
stand("kill-api")
|
||
killed = True
|
||
thread.join(THREAD_CLIENT_TIMEOUT_S)
|
||
assert not thread.is_alive()
|
||
# 201 до клиента не дошёл: перевод в ready ждал в триггере
|
||
assert outcome.get("status") != 201, outcome
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert pending in stored_ids(stand)
|
||
finally:
|
||
client.close()
|
||
stand("clear-fail-attachments")
|
||
if killed:
|
||
stand("start-api")
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
api.upload(alice, chat["id"], images.sample_png())
|
||
|
||
|
||
@pytest.mark.spec("attachments", "Исход фиксации неизвестен")
|
||
def test_ready_commit_outcome_unknown(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Перевод в ready падает на самом COMMIT (отложенный триггер): API не
|
||
знает, зафиксировано ли вложение, и объект не трогает (verify W2)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "commit", filename)
|
||
try:
|
||
started = time.monotonic()
|
||
resp = upload_raw(
|
||
api,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
after = attachment_rows(stand)
|
||
(pending,) = new_ids(before, after)
|
||
assert after[pending] == "pending"
|
||
# компенсации нет: вложение могло стать ready
|
||
assert pending in stored_ids(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Загрузка дольше grace")
|
||
def test_upload_outlives_grace(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Строка вставлена «на час старше», перевод в ready ждёт в триггере,
|
||
пока её не уберёт уборщик стенда: загрузка, пережившая grace, без
|
||
заморозки процесса (verify W3)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "outlive", filename)
|
||
# ожидание в триггере до 4,5 с: срок клиента больше
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
# не 201 (ready без объекта) и не 500 (дефект ветки)
|
||
assert_error(resp, 503)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
rows = attachment_rows(stand)
|
||
assert not new_ids(before, rows), rows
|
||
assert not stored_ids(stand) - set(before)
|
||
assert not orphan_objects(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "Сбой уборки после удаления объекта")
|
||
def test_cleaner_failure_after_object_delete(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Уборщик удалил объект, а удаление строки упало (08006), пока загрузка
|
||
ждёт перевода в ready: строка помечена до удаления объекта и ready уже
|
||
не станет (verify W1)."""
|
||
filename = fault_filename()
|
||
before = attachment_rows(stand)
|
||
stand("fail-attachments", "undeletable", filename)
|
||
slow = Api(str(api.client.base_url), timeout=THREAD_CLIENT_TIMEOUT_S)
|
||
try:
|
||
with slow:
|
||
resp = upload_raw(
|
||
slow,
|
||
alice,
|
||
chat,
|
||
images.sample_png(),
|
||
filename=filename,
|
||
)
|
||
assert_error(resp, 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
assert not ready_without_object(stand)
|
||
assert_invisible(api, alice, chat, pending)
|
||
finally:
|
||
stand("clear-fail-attachments")
|
||
# строка старше grace: следующий проход доводит уборку
|
||
returned = time.monotonic()
|
||
wait_cleaned(stand, pending, since=returned - CLEANER_GRACE_S)
|
||
assert not orphan_objects(stand)
|
||
assert not ready_without_object(stand)
|
||
|
||
|
||
@pytest.mark.spec("storage-consistency", "S3 недоступен во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "БД недоступна во время уборки")
|
||
@pytest.mark.spec("storage-consistency", "Готовые вложения не тронуты")
|
||
def test_cleaner_survives_outages(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
ready = api.upload(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
assert_error(upload_raw(api, alice, chat, data), 503)
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
# PostgreSQL недоступен, пока загрузка моложе grace
|
||
with outage(stand, "stop", "postgres"):
|
||
time.sleep(3)
|
||
# grace истёк, S3 всё ещё нет: проходы прерываются, строка на месте
|
||
sleep_until(started + CLEANER_GRACE_S + 3 * CLEANER_INTERVAL_S)
|
||
assert attachment_rows(stand).get(pending) == "pending"
|
||
# S3 вернулся: уборщик без перезапуска убирает строку за проход
|
||
returned = time.monotonic()
|
||
deadline = returned + CLEANER_INTERVAL_S + CLEANUP_SLACK_S
|
||
while pending in attachment_rows(stand):
|
||
assert time.monotonic() < deadline, (
|
||
f"pending {pending} is still there "
|
||
f"{time.monotonic() - returned:.1f}s after S3 came back"
|
||
)
|
||
time.sleep(0.5)
|
||
assert pending not in stored_ids(stand)
|
||
assert not orphan_objects(stand)
|
||
# готовое вложение старше grace на месте
|
||
meta = assert_ok(
|
||
api.attachment_meta(alice, ready["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == ready
|
||
assert meta["status"] == "ready"
|
||
assert_content(
|
||
api.attachment_content(alice, ready["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 остановлен")
|
||
def test_object_storage_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload(alice, chat["id"], data)
|
||
with outage(stand, "stop", "s3"):
|
||
resp, elapsed = timed(
|
||
lambda: api.attachment_content(bob, attachment["id"]),
|
||
)
|
||
assert_error(resp, 503)
|
||
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
|
||
meta = assert_ok(
|
||
api.attachment_meta(bob, attachment["id"]),
|
||
200,
|
||
"Attachment",
|
||
)
|
||
assert meta == attachment
|
||
message = api.send_message(
|
||
bob,
|
||
chat["id"],
|
||
"",
|
||
attachment_ids=[attachment["id"]],
|
||
)
|
||
# история с attachment_ids — из PostgreSQL, S3 ей не нужен
|
||
history = api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [message["id"]]
|
||
assert history[0]["attachment_ids"] == [attachment["id"]]
|
||
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
|
||
assert_content(
|
||
api.attachment_content(bob, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
|
||
|
||
def test_environment_does_not_change_limits(
|
||
api: Api,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Спек `service-configuration`, «Окружение не меняет лимит» (вне
|
||
сверки покрытия — сценарий в докстринге): обычный API стенда
|
||
отказов запущен с `MAX_UPLOAD_BYTES=1` и `MAX_IMAGE_PIXELS=1`
|
||
(`run_failures.py`, `LIMITS_ENV`), а `sample.png` (111 817 байт,
|
||
64 000 пикселей) всё равно принимается: лимиты — константы."""
|
||
limits = dict(
|
||
item.split("=", 1)
|
||
for item in os.environ.get("FAULT_LIMITS_ENV", "").split()
|
||
)
|
||
# без этих переменных на стенде тест ничего не доказывает
|
||
assert limits == {"MAX_UPLOAD_BYTES": "1", "MAX_IMAGE_PIXELS": "1"}, (
|
||
f"stand started without limit variables: {limits}"
|
||
)
|
||
api.upload(alice, chat["id"], images.sample_png(), filename="sample.png")
|
||
|
||
|
||
@pytest.mark.spec("attachments", "S3 завис")
|
||
@pytest.mark.spec("storage-consistency", "Зависшая загрузка убрана")
|
||
def test_object_storage_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
data = images.sample_png()
|
||
attachment = api.upload(alice, chat["id"], data)
|
||
before = attachment_rows(stand)
|
||
started = time.monotonic()
|
||
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
|
||
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
|
||
with slow, outage(stand, "pause", "s3"):
|
||
for call in (
|
||
lambda: upload_raw(slow, alice, chat, data),
|
||
lambda: slow.attachment_content(alice, attachment["id"]),
|
||
):
|
||
resp, elapsed = timed(call)
|
||
assert_error(resp, 503)
|
||
# срок вызова S3 и немного на шаги с БД, а не зависание
|
||
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
|
||
f"503 after {elapsed:.2f}s"
|
||
)
|
||
assert_content(
|
||
api.attachment_content(alice, attachment["id"]),
|
||
data,
|
||
"image/png",
|
||
)
|
||
# загрузка во время заморозки оставила строку pending; объект после
|
||
# разморозки не дописался, строку уборщик убирает после grace
|
||
(pending,) = new_ids(before, attachment_rows(stand))
|
||
wait_cleaned(stand, pending, since=started)
|
||
assert not orphan_objects(stand)
|