Files
mtusi2026lab4/student-tests/test_failures.py
T
ricoromaandClaude Opus 5.5 116d8d3bf0 feat: add image attachments to messages
- upload, metadata and download endpoints for PNG/JPEG attachments:
  limits before reading and decompression, type by content,
  structural integrity check without decoding pixels, safe filename
- write order: pending row with object key -> PUT to S3 -> ready;
  per-step DB sessions so no pool connection is held during transfers
- attachment_ids in messages (validation, idempotency hash, history);
  whitespace-only text allowed with an attachment (teacher, 2026-10-20)
- migration: attachments and message_attachments tables, text length
  CHECK relaxed with NOT VALID
- student tests, fault checks and driver commands for attachments;
  rollout profile starts from e1161f7 and checks the commit first
- sync attachments deltas into main specs; rules, AGENTS.md, PLAN.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 01:08:56 +03:00

692 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres,
Valkey и S3, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
идемпотентности 2 с, пул из одного соединения). На хранилища тесты действуют только через фикстуру
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
рабочее состояние. Решение 4 design change `failure-checks`.
"""
import statistics
import threading
import time
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import httpx
import pytest
import images
from api_client import API, Actor, Api, new_key
from contract import (
BEARER_CHALLENGE,
assert_content,
assert_error,
assert_ok,
assert_unauthorized,
)
pytestmark = pytest.mark.fault
type Stand = Callable[..., str]
RESTORE = {"stop": "start", "pause": "unpause"}
# запас к границам времени: планировщик Docker Desktop
SLACK_S = 0.3
# [cache] timeout_seconds и retry_after_seconds по умолчанию
CACHE_TIMEOUT_S = 0.2
CACHE_RETRY_AFTER_S = 5
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
SESSION_TTL_S = 2
IDEMPOTENCY_TTL_S = 2
POOL_TIMEOUT_S = 1
# срок проверки Postgres в /health/ready
READINESS_TIMEOUT_S = 2
# health без S3: проверка к S3 не обращается, ответ быстрее секунды (спек
# health, «Хранилище объектов недоступно»); срок вызова S3 — 5 с
OBJECT_STORAGE_HEALTH_S = 1
# [s3] timeout_seconds по умолчанию: срок одного вызова S3
S3_TIMEOUT_S = 5
# сколько секунд hook держит слоты max_connections; clear-connections
# освобождает их раньше
FILL_HOLD_S = 30
ME = f"{API}/users/me"
@contextmanager
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
"""`stop` или `pause` хранилища; в `finally` — обратно."""
stand(action, service)
try:
yield
finally:
stand(RESTORE[action], service)
def timed(
call: Callable[[], httpx.Response],
) -> tuple[httpx.Response, float]:
started = time.monotonic()
resp = call()
return resp, time.monotonic() - started
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
def wait_cache_ready(
api: Api,
stand: Stand,
actor: Actor,
chat: dict[str, Any],
) -> None:
"""Дождаться, пока чтение снова пишет карточку в кэш.
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
"""
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
stand("flush-cache")
while time.monotonic() < deadline:
assert_ok(read_chat(api, actor, chat), 200, "Chat")
if stand("cache-has-chat", chat["id"]) == "1":
return
time.sleep(0.5)
raise AssertionError("cache did not come back")
def chat_titles(api: Api, actor: Actor) -> set[str]:
return {
chat["title"]
for page in api.chat_pages(actor)
for chat in page["items"]
}
# ---------------------------------------------------------------------------
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
# ---------------------------------------------------------------------------
@pytest.mark.spec("sessions", "Истёкший токен")
def test_expired_token(tight_api: Api) -> None:
actor = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
time.sleep(SESSION_TTL_S + 1)
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "Срок не продлевается")
def test_use_does_not_extend_session(tight_api: Api) -> None:
actor = tight_api.register()
logged_in = time.monotonic()
successes = 0
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
successes += 1
time.sleep(0.5)
assert successes >= 3, "too few requests before expiry"
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
actor = api.register()
with outage(stand, "stop", "postgres"):
resp = api.request("GET", ME, actor=actor)
assert_error(resp, 503)
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
# ---------------------------------------------------------------------------
# Хранение: откат, остановленная БД, исчерпанный пул
# ---------------------------------------------------------------------------
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
def test_members_failure_rolls_back_chat(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
) -> None:
title = f"fault-{uuid.uuid4().hex}"
stand("fail-members", title)
try:
resp = api.request(
"POST",
f"{API}/chats",
actor=alice,
json={"title": title, "member_ids": [alice.id, bob.id]},
)
assert_error(resp, 500)
finally:
stand("clear-fail-members")
for actor in (alice, bob):
assert title not in chat_titles(api, actor)
assert stand("count-chats", title) == "0"
@pytest.mark.spec("data-storage", "БД остановлена")
@pytest.mark.spec("data-storage", "БД вернулась")
def test_database_stopped_and_back(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
error = assert_error(resp, 503)
assert took < 2, f"503 took {took:.2f}s"
for word in ("postgres", "sql", "connect"):
assert word not in error["message"].lower()
message = api.send_message(alice, chat["id"], "после возврата")
# запись во время отказа не зафиксирована, новая — видна
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
holder = Api(str(tight_api.client.base_url), timeout=15)
first: list[int] = []
def hold_connection() -> None:
first.append(holder.request("GET", ME, actor=actor).status_code)
with outage(stand, "pause", "postgres"):
thread = threading.Thread(target=hold_connection)
thread.start()
time.sleep(SLACK_S)
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
assert_error(resp, 503)
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
thread.join()
holder.close()
assert first == [503]
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
# hook оборвал соединение пула: при выдаче pool_pre_ping
# переподключается и получает «too many clients» (SQLSTATE 53300)
stand("fill-connections", str(FILL_HOLD_S))
try:
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
error = assert_error(resp, 503)
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
for word in ("postgres", "sql", "client", "connect"):
assert word not in error["message"].lower()
finally:
stand("clear-connections")
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
# сессии 2 с за время заполнения истёк — новый вход
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
# пачка одновременных запросов больше пула обычного API (10)
BURST_THREADS = 48
BURST_AFTER_BARRIER = 4
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
STATS_FLUSH_S = 11
def burst(api: Api, actor: Actor) -> None:
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
одновременно. Все ответы — 200."""
barrier = threading.Barrier(BURST_THREADS)
clients = [
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
]
responses: list[httpx.Response] = []
errors: list[BaseException] = []
lock = threading.Lock()
def worker(client: Api) -> None:
try:
got = [client.request("GET", ME, actor=actor)]
barrier.wait()
got += [
client.request("GET", ME, actor=actor)
for _ in range(BURST_AFTER_BARRIER)
]
except BaseException as e: # noqa: BLE001 — поднимется в тесте
barrier.abort()
with lock:
errors.append(e)
return
with lock:
responses.extend(got)
threads = [
threading.Thread(target=worker, args=(client,)) for client in clients
]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
for client in clients:
client.close()
assert not errors, f"burst failed: {errors[0]!r}"
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
for resp in responses:
assert_ok(resp, 200, "User")
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
def test_pool_keeps_connections(
api: Api,
tight_api: Api,
stand: Stand,
) -> None:
actor = api.register()
# соединение «тесного» API могли оборвать прежние тесты
# (fill-connections): его переподключение не должно попасть в окно
tight = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
# прогрев: пул обычного API набирает свой размер
for _ in range(2):
burst(api, actor)
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
# в общую статистику без ожидания блокировки и при занятой блокировке
# откладываются. Простаивающий backend сбрасывает их принудительно по
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
# попадали в окно по одному
time.sleep(STATS_FLUSH_S)
# калибровка: второй вызов видит сессию первого вызова hook, и только её
s1 = int(stand("db-sessions"))
s2 = int(stand("db-sessions"))
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
for _ in range(3):
burst(api, actor)
s3 = int(stand("db-sessions"))
opened = s3 - s2 - 1
assert opened == 0, (
f"PostgreSQL accepted {opened} new sessions during "
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
)
# ---------------------------------------------------------------------------
# Readiness
# ---------------------------------------------------------------------------
@pytest.mark.spec("health", "Хранилище не готово")
def test_readiness_without_database(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(lambda: api.request("GET", "/health/ready"))
assert_error(resp, 503)
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
live = api.request("GET", "/health/live")
assert live.status_code == 200
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Кэш недоступен")
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "valkey"):
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Хранилище объектов недоступно")
@pytest.mark.parametrize("action", ["stop", "pause"])
def test_readiness_without_object_storage(
api: Api,
stand: Stand,
action: str,
) -> None:
# pause: соединение с S3 есть, ответа нет — проверка, которая всё же
# ходила бы в S3, ждала бы срока вызова (5 с), а не отказа соединения
with outage(stand, action, "s3"):
for path in ("/health/ready", "/health/live"):
resp, took = timed(lambda path=path: api.request("GET", path))
assert assert_ok(resp, 200, "Health") == {"status": "ok"}
assert took < OBJECT_STORAGE_HEALTH_S, f"{path} took {took:.2f}s"
# ---------------------------------------------------------------------------
# Кэш карточки чата
# ---------------------------------------------------------------------------
@pytest.mark.spec("chat-cache", "Очистка кэша")
def test_flush_cache(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
stand("flush-cache")
assert stand("cache-has-chat", chat["id"]) == "0"
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert stand("cache-has-chat", chat["id"]) == "1"
@pytest.mark.spec("chat-cache", "Valkey остановлен")
def test_valkey_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
sent = [api.send_message(alice, chat["id"], "до отказа")]
with outage(stand, "stop", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
sent.append(assert_ok(resp, 201, "Message"))
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
assert api.all_messages(bob, chat["id"]) == sent
@pytest.mark.spec("chat-cache", "Valkey завис")
def test_valkey_paused(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
baseline = statistics.median(
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
)
with outage(stand, "pause", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
f"read took {took:.2f}s, baseline {baseline:.3f}s"
)
# окно пропуска: следующий запрос таймаута не ждёт
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
@pytest.mark.spec("chat-cache", "Valkey вернулся")
def test_valkey_back(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "valkey"):
assert_ok(read_chat(api, bob, chat), 200, "Chat")
# API не перезапускался; кэш снова в работе после окна пропуска
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
assert_ok(read_chat(api, bob, chat), 200, "Chat")
assert stand("cache-has-chat", chat["id"]) == "1"
hits = int(stand("cache-hits"))
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert int(stand("cache-hits")) > hits
# ---------------------------------------------------------------------------
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
# ---------------------------------------------------------------------------
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
def test_message_failure_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
# ключ вставляется первым: триггер роняет транзакцию уже после него
text = f"fault-{uuid.uuid4().hex}"
key = new_key()
stand("fail-messages", text)
try:
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 500)
finally:
stand("clear-fail-messages")
# ключ не занят: повтор выполняется заново, а не 500 и не 409
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
def test_database_outage_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
text = "при отказе БД"
key = new_key()
with outage(stand, "stop", "postgres"):
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 503)
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Ключ после срока")
def test_key_free_after_ttl(tight_api: Api) -> None:
alice = tight_api.register()
chat = tight_api.create_chat(alice)
key = new_key()
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
# в пределах срока ключ занят: другое тело — 409
resp = tight_api.post_message(
alice,
chat["id"],
key=key,
json={"text": "после срока"},
)
assert_error(resp, 409)
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
alice = tight_api.login(alice.user, alice.password)
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
assert second["id"] != first["id"]
history = tight_api.all_messages(alice, chat["id"])
assert [item["id"] for item in history] == [first["id"], second["id"]]
# ---------------------------------------------------------------------------
# Вложения: S3 остановлен или заморожен, счётчики строк и объектов — hook
# ---------------------------------------------------------------------------
def attachment_counts(stand: Stand) -> dict[str, int]:
"""Вложения стенда по статусам: `pending=N ready=M` от hook."""
return {
name: int(value)
for name, value in (
item.split("=") for item in stand("count-attachments").split()
)
}
def object_count(stand: Stand) -> int:
return int(stand("count-objects"))
def upload_raw(
api: Api,
actor: Actor,
chat: dict[str, Any],
data: Any,
**kwargs: Any,
) -> httpx.Response:
return api.post_attachment(actor, chat["id"], data, **kwargs)
@pytest.mark.spec("attachments", "Запись в S3 не удалась")
def test_upload_with_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
before = attachment_counts(stand)
with outage(stand, "stop", "s3"):
resp, elapsed = timed(lambda: upload_raw(api, alice, chat, data))
assert_error(resp, 503)
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
after = attachment_counts(stand)
# строка-намерение осталась, готовых не прибавилось; её уберёт
# уборщик change upload-consistency
assert after == {
"pending": before["pending"] + 1,
"ready": before["ready"],
}
api.send_message(alice, chat["id"], "текст без S3")
api.upload(alice, chat["id"], data)
@pytest.mark.spec("attachments", "Отклонённая загрузка ничего не оставляет")
def test_rejected_uploads_leave_nothing(
api: Api,
stand: Stand,
alice: Actor,
eve: Actor,
chat: dict[str, Any],
) -> None:
big = images.PNG_SIGNATURE + bytes(images.MAX_BYTES * 2)
def chunked() -> Iterator[bytes]:
for offset in range(0, len(big), 1024 * 1024):
yield big[offset : offset + 1024 * 1024]
# хотя бы одно готовое вложение: счётчики hook не нулевые случайно
api.upload(alice, chat["id"], images.sample_png())
before = (attachment_counts(stand), object_count(stand))
assert before[0]["ready"] >= 1, before
assert before[1] >= 1, before
rejected = (
(404, upload_raw(api, eve, chat, images.sample_png())),
(413, upload_raw(api, alice, chat, big)),
(413, upload_raw(api, alice, chat, chunked())),
(413, upload_raw(api, alice, chat, images.png_huge(8000, 5001))),
(
415,
upload_raw(
api,
alice,
chat,
images.sample_png(),
content_type="text/plain",
),
),
(415, upload_raw(api, alice, chat, images.png_idat_byte_changed())),
)
for status, resp in rejected:
assert_error(resp, status)
assert (attachment_counts(stand), object_count(stand)) == before
@pytest.mark.spec("attachments", "S3 остановлен")
def test_object_storage_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
attachment = api.upload(alice, chat["id"], data)
with outage(stand, "stop", "s3"):
resp, elapsed = timed(
lambda: api.attachment_content(bob, attachment["id"]),
)
assert_error(resp, 503)
assert elapsed < S3_TIMEOUT_S + SLACK_S, f"503 after {elapsed:.2f}s"
meta = assert_ok(
api.attachment_meta(bob, attachment["id"]),
200,
"Attachment",
)
assert meta == attachment
api.send_message(
bob,
chat["id"],
"",
attachment_ids=[attachment["id"]],
)
assert_ok(api.request("GET", "/health/ready"), 200, "Health")
assert_content(
api.attachment_content(bob, attachment["id"]),
data,
"image/png",
)
@pytest.mark.spec("attachments", "S3 завис")
def test_object_storage_paused(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
data = images.sample_png()
attachment = api.upload(alice, chat["id"], data)
# срок клиента больше срока вызова S3: ждём ответ API, а не свой обрыв
slow = Api(str(api.client.base_url), timeout=3 * S3_TIMEOUT_S)
with slow, outage(stand, "pause", "s3"):
for call in (
lambda: upload_raw(slow, alice, chat, data),
lambda: slow.attachment_content(alice, attachment["id"]),
):
resp, elapsed = timed(call)
assert_error(resp, 503)
# срок вызова S3 и немного на шаги с БД, а не зависание
assert elapsed < S3_TIMEOUT_S + 2 * SLACK_S, (
f"503 after {elapsed:.2f}s"
)
assert_content(
api.attachment_content(alice, attachment["id"]),
data,
"image/png",
)