755 lines
35 KiB
Python
755 lines
35 KiB
Python
"""Проверки отказов (уровень «4»): выполняет только `make failure-checks`.
|
||
|
||
Стенд и hook готовит `student-tests/stand.py failures`: свои PostgreSQL и
|
||
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
|
||
идемпотентности 2 с, пул из одного соединения, срок ожидания пула 1 с,
|
||
срок операции 2 с). На хранилища тесты действуют только через фикстуру
|
||
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
|
||
рабочее состояние (`outage`): упавшая проверка не ломает следующие.
|
||
|
||
Что проверяется: истечение сессии и то, что запросы её не продлевают;
|
||
недоступная БД — 503 без деталей и без вызова аутентификации, а не 401 и
|
||
не анонимный доступ; ограниченное ожидание при замёрзшей БД и
|
||
исчерпанном пуле; откат создания чата; readiness; обход кэша при отказе
|
||
Valkey и его возврат без рестарта API. С лабы 3 — ключ идемпотентности:
|
||
сбой записи сообщения откатывает и ключ, транзиентная 503 не фиксируется
|
||
результатом (повтор с тем же ключом проходит один раз), срок ключа.
|
||
С лабы 4 — вложения: отказ и заморозка S3 (503 в пределах срока, текст и
|
||
готовность работают), сбой фиксации `ready` после записанного объекта
|
||
(компенсация удаляет объект), kill реплики между PUT и фиксацией (строку
|
||
и объект убирает уборщик после grace), места загрузок (429 с
|
||
`Retry-After`) и срок тела (400), сверка bucket и БД с расхождениями на
|
||
обеих сторонах и под одновременными загрузками.
|
||
Границы времени — с запасом 0,3–1 с, но на порядок ниже «плохого»
|
||
поведения (секунды повторов redis-py, зависший до разморозки запрос).
|
||
"""
|
||
|
||
import hashlib
|
||
import os
|
||
import re
|
||
import statistics
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable, Iterator
|
||
from contextlib import contextmanager
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
import images
|
||
from helpers import (
|
||
BEARER_CHALLENGE,
|
||
CHATS,
|
||
ME,
|
||
SAMPLE_PNG,
|
||
assert_error,
|
||
assert_unauthorized,
|
||
attachment_content,
|
||
attachment_meta,
|
||
chat_ids,
|
||
create_chat,
|
||
headers,
|
||
login,
|
||
new_key,
|
||
public,
|
||
read_history,
|
||
register,
|
||
send,
|
||
upload,
|
||
uploaded,
|
||
)
|
||
|
||
pytestmark = pytest.mark.fault
|
||
|
||
type Stand = Callable[..., str]
|
||
|
||
RESTORE = {"stop": "start", "pause": "unpause"}
|
||
# запас к границам времени: планировщик Docker Desktop
|
||
SLACK = 0.3
|
||
# значения по умолчанию CACHE_TIMEOUT и CACHE_RETRY_AFTER
|
||
CACHE_TIMEOUT = 0.2
|
||
CACHE_RETRY_AFTER = 5.0
|
||
# окружение «тесного» API (TIGHT_ENV в stand.py)
|
||
SESSION_TTL = 2
|
||
IDEMPOTENCY_TTL = 2
|
||
POOL_TIMEOUT = 1.0
|
||
OPERATION_TIMEOUT = 2.0
|
||
# ограниченное закрытие соединения (DB_CLOSE_TIMEOUT) и срок readiness
|
||
CLOSE_TIMEOUT = 2.0
|
||
READINESS_TIMEOUT = 2.0
|
||
# лаба 4: срок вызова S3 (S3_TIMEOUT), уборщик стенда (CLEANER_ENV в
|
||
# stand.py) и «узкий по загрузкам» API (UPLOADS_ENV)
|
||
S3_TIMEOUT = 5.0
|
||
CLEANER_GRACE = float(os.environ.get("FAULT_CLEANER_GRACE", "25"))
|
||
CLEANER_INTERVAL = float(os.environ.get("FAULT_CLEANER_INTERVAL", "1"))
|
||
UPLOAD_WAIT_TIMEOUT = 1.0
|
||
UPLOAD_BODY_TIMEOUT = 2.0
|
||
|
||
|
||
@contextmanager
|
||
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
||
"""`stop` или `pause` хранилища; в `finally` — обратно."""
|
||
stand(action, service)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand(RESTORE[action], service)
|
||
|
||
|
||
def timed(call: Callable[[], httpx.Response]) -> tuple[httpx.Response, float]:
|
||
started = time.monotonic()
|
||
response = call()
|
||
return response, time.monotonic() - started
|
||
|
||
|
||
def read_chat(client: httpx.Client, actor: dict[str, Any], chat: dict[str, Any]):
|
||
return client.get(f"{CHATS}/{chat['id']}", headers=headers(actor))
|
||
|
||
|
||
def wait_cache_ready(
|
||
client: httpx.Client,
|
||
stand: Stand,
|
||
actor: dict[str, Any],
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
||
|
||
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
||
"""
|
||
deadline = time.monotonic() + CACHE_RETRY_AFTER + 2
|
||
stand("flush-cache")
|
||
while time.monotonic() < deadline:
|
||
assert read_chat(client, actor, chat).status_code == 200
|
||
if stand("cache-has-chat", chat["id"]) == "1":
|
||
return
|
||
time.sleep(0.5)
|
||
raise AssertionError("cache did not come back")
|
||
|
||
|
||
# --- сессии: срок 2 с на «тесном» API, БД остановлена — на обычном -----------
|
||
|
||
|
||
def test_expired_token_is_401(tight_client: httpx.Client) -> None:
|
||
actor = register(tight_client, "ttl")
|
||
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
|
||
time.sleep(SESSION_TTL + 1)
|
||
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
|
||
|
||
|
||
def test_requests_do_not_extend_session(tight_client: httpx.Client) -> None:
|
||
actor = register(tight_client, "ttl")
|
||
logged_in = time.monotonic()
|
||
successes = 0
|
||
while time.monotonic() - logged_in < SESSION_TTL - SLACK:
|
||
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
|
||
successes += 1
|
||
time.sleep(0.5)
|
||
assert successes >= 3, "too few requests before expiry"
|
||
time.sleep(max(0.0, logged_in + SESSION_TTL + 1 - time.monotonic()))
|
||
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
|
||
|
||
|
||
def test_token_while_database_stopped_is_503(
|
||
client: httpx.Client, stand: Stand
|
||
) -> None:
|
||
actor = register(client, "db")
|
||
with outage(stand, "stop", "postgres"):
|
||
response = client.get(ME, headers=headers(actor))
|
||
assert_error(response, 503, "unavailable")
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in response.headers
|
||
# тот же токен после возврата БД
|
||
assert client.get(ME, headers=headers(actor)).json() == public(actor)
|
||
|
||
|
||
# --- хранение: откат, остановленная и замёрзшая БД, исчерпанный пул ------------
|
||
|
||
|
||
def test_members_failure_rolls_back_chat(client: httpx.Client, stand: Stand) -> None:
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
title = f"fault-{uuid.uuid4().hex}"
|
||
stand("fail-members", title)
|
||
try:
|
||
response = client.post(
|
||
CHATS,
|
||
json={"title": title, "member_ids": [alice["id"], bob["id"]]},
|
||
headers=headers(alice),
|
||
)
|
||
assert_error(response, 500, "internal_error")
|
||
finally:
|
||
stand("clear-fail-members")
|
||
# ни чата, ни участников: вставка строки чата откатилась вместе с ними
|
||
for actor in (alice, bob):
|
||
assert chat_ids(client, actor) == []
|
||
assert stand("count-chats", title) == "0"
|
||
|
||
|
||
def test_database_stopped_and_back(client: httpx.Client, stand: Stand) -> None:
|
||
alice = register(client, "alice")
|
||
chat = create_chat(client, alice)
|
||
with outage(stand, "stop", "postgres"):
|
||
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
|
||
error = assert_error(response, 503, "unavailable")
|
||
print(f"postgres stopped: POST message -> 503 in {took:.3f}s")
|
||
assert took < 2, f"503 took {took:.2f}s"
|
||
for word in ("postgres", "sql", "connect", "asyncpg"):
|
||
assert word not in error["message"].lower()
|
||
after = send(client, alice, chat, "после возврата")
|
||
assert after.status_code == 201, after.text
|
||
# запись во время отказа не зафиксирована, новая — видна
|
||
assert read_history(client, alice, chat, limit=50)[0] == [after.json()]
|
||
|
||
|
||
def test_frozen_database_and_exhausted_pool(
|
||
tight_client: httpx.Client,
|
||
stand: Stand,
|
||
) -> None:
|
||
"""Замёрзшая БД держит запрос не дольше срока операции и закрытия
|
||
соединения; пока пул занят, следующий запрос ждёт не дольше срока пула."""
|
||
actor = register(tight_client, "pool")
|
||
first: list[tuple[int, float]] = []
|
||
|
||
def hold_connection() -> None:
|
||
with httpx.Client(base_url=str(tight_client.base_url), timeout=20) as holder:
|
||
response, took = timed(lambda: holder.get(ME, headers=headers(actor)))
|
||
first.append((response.status_code, took))
|
||
|
||
with outage(stand, "pause", "postgres"):
|
||
thread = threading.Thread(target=hold_connection)
|
||
thread.start()
|
||
time.sleep(SLACK)
|
||
response, took = timed(lambda: tight_client.get(ME, headers=headers(actor)))
|
||
assert_error(response, 503, "unavailable")
|
||
assert took < POOL_TIMEOUT + 0.5, f"pool wait took {took:.2f}s"
|
||
thread.join()
|
||
status, held = first[0]
|
||
print(
|
||
f"postgres paused: first request -> {status} in {held:.2f}s, "
|
||
f"second (pool of 1 busy) -> {response.status_code} in {took:.2f}s",
|
||
)
|
||
assert status == 503
|
||
assert held < OPERATION_TIMEOUT + CLOSE_TIMEOUT + 1, f"frozen db held {held:.2f}s"
|
||
# срок сессии 2 с мог истечь за время паузы: пул проверяем новым входом
|
||
fresh = register(tight_client, "pool")
|
||
assert tight_client.get(ME, headers=headers(fresh)).status_code == 200
|
||
|
||
|
||
# --- ключ идемпотентности (лаба 3) ----------------------------------------------
|
||
|
||
|
||
def test_message_failure_rolls_back_key(client: httpx.Client, stand: Stand) -> None:
|
||
"""Ключ вставлен первым, сообщение упало — откат убирает оба: повтор с
|
||
тем же ключом создаёт сообщение, а не 409 и не «пустой» повтор."""
|
||
alice = register(client, "alice")
|
||
chat = create_chat(client, alice)
|
||
text = f"fault-{uuid.uuid4().hex}"
|
||
key = new_key()
|
||
stand("fail-messages", text)
|
||
try:
|
||
assert_error(send(client, alice, chat, text, key=key), 500, "internal_error")
|
||
finally:
|
||
stand("clear-fail-messages")
|
||
assert stand("count-keys", chat["id"]) == "0"
|
||
assert read_history(client, alice, chat, limit=50)[0] == []
|
||
retry = send(client, alice, chat, text, key=key)
|
||
assert retry.status_code == 201, retry.text
|
||
assert read_history(client, alice, chat, limit=50)[0] == [retry.json()]
|
||
assert stand("count-keys", chat["id"]) == "1"
|
||
|
||
|
||
def test_unavailable_is_not_saved_as_result(client: httpx.Client, stand: Stand) -> None:
|
||
"""503 — транзиентный результат: тот же ключ после возврата БД даёт 201,
|
||
повтор — то же сообщение, в истории одно."""
|
||
alice = register(client, "alice")
|
||
chat = create_chat(client, alice)
|
||
key = new_key()
|
||
with outage(stand, "stop", "postgres"):
|
||
assert_error(
|
||
send(client, alice, chat, "неопределённо", key=key), 503, "unavailable"
|
||
)
|
||
first = send(client, alice, chat, "неопределённо", key=key)
|
||
assert first.status_code == 201, first.text
|
||
again = send(client, alice, chat, "неопределённо", key=key)
|
||
assert again.json() == first.json()
|
||
assert read_history(client, alice, chat, limit=50)[0] == [first.json()]
|
||
|
||
|
||
def test_key_expires_after_ttl(tight_client: httpx.Client) -> None:
|
||
"""Срок ключа 2 с на «тесном» API: в срок — повтор и 409, после срока
|
||
тот же ключ занимается заново (другое тело — новое сообщение, не 409)."""
|
||
alice = register(tight_client, "ttl")
|
||
chat = create_chat(tight_client, alice)
|
||
key = new_key()
|
||
first = send(tight_client, alice, chat, "в срок", key=key).json()
|
||
assert send(tight_client, alice, chat, "в срок", key=key).json() == first
|
||
assert_error(send(tight_client, alice, chat, "другое", key=key), 409, "conflict")
|
||
time.sleep(IDEMPOTENCY_TTL + 1)
|
||
# срок сессии на этом API тоже 2 с: новый вход
|
||
alice["token"] = login(tight_client, alice)
|
||
late = send(tight_client, alice, chat, "после срока", key=key)
|
||
assert late.status_code == 201, late.text
|
||
assert late.json()["id"] != first["id"]
|
||
items, _ = read_history(tight_client, alice, chat, limit=50)
|
||
assert [item["text"] for item in items] == ["в срок", "после срока"]
|
||
|
||
|
||
# --- readiness -------------------------------------------------------------------
|
||
|
||
|
||
def test_readiness_without_database(client: httpx.Client, stand: Stand) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
response, took = timed(lambda: client.get("/health/ready"))
|
||
assert_error(response, 503, "unavailable")
|
||
print(f"postgres stopped: /health/ready -> 503 in {took:.3f}s")
|
||
assert took < READINESS_TIMEOUT + 0.5, f"ready took {took:.2f}s"
|
||
assert client.get("/health/live").status_code == 200
|
||
assert client.get("/health/ready").status_code == 200
|
||
|
||
|
||
def test_readiness_without_cache(client: httpx.Client, stand: Stand) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert client.get("/health/ready").status_code == 200
|
||
|
||
|
||
# --- кэш карточки чата -------------------------------------------------------------
|
||
|
||
|
||
def test_flush_cache_refills_on_read(client: httpx.Client, stand: Stand) -> None:
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
chat = create_chat(client, alice, bob)
|
||
wait_cache_ready(client, stand, bob, chat)
|
||
stand("flush-cache")
|
||
assert stand("cache-has-chat", chat["id"]) == "0"
|
||
assert read_chat(client, bob, chat).json() == chat
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
|
||
|
||
def test_valkey_stopped_is_bypassed(client: httpx.Client, stand: Stand) -> None:
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
chat = create_chat(client, alice, bob)
|
||
sent = [send(client, alice, chat, "до отказа").json()]
|
||
with outage(stand, "stop", "valkey"):
|
||
response, read_took = timed(lambda: read_chat(client, bob, chat))
|
||
assert response.json() == chat
|
||
assert read_took < CACHE_TIMEOUT + SLACK, f"read took {read_took:.2f}s"
|
||
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
|
||
assert response.status_code == 201, response.text
|
||
print(
|
||
f"valkey stopped: GET chat -> 200 in {read_took:.3f}s, "
|
||
f"POST message -> 201 in {took:.3f}s",
|
||
)
|
||
assert took < CACHE_TIMEOUT + SLACK, f"send took {took:.2f}s"
|
||
sent.append(response.json())
|
||
assert read_history(client, bob, chat, limit=50)[0] == sent
|
||
|
||
|
||
def test_valkey_paused_costs_one_timeout(client: httpx.Client, stand: Stand) -> None:
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
chat = create_chat(client, alice, bob)
|
||
wait_cache_ready(client, stand, bob, chat)
|
||
baseline = statistics.median(
|
||
timed(lambda: read_chat(client, bob, chat))[1] for _ in range(5)
|
||
)
|
||
with outage(stand, "pause", "valkey"):
|
||
response, took = timed(lambda: read_chat(client, bob, chat))
|
||
assert response.json() == chat
|
||
assert took < baseline + CACHE_TIMEOUT + SLACK, (
|
||
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
||
)
|
||
# окно пропуска: следующий запрос срок не ждёт
|
||
response, second = timed(lambda: read_chat(client, bob, chat))
|
||
assert response.json() == chat
|
||
print(
|
||
f"valkey paused: baseline {baseline:.3f}s, first read {took:.3f}s, "
|
||
f"second read {second:.3f}s",
|
||
)
|
||
assert second < CACHE_TIMEOUT, f"second read took {second:.2f}s"
|
||
|
||
|
||
def test_valkey_back_without_api_restart(client: httpx.Client, stand: Stand) -> None:
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
chat = create_chat(client, alice, bob)
|
||
with outage(stand, "stop", "valkey"):
|
||
assert read_chat(client, bob, chat).status_code == 200
|
||
# API не перезапускался; кэш снова в работе после окна пропуска
|
||
time.sleep(CACHE_RETRY_AFTER + 0.5)
|
||
assert read_chat(client, bob, chat).status_code == 200
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
hits = int(stand("cache-hits"))
|
||
assert read_chat(client, bob, chat).json() == chat
|
||
assert int(stand("cache-hits")) > hits
|
||
|
||
|
||
# --- лаба 4: S3 недоступен ----------------------------------------------------------
|
||
|
||
|
||
def counts(stand: Stand, chat: dict[str, Any]) -> tuple[int, int]:
|
||
"""Строк вложений чата: (pending, ready)."""
|
||
pending, ready = stand("count-attachments", chat["id"]).split()
|
||
return int(pending), int(ready)
|
||
|
||
|
||
def ready(base_url: str) -> bool:
|
||
try:
|
||
return httpx.get(f"{base_url}/health/ready", timeout=2).status_code == 200
|
||
except httpx.HTTPError:
|
||
return False
|
||
|
||
|
||
def wait_until(check: Callable[[], bool], timeout: float, step: float = 0.5) -> float:
|
||
"""Секунды до выполнения условия; не выполнилось за срок — падение."""
|
||
started = time.monotonic()
|
||
while time.monotonic() - started < timeout:
|
||
if check():
|
||
return time.monotonic() - started
|
||
time.sleep(step)
|
||
raise AssertionError(f"condition not met in {timeout:.0f}s")
|
||
|
||
|
||
def test_upload_with_s3_stopped(client: httpx.Client, stand: Stand) -> None:
|
||
"""S3 остановлен: загрузка — 503 быстро, строка `pending` остаётся
|
||
уборщику, объекта нет; текст и готовность работают."""
|
||
alice = register(client, "alice")
|
||
chat = create_chat(client, alice)
|
||
with outage(stand, "stop", "s3"):
|
||
response, took = timed(lambda: upload(client, alice, chat))
|
||
error = assert_error(response, 503, "unavailable")
|
||
print(f"s3 stopped: upload -> 503 in {took:.3f}s")
|
||
assert took < S3_TIMEOUT, f"503 took {took:.2f}s"
|
||
for word in ("s3", "boto", "bucket", "seaweed", "http://"):
|
||
assert word not in error["message"].lower()
|
||
assert counts(stand, chat) == (1, 0)
|
||
assert send(client, alice, chat, "текст без S3").status_code == 201
|
||
assert client.get("/health/ready").status_code == 200
|
||
# тот же файл после возврата — 201
|
||
uploaded(client, alice, chat)
|
||
assert counts(stand, chat)[1] == 1
|
||
|
||
|
||
def test_upload_and_download_with_s3_paused(client: httpx.Client, stand: Stand) -> None:
|
||
"""Замороженный S3 держит загрузку и скачивание не дольше срока вызова;
|
||
метаданные и ссылка в сообщении без S3 работают."""
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
chat = create_chat(client, alice, bob)
|
||
ready = uploaded(client, alice, chat)
|
||
with outage(stand, "pause", "s3"):
|
||
response, upload_took = timed(lambda: upload(client, alice, chat))
|
||
assert_error(response, 503, "unavailable")
|
||
response, download_took = timed(
|
||
lambda: attachment_content(client, bob, ready["id"])
|
||
)
|
||
assert_error(response, 503, "unavailable")
|
||
print(
|
||
f"s3 paused: upload -> 503 in {upload_took:.2f}s, "
|
||
f"download -> 503 in {download_took:.2f}s",
|
||
)
|
||
assert upload_took < S3_TIMEOUT + 1.5, f"upload took {upload_took:.2f}s"
|
||
assert download_took < S3_TIMEOUT + 1.5, f"download took {download_took:.2f}s"
|
||
assert attachment_meta(client, bob, ready["id"]).json() == ready
|
||
linked = send(client, alice, chat, "", attachment_ids=[ready["id"]])
|
||
assert linked.status_code == 201, linked.text
|
||
content = attachment_content(client, bob, ready["id"])
|
||
assert content.content == SAMPLE_PNG
|
||
|
||
|
||
def test_readiness_without_object_storage(client: httpx.Client, stand: Stand) -> None:
|
||
"""Без S3 реплика остаётся готовой: иначе отказ S3 вывел бы из LB все
|
||
реплики, и текст перестал бы работать тоже."""
|
||
with outage(stand, "stop", "s3"):
|
||
assert client.get("/health/ready").status_code == 200
|
||
|
||
|
||
# --- лаба 4: согласованность БД и S3 ------------------------------------------------
|
||
|
||
|
||
def test_ready_failure_compensates_and_cleaner_removes_row(
|
||
client: httpx.Client, stand: Stand
|
||
) -> None:
|
||
"""Объект записан, а перевод в `ready` падает как отказ PostgreSQL: 503,
|
||
объект удалён до ответа (компенсация), строку `pending` убирает
|
||
уборщик после grace. Объектов без строки не остаётся."""
|
||
alice = register(client, "alice")
|
||
chat = create_chat(client, alice)
|
||
filename = f"fault-{uuid.uuid4().hex}.png"
|
||
objects_before = int(stand("count-objects"))
|
||
stand("fail-ready", filename)
|
||
try:
|
||
assert_error(upload(client, alice, chat, filename=filename), 503, "unavailable")
|
||
finally:
|
||
stand("clear-fail-ready")
|
||
assert counts(stand, chat) == (1, 0)
|
||
assert int(stand("count-objects")) == objects_before
|
||
took = wait_until(
|
||
lambda: counts(stand, chat) == (0, 0),
|
||
CLEANER_GRACE + CLEANER_INTERVAL + 10,
|
||
)
|
||
print(f"ready failed: object compensated at once, row removed after {took:.1f}s")
|
||
assert int(stand("count-objects")) == objects_before
|
||
|
||
|
||
def test_kill_between_put_and_ready(client: httpx.Client, stand: Stand) -> None:
|
||
"""Процесс API убит после PUT и до фиксации `ready`: клиент 201 не
|
||
получил, после grace уборщик убрал и строку, и объект."""
|
||
alice = register(client, "alice")
|
||
chat = create_chat(client, alice)
|
||
filename = f"stall-{uuid.uuid4().hex}.png"
|
||
objects_before = int(stand("count-objects"))
|
||
stand("stall-ready", filename, "5")
|
||
outcome: list[Any] = []
|
||
|
||
def slow_upload() -> None:
|
||
with httpx.Client(base_url=str(client.base_url), timeout=20) as own:
|
||
try:
|
||
outcome.append(upload(own, alice, chat, filename=filename).status_code)
|
||
except httpx.HTTPError as exc:
|
||
outcome.append(type(exc).__name__)
|
||
|
||
thread = threading.Thread(target=slow_upload)
|
||
try:
|
||
thread.start()
|
||
# объект уже записан, перевод в ready ждёт в триггере
|
||
wait_until(lambda: int(stand("count-objects")) == objects_before + 1, 10, 0.2)
|
||
stand("kill-api")
|
||
thread.join(timeout=20)
|
||
finally:
|
||
stand("clear-fail-ready")
|
||
stand("start-api")
|
||
print(f"killed between PUT and ready: client got {outcome}")
|
||
assert outcome
|
||
assert outcome[0] != 201
|
||
# keep-alive соединение фикстуры оборвано вместе с процессом: готовность
|
||
# — новым соединением, отказ соединения — «ещё не готов»
|
||
wait_until(lambda: ready(str(client.base_url)), 30)
|
||
assert counts(stand, chat) == (1, 0)
|
||
took = wait_until(
|
||
lambda: (
|
||
counts(stand, chat) == (0, 0)
|
||
and int(stand("count-objects")) == objects_before
|
||
),
|
||
CLEANER_GRACE + CLEANER_INTERVAL + 15,
|
||
)
|
||
print(f"cleaner removed row and object {took:.1f}s after the kill check")
|
||
|
||
|
||
# --- лаба 4: места загрузок и срок тела ---------------------------------------------
|
||
|
||
|
||
def slow_body(data: bytes, pause: float) -> Iterator[bytes]:
|
||
"""Тело в два приёма с паузой: загрузка держит место всё это время."""
|
||
half = len(data) // 2
|
||
yield data[:half]
|
||
time.sleep(pause)
|
||
yield data[half:]
|
||
|
||
|
||
def test_upload_slots_busy_is_429(uploads_client: httpx.Client) -> None:
|
||
"""Одно место и одно ожидающее: третья одновременная загрузка — сразу
|
||
429 с `Retry-After`, ожидавшая — 429 по сроку ожидания места, у отказов
|
||
нет ни строки, ни объекта. Лимиты контракта окружение не меняет."""
|
||
alice = register(uploads_client, "alice")
|
||
chat = create_chat(uploads_client, alice)
|
||
url = str(uploads_client.base_url)
|
||
results: dict[str, httpx.Response] = {}
|
||
|
||
def holder() -> None:
|
||
with httpx.Client(base_url=url, timeout=20) as own:
|
||
# медленное тело в пределах срока тела: место занято ~1,5 с
|
||
results["holder"] = upload(own, alice, chat, slow_body(SAMPLE_PNG, 1.5))
|
||
|
||
def waiter() -> None:
|
||
with httpx.Client(base_url=url, timeout=20) as own:
|
||
results["waiter"] = upload(own, alice, chat)
|
||
|
||
first = threading.Thread(target=holder)
|
||
first.start()
|
||
time.sleep(0.4)
|
||
second = threading.Thread(target=waiter)
|
||
second.start()
|
||
time.sleep(0.2)
|
||
third, took = timed(lambda: upload(uploads_client, alice, chat))
|
||
first.join()
|
||
second.join()
|
||
error = assert_error(third, 429, "rate_limited")
|
||
assert third.headers["Retry-After"] == "1"
|
||
print(f"slots busy: third upload -> 429 in {took:.3f}s ({error['message']})")
|
||
assert took < UPLOAD_WAIT_TIMEOUT, f"queue-full 429 took {took:.2f}s"
|
||
assert_error(results["waiter"], 429, "rate_limited")
|
||
assert results["holder"].status_code == 201, results["holder"].text
|
||
# MAX_UPLOAD_BYTES=1 в окружении этого API лимит не изменил
|
||
assert results["holder"].json()["size_bytes"] == len(SAMPLE_PNG)
|
||
|
||
|
||
def test_upload_body_timeout_is_400(uploads_client: httpx.Client, stand: Stand) -> None:
|
||
alice = register(uploads_client, "alice")
|
||
chat = create_chat(uploads_client, alice)
|
||
response, took = timed(
|
||
lambda: upload(
|
||
uploads_client, alice, chat, slow_body(SAMPLE_PNG, UPLOAD_BODY_TIMEOUT + 1)
|
||
)
|
||
)
|
||
error = assert_error(response, 400, "invalid_request")
|
||
print(f"slow body: -> 400 in {took:.2f}s ({error['message']})")
|
||
assert counts(stand, chat) == (0, 0)
|
||
# место освобождено: следующая загрузка проходит
|
||
assert upload(uploads_client, alice, chat).status_code == 201
|
||
|
||
|
||
# --- лаба 4, трек 2: сверка bucket и БД ---------------------------------------------
|
||
|
||
|
||
@contextmanager
|
||
def cleaner_stopped(stand: Stand) -> Iterator[None]:
|
||
"""Без уборщика: зависшие загрузки сверки он не убрал бы раньше неё."""
|
||
stand("stop-cleaner")
|
||
try:
|
||
yield
|
||
finally:
|
||
stand("start-cleaner")
|
||
|
||
|
||
def reconcile(stand: Stand, *args: str) -> tuple[int, str]:
|
||
"""Разовая сверка в контейнере уборщика: код выхода и журнал."""
|
||
output = stand("reconcile", *args)
|
||
code = int(output.rsplit("exit_code=", 1)[1])
|
||
return code, output
|
||
|
||
|
||
def findings(output: str, verb: str = "found") -> dict[str, str]:
|
||
"""object_key → kind из строк «storage discrepancy found|fixed»."""
|
||
pattern = rf"storage discrepancy {verb} kind=(\S+) object_key=(\S+)"
|
||
return {key: kind for kind, key in re.findall(pattern, output)}
|
||
|
||
|
||
def test_reconcile_finds_and_fixes_both_sides(
|
||
client: httpx.Client, stand: Stand
|
||
) -> None:
|
||
"""Расхождения с обеих сторон: dry-run находит и ничего не меняет,
|
||
`--apply` исправляет безопасное, чужой объект остаётся (код 3), повтор
|
||
находит только его; вернувшийся оригинал снимает пометку."""
|
||
alice, bob = register(client, "alice"), register(client, "bob")
|
||
chat = create_chat(client, alice, bob)
|
||
prefix = stand("prefix")
|
||
with cleaner_stopped(stand):
|
||
healthy = uploaded(client, alice, chat)
|
||
missing = uploaded(client, alice, chat, images.png(64, 64, seed=1))
|
||
resized = uploaded(client, alice, chat, images.png(64, 64, seed=2))
|
||
linked = send(client, alice, chat, "", attachment_ids=[missing["id"]])
|
||
assert linked.status_code == 201
|
||
with outage(stand, "stop", "s3"):
|
||
assert_error(upload(client, alice, chat), 503, "unavailable")
|
||
assert counts(stand, chat) == (1, 3)
|
||
orphan_key = f"{prefix}{uuid.uuid4()}/original"
|
||
foreign_key = f"{prefix}not-an-attachment.txt"
|
||
stand("put-object", orphan_key, data=b"orphan bytes")
|
||
stand("put-object", foreign_key, data=b"foreign bytes")
|
||
missing_key = stand("attachment", missing["id"]).split()[3]
|
||
stand("delete-object", missing_key)
|
||
stand("set-size", resized["id"], str(resized["size_bytes"] + 1))
|
||
# строки «старше grace»; у объекта время записи не подделать — ждём
|
||
(pending_id,) = stand("pending-ids", chat["id"]).split()
|
||
for attachment_id in (healthy["id"], missing["id"], resized["id"], pending_id):
|
||
stand("age-row", attachment_id, str(CLEANER_GRACE + 60))
|
||
time.sleep(CLEANER_GRACE + 1)
|
||
# до пометки байты пропавшего объекта — 500, а не «чужой» 404
|
||
assert_error(
|
||
attachment_content(client, bob, missing["id"]), 500, "internal_error"
|
||
)
|
||
|
||
objects_before = int(stand("count-objects"))
|
||
code, output = reconcile(stand)
|
||
found = findings(output)
|
||
print("dry-run:", {key.rsplit("/", 2)[-2]: kind for key, kind in found.items()})
|
||
assert code == 3
|
||
assert found.get(orphan_key) == "orphan_object"
|
||
assert found.get(missing_key) == "missing_object"
|
||
assert (
|
||
found.get(stand("attachment", resized["id"]).split()[3]) == "size_mismatch"
|
||
)
|
||
assert found.get(foreign_key) == "foreign_object"
|
||
assert "stale_pending" in found.values()
|
||
assert int(stand("count-objects")) == objects_before
|
||
assert counts(stand, chat) == (1, 3)
|
||
|
||
code, output = reconcile(stand, "--apply")
|
||
fixed = findings(output, "fixed")
|
||
assert code == 3, output[-2000:]
|
||
assert fixed.get(orphan_key) == "orphan_object"
|
||
assert fixed.get(missing_key) == "missing_object"
|
||
assert stand("object", orphan_key) == "none"
|
||
assert counts(stand, chat) == (0, 3)
|
||
# потерянный оригинал: метаданные 200, байты 404, ссылка — по-прежнему
|
||
assert attachment_meta(client, bob, missing["id"]).status_code == 200
|
||
assert_error(attachment_content(client, bob, missing["id"]), 404, "not_found")
|
||
assert attachment_content(client, bob, healthy["id"]).content == SAMPLE_PNG
|
||
|
||
code, output = reconcile(stand)
|
||
assert code == 3
|
||
assert set(findings(output).values()) == {"foreign_object"}
|
||
stand("delete-object", foreign_key)
|
||
# оригинал вернулся (тот же SHA-256): пометка снимается
|
||
stand("put-object", missing_key, data=images.png(64, 64, seed=1))
|
||
code, output = reconcile(stand, "--apply")
|
||
assert findings(output, "fixed").get(missing_key) == "restored_object"
|
||
assert code == 0, output[-2000:]
|
||
restored = attachment_content(client, bob, missing["id"])
|
||
assert hashlib.sha256(restored.content).hexdigest() == missing["sha256"]
|
||
code, output = reconcile(stand)
|
||
summary = [line for line in output.splitlines() if "pass finished" in line]
|
||
print("after fixes:", summary[-1].split("pass finished", 1)[1][:400])
|
||
assert code == 0
|
||
|
||
|
||
def test_reconcile_apply_under_uploads(new_client: Any, stand: Stand) -> None:
|
||
"""`--apply` раз за разом под одновременными загрузками и отправками
|
||
четырёх клиентов: ни одно подтверждённое вложение не удалено и не
|
||
помечено — каждое 201 скачивается тем же SHA-256."""
|
||
setup = new_client()
|
||
alice, bob = register(setup, "alice"), register(setup, "bob")
|
||
chat = create_chat(setup, alice, bob)
|
||
stop = threading.Event()
|
||
done: list[tuple[str, str]] = []
|
||
errors: list[str] = []
|
||
touched: set[str] = set()
|
||
|
||
def worker(index: int) -> None:
|
||
own = new_client()
|
||
number = 0
|
||
while not stop.is_set():
|
||
body = images.png(48, 48, seed=index * 1000 + number)
|
||
number += 1
|
||
response = upload(own, alice, chat, body)
|
||
if response.status_code != 201:
|
||
errors.append(f"upload {response.status_code}")
|
||
continue
|
||
attachment = response.json()
|
||
done.append((attachment["id"], attachment["sha256"]))
|
||
sent = send(own, alice, chat, "", attachment_ids=[attachment["id"]])
|
||
if sent.status_code != 201:
|
||
errors.append(f"send {sent.status_code}")
|
||
|
||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)]
|
||
for thread in threads:
|
||
thread.start()
|
||
passes = []
|
||
try:
|
||
deadline = time.monotonic() + 15
|
||
while time.monotonic() < deadline:
|
||
code, output = reconcile(stand, "--apply")
|
||
passes.append(code)
|
||
# id вложения — предпоследняя часть ключа <база>/attachments/<id>/original
|
||
touched |= {key.split("/")[-2] for key in findings(output, "fixed")}
|
||
finally:
|
||
stop.set()
|
||
for thread in threads:
|
||
thread.join()
|
||
print(f"reconcile --apply passes: {passes}; uploads: {len(done)}; errors: {errors}")
|
||
assert done
|
||
assert not errors
|
||
assert not touched & {attachment_id for attachment_id, _ in done}
|
||
for attachment_id, sha256 in done:
|
||
content = attachment_content(setup, bob, attachment_id)
|
||
assert content.status_code == 200, content.text
|
||
assert hashlib.sha256(content.content).hexdigest() == sha256
|