Files
mtusi2026lab4/student-tests/test_storage_failures.py
T

100 lines
3.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Недоступность S3: контейнер s3 останавливается через docker compose
(volume сохраняется), затем запускается снова.
Ожидаемое поведение: загрузка и скачивание — быстрый 503 в формате контракта,
readiness — 503 (вложения входят в обязательный путь lab4), но метаданные
и сообщения продолжают работать: nginx не исключает реплики по readiness.
После восстановления незавершённых строк не остаётся: компенсация или
janitor их дочищают.
Запуск явно: make test-s3-failure (RUN_FAILURE_TESTS=1).
"""
import os
import time
import httpx
import pytest
from helpers import (
assert_error,
auth,
compose,
png_bytes,
psql,
run_janitor,
upload_raw,
)
pytestmark = pytest.mark.skipif(
os.environ.get("RUN_FAILURE_TESTS") != "1",
reason="останавливает контейнер s3: запускайте через make test-s3-failure",
)
# connect timeout 2 с × 2 попытки + проверка изображения
BOUNDED_RESPONSE_SEC = 6
READY_TIMEOUT_SEC = 60
def unfinished_rows(chat_id: str) -> int:
return int(
psql(
f"SELECT count(*) FROM attachments "
f"WHERE chat_id = '{chat_id}' AND status <> 'ready';"
)
)
def wait_upload_works(api: httpx.Client, user: dict, chat_id: str) -> httpx.Response:
deadline = time.monotonic() + READY_TIMEOUT_SEC
while True:
response = upload_raw(api, user, chat_id, png_bytes())
if response.status_code == 201 or time.monotonic() > deadline:
return response
time.sleep(1)
def test_s3_outage_is_isolated_and_leaves_no_garbage(
api: httpx.Client, make_user, make_chat, upload, send_attachments
) -> None:
alice = make_user("alice")
bob = make_user("bob")
chat = make_chat(alice, [alice, bob])
before = upload(alice, chat)
compose("stop", "s3")
try:
started = time.monotonic()
failed_upload = upload_raw(api, alice, chat["id"], png_bytes(64, 64))
elapsed = time.monotonic() - started
assert_error(failed_upload, 503, "unavailable")
assert elapsed < BOUNDED_RESPONSE_SEC, elapsed
assert "s3" not in failed_upload.json()["error"]["message"].lower()
download = api.get(
f"/api/v1/attachments/{before['id']}/content", headers=auth(bob)
)
assert_error(download, 503, "unavailable")
# Путь без байтов S3 работает: метаданные и сообщения
metadata = api.get(f"/api/v1/attachments/{before['id']}", headers=auth(bob))
assert metadata.status_code == 200 and metadata.json() == before
message = send_attachments(alice, chat, [before["id"]], text="S3 лежит")
assert message.status_code == 201, message.text
assert api.get("/health/live").status_code == 200
started = time.monotonic()
assert_error(api.get("/health/ready"), 503, "unavailable")
assert time.monotonic() - started < BOUNDED_RESPONSE_SEC
finally:
compose("up", "-d", "--wait", "s3")
after = wait_upload_works(api, alice, chat["id"])
assert after.status_code == 201, after.text
content = api.get(f"/api/v1/attachments/{before['id']}/content", headers=auth(bob))
assert content.status_code == 200
# Компенсация не смогла удалить объект при лежащем S3: строка deleting
# дочищается janitor независимо от grace period
run_janitor()
assert unfinished_rows(chat["id"]) == 0