forked from mtusi2026/mtusi2026lab4
100 lines
3.8 KiB
Python
100 lines
3.8 KiB
Python
"""
|
||
Недоступность S3: контейнер s3 останавливается через docker compose
|
||
(volume сохраняется), затем запускается снова.
|
||
|
||
Ожидаемое поведение: загрузка и скачивание — быстрый 503 в формате контракта,
|
||
readiness — 503 (вложения входят в обязательный путь lab4), но метаданные
|
||
и сообщения продолжают работать: nginx не исключает реплики по readiness.
|
||
После восстановления незавершённых строк не остаётся: компенсация или
|
||
janitor их дочищают.
|
||
|
||
Запуск явно: make test-s3-failure (RUN_FAILURE_TESTS=1).
|
||
"""
|
||
|
||
import os
|
||
import time
|
||
|
||
import httpx
|
||
import pytest
|
||
from helpers import (
|
||
assert_error,
|
||
auth,
|
||
compose,
|
||
png_bytes,
|
||
psql,
|
||
run_janitor,
|
||
upload_raw,
|
||
)
|
||
|
||
pytestmark = pytest.mark.skipif(
|
||
os.environ.get("RUN_FAILURE_TESTS") != "1",
|
||
reason="останавливает контейнер s3: запускайте через make test-s3-failure",
|
||
)
|
||
|
||
# connect timeout 2 с × 2 попытки + проверка изображения
|
||
BOUNDED_RESPONSE_SEC = 6
|
||
READY_TIMEOUT_SEC = 60
|
||
|
||
|
||
def unfinished_rows(chat_id: str) -> int:
|
||
return int(
|
||
psql(
|
||
f"SELECT count(*) FROM attachments "
|
||
f"WHERE chat_id = '{chat_id}' AND status <> 'ready';"
|
||
)
|
||
)
|
||
|
||
|
||
def wait_upload_works(api: httpx.Client, user: dict, chat_id: str) -> httpx.Response:
|
||
deadline = time.monotonic() + READY_TIMEOUT_SEC
|
||
while True:
|
||
response = upload_raw(api, user, chat_id, png_bytes())
|
||
if response.status_code == 201 or time.monotonic() > deadline:
|
||
return response
|
||
time.sleep(1)
|
||
|
||
|
||
def test_s3_outage_is_isolated_and_leaves_no_garbage(
|
||
api: httpx.Client, make_user, make_chat, upload, send_attachments
|
||
) -> None:
|
||
alice = make_user("alice")
|
||
bob = make_user("bob")
|
||
chat = make_chat(alice, [alice, bob])
|
||
before = upload(alice, chat)
|
||
|
||
compose("stop", "s3")
|
||
try:
|
||
started = time.monotonic()
|
||
failed_upload = upload_raw(api, alice, chat["id"], png_bytes(64, 64))
|
||
elapsed = time.monotonic() - started
|
||
assert_error(failed_upload, 503, "unavailable")
|
||
assert elapsed < BOUNDED_RESPONSE_SEC, elapsed
|
||
assert "s3" not in failed_upload.json()["error"]["message"].lower()
|
||
|
||
download = api.get(
|
||
f"/api/v1/attachments/{before['id']}/content", headers=auth(bob)
|
||
)
|
||
assert_error(download, 503, "unavailable")
|
||
|
||
# Путь без байтов S3 работает: метаданные и сообщения
|
||
metadata = api.get(f"/api/v1/attachments/{before['id']}", headers=auth(bob))
|
||
assert metadata.status_code == 200 and metadata.json() == before
|
||
message = send_attachments(alice, chat, [before["id"]], text="S3 лежит")
|
||
assert message.status_code == 201, message.text
|
||
assert api.get("/health/live").status_code == 200
|
||
started = time.monotonic()
|
||
assert_error(api.get("/health/ready"), 503, "unavailable")
|
||
assert time.monotonic() - started < BOUNDED_RESPONSE_SEC
|
||
finally:
|
||
compose("up", "-d", "--wait", "s3")
|
||
|
||
after = wait_upload_works(api, alice, chat["id"])
|
||
assert after.status_code == 201, after.text
|
||
content = api.get(f"/api/v1/attachments/{before['id']}/content", headers=auth(bob))
|
||
assert content.status_code == 200
|
||
|
||
# Компенсация не смогла удалить объект при лежащем S3: строка deleting
|
||
# дочищается janitor независимо от grace period
|
||
run_janitor()
|
||
assert unfinished_rows(chat["id"]) == 0
|