- X-Instance-Id в каждом ответе; обязательный Idempotency-Key, таблица idempotency_keys (миграция 0002, одна транзакция, конкурентные повторы через уникальный индекс) - graceful shutdown: SIGTERM -> ready=503 -> drain 3 с -> закрытие порта - compose: lb (HAProxy) + api x2 без портов + разовый migrate + postgres, лимиты CPU/RAM - хуки stop-one/restart/restore-replicas, scripts/experiments.py (failover, idempotency, rolling) - свои тесты: 44 (13 новых); evidence: smoke, pytest, persistence, failover, эксперименты
163 lines
7.4 KiB
Python
163 lines
7.4 KiB
Python
import datetime as dt
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from fastapi.testclient import TestClient
|
|
from sqlalchemy import text
|
|
|
|
from conftest import assert_error, with_key
|
|
from core.config import settings
|
|
from core.db import engine
|
|
from main import app
|
|
from utils import lifecycle
|
|
|
|
|
|
def messages_path(chat):
|
|
return "/api/v1/chats/" + chat["id"] + "/messages"
|
|
|
|
|
|
def count_messages():
|
|
with engine.connect() as connection:
|
|
return connection.scalar(text("SELECT count(*) FROM messages"))
|
|
|
|
|
|
def test_missing_or_invalid_key_returns_400(client, chat, alice):
|
|
path = messages_path(chat)
|
|
assert_error(client.post(path, headers=alice["auth"], json={"text": "без ключа"}), 400, "invalid_request")
|
|
# Только ASCII: HTTP-заголовок с кириллицей тестовый клиент отправить не может.
|
|
for key in ("", "a b", "x" * 129, "bad/key", "bad,key"):
|
|
response = client.post(path, headers={**alice["auth"], "Idempotency-Key": key}, json={"text": "плохой ключ"})
|
|
assert_error(response, 400, "invalid_request")
|
|
assert count_messages() == 0
|
|
|
|
|
|
def test_valid_key_alphabet_and_max_length(client, chat, alice):
|
|
path = messages_path(chat)
|
|
for key in ("A-z_0.9:x", "k" * 128):
|
|
assert client.post(path, headers=with_key(alice["auth"], key), json={"text": key[:5]}).status_code == 201
|
|
|
|
|
|
def test_replay_returns_same_message_without_duplicate(client, chat, alice):
|
|
path = messages_path(chat)
|
|
headers = with_key(alice["auth"], "replay-1")
|
|
first = client.post(path, headers=headers, json={"text": "один раз"})
|
|
second = client.post(path, headers=headers, json={"text": "один раз"})
|
|
assert first.status_code == second.status_code == 201
|
|
assert first.json() == second.json()
|
|
assert count_messages() == 1
|
|
|
|
|
|
def test_json_key_order_and_whitespace_do_not_matter(client, chat, alice):
|
|
path = messages_path(chat)
|
|
headers = {**with_key(alice["auth"], "order-1"), "Content-Type": "application/json"}
|
|
first = client.post(path, headers=headers, content='{"text": "тот же текст"}')
|
|
second = client.post(path, headers=headers, content='{ "text" :"тот же текст" }')
|
|
assert first.status_code == second.status_code == 201
|
|
assert first.json() == second.json()
|
|
assert count_messages() == 1
|
|
|
|
|
|
def test_same_key_with_other_body_returns_409(client, chat, alice):
|
|
path = messages_path(chat)
|
|
headers = with_key(alice["auth"], "conflict-1")
|
|
assert client.post(path, headers=headers, json={"text": "первое"}).status_code == 201
|
|
assert_error(client.post(path, headers=headers, json={"text": "другое"}), 409, "conflict")
|
|
assert count_messages() == 1
|
|
|
|
|
|
def test_key_scope_is_user_and_chat(client, chat, alice, bob):
|
|
path = messages_path(chat)
|
|
a = client.post(path, headers=with_key(alice["auth"], "shared"), json={"text": "alice"})
|
|
b = client.post(path, headers=with_key(bob["auth"], "shared"), json={"text": "bob"})
|
|
assert a.status_code == b.status_code == 201
|
|
assert a.json()["id"] != b.json()["id"]
|
|
|
|
other = client.post("/api/v1/chats", headers=alice["auth"],
|
|
json={"title": "Другой", "member_ids": [alice["id"]]}).json()
|
|
c = client.post(messages_path(other), headers=with_key(alice["auth"], "shared"), json={"text": "alice"})
|
|
assert c.status_code == 201
|
|
assert c.json()["id"] != a.json()["id"]
|
|
assert count_messages() == 3
|
|
|
|
|
|
def test_replay_still_checks_session_and_membership(client, chat, alice, eve):
|
|
path = messages_path(chat)
|
|
assert client.post(path, headers=with_key(alice["auth"], "k1"), json={"text": "секрет"}).status_code == 201
|
|
# Посторонний с тем же ключом не получает чужое сообщение.
|
|
assert_error(client.post(path, headers=with_key(eve["auth"], "k1"), json={"text": "секрет"}), 404, "not_found")
|
|
# Без токена повтор тоже отклоняется.
|
|
assert_error(client.post(path, headers={"Idempotency-Key": "k1"}, json={"text": "секрет"}), 401, "unauthorized")
|
|
|
|
|
|
def test_failed_request_does_not_consume_key(client, chat, alice):
|
|
path = messages_path(chat)
|
|
headers = with_key(alice["auth"], "retry-after-error")
|
|
assert_error(client.post(path, headers=headers, json={"text": ""}), 400, "invalid_request")
|
|
# Тот же ключ после ошибки валидации можно использовать для корректного запроса.
|
|
assert client.post(path, headers=headers, json={"text": "теперь верно"}).status_code == 201
|
|
|
|
|
|
def test_expired_key_is_reusable(client, chat, alice):
|
|
path = messages_path(chat)
|
|
headers = with_key(alice["auth"], "ttl-1")
|
|
first = client.post(path, headers=headers, json={"text": "старое"}).json()
|
|
old = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=settings.idempotency.ttl_seconds + 60)
|
|
with engine.begin() as connection:
|
|
connection.execute(text("UPDATE idempotency_keys SET created_at = :old"), {"old": old})
|
|
# Спустя TTL тот же ключ может описывать уже другое сообщение.
|
|
second = client.post(path, headers=headers, json={"text": "новое"})
|
|
assert second.status_code == 201
|
|
assert second.json()["id"] != first["id"]
|
|
assert client.post(path, headers=headers, json={"text": "новое"}).json() == second.json()
|
|
|
|
|
|
def test_concurrent_requests_with_one_key_create_one_message(chat, alice):
|
|
path = messages_path(chat)
|
|
headers = with_key(alice["auth"], "parallel-1")
|
|
|
|
def send(_):
|
|
# Отдельный клиент на поток (без `with`: lifespan закрыл бы общий пул соединений).
|
|
return TestClient(app).post(path, headers=headers, json={"text": "гонка"})
|
|
|
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
|
responses = list(pool.map(send, range(16)))
|
|
|
|
assert {r.status_code for r in responses} == {201}
|
|
assert len({r.json()["id"] for r in responses}) == 1
|
|
assert count_messages() == 1
|
|
|
|
|
|
def test_concurrent_different_bodies_one_key_only_one_wins(chat, alice):
|
|
path = messages_path(chat)
|
|
headers = with_key(alice["auth"], "parallel-2")
|
|
|
|
def send(i):
|
|
return TestClient(app).post(path, headers=headers, json={"text": f"вариант {i % 2}"})
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
responses = list(pool.map(send, range(8)))
|
|
|
|
statuses = sorted(r.status_code for r in responses)
|
|
assert set(statuses) <= {201, 409} and 201 in statuses
|
|
assert count_messages() == 1
|
|
|
|
|
|
def test_instance_id_header_is_present_and_stable(client, alice):
|
|
first = client.get("/health/live")
|
|
second = client.get("/api/v1/users/me", headers=alice["auth"])
|
|
error = client.get("/api/v1/users/me")
|
|
assert first.headers["X-Instance-Id"] == settings.run.instance_id
|
|
assert second.headers["X-Instance-Id"] == error.headers["X-Instance-Id"] == first.headers["X-Instance-Id"]
|
|
assert error.status_code == 401
|
|
|
|
|
|
def test_ready_returns_503_while_draining(client):
|
|
assert client.get("/health/ready").status_code == 200
|
|
lifecycle._draining.set()
|
|
try:
|
|
ready = client.get("/health/ready")
|
|
assert_error(ready, 503, "unavailable")
|
|
assert client.get("/health/live").status_code == 200 # процесс жив и обслуживает запросы
|
|
finally:
|
|
lifecycle._draining.clear()
|
|
assert client.get("/health/ready").status_code == 200
|