forked from mtusi2026/mtusi2026lab4
Исходники, Dockerfile, compose.yaml, скрипты и собственные тесты скопированы из mtusi2026lab3 (ветка solution/bvt2302-bodanyuk, b6847e2) как есть; изменения lab4 — отдельными коммитами. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
282 lines
13 KiB
Python
282 lines
13 KiB
Python
"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
|
|
|
|
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
|
|
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
|
|
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
|
|
undone in `finally`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
import httpx2 as httpx
|
|
import pytest
|
|
|
|
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
|
|
def messages_in_db(chat: dict) -> int:
|
|
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
|
|
|
|
|
|
def key_rows(actor: dict, key: str) -> int:
|
|
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
|
|
|
|
|
|
@pytest.fixture
|
|
def pair(api):
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
return alice, bob, api.chat(alice, bob)
|
|
|
|
|
|
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
|
|
alice, bob, chat = pair
|
|
key = "replay-" + uuid.uuid4().hex
|
|
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
|
|
served = {first.headers["x-instance-id"]}
|
|
for _ in range(20):
|
|
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
|
|
assert again.status_code == 201 and again.json() == first.json()
|
|
served.add(again.headers["x-instance-id"])
|
|
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
|
|
assert api.history(bob, chat, 100)[0] == [first.json()]
|
|
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
|
|
|
|
|
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
|
|
alice, bob, chat = pair
|
|
key = uuid.uuid4().hex
|
|
original = api.send(alice, chat, "оригинал", key=key)
|
|
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
|
|
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
|
|
assert api.send(alice, chat, "оригинал", key=key) == original
|
|
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
|
|
|
|
|
|
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
|
|
alice, _, chat = pair
|
|
key = uuid.uuid4().hex
|
|
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
|
|
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
|
|
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
|
|
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
|
|
assert messages_in_db(chat) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
|
|
def test_missing_or_invalid_key_is_400(api, pair, value):
|
|
alice, _, chat = pair
|
|
raw_value = value.encode("utf-8")
|
|
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
|
|
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
|
|
assert error["error"]["code"] == "invalid_request"
|
|
assert messages_in_db(chat) == 0
|
|
|
|
|
|
def test_repeated_key_header_is_400(pair):
|
|
alice, _, chat = pair
|
|
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
|
|
headers=[("Authorization", "Bearer " + alice["token"]),
|
|
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
|
|
assert response.status_code == 400 and response.headers["x-instance-id"]
|
|
assert messages_in_db(chat) == 0
|
|
|
|
|
|
def test_longest_key_and_every_allowed_character(api, pair):
|
|
alice, _, chat = pair
|
|
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
|
|
first = api.send(alice, chat, key[:20], key=key)
|
|
assert api.send(alice, chat, key[:20], key=key) == first
|
|
assert messages_in_db(chat) == 2
|
|
|
|
|
|
def test_scope_is_user_plus_chat_plus_key(api, pair):
|
|
alice, bob, chat = pair
|
|
other = api.chat(alice, bob, title="второй")
|
|
key = "shared-key"
|
|
a1 = api.send(alice, chat, "текст", key=key)
|
|
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
|
|
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
|
|
assert len({a1["id"], b1["id"], a2["id"]}) == 3
|
|
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
|
|
|
|
|
|
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
|
|
alice, bob, chat = pair
|
|
eve = api.user("eve")
|
|
key = uuid.uuid4().hex
|
|
api.send(alice, chat, "секрет", key=key)
|
|
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
|
|
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
|
|
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
|
|
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
|
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
|
|
assert messages_in_db(chat) == 1
|
|
|
|
|
|
def fire_together(workers: int, request) -> list:
|
|
barrier = threading.Barrier(workers)
|
|
results: list = [None] * workers
|
|
|
|
def body(index: int) -> None:
|
|
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
|
|
barrier.wait()
|
|
try:
|
|
results[index] = request(client, index)
|
|
except BaseException as exc: # noqa: BLE001 - asserted by the test
|
|
results[index] = exc
|
|
finally:
|
|
client.close()
|
|
|
|
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
return results
|
|
|
|
|
|
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
|
|
alice, bob, chat = pair
|
|
key = "race-" + uuid.uuid4().hex
|
|
|
|
def post(client: httpx.Client, _index: int):
|
|
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
|
|
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
|
return r.status_code, r.headers.get("x-instance-id"), r.json()
|
|
|
|
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
|
|
results = fire_together(24, post)
|
|
assert all(isinstance(r, tuple) for r in results), results
|
|
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
|
|
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
|
|
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
|
|
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
|
|
|
|
|
def test_concurrent_same_key_different_bodies_one_wins(pair):
|
|
alice, _, chat = pair
|
|
key = "conflict-" + uuid.uuid4().hex
|
|
|
|
def post(client: httpx.Client, index: int):
|
|
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
|
|
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
|
return r.status_code, r.json()
|
|
|
|
results = fire_together(20, post)
|
|
winners = [body for status, body in results if status == 201]
|
|
losers = [body for status, body in results if status == 409]
|
|
assert len(winners) + len(losers) == 20, results
|
|
assert winners and len({str(b) for b in winners}) == 1
|
|
assert all(b["error"]["code"] == "conflict" for b in losers)
|
|
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
|
|
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
|
|
assert messages_in_db(chat) == 1
|
|
|
|
|
|
def inserts_into_keys() -> int:
|
|
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
|
|
|
|
|
|
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
|
|
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
|
|
transaction. Expected: 500 for the client, no key row and no message left behind
|
|
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
|
|
alice, bob, chat = pair
|
|
marker = "boom-" + uuid.uuid4().hex
|
|
key = uuid.uuid4().hex
|
|
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
|
|
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
|
|
"RETURN NEW; END $$; "
|
|
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
|
|
try:
|
|
before = inserts_into_keys()
|
|
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
|
|
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
|
|
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
|
|
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
|
|
while inserts_into_keys() == before and time.monotonic() < deadline:
|
|
time.sleep(0.5)
|
|
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
|
|
finally:
|
|
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
|
|
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
|
|
assert api.send(alice, chat, marker, key=key) == retried
|
|
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
|
|
|
|
|
|
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
|
|
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
|
|
f"SELECT pg_sleep({seconds}); COMMIT;")
|
|
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
|
|
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
def wait_lock_taken(chat: dict) -> None:
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
|
|
"AND pid <> pg_backend_pid()") != "0":
|
|
return
|
|
time.sleep(0.2)
|
|
raise AssertionError("the row lock was not taken")
|
|
|
|
|
|
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
|
|
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
|
|
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
|
|
alice, bob, chat = pair
|
|
key = uuid.uuid4().hex
|
|
locker = lock_chat_row(chat, 1.0)
|
|
try:
|
|
wait_lock_taken(chat)
|
|
with pytest.raises(httpx.TimeoutException):
|
|
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
|
|
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
|
finally:
|
|
locker.wait(timeout=30)
|
|
deadline = time.monotonic() + 10
|
|
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
|
|
time.sleep(0.2)
|
|
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
|
|
assert stored, "the server-side transaction was expected to commit after the lock was released"
|
|
retried = api.send(alice, chat, "неопределённо", key=key)
|
|
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
|
|
|
|
|
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
|
|
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
|
|
nothing; after the lock is gone the same key creates the message exactly once."""
|
|
alice, bob, chat = pair
|
|
key = uuid.uuid4().hex
|
|
locker = lock_chat_row(chat, 4.0)
|
|
try:
|
|
wait_lock_taken(chat)
|
|
started = time.monotonic()
|
|
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
|
|
assert time.monotonic() - started < 3.6
|
|
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
|
|
finally:
|
|
locker.wait(timeout=30)
|
|
created = api.send(alice, chat, "после блокировки", key=key)
|
|
assert api.send(alice, chat, "после блокировки", key=key) == created
|
|
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
|
|
|
|
|
|
def test_replay_after_the_serving_replica_is_stopped(api, pair):
|
|
alice, bob, chat = pair
|
|
key = uuid.uuid4().hex
|
|
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
|
|
served_by = first.headers["x-instance-id"]
|
|
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
|
|
with replica_stopped(replica):
|
|
for _ in range(5):
|
|
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
|
|
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
|
|
assert messages_in_db(chat) == 1
|