Files
mtusi2026lab4/student-tests/test_idempotency.py
T
Alexey BodanyukandClaude Opus 5.5 d916d09f7b перенос: реализация lab3 без изменений (API, миграции 0001-0003, HAProxy, Valkey, тесты)
Исходники, Dockerfile, compose.yaml, скрипты и собственные тесты скопированы из
mtusi2026lab3 (ветка solution/bvt2302-bodanyuk, b6847e2) как есть; изменения lab4 —
отдельными коммитами.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 12:55:58 +03:00

282 lines
13 KiB
Python

"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
undone in `finally`.
"""
from __future__ import annotations
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
pytestmark = pytest.mark.stack
def messages_in_db(chat: dict) -> int:
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
def key_rows(actor: dict, key: str) -> int:
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
@pytest.fixture
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
alice, bob, chat = pair
key = "replay-" + uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
served = {first.headers["x-instance-id"]}
for _ in range(20):
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
assert again.status_code == 201 and again.json() == first.json()
served.add(again.headers["x-instance-id"])
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
assert api.history(bob, chat, 100)[0] == [first.json()]
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
original = api.send(alice, chat, "оригинал", key=key)
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
assert api.send(alice, chat, "оригинал", key=key) == original
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
alice, _, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
assert messages_in_db(chat) == 1
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
def test_missing_or_invalid_key_is_400(api, pair, value):
alice, _, chat = pair
raw_value = value.encode("utf-8")
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
assert error["error"]["code"] == "invalid_request"
assert messages_in_db(chat) == 0
def test_repeated_key_header_is_400(pair):
alice, _, chat = pair
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
headers=[("Authorization", "Bearer " + alice["token"]),
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
assert response.status_code == 400 and response.headers["x-instance-id"]
assert messages_in_db(chat) == 0
def test_longest_key_and_every_allowed_character(api, pair):
alice, _, chat = pair
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
first = api.send(alice, chat, key[:20], key=key)
assert api.send(alice, chat, key[:20], key=key) == first
assert messages_in_db(chat) == 2
def test_scope_is_user_plus_chat_plus_key(api, pair):
alice, bob, chat = pair
other = api.chat(alice, bob, title="второй")
key = "shared-key"
a1 = api.send(alice, chat, "текст", key=key)
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
assert len({a1["id"], b1["id"], a2["id"]}) == 3
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
alice, bob, chat = pair
eve = api.user("eve")
key = uuid.uuid4().hex
api.send(alice, chat, "секрет", key=key)
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
assert messages_in_db(chat) == 1
def fire_together(workers: int, request) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
barrier.wait()
try:
results[index] = request(client, index)
except BaseException as exc: # noqa: BLE001 - asserted by the test
results[index] = exc
finally:
client.close()
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
alice, bob, chat = pair
key = "race-" + uuid.uuid4().hex
def post(client: httpx.Client, _index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.headers.get("x-instance-id"), r.json()
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
results = fire_together(24, post)
assert all(isinstance(r, tuple) for r in results), results
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_concurrent_same_key_different_bodies_one_wins(pair):
alice, _, chat = pair
key = "conflict-" + uuid.uuid4().hex
def post(client: httpx.Client, index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.json()
results = fire_together(20, post)
winners = [body for status, body in results if status == 201]
losers = [body for status, body in results if status == 409]
assert len(winners) + len(losers) == 20, results
assert winners and len({str(b) for b in winners}) == 1
assert all(b["error"]["code"] == "conflict" for b in losers)
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
assert messages_in_db(chat) == 1
def inserts_into_keys() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
transaction. Expected: 500 for the client, no key row and no message left behind
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
alice, bob, chat = pair
marker = "boom-" + uuid.uuid4().hex
key = uuid.uuid4().hex
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
"RETURN NEW; END $$; "
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
try:
before = inserts_into_keys()
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
while inserts_into_keys() == before and time.monotonic() < deadline:
time.sleep(0.5)
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
finally:
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
assert api.send(alice, chat, marker, key=key) == retried
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
f"SELECT pg_sleep({seconds}); COMMIT;")
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_lock_taken(chat: dict) -> None:
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
"AND pid <> pg_backend_pid()") != "0":
return
time.sleep(0.2)
raise AssertionError("the row lock was not taken")
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 1.0)
try:
wait_lock_taken(chat)
with pytest.raises(httpx.TimeoutException):
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
finally:
locker.wait(timeout=30)
deadline = time.monotonic() + 10
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
time.sleep(0.2)
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
assert stored, "the server-side transaction was expected to commit after the lock was released"
retried = api.send(alice, chat, "неопределённо", key=key)
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
nothing; after the lock is gone the same key creates the message exactly once."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 4.0)
try:
wait_lock_taken(chat)
started = time.monotonic()
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
assert time.monotonic() - started < 3.6
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
finally:
locker.wait(timeout=30)
created = api.send(alice, chat, "после блокировки", key=key)
assert api.send(alice, chat, "после блокировки", key=key) == created
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
def test_replay_after_the_serving_replica_is_stopped(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
served_by = first.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica):
for _ in range(5):
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
assert messages_in_db(chat) == 1