"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer. Strict assertions: exact HTTP codes for every request, identical bodies for replays, the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always undone in `finally`. """ from __future__ import annotations import subprocess import threading import time import uuid import httpx2 as httpx import pytest from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas pytestmark = pytest.mark.stack def messages_in_db(chat: dict) -> int: return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'")) def key_rows(actor: dict, key: str) -> int: return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'")) @pytest.fixture def pair(api): alice, bob = api.user("alice"), api.user("bob") return alice, bob, api.chat(alice, bob) def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair): alice, bob, chat = pair key = "replay-" + uuid.uuid4().hex first = api.post_message(alice, chat, {"text": "один раз"}, key=key) served = {first.headers["x-instance-id"]} for _ in range(20): again = api.post_message(alice, chat, {"text": "один раз"}, key=key) assert again.status_code == 201 and again.json() == first.json() served.add(again.headers["x-instance-id"]) assert len(served) >= 2, "replays must be answered by different replicas, not a local cache" assert api.history(bob, chat, 100)[0] == [first.json()] assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair): alice, bob, chat = pair key = uuid.uuid4().hex original = api.send(alice, chat, "оригинал", key=key) for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}): assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict" assert api.send(alice, chat, "оригинал", key=key) == original assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1 def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair): alice, _, chat = pair key = uuid.uuid4().hex first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json() for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}', '{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'): assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first assert messages_in_db(chat) == 1 @pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"]) def test_missing_or_invalid_key_is_400(api, pair, value): alice, _, chat = pair raw_value = value.encode("utf-8") error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json() assert error["error"]["code"] == "invalid_request" assert messages_in_db(chat) == 0 def test_repeated_key_header_is_400(pair): alice, _, chat = pair response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10, headers=[("Authorization", "Bearer " + alice["token"]), ("Idempotency-Key", "one"), ("Idempotency-Key", "two")]) assert response.status_code == 400 and response.headers["x-instance-id"] assert messages_in_db(chat) == 0 def test_longest_key_and_every_allowed_character(api, pair): alice, _, chat = pair for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex): first = api.send(alice, chat, key[:20], key=key) assert api.send(alice, chat, key[:20], key=key) == first assert messages_in_db(chat) == 2 def test_scope_is_user_plus_chat_plus_key(api, pair): alice, bob, chat = pair other = api.chat(alice, bob, title="второй") key = "shared-key" a1 = api.send(alice, chat, "текст", key=key) b1 = api.send(bob, chat, "текст", key=key) # another user: independent a2 = api.send(alice, other, "текст", key=key) # another chat: independent assert len({a1["id"], b1["id"], a2["id"]}) == 3 assert messages_in_db(chat) == 2 and messages_in_db(other) == 1 def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair): alice, bob, chat = pair eve = api.user("eve") key = uuid.uuid4().hex api.send(alice, chat, "секрет", key=key) for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found" api.post_message(eve, chat, {"text": "другое"}, 404, key=key) api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again assert messages_in_db(chat) == 1 def fire_together(workers: int, request) -> list: barrier = threading.Barrier(workers) results: list = [None] * workers def body(index: int) -> None: client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them barrier.wait() try: results[index] = request(client, index) except BaseException as exc: # noqa: BLE001 - asserted by the test results[index] = exc finally: client.close() threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)] for t in threads: t.start() for t in threads: t.join() return results def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair): alice, bob, chat = pair key = "race-" + uuid.uuid4().hex def post(client: httpx.Client, _index: int): r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"}, headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) return r.status_code, r.headers.get("x-instance-id"), r.json() for _ in range(3): # three rounds: the first creates, later rounds replay concurrently results = fire_together(24, post) assert all(isinstance(r, tuple) for r in results), results assert [r[0] for r in results] == [201] * 24, [r[0] for r in results] assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message" assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas" assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 def test_concurrent_same_key_different_bodies_one_wins(pair): alice, _, chat = pair key = "conflict-" + uuid.uuid4().hex def post(client: httpx.Client, index: int): r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"}, headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) return r.status_code, r.json() results = fire_together(20, post) winners = [body for status, body in results if status == 201] losers = [body for status, body in results if status == 409] assert len(winners) + len(losers) == 20, results assert winners and len({str(b) for b in winners}) == 1 assert all(b["error"]["code"] == "conflict" for b in losers) assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"} assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"]) assert messages_in_db(chat) == 1 def inserts_into_keys() -> int: return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'")) def test_failure_after_the_key_insert_rolls_everything_back(api, pair): """A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same transaction. Expected: 500 for the client, no key row and no message left behind (n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds.""" alice, bob, chat = pair marker = "boom-" + uuid.uuid4().hex key = uuid.uuid4().hex psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ " f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; " "RETURN NEW; END $$; " "CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()") try: before = inserts_into_keys() error = api.post_message(alice, chat, {"text": marker}, 500, key=key) assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 deadline = time.monotonic() + 20 # table statistics are flushed asynchronously while inserts_into_keys() == before and time.monotonic() < deadline: time.sleep(0.5) assert inserts_into_keys() > before, "the key INSERT never ran before the failure" finally: psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()") retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded assert api.send(alice, chat, marker, key=key) == retried assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1 def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen: sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; " f"SELECT pg_sleep({seconds}); COMMIT;") return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def wait_lock_taken(chat: dict) -> None: deadline = time.monotonic() + 15 while time.monotonic() < deadline: if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' " "AND pid <> pg_backend_pid()") != "0": return time.sleep(0.2) raise AssertionError("the row lock was not taken") def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair): """Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one.""" alice, bob, chat = pair key = uuid.uuid4().hex locker = lock_chat_row(chat, 1.0) try: wait_lock_taken(chat) with pytest.raises(httpx.TimeoutException): httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4, headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) finally: locker.wait(timeout=30) deadline = time.monotonic() + 10 while messages_in_db(chat) == 0 and time.monotonic() < deadline: time.sleep(0.2) stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'") assert stored, "the server-side transaction was expected to commit after the lock was released" retried = api.send(alice, chat, "неопределённо", key=key) assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1 def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair): """The row lock outlives lock_timeout: the API answers 503 within its deadline and stores nothing; after the lock is gone the same key creates the message exactly once.""" alice, bob, chat = pair key = uuid.uuid4().hex locker = lock_chat_row(chat, 4.0) try: wait_lock_taken(chat) started = time.monotonic() api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key) assert time.monotonic() - started < 3.6 assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 finally: locker.wait(timeout=30) created = api.send(alice, chat, "после блокировки", key=key) assert api.send(alice, chat, "после блокировки", key=key) == created assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1 def test_replay_after_the_serving_replica_is_stopped(api, pair): alice, bob, chat = pair key = uuid.uuid4().hex first = api.post_message(alice, chat, {"text": "до остановки"}, key=key) served_by = first.headers["x-instance-id"] replica = next(r for r in running_replicas() if r["hostname"] == served_by) with replica_stopped(replica): for _ in range(5): again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key) assert again.json() == first.json() and again.headers["x-instance-id"] != served_by assert messages_in_db(chat) == 1