forked from mtusi2026/mtusi2026lab3
test: строгие тесты lab3 — идемпотентность, балансировка, отказы реплик
- test_idempotency: точные коды и тела повторов через разные реплики, 24
одновременных запроса с одним ключом -> ровно одно сообщение и одна строка
ключа в БД, 409 при другом body, scope user+chat+key, 404 постороннему,
откат после сбоя на втором INSERT (триггер, n_tup_ins), неопределённый
POST (клиентский timeout) и lock_timeout с повтором тем же ключом.
- test_balancing: X-Instance-Id = реальные контейнеры, нет sticky,
LB исключает живую, но неготовую реплику (pg_hba reject), graceful stop
каждой реплики под нагрузкой без единой ошибки, SIGKILL — ограниченные
ошибки и восстановление, полный restart и scale up/down без рестарта LB,
политика compose (лимиты, порты, secrets, digest), миграции один раз.
- Входы из замечаний к чужим PR: курсоры с []/{}/числами/пустые/пробел,
байт FF, TEXT/text, дубли ключей, Bearer é, UUID в разном регистре, тело
при входе, пароль "я:"*64. Тесты lab2 переведены на Idempotency-Key и LB;
все отказы восстанавливаются в finally и ждут UP всех реплик в HAProxy.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
3f12698676
commit
78b95acabf
+81
-16
@@ -2,9 +2,10 @@
|
||||
|
||||
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
|
||||
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
|
||||
stand (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database
|
||||
checks (psql inside the postgres container) and failure injection. Every failure
|
||||
test restores the stopped/paused service in `finally`.
|
||||
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
|
||||
`docker compose` for database checks (psql inside the postgres container), the HAProxy
|
||||
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
|
||||
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
|
||||
|
||||
python -m pytest student-tests # everything (stand must be up)
|
||||
python -m pytest student-tests -m "not stack" # unit only
|
||||
@@ -27,10 +28,13 @@ from jsonschema import Draft202012Validator
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
import lbctl # noqa: E402
|
||||
|
||||
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
|
||||
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
@@ -74,26 +78,32 @@ def wait_ready(timeout: float = 90) -> None:
|
||||
|
||||
|
||||
def wait_healthy(*services: str, timeout: float = 120) -> None:
|
||||
"""Poll Docker health until every service is healthy again. `compose up --wait` is not used:
|
||||
it gives up at once on a container that was marked unhealthy while paused."""
|
||||
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
|
||||
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
|
||||
deadline = time.monotonic() + timeout
|
||||
pending = set(services)
|
||||
while pending and time.monotonic() < deadline:
|
||||
for service in list(pending):
|
||||
ids = compose("ps", "-q", service, check=False).stdout.split()
|
||||
if ids:
|
||||
state = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
if state == "healthy":
|
||||
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
if states and all(state == "healthy" for state in states):
|
||||
pending.discard(service)
|
||||
time.sleep(0.5)
|
||||
assert not pending, f"not healthy: {pending}"
|
||||
|
||||
|
||||
def wait_stand(replicas: int = REPLICAS) -> None:
|
||||
"""Readiness through the LB AND every replica back in HAProxy rotation."""
|
||||
wait_ready()
|
||||
lbctl.wait_up(replicas, timeout=90)
|
||||
|
||||
|
||||
def restart_services(*services: str) -> None:
|
||||
compose("restart", *services)
|
||||
wait_healthy(*services)
|
||||
wait_ready()
|
||||
wait_stand()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
@@ -106,7 +116,7 @@ def service_down(service: str, how: str = "stop"):
|
||||
finally:
|
||||
compose("unpause" if how == "pause" else "start", service)
|
||||
wait_healthy(service)
|
||||
wait_ready()
|
||||
wait_stand()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
@@ -116,14 +126,33 @@ def api_with_env(**overrides: str):
|
||||
env = {**os.environ, **overrides}
|
||||
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
|
||||
capture_output=True, text=True, check=True, timeout=180)
|
||||
wait_ready()
|
||||
wait_stand()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
clean = {k: v for k, v in os.environ.items() if k not in overrides}
|
||||
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
|
||||
capture_output=True, text=True, check=True, timeout=180)
|
||||
wait_ready()
|
||||
wait_stand()
|
||||
|
||||
|
||||
def running_replicas() -> list[dict]:
|
||||
return [r for r in lbctl.replicas() if r["state"] == "running"]
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def replica_stopped(replica: dict, mode: str = "stop"):
|
||||
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
|
||||
and wait until HAProxy has every replica UP."""
|
||||
assert mode in {"stop", "kill"}
|
||||
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
|
||||
capture_output=True, text=True, check=True, timeout=60)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
|
||||
wait_healthy("api")
|
||||
wait_stand()
|
||||
|
||||
|
||||
def api_env(name: str) -> str:
|
||||
@@ -139,7 +168,10 @@ class Api:
|
||||
self.client = httpx.Client(base_url=base_url, timeout=15)
|
||||
|
||||
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
|
||||
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None):
|
||||
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
|
||||
lb_error_ok: bool = False):
|
||||
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
|
||||
accepted without X-Instance-Id; every response of an API replica must carry it."""
|
||||
h = dict(headers or {})
|
||||
if token is not None:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
@@ -151,6 +183,9 @@ class Api:
|
||||
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
|
||||
request_id = response.headers.get("x-request-id")
|
||||
assert request_id, "every response must carry X-Request-Id"
|
||||
if not response.headers.get("x-instance-id"):
|
||||
assert lb_error_ok and response.status_code in (502, 503, 504), \
|
||||
("missing X-Instance-Id", method, path, response.status_code)
|
||||
if response.status_code >= 400:
|
||||
body = response.json()
|
||||
assert schema_errors(body, "Error") == [], response.text
|
||||
@@ -186,9 +221,18 @@ class Api:
|
||||
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||||
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
|
||||
|
||||
def send(self, actor: dict, chat: dict, text: str) -> dict:
|
||||
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=actor["token"],
|
||||
json_body={"text": text}, schema="Message").json()
|
||||
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
|
||||
raw: bytes | None = None, headers: dict | None = None, **kwargs):
|
||||
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
|
||||
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
|
||||
if raw is not None:
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
|
||||
json_body=body if raw is None else None, raw=raw, headers=h,
|
||||
schema="Message" if expected == 201 else None, **kwargs)
|
||||
|
||||
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
|
||||
return self.post_message(actor, chat, {"text": text}, key=key).json()
|
||||
|
||||
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
|
||||
"""Walk all pages; returns (items, number_of_pages)."""
|
||||
@@ -224,3 +268,24 @@ def pytest_collection_modifyitems(config, items):
|
||||
@pytest.fixture(scope="session")
|
||||
def api() -> Api:
|
||||
return Api()
|
||||
|
||||
|
||||
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
|
||||
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
|
||||
script = str(ROOT / "scripts" / name)
|
||||
if os.name == "nt":
|
||||
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
|
||||
else:
|
||||
command = [script]
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
|
||||
timeout=timeout, env={**os.environ, **env})
|
||||
|
||||
|
||||
def probe_inside(replica: dict, path: str) -> int:
|
||||
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
|
||||
code = ("import urllib.request as u, urllib.error as e\n"
|
||||
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
|
||||
"except e.HTTPError as x: print(x.code)") % path
|
||||
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
|
||||
timeout=30).stdout.strip()
|
||||
return int(out or 0)
|
||||
|
||||
+105
-2
@@ -108,7 +108,7 @@ def test_unicode_limits_are_code_points_and_nul_is_stored(api):
|
||||
assert chat["title"] == "T\u0000itle"
|
||||
assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world"
|
||||
assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000
|
||||
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=actor["token"], json_body={"text": "я" * 4001})
|
||||
api.post_message(actor, chat, {"text": "я" * 4001}, 400)
|
||||
texts = [m["text"] for m in api.history(actor, chat, 100)[0]]
|
||||
assert texts == ["hello\u0000world", "🙂" * 4000]
|
||||
|
||||
@@ -136,7 +136,7 @@ def test_foreign_chat_is_indistinguishable_from_missing(api):
|
||||
base = f"/api/v1/chats/{chat_id}"
|
||||
errors = [api.call("GET", base, 404, token=eve["token"]).json(),
|
||||
api.call("GET", base + "/messages", 404, token=eve["token"]).json(),
|
||||
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}).json(),
|
||||
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(),
|
||||
api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()]
|
||||
assert len({e["error"]["message"] for e in errors}) == 1
|
||||
api.call("GET", "/api/v1/chats/not-a-uuid", 401)
|
||||
@@ -199,3 +199,106 @@ def test_read_after_write(api):
|
||||
message = api.send(alice, chat, f"m{i}")
|
||||
last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1]
|
||||
assert last == message
|
||||
|
||||
|
||||
# ================================================================== lab 3: inputs found by the reviewer in other PRs
|
||||
def b64(raw: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def bad_cursors(owner_id: str) -> list[str]:
|
||||
"""Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time."""
|
||||
good_uuid = str(uuid.uuid4())
|
||||
return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"),
|
||||
b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'),
|
||||
b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()),
|
||||
b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()),
|
||||
b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()),
|
||||
b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()),
|
||||
b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())]
|
||||
|
||||
|
||||
def test_malformed_cursor_values_are_400_on_both_lists(api):
|
||||
alice = api.user("cur")
|
||||
chat = api.chat(alice)
|
||||
for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]):
|
||||
quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20")
|
||||
for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
|
||||
error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json()
|
||||
assert error["error"]["code"] == "invalid_request", (path, cursor)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}',
|
||||
b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}',
|
||||
b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}',
|
||||
b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}',
|
||||
b'{"text":" \n\t "}'])
|
||||
def test_bad_message_body_is_400_and_nothing_is_stored(api, raw):
|
||||
alice = api.user("body")
|
||||
chat = api.chat(alice)
|
||||
error = api.post_message(alice, chat, None, 400, raw=raw).json()
|
||||
assert error["error"]["code"] == "invalid_request"
|
||||
assert api.history(alice, chat, 100)[0] == []
|
||||
|
||||
|
||||
def test_ff_byte_body_on_registration_is_contract_error(api):
|
||||
response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"})
|
||||
assert response.json()["error"]["request_id"] == response.headers["x-request-id"]
|
||||
|
||||
|
||||
def test_text_is_stored_verbatim_and_length_counts_raw_text(api):
|
||||
alice = api.user("verbatim")
|
||||
chat = api.chat(alice)
|
||||
padded = " строка\nс пробелами "
|
||||
assert api.send(alice, chat, padded)["text"] == padded
|
||||
api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming
|
||||
assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded]
|
||||
|
||||
|
||||
def test_unknown_attachment_ids_are_404(api):
|
||||
alice = api.user("att")
|
||||
chat = api.chat(alice)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates
|
||||
assert api.history(alice, chat, 100)[0] == []
|
||||
|
||||
|
||||
def test_member_ids_are_compared_as_uuids_not_strings(api):
|
||||
alice, bob = api.user("case"), api.user("case")
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
|
||||
json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]})
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
|
||||
json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())})
|
||||
chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"],
|
||||
json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json()
|
||||
assert chat["member_ids"] == [alice["id"], bob["id"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43])
|
||||
def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header):
|
||||
api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")})
|
||||
|
||||
|
||||
def test_login_with_a_body_is_rejected(api):
|
||||
actor = api.register("login_body")
|
||||
auth = {"Authorization": basic(actor["username"], actor["password"])}
|
||||
api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"})
|
||||
api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth)
|
||||
api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session")
|
||||
|
||||
|
||||
def test_password_range_is_12_to_128_code_points(api):
|
||||
api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"})
|
||||
actor = api.user("pw128", password="я:" * 64)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
|
||||
|
||||
def test_every_api_response_carries_instance_and_request_id(api):
|
||||
"""X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it)."""
|
||||
api.call("GET", "/health/live", 200)
|
||||
api.call("GET", "/health/ready", 200)
|
||||
api.call("GET", "/no/such/route", 404)
|
||||
api.call("PUT", "/api/v1/users", 405)
|
||||
api.call("GET", "/api/v1/users/me", 401)
|
||||
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
|
||||
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
|
||||
|
||||
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
|
||||
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx2 as httpx
|
||||
import pytest
|
||||
|
||||
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
|
||||
running_replicas, wait_healthy, wait_stand
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
|
||||
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ level 3: replicas, no affinity
|
||||
def test_one_session_is_served_by_every_real_replica(api):
|
||||
actor = api.user("rr")
|
||||
seen = instances_seen(api, actor["token"])
|
||||
running = {r["hostname"] for r in running_replicas()}
|
||||
assert len(running) == REPLICAS >= 2
|
||||
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
|
||||
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
|
||||
# so there is no sticky session hiding split state.
|
||||
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
|
||||
assert switches >= len(seen) // 2, seen
|
||||
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
|
||||
|
||||
|
||||
def test_write_on_one_replica_is_read_on_the_other(api):
|
||||
alice, bob = api.user("rw"), api.user("rw")
|
||||
chat = api.chat(alice, bob)
|
||||
for i in range(10):
|
||||
sent = api.post_message(alice, chat, {"text": f"m{i}"})
|
||||
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
|
||||
assert read.json()["items"][-1] == sent.json()
|
||||
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
|
||||
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
||||
assert r.headers["x-instance-id"]
|
||||
assert logout.headers["x-instance-id"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ readiness drives the LB
|
||||
def hba_file() -> str:
|
||||
return psql("SHOW hba_file")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def isolated_from_postgres(replica: dict):
|
||||
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
|
||||
connections), while it keeps running and answering /health/live. Always undone."""
|
||||
ip = replica["ips"][0]
|
||||
path = hba_file()
|
||||
marker = "lab3-test-isolate"
|
||||
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
|
||||
try:
|
||||
psql("SELECT pg_reload_conf()")
|
||||
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
|
||||
yield replica
|
||||
finally:
|
||||
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
|
||||
psql("SELECT pg_reload_conf()")
|
||||
wait_stand()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position):
|
||||
"""The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is
|
||||
200, /health/ready 503 -> HAProxy must take it out; no business request may reach it."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
actor = api.user("ready")
|
||||
with isolated_from_postgres(replica):
|
||||
started = time.monotonic()
|
||||
while replica["hostname"] in lbctl.up_instances():
|
||||
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
|
||||
time.sleep(0.1)
|
||||
detected = time.monotonic() - started
|
||||
assert probe_inside(replica, "/health/live") == 200 # the process is alive ...
|
||||
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
|
||||
seen = instances_seen(api, actor["token"], 40)
|
||||
assert replica["hostname"] not in seen and len(seen) == 40
|
||||
for i in range(20): # writes keep working, only through the ready replicas
|
||||
assert api.register(f"r{i}")["id"]
|
||||
print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ stopping replicas under traffic
|
||||
class Traffic:
|
||||
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
|
||||
|
||||
def __init__(self, api: Api, workers: int = 4):
|
||||
self.alice, self.bob = api.user("load"), api.user("load")
|
||||
self.chat = api.chat(self.alice, self.bob)
|
||||
self.workers = workers
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
|
||||
self.confirmed: list[dict] = []
|
||||
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
|
||||
|
||||
def record(self, kind: str, status: int, instance: str | None) -> None:
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), kind, status, instance))
|
||||
|
||||
def run(self, index: int) -> None:
|
||||
client = httpx.Client(base_url=BASE_URL, timeout=3)
|
||||
auth = {"Authorization": "Bearer " + self.alice["token"]}
|
||||
n = 0
|
||||
key, text = uuid.uuid4().hex, f"w{index}-0"
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
|
||||
headers={**auth, "Idempotency-Key": key})
|
||||
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
|
||||
if r.status_code == 201:
|
||||
with self.lock:
|
||||
self.confirmed.append(r.json())
|
||||
n += 1
|
||||
key, text = uuid.uuid4().hex, f"w{index}-{n}"
|
||||
# any other outcome: keep key and text, retry the same request
|
||||
except httpx.HTTPError:
|
||||
self.record("POST", 0, None)
|
||||
try:
|
||||
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
|
||||
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
|
||||
except httpx.HTTPError:
|
||||
self.record("GET", 0, None)
|
||||
time.sleep(0.02)
|
||||
client.close()
|
||||
|
||||
def __enter__(self) -> "Traffic":
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
time.sleep(1.0)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self.stop.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=10)
|
||||
|
||||
def failures(self, since: float = 0.0) -> list[tuple]:
|
||||
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
|
||||
|
||||
def check_history(self, api: Api) -> None:
|
||||
history = api.history(self.bob, self.chat, 100)[0]
|
||||
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
|
||||
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
|
||||
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
|
||||
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
with Traffic(api) as traffic:
|
||||
started = time.monotonic()
|
||||
with replica_stopped(replica, "stop"):
|
||||
stopped_after = time.monotonic() - started
|
||||
time.sleep(3)
|
||||
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
|
||||
assert after and replica["hostname"] not in {e[3] for e in after}
|
||||
time.sleep(1)
|
||||
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
|
||||
check=True).stdout)[0]["State"]
|
||||
assert traffic.failures() == [], traffic.failures()[:10]
|
||||
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
|
||||
assert state["Status"] == "running" # restored by replica_stopped
|
||||
traffic.check_history(api)
|
||||
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
|
||||
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
|
||||
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
with Traffic(api) as traffic:
|
||||
killed_at = time.monotonic()
|
||||
with replica_stopped(replica, "kill"):
|
||||
time.sleep(4)
|
||||
failures = traffic.failures(killed_at)
|
||||
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
|
||||
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
|
||||
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
|
||||
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
|
||||
assert late and replica["hostname"] not in {e[3] for e in late}
|
||||
assert last_failure <= 10
|
||||
traffic.check_history(api)
|
||||
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
|
||||
|
||||
|
||||
def test_graceful_shutdown_order_in_logs(api):
|
||||
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
|
||||
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
|
||||
with replica_stopped(replica, "stop"):
|
||||
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
|
||||
text=True, encoding="utf-8").stdout
|
||||
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
|
||||
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
|
||||
check=True).stdout)[0]["State"]
|
||||
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
|
||||
names = [e["event"] for e in events]
|
||||
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
|
||||
t = {e["event"]: e["ts"] for e in events}
|
||||
assert t["drain_started"] < t["shutdown_requested"]
|
||||
assert state["ExitCode"] == 0 and not state["OOMKilled"]
|
||||
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ discovery: restart and scaling
|
||||
def test_full_restart_lb_rediscovers_all_replicas(api):
|
||||
actor = api.user("restart")
|
||||
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
|
||||
run_hook("restart.sh")
|
||||
wait_healthy("postgres", "valkey", "api")
|
||||
wait_stand()
|
||||
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
|
||||
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
|
||||
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
|
||||
|
||||
|
||||
def test_scale_up_and_down_is_picked_up_without_errors(api):
|
||||
with Traffic(api, workers=2) as traffic:
|
||||
try:
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
|
||||
timeout=180)
|
||||
lbctl.wait_up(REPLICAS + 1, timeout=30)
|
||||
actor = api.user("scale")
|
||||
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
|
||||
timeout=180)
|
||||
wait_stand()
|
||||
time.sleep(1)
|
||||
assert traffic.failures() == [], traffic.failures()[:10]
|
||||
traffic.check_history(api)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ level 4: deployment policy
|
||||
def test_compose_policy_limits_ports_secrets_images():
|
||||
config = json.loads(compose("config", "--format", "json").stdout)
|
||||
services = config["services"]
|
||||
assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"}
|
||||
for name, svc in services.items():
|
||||
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
|
||||
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
|
||||
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
|
||||
if name in ("api", "migrate"):
|
||||
assert "build" in svc # built from this repository's source
|
||||
else:
|
||||
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
|
||||
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
|
||||
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
|
||||
assert set(published) == {"lb"} and all(p["host_ip"] == "127.0.0.1" for p in published["lb"])
|
||||
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
|
||||
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"],
|
||||
"migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []}
|
||||
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
|
||||
for name, svc in services.items()}
|
||||
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
|
||||
|
||||
|
||||
def test_running_containers_have_limits_and_no_bypass_ports():
|
||||
ids = compose("ps", "-q").stdout.split()
|
||||
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
|
||||
check=True).stdout):
|
||||
name = item["Name"]
|
||||
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
|
||||
bindings = item["HostConfig"]["PortBindings"] or {}
|
||||
if "-lb-" in name:
|
||||
assert all(b["HostIp"] == "127.0.0.1" for v in bindings.values() for b in v)
|
||||
else:
|
||||
assert not bindings, f"{name} publishes ports in bypass of the LB"
|
||||
|
||||
|
||||
def test_migrations_ran_once_by_the_migrate_service():
|
||||
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
|
||||
assert applied == ["0001_init", "0002_idempotency_keys"]
|
||||
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
|
||||
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
|
||||
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
|
||||
api_logs = compose("logs", "--no-log-prefix", "api").stdout
|
||||
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate
|
||||
@@ -76,7 +76,8 @@ def test_migrations_rerun_is_a_noop():
|
||||
users_before = psql("SELECT count(*) FROM users")
|
||||
result = compose("run", "--rm", "--no-deps", "migrate")
|
||||
assert "0001_init: already applied, skipped" in result.stdout
|
||||
assert psql("SELECT count(*) FROM schema_migrations") == "1"
|
||||
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
|
||||
assert psql("SELECT count(*) FROM schema_migrations") == "2"
|
||||
assert psql("SELECT count(*) FROM users") == users_before
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
|
||||
Every test stops/pauses a real container and restores it in `finally` (service_down).
|
||||
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
|
||||
|
||||
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
|
||||
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
|
||||
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
|
||||
for a valid token. Liveness/readiness of the processes are checked inside each container.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,7 +15,7 @@ import time
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT, restart_services, service_down
|
||||
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
@@ -20,10 +25,16 @@ CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
|
||||
|
||||
def timed(api, *args, **kwargs):
|
||||
started = time.monotonic()
|
||||
response = api.call(*args, **kwargs)
|
||||
response = api.call(*args, lb_error_ok=True, **kwargs)
|
||||
return response, time.monotonic() - started
|
||||
|
||||
|
||||
def processes_alive_but_not_ready() -> None:
|
||||
for replica in running_replicas():
|
||||
assert probe_inside(replica, "/health/live") == 200
|
||||
assert probe_inside(replica, "/health/ready") == 503
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prepared(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
@@ -46,9 +57,9 @@ def test_valkey_failure(api, prepared, how):
|
||||
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
|
||||
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
|
||||
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
|
||||
api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected
|
||||
api.call("GET", "/health/ready", 503)
|
||||
api.call("GET", "/health/live", 200)
|
||||
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
|
||||
api.call("GET", "/health/ready", 503, lb_error_ok=True)
|
||||
processes_alive_but_not_ready()
|
||||
check_intact(api, alice, bob, chat, message)
|
||||
|
||||
|
||||
@@ -60,9 +71,9 @@ def test_postgres_failure(api, prepared, how):
|
||||
response, took = timed(api, "GET", path, 503, token=alice["token"])
|
||||
assert response.json()["error"]["code"] == "unavailable", path
|
||||
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
|
||||
api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected
|
||||
api.call("GET", "/health/ready", 503)
|
||||
api.call("GET", "/health/live", 200)
|
||||
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
|
||||
api.call("GET", "/health/ready", 503, lb_error_ok=True)
|
||||
processes_alive_but_not_ready()
|
||||
check_intact(api, alice, bob, chat, message)
|
||||
|
||||
|
||||
@@ -77,6 +88,7 @@ def test_locked_table_gives_bounded_503(api, prepared):
|
||||
time.sleep(1.5) # let the lock be taken
|
||||
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
|
||||
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
|
||||
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
|
||||
finally:
|
||||
locker.wait(timeout=30)
|
||||
check_intact(api, alice, bob, chat, message)
|
||||
@@ -88,7 +100,7 @@ def test_logout_during_valkey_outage_is_never_lost(api, prepared):
|
||||
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
|
||||
alice, bob, chat, message = prepared
|
||||
with service_down("valkey", "stop"):
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"])
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
|
||||
|
||||
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
|
||||
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
|
||||
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
|
||||
undone in `finally`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx2 as httpx
|
||||
import pytest
|
||||
|
||||
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def messages_in_db(chat: dict) -> int:
|
||||
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
|
||||
|
||||
|
||||
def key_rows(actor: dict, key: str) -> int:
|
||||
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pair(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
return alice, bob, api.chat(alice, bob)
|
||||
|
||||
|
||||
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
|
||||
alice, bob, chat = pair
|
||||
key = "replay-" + uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
|
||||
served = {first.headers["x-instance-id"]}
|
||||
for _ in range(20):
|
||||
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
|
||||
assert again.status_code == 201 and again.json() == first.json()
|
||||
served.add(again.headers["x-instance-id"])
|
||||
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
|
||||
assert api.history(bob, chat, 100)[0] == [first.json()]
|
||||
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
original = api.send(alice, chat, "оригинал", key=key)
|
||||
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
|
||||
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
|
||||
assert api.send(alice, chat, "оригинал", key=key) == original
|
||||
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
|
||||
alice, _, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
|
||||
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
|
||||
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
|
||||
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
|
||||
assert messages_in_db(chat) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
|
||||
def test_missing_or_invalid_key_is_400(api, pair, value):
|
||||
alice, _, chat = pair
|
||||
raw_value = value.encode("utf-8")
|
||||
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
|
||||
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
|
||||
assert error["error"]["code"] == "invalid_request"
|
||||
assert messages_in_db(chat) == 0
|
||||
|
||||
|
||||
def test_repeated_key_header_is_400(pair):
|
||||
alice, _, chat = pair
|
||||
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
|
||||
headers=[("Authorization", "Bearer " + alice["token"]),
|
||||
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
|
||||
assert response.status_code == 400 and response.headers["x-instance-id"]
|
||||
assert messages_in_db(chat) == 0
|
||||
|
||||
|
||||
def test_longest_key_and_every_allowed_character(api, pair):
|
||||
alice, _, chat = pair
|
||||
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
|
||||
first = api.send(alice, chat, key[:20], key=key)
|
||||
assert api.send(alice, chat, key[:20], key=key) == first
|
||||
assert messages_in_db(chat) == 2
|
||||
|
||||
|
||||
def test_scope_is_user_plus_chat_plus_key(api, pair):
|
||||
alice, bob, chat = pair
|
||||
other = api.chat(alice, bob, title="второй")
|
||||
key = "shared-key"
|
||||
a1 = api.send(alice, chat, "текст", key=key)
|
||||
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
|
||||
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
|
||||
assert len({a1["id"], b1["id"], a2["id"]}) == 3
|
||||
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
|
||||
|
||||
|
||||
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
|
||||
alice, bob, chat = pair
|
||||
eve = api.user("eve")
|
||||
key = uuid.uuid4().hex
|
||||
api.send(alice, chat, "секрет", key=key)
|
||||
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
|
||||
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
|
||||
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
|
||||
assert messages_in_db(chat) == 1
|
||||
|
||||
|
||||
def fire_together(workers: int, request) -> list:
|
||||
barrier = threading.Barrier(workers)
|
||||
results: list = [None] * workers
|
||||
|
||||
def body(index: int) -> None:
|
||||
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
|
||||
barrier.wait()
|
||||
try:
|
||||
results[index] = request(client, index)
|
||||
except BaseException as exc: # noqa: BLE001 - asserted by the test
|
||||
results[index] = exc
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return results
|
||||
|
||||
|
||||
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
|
||||
alice, bob, chat = pair
|
||||
key = "race-" + uuid.uuid4().hex
|
||||
|
||||
def post(client: httpx.Client, _index: int):
|
||||
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
||||
return r.status_code, r.headers.get("x-instance-id"), r.json()
|
||||
|
||||
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
|
||||
results = fire_together(24, post)
|
||||
assert all(isinstance(r, tuple) for r in results), results
|
||||
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
|
||||
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
|
||||
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
|
||||
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_concurrent_same_key_different_bodies_one_wins(pair):
|
||||
alice, _, chat = pair
|
||||
key = "conflict-" + uuid.uuid4().hex
|
||||
|
||||
def post(client: httpx.Client, index: int):
|
||||
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
||||
return r.status_code, r.json()
|
||||
|
||||
results = fire_together(20, post)
|
||||
winners = [body for status, body in results if status == 201]
|
||||
losers = [body for status, body in results if status == 409]
|
||||
assert len(winners) + len(losers) == 20, results
|
||||
assert winners and len({str(b) for b in winners}) == 1
|
||||
assert all(b["error"]["code"] == "conflict" for b in losers)
|
||||
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
|
||||
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
|
||||
assert messages_in_db(chat) == 1
|
||||
|
||||
|
||||
def inserts_into_keys() -> int:
|
||||
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
|
||||
|
||||
|
||||
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
|
||||
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
|
||||
transaction. Expected: 500 for the client, no key row and no message left behind
|
||||
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
|
||||
alice, bob, chat = pair
|
||||
marker = "boom-" + uuid.uuid4().hex
|
||||
key = uuid.uuid4().hex
|
||||
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
|
||||
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
|
||||
"RETURN NEW; END $$; "
|
||||
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
|
||||
try:
|
||||
before = inserts_into_keys()
|
||||
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
|
||||
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
|
||||
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
|
||||
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
|
||||
while inserts_into_keys() == before and time.monotonic() < deadline:
|
||||
time.sleep(0.5)
|
||||
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
|
||||
finally:
|
||||
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
|
||||
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
|
||||
assert api.send(alice, chat, marker, key=key) == retried
|
||||
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
|
||||
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
|
||||
f"SELECT pg_sleep({seconds}); COMMIT;")
|
||||
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
|
||||
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_lock_taken(chat: dict) -> None:
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
|
||||
"AND pid <> pg_backend_pid()") != "0":
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise AssertionError("the row lock was not taken")
|
||||
|
||||
|
||||
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
|
||||
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
|
||||
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
locker = lock_chat_row(chat, 1.0)
|
||||
try:
|
||||
wait_lock_taken(chat)
|
||||
with pytest.raises(httpx.TimeoutException):
|
||||
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
||||
finally:
|
||||
locker.wait(timeout=30)
|
||||
deadline = time.monotonic() + 10
|
||||
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
|
||||
time.sleep(0.2)
|
||||
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
|
||||
assert stored, "the server-side transaction was expected to commit after the lock was released"
|
||||
retried = api.send(alice, chat, "неопределённо", key=key)
|
||||
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
|
||||
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
|
||||
nothing; after the lock is gone the same key creates the message exactly once."""
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
locker = lock_chat_row(chat, 4.0)
|
||||
try:
|
||||
wait_lock_taken(chat)
|
||||
started = time.monotonic()
|
||||
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
|
||||
assert time.monotonic() - started < 3.6
|
||||
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
|
||||
finally:
|
||||
locker.wait(timeout=30)
|
||||
created = api.send(alice, chat, "после блокировки", key=key)
|
||||
assert api.send(alice, chat, "после блокировки", key=key) == created
|
||||
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_replay_after_the_serving_replica_is_stopped(api, pair):
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
|
||||
served_by = first.headers["x-instance-id"]
|
||||
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
|
||||
with replica_stopped(replica):
|
||||
for _ in range(5):
|
||||
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
|
||||
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
|
||||
assert messages_in_db(chat) == 1
|
||||
@@ -57,8 +57,9 @@ def test_postgres_session_store(api):
|
||||
restart_services("api")
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
with service_down("postgres", "stop"):
|
||||
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"]).json()["error"]["code"] == "unavailable"
|
||||
api.call("GET", "/api/v1/users/me", 401)
|
||||
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"],
|
||||
lb_error_ok=True).json()["error"]["code"] == "unavailable"
|
||||
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
|
||||
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
|
||||
|
||||
@@ -132,3 +132,60 @@ def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
|
||||
with pytest.raises(ConfigError) as info:
|
||||
Settings.from_env()
|
||||
assert "u:p" not in str(info.value)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id
|
||||
@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"])
|
||||
def test_valid_idempotency_keys(key):
|
||||
assert validation.idempotency_key(key) == key
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"])
|
||||
def test_invalid_idempotency_keys(key):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.idempotency_key(key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}'])
|
||||
def test_duplicate_keys_and_garbage_json_are_rejected(raw):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.parse_json_object(raw)
|
||||
|
||||
|
||||
def test_attachment_ids_absent_equals_empty_and_is_validated():
|
||||
assert validation.attachment_ids_field({"text": "x"}) == []
|
||||
assert validation.attachment_ids_field({"attachment_ids": []}) == []
|
||||
one = "11111111-1111-4111-8111-111111111111"
|
||||
assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one]
|
||||
for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.attachment_ids_field({"attachment_ids": bad})
|
||||
|
||||
|
||||
def test_request_fingerprint_ignores_key_order_and_default_attachments():
|
||||
from messenger.app import request_fingerprint
|
||||
assert request_fingerprint("привет", []) == request_fingerprint("привет", [])
|
||||
assert request_fingerprint("привет", []) != request_fingerprint("привет ", [])
|
||||
assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", [])
|
||||
one = "11111111-1111-4111-8111-111111111111"
|
||||
assert request_fingerprint("x", [one]) != request_fingerprint("x", [])
|
||||
|
||||
|
||||
def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch):
|
||||
import socket
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
monkeypatch.delenv("INSTANCE_ID", raising=False)
|
||||
assert Settings.from_env().instance_id == socket.gethostname()
|
||||
monkeypatch.setenv("INSTANCE_ID", "replica-a")
|
||||
assert Settings.from_env().instance_id == "replica-a"
|
||||
for bad in ("with space", "x" * 65, "ip/1"):
|
||||
monkeypatch.setenv("INSTANCE_ID", bad)
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
|
||||
|
||||
def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600")
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
|
||||
Reference in New Issue
Block a user