test: строгие тесты lab3 — идемпотентность, балансировка, отказы реплик

- test_idempotency: точные коды и тела повторов через разные реплики, 24
  одновременных запроса с одним ключом -> ровно одно сообщение и одна строка
  ключа в БД, 409 при другом body, scope user+chat+key, 404 постороннему,
  откат после сбоя на втором INSERT (триггер, n_tup_ins), неопределённый
  POST (клиентский timeout) и lock_timeout с повтором тем же ключом.
- test_balancing: X-Instance-Id = реальные контейнеры, нет sticky,
  LB исключает живую, но неготовую реплику (pg_hba reject), graceful stop
  каждой реплики под нагрузкой без единой ошибки, SIGKILL — ограниченные
  ошибки и восстановление, полный restart и scale up/down без рестарта LB,
  политика compose (лимиты, порты, secrets, digest), миграции один раз.
- Входы из замечаний к чужим PR: курсоры с []/{}/числами/пустые/пробел,
  байт FF, TEXT/text, дубли ключей, Bearer é, UUID в разном регистре, тело
  при входе, пароль "я:"*64. Тесты lab2 переведены на Idempotency-Key и LB;
  все отказы восстанавливаются в finally и ждут UP всех реплик в HAProxy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-16 13:32:13 +03:00
co-authored by Claude Opus 5.5
parent 3f12698676
commit 78b95acabf
8 changed files with 848 additions and 30 deletions
+81 -16
View File
@@ -2,9 +2,10 @@
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
stand (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database
checks (psql inside the postgres container) and failure injection. Every failure
test restores the stopped/paused service in `finally`.
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
`docker compose` for database checks (psql inside the postgres container), the HAProxy
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
python -m pytest student-tests # everything (stand must be up)
python -m pytest student-tests -m "not stack" # unit only
@@ -27,10 +28,13 @@ from jsonschema import Draft202012Validator
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "scripts"))
import lbctl # noqa: E402
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
def pytest_configure(config):
@@ -74,26 +78,32 @@ def wait_ready(timeout: float = 90) -> None:
def wait_healthy(*services: str, timeout: float = 120) -> None:
"""Poll Docker health until every service is healthy again. `compose up --wait` is not used:
it gives up at once on a container that was marked unhealthy while paused."""
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
deadline = time.monotonic() + timeout
pending = set(services)
while pending and time.monotonic() < deadline:
for service in list(pending):
ids = compose("ps", "-q", service, check=False).stdout.split()
if ids:
state = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]],
capture_output=True, text=True).stdout.strip()
if state == "healthy":
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
capture_output=True, text=True).stdout.split()
if states and all(state == "healthy" for state in states):
pending.discard(service)
time.sleep(0.5)
assert not pending, f"not healthy: {pending}"
def wait_stand(replicas: int = REPLICAS) -> None:
"""Readiness through the LB AND every replica back in HAProxy rotation."""
wait_ready()
lbctl.wait_up(replicas, timeout=90)
def restart_services(*services: str) -> None:
compose("restart", *services)
wait_healthy(*services)
wait_ready()
wait_stand()
@contextlib.contextmanager
@@ -106,7 +116,7 @@ def service_down(service: str, how: str = "stop"):
finally:
compose("unpause" if how == "pause" else "start", service)
wait_healthy(service)
wait_ready()
wait_stand()
@contextlib.contextmanager
@@ -116,14 +126,33 @@ def api_with_env(**overrides: str):
env = {**os.environ, **overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
capture_output=True, text=True, check=True, timeout=180)
wait_ready()
wait_stand()
try:
yield
finally:
clean = {k: v for k, v in os.environ.items() if k not in overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
capture_output=True, text=True, check=True, timeout=180)
wait_ready()
wait_stand()
def running_replicas() -> list[dict]:
return [r for r in lbctl.replicas() if r["state"] == "running"]
@contextlib.contextmanager
def replica_stopped(replica: dict, mode: str = "stop"):
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
and wait until HAProxy has every replica UP."""
assert mode in {"stop", "kill"}
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
capture_output=True, text=True, check=True, timeout=60)
try:
yield
finally:
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
wait_healthy("api")
wait_stand()
def api_env(name: str) -> str:
@@ -139,7 +168,10 @@ class Api:
self.client = httpx.Client(base_url=base_url, timeout=15)
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None):
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
lb_error_ok: bool = False):
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
accepted without X-Instance-Id; every response of an API replica must carry it."""
h = dict(headers or {})
if token is not None:
h["Authorization"] = "Bearer " + token
@@ -151,6 +183,9 @@ class Api:
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
request_id = response.headers.get("x-request-id")
assert request_id, "every response must carry X-Request-Id"
if not response.headers.get("x-instance-id"):
assert lb_error_ok and response.status_code in (502, 503, 504), \
("missing X-Instance-Id", method, path, response.status_code)
if response.status_code >= 400:
body = response.json()
assert schema_errors(body, "Error") == [], response.text
@@ -186,9 +221,18 @@ class Api:
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
def send(self, actor: dict, chat: dict, text: str) -> dict:
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=actor["token"],
json_body={"text": text}, schema="Message").json()
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
raw: bytes | None = None, headers: dict | None = None, **kwargs):
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
if raw is not None:
h.setdefault("Content-Type", "application/json")
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
json_body=body if raw is None else None, raw=raw, headers=h,
schema="Message" if expected == 201 else None, **kwargs)
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
return self.post_message(actor, chat, {"text": text}, key=key).json()
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
"""Walk all pages; returns (items, number_of_pages)."""
@@ -224,3 +268,24 @@ def pytest_collection_modifyitems(config, items):
@pytest.fixture(scope="session")
def api() -> Api:
return Api()
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
script = str(ROOT / "scripts" / name)
if os.name == "nt":
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
else:
command = [script]
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
timeout=timeout, env={**os.environ, **env})
def probe_inside(replica: dict, path: str) -> int:
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
code = ("import urllib.request as u, urllib.error as e\n"
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
"except e.HTTPError as x: print(x.code)") % path
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
timeout=30).stdout.strip()
return int(out or 0)
+105 -2
View File
@@ -108,7 +108,7 @@ def test_unicode_limits_are_code_points_and_nul_is_stored(api):
assert chat["title"] == "T\u0000itle"
assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world"
assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=actor["token"], json_body={"text": "я" * 4001})
api.post_message(actor, chat, {"text": "я" * 4001}, 400)
texts = [m["text"] for m in api.history(actor, chat, 100)[0]]
assert texts == ["hello\u0000world", "🙂" * 4000]
@@ -136,7 +136,7 @@ def test_foreign_chat_is_indistinguishable_from_missing(api):
base = f"/api/v1/chats/{chat_id}"
errors = [api.call("GET", base, 404, token=eve["token"]).json(),
api.call("GET", base + "/messages", 404, token=eve["token"]).json(),
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}).json(),
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(),
api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()]
assert len({e["error"]["message"] for e in errors}) == 1
api.call("GET", "/api/v1/chats/not-a-uuid", 401)
@@ -199,3 +199,106 @@ def test_read_after_write(api):
message = api.send(alice, chat, f"m{i}")
last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1]
assert last == message
# ================================================================== lab 3: inputs found by the reviewer in other PRs
def b64(raw: bytes) -> str:
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
def bad_cursors(owner_id: str) -> list[str]:
"""Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time."""
good_uuid = str(uuid.uuid4())
return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"),
b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'),
b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()),
b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()),
b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()),
b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()),
b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())]
def test_malformed_cursor_values_are_400_on_both_lists(api):
alice = api.user("cur")
chat = api.chat(alice)
for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]):
quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20")
for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json()
assert error["error"]["code"] == "invalid_request", (path, cursor)
@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}',
b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}',
b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}',
b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}',
b'{"text":" \n\t "}'])
def test_bad_message_body_is_400_and_nothing_is_stored(api, raw):
alice = api.user("body")
chat = api.chat(alice)
error = api.post_message(alice, chat, None, 400, raw=raw).json()
assert error["error"]["code"] == "invalid_request"
assert api.history(alice, chat, 100)[0] == []
def test_ff_byte_body_on_registration_is_contract_error(api):
response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"})
assert response.json()["error"]["request_id"] == response.headers["x-request-id"]
def test_text_is_stored_verbatim_and_length_counts_raw_text(api):
alice = api.user("verbatim")
chat = api.chat(alice)
padded = " строка\nс пробелами "
assert api.send(alice, chat, padded)["text"] == padded
api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming
assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded]
def test_unknown_attachment_ids_are_404(api):
alice = api.user("att")
chat = api.chat(alice)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates
assert api.history(alice, chat, 100)[0] == []
def test_member_ids_are_compared_as_uuids_not_strings(api):
alice, bob = api.user("case"), api.user("case")
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]})
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())})
chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"],
json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json()
assert chat["member_ids"] == [alice["id"], bob["id"]]
@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43])
def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header):
api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")})
def test_login_with_a_body_is_rejected(api):
actor = api.register("login_body")
auth = {"Authorization": basic(actor["username"], actor["password"])}
api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"})
api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth)
api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session")
def test_password_range_is_12_to_128_code_points(api):
api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"})
actor = api.user("pw128", password="я:" * 64)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_every_api_response_carries_instance_and_request_id(api):
"""X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it)."""
api.call("GET", "/health/live", 200)
api.call("GET", "/health/ready", 200)
api.call("GET", "/no/such/route", 404)
api.call("PUT", "/api/v1/users", 405)
api.call("GET", "/api/v1/users/me", 401)
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
+298
View File
@@ -0,0 +1,298 @@
"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
"""
from __future__ import annotations
import contextlib
import json
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
running_replicas, wait_healthy, wait_stand
pytestmark = pytest.mark.stack
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
# ------------------------------------------------------------------ level 3: replicas, no affinity
def test_one_session_is_served_by_every_real_replica(api):
actor = api.user("rr")
seen = instances_seen(api, actor["token"])
running = {r["hostname"] for r in running_replicas()}
assert len(running) == REPLICAS >= 2
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
# so there is no sticky session hiding split state.
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
assert switches >= len(seen) // 2, seen
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
def test_write_on_one_replica_is_read_on_the_other(api):
alice, bob = api.user("rw"), api.user("rw")
chat = api.chat(alice, bob)
for i in range(10):
sent = api.post_message(alice, chat, {"text": f"m{i}"})
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
assert read.json()["items"][-1] == sent.json()
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
assert r.headers["x-instance-id"]
assert logout.headers["x-instance-id"]
# ------------------------------------------------------------------ readiness drives the LB
def hba_file() -> str:
return psql("SHOW hba_file")
@contextlib.contextmanager
def isolated_from_postgres(replica: dict):
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
connections), while it keeps running and answering /health/live. Always undone."""
ip = replica["ips"][0]
path = hba_file()
marker = "lab3-test-isolate"
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
try:
psql("SELECT pg_reload_conf()")
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
yield replica
finally:
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
psql("SELECT pg_reload_conf()")
wait_stand()
@pytest.mark.parametrize("position", range(REPLICAS))
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position):
"""The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is
200, /health/ready 503 -> HAProxy must take it out; no business request may reach it."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
actor = api.user("ready")
with isolated_from_postgres(replica):
started = time.monotonic()
while replica["hostname"] in lbctl.up_instances():
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
time.sleep(0.1)
detected = time.monotonic() - started
assert probe_inside(replica, "/health/live") == 200 # the process is alive ...
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
seen = instances_seen(api, actor["token"], 40)
assert replica["hostname"] not in seen and len(seen) == 40
for i in range(20): # writes keep working, only through the ready replicas
assert api.register(f"r{i}")["id"]
print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s")
# ------------------------------------------------------------------ stopping replicas under traffic
class Traffic:
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
def __init__(self, api: Api, workers: int = 4):
self.alice, self.bob = api.user("load"), api.user("load")
self.chat = api.chat(self.alice, self.bob)
self.workers = workers
self.stop = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
self.confirmed: list[dict] = []
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
def record(self, kind: str, status: int, instance: str | None) -> None:
with self.lock:
self.events.append((time.monotonic(), kind, status, instance))
def run(self, index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=3)
auth = {"Authorization": "Bearer " + self.alice["token"]}
n = 0
key, text = uuid.uuid4().hex, f"w{index}-0"
while not self.stop.is_set():
try:
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
headers={**auth, "Idempotency-Key": key})
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
if r.status_code == 201:
with self.lock:
self.confirmed.append(r.json())
n += 1
key, text = uuid.uuid4().hex, f"w{index}-{n}"
# any other outcome: keep key and text, retry the same request
except httpx.HTTPError:
self.record("POST", 0, None)
try:
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
except httpx.HTTPError:
self.record("GET", 0, None)
time.sleep(0.02)
client.close()
def __enter__(self) -> "Traffic":
for t in self.threads:
t.start()
time.sleep(1.0)
return self
def __exit__(self, *exc) -> None:
self.stop.set()
for t in self.threads:
t.join(timeout=10)
def failures(self, since: float = 0.0) -> list[tuple]:
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
def check_history(self, api: Api) -> None:
history = api.history(self.bob, self.chat, 100)[0]
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
@pytest.mark.parametrize("position", range(REPLICAS))
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
started = time.monotonic()
with replica_stopped(replica, "stop"):
stopped_after = time.monotonic() - started
time.sleep(3)
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
assert after and replica["hostname"] not in {e[3] for e in after}
time.sleep(1)
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
assert traffic.failures() == [], traffic.failures()[:10]
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
assert state["Status"] == "running" # restored by replica_stopped
traffic.check_history(api)
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
@pytest.mark.parametrize("position", range(REPLICAS))
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
killed_at = time.monotonic()
with replica_stopped(replica, "kill"):
time.sleep(4)
failures = traffic.failures(killed_at)
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
assert late and replica["hostname"] not in {e[3] for e in late}
assert last_failure <= 10
traffic.check_history(api)
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
def test_graceful_shutdown_order_in_logs(api):
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
with replica_stopped(replica, "stop"):
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
text=True, encoding="utf-8").stdout
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
names = [e["event"] for e in events]
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
t = {e["event"]: e["ts"] for e in events}
assert t["drain_started"] < t["shutdown_requested"]
assert state["ExitCode"] == 0 and not state["OOMKilled"]
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
# ------------------------------------------------------------------ discovery: restart and scaling
def test_full_restart_lb_rediscovers_all_replicas(api):
actor = api.user("restart")
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
run_hook("restart.sh")
wait_healthy("postgres", "valkey", "api")
wait_stand()
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
def test_scale_up_and_down_is_picked_up_without_errors(api):
with Traffic(api, workers=2) as traffic:
try:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
timeout=180)
lbctl.wait_up(REPLICAS + 1, timeout=30)
actor = api.user("scale")
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
timeout=180)
wait_stand()
time.sleep(1)
assert traffic.failures() == [], traffic.failures()[:10]
traffic.check_history(api)
# ------------------------------------------------------------------ level 4: deployment policy
def test_compose_policy_limits_ports_secrets_images():
config = json.loads(compose("config", "--format", "json").stdout)
services = config["services"]
assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"}
for name, svc in services.items():
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
if name in ("api", "migrate"):
assert "build" in svc # built from this repository's source
else:
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
assert set(published) == {"lb"} and all(p["host_ip"] == "127.0.0.1" for p in published["lb"])
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"],
"migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []}
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
def test_running_containers_have_limits_and_no_bypass_ports():
ids = compose("ps", "-q").stdout.split()
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
name = item["Name"]
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
bindings = item["HostConfig"]["PortBindings"] or {}
if "-lb-" in name:
assert all(b["HostIp"] == "127.0.0.1" for v in bindings.values() for b in v)
else:
assert not bindings, f"{name} publishes ports in bypass of the LB"
def test_migrations_ran_once_by_the_migrate_service():
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
assert applied == ["0001_init", "0002_idempotency_keys"]
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
api_logs = compose("logs", "--no-log-prefix", "api").stdout
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate
+2 -1
View File
@@ -76,7 +76,8 @@ def test_migrations_rerun_is_a_noop():
users_before = psql("SELECT count(*) FROM users")
result = compose("run", "--rm", "--no-deps", "migrate")
assert "0001_init: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "1"
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "2"
assert psql("SELECT count(*) FROM users") == users_before
+21 -9
View File
@@ -2,6 +2,11 @@
Every test stops/pauses a real container and restores it in `finally` (service_down).
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
for a valid token. Liveness/readiness of the processes are checked inside each container.
"""
from __future__ import annotations
@@ -10,7 +15,7 @@ import time
import pytest
from conftest import ROOT, restart_services, service_down
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
pytestmark = pytest.mark.stack
@@ -20,10 +25,16 @@ CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
def timed(api, *args, **kwargs):
started = time.monotonic()
response = api.call(*args, **kwargs)
response = api.call(*args, lb_error_ok=True, **kwargs)
return response, time.monotonic() - started
def processes_alive_but_not_ready() -> None:
for replica in running_replicas():
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503
@pytest.fixture
def prepared(api):
alice, bob = api.user("alice"), api.user("bob")
@@ -46,9 +57,9 @@ def test_valkey_failure(api, prepared, how):
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected
api.call("GET", "/health/ready", 503)
api.call("GET", "/health/live", 200)
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
@@ -60,9 +71,9 @@ def test_postgres_failure(api, prepared, how):
response, took = timed(api, "GET", path, 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable", path
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected
api.call("GET", "/health/ready", 503)
api.call("GET", "/health/live", 200)
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
@@ -77,6 +88,7 @@ def test_locked_table_gives_bounded_503(api, prepared):
time.sleep(1.5) # let the lock be taken
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
finally:
locker.wait(timeout=30)
check_intact(api, alice, bob, chat, message)
@@ -88,7 +100,7 @@ def test_logout_during_valkey_outage_is_never_lost(api, prepared):
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
alice, bob, chat, message = prepared
with service_down("valkey", "stop"):
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
+281
View File
@@ -0,0 +1,281 @@
"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
undone in `finally`.
"""
from __future__ import annotations
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
pytestmark = pytest.mark.stack
def messages_in_db(chat: dict) -> int:
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
def key_rows(actor: dict, key: str) -> int:
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
@pytest.fixture
def pair(api):
alice, bob = api.user("alice"), api.user("bob")
return alice, bob, api.chat(alice, bob)
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
alice, bob, chat = pair
key = "replay-" + uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
served = {first.headers["x-instance-id"]}
for _ in range(20):
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
assert again.status_code == 201 and again.json() == first.json()
served.add(again.headers["x-instance-id"])
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
assert api.history(bob, chat, 100)[0] == [first.json()]
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
original = api.send(alice, chat, "оригинал", key=key)
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
assert api.send(alice, chat, "оригинал", key=key) == original
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
alice, _, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
assert messages_in_db(chat) == 1
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
def test_missing_or_invalid_key_is_400(api, pair, value):
alice, _, chat = pair
raw_value = value.encode("utf-8")
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
assert error["error"]["code"] == "invalid_request"
assert messages_in_db(chat) == 0
def test_repeated_key_header_is_400(pair):
alice, _, chat = pair
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
headers=[("Authorization", "Bearer " + alice["token"]),
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
assert response.status_code == 400 and response.headers["x-instance-id"]
assert messages_in_db(chat) == 0
def test_longest_key_and_every_allowed_character(api, pair):
alice, _, chat = pair
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
first = api.send(alice, chat, key[:20], key=key)
assert api.send(alice, chat, key[:20], key=key) == first
assert messages_in_db(chat) == 2
def test_scope_is_user_plus_chat_plus_key(api, pair):
alice, bob, chat = pair
other = api.chat(alice, bob, title="второй")
key = "shared-key"
a1 = api.send(alice, chat, "текст", key=key)
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
assert len({a1["id"], b1["id"], a2["id"]}) == 3
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
alice, bob, chat = pair
eve = api.user("eve")
key = uuid.uuid4().hex
api.send(alice, chat, "секрет", key=key)
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
assert messages_in_db(chat) == 1
def fire_together(workers: int, request) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
barrier.wait()
try:
results[index] = request(client, index)
except BaseException as exc: # noqa: BLE001 - asserted by the test
results[index] = exc
finally:
client.close()
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
alice, bob, chat = pair
key = "race-" + uuid.uuid4().hex
def post(client: httpx.Client, _index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.headers.get("x-instance-id"), r.json()
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
results = fire_together(24, post)
assert all(isinstance(r, tuple) for r in results), results
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_concurrent_same_key_different_bodies_one_wins(pair):
alice, _, chat = pair
key = "conflict-" + uuid.uuid4().hex
def post(client: httpx.Client, index: int):
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
return r.status_code, r.json()
results = fire_together(20, post)
winners = [body for status, body in results if status == 201]
losers = [body for status, body in results if status == 409]
assert len(winners) + len(losers) == 20, results
assert winners and len({str(b) for b in winners}) == 1
assert all(b["error"]["code"] == "conflict" for b in losers)
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
assert messages_in_db(chat) == 1
def inserts_into_keys() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
transaction. Expected: 500 for the client, no key row and no message left behind
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
alice, bob, chat = pair
marker = "boom-" + uuid.uuid4().hex
key = uuid.uuid4().hex
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
"RETURN NEW; END $$; "
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
try:
before = inserts_into_keys()
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
while inserts_into_keys() == before and time.monotonic() < deadline:
time.sleep(0.5)
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
finally:
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
assert api.send(alice, chat, marker, key=key) == retried
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
f"SELECT pg_sleep({seconds}); COMMIT;")
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def wait_lock_taken(chat: dict) -> None:
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
"AND pid <> pg_backend_pid()") != "0":
return
time.sleep(0.2)
raise AssertionError("the row lock was not taken")
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 1.0)
try:
wait_lock_taken(chat)
with pytest.raises(httpx.TimeoutException):
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
finally:
locker.wait(timeout=30)
deadline = time.monotonic() + 10
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
time.sleep(0.2)
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
assert stored, "the server-side transaction was expected to commit after the lock was released"
retried = api.send(alice, chat, "неопределённо", key=key)
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
nothing; after the lock is gone the same key creates the message exactly once."""
alice, bob, chat = pair
key = uuid.uuid4().hex
locker = lock_chat_row(chat, 4.0)
try:
wait_lock_taken(chat)
started = time.monotonic()
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
assert time.monotonic() - started < 3.6
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
finally:
locker.wait(timeout=30)
created = api.send(alice, chat, "после блокировки", key=key)
assert api.send(alice, chat, "после блокировки", key=key) == created
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
def test_replay_after_the_serving_replica_is_stopped(api, pair):
alice, bob, chat = pair
key = uuid.uuid4().hex
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
served_by = first.headers["x-instance-id"]
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
with replica_stopped(replica):
for _ in range(5):
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
assert messages_in_db(chat) == 1
+3 -2
View File
@@ -57,8 +57,9 @@ def test_postgres_session_store(api):
restart_services("api")
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
with service_down("postgres", "stop"):
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"]).json()["error"]["code"] == "unavailable"
api.call("GET", "/api/v1/users/me", 401)
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"],
lb_error_ok=True).json()["error"]["code"] == "unavailable"
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
+57
View File
@@ -132,3 +132,60 @@ def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
with pytest.raises(ConfigError) as info:
Settings.from_env()
assert "u:p" not in str(info.value)
# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id
@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"])
def test_valid_idempotency_keys(key):
assert validation.idempotency_key(key) == key
@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"])
def test_invalid_idempotency_keys(key):
with pytest.raises(InvalidRequest):
validation.idempotency_key(key)
@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}'])
def test_duplicate_keys_and_garbage_json_are_rejected(raw):
with pytest.raises(InvalidRequest):
validation.parse_json_object(raw)
def test_attachment_ids_absent_equals_empty_and_is_validated():
assert validation.attachment_ids_field({"text": "x"}) == []
assert validation.attachment_ids_field({"attachment_ids": []}) == []
one = "11111111-1111-4111-8111-111111111111"
assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one]
for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11):
with pytest.raises(InvalidRequest):
validation.attachment_ids_field({"attachment_ids": bad})
def test_request_fingerprint_ignores_key_order_and_default_attachments():
from messenger.app import request_fingerprint
assert request_fingerprint("привет", []) == request_fingerprint("привет", [])
assert request_fingerprint("привет", []) != request_fingerprint("привет ", [])
assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", [])
one = "11111111-1111-4111-8111-111111111111"
assert request_fingerprint("x", [one]) != request_fingerprint("x", [])
def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch):
import socket
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.delenv("INSTANCE_ID", raising=False)
assert Settings.from_env().instance_id == socket.gethostname()
monkeypatch.setenv("INSTANCE_ID", "replica-a")
assert Settings.from_env().instance_id == "replica-a"
for bad in ("with space", "x" * 65, "ip/1"):
monkeypatch.setenv("INSTANCE_ID", bad)
with pytest.raises(ConfigError):
Settings.from_env()
def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch):
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600")
with pytest.raises(ConfigError):
Settings.from_env()