diff --git a/student-tests/conftest.py b/student-tests/conftest.py index f09a6ad..68b9e15 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -2,9 +2,10 @@ * `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services. * `test_*` in this folder — marker `stack`: black-box HTTP against the running compose - stand (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database - checks (psql inside the postgres container) and failure injection. Every failure - test restores the stopped/paused service in `finally`. + stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus + `docker compose` for database checks (psql inside the postgres container), the HAProxy + runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the + stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP. python -m pytest student-tests # everything (stand must be up) python -m pytest student-tests -m "not stack" # unit only @@ -27,10 +28,13 @@ from jsonschema import Draft202012Validator ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) +import lbctl # noqa: E402 SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8")) BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose +REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs def pytest_configure(config): @@ -74,26 +78,32 @@ def wait_ready(timeout: float = 90) -> None: def wait_healthy(*services: str, timeout: float = 120) -> None: - """Poll Docker health until every service is healthy again. `compose up --wait` is not used: - it gives up at once on a container that was marked unhealthy while paused.""" + """Poll Docker health until EVERY container of every service is healthy again. `compose up + --wait` is not used: it gives up at once on a container marked unhealthy while paused.""" deadline = time.monotonic() + timeout pending = set(services) while pending and time.monotonic() < deadline: for service in list(pending): ids = compose("ps", "-q", service, check=False).stdout.split() if ids: - state = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], - capture_output=True, text=True).stdout.strip() - if state == "healthy": + states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids], + capture_output=True, text=True).stdout.split() + if states and all(state == "healthy" for state in states): pending.discard(service) time.sleep(0.5) assert not pending, f"not healthy: {pending}" +def wait_stand(replicas: int = REPLICAS) -> None: + """Readiness through the LB AND every replica back in HAProxy rotation.""" + wait_ready() + lbctl.wait_up(replicas, timeout=90) + + def restart_services(*services: str) -> None: compose("restart", *services) wait_healthy(*services) - wait_ready() + wait_stand() @contextlib.contextmanager @@ -106,7 +116,7 @@ def service_down(service: str, how: str = "stop"): finally: compose("unpause" if how == "pause" else "start", service) wait_healthy(service) - wait_ready() + wait_stand() @contextlib.contextmanager @@ -116,14 +126,33 @@ def api_with_env(**overrides: str): env = {**os.environ, **overrides} subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env, capture_output=True, text=True, check=True, timeout=180) - wait_ready() + wait_stand() try: yield finally: clean = {k: v for k, v in os.environ.items() if k not in overrides} subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean, capture_output=True, text=True, check=True, timeout=180) - wait_ready() + wait_stand() + + +def running_replicas() -> list[dict]: + return [r for r in lbctl.replicas() if r["state"] == "running"] + + +@contextlib.contextmanager +def replica_stopped(replica: dict, mode: str = "stop"): + """Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again + and wait until HAProxy has every replica UP.""" + assert mode in {"stop", "kill"} + subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]], + capture_output=True, text=True, check=True, timeout=60) + try: + yield + finally: + subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60) + wait_healthy("api") + wait_stand() def api_env(name: str) -> str: @@ -139,7 +168,10 @@ class Api: self.client = httpx.Client(base_url=base_url, timeout=15) def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None, - json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None): + json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None, + lb_error_ok: bool = False): + """lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is + accepted without X-Instance-Id; every response of an API replica must carry it.""" h = dict(headers or {}) if token is not None: h["Authorization"] = "Bearer " + token @@ -151,6 +183,9 @@ class Api: assert response.status_code in allowed, (method, path, response.status_code, response.text[:300]) request_id = response.headers.get("x-request-id") assert request_id, "every response must carry X-Request-Id" + if not response.headers.get("x-instance-id"): + assert lb_error_ok and response.status_code in (502, 503, 504), \ + ("missing X-Instance-Id", method, path, response.status_code) if response.status_code >= 400: body = response.json() assert schema_errors(body, "Error") == [], response.text @@ -186,9 +221,18 @@ class Api: return self.call("POST", "/api/v1/chats", 201, token=owner["token"], json_body={"title": title, "member_ids": ids}, schema="Chat").json() - def send(self, actor: dict, chat: dict, text: str) -> dict: - return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=actor["token"], - json_body={"text": text}, schema="Message").json() + def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None, + raw: bytes | None = None, headers: dict | None = None, **kwargs): + """POST a message with an Idempotency-Key (a fresh one unless `key` is given).""" + h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})} + if raw is not None: + h.setdefault("Content-Type", "application/json") + return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"], + json_body=body if raw is None else None, raw=raw, headers=h, + schema="Message" if expected == 201 else None, **kwargs) + + def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict: + return self.post_message(actor, chat, {"text": text}, key=key).json() def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: """Walk all pages; returns (items, number_of_pages).""" @@ -224,3 +268,24 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture(scope="session") def api() -> Api: return Api() + + +def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess: + """Run scripts/ (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers.""" + script = str(ROOT / "scripts" / name) + if os.name == "nt": + command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script] + else: + command = [script] + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True, + timeout=timeout, env={**os.environ, **env}) + + +def probe_inside(replica: dict, path: str) -> int: + """HTTP status of `path` asked inside the replica container itself (bypassing the LB).""" + code = ("import urllib.request as u, urllib.error as e\n" + "try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n" + "except e.HTTPError as x: print(x.code)") % path + out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True, + timeout=30).stdout.strip() + return int(out or 0) diff --git a/student-tests/test_api.py b/student-tests/test_api.py index d99077e..7339580 100644 --- a/student-tests/test_api.py +++ b/student-tests/test_api.py @@ -108,7 +108,7 @@ def test_unicode_limits_are_code_points_and_nul_is_stored(api): assert chat["title"] == "T\u0000itle" assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world" assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000 - api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=actor["token"], json_body={"text": "я" * 4001}) + api.post_message(actor, chat, {"text": "я" * 4001}, 400) texts = [m["text"] for m in api.history(actor, chat, 100)[0]] assert texts == ["hello\u0000world", "🙂" * 4000] @@ -136,7 +136,7 @@ def test_foreign_chat_is_indistinguishable_from_missing(api): base = f"/api/v1/chats/{chat_id}" errors = [api.call("GET", base, 404, token=eve["token"]).json(), api.call("GET", base + "/messages", 404, token=eve["token"]).json(), - api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}).json(), + api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(), api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()] assert len({e["error"]["message"] for e in errors}) == 1 api.call("GET", "/api/v1/chats/not-a-uuid", 401) @@ -199,3 +199,106 @@ def test_read_after_write(api): message = api.send(alice, chat, f"m{i}") last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1] assert last == message + + +# ================================================================== lab 3: inputs found by the reviewer in other PRs +def b64(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def bad_cursors(owner_id: str) -> list[str]: + """Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time.""" + good_uuid = str(uuid.uuid4()) + return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"), + b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'), + b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()), + b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()), + b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()), + b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()), + b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())] + + +def test_malformed_cursor_values_are_400_on_both_lists(api): + alice = api.user("cur") + chat = api.chat(alice) + for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]): + quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20") + for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json() + assert error["error"]["code"] == "invalid_request", (path, cursor) + + +@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}', + b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}', + b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}', + b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}', + b'{"text":" \n\t "}']) +def test_bad_message_body_is_400_and_nothing_is_stored(api, raw): + alice = api.user("body") + chat = api.chat(alice) + error = api.post_message(alice, chat, None, 400, raw=raw).json() + assert error["error"]["code"] == "invalid_request" + assert api.history(alice, chat, 100)[0] == [] + + +def test_ff_byte_body_on_registration_is_contract_error(api): + response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"}) + assert response.json()["error"]["request_id"] == response.headers["x-request-id"] + + +def test_text_is_stored_verbatim_and_length_counts_raw_text(api): + alice = api.user("verbatim") + chat = api.chat(alice) + padded = " строка\nс пробелами " + assert api.send(alice, chat, padded)["text"] == padded + api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming + assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded] + + +def test_unknown_attachment_ids_are_404(api): + alice = api.user("att") + chat = api.chat(alice) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates + assert api.history(alice, chat, 100)[0] == [] + + +def test_member_ids_are_compared_as_uuids_not_strings(api): + alice, bob = api.user("case"), api.user("case") + api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]}) + api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())}) + chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json() + assert chat["member_ids"] == [alice["id"], bob["id"]] + + +@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43]) +def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header): + api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")}) + + +def test_login_with_a_body_is_rejected(api): + actor = api.register("login_body") + auth = {"Authorization": basic(actor["username"], actor["password"])} + api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"}) + api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth) + api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session") + + +def test_password_range_is_12_to_128_code_points(api): + api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"}) + actor = api.user("pw128", password="я:" * 64) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_every_api_response_carries_instance_and_request_id(api): + """X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it).""" + api.call("GET", "/health/live", 200) + api.call("GET", "/health/ready", 200) + api.call("GET", "/no/such/route", 404) + api.call("PUT", "/api/v1/users", 405) + api.call("GET", "/api/v1/users/me", 401) + response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"}) + assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..edf9dd6 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,298 @@ +"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4). + +Each test that stops, kills, rescales or isolates a replica restores the stand in `finally` +and waits until HAProxy reports every replica UP again (conftest.wait_stand). +""" +from __future__ import annotations + +import contextlib +import json +import subprocess +import threading +import time +import uuid + +import httpx2 as httpx +import pytest + +from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \ + running_replicas, wait_healthy, wait_stand + +pytestmark = pytest.mark.stack + + +def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]: + return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)] + + +# ------------------------------------------------------------------ level 3: replicas, no affinity +def test_one_session_is_served_by_every_real_replica(api): + actor = api.user("rr") + seen = instances_seen(api, actor["token"]) + running = {r["hostname"] for r in running_replicas()} + assert len(running) == REPLICAS >= 2 + assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers" + # Round robin on ONE keep-alive client connection: consecutive requests switch replicas, + # so there is no sticky session hiding split state. + switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b) + assert switches >= len(seen) // 2, seen + assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running + + +def test_write_on_one_replica_is_read_on_the_other(api): + alice, bob = api.user("rw"), api.user("rw") + chat = api.chat(alice, bob) + for i in range(10): + sent = api.post_message(alice, chat, {"text": f"m{i}"}) + read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]) + assert read.json()["items"][-1] == sent.json() + logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + for _ in range(6): # the revocation is visible on every replica at once (shared Valkey) + r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + assert r.headers["x-instance-id"] + assert logout.headers["x-instance-id"] + + +# ------------------------------------------------------------------ readiness drives the LB +def hba_file() -> str: + return psql("SHOW hba_file") + + +@contextlib.contextmanager +def isolated_from_postgres(replica: dict): + """Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its + connections), while it keeps running and answering /health/live. Always undone.""" + ip = replica["ips"][0] + path = hba_file() + marker = "lab3-test-isolate" + compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}") + try: + psql("SELECT pg_reload_conf()") + psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'") + yield replica + finally: + compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}") + psql("SELECT pg_reload_conf()") + wait_stand() + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position): + """The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is + 200, /health/ready 503 -> HAProxy must take it out; no business request may reach it.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + actor = api.user("ready") + with isolated_from_postgres(replica): + started = time.monotonic() + while replica["hostname"] in lbctl.up_instances(): + assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s" + time.sleep(0.1) + detected = time.monotonic() - started + assert probe_inside(replica, "/health/live") == 200 # the process is alive ... + assert probe_inside(replica, "/health/ready") == 503 # ... but not ready + seen = instances_seen(api, actor["token"], 40) + assert replica["hostname"] not in seen and len(seen) == 40 + for i in range(20): # writes keep working, only through the ready replicas + assert api.register(f"r{i}")["id"] + print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s") + + +# ------------------------------------------------------------------ stopping replicas under traffic +class Traffic: + """Background writers/readers through the LB; uncertain POSTs are retried with the SAME key.""" + + def __init__(self, api: Api, workers: int = 4): + self.alice, self.bob = api.user("load"), api.user("load") + self.chat = api.chat(self.alice, self.bob) + self.workers = workers + self.stop = threading.Event() + self.lock = threading.Lock() + self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance) + self.confirmed: list[dict] = [] + self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)] + + def record(self, kind: str, status: int, instance: str | None) -> None: + with self.lock: + self.events.append((time.monotonic(), kind, status, instance)) + + def run(self, index: int) -> None: + client = httpx.Client(base_url=BASE_URL, timeout=3) + auth = {"Authorization": "Bearer " + self.alice["token"]} + n = 0 + key, text = uuid.uuid4().hex, f"w{index}-0" + while not self.stop.is_set(): + try: + r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text}, + headers={**auth, "Idempotency-Key": key}) + self.record("POST", r.status_code, r.headers.get("x-instance-id")) + if r.status_code == 201: + with self.lock: + self.confirmed.append(r.json()) + n += 1 + key, text = uuid.uuid4().hex, f"w{index}-{n}" + # any other outcome: keep key and text, retry the same request + except httpx.HTTPError: + self.record("POST", 0, None) + try: + r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth) + self.record("GET", r.status_code, r.headers.get("x-instance-id")) + except httpx.HTTPError: + self.record("GET", 0, None) + time.sleep(0.02) + client.close() + + def __enter__(self) -> "Traffic": + for t in self.threads: + t.start() + time.sleep(1.0) + return self + + def __exit__(self, *exc) -> None: + self.stop.set() + for t in self.threads: + t.join(timeout=10) + + def failures(self, since: float = 0.0) -> list[tuple]: + return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)] + + def check_history(self, api: Api) -> None: + history = api.history(self.bob, self.chat, 100)[0] + ids, texts = [m["id"] for m in history], [m["text"] for m in history] + assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry" + assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost" + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_graceful_stop_of_each_replica_loses_no_request(api, position): + """SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + with Traffic(api) as traffic: + started = time.monotonic() + with replica_stopped(replica, "stop"): + stopped_after = time.monotonic() - started + time.sleep(3) + after = [e for e in traffic.events if e[0] > time.monotonic() - 2] + assert after and replica["hostname"] not in {e[3] for e in after} + time.sleep(1) + state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, + check=True).stdout)[0]["State"] + assert traffic.failures() == [], traffic.failures()[:10] + assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)" + assert state["Status"] == "running" # restored by replica_stopped + traffic.check_history(api) + print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed") + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position): + """SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within + 10 s, the killed replica gets nothing after detection, retries do not duplicate.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + with Traffic(api) as traffic: + killed_at = time.monotonic() + with replica_stopped(replica, "kill"): + time.sleep(4) + failures = traffic.failures(killed_at) + last_failure = max((e[0] for e in failures), default=killed_at) - killed_at + post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201] + assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s" + late = [e for e in traffic.events if e[0] > killed_at + 3.5] + assert late and replica["hostname"] not in {e[3] for e in late} + assert last_failure <= 10 + traffic.check_history(api) + print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s") + + +def test_graceful_shutdown_order_in_logs(api): + """Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[-1] + since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1)) + with replica_stopped(replica, "stop"): + logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True, + text=True, encoding="utf-8").stdout + lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout + state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, + check=True).stdout)[0]["State"] + events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] + names = [e["event"] for e in events] + assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped") + t = {e["event"]: e["ts"] for e in events} + assert t["drain_started"] < t["shutdown_requested"] + assert state["ExitCode"] == 0 and not state["OOMKilled"] + assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs) + + +# ------------------------------------------------------------------ discovery: restart and scaling +def test_full_restart_lb_rediscovers_all_replicas(api): + actor = api.user("restart") + before = {r["hostname"]: r["started_at"] for r in running_replicas()} + run_hook("restart.sh") + wait_healthy("postgres", "valkey", "api") + wait_stand() + after = {r["hostname"]: r["started_at"] for r in running_replicas()} + assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted" + assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed + + +def test_scale_up_and_down_is_picked_up_without_errors(api): + with Traffic(api, workers=2) as traffic: + try: + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api", + timeout=180) + lbctl.wait_up(REPLICAS + 1, timeout=30) + actor = api.user("scale") + assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1 + finally: + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api", + timeout=180) + wait_stand() + time.sleep(1) + assert traffic.failures() == [], traffic.failures()[:10] + traffic.check_history(api) + + +# ------------------------------------------------------------------ level 4: deployment policy +def test_compose_policy_limits_ports_secrets_images(): + config = json.loads(compose("config", "--format", "json").stdout) + services = config["services"] + assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"} + for name, svc in services.items(): + limits = svc.get("deploy", {}).get("resources", {}).get("limits", {}) + assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits" + assert "container_name" not in svc, f"{name}: container_name prevents scaling" + if name in ("api", "migrate"): + assert "build" in svc # built from this repository's source + else: + assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest" + assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports") + published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")} + assert set(published) == {"lb"} and all(p["host_ip"] == "127.0.0.1" for p in published["lb"]) + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()} + assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], + "migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []} + volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] + for name, svc in services.items()} + assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []} + + +def test_running_containers_have_limits_and_no_bypass_ports(): + ids = compose("ps", "-q").stdout.split() + for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True, + check=True).stdout): + name = item["Name"] + assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name + bindings = item["HostConfig"]["PortBindings"] or {} + if "-lb-" in name: + assert all(b["HostIp"] == "127.0.0.1" for v in bindings.values() for b in v) + else: + assert not bindings, f"{name} publishes ports in bypass of the LB" + + +def test_migrations_ran_once_by_the_migrate_service(): + applied = psql("SELECT version FROM schema_migrations ORDER BY version").split() + assert applied == ["0001_init", "0002_idempotency_keys"] + migrate = compose("ps", "-a", "--format", "json", "migrate").stdout + rows = [json.loads(line) for line in migrate.splitlines() if line.strip()] + assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows) + api_logs = compose("logs", "--no-log-prefix", "api").stdout + assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate diff --git a/student-tests/test_db.py b/student-tests/test_db.py index fd065e1..3e250c6 100644 --- a/student-tests/test_db.py +++ b/student-tests/test_db.py @@ -76,7 +76,8 @@ def test_migrations_rerun_is_a_noop(): users_before = psql("SELECT count(*) FROM users") result = compose("run", "--rm", "--no-deps", "migrate") assert "0001_init: already applied, skipped" in result.stdout - assert psql("SELECT count(*) FROM schema_migrations") == "1" + assert "0002_idempotency_keys: already applied, skipped" in result.stdout + assert psql("SELECT count(*) FROM schema_migrations") == "2" assert psql("SELECT count(*) FROM users") == users_before diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py index 0dce65a..899d29b 100644 --- a/student-tests/test_failures.py +++ b/student-tests/test_failures.py @@ -2,6 +2,11 @@ Every test stops/pauses a real container and restores it in `finally` (service_down). Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5. + +Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer +503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same +contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401 +for a valid token. Liveness/readiness of the processes are checked inside each container. """ from __future__ import annotations @@ -10,7 +15,7 @@ import time import pytest -from conftest import ROOT, restart_services, service_down +from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down pytestmark = pytest.mark.stack @@ -20,10 +25,16 @@ CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead def timed(api, *args, **kwargs): started = time.monotonic() - response = api.call(*args, **kwargs) + response = api.call(*args, lb_error_ok=True, **kwargs) return response, time.monotonic() - started +def processes_alive_but_not_ready() -> None: + for replica in running_replicas(): + assert probe_inside(replica, "/health/live") == 200 + assert probe_inside(replica, "/health/ready") == 503 + + @pytest.fixture def prepared(api): alice, bob = api.user("alice"), api.user("bob") @@ -46,9 +57,9 @@ def test_valkey_failure(api, prepared, how): response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"]) assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) - api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected - api.call("GET", "/health/ready", 503) - api.call("GET", "/health/live", 200) + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access + api.call("GET", "/health/ready", 503, lb_error_ok=True) + processes_alive_but_not_ready() check_intact(api, alice, bob, chat, message) @@ -60,9 +71,9 @@ def test_postgres_failure(api, prepared, how): response, took = timed(api, "GET", path, 503, token=alice["token"]) assert response.json()["error"]["code"] == "unavailable", path assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause) - api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected - api.call("GET", "/health/ready", 503) - api.call("GET", "/health/live", 200) + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access + api.call("GET", "/health/ready", 503, lb_error_ok=True) + processes_alive_but_not_ready() check_intact(api, alice, bob, chat, message) @@ -77,6 +88,7 @@ def test_locked_table_gives_bounded_503(api, prepared): time.sleep(1.5) # let the lock be taken response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took + assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready" finally: locker.wait(timeout=30) check_intact(api, alice, bob, chat, message) @@ -88,7 +100,7 @@ def test_logout_during_valkey_outage_is_never_lost(api, prepared): after recovery logout works, and the revocation survives a Valkey and API restart (AOF).""" alice, bob, chat, message = prepared with service_down("valkey", "stop"): - api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True) api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..a1e2bcc --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,281 @@ +"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer. + +Strict assertions: exact HTTP codes for every request, identical bodies for replays, +the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica +answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always +undone in `finally`. +""" +from __future__ import annotations + +import subprocess +import threading +import time +import uuid + +import httpx2 as httpx +import pytest + +from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas + +pytestmark = pytest.mark.stack + + +def messages_in_db(chat: dict) -> int: + return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'")) + + +def key_rows(actor: dict, key: str) -> int: + return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'")) + + +@pytest.fixture +def pair(api): + alice, bob = api.user("alice"), api.user("bob") + return alice, bob, api.chat(alice, bob) + + +def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair): + alice, bob, chat = pair + key = "replay-" + uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "один раз"}, key=key) + served = {first.headers["x-instance-id"]} + for _ in range(20): + again = api.post_message(alice, chat, {"text": "один раз"}, key=key) + assert again.status_code == 201 and again.json() == first.json() + served.add(again.headers["x-instance-id"]) + assert len(served) >= 2, "replays must be answered by different replicas, not a local cache" + assert api.history(bob, chat, 100)[0] == [first.json()] + assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair): + alice, bob, chat = pair + key = uuid.uuid4().hex + original = api.send(alice, chat, "оригинал", key=key) + for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}): + assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict" + assert api.send(alice, chat, "оригинал", key=key) == original + assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1 + + +def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair): + alice, _, chat = pair + key = uuid.uuid4().hex + first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json() + for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}', + '{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'): + assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first + assert messages_in_db(chat) == 1 + + +@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"]) +def test_missing_or_invalid_key_is_400(api, pair, value): + alice, _, chat = pair + raw_value = value.encode("utf-8") + error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json() + assert error["error"]["code"] == "invalid_request" + assert messages_in_db(chat) == 0 + + +def test_repeated_key_header_is_400(pair): + alice, _, chat = pair + response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10, + headers=[("Authorization", "Bearer " + alice["token"]), + ("Idempotency-Key", "one"), ("Idempotency-Key", "two")]) + assert response.status_code == 400 and response.headers["x-instance-id"] + assert messages_in_db(chat) == 0 + + +def test_longest_key_and_every_allowed_character(api, pair): + alice, _, chat = pair + for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex): + first = api.send(alice, chat, key[:20], key=key) + assert api.send(alice, chat, key[:20], key=key) == first + assert messages_in_db(chat) == 2 + + +def test_scope_is_user_plus_chat_plus_key(api, pair): + alice, bob, chat = pair + other = api.chat(alice, bob, title="второй") + key = "shared-key" + a1 = api.send(alice, chat, "текст", key=key) + b1 = api.send(bob, chat, "текст", key=key) # another user: independent + a2 = api.send(alice, other, "текст", key=key) # another chat: independent + assert len({a1["id"], b1["id"], a2["id"]}) == 3 + assert messages_in_db(chat) == 2 and messages_in_db(other) == 1 + + +def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair): + alice, bob, chat = pair + eve = api.user("eve") + key = uuid.uuid4().hex + api.send(alice, chat, "секрет", key=key) + for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message + assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found" + api.post_message(eve, chat, {"text": "другое"}, 404, key=key) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again + assert messages_in_db(chat) == 1 + + +def fire_together(workers: int, request) -> list: + barrier = threading.Barrier(workers) + results: list = [None] * workers + + def body(index: int) -> None: + client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them + barrier.wait() + try: + results[index] = request(client, index) + except BaseException as exc: # noqa: BLE001 - asserted by the test + results[index] = exc + finally: + client.close() + + threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + return results + + +def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair): + alice, bob, chat = pair + key = "race-" + uuid.uuid4().hex + + def post(client: httpx.Client, _index: int): + r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"}, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + return r.status_code, r.headers.get("x-instance-id"), r.json() + + for _ in range(3): # three rounds: the first creates, later rounds replay concurrently + results = fire_together(24, post) + assert all(isinstance(r, tuple) for r in results), results + assert [r[0] for r in results] == [201] * 24, [r[0] for r in results] + assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message" + assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas" + assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_concurrent_same_key_different_bodies_one_wins(pair): + alice, _, chat = pair + key = "conflict-" + uuid.uuid4().hex + + def post(client: httpx.Client, index: int): + r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"}, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + return r.status_code, r.json() + + results = fire_together(20, post) + winners = [body for status, body in results if status == 201] + losers = [body for status, body in results if status == 409] + assert len(winners) + len(losers) == 20, results + assert winners and len({str(b) for b in winners}) == 1 + assert all(b["error"]["code"] == "conflict" for b in losers) + assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"} + assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"]) + assert messages_in_db(chat) == 1 + + +def inserts_into_keys() -> int: + return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'")) + + +def test_failure_after_the_key_insert_rolls_everything_back(api, pair): + """A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same + transaction. Expected: 500 for the client, no key row and no message left behind + (n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds.""" + alice, bob, chat = pair + marker = "boom-" + uuid.uuid4().hex + key = uuid.uuid4().hex + psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ " + f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; " + "RETURN NEW; END $$; " + "CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()") + try: + before = inserts_into_keys() + error = api.post_message(alice, chat, {"text": marker}, 500, key=key) + assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text + assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 + deadline = time.monotonic() + 20 # table statistics are flushed asynchronously + while inserts_into_keys() == before and time.monotonic() < deadline: + time.sleep(0.5) + assert inserts_into_keys() > before, "the key INSERT never ran before the failure" + finally: + psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()") + retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded + assert api.send(alice, chat, marker, key=key) == retried + assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1 + + +def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen: + sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; " + f"SELECT pg_sleep({seconds}); COMMIT;") + return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", + "messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def wait_lock_taken(chat: dict) -> None: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' " + "AND pid <> pg_backend_pid()") != "0": + return + time.sleep(0.2) + raise AssertionError("the row lock was not taken") + + +def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair): + """Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row + lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one.""" + alice, bob, chat = pair + key = uuid.uuid4().hex + locker = lock_chat_row(chat, 1.0) + try: + wait_lock_taken(chat) + with pytest.raises(httpx.TimeoutException): + httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + finally: + locker.wait(timeout=30) + deadline = time.monotonic() + 10 + while messages_in_db(chat) == 0 and time.monotonic() < deadline: + time.sleep(0.2) + stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'") + assert stored, "the server-side transaction was expected to commit after the lock was released" + retried = api.send(alice, chat, "неопределённо", key=key) + assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair): + """The row lock outlives lock_timeout: the API answers 503 within its deadline and stores + nothing; after the lock is gone the same key creates the message exactly once.""" + alice, bob, chat = pair + key = uuid.uuid4().hex + locker = lock_chat_row(chat, 4.0) + try: + wait_lock_taken(chat) + started = time.monotonic() + api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key) + assert time.monotonic() - started < 3.6 + assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 + finally: + locker.wait(timeout=30) + created = api.send(alice, chat, "после блокировки", key=key) + assert api.send(alice, chat, "после блокировки", key=key) == created + assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1 + + +def test_replay_after_the_serving_replica_is_stopped(api, pair): + alice, bob, chat = pair + key = uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "до остановки"}, key=key) + served_by = first.headers["x-instance-id"] + replica = next(r for r in running_replicas() if r["hostname"] == served_by) + with replica_stopped(replica): + for _ in range(5): + again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key) + assert again.json() == first.json() and again.headers["x-instance-id"] != served_by + assert messages_in_db(chat) == 1 diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py index 245db76..8cbda00 100644 --- a/student-tests/test_sessions.py +++ b/student-tests/test_sessions.py @@ -57,8 +57,9 @@ def test_postgres_session_store(api): restart_services("api") api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) with service_down("postgres", "stop"): - assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"]).json()["error"]["code"] == "unavailable" - api.call("GET", "/api/v1/users/me", 401) + assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"], + lb_error_ok=True).json()["error"]["code"] == "unavailable" + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"]) api.call("GET", "/api/v1/users/me", 401, token=actor["token"]) diff --git a/student-tests/unit/test_unit.py b/student-tests/unit/test_unit.py index 3461286..ebbc5b7 100644 --- a/student-tests/unit/test_unit.py +++ b/student-tests/unit/test_unit.py @@ -132,3 +132,60 @@ def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch): with pytest.raises(ConfigError) as info: Settings.from_env() assert "u:p" not in str(info.value) + + +# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id +@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"]) +def test_valid_idempotency_keys(key): + assert validation.idempotency_key(key) == key + + +@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"]) +def test_invalid_idempotency_keys(key): + with pytest.raises(InvalidRequest): + validation.idempotency_key(key) + + +@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}']) +def test_duplicate_keys_and_garbage_json_are_rejected(raw): + with pytest.raises(InvalidRequest): + validation.parse_json_object(raw) + + +def test_attachment_ids_absent_equals_empty_and_is_validated(): + assert validation.attachment_ids_field({"text": "x"}) == [] + assert validation.attachment_ids_field({"attachment_ids": []}) == [] + one = "11111111-1111-4111-8111-111111111111" + assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one] + for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11): + with pytest.raises(InvalidRequest): + validation.attachment_ids_field({"attachment_ids": bad}) + + +def test_request_fingerprint_ignores_key_order_and_default_attachments(): + from messenger.app import request_fingerprint + assert request_fingerprint("привет", []) == request_fingerprint("привет", []) + assert request_fingerprint("привет", []) != request_fingerprint("привет ", []) + assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", []) + one = "11111111-1111-4111-8111-111111111111" + assert request_fingerprint("x", [one]) != request_fingerprint("x", []) + + +def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch): + import socket + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.delenv("INSTANCE_ID", raising=False) + assert Settings.from_env().instance_id == socket.gethostname() + monkeypatch.setenv("INSTANCE_ID", "replica-a") + assert Settings.from_env().instance_id == "replica-a" + for bad in ("with space", "x" * 65, "ip/1"): + monkeypatch.setenv("INSTANCE_ID", bad) + with pytest.raises(ConfigError): + Settings.from_env() + + +def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600") + with pytest.raises(ConfigError): + Settings.from_env()