diff --git a/bench/track1/compose.bench.yaml b/bench/track1/compose.bench.yaml index 1d52cc4..5571415 100644 --- a/bench/track1/compose.bench.yaml +++ b/bench/track1/compose.bench.yaml @@ -21,6 +21,7 @@ services: image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34 profiles: ["bench"] cpuset: "${K6_CPUSET:-0-1}" + networks: [edge] # reaches the LB only, like an external client environment: BASE_URL: http://lb:8080 volumes: diff --git a/bench/track1/run.py b/bench/track1/run.py index d076543..825d435 100644 --- a/bench/track1/run.py +++ b/bench/track1/run.py @@ -123,6 +123,19 @@ def start_stand(cfg: dict) -> None: raise RuntimeError("not all replicas UP in HAProxy") +def actual_limits() -> dict: + """What Docker really applied (not what the config intended): CPU quota, memory, cpuset.""" + ids = compose("ps", "-q").split() + result = {} + for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True, + check=True).stdout): + host = item["HostConfig"] + result[item["Name"].lstrip("/")[len(PROJECT) + 1:]] = { + "cpus": host["NanoCpus"] / 1e9, "memory_mib": host["Memory"] // (1024 * 1024), + "cpuset": host["CpusetCpus"], "image": item["Config"]["Image"]} + return result + + def make_chats(users: int) -> list[dict]: created = [] stamp = uuid.uuid4().hex[:8] @@ -171,6 +184,11 @@ class Sampler(threading.Thread): def window(self, start: float, end: float) -> dict: rows = [r for t, r in self.samples if start <= t <= end] result = {} + # All API replicas summed PER SAMPLE first, then the median over time (not a sum of medians). + totals = [sum(v["cpu"] for k, v in r.items() if k.startswith("api-")) for r in rows] + if totals: + result["api_total"] = {"cpu_median": round(statistics.median(totals), 1), "cpu_max": round(max(totals), 1), + "samples": len(totals)} for name in sorted({n for r in rows for n in r}): values = [r[name]["cpu"] for r in rows if name in r] result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1), @@ -281,8 +299,13 @@ def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> di def flat(r: dict) -> dict: cpu = r["cpu"] - api_cpu = sum(v["cpu_median"] for k, v in cpu.items() if k.startswith("api-")) + api_cpu = cpu.get("api_total", {}).get("cpu_median", 0.0) + limits = r.get("limits", {}) + api_limits = [v for k, v in limits.items() if k.startswith("api-")] return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"], + "replicas_running": len(api_limits), "api_cpus_limit_total": round(sum(v["cpus"] for v in api_limits), 2), + "api_mem_mib_each": api_limits[0]["memory_mib"] if api_limits else None, + "pg_cpus_limit": limits.get("postgres-1", {}).get("cpus"), "requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"], "p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"], "api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"), @@ -333,6 +356,7 @@ def main() -> int: name = config_name(cfg) print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True) start_stand(cfg) + limits = actual_limits() chats_dir = tempfile.mkdtemp(prefix="track1-") try: Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8") @@ -341,6 +365,7 @@ def main() -> int: for rate in rates: result = run_step(cfg, rate, args, chats_dir, sampler) result["repeat"] = repeat + result["limits"] = limits (out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text( json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") row = flat(result) diff --git a/compose.yaml b/compose.yaml index 2a8015d..53789a7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -6,6 +6,8 @@ # Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb. # Images are pinned by version AND digest. Passwords are passed only as secret files, and # every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only). +# Two networks: `edge` (lb <-> api) and `data` (api/migrate <-> postgres/valkey, internal): +# the LB cannot reach the stores, and the stores are not reachable from the edge. name: mtusi-lab3 x-api-env: &api-env @@ -46,6 +48,7 @@ services: start_period: 10s stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery restart: unless-stopped + networks: [data] deploy: resources: limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"} @@ -73,6 +76,7 @@ services: retries: 30 start_period: 5s restart: unless-stopped + networks: [data] deploy: resources: limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} @@ -95,6 +99,7 @@ services: healthcheck: disable: true restart: "no" + networks: [data] deploy: resources: limits: {cpus: "0.5", memory: 256m} @@ -125,6 +130,7 @@ services: condition: service_completed_successfully stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s) restart: unless-stopped + networks: [edge, data] deploy: replicas: ${API_REPLICAS:-2} resources: @@ -147,10 +153,16 @@ services: api: condition: service_healthy restart: unless-stopped + networks: [edge] deploy: resources: limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} +networks: + edge: + data: + internal: true # no route outside: stores talk only to api/migrate + volumes: pgdata: valkeydata: diff --git a/scripts/lbctl.py b/scripts/lbctl.py index 755d4c6..259d35e 100644 --- a/scripts/lbctl.py +++ b/scripts/lbctl.py @@ -52,6 +52,8 @@ def replicas(all_states: bool = True) -> list[dict]: "started_at": item["State"]["StartedAt"], "pid": item["State"]["Pid"], "ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")), + # compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...} + "networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")}, }) return sorted(result, key=lambda r: r["name"]) diff --git a/src/messenger/db.py b/src/messenger/db.py index 6c753a6..33f4126 100644 --- a/src/messenger/db.py +++ b/src/messenger/db.py @@ -8,12 +8,21 @@ Three independent limits keep a request from hanging when the database misbehave * client deadline — covers what the server cannot report, e.g. a paused/frozen PostgreSQL container or a dead network. -The client deadline needs care: when a psycopg query is cancelled, psycopg sends a -cancel request to the server and waits for it (up to 5 s + 5 s) before giving up. -With a frozen server that would stretch a 3 s deadline to 13 s. Therefore every unit -of work runs as its own task: at the deadline the request stops waiting and gets 503 -at once, while the task finishes its cleanup (cancel, close, return to the pool) in -the background. Any failure becomes 503 `unavailable` — never 401, never a hang. +The client deadline needs care. Every unit of work runs as its own task; at the deadline +the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a +psycopg query makes psycopg send a cancel request, and libpq starts that connection +(PQcancelStart) synchronously in the event-loop thread, resolving the host name first. +When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network +disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute. +The abandoned task instead ends by itself, bounded by: + +* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network + breaks the socket within seconds, the connection is discarded by the pool; +* `statement_timeout` / `lock_timeout` on a live but slow server; +* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits + at most the pool timeout and gets 503. + +Any failure becomes 503 `unavailable` — never 401, never a hang. """ from __future__ import annotations @@ -65,6 +74,12 @@ class Database: "options": options, "connect_timeout": max(1, int(settings.db_timeout_seconds)), "application_name": "messenger-api", + # Detect a dead peer / lost network without any cancel request (see module doc). + "keepalives": 1, + "keepalives_idle": 5, + "keepalives_interval": 1, + "keepalives_count": 3, + "tcp_user_timeout": int(settings.db_timeout_seconds * 1000), }, ) @@ -95,8 +110,8 @@ class Database: raise Unavailable("База данных временно недоступна") from None finally: if not task.done(): - # Deadline or cancelled request: stop waiting now, let psycopg clean up later. - task.cancel() + # Deadline or cancelled request: stop waiting now. The task is left to finish + # on its own (no task.cancel(): that would trigger psycopg's blocking cancel). task.add_done_callback(_forget) async def ping(self, timeout: float = 1.0) -> bool: diff --git a/student-tests/test_api.py b/student-tests/test_api.py index 7339580..13fc2d2 100644 --- a/student-tests/test_api.py +++ b/student-tests/test_api.py @@ -302,3 +302,43 @@ def test_every_api_response_carries_instance_and_request_id(api): api.call("GET", "/api/v1/users/me", 401) response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"}) assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log + + +@pytest.mark.parametrize("blank", [" ", "  ", "  \t "]) +def test_unicode_whitespace_only_strings_are_400(api, blank): + alice = api.user("blank") + chat = api.chat(alice) + api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank, + "password": PASSWORD}) + api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]}) + api.post_message(alice, chat, {"text": blank}, 400) + assert api.history(alice, chat, 100)[0] == [] + + +@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}', + '{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}', + b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}']) +def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw): + api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"}) + api.call("GET", "/health/ready", 200) + + +def test_more_than_100_members_is_400(api): + owner = api.user("big") + others = [api.register(f"m{i}")["id"] for i in range(100)] + api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]}) + chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"], + json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json() + assert len(chat["member_ids"]) == 100 + + +def test_cursor_of_one_list_is_rejected_by_the_other(api): + alice = api.user("cross") + chats = [api.chat(alice, title=f"c{i}") for i in range(2)] + for _ in range(2): + api.send(alice, chats[0], "m") + message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200, + token=alice["token"]).json()["next_cursor"] + chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"] + api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"]) + api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py index edf9dd6..d1b2eed 100644 --- a/student-tests/test_balancing.py +++ b/student-tests/test_balancing.py @@ -62,7 +62,7 @@ def hba_file() -> str: def isolated_from_postgres(replica: dict): """Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its connections), while it keeps running and answering /health/live. Always undone.""" - ip = replica["ips"][0] + ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network path = hba_file() marker = "lab3-test-isolate" compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}") @@ -73,28 +73,51 @@ def isolated_from_postgres(replica: dict): finally: compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}") psql("SELECT pg_reload_conf()") + lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed wait_stand() +@contextlib.contextmanager +def disconnected_from_data_network(replica: dict): + """`docker network disconnect _data `: the replica keeps its edge link to the + LB but loses PostgreSQL AND Valkey. Always reconnected.""" + network = next(n for n in json.loads(subprocess.run( + ["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]], + capture_output=True, text=True, check=True).stdout) if n.endswith("_data")) + subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True) + try: + yield + finally: + subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True) + lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout) + wait_stand() + + +@pytest.mark.parametrize("how", ["pg_hba", "network"]) @pytest.mark.parametrize("position", range(REPLICAS)) -def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position): - """The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is - 200, /health/ready 503 -> HAProxy must take it out; no business request may reach it.""" +def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how): + """The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data + network (docker network disconnect) but keeps running. /health/live 200, /health/ready + 503 -> HAProxy must take it out; no business request may reach it.""" replica = sorted(running_replicas(), key=lambda r: r["name"])[position] actor = api.user("ready") - with isolated_from_postgres(replica): + isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network + with isolate(replica): started = time.monotonic() while replica["hostname"] in lbctl.up_instances(): assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s" time.sleep(0.1) detected = time.monotonic() - started - assert probe_inside(replica, "/health/live") == 200 # the process is alive ... - assert probe_inside(replica, "/health/ready") == 503 # ... but not ready + for _ in range(3): # the process stays responsive (event loop never blocked) ... + started_probe = time.monotonic() + assert probe_inside(replica, "/health/live") == 200 + assert probe_inside(replica, "/health/ready") == 503 # ... but not ready + assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable" seen = instances_seen(api, actor["token"], 40) assert replica["hostname"] not in seen and len(seen) == 40 for i in range(20): # writes keep working, only through the ready replicas assert api.register(f"r{i}")["id"] - print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s") + print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s") # ------------------------------------------------------------------ stopping replicas under traffic @@ -273,6 +296,10 @@ def test_compose_policy_limits_ports_secrets_images(): volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] for name, svc in services.items()} assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []} + networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()} + assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"], + "lb": ["edge"]}, networks # the LB cannot reach the stores at all + assert config["networks"]["data"].get("internal") is True def test_running_containers_have_limits_and_no_bypass_ports():