fix: реплика не «замерзает» при потере сети к PostgreSQL; сети edge/data

Найдено тестом сценария рецензента (docker network disconnect одной API
от сети хранилищ): по дедлайну задача psycopg отменялась, psycopg слал
cancel-запрос, а libpq PQcancelStart синхронно резолвит имя хоста в потоке
event loop — вся реплика, включая /health/live, висела ~1 минуту и не
возвращалась в ротацию сама. Теперь задача по дедлайну не отменяется
(запрос сразу получает 503, задача доживает сама), а мёртвые соединения
рвёт TCP keepalive + tcp_user_timeout = DB_TIMEOUT_SECONDS.

- compose: сеть edge (lb, api) и internal-сеть data (api, migrate,
  postgres, valkey) — LB не видит хранилища.
- тесты: неготовая реплика (pg_hba и network disconnect) выводится из
  ротации, процесс отвечает на live, после восстановления сам UP за <= 15 с;
  пробельные Unicode-строки, 1e1000, байт FF в пароле, 101 участник,
  курсор одного списка в другом.
- bench/track1: CPU реплик суммируется по каждому замеру, в raw и CSV —
  фактические лимиты контейнеров из docker inspect.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-19 14:01:11 +03:00
co-authored by Claude Opus 5.5
parent 2033440cdb
commit 3f45eb1b51
7 changed files with 139 additions and 17 deletions
+1
View File
@@ -21,6 +21,7 @@ services:
image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34 image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34
profiles: ["bench"] profiles: ["bench"]
cpuset: "${K6_CPUSET:-0-1}" cpuset: "${K6_CPUSET:-0-1}"
networks: [edge] # reaches the LB only, like an external client
environment: environment:
BASE_URL: http://lb:8080 BASE_URL: http://lb:8080
volumes: volumes:
+26 -1
View File
@@ -123,6 +123,19 @@ def start_stand(cfg: dict) -> None:
raise RuntimeError("not all replicas UP in HAProxy") raise RuntimeError("not all replicas UP in HAProxy")
def actual_limits() -> dict:
"""What Docker really applied (not what the config intended): CPU quota, memory, cpuset."""
ids = compose("ps", "-q").split()
result = {}
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
host = item["HostConfig"]
result[item["Name"].lstrip("/")[len(PROJECT) + 1:]] = {
"cpus": host["NanoCpus"] / 1e9, "memory_mib": host["Memory"] // (1024 * 1024),
"cpuset": host["CpusetCpus"], "image": item["Config"]["Image"]}
return result
def make_chats(users: int) -> list[dict]: def make_chats(users: int) -> list[dict]:
created = [] created = []
stamp = uuid.uuid4().hex[:8] stamp = uuid.uuid4().hex[:8]
@@ -171,6 +184,11 @@ class Sampler(threading.Thread):
def window(self, start: float, end: float) -> dict: def window(self, start: float, end: float) -> dict:
rows = [r for t, r in self.samples if start <= t <= end] rows = [r for t, r in self.samples if start <= t <= end]
result = {} result = {}
# All API replicas summed PER SAMPLE first, then the median over time (not a sum of medians).
totals = [sum(v["cpu"] for k, v in r.items() if k.startswith("api-")) for r in rows]
if totals:
result["api_total"] = {"cpu_median": round(statistics.median(totals), 1), "cpu_max": round(max(totals), 1),
"samples": len(totals)}
for name in sorted({n for r in rows for n in r}): for name in sorted({n for r in rows for n in r}):
values = [r[name]["cpu"] for r in rows if name in r] values = [r[name]["cpu"] for r in rows if name in r]
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1), result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
@@ -281,8 +299,13 @@ def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> di
def flat(r: dict) -> dict: def flat(r: dict) -> dict:
cpu = r["cpu"] cpu = r["cpu"]
api_cpu = sum(v["cpu_median"] for k, v in cpu.items() if k.startswith("api-")) api_cpu = cpu.get("api_total", {}).get("cpu_median", 0.0)
limits = r.get("limits", {})
api_limits = [v for k, v in limits.items() if k.startswith("api-")]
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"], return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
"replicas_running": len(api_limits), "api_cpus_limit_total": round(sum(v["cpus"] for v in api_limits), 2),
"api_mem_mib_each": api_limits[0]["memory_mib"] if api_limits else None,
"pg_cpus_limit": limits.get("postgres-1", {}).get("cpus"),
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"], "requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"], "p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"), "api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
@@ -333,6 +356,7 @@ def main() -> int:
name = config_name(cfg) name = config_name(cfg)
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True) print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
start_stand(cfg) start_stand(cfg)
limits = actual_limits()
chats_dir = tempfile.mkdtemp(prefix="track1-") chats_dir = tempfile.mkdtemp(prefix="track1-")
try: try:
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8") Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
@@ -341,6 +365,7 @@ def main() -> int:
for rate in rates: for rate in rates:
result = run_step(cfg, rate, args, chats_dir, sampler) result = run_step(cfg, rate, args, chats_dir, sampler)
result["repeat"] = repeat result["repeat"] = repeat
result["limits"] = limits
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text( (out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
row = flat(result) row = flat(result)
+12
View File
@@ -6,6 +6,8 @@
# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb. # Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb.
# Images are pinned by version AND digest. Passwords are passed only as secret files, and # Images are pinned by version AND digest. Passwords are passed only as secret files, and
# every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only). # every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only).
# Two networks: `edge` (lb <-> api) and `data` (api/migrate <-> postgres/valkey, internal):
# the LB cannot reach the stores, and the stores are not reachable from the edge.
name: mtusi-lab3 name: mtusi-lab3
x-api-env: &api-env x-api-env: &api-env
@@ -46,6 +48,7 @@ services:
start_period: 10s start_period: 10s
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
restart: unless-stopped restart: unless-stopped
networks: [data]
deploy: deploy:
resources: resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"} limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
@@ -73,6 +76,7 @@ services:
retries: 30 retries: 30
start_period: 5s start_period: 5s
restart: unless-stopped restart: unless-stopped
networks: [data]
deploy: deploy:
resources: resources:
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
@@ -95,6 +99,7 @@ services:
healthcheck: healthcheck:
disable: true disable: true
restart: "no" restart: "no"
networks: [data]
deploy: deploy:
resources: resources:
limits: {cpus: "0.5", memory: 256m} limits: {cpus: "0.5", memory: 256m}
@@ -125,6 +130,7 @@ services:
condition: service_completed_successfully condition: service_completed_successfully
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s) stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
restart: unless-stopped restart: unless-stopped
networks: [edge, data]
deploy: deploy:
replicas: ${API_REPLICAS:-2} replicas: ${API_REPLICAS:-2}
resources: resources:
@@ -147,10 +153,16 @@ services:
api: api:
condition: service_healthy condition: service_healthy
restart: unless-stopped restart: unless-stopped
networks: [edge]
deploy: deploy:
resources: resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
networks:
edge:
data:
internal: true # no route outside: stores talk only to api/migrate
volumes: volumes:
pgdata: pgdata:
valkeydata: valkeydata:
+2
View File
@@ -52,6 +52,8 @@ def replicas(all_states: bool = True) -> list[dict]:
"started_at": item["State"]["StartedAt"], "started_at": item["State"]["StartedAt"],
"pid": item["State"]["Pid"], "pid": item["State"]["Pid"],
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")), "ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
}) })
return sorted(result, key=lambda r: r["name"]) return sorted(result, key=lambda r: r["name"])
+23 -8
View File
@@ -8,12 +8,21 @@ Three independent limits keep a request from hanging when the database misbehave
* client deadline — covers what the server cannot report, e.g. a * client deadline — covers what the server cannot report, e.g. a
paused/frozen PostgreSQL container or a dead network. paused/frozen PostgreSQL container or a dead network.
The client deadline needs care: when a psycopg query is cancelled, psycopg sends a The client deadline needs care. Every unit of work runs as its own task; at the deadline
cancel request to the server and waits for it (up to 5 s + 5 s) before giving up. the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
With a frozen server that would stretch a 3 s deadline to 13 s. Therefore every unit psycopg query makes psycopg send a cancel request, and libpq starts that connection
of work runs as its own task: at the deadline the request stops waiting and gets 503 (PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
at once, while the task finishes its cleanup (cancel, close, return to the pool) in When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
the background. Any failure becomes 503 `unavailable` — never 401, never a hang. disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
The abandoned task instead ends by itself, bounded by:
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
breaks the socket within seconds, the connection is discarded by the pool;
* `statement_timeout` / `lock_timeout` on a live but slow server;
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
at most the pool timeout and gets 503.
Any failure becomes 503 `unavailable` — never 401, never a hang.
""" """
from __future__ import annotations from __future__ import annotations
@@ -65,6 +74,12 @@ class Database:
"options": options, "options": options,
"connect_timeout": max(1, int(settings.db_timeout_seconds)), "connect_timeout": max(1, int(settings.db_timeout_seconds)),
"application_name": "messenger-api", "application_name": "messenger-api",
# Detect a dead peer / lost network without any cancel request (see module doc).
"keepalives": 1,
"keepalives_idle": 5,
"keepalives_interval": 1,
"keepalives_count": 3,
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
}, },
) )
@@ -95,8 +110,8 @@ class Database:
raise Unavailable("База данных временно недоступна") from None raise Unavailable("База данных временно недоступна") from None
finally: finally:
if not task.done(): if not task.done():
# Deadline or cancelled request: stop waiting now, let psycopg clean up later. # Deadline or cancelled request: stop waiting now. The task is left to finish
task.cancel() # on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
task.add_done_callback(_forget) task.add_done_callback(_forget)
async def ping(self, timeout: float = 1.0) -> bool: async def ping(self, timeout: float = 1.0) -> bool:
+40
View File
@@ -302,3 +302,43 @@ def test_every_api_response_carries_instance_and_request_id(api):
api.call("GET", "/api/v1/users/me", 401) api.call("GET", "/api/v1/users/me", 401)
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"}) response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
@pytest.mark.parametrize("blank", [" ", "  ", "  \t "])
def test_unicode_whitespace_only_strings_are_400(api, blank):
alice = api.user("blank")
chat = api.chat(alice)
api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank,
"password": PASSWORD})
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]})
api.post_message(alice, chat, {"text": blank}, 400)
assert api.history(alice, chat, 100)[0] == []
@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}',
'{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}',
b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}'])
def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
api.call("GET", "/health/ready", 200)
def test_more_than_100_members_is_400(api):
owner = api.user("big")
others = [api.register(f"m{i}")["id"] for i in range(100)]
api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]})
chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json()
assert len(chat["member_ids"]) == 100
def test_cursor_of_one_list_is_rejected_by_the_other(api):
alice = api.user("cross")
chats = [api.chat(alice, title=f"c{i}") for i in range(2)]
for _ in range(2):
api.send(alice, chats[0], "m")
message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200,
token=alice["token"]).json()["next_cursor"]
chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"])
api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"])
+35 -8
View File
@@ -62,7 +62,7 @@ def hba_file() -> str:
def isolated_from_postgres(replica: dict): def isolated_from_postgres(replica: dict):
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its """Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
connections), while it keeps running and answering /health/live. Always undone.""" connections), while it keeps running and answering /health/live. Always undone."""
ip = replica["ips"][0] ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
path = hba_file() path = hba_file()
marker = "lab3-test-isolate" marker = "lab3-test-isolate"
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}") compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
@@ -73,28 +73,51 @@ def isolated_from_postgres(replica: dict):
finally: finally:
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}") compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
psql("SELECT pg_reload_conf()") psql("SELECT pg_reload_conf()")
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
wait_stand() wait_stand()
@contextlib.contextmanager
def disconnected_from_data_network(replica: dict):
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
LB but loses PostgreSQL AND Valkey. Always reconnected."""
network = next(n for n in json.loads(subprocess.run(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
try:
yield
finally:
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
wait_stand()
@pytest.mark.parametrize("how", ["pg_hba", "network"])
@pytest.mark.parametrize("position", range(REPLICAS)) @pytest.mark.parametrize("position", range(REPLICAS))
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position): def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
"""The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is """The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
200, /health/ready 503 -> HAProxy must take it out; no business request may reach it.""" network (docker network disconnect) but keeps running. /health/live 200, /health/ready
503 -> HAProxy must take it out; no business request may reach it."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position] replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
actor = api.user("ready") actor = api.user("ready")
with isolated_from_postgres(replica): isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
with isolate(replica):
started = time.monotonic() started = time.monotonic()
while replica["hostname"] in lbctl.up_instances(): while replica["hostname"] in lbctl.up_instances():
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s" assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
time.sleep(0.1) time.sleep(0.1)
detected = time.monotonic() - started detected = time.monotonic() - started
assert probe_inside(replica, "/health/live") == 200 # the process is alive ... for _ in range(3): # the process stays responsive (event loop never blocked) ...
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready started_probe = time.monotonic()
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
seen = instances_seen(api, actor["token"], 40) seen = instances_seen(api, actor["token"], 40)
assert replica["hostname"] not in seen and len(seen) == 40 assert replica["hostname"] not in seen and len(seen) == 40
for i in range(20): # writes keep working, only through the ready replicas for i in range(20): # writes keep working, only through the ready replicas
assert api.register(f"r{i}")["id"] assert api.register(f"r{i}")["id"]
print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s") print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
# ------------------------------------------------------------------ stopping replicas under traffic # ------------------------------------------------------------------ stopping replicas under traffic
@@ -273,6 +296,10 @@ def test_compose_policy_limits_ports_secrets_images():
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()} for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []} assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"],
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
assert config["networks"]["data"].get("internal") is True
def test_running_containers_have_limits_and_no_bypass_ports(): def test_running_containers_have_limits_and_no_bypass_ports():