fix: реплика не «замерзает» при потере сети к PostgreSQL; сети edge/data

Найдено тестом сценария рецензента (docker network disconnect одной API
от сети хранилищ): по дедлайну задача psycopg отменялась, psycopg слал
cancel-запрос, а libpq PQcancelStart синхронно резолвит имя хоста в потоке
event loop — вся реплика, включая /health/live, висела ~1 минуту и не
возвращалась в ротацию сама. Теперь задача по дедлайну не отменяется
(запрос сразу получает 503, задача доживает сама), а мёртвые соединения
рвёт TCP keepalive + tcp_user_timeout = DB_TIMEOUT_SECONDS.

- compose: сеть edge (lb, api) и internal-сеть data (api, migrate,
  postgres, valkey) — LB не видит хранилища.
- тесты: неготовая реплика (pg_hba и network disconnect) выводится из
  ротации, процесс отвечает на live, после восстановления сам UP за <= 15 с;
  пробельные Unicode-строки, 1e1000, байт FF в пароле, 101 участник,
  курсор одного списка в другом.
- bench/track1: CPU реплик суммируется по каждому замеру, в raw и CSV —
  фактические лимиты контейнеров из docker inspect.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-19 14:01:11 +03:00
co-authored by Claude Opus 5.5
parent 2033440cdb
commit 3f45eb1b51
7 changed files with 139 additions and 17 deletions
+1
View File
@@ -21,6 +21,7 @@ services:
image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34
profiles: ["bench"]
cpuset: "${K6_CPUSET:-0-1}"
networks: [edge] # reaches the LB only, like an external client
environment:
BASE_URL: http://lb:8080
volumes:
+26 -1
View File
@@ -123,6 +123,19 @@ def start_stand(cfg: dict) -> None:
raise RuntimeError("not all replicas UP in HAProxy")
def actual_limits() -> dict:
"""What Docker really applied (not what the config intended): CPU quota, memory, cpuset."""
ids = compose("ps", "-q").split()
result = {}
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
host = item["HostConfig"]
result[item["Name"].lstrip("/")[len(PROJECT) + 1:]] = {
"cpus": host["NanoCpus"] / 1e9, "memory_mib": host["Memory"] // (1024 * 1024),
"cpuset": host["CpusetCpus"], "image": item["Config"]["Image"]}
return result
def make_chats(users: int) -> list[dict]:
created = []
stamp = uuid.uuid4().hex[:8]
@@ -171,6 +184,11 @@ class Sampler(threading.Thread):
def window(self, start: float, end: float) -> dict:
rows = [r for t, r in self.samples if start <= t <= end]
result = {}
# All API replicas summed PER SAMPLE first, then the median over time (not a sum of medians).
totals = [sum(v["cpu"] for k, v in r.items() if k.startswith("api-")) for r in rows]
if totals:
result["api_total"] = {"cpu_median": round(statistics.median(totals), 1), "cpu_max": round(max(totals), 1),
"samples": len(totals)}
for name in sorted({n for r in rows for n in r}):
values = [r[name]["cpu"] for r in rows if name in r]
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
@@ -281,8 +299,13 @@ def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> di
def flat(r: dict) -> dict:
cpu = r["cpu"]
api_cpu = sum(v["cpu_median"] for k, v in cpu.items() if k.startswith("api-"))
api_cpu = cpu.get("api_total", {}).get("cpu_median", 0.0)
limits = r.get("limits", {})
api_limits = [v for k, v in limits.items() if k.startswith("api-")]
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
"replicas_running": len(api_limits), "api_cpus_limit_total": round(sum(v["cpus"] for v in api_limits), 2),
"api_mem_mib_each": api_limits[0]["memory_mib"] if api_limits else None,
"pg_cpus_limit": limits.get("postgres-1", {}).get("cpus"),
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
@@ -333,6 +356,7 @@ def main() -> int:
name = config_name(cfg)
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
start_stand(cfg)
limits = actual_limits()
chats_dir = tempfile.mkdtemp(prefix="track1-")
try:
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
@@ -341,6 +365,7 @@ def main() -> int:
for rate in rates:
result = run_step(cfg, rate, args, chats_dir, sampler)
result["repeat"] = repeat
result["limits"] = limits
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
row = flat(result)
+12
View File
@@ -6,6 +6,8 @@
# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb.
# Images are pinned by version AND digest. Passwords are passed only as secret files, and
# every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only).
# Two networks: `edge` (lb <-> api) and `data` (api/migrate <-> postgres/valkey, internal):
# the LB cannot reach the stores, and the stores are not reachable from the edge.
name: mtusi-lab3
x-api-env: &api-env
@@ -46,6 +48,7 @@ services:
start_period: 10s
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
restart: unless-stopped
networks: [data]
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
@@ -73,6 +76,7 @@ services:
retries: 30
start_period: 5s
restart: unless-stopped
networks: [data]
deploy:
resources:
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
@@ -95,6 +99,7 @@ services:
healthcheck:
disable: true
restart: "no"
networks: [data]
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
@@ -125,6 +130,7 @@ services:
condition: service_completed_successfully
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
restart: unless-stopped
networks: [edge, data]
deploy:
replicas: ${API_REPLICAS:-2}
resources:
@@ -147,10 +153,16 @@ services:
api:
condition: service_healthy
restart: unless-stopped
networks: [edge]
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
networks:
edge:
data:
internal: true # no route outside: stores talk only to api/migrate
volumes:
pgdata:
valkeydata:
+2
View File
@@ -52,6 +52,8 @@ def replicas(all_states: bool = True) -> list[dict]:
"started_at": item["State"]["StartedAt"],
"pid": item["State"]["Pid"],
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
})
return sorted(result, key=lambda r: r["name"])
+23 -8
View File
@@ -8,12 +8,21 @@ Three independent limits keep a request from hanging when the database misbehave
* client deadline — covers what the server cannot report, e.g. a
paused/frozen PostgreSQL container or a dead network.
The client deadline needs care: when a psycopg query is cancelled, psycopg sends a
cancel request to the server and waits for it (up to 5 s + 5 s) before giving up.
With a frozen server that would stretch a 3 s deadline to 13 s. Therefore every unit
of work runs as its own task: at the deadline the request stops waiting and gets 503
at once, while the task finishes its cleanup (cancel, close, return to the pool) in
the background. Any failure becomes 503 `unavailable` — never 401, never a hang.
The client deadline needs care. Every unit of work runs as its own task; at the deadline
the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
psycopg query makes psycopg send a cancel request, and libpq starts that connection
(PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
The abandoned task instead ends by itself, bounded by:
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
breaks the socket within seconds, the connection is discarded by the pool;
* `statement_timeout` / `lock_timeout` on a live but slow server;
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
at most the pool timeout and gets 503.
Any failure becomes 503 `unavailable` — never 401, never a hang.
"""
from __future__ import annotations
@@ -65,6 +74,12 @@ class Database:
"options": options,
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
"application_name": "messenger-api",
# Detect a dead peer / lost network without any cancel request (see module doc).
"keepalives": 1,
"keepalives_idle": 5,
"keepalives_interval": 1,
"keepalives_count": 3,
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
},
)
@@ -95,8 +110,8 @@ class Database:
raise Unavailable("База данных временно недоступна") from None
finally:
if not task.done():
# Deadline or cancelled request: stop waiting now, let psycopg clean up later.
task.cancel()
# Deadline or cancelled request: stop waiting now. The task is left to finish
# on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
task.add_done_callback(_forget)
async def ping(self, timeout: float = 1.0) -> bool:
+40
View File
@@ -302,3 +302,43 @@ def test_every_api_response_carries_instance_and_request_id(api):
api.call("GET", "/api/v1/users/me", 401)
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
@pytest.mark.parametrize("blank", [" ", "  ", "  \t "])
def test_unicode_whitespace_only_strings_are_400(api, blank):
alice = api.user("blank")
chat = api.chat(alice)
api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank,
"password": PASSWORD})
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]})
api.post_message(alice, chat, {"text": blank}, 400)
assert api.history(alice, chat, 100)[0] == []
@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}',
'{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}',
b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}'])
def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
api.call("GET", "/health/ready", 200)
def test_more_than_100_members_is_400(api):
owner = api.user("big")
others = [api.register(f"m{i}")["id"] for i in range(100)]
api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]})
chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json()
assert len(chat["member_ids"]) == 100
def test_cursor_of_one_list_is_rejected_by_the_other(api):
alice = api.user("cross")
chats = [api.chat(alice, title=f"c{i}") for i in range(2)]
for _ in range(2):
api.send(alice, chats[0], "m")
message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200,
token=alice["token"]).json()["next_cursor"]
chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"])
api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"])
+35 -8
View File
@@ -62,7 +62,7 @@ def hba_file() -> str:
def isolated_from_postgres(replica: dict):
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
connections), while it keeps running and answering /health/live. Always undone."""
ip = replica["ips"][0]
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
path = hba_file()
marker = "lab3-test-isolate"
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
@@ -73,28 +73,51 @@ def isolated_from_postgres(replica: dict):
finally:
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
psql("SELECT pg_reload_conf()")
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
wait_stand()
@contextlib.contextmanager
def disconnected_from_data_network(replica: dict):
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
LB but loses PostgreSQL AND Valkey. Always reconnected."""
network = next(n for n in json.loads(subprocess.run(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
try:
yield
finally:
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
wait_stand()
@pytest.mark.parametrize("how", ["pg_hba", "network"])
@pytest.mark.parametrize("position", range(REPLICAS))
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position):
"""The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is
200, /health/ready 503 -> HAProxy must take it out; no business request may reach it."""
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
503 -> HAProxy must take it out; no business request may reach it."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
actor = api.user("ready")
with isolated_from_postgres(replica):
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
with isolate(replica):
started = time.monotonic()
while replica["hostname"] in lbctl.up_instances():
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
time.sleep(0.1)
detected = time.monotonic() - started
assert probe_inside(replica, "/health/live") == 200 # the process is alive ...
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
for _ in range(3): # the process stays responsive (event loop never blocked) ...
started_probe = time.monotonic()
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
seen = instances_seen(api, actor["token"], 40)
assert replica["hostname"] not in seen and len(seen) == 40
for i in range(20): # writes keep working, only through the ready replicas
assert api.register(f"r{i}")["id"]
print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s")
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
# ------------------------------------------------------------------ stopping replicas under traffic
@@ -273,6 +296,10 @@ def test_compose_policy_limits_ports_secrets_images():
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"],
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
assert config["networks"]["data"].get("internal") is True
def test_running_containers_have_limits_and_no_bypass_ports():