fix: реплика не «замерзает» при потере сети к PostgreSQL; сети edge/data
Найдено тестом сценария рецензента (docker network disconnect одной API от сети хранилищ): по дедлайну задача psycopg отменялась, psycopg слал cancel-запрос, а libpq PQcancelStart синхронно резолвит имя хоста в потоке event loop — вся реплика, включая /health/live, висела ~1 минуту и не возвращалась в ротацию сама. Теперь задача по дедлайну не отменяется (запрос сразу получает 503, задача доживает сама), а мёртвые соединения рвёт TCP keepalive + tcp_user_timeout = DB_TIMEOUT_SECONDS. - compose: сеть edge (lb, api) и internal-сеть data (api, migrate, postgres, valkey) — LB не видит хранилища. - тесты: неготовая реплика (pg_hba и network disconnect) выводится из ротации, процесс отвечает на live, после восстановления сам UP за <= 15 с; пробельные Unicode-строки, 1e1000, байт FF в пароле, 101 участник, курсор одного списка в другом. - bench/track1: CPU реплик суммируется по каждому замеру, в raw и CSV — фактические лимиты контейнеров из docker inspect. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2033440cdb
commit
3f45eb1b51
@@ -21,6 +21,7 @@ services:
|
||||
image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34
|
||||
profiles: ["bench"]
|
||||
cpuset: "${K6_CPUSET:-0-1}"
|
||||
networks: [edge] # reaches the LB only, like an external client
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
volumes:
|
||||
|
||||
+26
-1
@@ -123,6 +123,19 @@ def start_stand(cfg: dict) -> None:
|
||||
raise RuntimeError("not all replicas UP in HAProxy")
|
||||
|
||||
|
||||
def actual_limits() -> dict:
|
||||
"""What Docker really applied (not what the config intended): CPU quota, memory, cpuset."""
|
||||
ids = compose("ps", "-q").split()
|
||||
result = {}
|
||||
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
|
||||
check=True).stdout):
|
||||
host = item["HostConfig"]
|
||||
result[item["Name"].lstrip("/")[len(PROJECT) + 1:]] = {
|
||||
"cpus": host["NanoCpus"] / 1e9, "memory_mib": host["Memory"] // (1024 * 1024),
|
||||
"cpuset": host["CpusetCpus"], "image": item["Config"]["Image"]}
|
||||
return result
|
||||
|
||||
|
||||
def make_chats(users: int) -> list[dict]:
|
||||
created = []
|
||||
stamp = uuid.uuid4().hex[:8]
|
||||
@@ -171,6 +184,11 @@ class Sampler(threading.Thread):
|
||||
def window(self, start: float, end: float) -> dict:
|
||||
rows = [r for t, r in self.samples if start <= t <= end]
|
||||
result = {}
|
||||
# All API replicas summed PER SAMPLE first, then the median over time (not a sum of medians).
|
||||
totals = [sum(v["cpu"] for k, v in r.items() if k.startswith("api-")) for r in rows]
|
||||
if totals:
|
||||
result["api_total"] = {"cpu_median": round(statistics.median(totals), 1), "cpu_max": round(max(totals), 1),
|
||||
"samples": len(totals)}
|
||||
for name in sorted({n for r in rows for n in r}):
|
||||
values = [r[name]["cpu"] for r in rows if name in r]
|
||||
result[name] = {"cpu_median": round(statistics.median(values), 1), "cpu_max": round(max(values), 1),
|
||||
@@ -281,8 +299,13 @@ def run_step(cfg: dict, rate: int, args, chats_dir: str, sampler: Sampler) -> di
|
||||
|
||||
def flat(r: dict) -> dict:
|
||||
cpu = r["cpu"]
|
||||
api_cpu = sum(v["cpu_median"] for k, v in cpu.items() if k.startswith("api-"))
|
||||
api_cpu = cpu.get("api_total", {}).get("cpu_median", 0.0)
|
||||
limits = r.get("limits", {})
|
||||
api_limits = [v for k, v in limits.items() if k.startswith("api-")]
|
||||
return {"config": r["config"], "repeat": r["repeat"], "rate": r["rate"], "ok_rps": r["ok_rps"],
|
||||
"replicas_running": len(api_limits), "api_cpus_limit_total": round(sum(v["cpus"] for v in api_limits), 2),
|
||||
"api_mem_mib_each": api_limits[0]["memory_mib"] if api_limits else None,
|
||||
"pg_cpus_limit": limits.get("postgres-1", {}).get("cpus"),
|
||||
"requests": r["requests"], "ok": r["ok"], "dropped": r["dropped"], "error_ratio": r["error_ratio"],
|
||||
"p50_ms": r["ok_p50_ms"], "p95_ms": r["ok_p95_ms"], "p99_ms": r["ok_p99_ms"],
|
||||
"api_cpu_pct_total": round(api_cpu, 1), "pg_cpu_pct": cpu.get("postgres-1", {}).get("cpu_median"),
|
||||
@@ -333,6 +356,7 @@ def main() -> int:
|
||||
name = config_name(cfg)
|
||||
print(f"== {args.series}/{args.label} {name} repeat {repeat}: fresh stand", flush=True)
|
||||
start_stand(cfg)
|
||||
limits = actual_limits()
|
||||
chats_dir = tempfile.mkdtemp(prefix="track1-")
|
||||
try:
|
||||
Path(chats_dir, "chats.json").write_text(json.dumps(make_chats(args.users)), encoding="utf-8")
|
||||
@@ -341,6 +365,7 @@ def main() -> int:
|
||||
for rate in rates:
|
||||
result = run_step(cfg, rate, args, chats_dir, sampler)
|
||||
result["repeat"] = repeat
|
||||
result["limits"] = limits
|
||||
(out / "raw" / f"{name}-r{repeat}-{rate}.json").write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
row = flat(result)
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb.
|
||||
# Images are pinned by version AND digest. Passwords are passed only as secret files, and
|
||||
# every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only).
|
||||
# Two networks: `edge` (lb <-> api) and `data` (api/migrate <-> postgres/valkey, internal):
|
||||
# the LB cannot reach the stores, and the stores are not reachable from the edge.
|
||||
name: mtusi-lab3
|
||||
|
||||
x-api-env: &api-env
|
||||
@@ -46,6 +48,7 @@ services:
|
||||
start_period: 10s
|
||||
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
|
||||
@@ -73,6 +76,7 @@ services:
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
|
||||
@@ -95,6 +99,7 @@ services:
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256m}
|
||||
@@ -125,6 +130,7 @@ services:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
|
||||
restart: unless-stopped
|
||||
networks: [edge, data]
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
@@ -147,10 +153,16 @@ services:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
networks: [edge]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
|
||||
|
||||
networks:
|
||||
edge:
|
||||
data:
|
||||
internal: true # no route outside: stores talk only to api/migrate
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
|
||||
@@ -52,6 +52,8 @@ def replicas(all_states: bool = True) -> list[dict]:
|
||||
"started_at": item["State"]["StartedAt"],
|
||||
"pid": item["State"]["Pid"],
|
||||
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
|
||||
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
|
||||
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
|
||||
})
|
||||
return sorted(result, key=lambda r: r["name"])
|
||||
|
||||
|
||||
+23
-8
@@ -8,12 +8,21 @@ Three independent limits keep a request from hanging when the database misbehave
|
||||
* client deadline — covers what the server cannot report, e.g. a
|
||||
paused/frozen PostgreSQL container or a dead network.
|
||||
|
||||
The client deadline needs care: when a psycopg query is cancelled, psycopg sends a
|
||||
cancel request to the server and waits for it (up to 5 s + 5 s) before giving up.
|
||||
With a frozen server that would stretch a 3 s deadline to 13 s. Therefore every unit
|
||||
of work runs as its own task: at the deadline the request stops waiting and gets 503
|
||||
at once, while the task finishes its cleanup (cancel, close, return to the pool) in
|
||||
the background. Any failure becomes 503 `unavailable` — never 401, never a hang.
|
||||
The client deadline needs care. Every unit of work runs as its own task; at the deadline
|
||||
the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
|
||||
psycopg query makes psycopg send a cancel request, and libpq starts that connection
|
||||
(PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
|
||||
When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
|
||||
disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
|
||||
The abandoned task instead ends by itself, bounded by:
|
||||
|
||||
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
|
||||
breaks the socket within seconds, the connection is discarded by the pool;
|
||||
* `statement_timeout` / `lock_timeout` on a live but slow server;
|
||||
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
|
||||
at most the pool timeout and gets 503.
|
||||
|
||||
Any failure becomes 503 `unavailable` — never 401, never a hang.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -65,6 +74,12 @@ class Database:
|
||||
"options": options,
|
||||
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
|
||||
"application_name": "messenger-api",
|
||||
# Detect a dead peer / lost network without any cancel request (see module doc).
|
||||
"keepalives": 1,
|
||||
"keepalives_idle": 5,
|
||||
"keepalives_interval": 1,
|
||||
"keepalives_count": 3,
|
||||
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -95,8 +110,8 @@ class Database:
|
||||
raise Unavailable("База данных временно недоступна") from None
|
||||
finally:
|
||||
if not task.done():
|
||||
# Deadline or cancelled request: stop waiting now, let psycopg clean up later.
|
||||
task.cancel()
|
||||
# Deadline or cancelled request: stop waiting now. The task is left to finish
|
||||
# on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
|
||||
task.add_done_callback(_forget)
|
||||
|
||||
async def ping(self, timeout: float = 1.0) -> bool:
|
||||
|
||||
@@ -302,3 +302,43 @@ def test_every_api_response_carries_instance_and_request_id(api):
|
||||
api.call("GET", "/api/v1/users/me", 401)
|
||||
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
|
||||
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blank", [" ", " ", " \t "])
|
||||
def test_unicode_whitespace_only_strings_are_400(api, blank):
|
||||
alice = api.user("blank")
|
||||
chat = api.chat(alice)
|
||||
api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank,
|
||||
"password": PASSWORD})
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]})
|
||||
api.post_message(alice, chat, {"text": blank}, 400)
|
||||
assert api.history(alice, chat, 100)[0] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}',
|
||||
'{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}',
|
||||
b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}'])
|
||||
def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw):
|
||||
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
|
||||
api.call("GET", "/health/ready", 200)
|
||||
|
||||
|
||||
def test_more_than_100_members_is_400(api):
|
||||
owner = api.user("big")
|
||||
others = [api.register(f"m{i}")["id"] for i in range(100)]
|
||||
api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]})
|
||||
chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||||
json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json()
|
||||
assert len(chat["member_ids"]) == 100
|
||||
|
||||
|
||||
def test_cursor_of_one_list_is_rejected_by_the_other(api):
|
||||
alice = api.user("cross")
|
||||
chats = [api.chat(alice, title=f"c{i}") for i in range(2)]
|
||||
for _ in range(2):
|
||||
api.send(alice, chats[0], "m")
|
||||
message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200,
|
||||
token=alice["token"]).json()["next_cursor"]
|
||||
chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
|
||||
api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"])
|
||||
api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"])
|
||||
|
||||
@@ -62,7 +62,7 @@ def hba_file() -> str:
|
||||
def isolated_from_postgres(replica: dict):
|
||||
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
|
||||
connections), while it keeps running and answering /health/live. Always undone."""
|
||||
ip = replica["ips"][0]
|
||||
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
|
||||
path = hba_file()
|
||||
marker = "lab3-test-isolate"
|
||||
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
|
||||
@@ -73,28 +73,51 @@ def isolated_from_postgres(replica: dict):
|
||||
finally:
|
||||
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
|
||||
psql("SELECT pg_reload_conf()")
|
||||
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
|
||||
wait_stand()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def disconnected_from_data_network(replica: dict):
|
||||
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
|
||||
LB but loses PostgreSQL AND Valkey. Always reconnected."""
|
||||
network = next(n for n in json.loads(subprocess.run(
|
||||
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
|
||||
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
|
||||
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
|
||||
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
|
||||
wait_stand()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("how", ["pg_hba", "network"])
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position):
|
||||
"""The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is
|
||||
200, /health/ready 503 -> HAProxy must take it out; no business request may reach it."""
|
||||
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
|
||||
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
|
||||
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
|
||||
503 -> HAProxy must take it out; no business request may reach it."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
actor = api.user("ready")
|
||||
with isolated_from_postgres(replica):
|
||||
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
|
||||
with isolate(replica):
|
||||
started = time.monotonic()
|
||||
while replica["hostname"] in lbctl.up_instances():
|
||||
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
|
||||
time.sleep(0.1)
|
||||
detected = time.monotonic() - started
|
||||
assert probe_inside(replica, "/health/live") == 200 # the process is alive ...
|
||||
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
|
||||
for _ in range(3): # the process stays responsive (event loop never blocked) ...
|
||||
started_probe = time.monotonic()
|
||||
assert probe_inside(replica, "/health/live") == 200
|
||||
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
|
||||
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
|
||||
seen = instances_seen(api, actor["token"], 40)
|
||||
assert replica["hostname"] not in seen and len(seen) == 40
|
||||
for i in range(20): # writes keep working, only through the ready replicas
|
||||
assert api.register(f"r{i}")["id"]
|
||||
print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s")
|
||||
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ stopping replicas under traffic
|
||||
@@ -273,6 +296,10 @@ def test_compose_policy_limits_ports_secrets_images():
|
||||
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
|
||||
for name, svc in services.items()}
|
||||
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
|
||||
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
|
||||
assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"],
|
||||
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
|
||||
assert config["networks"]["data"].get("internal") is True
|
||||
|
||||
|
||||
def test_running_containers_have_limits_and_no_bypass_ports():
|
||||
|
||||
Reference in New Issue
Block a user