The API service now names every timeout that the chain depends on: DRAIN_SECONDS, DB_ACQUIRE_TIMEOUT_SECONDS and DB_COMMAND_TIMEOUT_SECONDS join REQUEST_TIMEOUT_SECONDS and SHUTDOWN_TIMEOUT_SECONDS in compose, and the image gets the drain default. A unit test reads the compose defaults, the HAProxy check and timeout settings and the stop hook timeout, and checks that the chain shrinks inwards: LB server 10 s > request 8 s > pool wait plus statement 3 s, readiness 0.8 s < check 1 s, drain 2 s > worst detection 1.5 s, shutdown 5 s leaves 3 s for accepted requests, and shutdown < hook 6 s < stop_grace_period 8 s < 10 s. It also pins the declared detection maxima: 1.5 s for a 503 or a refused connection and 3.5 s for a hung replica. The k6 load generator joins the stand under the bench profile: pinned by digest, edge network only, 2 CPU and 512 MiB, no secrets. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
61 lines
2.5 KiB
Python
61 lines
2.5 KiB
Python
import re
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import yaml
|
||
|
||
from messenger.shared.infrastructure.postgres import PING_TIMEOUT_SECONDS
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
DEFAULT = re.compile(r"\$\{\w+:?-([^}]*)\}")
|
||
UNITS = {"ms": 0.001, "s": 1.0, "": 1.0}
|
||
|
||
|
||
def seconds(value: object) -> float:
|
||
found = re.fullmatch(r"([\d.]+)(ms|s|)", str(value))
|
||
assert found is not None, value
|
||
return float(found[1]) * UNITS[found[2]]
|
||
|
||
|
||
def services() -> dict[str, Any]:
|
||
return yaml.safe_load(DEFAULT.sub(r"\1", (ROOT / "compose.yaml").read_text()))["services"]
|
||
|
||
|
||
def balancer() -> dict[str, float]:
|
||
text = (ROOT / "infra/haproxy/haproxy.cfg").read_text()
|
||
found = {
|
||
f"timeout {name}": seconds(value)
|
||
for name, value in re.findall(r"timeout ([\w-]+) (\S+)\n", text)
|
||
}
|
||
found["hard-stop-after"] = seconds(re.findall(r"hard-stop-after (\S+)", text)[0])
|
||
check = re.findall(r"default-server .*", text)[0]
|
||
for key in ("inter", "fastinter", "fall", "rise"):
|
||
found[key] = seconds(re.findall(rf"\b{key} (\S+)", check)[0])
|
||
return found
|
||
|
||
|
||
def test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget() -> None:
|
||
lb, stand = balancer(), services()
|
||
api = {k: seconds(v) for k, v in stand["api"]["environment"].items() if k.endswith("SECONDS")}
|
||
hook = seconds(
|
||
re.findall(r"STOP_TIMEOUT:-(\d+)", (ROOT / "scripts/stop-one.sh").read_text())[0]
|
||
)
|
||
grace = seconds(stand["api"]["stop_grace_period"])
|
||
detection = lb["inter"] + (lb["fall"] - 1) * lb["fastinter"]
|
||
hung = detection + lb["fall"] * lb["timeout check"]
|
||
database = api["DB_ACQUIRE_TIMEOUT_SECONDS"] + api["DB_COMMAND_TIMEOUT_SECONDS"]
|
||
drain, shutdown = api["DRAIN_SECONDS"], api["SHUTDOWN_TIMEOUT_SECONDS"]
|
||
print(
|
||
f"обнаружение {detection} с (зависание {hung} с), окно {drain} с, остановка {shutdown} с,"
|
||
f" хук {hook} с, stop_grace_period {grace} с; LB server {lb['timeout server']} с >"
|
||
f" запрос {api['REQUEST_TIMEOUT_SECONDS']} с > БД {database} с;"
|
||
f" readiness {2 * PING_TIMEOUT_SECONDS} с < check {lb['timeout check']} с"
|
||
)
|
||
assert lb["timeout server"] > api["REQUEST_TIMEOUT_SECONDS"] > database
|
||
assert lb["timeout check"] > 2 * PING_TIMEOUT_SECONDS
|
||
assert drain > detection
|
||
assert shutdown - drain >= database
|
||
assert shutdown < hook < grace < 10
|
||
assert lb["hard-stop-after"] < seconds(stand["lb"]["stop_grace_period"])
|
||
assert (detection, hung) == (1.5, 3.5)
|