diff --git a/.env.example b/.env.example index ee70667f..91f9ec42 100644 --- a/.env.example +++ b/.env.example @@ -12,7 +12,8 @@ CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 # Solution-specific: published LB ports, replica limits, workers and pool per replica, -# LB queue per replica, PostgreSQL limits, cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone: +# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL +# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone: # only the load balancer is published. LB_HTTP_PORT=8080 LB_STATS_PORT=8404 @@ -21,7 +22,10 @@ API_MEMORY=512m WEB_CONCURRENCY=2 DB_POOL_MAX_SIZE=8 API_SERVER_MAXCONN=64 +DRAIN_SECONDS=2 PG_MAX_CONNECTIONS=200 PG_CPUS=2 PG_MEMORY=1g +K6_CPUS=2 +K6_MEMORY=512m CURSOR_SECRET_FILE=/run/secrets/cursor_secret diff --git a/Dockerfile b/Dockerfile index d4d9f201..1f4e6fed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ ENV PYTHONUNBUFFERED=1 \ LOG_LEVEL=INFO \ WEB_CONCURRENCY=2 \ REQUEST_TIMEOUT_SECONDS=8 \ + DRAIN_SECONDS=2 \ SHUTDOWN_TIMEOUT_SECONDS=5 RUN groupadd --system --gid 10001 messenger \ diff --git a/compose.yaml b/compose.yaml index 7339cbe9..b413aecb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -42,6 +42,9 @@ services: WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8} + DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1} + DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2} + DRAIN_SECONDS: ${DRAIN_SECONDS:-2} SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} LOG_LEVEL: ${LOG_LEVEL:-INFO} secrets: [database_url, cache_url, cursor_secret] @@ -149,6 +152,17 @@ services: resources: limits: {cpus: "0.5", memory: 128m} + k6: + image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 + profiles: [bench] + environment: + BASE_URL: http://lb:8080 + volumes: [./infra/k6:/scripts:ro] + networks: [edge] + deploy: + resources: + limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"} + secrets: postgres_password: file: ./secrets/postgres_password diff --git a/openspec/changes/add-lab3-failover-operations/tasks.md b/openspec/changes/add-lab3-failover-operations/tasks.md index 68138284..e4e758ce 100644 --- a/openspec/changes/add-lab3-failover-operations/tasks.md +++ b/openspec/changes/add-lab3-failover-operations/tasks.md @@ -7,9 +7,9 @@ ## 2. Стенд и таймауты -- [ ] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile -- [ ] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh` -- [ ] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"` +- [x] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile +- [x] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh` +- [x] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"` ## 3. Нагрузка и зонд diff --git a/student-tests/unit/test_timeouts.py b/student-tests/unit/test_timeouts.py new file mode 100644 index 00000000..4050372a --- /dev/null +++ b/student-tests/unit/test_timeouts.py @@ -0,0 +1,60 @@ +import re +from pathlib import Path +from typing import Any + +import yaml + +from messenger.shared.infrastructure.postgres import PING_TIMEOUT_SECONDS + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT = re.compile(r"\$\{\w+:?-([^}]*)\}") +UNITS = {"ms": 0.001, "s": 1.0, "": 1.0} + + +def seconds(value: object) -> float: + found = re.fullmatch(r"([\d.]+)(ms|s|)", str(value)) + assert found is not None, value + return float(found[1]) * UNITS[found[2]] + + +def services() -> dict[str, Any]: + return yaml.safe_load(DEFAULT.sub(r"\1", (ROOT / "compose.yaml").read_text()))["services"] + + +def balancer() -> dict[str, float]: + text = (ROOT / "infra/haproxy/haproxy.cfg").read_text() + found = { + f"timeout {name}": seconds(value) + for name, value in re.findall(r"timeout ([\w-]+) (\S+)\n", text) + } + found["hard-stop-after"] = seconds(re.findall(r"hard-stop-after (\S+)", text)[0]) + check = re.findall(r"default-server .*", text)[0] + for key in ("inter", "fastinter", "fall", "rise"): + found[key] = seconds(re.findall(rf"\b{key} (\S+)", check)[0]) + return found + + +def test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget() -> None: + lb, stand = balancer(), services() + api = {k: seconds(v) for k, v in stand["api"]["environment"].items() if k.endswith("SECONDS")} + hook = seconds( + re.findall(r"STOP_TIMEOUT:-(\d+)", (ROOT / "scripts/stop-one.sh").read_text())[0] + ) + grace = seconds(stand["api"]["stop_grace_period"]) + detection = lb["inter"] + (lb["fall"] - 1) * lb["fastinter"] + hung = detection + lb["fall"] * lb["timeout check"] + database = api["DB_ACQUIRE_TIMEOUT_SECONDS"] + api["DB_COMMAND_TIMEOUT_SECONDS"] + drain, shutdown = api["DRAIN_SECONDS"], api["SHUTDOWN_TIMEOUT_SECONDS"] + print( + f"обнаружение {detection} с (зависание {hung} с), окно {drain} с, остановка {shutdown} с," + f" хук {hook} с, stop_grace_period {grace} с; LB server {lb['timeout server']} с >" + f" запрос {api['REQUEST_TIMEOUT_SECONDS']} с > БД {database} с;" + f" readiness {2 * PING_TIMEOUT_SECONDS} с < check {lb['timeout check']} с" + ) + assert lb["timeout server"] > api["REQUEST_TIMEOUT_SECONDS"] > database + assert lb["timeout check"] > 2 * PING_TIMEOUT_SECONDS + assert drain > detection + assert shutdown - drain >= database + assert shutdown < hook < grace < 10 + assert lb["hard-stop-after"] < seconds(stand["lb"]["stop_grace_period"]) + assert (detection, hung) == (1.5, 3.5)