forked from mtusi2026/mtusi2026lab3
Pin the timeout chain in the stand and test it
The API service now names every timeout that the chain depends on: DRAIN_SECONDS, DB_ACQUIRE_TIMEOUT_SECONDS and DB_COMMAND_TIMEOUT_SECONDS join REQUEST_TIMEOUT_SECONDS and SHUTDOWN_TIMEOUT_SECONDS in compose, and the image gets the drain default. A unit test reads the compose defaults, the HAProxy check and timeout settings and the stop hook timeout, and checks that the chain shrinks inwards: LB server 10 s > request 8 s > pool wait plus statement 3 s, readiness 0.8 s < check 1 s, drain 2 s > worst detection 1.5 s, shutdown 5 s leaves 3 s for accepted requests, and shutdown < hook 6 s < stop_grace_period 8 s < 10 s. It also pins the declared detection maxima: 1.5 s for a 503 or a refused connection and 3.5 s for a hung replica. The k6 load generator joins the stand under the bench profile: pinned by digest, edge network only, 2 CPU and 512 MiB, no secrets. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6fb4587ef6
commit
85ed37914d
+5
-1
@@ -12,7 +12,8 @@ CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Solution-specific: published LB ports, replica limits, workers and pool per replica,
|
||||
# LB queue per replica, PostgreSQL limits, cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
|
||||
# LB queue per replica, drain window before closing the listener on SIGTERM, PostgreSQL
|
||||
# limits, load generator limits (profile bench), cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
|
||||
# only the load balancer is published.
|
||||
LB_HTTP_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
@@ -21,7 +22,10 @@ API_MEMORY=512m
|
||||
WEB_CONCURRENCY=2
|
||||
DB_POOL_MAX_SIZE=8
|
||||
API_SERVER_MAXCONN=64
|
||||
DRAIN_SECONDS=2
|
||||
PG_MAX_CONNECTIONS=200
|
||||
PG_CPUS=2
|
||||
PG_MEMORY=1g
|
||||
K6_CPUS=2
|
||||
K6_MEMORY=512m
|
||||
CURSOR_SECRET_FILE=/run/secrets/cursor_secret
|
||||
|
||||
@@ -26,6 +26,7 @@ ENV PYTHONUNBUFFERED=1 \
|
||||
LOG_LEVEL=INFO \
|
||||
WEB_CONCURRENCY=2 \
|
||||
REQUEST_TIMEOUT_SECONDS=8 \
|
||||
DRAIN_SECONDS=2 \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
|
||||
RUN groupadd --system --gid 10001 messenger \
|
||||
|
||||
@@ -42,6 +42,9 @@ services:
|
||||
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
|
||||
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
|
||||
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
|
||||
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
secrets: [database_url, cache_url, cursor_secret]
|
||||
@@ -149,6 +152,17 @@ services:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128m}
|
||||
|
||||
k6:
|
||||
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
|
||||
profiles: [bench]
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
volumes: [./infra/k6:/scripts:ro]
|
||||
networks: [edge]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${K6_CPUS:-2}", memory: "${K6_MEMORY:-512m}"}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
|
||||
@@ -7,9 +7,9 @@
|
||||
|
||||
## 2. Стенд и таймауты
|
||||
|
||||
- [ ] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile
|
||||
- [ ] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh`
|
||||
- [ ] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"`
|
||||
- [x] 2.1 `compose.yaml`: явные `DRAIN_SECONDS`, `DB_ACQUIRE_TIMEOUT_SECONDS`, `DB_COMMAND_TIMEOUT_SECONDS` у `api`; сервис `k6` под профилем `bench`; `.env.example`, Dockerfile
|
||||
- [x] 2.2 Тест цепочки таймаутов по `compose.yaml`, `haproxy.cfg` и `scripts/stop-one.sh`
|
||||
- [x] 2.3 Проверка группы: `docker compose --profile bench config --quiet`, `make student-test -k "timeouts or drain"`
|
||||
|
||||
## 3. Нагрузка и зонд
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from messenger.shared.infrastructure.postgres import PING_TIMEOUT_SECONDS
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT = re.compile(r"\$\{\w+:?-([^}]*)\}")
|
||||
UNITS = {"ms": 0.001, "s": 1.0, "": 1.0}
|
||||
|
||||
|
||||
def seconds(value: object) -> float:
|
||||
found = re.fullmatch(r"([\d.]+)(ms|s|)", str(value))
|
||||
assert found is not None, value
|
||||
return float(found[1]) * UNITS[found[2]]
|
||||
|
||||
|
||||
def services() -> dict[str, Any]:
|
||||
return yaml.safe_load(DEFAULT.sub(r"\1", (ROOT / "compose.yaml").read_text()))["services"]
|
||||
|
||||
|
||||
def balancer() -> dict[str, float]:
|
||||
text = (ROOT / "infra/haproxy/haproxy.cfg").read_text()
|
||||
found = {
|
||||
f"timeout {name}": seconds(value)
|
||||
for name, value in re.findall(r"timeout ([\w-]+) (\S+)\n", text)
|
||||
}
|
||||
found["hard-stop-after"] = seconds(re.findall(r"hard-stop-after (\S+)", text)[0])
|
||||
check = re.findall(r"default-server .*", text)[0]
|
||||
for key in ("inter", "fastinter", "fall", "rise"):
|
||||
found[key] = seconds(re.findall(rf"\b{key} (\S+)", check)[0])
|
||||
return found
|
||||
|
||||
|
||||
def test_timeouts_shrink_inwards_and_shutdown_fits_the_stop_budget() -> None:
|
||||
lb, stand = balancer(), services()
|
||||
api = {k: seconds(v) for k, v in stand["api"]["environment"].items() if k.endswith("SECONDS")}
|
||||
hook = seconds(
|
||||
re.findall(r"STOP_TIMEOUT:-(\d+)", (ROOT / "scripts/stop-one.sh").read_text())[0]
|
||||
)
|
||||
grace = seconds(stand["api"]["stop_grace_period"])
|
||||
detection = lb["inter"] + (lb["fall"] - 1) * lb["fastinter"]
|
||||
hung = detection + lb["fall"] * lb["timeout check"]
|
||||
database = api["DB_ACQUIRE_TIMEOUT_SECONDS"] + api["DB_COMMAND_TIMEOUT_SECONDS"]
|
||||
drain, shutdown = api["DRAIN_SECONDS"], api["SHUTDOWN_TIMEOUT_SECONDS"]
|
||||
print(
|
||||
f"обнаружение {detection} с (зависание {hung} с), окно {drain} с, остановка {shutdown} с,"
|
||||
f" хук {hook} с, stop_grace_period {grace} с; LB server {lb['timeout server']} с >"
|
||||
f" запрос {api['REQUEST_TIMEOUT_SECONDS']} с > БД {database} с;"
|
||||
f" readiness {2 * PING_TIMEOUT_SECONDS} с < check {lb['timeout check']} с"
|
||||
)
|
||||
assert lb["timeout server"] > api["REQUEST_TIMEOUT_SECONDS"] > database
|
||||
assert lb["timeout check"] > 2 * PING_TIMEOUT_SECONDS
|
||||
assert drain > detection
|
||||
assert shutdown - drain >= database
|
||||
assert shutdown < hook < grace < 10
|
||||
assert lb["hard-stop-after"] < seconds(stand["lb"]["stop_grace_period"])
|
||||
assert (detection, hung) == (1.5, 3.5)
|
||||
Reference in New Issue
Block a user