diff --git a/.env.example b/.env.example index 26557598..ee70667f 100644 --- a/.env.example +++ b/.env.example @@ -11,8 +11,17 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -# Solution-specific: host ports of the API and of the short TTL instance, workers, cursor key file. -API_PORT=8080 -SHORTTTL_PORT=8082 -WEB_CONCURRENCY=4 +# Solution-specific: published LB ports, replica limits, workers and pool per replica, +# LB queue per replica, PostgreSQL limits, cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone: +# only the load balancer is published. +LB_HTTP_PORT=8080 +LB_STATS_PORT=8404 +API_CPUS=1 +API_MEMORY=512m +WEB_CONCURRENCY=2 +DB_POOL_MAX_SIZE=8 +API_SERVER_MAXCONN=64 +PG_MAX_CONNECTIONS=200 +PG_CPUS=2 +PG_MEMORY=1g CURSOR_SECRET_FILE=/run/secrets/cursor_secret diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index 5ba96a5e..0bca8e17 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -19,7 +19,7 @@ jobs: - run: make student-test VENV=.venv acceptance: - name: Стенд Compose, smoke, рестарт и отказы зависимостей + name: Стенд Compose за HAProxy, smoke, рестарт и отказы зависимостей runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,27 +31,31 @@ jobs: - name: Поднять стенд из чистого clone run: make up - - name: Публичный smoke + - name: Публичный smoke через балансировщик run: make test - - name: Рестарт API и хранилищ с сохранением volumes + - name: Рестарт LB, API и хранилищ с сохранением volumes run: make test-persistence ACTION_SCRIPT=scripts/restart.sh - - name: Отказы Valkey и PostgreSQL, expiry, гонка и rollback + - name: Отказы Valkey и PostgreSQL, гонка и rollback run: make test-faults + - name: Наружу опубликован только балансировщик + run: | + docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" {print; bad=1} END {exit bad}' + - name: Процессы работают не от root run: | docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}' - - name: Корректная остановка API по SIGTERM + - name: Корректная остановка реплик API по SIGTERM run: | start=$(date +%s) docker compose kill --signal SIGTERM api - code=$(docker wait "$(docker compose ps -aq api)") + codes=$(docker wait $(docker compose ps -aq api) | sort -u) elapsed=$(( $(date +%s) - start )) - echo "остановка за ${elapsed} с, код завершения ${code}" - test "$elapsed" -le 10 && test "$code" = "0" + echo "остановка за ${elapsed} с, коды завершения ${codes}" + test "$elapsed" -le 10 && test "$codes" = "0" - name: Журналы стенда if: always() diff --git a/Dockerfile b/Dockerfile index f5b09847..d4d9f201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,9 @@ ENV PYTHONUNBUFFERED=1 \ HOST=0.0.0.0 \ PORT=8080 \ LOG_LEVEL=INFO \ - WEB_CONCURRENCY=4 \ - REQUEST_TIMEOUT_SECONDS=15 \ - SHUTDOWN_TIMEOUT_SECONDS=10 + WEB_CONCURRENCY=2 \ + REQUEST_TIMEOUT_SECONDS=8 \ + SHUTDOWN_TIMEOUT_SECONDS=5 RUN groupadd --system --gid 10001 messenger \ && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger diff --git a/Makefile b/Makefile index 08d7b9c9..8f96d97c 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,6 @@ VENV ?= .venv UV ?= uv COMPOSE ?= docker compose PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q -SHORTTTL_PORT ?= 8082 -SHORTTTL_URL ?= http://localhost:$(SHORTTTL_PORT) .PHONY: help check validate test load test-persistence test-failover \ install lint format typecheck student-test \ @@ -65,11 +63,11 @@ up: secrets $(COMPOSE) up -d --build --wait down: - $(COMPOSE) --profile faults down + $(COMPOSE) down reset-data: @echo "Удаляются volumes с учебными данными стенда" - $(COMPOSE) --profile faults down -v --remove-orphans + $(COMPOSE) down -v --remove-orphans migrate: $(COMPOSE) run --rm migrate @@ -92,8 +90,7 @@ cache-evidence: cat evidence/cache.txt test-faults: - $(COMPOSE) --profile faults up -d --wait api-shortttl - BASE_URL="$(BASE_URL)" SHORTTTL_URL="$(SHORTTTL_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack + BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack seed: $(PSQL) -f - < scripts/sql/seed-history.sql diff --git a/compose.yaml b/compose.yaml index c1c8daba..7339cbe9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,27 +1,106 @@ x-app: &app - image: mtusi-messenger:lab2 - build: . - environment: &app-environment - DATABASE_URL_FILE: /run/secrets/database_url - CACHE_URL_FILE: /run/secrets/cache_url - CURSOR_SECRET_FILE: /run/secrets/cursor_secret - SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} - WEB_CONCURRENCY: ${WEB_CONCURRENCY:-4} - LOG_LEVEL: ${LOG_LEVEL:-INFO} - secrets: [database_url, cache_url, cursor_secret] + image: ${API_IMAGE:-mtusi-messenger:lab3} networks: [backend] -x-ready: &ready - test: ["CMD", "python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"] - interval: 3s - timeout: 3s - retries: 5 - start_period: 15s - services: + lb: + image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b + environment: + API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64} + volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro] + ports: + - "127.0.0.1:${LB_HTTP_PORT:-8080}:8080" + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + networks: [edge, backend] + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] + interval: 2s + timeout: 2s + retries: 3 + start_period: 10s + sysctls: + net.ipv4.ip_local_port_range: "10240 65535" + net.ipv4.tcp_tw_reuse: "1" + stop_signal: SIGUSR1 + stop_grace_period: 8s + restart: unless-stopped + read_only: true + cap_drop: [ALL] + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1}", memory: 128m} + + api: + <<: *app + build: . + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} + REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + secrets: [database_url, cache_url, cursor_secret] + healthcheck: + test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + stop_grace_period: 8s + restart: unless-stopped + read_only: true + tmpfs: [/tmp] + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_started + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-512m}"} + + migrate: + <<: *app + pull_policy: never + command: ["python", "-m", "messenger.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + LOG_LEVEL: ${LOG_LEVEL:-INFO} + secrets: [database_url] + healthcheck: + disable: true + restart: "no" + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} + postgres: image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b user: "70:70" + command: + - postgres + - -c + - max_connections=${PG_MAX_CONNECTIONS:-200} + - -c + - shared_buffers=${PG_SHARED_BUFFERS:-256MB} + - -c + - shared_preload_libraries=pg_stat_statements + - -c + - pg_stat_statements.track=top + - -c + - track_io_timing=on + - -c + - idle_in_transaction_session_timeout=10s environment: POSTGRES_DB: messenger POSTGRES_USER: messenger @@ -38,6 +117,9 @@ services: stop_grace_period: 10s restart: unless-stopped networks: [backend] + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2}", memory: "${PG_MEMORY:-1g}"} valkey: image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a @@ -63,45 +145,9 @@ services: stop_grace_period: 10s restart: unless-stopped networks: [backend] - - migrate: - <<: *app - command: ["python", "-m", "messenger.migrate"] - healthcheck: - disable: true - restart: "no" - depends_on: - postgres: - condition: service_healthy - - api: - <<: *app - ports: ["127.0.0.1:${API_PORT:-8080}:8080"] - healthcheck: *ready - stop_grace_period: 10s - restart: unless-stopped - depends_on: - migrate: - condition: service_completed_successfully - valkey: - condition: service_started - - api-shortttl: - <<: *app - profiles: [faults] - environment: - <<: *app-environment - SESSION_TTL_SECONDS: "2" - WEB_CONCURRENCY: "1" - ports: ["127.0.0.1:${SHORTTTL_PORT:-8082}:8080"] - healthcheck: *ready - stop_grace_period: 10s - restart: unless-stopped - depends_on: - migrate: - condition: service_completed_successfully - valkey: - condition: service_started + deploy: + resources: + limits: {cpus: "0.5", memory: 128m} secrets: postgres_password: @@ -120,4 +166,6 @@ volumes: valkeydata: networks: + edge: backend: + internal: true diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..421b78f9 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,67 @@ +global + log stdout format raw local0 info + maxconn 4096 + hard-stop-after 5s + tune.bufsize 32768 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + +defaults + mode http + log global + option dontlognull + option dontlog-normal + log-format "%ci [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] %HM %HPO" + option http-server-close + option redispatch 1 + retries 2 + retry-on conn-failure empty-response + timeout connect 500ms + timeout client 30s + timeout server 10s + timeout http-request 5s + timeout http-keep-alive 5s + timeout queue 3s + timeout check 1s + http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id)]"}}' + http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id)]"}}' + http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id)]"}}' + +frontend public + bind :8080 + option forwardfor + http-request set-var(txn.request_id) req.fhdr(X-Request-Id) if { req.fhdr(X-Request-Id) -m len 1:200 } + http-request set-var(txn.request_id) uuid(4) unless { var(txn.request_id) -m found } + http-request set-header X-Request-Id %[var(txn.request_id)] + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + acl message_path path_reg ^/api/v1/chats/[^/]+/messages$ + acl has_key req.hdr(Idempotency-Key) -m found + http-request set-var(txn.replay_safe) bool(1) if METH_GET + http-request set-var(txn.replay_safe) bool(1) if METH_POST message_path has_key + http-request disable-l7-retry unless { var(txn.replay_safe) -m bool } + default-server check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none maxconn "${API_SERVER_MAXCONN-64}" + server-template api 8 api:8080 + +frontend stats + bind :8404 + monitor-uri /healthz + monitor fail if { nbsrv(api) lt 1 } + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 5s diff --git a/openspec/changes/add-lab3-load-balanced-replicas/tasks.md b/openspec/changes/add-lab3-load-balanced-replicas/tasks.md index dc42be43..bf4bd921 100644 --- a/openspec/changes/add-lab3-load-balanced-replicas/tasks.md +++ b/openspec/changes/add-lab3-load-balanced-replicas/tasks.md @@ -24,10 +24,10 @@ ## 5. Стенд -- [ ] 5.1 `infra/haproxy/haproxy.cfg` из заготовки: round robin, `httpchk /health/ready`, Docker DNS, политика повторов, JSON-ошибки LB, статистика -- [ ] 5.2 `compose.yaml`: `lb`, `api` с `deploy.replicas` без портов, сети `edge`/`backend`, лимиты всех сервисов, параметры PostgreSQL, таймауты и `stop_grace_period`; удалить `api-shortttl` -- [ ] 5.3 `.env.example`, Dockerfile (умолчания таймаутов и воркеров), Makefile (`up`, `down`, `restore`, `lb-check`, `test-faults` без `api-shortttl`), CI-job под лабу 3 -- [ ] 5.4 Проверка группы: `docker compose config --quiet`, `haproxy -c`, `make up`, опубликованы только порты `lb` +- [x] 5.1 `infra/haproxy/haproxy.cfg` из заготовки: round robin, `httpchk /health/ready`, Docker DNS, политика повторов, JSON-ошибки LB, статистика +- [x] 5.2 `compose.yaml`: `lb`, `api` с `deploy.replicas` без портов, сети `edge`/`backend`, лимиты всех сервисов, параметры PostgreSQL, таймауты и `stop_grace_period`; удалить `api-shortttl` +- [x] 5.3 `.env.example`, Dockerfile (умолчания таймаутов и воркеров), Makefile (`up`, `down`, `restore`, `lb-check`, `test-faults` без `api-shortttl`), CI-job под лабу 3 +- [x] 5.4 Проверка группы: `docker compose config --quiet`, `haproxy -c`, `make up`, опубликованы только порты `lb` ## 6. Хуки и зонд diff --git a/src/messenger/gunicorn_conf.py b/src/messenger/gunicorn_conf.py index b9c734be..1b168029 100644 --- a/src/messenger/gunicorn_conf.py +++ b/src/messenger/gunicorn_conf.py @@ -23,19 +23,19 @@ class MessengerWorker(UvicornWorker): "access_log": False, "server_header": False, "date_header": False, - "timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "10")), + "timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")), } bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}" -workers = int(os.environ.get("WEB_CONCURRENCY", "4")) +workers = int(os.environ.get("WEB_CONCURRENCY", "2")) worker_class = "messenger.gunicorn_conf.MessengerWorker" logger_class = StructlogGunicornLogger loglevel = LOG_LEVEL.lower() accesslog = None errorlog = "-" -timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "15")) + 5) -graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "10")) +timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5) +graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) keepalive = 5 preload_app = False control_socket_disable = True diff --git a/src/messenger/shared/entrypoints/config.py b/src/messenger/shared/entrypoints/config.py index 72312868..c3e77663 100644 --- a/src/messenger/shared/entrypoints/config.py +++ b/src/messenger/shared/entrypoints/config.py @@ -6,5 +6,5 @@ from messenger.shared.config import Settings class TransportSettings(Settings): docs: bool = False max_body_bytes: int = Field(default=1_048_576, ge=4_096) - request_timeout_seconds: float = Field(default=15.0, gt=0, le=300) - shutdown_timeout_seconds: int = Field(default=10, ge=1, le=10) + request_timeout_seconds: float = Field(default=8.0, gt=0, le=300) + shutdown_timeout_seconds: int = Field(default=5, ge=1, le=10) diff --git a/student-tests/stack/test_faults.py b/student-tests/stack/test_faults.py index 7008b283..4c5ab055 100644 --- a/student-tests/stack/test_faults.py +++ b/student-tests/stack/test_faults.py @@ -9,7 +9,6 @@ from stand import ( BASE_URL, BASIC, BEARER, - SHORTTTL_URL, basic, bearer, compose, @@ -89,18 +88,6 @@ async def test_logout_then_token_is_dead(http: AsyncClient) -> None: assert again.headers["WWW-Authenticate"] == BEARER -async def test_short_ttl_session_expires() -> None: - async with AsyncClient(base_url=SHORTTTL_URL, timeout=10, trust_env=False) as short: - _, token = await enroll(short, "expiry") - fresh = await short.get("/api/v1/users/me", headers=bearer(token)) - await asyncio.sleep(3) - stale = await short.get("/api/v1/users/me", headers=bearer(token)) - print(f"SESSION_TTL_SECONDS=2: сразу {fresh.status_code}, через 3 с {stale.status_code}") - assert fresh.status_code == 200 - assert stale.status_code == 401 - assert stale.headers["WWW-Authenticate"] == BEARER - - async def test_valkey_serves_repeated_checks(http: AsyncClient) -> None: _, token = await enroll(http, "cache") await http.get("/api/v1/users/me", headers=bearer(token)) diff --git a/student-tests/stand.py b/student-tests/stand.py index 8e90e739..3fc7ba1d 100644 --- a/student-tests/stand.py +++ b/student-tests/stand.py @@ -12,7 +12,6 @@ from httpx import AsyncClient, Response, TransportError ROOT = Path(__file__).resolve().parents[1] BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080") -SHORTTTL_URL = os.environ.get("SHORTTTL_URL", "http://localhost:8082") PASSWORD = "Пароль-для-стенда-01" BASIC = 'Basic realm="messenger", charset="UTF-8"' BEARER = 'Bearer realm="messenger"' @@ -43,7 +42,13 @@ def valkey(command: str) -> str: def started_at(service: str) -> str: return subprocess.run( - ["docker", "inspect", "--format", "{{.State.StartedAt}}", compose("ps", "-q", service)], + [ + "docker", + "inspect", + "--format", + "{{.State.StartedAt}}", + *compose("ps", "-q", service).split(), + ], check=True, capture_output=True, text=True,