Run the API as replicas behind HAProxy

compose.yaml now runs one api service with API_REPLICAS replicas and
no published ports, container name or hostname, behind lb, HAProxy
3.2.26 LTS pinned by digest and the only service with ports, bound to
127.0.0.1 (LB_HTTP_PORT for HTTP, LB_STATS_PORT for read-only stats).
The balancer finds replicas through Docker DNS with a server template,
checks GET /health/ready every second (fall 2, rise 2), balances round
robin without stickiness, retries connection failures on another
replica and allows a retry after a sent request only for GET and for
message posts carrying Idempotency-Key, never for users, chats or
login. Its own 502/503/504 answers use the contract error body with
the forwarded X-Request-Id.

The network is split into edge and an internal backend, every service
gets CPU and memory limits, the API runs read-only with a tmpfs /tmp,
two workers per replica, an 8 s request limit and a 5 s graceful stop
inside an 8 s stop_grace_period. Only api builds the image, migrate
reuses it once before the replicas start. PostgreSQL loads
pg_stat_statements and allows 200 connections. The short TTL instance
of lab 2 published the API around the balancer and is removed; its
expiry check stays as an in-process test.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-28 11:43:43 +03:00
co-authored by Claude Opus 5.5
parent 7bf28fc6ed
commit dc2015473f
11 changed files with 219 additions and 102 deletions
+13 -4
View File
@@ -11,8 +11,17 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# Solution-specific: host ports of the API and of the short TTL instance, workers, cursor key file.
API_PORT=8080
SHORTTTL_PORT=8082
WEB_CONCURRENCY=4
# Solution-specific: published LB ports, replica limits, workers and pool per replica,
# LB queue per replica, PostgreSQL limits, cursor key file. API_PORT/SHORTTTL_PORT of lab2 are gone:
# only the load balancer is published.
LB_HTTP_PORT=8080
LB_STATS_PORT=8404
API_CPUS=1
API_MEMORY=512m
WEB_CONCURRENCY=2
DB_POOL_MAX_SIZE=8
API_SERVER_MAXCONN=64
PG_MAX_CONNECTIONS=200
PG_CPUS=2
PG_MEMORY=1g
CURSOR_SECRET_FILE=/run/secrets/cursor_secret
+12 -8
View File
@@ -19,7 +19,7 @@ jobs:
- run: make student-test VENV=.venv
acceptance:
name: Стенд Compose, smoke, рестарт и отказы зависимостей
name: Стенд Compose за HAProxy, smoke, рестарт и отказы зависимостей
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -31,27 +31,31 @@ jobs:
- name: Поднять стенд из чистого clone
run: make up
- name: Публичный smoke
- name: Публичный smoke через балансировщик
run: make test
- name: Рестарт API и хранилищ с сохранением volumes
- name: Рестарт LB, API и хранилищ с сохранением volumes
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh
- name: Отказы Valkey и PostgreSQL, expiry, гонка и rollback
- name: Отказы Valkey и PostgreSQL, гонка и rollback
run: make test-faults
- name: Наружу опубликован только балансировщик
run: |
docker compose ps --format '{{.Service}} {{.Ports}}' | awk '$2 != "" && $1 != "lb" {print; bad=1} END {exit bad}'
- name: Процессы работают не от root
run: |
docker compose top | awk 'NR>1 && $3=="0" {print; bad=1} END {exit bad}'
- name: Корректная остановка API по SIGTERM
- name: Корректная остановка реплик API по SIGTERM
run: |
start=$(date +%s)
docker compose kill --signal SIGTERM api
code=$(docker wait "$(docker compose ps -aq api)")
codes=$(docker wait $(docker compose ps -aq api) | sort -u)
elapsed=$(( $(date +%s) - start ))
echo "остановка за ${elapsed} с, код завершения ${code}"
test "$elapsed" -le 10 && test "$code" = "0"
echo "остановка за ${elapsed} с, коды завершения ${codes}"
test "$elapsed" -le 10 && test "$codes" = "0"
- name: Журналы стенда
if: always()
+3 -3
View File
@@ -24,9 +24,9 @@ ENV PYTHONUNBUFFERED=1 \
HOST=0.0.0.0 \
PORT=8080 \
LOG_LEVEL=INFO \
WEB_CONCURRENCY=4 \
REQUEST_TIMEOUT_SECONDS=15 \
SHUTDOWN_TIMEOUT_SECONDS=10
WEB_CONCURRENCY=2 \
REQUEST_TIMEOUT_SECONDS=8 \
SHUTDOWN_TIMEOUT_SECONDS=5
RUN groupadd --system --gid 10001 messenger \
&& useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
+3 -6
View File
@@ -8,8 +8,6 @@ VENV ?= .venv
UV ?= uv
COMPOSE ?= docker compose
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
SHORTTTL_PORT ?= 8082
SHORTTTL_URL ?= http://localhost:$(SHORTTTL_PORT)
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
@@ -65,11 +63,11 @@ up: secrets
$(COMPOSE) up -d --build --wait
down:
$(COMPOSE) --profile faults down
$(COMPOSE) down
reset-data:
@echo "Удаляются volumes с учебными данными стенда"
$(COMPOSE) --profile faults down -v --remove-orphans
$(COMPOSE) down -v --remove-orphans
migrate:
$(COMPOSE) run --rm migrate
@@ -92,8 +90,7 @@ cache-evidence:
cat evidence/cache.txt
test-faults:
$(COMPOSE) --profile faults up -d --wait api-shortttl
BASE_URL="$(BASE_URL)" SHORTTTL_URL="$(SHORTTTL_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
seed:
$(PSQL) -f - < scripts/sql/seed-history.sql
+104 -56
View File
@@ -1,27 +1,106 @@
x-app: &app
image: mtusi-messenger:lab2
build: .
environment: &app-environment
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-4}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url, cache_url, cursor_secret]
image: ${API_IMAGE:-mtusi-messenger:lab3}
networks: [backend]
x-ready: &ready
test: ["CMD", "python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"]
interval: 3s
timeout: 3s
retries: 5
start_period: 15s
services:
lb:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
environment:
API_SERVER_MAXCONN: ${API_SERVER_MAXCONN:-64}
volumes: [./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro]
ports:
- "127.0.0.1:${LB_HTTP_PORT:-8080}:8080"
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
networks: [edge, backend]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
sysctls:
net.ipv4.ip_local_port_range: "10240 65535"
net.ipv4.tcp_tw_reuse: "1"
stop_signal: SIGUSR1
stop_grace_period: 8s
restart: unless-stopped
read_only: true
cap_drop: [ALL]
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1}", memory: 128m}
api:
<<: *app
build: .
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url, cache_url, cursor_secret]
healthcheck:
test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"]
interval: 5s
timeout: 3s
retries: 3
start_period: 30s
start_interval: 1s
stop_grace_period: 8s
restart: unless-stopped
read_only: true
tmpfs: [/tmp]
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_started
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "${API_CPUS:-1}", memory: "${API_MEMORY:-512m}"}
migrate:
<<: *app
pull_policy: never
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url]
healthcheck:
disable: true
restart: "no"
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits: {cpus: "0.5", memory: 256m}
postgres:
image: postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b
user: "70:70"
command:
- postgres
- -c
- max_connections=${PG_MAX_CONNECTIONS:-200}
- -c
- shared_buffers=${PG_SHARED_BUFFERS:-256MB}
- -c
- shared_preload_libraries=pg_stat_statements
- -c
- pg_stat_statements.track=top
- -c
- track_io_timing=on
- -c
- idle_in_transaction_session_timeout=10s
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
@@ -38,6 +117,9 @@ services:
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2}", memory: "${PG_MEMORY:-1g}"}
valkey:
image: valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a
@@ -63,45 +145,9 @@ services:
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
migrate:
<<: *app
command: ["python", "-m", "messenger.migrate"]
healthcheck:
disable: true
restart: "no"
depends_on:
postgres:
condition: service_healthy
api:
<<: *app
ports: ["127.0.0.1:${API_PORT:-8080}:8080"]
healthcheck: *ready
stop_grace_period: 10s
restart: unless-stopped
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_started
api-shortttl:
<<: *app
profiles: [faults]
environment:
<<: *app-environment
SESSION_TTL_SECONDS: "2"
WEB_CONCURRENCY: "1"
ports: ["127.0.0.1:${SHORTTTL_PORT:-8082}:8080"]
healthcheck: *ready
stop_grace_period: 10s
restart: unless-stopped
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_started
deploy:
resources:
limits: {cpus: "0.5", memory: 128m}
secrets:
postgres_password:
@@ -120,4 +166,6 @@ volumes:
valkeydata:
networks:
edge:
backend:
internal: true
+67
View File
@@ -0,0 +1,67 @@
global
log stdout format raw local0 info
maxconn 4096
hard-stop-after 5s
tune.bufsize 32768
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold nx 2s
hold other 2s
hold refused 2s
hold timeout 2s
defaults
mode http
log global
option dontlognull
option dontlog-normal
log-format "%ci [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] %HM %HPO"
option http-server-close
option redispatch 1
retries 2
retry-on conn-failure empty-response
timeout connect 500ms
timeout client 30s
timeout server 10s
timeout http-request 5s
timeout http-keep-alive 5s
timeout queue 3s
timeout check 1s
http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id)]"}}'
http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id)]"}}'
http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id)]"}}'
frontend public
bind :8080
option forwardfor
http-request set-var(txn.request_id) req.fhdr(X-Request-Id) if { req.fhdr(X-Request-Id) -m len 1:200 }
http-request set-var(txn.request_id) uuid(4) unless { var(txn.request_id) -m found }
http-request set-header X-Request-Id %[var(txn.request_id)]
default_backend api
backend api
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
http-check expect status 200
acl message_path path_reg ^/api/v1/chats/[^/]+/messages$
acl has_key req.hdr(Idempotency-Key) -m found
http-request set-var(txn.replay_safe) bool(1) if METH_GET
http-request set-var(txn.replay_safe) bool(1) if METH_POST message_path has_key
http-request disable-l7-retry unless { var(txn.replay_safe) -m bool }
default-server check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none maxconn "${API_SERVER_MAXCONN-64}"
server-template api 8 api:8080
frontend stats
bind :8404
monitor-uri /healthz
monitor fail if { nbsrv(api) lt 1 }
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /stats
stats refresh 5s
@@ -24,10 +24,10 @@
## 5. Стенд
- [ ] 5.1 `infra/haproxy/haproxy.cfg` из заготовки: round robin, `httpchk /health/ready`, Docker DNS, политика повторов, JSON-ошибки LB, статистика
- [ ] 5.2 `compose.yaml`: `lb`, `api` с `deploy.replicas` без портов, сети `edge`/`backend`, лимиты всех сервисов, параметры PostgreSQL, таймауты и `stop_grace_period`; удалить `api-shortttl`
- [ ] 5.3 `.env.example`, Dockerfile (умолчания таймаутов и воркеров), Makefile (`up`, `down`, `restore`, `lb-check`, `test-faults` без `api-shortttl`), CI-job под лабу 3
- [ ] 5.4 Проверка группы: `docker compose config --quiet`, `haproxy -c`, `make up`, опубликованы только порты `lb`
- [x] 5.1 `infra/haproxy/haproxy.cfg` из заготовки: round robin, `httpchk /health/ready`, Docker DNS, политика повторов, JSON-ошибки LB, статистика
- [x] 5.2 `compose.yaml`: `lb`, `api` с `deploy.replicas` без портов, сети `edge`/`backend`, лимиты всех сервисов, параметры PostgreSQL, таймауты и `stop_grace_period`; удалить `api-shortttl`
- [x] 5.3 `.env.example`, Dockerfile (умолчания таймаутов и воркеров), Makefile (`up`, `down`, `restore`, `lb-check`, `test-faults` без `api-shortttl`), CI-job под лабу 3
- [x] 5.4 Проверка группы: `docker compose config --quiet`, `haproxy -c`, `make up`, опубликованы только порты `lb`
## 6. Хуки и зонд
+4 -4
View File
@@ -23,19 +23,19 @@ class MessengerWorker(UvicornWorker):
"access_log": False,
"server_header": False,
"date_header": False,
"timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "10")),
"timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")),
}
bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}"
workers = int(os.environ.get("WEB_CONCURRENCY", "4"))
workers = int(os.environ.get("WEB_CONCURRENCY", "2"))
worker_class = "messenger.gunicorn_conf.MessengerWorker"
logger_class = StructlogGunicornLogger
loglevel = LOG_LEVEL.lower()
accesslog = None
errorlog = "-"
timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "15")) + 5)
graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "10"))
timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5)
graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
keepalive = 5
preload_app = False
control_socket_disable = True
+2 -2
View File
@@ -6,5 +6,5 @@ from messenger.shared.config import Settings
class TransportSettings(Settings):
docs: bool = False
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
request_timeout_seconds: float = Field(default=15.0, gt=0, le=300)
shutdown_timeout_seconds: int = Field(default=10, ge=1, le=10)
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
shutdown_timeout_seconds: int = Field(default=5, ge=1, le=10)
-13
View File
@@ -9,7 +9,6 @@ from stand import (
BASE_URL,
BASIC,
BEARER,
SHORTTTL_URL,
basic,
bearer,
compose,
@@ -89,18 +88,6 @@ async def test_logout_then_token_is_dead(http: AsyncClient) -> None:
assert again.headers["WWW-Authenticate"] == BEARER
async def test_short_ttl_session_expires() -> None:
async with AsyncClient(base_url=SHORTTTL_URL, timeout=10, trust_env=False) as short:
_, token = await enroll(short, "expiry")
fresh = await short.get("/api/v1/users/me", headers=bearer(token))
await asyncio.sleep(3)
stale = await short.get("/api/v1/users/me", headers=bearer(token))
print(f"SESSION_TTL_SECONDS=2: сразу {fresh.status_code}, через 3 с {stale.status_code}")
assert fresh.status_code == 200
assert stale.status_code == 401
assert stale.headers["WWW-Authenticate"] == BEARER
async def test_valkey_serves_repeated_checks(http: AsyncClient) -> None:
_, token = await enroll(http, "cache")
await http.get("/api/v1/users/me", headers=bearer(token))
+7 -2
View File
@@ -12,7 +12,6 @@ from httpx import AsyncClient, Response, TransportError
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080")
SHORTTTL_URL = os.environ.get("SHORTTTL_URL", "http://localhost:8082")
PASSWORD = "Пароль-для-стенда-01"
BASIC = 'Basic realm="messenger", charset="UTF-8"'
BEARER = 'Bearer realm="messenger"'
@@ -43,7 +42,13 @@ def valkey(command: str) -> str:
def started_at(service: str) -> str:
return subprocess.run(
["docker", "inspect", "--format", "{{.State.StartedAt}}", compose("ps", "-q", service)],
[
"docker",
"inspect",
"--format",
"{{.State.StartedAt}}",
*compose("ps", "-q", service).split(),
],
check=True,
capture_output=True,
text=True,