Разделить стенд и настройки нагрузочных экспериментов, ограничить число запросов на реплику в HAProxy и передавать X-Request-Id от балансировщика в API, сократить комментарии

This commit is contained in:
2026-11-04 12:23:14 +03:00
parent 7fbd781ca3
commit ce47c138e4
7 changed files with 94 additions and 111 deletions
+2
View File
@@ -19,3 +19,5 @@ node_modules
evidence
.DS_Store
compose*.yaml
bench-out
bin
+6 -12
View File
@@ -1,8 +1,5 @@
# Build stage: compile static binaries with the exact toolchain pinned in go.mod.
# Modules live in their own layer, rebuilt only when go.mod or go.sum change.
# Only the Go sources are copied after it, and the compiler's cache persists
# between builds: a change elsewhere in the repository does not trigger a full
# recompilation, which matters when a rolling update rebuilds the image.
# Modules in their own layer and a persistent build cache keep the rebuild of
# a rolling update short.
FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build
WORKDIR /src
COPY go.mod go.sum ./
@@ -13,8 +10,7 @@ COPY migrations ./migrations
RUN --mount=type=cache,target=/root/.cache/go-build \
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate
# Load generator of the experiments, an image of its own: the API image does
# not carry it. Built only for `docker compose --profile bench`.
# The load generator gets an image of its own; the API image does not carry it.
FROM build AS loadgen-build
COPY tools ./tools
RUN --mount=type=cache,target=/root/.cache/go-build \
@@ -26,9 +22,7 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen
USER app
ENTRYPOINT ["/usr/local/bin/loadgen"]
# Runtime stage, the last one so that `build: .` builds it: minimal image,
# unprivileged user, the server is PID 1. The same image runs the one-shot
# migrate service with its own entrypoint.
# Last, so that `build: .` builds it; it also runs the migrate service.
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
RUN adduser -D -H -u 10001 app
COPY --from=build /out/server /out/migrate /usr/local/bin/
@@ -37,8 +31,8 @@ ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab
EXPOSE 8080
# The short start interval lets `compose up --wait` and the rolling update see
# a new replica as healthy about a second after it is ready.
# The start interval lets a rolling update see a new replica healthy within
# about a second.
HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \
CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1
ENTRYPOINT ["/usr/local/bin/server"]
+47
View File
@@ -0,0 +1,47 @@
# Load-test override of compose.yaml, used only by scripts/bench: every value
# an experiment varies, and the load generator. Defaults equal compose.yaml.
services:
lb:
environment:
LB_QUIET: ${LB_QUIET:-0}
API_MAXCONN: ${API_MAXCONN:-20}
deploy:
resources:
limits:
cpus: "${LB_CPUS:-0.5}"
api:
environment:
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB}
APP_DEBUG_ADDR: ":9090"
deploy:
resources:
limits:
cpus: "${API_CPUS:-1.0}"
memory: ${API_MEMORY:-256M}
postgres:
deploy:
resources:
limits:
cpus: "${PG_CPUS:-1.0}"
memory: ${PG_MEMORY:-512M}
loadgen:
profiles: ["bench"]
build:
context: .
target: loadgen
image: mtusi-messenger-loadgen:lab3
environment:
GOMEMLIMIT: 900MiB
volumes:
- ${BENCH_OUT:-./bench-out}:/out
networks:
- edge
deploy:
resources:
limits:
cpus: "${LOADGEN_CPUS:-2.0}"
memory: 1G
+14 -60
View File
@@ -1,12 +1,6 @@
# Local stand: HAProxy in front of interchangeable API replicas over shared
# PostgreSQL (users, chats, messages, idempotency keys) and Valkey (sessions).
# Only the balancer is published, on loopback. Credentials are files in
# ./secrets created by `make secrets`; named volumes keep the data across
# restarts, and only `make destroy` removes them.
#
# Two networks: edge joins the balancer and the API, data joins the API, the
# migrations and the stores and has no way out. The balancer cannot reach the
# stores, and the stores cannot reach anything outside.
# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data
# network has no way out. The load-test knobs are in compose.bench.yaml.
name: lab3
services:
@@ -14,21 +8,13 @@ services:
image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
environment:
# 1 silences the log line of every successful request during load runs.
LB_QUIET: ${LB_QUIET:-0}
# One HAProxy thread per CPU of the limit below.
LB_THREADS: ${LB_THREADS:-1}
ports:
# Loopback only; override with PUBLISH=127.0.0.1:28080 when 8080 is busy.
- "${PUBLISH:-127.0.0.1:8080}:8080"
# Read-only statistics page: state and checks of every replica.
- "${STATS_PUBLISH:-127.0.0.1:8404}:8404"
networks:
- edge
healthcheck:
# Through the balancer itself: healthy once it can route to a ready
# replica, so `compose up --wait` means "ready through the entry point".
# Through the balancer: `up --wait` returns once a replica is reachable.
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
@@ -38,14 +24,12 @@ services:
deploy:
resources:
limits:
cpus: "${LB_CPUS:-0.5}"
cpus: "0.5"
memory: 128M
api:
build: .
image: mtusi-messenger:lab3
# No container_name and no ports: the service scales, and clients reach
# the replicas only through the balancer.
env_file:
- path: .env
required: false
@@ -54,13 +38,8 @@ services:
SESSION_STORE_URL_FILE: /run/secrets/session_store_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
# Above the balancer's detection time of 1.5 s (haproxy.cfg).
APP_DRAIN_DELAY: ${APP_DRAIN_DELAY:-2s}
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
# About 90 % of the memory limit below; Go does not read the cgroup limit.
GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB}
# Empty disables the internal expvar listener; the benchmark sets :9090.
APP_DEBUG_ADDR: ${APP_DEBUG_ADDR:-}
# Go does not read the cgroup memory limit.
GOMEMLIMIT: 230MiB
secrets:
- database_url
- session_store_url
@@ -74,16 +53,15 @@ services:
condition: service_healthy
valkey:
condition: service_healthy
# Drain delay 2 s + shutdown timeout 8 s fit before Docker sends SIGKILL.
# Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL.
stop_grace_period: 15s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPUS:-1.0}"
memory: ${API_MEMORY:-256M}
cpus: "1.0"
memory: 256M
# One-shot: applies the schema once before the replicas start, then exits.
migrate:
image: mtusi-messenger:lab3
entrypoint: ["/usr/local/bin/migrate"]
@@ -108,8 +86,7 @@ services:
postgres:
image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873
# Ten replicas with ten connections each fit during a rolling update of
# five, plus migrate and psql sessions.
# 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql.
command: ["postgres", "-c", "max_connections=150"]
environment:
POSTGRES_DB: messenger
@@ -117,28 +94,24 @@ services:
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
# Since PostgreSQL 18 the image keeps PGDATA in a versioned directory
# below /var/lib/postgresql, so the volume is mounted on the parent.
# PostgreSQL 18 keeps PGDATA in a versioned directory below this one.
volumes:
- pgdata:/var/lib/postgresql
networks:
- data
healthcheck:
# Over TCP: during first initialisation the image runs a temporary
# server on the Unix socket only, which must not count as ready.
# Over TCP: the first initialisation runs a temporary socket-only server.
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
# A clean shutdown writes a checkpoint instead of leaving crash recovery
# to the next start.
stop_grace_period: 30s
deploy:
resources:
limits:
cpus: "${PG_CPUS:-1.0}"
memory: ${PG_MEMORY:-512M}
cpus: "1.0"
memory: 512M
valkey:
image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
@@ -165,25 +138,6 @@ services:
# maxmemory 256mb (valkey.conf) plus the server's own overhead.
memory: 320M
# Load generator of the experiments (tools/loadgen). Only
# `docker compose --profile bench run --rm loadgen ...` starts it; it talks to
# the balancer and, for metrics, to the replicas' internal debug port.
loadgen:
profiles: ["bench"]
build:
context: .
target: loadgen
image: mtusi-messenger-loadgen:lab3
volumes:
- ${BENCH_OUT:-./bench-out}:/out
networks:
- edge
deploy:
resources:
limits:
cpus: "${LOADGEN_CPUS:-1.5}"
memory: 512M
volumes:
pgdata:
valkeydata:
+20 -29
View File
@@ -1,45 +1,39 @@
# The single entry point: round-robin over the API replicas found through
# Docker DNS, with active readiness checks and no client stickiness. Replicas
# are interchangeable: sessions live in Valkey, everything else in PostgreSQL.
# Docker DNS, with active readiness checks and no client stickiness.
global
log stdout format raw local0 info
maxconn 20000
# One thread per CPU of the container limit; by default HAProxy would
# start one per CPU of the host and fight the CPU quota.
nbthread "${LB_THREADS}"
# The container has at most one CPU; by default HAProxy starts a thread
# per host CPU and fights the quota.
nbthread 1
defaults
mode http
log global
option dontlognull
# Inside the Docker network a replica accepts or refuses at once; a
# second means it is gone.
timeout connect 1s
timeout client 30s
# Above the API's own 5 s request deadline: a slow request ends in the
# API's 503 with its cause logged, and this timeout only cuts off a
# replica that hangs entirely.
# Above the API's 5 s request deadline: the API answers a slow request
# itself, this only cuts off a hung replica.
timeout server 7s
timeout http-request 10s
timeout http-keep-alive 10s
timeout check 2s
# A request is re-sent, to another replica, only when the connection
# failed and so nothing reached the API. A timeout or an empty response
# may follow a committed write, and POST /users or /chats must never be
# repeated behind the client's back (contract, idempotency section).
# Below the 500 ms latency objective: excess load gets a fast 503.
timeout queue 300ms
# Re-sent only when nothing reached the API: a timeout or an empty
# response may follow a committed write.
retries 2
option redispatch 1
retry-on conn-failure
# Errors produced by the balancer itself use the contract's envelope. They
# carry X-Request-Id but no X-Instance-Id: no replica answered.
# Errors produced by the balancer itself use the contract's envelope.
http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]"
http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]"
http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]"
# Docker's embedded DNS: the service name `api` resolves to every running
# replica. Re-resolved every second, so scaled, restarted or replaced
# replicas join and stopped ones leave without a reload.
# `api` resolves to every running replica, re-resolved every second; an
# address that disappears leaves at once.
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
@@ -52,10 +46,9 @@ resolvers docker
frontend entry
bind :8080
option httplog
# 32 hex characters, like the request ids of the API.
unique-id-format %[uuid(4),regsub(-,,g)]
# The access log names the replica that answered and its request id, which
# is the key to the replica's own log line.
# Replaces any client value; the API keeps it, so one id spans both logs.
http-request set-header X-Request-Id %[unique-id]
capture response header X-Instance-Id len 32
capture response header X-Request-Id len 40
http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 }
@@ -66,13 +59,11 @@ backend replicas
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
# Detection: a replica that fails readiness is out after the next check
# (inter 1 s) plus one fast re-check (fastinter 0.5 s), so within 1.5 s.
# A stopping API keeps serving for APP_DRAIN_DELAY (2 s) after readiness
# turns 503, longer than this, and only then closes its listener. Back in
# rotation after two good checks (rise 2), about 2 s.
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4
# Ten slots: up to five replicas, doubled while docker rollout replaces them.
# Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay.
# maxconn 20: the replica's 10 pooled database connections plus requests
# that need no database; the rest waits in the queue. 0 means no limit.
default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4
# Up to five replicas, doubled while docker rollout replaces them.
server-template api 10 api:8080
listen stats
+2 -10
View File
@@ -1,13 +1,6 @@
-- +goose Up
-- One row per Idempotency-Key a user has sent to one chat. The contract scope
-- is user + method + chat path + key; only POST /chats/{chat_id}/messages
-- accepts keys, so (user_id, chat_id) already fixes the method and the path.
-- The primary key is what makes concurrent retries through different
-- replicas safe: the second insert of a key waits for the first one's commit.
-- The row references the message instead of storing a response: messages
-- are immutable, so a replay is answered from the source of truth.
-- The table only adds to the schema: the previous API version keeps working
-- while a rolling update runs this migration.
-- Keys of POST /chats/{chat_id}/messages: the contract scope user + method +
-- chat path + key is (user_id, chat_id, key), as no other endpoint takes keys.
CREATE TABLE message_idempotency_keys (
user_id uuid NOT NULL,
chat_id uuid NOT NULL,
@@ -18,7 +11,6 @@ CREATE TABLE message_idempotency_keys (
FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id)
);
-- Serves the periodic removal of keys older than the retention period.
CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at);
-- +goose Down
+3
View File
@@ -1,5 +1,8 @@
# Shared helpers of the experiment scripts; sourced, not run.
# The generator and the knobs the experiments vary live in compose.bench.yaml.
export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml
# abspath prints the absolute path of an existing directory in the form the
# Docker CLI accepts for a bind mount (a Windows path under Git Bash).
abspath() {