Разделить стенд и настройки нагрузочных экспериментов, ограничить число запросов на реплику в HAProxy и передавать X-Request-Id от балансировщика в API, сократить комментарии
This commit is contained in:
@@ -19,3 +19,5 @@ node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
compose*.yaml
|
||||
bench-out
|
||||
bin
|
||||
|
||||
+6
-12
@@ -1,8 +1,5 @@
|
||||
# Build stage: compile static binaries with the exact toolchain pinned in go.mod.
|
||||
# Modules live in their own layer, rebuilt only when go.mod or go.sum change.
|
||||
# Only the Go sources are copied after it, and the compiler's cache persists
|
||||
# between builds: a change elsewhere in the repository does not trigger a full
|
||||
# recompilation, which matters when a rolling update rebuilds the image.
|
||||
# Modules in their own layer and a persistent build cache keep the rebuild of
|
||||
# a rolling update short.
|
||||
FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
@@ -13,8 +10,7 @@ COPY migrations ./migrations
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate
|
||||
|
||||
# Load generator of the experiments, an image of its own: the API image does
|
||||
# not carry it. Built only for `docker compose --profile bench`.
|
||||
# The load generator gets an image of its own; the API image does not carry it.
|
||||
FROM build AS loadgen-build
|
||||
COPY tools ./tools
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
@@ -26,9 +22,7 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen
|
||||
USER app
|
||||
ENTRYPOINT ["/usr/local/bin/loadgen"]
|
||||
|
||||
# Runtime stage, the last one so that `build: .` builds it: minimal image,
|
||||
# unprivileged user, the server is PID 1. The same image runs the one-shot
|
||||
# migrate service with its own entrypoint.
|
||||
# Last, so that `build: .` builds it; it also runs the migrate service.
|
||||
FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40
|
||||
RUN adduser -D -H -u 10001 app
|
||||
COPY --from=build /out/server /out/migrate /usr/local/bin/
|
||||
@@ -37,8 +31,8 @@ ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab
|
||||
EXPOSE 8080
|
||||
# The short start interval lets `compose up --wait` and the rolling update see
|
||||
# a new replica as healthy about a second after it is ready.
|
||||
# The start interval lets a rolling update see a new replica healthy within
|
||||
# about a second.
|
||||
HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \
|
||||
CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1
|
||||
ENTRYPOINT ["/usr/local/bin/server"]
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# Load-test override of compose.yaml, used only by scripts/bench: every value
|
||||
# an experiment varies, and the load generator. Defaults equal compose.yaml.
|
||||
services:
|
||||
lb:
|
||||
environment:
|
||||
LB_QUIET: ${LB_QUIET:-0}
|
||||
API_MAXCONN: ${API_MAXCONN:-20}
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${LB_CPUS:-0.5}"
|
||||
|
||||
api:
|
||||
environment:
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
|
||||
GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB}
|
||||
APP_DEBUG_ADDR: ":9090"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-1.0}"
|
||||
memory: ${API_MEMORY:-256M}
|
||||
|
||||
postgres:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${PG_CPUS:-1.0}"
|
||||
memory: ${PG_MEMORY:-512M}
|
||||
|
||||
loadgen:
|
||||
profiles: ["bench"]
|
||||
build:
|
||||
context: .
|
||||
target: loadgen
|
||||
image: mtusi-messenger-loadgen:lab3
|
||||
environment:
|
||||
GOMEMLIMIT: 900MiB
|
||||
volumes:
|
||||
- ${BENCH_OUT:-./bench-out}:/out
|
||||
networks:
|
||||
- edge
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${LOADGEN_CPUS:-2.0}"
|
||||
memory: 1G
|
||||
+14
-60
@@ -1,12 +1,6 @@
|
||||
# Local stand: HAProxy in front of interchangeable API replicas over shared
|
||||
# PostgreSQL (users, chats, messages, idempotency keys) and Valkey (sessions).
|
||||
# Only the balancer is published, on loopback. Credentials are files in
|
||||
# ./secrets created by `make secrets`; named volumes keep the data across
|
||||
# restarts, and only `make destroy` removes them.
|
||||
#
|
||||
# Two networks: edge joins the balancer and the API, data joins the API, the
|
||||
# migrations and the stores and has no way out. The balancer cannot reach the
|
||||
# stores, and the stores cannot reach anything outside.
|
||||
# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data
|
||||
# network has no way out. The load-test knobs are in compose.bench.yaml.
|
||||
name: lab3
|
||||
|
||||
services:
|
||||
@@ -14,21 +8,13 @@ services:
|
||||
image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
environment:
|
||||
# 1 silences the log line of every successful request during load runs.
|
||||
LB_QUIET: ${LB_QUIET:-0}
|
||||
# One HAProxy thread per CPU of the limit below.
|
||||
LB_THREADS: ${LB_THREADS:-1}
|
||||
ports:
|
||||
# Loopback only; override with PUBLISH=127.0.0.1:28080 when 8080 is busy.
|
||||
- "${PUBLISH:-127.0.0.1:8080}:8080"
|
||||
# Read-only statistics page: state and checks of every replica.
|
||||
- "${STATS_PUBLISH:-127.0.0.1:8404}:8404"
|
||||
networks:
|
||||
- edge
|
||||
healthcheck:
|
||||
# Through the balancer itself: healthy once it can route to a ready
|
||||
# replica, so `compose up --wait` means "ready through the entry point".
|
||||
# Through the balancer: `up --wait` returns once a replica is reachable.
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
@@ -38,14 +24,12 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${LB_CPUS:-0.5}"
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtusi-messenger:lab3
|
||||
# No container_name and no ports: the service scales, and clients reach
|
||||
# the replicas only through the balancer.
|
||||
env_file:
|
||||
- path: .env
|
||||
required: false
|
||||
@@ -54,13 +38,8 @@ services:
|
||||
SESSION_STORE_URL_FILE: /run/secrets/session_store_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
# Above the balancer's detection time of 1.5 s (haproxy.cfg).
|
||||
APP_DRAIN_DELAY: ${APP_DRAIN_DELAY:-2s}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
|
||||
# About 90 % of the memory limit below; Go does not read the cgroup limit.
|
||||
GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB}
|
||||
# Empty disables the internal expvar listener; the benchmark sets :9090.
|
||||
APP_DEBUG_ADDR: ${APP_DEBUG_ADDR:-}
|
||||
# Go does not read the cgroup memory limit.
|
||||
GOMEMLIMIT: 230MiB
|
||||
secrets:
|
||||
- database_url
|
||||
- session_store_url
|
||||
@@ -74,16 +53,15 @@ services:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
# Drain delay 2 s + shutdown timeout 8 s fit before Docker sends SIGKILL.
|
||||
# Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL.
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-1.0}"
|
||||
memory: ${API_MEMORY:-256M}
|
||||
cpus: "1.0"
|
||||
memory: 256M
|
||||
|
||||
# One-shot: applies the schema once before the replicas start, then exits.
|
||||
migrate:
|
||||
image: mtusi-messenger:lab3
|
||||
entrypoint: ["/usr/local/bin/migrate"]
|
||||
@@ -108,8 +86,7 @@ services:
|
||||
|
||||
postgres:
|
||||
image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873
|
||||
# Ten replicas with ten connections each fit during a rolling update of
|
||||
# five, plus migrate and psql sessions.
|
||||
# 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql.
|
||||
command: ["postgres", "-c", "max_connections=150"]
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
@@ -117,28 +94,24 @@ services:
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
# Since PostgreSQL 18 the image keeps PGDATA in a versioned directory
|
||||
# below /var/lib/postgresql, so the volume is mounted on the parent.
|
||||
# PostgreSQL 18 keeps PGDATA in a versioned directory below this one.
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql
|
||||
networks:
|
||||
- data
|
||||
healthcheck:
|
||||
# Over TCP: during first initialisation the image runs a temporary
|
||||
# server on the Unix socket only, which must not count as ready.
|
||||
# Over TCP: the first initialisation runs a temporary socket-only server.
|
||||
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
# A clean shutdown writes a checkpoint instead of leaving crash recovery
|
||||
# to the next start.
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${PG_CPUS:-1.0}"
|
||||
memory: ${PG_MEMORY:-512M}
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11
|
||||
@@ -165,25 +138,6 @@ services:
|
||||
# maxmemory 256mb (valkey.conf) plus the server's own overhead.
|
||||
memory: 320M
|
||||
|
||||
# Load generator of the experiments (tools/loadgen). Only
|
||||
# `docker compose --profile bench run --rm loadgen ...` starts it; it talks to
|
||||
# the balancer and, for metrics, to the replicas' internal debug port.
|
||||
loadgen:
|
||||
profiles: ["bench"]
|
||||
build:
|
||||
context: .
|
||||
target: loadgen
|
||||
image: mtusi-messenger-loadgen:lab3
|
||||
volumes:
|
||||
- ${BENCH_OUT:-./bench-out}:/out
|
||||
networks:
|
||||
- edge
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${LOADGEN_CPUS:-1.5}"
|
||||
memory: 512M
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
|
||||
+20
-29
@@ -1,45 +1,39 @@
|
||||
# The single entry point: round-robin over the API replicas found through
|
||||
# Docker DNS, with active readiness checks and no client stickiness. Replicas
|
||||
# are interchangeable: sessions live in Valkey, everything else in PostgreSQL.
|
||||
# Docker DNS, with active readiness checks and no client stickiness.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 20000
|
||||
# One thread per CPU of the container limit; by default HAProxy would
|
||||
# start one per CPU of the host and fight the CPU quota.
|
||||
nbthread "${LB_THREADS}"
|
||||
# The container has at most one CPU; by default HAProxy starts a thread
|
||||
# per host CPU and fights the quota.
|
||||
nbthread 1
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option dontlognull
|
||||
# Inside the Docker network a replica accepts or refuses at once; a
|
||||
# second means it is gone.
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
# Above the API's own 5 s request deadline: a slow request ends in the
|
||||
# API's 503 with its cause logged, and this timeout only cuts off a
|
||||
# replica that hangs entirely.
|
||||
# Above the API's 5 s request deadline: the API answers a slow request
|
||||
# itself, this only cuts off a hung replica.
|
||||
timeout server 7s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 10s
|
||||
timeout check 2s
|
||||
# A request is re-sent, to another replica, only when the connection
|
||||
# failed and so nothing reached the API. A timeout or an empty response
|
||||
# may follow a committed write, and POST /users or /chats must never be
|
||||
# repeated behind the client's back (contract, idempotency section).
|
||||
# Below the 500 ms latency objective: excess load gets a fast 503.
|
||||
timeout queue 300ms
|
||||
# Re-sent only when nothing reached the API: a timeout or an empty
|
||||
# response may follow a committed write.
|
||||
retries 2
|
||||
option redispatch 1
|
||||
retry-on conn-failure
|
||||
# Errors produced by the balancer itself use the contract's envelope. They
|
||||
# carry X-Request-Id but no X-Instance-Id: no replica answered.
|
||||
# Errors produced by the balancer itself use the contract's envelope.
|
||||
http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]"
|
||||
http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]"
|
||||
http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]"
|
||||
|
||||
# Docker's embedded DNS: the service name `api` resolves to every running
|
||||
# replica. Re-resolved every second, so scaled, restarted or replaced
|
||||
# replicas join and stopped ones leave without a reload.
|
||||
# `api` resolves to every running replica, re-resolved every second; an
|
||||
# address that disappears leaves at once.
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
@@ -52,10 +46,9 @@ resolvers docker
|
||||
frontend entry
|
||||
bind :8080
|
||||
option httplog
|
||||
# 32 hex characters, like the request ids of the API.
|
||||
unique-id-format %[uuid(4),regsub(-,,g)]
|
||||
# The access log names the replica that answered and its request id, which
|
||||
# is the key to the replica's own log line.
|
||||
# Replaces any client value; the API keeps it, so one id spans both logs.
|
||||
http-request set-header X-Request-Id %[unique-id]
|
||||
capture response header X-Instance-Id len 32
|
||||
capture response header X-Request-Id len 40
|
||||
http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 }
|
||||
@@ -66,13 +59,11 @@ backend replicas
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
# Detection: a replica that fails readiness is out after the next check
|
||||
# (inter 1 s) plus one fast re-check (fastinter 0.5 s), so within 1.5 s.
|
||||
# A stopping API keeps serving for APP_DRAIN_DELAY (2 s) after readiness
|
||||
# turns 503, longer than this, and only then closes its listener. Back in
|
||||
# rotation after two good checks (rise 2), about 2 s.
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4
|
||||
# Ten slots: up to five replicas, doubled while docker rollout replaces them.
|
||||
# Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay.
|
||||
# maxconn 20: the replica's 10 pooled database connections plus requests
|
||||
# that need no database; the rest waits in the queue. 0 means no limit.
|
||||
default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4
|
||||
# Up to five replicas, doubled while docker rollout replaces them.
|
||||
server-template api 10 api:8080
|
||||
|
||||
listen stats
|
||||
|
||||
@@ -1,13 +1,6 @@
|
||||
-- +goose Up
|
||||
-- One row per Idempotency-Key a user has sent to one chat. The contract scope
|
||||
-- is user + method + chat path + key; only POST /chats/{chat_id}/messages
|
||||
-- accepts keys, so (user_id, chat_id) already fixes the method and the path.
|
||||
-- The primary key is what makes concurrent retries through different
|
||||
-- replicas safe: the second insert of a key waits for the first one's commit.
|
||||
-- The row references the message instead of storing a response: messages
|
||||
-- are immutable, so a replay is answered from the source of truth.
|
||||
-- The table only adds to the schema: the previous API version keeps working
|
||||
-- while a rolling update runs this migration.
|
||||
-- Keys of POST /chats/{chat_id}/messages: the contract scope user + method +
|
||||
-- chat path + key is (user_id, chat_id, key), as no other endpoint takes keys.
|
||||
CREATE TABLE message_idempotency_keys (
|
||||
user_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
@@ -18,7 +11,6 @@ CREATE TABLE message_idempotency_keys (
|
||||
FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id)
|
||||
);
|
||||
|
||||
-- Serves the periodic removal of keys older than the retention period.
|
||||
CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at);
|
||||
|
||||
-- +goose Down
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# Shared helpers of the experiment scripts; sourced, not run.
|
||||
|
||||
# The generator and the knobs the experiments vary live in compose.bench.yaml.
|
||||
export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml
|
||||
|
||||
# abspath prints the absolute path of an existing directory in the form the
|
||||
# Docker CLI accepts for a bind mount (a Windows path under Git Bash).
|
||||
abspath() {
|
||||
|
||||
Reference in New Issue
Block a user