From ce47c138e4744d81be2cf9934bcd50f59cf027ec Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 12:23:14 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B0=D0=B7=D0=B4=D0=B5=D0=BB=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=81=D1=82=D0=B5=D0=BD=D0=B4=20=D0=B8=20=D0=BD?= =?UTF-8?q?=D0=B0=D1=81=D1=82=D1=80=D0=BE=D0=B9=D0=BA=D0=B8=20=D0=BD=D0=B0?= =?UTF-8?q?=D0=B3=D1=80=D1=83=D0=B7=D0=BE=D1=87=D0=BD=D1=8B=D1=85=20=D1=8D?= =?UTF-8?q?=D0=BA=D1=81=D0=BF=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=BE=D0=B2,=20=D0=BE=D0=B3=D1=80=D0=B0=D0=BD=D0=B8=D1=87?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D1=87=D0=B8=D1=81=D0=BB=D0=BE=20=D0=B7?= =?UTF-8?q?=D0=B0=D0=BF=D1=80=D0=BE=D1=81=D0=BE=D0=B2=20=D0=BD=D0=B0=20?= =?UTF-8?q?=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D1=83=20=D0=B2=20HAProxy?= =?UTF-8?q?=20=D0=B8=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B4=D0=B0=D0=B2=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20X-Request-Id=20=D0=BE=D1=82=20=D0=B1=D0=B0=D0=BB?= =?UTF-8?q?=D0=B0=D0=BD=D1=81=D0=B8=D1=80=D0=BE=D0=B2=D1=89=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B2=20API,=20=D1=81=D0=BE=D0=BA=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82?= =?UTF-8?q?=D0=B0=D1=80=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 + Dockerfile | 18 ++--- compose.bench.yaml | 47 ++++++++++++ compose.yaml | 74 ++++--------------- infra/haproxy/haproxy.cfg | 49 +++++------- migrations/00002_message_idempotency_keys.sql | 12 +-- scripts/bench/lib.sh | 3 + 7 files changed, 94 insertions(+), 111 deletions(-) create mode 100644 compose.bench.yaml diff --git a/.dockerignore b/.dockerignore index c872c535..95cf4c12 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,3 +19,5 @@ node_modules evidence .DS_Store compose*.yaml +bench-out +bin diff --git a/Dockerfile b/Dockerfile index f32eecf5..2a0b1902 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,5 @@ -# Build stage: compile static binaries with the exact toolchain pinned in go.mod. -# Modules live in their own layer, rebuilt only when go.mod or go.sum change. -# Only the Go sources are copied after it, and the compiler's cache persists -# between builds: a change elsewhere in the repository does not trigger a full -# recompilation, which matters when a rolling update rebuilds the image. +# Modules in their own layer and a persistent build cache keep the rebuild of +# a rolling update short. FROM golang:1.27.1-alpine3.23@sha256:d9e2f2f07b10cc922da3e80e035c3058810b328d5aef82d2c63680967c5e2ec9 AS build WORKDIR /src COPY go.mod go.sum ./ @@ -13,8 +10,7 @@ COPY migrations ./migrations RUN --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOFLAGS=-mod=readonly go build -trimpath -ldflags="-s -w" -o /out/ ./cmd/server ./cmd/migrate -# Load generator of the experiments, an image of its own: the API image does -# not carry it. Built only for `docker compose --profile bench`. +# The load generator gets an image of its own; the API image does not carry it. FROM build AS loadgen-build COPY tools ./tools RUN --mount=type=cache,target=/root/.cache/go-build \ @@ -26,9 +22,7 @@ COPY --from=loadgen-build /out/loadgen /usr/local/bin/loadgen USER app ENTRYPOINT ["/usr/local/bin/loadgen"] -# Runtime stage, the last one so that `build: .` builds it: minimal image, -# unprivileged user, the server is PID 1. The same image runs the one-shot -# migrate service with its own entrypoint. +# Last, so that `build: .` builds it; it also runs the migrate service. FROM alpine:3.23@sha256:fd791d74b68913cbb027c6546007b3f0d3bc45125f797758156952bc2d6daf40 RUN adduser -D -H -u 10001 app COPY --from=build /out/server /out/migrate /usr/local/bin/ @@ -37,8 +31,8 @@ ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ APP_ENV=lab EXPOSE 8080 -# The short start interval lets `compose up --wait` and the rolling update see -# a new replica as healthy about a second after it is ready. +# The start interval lets a rolling update see a new replica healthy within +# about a second. HEALTHCHECK --interval=5s --timeout=2s --start-period=15s --start-interval=1s --retries=3 \ CMD wget -qO- "http://127.0.0.1:${APP_PORT:-8080}/health/ready" >/dev/null || exit 1 ENTRYPOINT ["/usr/local/bin/server"] diff --git a/compose.bench.yaml b/compose.bench.yaml new file mode 100644 index 00000000..0a61825e --- /dev/null +++ b/compose.bench.yaml @@ -0,0 +1,47 @@ +# Load-test override of compose.yaml, used only by scripts/bench: every value +# an experiment varies, and the load generator. Defaults equal compose.yaml. +services: + lb: + environment: + LB_QUIET: ${LB_QUIET:-0} + API_MAXCONN: ${API_MAXCONN:-20} + deploy: + resources: + limits: + cpus: "${LB_CPUS:-0.5}" + + api: + environment: + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} + APP_DEBUG_ADDR: ":9090" + deploy: + resources: + limits: + cpus: "${API_CPUS:-1.0}" + memory: ${API_MEMORY:-256M} + + postgres: + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEMORY:-512M} + + loadgen: + profiles: ["bench"] + build: + context: . + target: loadgen + image: mtusi-messenger-loadgen:lab3 + environment: + GOMEMLIMIT: 900MiB + volumes: + - ${BENCH_OUT:-./bench-out}:/out + networks: + - edge + deploy: + resources: + limits: + cpus: "${LOADGEN_CPUS:-2.0}" + memory: 1G diff --git a/compose.yaml b/compose.yaml index 5e1421eb..09b76e75 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,12 +1,6 @@ # Local stand: HAProxy in front of interchangeable API replicas over shared -# PostgreSQL (users, chats, messages, idempotency keys) and Valkey (sessions). -# Only the balancer is published, on loopback. Credentials are files in -# ./secrets created by `make secrets`; named volumes keep the data across -# restarts, and only `make destroy` removes them. -# -# Two networks: edge joins the balancer and the API, data joins the API, the -# migrations and the stores and has no way out. The balancer cannot reach the -# stores, and the stores cannot reach anything outside. +# PostgreSQL and Valkey. Only the balancer is published, on loopback; the data +# network has no way out. The load-test knobs are in compose.bench.yaml. name: lab3 services: @@ -14,21 +8,13 @@ services: image: haproxy:3.4.7-alpine3.24@sha256:2caaf8b1a1ef67f3f1ea9b721f2821675d3ba4500fb518e23884fecb93e8dbaa volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro - environment: - # 1 silences the log line of every successful request during load runs. - LB_QUIET: ${LB_QUIET:-0} - # One HAProxy thread per CPU of the limit below. - LB_THREADS: ${LB_THREADS:-1} ports: - # Loopback only; override with PUBLISH=127.0.0.1:28080 when 8080 is busy. - "${PUBLISH:-127.0.0.1:8080}:8080" - # Read-only statistics page: state and checks of every replica. - "${STATS_PUBLISH:-127.0.0.1:8404}:8404" networks: - edge healthcheck: - # Through the balancer itself: healthy once it can route to a ready - # replica, so `compose up --wait` means "ready through the entry point". + # Through the balancer: `up --wait` returns once a replica is reachable. test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] interval: 5s timeout: 3s @@ -38,14 +24,12 @@ services: deploy: resources: limits: - cpus: "${LB_CPUS:-0.5}" + cpus: "0.5" memory: 128M api: build: . image: mtusi-messenger:lab3 - # No container_name and no ports: the service scales, and clients reach - # the replicas only through the balancer. env_file: - path: .env required: false @@ -54,13 +38,8 @@ services: SESSION_STORE_URL_FILE: /run/secrets/session_store_url SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - # Above the balancer's detection time of 1.5 s (haproxy.cfg). - APP_DRAIN_DELAY: ${APP_DRAIN_DELAY:-2s} - DB_MAX_CONNS: ${DB_MAX_CONNS:-10} - # About 90 % of the memory limit below; Go does not read the cgroup limit. - GOMEMLIMIT: ${API_GOMEMLIMIT:-230MiB} - # Empty disables the internal expvar listener; the benchmark sets :9090. - APP_DEBUG_ADDR: ${APP_DEBUG_ADDR:-} + # Go does not read the cgroup memory limit. + GOMEMLIMIT: 230MiB secrets: - database_url - session_store_url @@ -74,16 +53,15 @@ services: condition: service_healthy valkey: condition: service_healthy - # Drain delay 2 s + shutdown timeout 8 s fit before Docker sends SIGKILL. + # Drain delay 2 s plus shutdown timeout 8 s, before SIGKILL. stop_grace_period: 15s deploy: replicas: ${API_REPLICAS:-2} resources: limits: - cpus: "${API_CPUS:-1.0}" - memory: ${API_MEMORY:-256M} + cpus: "1.0" + memory: 256M - # One-shot: applies the schema once before the replicas start, then exits. migrate: image: mtusi-messenger:lab3 entrypoint: ["/usr/local/bin/migrate"] @@ -108,8 +86,7 @@ services: postgres: image: postgres:18.6-alpine3.24@sha256:77f585114c32fbca283dc835b0596f4e52b51b4c6662d7810b2f4084f60a1873 - # Ten replicas with ten connections each fit during a rolling update of - # five, plus migrate and psql sessions. + # 10 balancer slots x DB_MAX_CONNS 10, plus migrate and psql. command: ["postgres", "-c", "max_connections=150"] environment: POSTGRES_DB: messenger @@ -117,28 +94,24 @@ services: POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password secrets: - postgres_password - # Since PostgreSQL 18 the image keeps PGDATA in a versioned directory - # below /var/lib/postgresql, so the volume is mounted on the parent. + # PostgreSQL 18 keeps PGDATA in a versioned directory below this one. volumes: - pgdata:/var/lib/postgresql networks: - data healthcheck: - # Over TCP: during first initialisation the image runs a temporary - # server on the Unix socket only, which must not count as ready. + # Over TCP: the first initialisation runs a temporary socket-only server. test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] interval: 5s timeout: 3s retries: 10 start_period: 10s - # A clean shutdown writes a checkpoint instead of leaving crash recovery - # to the next start. stop_grace_period: 30s deploy: resources: limits: - cpus: "${PG_CPUS:-1.0}" - memory: ${PG_MEMORY:-512M} + cpus: "1.0" + memory: 512M valkey: image: valkey/valkey:9.1.2-alpine3.24@sha256:48332870af354a799964c0012ae1194a0bf2bf894eb508f945810596dc2d8d11 @@ -165,25 +138,6 @@ services: # maxmemory 256mb (valkey.conf) plus the server's own overhead. memory: 320M - # Load generator of the experiments (tools/loadgen). Only - # `docker compose --profile bench run --rm loadgen ...` starts it; it talks to - # the balancer and, for metrics, to the replicas' internal debug port. - loadgen: - profiles: ["bench"] - build: - context: . - target: loadgen - image: mtusi-messenger-loadgen:lab3 - volumes: - - ${BENCH_OUT:-./bench-out}:/out - networks: - - edge - deploy: - resources: - limits: - cpus: "${LOADGEN_CPUS:-1.5}" - memory: 512M - volumes: pgdata: valkeydata: diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 861a97d8..7cb20e25 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -1,45 +1,39 @@ # The single entry point: round-robin over the API replicas found through -# Docker DNS, with active readiness checks and no client stickiness. Replicas -# are interchangeable: sessions live in Valkey, everything else in PostgreSQL. +# Docker DNS, with active readiness checks and no client stickiness. global log stdout format raw local0 info maxconn 20000 - # One thread per CPU of the container limit; by default HAProxy would - # start one per CPU of the host and fight the CPU quota. - nbthread "${LB_THREADS}" + # The container has at most one CPU; by default HAProxy starts a thread + # per host CPU and fights the quota. + nbthread 1 defaults mode http log global option dontlognull - # Inside the Docker network a replica accepts or refuses at once; a - # second means it is gone. timeout connect 1s timeout client 30s - # Above the API's own 5 s request deadline: a slow request ends in the - # API's 503 with its cause logged, and this timeout only cuts off a - # replica that hangs entirely. + # Above the API's 5 s request deadline: the API answers a slow request + # itself, this only cuts off a hung replica. timeout server 7s timeout http-request 10s timeout http-keep-alive 10s timeout check 2s - # A request is re-sent, to another replica, only when the connection - # failed and so nothing reached the API. A timeout or an empty response - # may follow a committed write, and POST /users or /chats must never be - # repeated behind the client's back (contract, idempotency section). + # Below the 500 ms latency objective: excess load gets a fast 503. + timeout queue 300ms + # Re-sent only when nothing reached the API: a timeout or an empty + # response may follow a committed write. retries 2 option redispatch 1 retry-on conn-failure - # Errors produced by the balancer itself use the contract's envelope. They - # carry X-Request-Id but no X-Instance-Id: no replica answered. + # Errors produced by the balancer itself use the contract's envelope. http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica failed to answer","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"no API replica is available","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"the API replica did not answer in time","request_id":"%[unique-id]"}}' hdr X-Request-Id "%[unique-id]" -# Docker's embedded DNS: the service name `api` resolves to every running -# replica. Re-resolved every second, so scaled, restarted or replaced -# replicas join and stopped ones leave without a reload. +# `api` resolves to every running replica, re-resolved every second; an +# address that disappears leaves at once. resolvers docker nameserver dns 127.0.0.11:53 resolve_retries 3 @@ -52,10 +46,9 @@ resolvers docker frontend entry bind :8080 option httplog - # 32 hex characters, like the request ids of the API. unique-id-format %[uuid(4),regsub(-,,g)] - # The access log names the replica that answered and its request id, which - # is the key to the replica's own log line. + # Replaces any client value; the API keeps it, so one id spans both logs. + http-request set-header X-Request-Id %[unique-id] capture response header X-Instance-Id len 32 capture response header X-Request-Id len 40 http-after-response set-log-level silent if { status lt 500 } { env(LB_QUIET) -m str 1 } @@ -66,13 +59,11 @@ backend replicas option httpchk http-check send meth GET uri /health/ready http-check expect status 200 - # Detection: a replica that fails readiness is out after the next check - # (inter 1 s) plus one fast re-check (fastinter 0.5 s), so within 1.5 s. - # A stopping API keeps serving for APP_DRAIN_DELAY (2 s) after readiness - # turns 503, longer than this, and only then closes its listener. Back in - # rotation after two good checks (rise 2), about 2 s. - default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 - # Ten slots: up to five replicas, doubled while docker rollout replaces them. + # Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay. + # maxconn 20: the replica's 10 pooled database connections plus requests + # that need no database; the rest waits in the queue. 0 means no limit. + default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 + # Up to five replicas, doubled while docker rollout replaces them. server-template api 10 api:8080 listen stats diff --git a/migrations/00002_message_idempotency_keys.sql b/migrations/00002_message_idempotency_keys.sql index 2f35f086..6fe78f6f 100644 --- a/migrations/00002_message_idempotency_keys.sql +++ b/migrations/00002_message_idempotency_keys.sql @@ -1,13 +1,6 @@ -- +goose Up --- One row per Idempotency-Key a user has sent to one chat. The contract scope --- is user + method + chat path + key; only POST /chats/{chat_id}/messages --- accepts keys, so (user_id, chat_id) already fixes the method and the path. --- The primary key is what makes concurrent retries through different --- replicas safe: the second insert of a key waits for the first one's commit. --- The row references the message instead of storing a response: messages --- are immutable, so a replay is answered from the source of truth. --- The table only adds to the schema: the previous API version keeps working --- while a rolling update runs this migration. +-- Keys of POST /chats/{chat_id}/messages: the contract scope user + method + +-- chat path + key is (user_id, chat_id, key), as no other endpoint takes keys. CREATE TABLE message_idempotency_keys ( user_id uuid NOT NULL, chat_id uuid NOT NULL, @@ -18,7 +11,6 @@ CREATE TABLE message_idempotency_keys ( FOREIGN KEY (chat_id, message_id) REFERENCES messages (chat_id, id) ); --- Serves the periodic removal of keys older than the retention period. CREATE INDEX message_idempotency_keys_created_at_idx ON message_idempotency_keys (created_at); -- +goose Down diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index d7b9d48c..2b8c3115 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -1,5 +1,8 @@ # Shared helpers of the experiment scripts; sourced, not run. +# The generator and the knobs the experiments vary live in compose.bench.yaml. +export COMPOSE_PATH_SEPARATOR=: COMPOSE_FILE=compose.yaml:compose.bench.yaml + # abspath prints the absolute path of an existing directory in the form the # Docker CLI accepts for a bind mount (a Windows path under Git Bash). abspath() {