From e9bc51288bf225f18516e0cc11996434a6812b4d Mon Sep 17 00:00:00 2001 From: aegor Date: Sun, 1 Nov 2026 14:50:16 +0300 Subject: [PATCH] feat: add haproxy load balancer, api replicas and graceful shutdown --- Dockerfile | 4 ++-- compose.yaml | 19 ++++++++++++--- infra/haproxy/haproxy.cfg | 41 +++++++++++++++++++++++++++++++++ scripts/restart.sh | 4 ++-- scripts/stop-one.sh | 10 ++++++++ src/health/health.controller.ts | 4 ++++ src/health/shutdown.ts | 26 +++++++++++++++++++++ src/main.ts | 3 ++- 8 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 infra/haproxy/haproxy.cfg create mode 100755 scripts/stop-one.sh create mode 100644 src/health/shutdown.ts diff --git a/Dockerfile b/Dockerfile index e433a411..8cc8ac25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ COPY contracts/openapi.json ./contracts/openapi.json USER node EXPOSE 8080 -HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ - CMD node -e "require('http').get('http://127.0.0.1:8080/health/live',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))" +HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \ + CMD node -e "require('http').get('http://127.0.0.1:8080/health/ready',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))" CMD ["node", "dist/main.js"] diff --git a/compose.yaml b/compose.yaml index 0f296350..d7b3c185 100644 --- a/compose.yaml +++ b/compose.yaml @@ -78,12 +78,14 @@ services: APP_ENV: ${APP_ENV:-lab} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SHUTDOWN_DELAY_MS: 3000 DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url - ports: - - "127.0.0.1:${APP_PORT:-8080}:8080" volumes: - secrets:/run/secrets:ro + deploy: + replicas: ${API_REPLICAS:-2} + stop_grace_period: 10s depends_on: migrate: condition: service_completed_successfully @@ -91,6 +93,17 @@ services: condition: service_started restart: unless-stopped + lb: + image: haproxy:3.0.6-alpine + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + restart: unless-stopped + tools: profiles: [tools] build: @@ -100,7 +113,7 @@ services: WORKDIR /work image: mtusi-messenger-lab3-tools:dev environment: - BASE_URL: http://api:8080 + BASE_URL: http://lb:8080 volumes: - .:/work - /var/run/docker.sock:/var/run/docker.sock diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..d8c9285d --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,41 @@ +global + log stdout format raw local0 info + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout queue 5s + timeout check 1s + retries 2 + option redispatch 1 + retry-on conn-failure + unique-id-format %[uuid()] + http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"bad gateway\",\"request_id\":\"%[unique-id]\"}}" + http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no API replica available\",\"request_id\":\"%[unique-id]\"}}" + http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"gateway timeout\",\"request_id\":\"%[unique-id]\"}}" + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold obsolete 5s + accepted_payload_size 8192 + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 1-4 api:8080 diff --git a/scripts/restart.sh b/scripts/restart.sh index 7ff5ed98..783157a4 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,8 +2,8 @@ set -eu cd "$(dirname "$0")/.." started() { - docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -q postgres redis api) + docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb) } started -docker compose restart postgres redis api +docker compose restart postgres redis api lb started diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..7ce356ce --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +labels=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$TARGET_INSTANCE_ID") +if [ "$labels" != "messenger-lab3/api" ]; then + echo "refusing to stop $TARGET_INSTANCE_ID: not an api replica of messenger-lab3 ($labels)" >&2 + exit 1 +fi +docker stop -t 8 "$TARGET_INSTANCE_ID" >/dev/null +docker inspect -f '{{.Name}} {{.State.Status}} finished={{.State.FinishedAt}}' "$TARGET_INSTANCE_ID" diff --git a/src/health/health.controller.ts b/src/health/health.controller.ts index f691b565..2a58c44e 100644 --- a/src/health/health.controller.ts +++ b/src/health/health.controller.ts @@ -1,6 +1,7 @@ import { Controller, Get } from '@nestjs/common'; import { ApiError } from '../common/filters/api-error'; import { PrismaService } from '../db/prisma.service'; +import { isDraining } from './shutdown'; @Controller('health') export class HealthController { @@ -13,6 +14,9 @@ export class HealthController { @Get('ready') async ready(): Promise<{ status: 'ok' }> { + if (isDraining()) { + throw new ApiError('unavailable', 'shutting down'); + } try { await this.prisma.$queryRaw`SELECT 1`; } catch { diff --git a/src/health/shutdown.ts b/src/health/shutdown.ts new file mode 100644 index 00000000..329f27da --- /dev/null +++ b/src/health/shutdown.ts @@ -0,0 +1,26 @@ +import { INestApplication, Logger } from '@nestjs/common'; + +let draining = false; + +export function isDraining(): boolean { + return draining; +} + +export function enableGracefulShutdown(app: INestApplication): void { + const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000); + const shutdown = (signal: string): void => { + if (draining) { + return; + } + draining = true; + Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown'); + setTimeout(() => { + app.close().then( + () => process.exit(0), + () => process.exit(1), + ); + }, delayMs); + }; + process.once('SIGTERM', () => shutdown('SIGTERM')); + process.once('SIGINT', () => shutdown('SIGINT')); +} diff --git a/src/main.ts b/src/main.ts index a71214b0..4e6043cf 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,6 +7,7 @@ import { import { AppModule } from './app.module'; import { configureApp } from './configure-app'; import { setupSwagger } from './docs/swagger'; +import { enableGracefulShutdown } from './health/shutdown'; async function bootstrap(): Promise { const app = await NestFactory.create( @@ -19,7 +20,7 @@ async function bootstrap(): Promise { configureApp(app); setupSwagger(app); - app.enableShutdownHooks(); + enableGracefulShutdown(app); const host = process.env.APP_HOST ?? '0.0.0.0'; const port = Number(process.env.APP_PORT ?? 8080);