feat: add haproxy load balancer, api replicas and graceful shutdown

This commit is contained in:
2026-11-01 14:50:16 +03:00
parent cdcff89d25
commit e9bc51288b
8 changed files with 103 additions and 8 deletions
+2 -2
View File
@@ -38,7 +38,7 @@ COPY contracts/openapi.json ./contracts/openapi.json
USER node
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD node -e "require('http').get('http://127.0.0.1:8080/health/live',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \
CMD node -e "require('http').get('http://127.0.0.1:8080/health/ready',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
CMD ["node", "dist/main.js"]
+16 -3
View File
@@ -78,12 +78,14 @@ services:
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SHUTDOWN_DELAY_MS: 3000
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- secrets:/run/secrets:ro
deploy:
replicas: ${API_REPLICAS:-2}
stop_grace_period: 10s
depends_on:
migrate:
condition: service_completed_successfully
@@ -91,6 +93,17 @@ services:
condition: service_started
restart: unless-stopped
lb:
image: haproxy:3.0.6-alpine
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
restart: unless-stopped
tools:
profiles: [tools]
build:
@@ -100,7 +113,7 @@ services:
WORKDIR /work
image: mtusi-messenger-lab3-tools:dev
environment:
BASE_URL: http://api:8080
BASE_URL: http://lb:8080
volumes:
- .:/work
- /var/run/docker.sock:/var/run/docker.sock
+41
View File
@@ -0,0 +1,41 @@
global
log stdout format raw local0 info
defaults
mode http
log global
option httplog
option dontlognull
timeout connect 1s
timeout client 30s
timeout server 10s
timeout http-request 10s
timeout queue 5s
timeout check 1s
retries 2
option redispatch 1
retry-on conn-failure
unique-id-format %[uuid()]
http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"bad gateway\",\"request_id\":\"%[unique-id]\"}}"
http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no API replica available\",\"request_id\":\"%[unique-id]\"}}"
http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"gateway timeout\",\"request_id\":\"%[unique-id]\"}}"
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 5s
hold obsolete 5s
accepted_payload_size 8192
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
server-template api 1-4 api:8080
+2 -2
View File
@@ -2,8 +2,8 @@
set -eu
cd "$(dirname "$0")/.."
started() {
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -q postgres redis api)
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb)
}
started
docker compose restart postgres redis api
docker compose restart postgres redis api lb
started
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
labels=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$TARGET_INSTANCE_ID")
if [ "$labels" != "messenger-lab3/api" ]; then
echo "refusing to stop $TARGET_INSTANCE_ID: not an api replica of messenger-lab3 ($labels)" >&2
exit 1
fi
docker stop -t 8 "$TARGET_INSTANCE_ID" >/dev/null
docker inspect -f '{{.Name}} {{.State.Status}} finished={{.State.FinishedAt}}' "$TARGET_INSTANCE_ID"
+4
View File
@@ -1,6 +1,7 @@
import { Controller, Get } from '@nestjs/common';
import { ApiError } from '../common/filters/api-error';
import { PrismaService } from '../db/prisma.service';
import { isDraining } from './shutdown';
@Controller('health')
export class HealthController {
@@ -13,6 +14,9 @@ export class HealthController {
@Get('ready')
async ready(): Promise<{ status: 'ok' }> {
if (isDraining()) {
throw new ApiError('unavailable', 'shutting down');
}
try {
await this.prisma.$queryRaw`SELECT 1`;
} catch {
+26
View File
@@ -0,0 +1,26 @@
import { INestApplication, Logger } from '@nestjs/common';
let draining = false;
export function isDraining(): boolean {
return draining;
}
export function enableGracefulShutdown(app: INestApplication): void {
const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000);
const shutdown = (signal: string): void => {
if (draining) {
return;
}
draining = true;
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
setTimeout(() => {
app.close().then(
() => process.exit(0),
() => process.exit(1),
);
}, delayMs);
};
process.once('SIGTERM', () => shutdown('SIGTERM'));
process.once('SIGINT', () => shutdown('SIGINT'));
}
+2 -1
View File
@@ -7,6 +7,7 @@ import {
import { AppModule } from './app.module';
import { configureApp } from './configure-app';
import { setupSwagger } from './docs/swagger';
import { enableGracefulShutdown } from './health/shutdown';
async function bootstrap(): Promise<void> {
const app = await NestFactory.create<NestFastifyApplication>(
@@ -19,7 +20,7 @@ async function bootstrap(): Promise<void> {
configureApp(app);
setupSwagger(app);
app.enableShutdownHooks();
enableGracefulShutdown(app);
const host = process.env.APP_HOST ?? '0.0.0.0';
const port = Number(process.env.APP_PORT ?? 8080);