forked from mtusi2026/mtusi2026lab3
feat: add haproxy load balancer, api replicas and graceful shutdown
This commit is contained in:
+2
-2
@@ -38,7 +38,7 @@ COPY contracts/openapi.json ./contracts/openapi.json
|
||||
USER node
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD node -e "require('http').get('http://127.0.0.1:8080/health/live',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
|
||||
HEALTHCHECK --interval=5s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD node -e "require('http').get('http://127.0.0.1:8080/health/ready',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
|
||||
|
||||
CMD ["node", "dist/main.js"]
|
||||
|
||||
+16
-3
@@ -78,12 +78,14 @@ services:
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_DELAY_MS: 3000
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- secrets:/run/secrets:ro
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
stop_grace_period: 10s
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
@@ -91,6 +93,17 @@ services:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.6-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
tools:
|
||||
profiles: [tools]
|
||||
build:
|
||||
@@ -100,7 +113,7 @@ services:
|
||||
WORKDIR /work
|
||||
image: mtusi-messenger-lab3-tools:dev
|
||||
environment:
|
||||
BASE_URL: http://api:8080
|
||||
BASE_URL: http://lb:8080
|
||||
volumes:
|
||||
- .:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout queue 5s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
option redispatch 1
|
||||
retry-on conn-failure
|
||||
unique-id-format %[uuid()]
|
||||
http-error status 502 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"bad gateway\",\"request_id\":\"%[unique-id]\"}}"
|
||||
http-error status 503 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"no API replica available\",\"request_id\":\"%[unique-id]\"}}"
|
||||
http-error status 504 content-type application/json hdr X-Request-Id "%[unique-id]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"gateway timeout\",\"request_id\":\"%[unique-id]\"}}"
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 5s
|
||||
hold obsolete 5s
|
||||
accepted_payload_size 8192
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
server-template api 1-4 api:8080
|
||||
+2
-2
@@ -2,8 +2,8 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
started() {
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -q postgres redis api)
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb)
|
||||
}
|
||||
started
|
||||
docker compose restart postgres redis api
|
||||
docker compose restart postgres redis api lb
|
||||
started
|
||||
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
labels=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$TARGET_INSTANCE_ID")
|
||||
if [ "$labels" != "messenger-lab3/api" ]; then
|
||||
echo "refusing to stop $TARGET_INSTANCE_ID: not an api replica of messenger-lab3 ($labels)" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop -t 8 "$TARGET_INSTANCE_ID" >/dev/null
|
||||
docker inspect -f '{{.Name}} {{.State.Status}} finished={{.State.FinishedAt}}' "$TARGET_INSTANCE_ID"
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { ApiError } from '../common/filters/api-error';
|
||||
import { PrismaService } from '../db/prisma.service';
|
||||
import { isDraining } from './shutdown';
|
||||
|
||||
@Controller('health')
|
||||
export class HealthController {
|
||||
@@ -13,6 +14,9 @@ export class HealthController {
|
||||
|
||||
@Get('ready')
|
||||
async ready(): Promise<{ status: 'ok' }> {
|
||||
if (isDraining()) {
|
||||
throw new ApiError('unavailable', 'shutting down');
|
||||
}
|
||||
try {
|
||||
await this.prisma.$queryRaw`SELECT 1`;
|
||||
} catch {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { INestApplication, Logger } from '@nestjs/common';
|
||||
|
||||
let draining = false;
|
||||
|
||||
export function isDraining(): boolean {
|
||||
return draining;
|
||||
}
|
||||
|
||||
export function enableGracefulShutdown(app: INestApplication): void {
|
||||
const delayMs = Number(process.env.SHUTDOWN_DELAY_MS ?? 3000);
|
||||
const shutdown = (signal: string): void => {
|
||||
if (draining) {
|
||||
return;
|
||||
}
|
||||
draining = true;
|
||||
Logger.log(`${signal}: readiness → 503, closing in ${delayMs} ms`, 'Shutdown');
|
||||
setTimeout(() => {
|
||||
app.close().then(
|
||||
() => process.exit(0),
|
||||
() => process.exit(1),
|
||||
);
|
||||
}, delayMs);
|
||||
};
|
||||
process.once('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.once('SIGINT', () => shutdown('SIGINT'));
|
||||
}
|
||||
+2
-1
@@ -7,6 +7,7 @@ import {
|
||||
import { AppModule } from './app.module';
|
||||
import { configureApp } from './configure-app';
|
||||
import { setupSwagger } from './docs/swagger';
|
||||
import { enableGracefulShutdown } from './health/shutdown';
|
||||
|
||||
async function bootstrap(): Promise<void> {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
@@ -19,7 +20,7 @@ async function bootstrap(): Promise<void> {
|
||||
|
||||
configureApp(app);
|
||||
setupSwagger(app);
|
||||
app.enableShutdownHooks();
|
||||
enableGracefulShutdown(app);
|
||||
|
||||
const host = process.env.APP_HOST ?? '0.0.0.0';
|
||||
const port = Number(process.env.APP_PORT ?? 8080);
|
||||
|
||||
Reference in New Issue
Block a user