forked from mtusi2026/mtusi2026lab3
lab3: стенд HAProxy + реплики под compose, лимиты, хуки остановки и квиесценции
- compose.yaml: проект mtusi-lab3, HAProxy как единственная точка входа,
deploy.replicas ${API_REPLICAS:-2}, CPU/RAM-лимиты на каждый сервис,
stop_grace_period 20s, лимиты из переменных (API_CPUS_PER_REPLICA и т.д.),
loopback-порты PostgreSQL/Redis только для операторских проверок; у API
портов нет; container_name нигде не задан.
- infra/haproxy/haproxy.cfg: проба GET /internal/lb-check, inter 500ms/fall 2
(~1s обнаружение), server-template + DNS Docker вместо статического списка,
stats на 127.0.0.1:8404.
- Дрейн: SIGTERM -> реплика сразу перестаёт проходить пробу LB -> пауза
DRAIN_DELAY_MS -> закрытие сервера и пулов. Nest уничтожает провайдеров
раньше beforeApplicationShutdown, поэтому последовательность вынесена в
main.ts, а HealthService отвечает только за флаг not-ready.
- scripts/stop-one.sh (остановка ровно одной реплики по X-Instance-Id),
scripts/start-replica.sh (восстановление до API_REPLICAS), scripts/quiesce.sh
(остановка/возврат посторонних контейнеров хоста), restart.sh перезапускает
и LB, scripts/app-e2e.sh (host-side e2e через loopback-порты).
- Makefile: stand-up/down/status, stand-replica, wait-ready, quiesce-up/down,
test-e2e, test-failover-all, rolling-update, track1, evidence.
- Dockerfile: ARG/ENV BUILD_ID для доказательства rolling update.
Проверено: make test, make test-e2e, make test-persistence (restart.sh),
make test-failover (stop-one.sh, восстановление через start-replica.sh),
make test-dependencies; docker inspect подтверждает NanoCpus/Memory и
отсутствие опубликованных портов у api.
This commit is contained in:
@@ -6,11 +6,9 @@ describe('HealthService', () => {
|
||||
it('reports liveness and readiness when dependencies are healthy', async () => {
|
||||
const database = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const redis = { health: vi.fn().mockResolvedValue(undefined) };
|
||||
const config = { get: vi.fn().mockReturnValue(0) };
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
config as never,
|
||||
);
|
||||
expect(service.getLiveness()).toEqual({ status: 'ok' });
|
||||
await expect(service.getReadiness()).resolves.toEqual({ status: 'ok' });
|
||||
@@ -24,7 +22,6 @@ describe('HealthService', () => {
|
||||
const service = new HealthService(
|
||||
database as never,
|
||||
redis as never,
|
||||
{ get: vi.fn() } as never,
|
||||
);
|
||||
await expect(service.getReadiness()).rejects.toBeInstanceOf(
|
||||
ServiceUnavailableException,
|
||||
@@ -35,7 +32,6 @@ describe('HealthService', () => {
|
||||
const service = new HealthService(
|
||||
{ health: vi.fn() } as never,
|
||||
{ health: vi.fn() } as never,
|
||||
{ get: vi.fn() } as never,
|
||||
);
|
||||
service.setDraining();
|
||||
expect(service.getLiveness()).toEqual({ status: 'ok' });
|
||||
@@ -48,7 +44,6 @@ describe('HealthService', () => {
|
||||
const service = new HealthService(
|
||||
{ health: vi.fn().mockRejectedValue(new Error('down')) } as never,
|
||||
{ health: vi.fn().mockRejectedValue(new Error('down')) } as never,
|
||||
{ get: vi.fn() } as never,
|
||||
);
|
||||
expect(service.getLbCheck()).toEqual({ status: 'ok' });
|
||||
service.setDraining();
|
||||
|
||||
@@ -4,10 +4,8 @@ import {
|
||||
Module,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { DatabaseClient } from '../database/database.client.js';
|
||||
import { RedisClient } from '../redis/redis.client.js';
|
||||
import type { AppConfig } from '../config/app-config.schema.js';
|
||||
|
||||
@Injectable()
|
||||
export class HealthService {
|
||||
@@ -15,7 +13,6 @@ export class HealthService {
|
||||
constructor(
|
||||
private readonly database: DatabaseClient,
|
||||
private readonly redis: RedisClient,
|
||||
private readonly config: ConfigService<AppConfig>,
|
||||
) {}
|
||||
getLiveness() {
|
||||
return { status: 'ok' as const };
|
||||
@@ -40,16 +37,13 @@ export class HealthService {
|
||||
}
|
||||
return { status: 'ok' as const };
|
||||
}
|
||||
// Called on SIGTERM before the drain delay: the replica turns not-ready for
|
||||
// both the LB probe and the operator probe while it finishes in-flight
|
||||
// requests. The delay itself lives in main.ts, because Nest destroys
|
||||
// providers (closing the pools) before this class could serve anyone.
|
||||
setDraining() {
|
||||
this.ready = false;
|
||||
}
|
||||
async beforeApplicationShutdown(signal?: string): Promise<void> {
|
||||
this.setDraining();
|
||||
const delay = this.config.get<number>('DRAIN_DELAY_MS') ?? 5000;
|
||||
if (signal && delay > 0) {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Global()
|
||||
|
||||
+24
-1
@@ -7,6 +7,7 @@ import {
|
||||
NestFastifyApplication,
|
||||
} from '@nestjs/platform-fastify';
|
||||
import { AppModule } from './app.module.js';
|
||||
import { HealthService } from './health/health.service.js';
|
||||
import { MetricsService } from './metrics/metrics.service.js';
|
||||
import { RequestIdInterceptor } from './common/http/request-id.interceptor.js';
|
||||
|
||||
@@ -29,12 +30,34 @@ async function bootstrap(): Promise<void> {
|
||||
transform: true,
|
||||
}),
|
||||
);
|
||||
app.enableShutdownHooks();
|
||||
|
||||
const config = app.get(ConfigService);
|
||||
const port = config.getOrThrow<number>('APP_PORT');
|
||||
const host = config.getOrThrow<string>('APP_HOST');
|
||||
await app.listen(port, host);
|
||||
|
||||
// Drain before teardown. Nest's own shutdown sequence destroys providers
|
||||
// (closing the database pool and Redis) before `beforeApplicationShutdown`
|
||||
// runs, so relying on it would keep the replica listed in the load balancer
|
||||
// while it can no longer serve: SIGTERM must first turn the replica
|
||||
// not-ready for the LB probe, then wait DRAIN_DELAY_MS, and only then close
|
||||
// the server and the pools.
|
||||
const health = app.get(HealthService);
|
||||
const drainDelayMs = config.getOrThrow<number>('DRAIN_DELAY_MS');
|
||||
let shuttingDown = false;
|
||||
const shutdown = async (): Promise<void> => {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
health.setDraining();
|
||||
if (drainDelayMs > 0)
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, drainDelayMs));
|
||||
await app.close();
|
||||
process.exit(0);
|
||||
};
|
||||
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const)
|
||||
process.once(signal, () => {
|
||||
void shutdown();
|
||||
});
|
||||
}
|
||||
|
||||
await bootstrap();
|
||||
|
||||
Reference in New Issue
Block a user