lab3: стенд HAProxy + реплики под compose, лимиты, хуки остановки и квиесценции

- compose.yaml: проект mtusi-lab3, HAProxy как единственная точка входа,
  deploy.replicas ${API_REPLICAS:-2}, CPU/RAM-лимиты на каждый сервис,
  stop_grace_period 20s, лимиты из переменных (API_CPUS_PER_REPLICA и т.д.),
  loopback-порты PostgreSQL/Redis только для операторских проверок; у API
  портов нет; container_name нигде не задан.
- infra/haproxy/haproxy.cfg: проба GET /internal/lb-check, inter 500ms/fall 2
  (~1s обнаружение), server-template + DNS Docker вместо статического списка,
  stats на 127.0.0.1:8404.
- Дрейн: SIGTERM -> реплика сразу перестаёт проходить пробу LB -> пауза
  DRAIN_DELAY_MS -> закрытие сервера и пулов. Nest уничтожает провайдеров
  раньше beforeApplicationShutdown, поэтому последовательность вынесена в
  main.ts, а HealthService отвечает только за флаг not-ready.
- scripts/stop-one.sh (остановка ровно одной реплики по X-Instance-Id),
  scripts/start-replica.sh (восстановление до API_REPLICAS), scripts/quiesce.sh
  (остановка/возврат посторонних контейнеров хоста), restart.sh перезапускает
  и LB, scripts/app-e2e.sh (host-side e2e через loopback-порты).
- Makefile: stand-up/down/status, stand-replica, wait-ready, quiesce-up/down,
  test-e2e, test-failover-all, rolling-update, track1, evidence.
- Dockerfile: ARG/ENV BUILD_ID для доказательства rolling update.

Проверено: make test, make test-e2e, make test-persistence (restart.sh),
make test-failover (stop-one.sh, восстановление через start-replica.sh),
make test-dependencies; docker inspect подтверждает NanoCpus/Memory и
отсутствие опубликованных портов у api.
This commit is contained in:
2026-10-15 21:14:17 +03:00
parent e9cd28d6ba
commit 917ed1a734
14 changed files with 464 additions and 82 deletions
-5
View File
@@ -6,11 +6,9 @@ describe('HealthService', () => {
it('reports liveness and readiness when dependencies are healthy', async () => {
const database = { health: vi.fn().mockResolvedValue(undefined) };
const redis = { health: vi.fn().mockResolvedValue(undefined) };
const config = { get: vi.fn().mockReturnValue(0) };
const service = new HealthService(
database as never,
redis as never,
config as never,
);
expect(service.getLiveness()).toEqual({ status: 'ok' });
await expect(service.getReadiness()).resolves.toEqual({ status: 'ok' });
@@ -24,7 +22,6 @@ describe('HealthService', () => {
const service = new HealthService(
database as never,
redis as never,
{ get: vi.fn() } as never,
);
await expect(service.getReadiness()).rejects.toBeInstanceOf(
ServiceUnavailableException,
@@ -35,7 +32,6 @@ describe('HealthService', () => {
const service = new HealthService(
{ health: vi.fn() } as never,
{ health: vi.fn() } as never,
{ get: vi.fn() } as never,
);
service.setDraining();
expect(service.getLiveness()).toEqual({ status: 'ok' });
@@ -48,7 +44,6 @@ describe('HealthService', () => {
const service = new HealthService(
{ health: vi.fn().mockRejectedValue(new Error('down')) } as never,
{ health: vi.fn().mockRejectedValue(new Error('down')) } as never,
{ get: vi.fn() } as never,
);
expect(service.getLbCheck()).toEqual({ status: 'ok' });
service.setDraining();
+4 -10
View File
@@ -4,10 +4,8 @@ import {
Module,
ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { DatabaseClient } from '../database/database.client.js';
import { RedisClient } from '../redis/redis.client.js';
import type { AppConfig } from '../config/app-config.schema.js';
@Injectable()
export class HealthService {
@@ -15,7 +13,6 @@ export class HealthService {
constructor(
private readonly database: DatabaseClient,
private readonly redis: RedisClient,
private readonly config: ConfigService<AppConfig>,
) {}
getLiveness() {
return { status: 'ok' as const };
@@ -40,16 +37,13 @@ export class HealthService {
}
return { status: 'ok' as const };
}
// Called on SIGTERM before the drain delay: the replica turns not-ready for
// both the LB probe and the operator probe while it finishes in-flight
// requests. The delay itself lives in main.ts, because Nest destroys
// providers (closing the pools) before this class could serve anyone.
setDraining() {
this.ready = false;
}
async beforeApplicationShutdown(signal?: string): Promise<void> {
this.setDraining();
const delay = this.config.get<number>('DRAIN_DELAY_MS') ?? 5000;
if (signal && delay > 0) {
await new Promise<void>((resolve) => setTimeout(resolve, delay));
}
}
}
@Global()
+24 -1
View File
@@ -7,6 +7,7 @@ import {
NestFastifyApplication,
} from '@nestjs/platform-fastify';
import { AppModule } from './app.module.js';
import { HealthService } from './health/health.service.js';
import { MetricsService } from './metrics/metrics.service.js';
import { RequestIdInterceptor } from './common/http/request-id.interceptor.js';
@@ -29,12 +30,34 @@ async function bootstrap(): Promise<void> {
transform: true,
}),
);
app.enableShutdownHooks();
const config = app.get(ConfigService);
const port = config.getOrThrow<number>('APP_PORT');
const host = config.getOrThrow<string>('APP_HOST');
await app.listen(port, host);
// Drain before teardown. Nest's own shutdown sequence destroys providers
// (closing the database pool and Redis) before `beforeApplicationShutdown`
// runs, so relying on it would keep the replica listed in the load balancer
// while it can no longer serve: SIGTERM must first turn the replica
// not-ready for the LB probe, then wait DRAIN_DELAY_MS, and only then close
// the server and the pools.
const health = app.get(HealthService);
const drainDelayMs = config.getOrThrow<number>('DRAIN_DELAY_MS');
let shuttingDown = false;
const shutdown = async (): Promise<void> => {
if (shuttingDown) return;
shuttingDown = true;
health.setDraining();
if (drainDelayMs > 0)
await new Promise<void>((resolve) => setTimeout(resolve, drainDelayMs));
await app.close();
process.exit(0);
};
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP'] as const)
process.once(signal, () => {
void shutdown();
});
}
await bootstrap();