Files
mtusi2026lab3/app/test/dependency-failures.e2e-spec.ts

800 lines
24 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { execFile } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { setTimeout as sleep } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { SessionResponse } from '../src/auth/auth.service.js';
import type { Chat } from '../src/chats/chat.js';
import type { ApiErrorBody } from '../src/common/errors/api-error.js';
import type { Page } from '../src/common/http/page.js';
import type { Message } from '../src/messages/message.js';
import type { User } from '../src/users/user.js';
// Automated counterpart of the manual Redis/PostgreSQL outage checks. Four
// topologies, because the load balancer probes /health/ready (the same route as
// the container healthcheck):
// 1. Redis outage — shared dependency: every replica turns not-ready, HAProxy
// removes all backends and answers 503 itself, while the replica's own
// readiness stays a bounded API 503 with request_id.
// 2. PostgreSQL outage — same shared-dependency topology, confirmed data and
// the session survive the restart.
// 3. Per-replica PostgreSQL fault — only the selected replica loses its own
// path to the database (temporary pg_hba reject entry for its address plus
// connection termination): that slot leaves rotation, the healthy replica
// keeps serving, and the fault is always undone.
// 4. Pool saturation — a busy shared pool is overload, not a lost dependency:
// the replicas stay in rotation because the probe uses its own connection.
//
// It is opt-in because it needs control over the Docker Compose project:
// make test-dependencies
// cd app && E2E_DEPENDENCY_FAULTS=1 npm run test:dependencies
const enabled = process.env.E2E_DEPENDENCY_FAULTS === '1';
const baseUrl = (process.env.API_BASE_URL ?? 'http://localhost:8080').replace(
/\/+$/u,
'',
);
const composeDir =
process.env.E2E_COMPOSE_DIR ??
fileURLToPath(new URL('../..', import.meta.url));
const composeEnvFile = process.env.E2E_COMPOSE_ENV_FILE ?? '.env';
const redisService = process.env.E2E_REDIS_SERVICE ?? 'redis';
const postgresService = process.env.E2E_POSTGRES_SERVICE ?? 'postgres';
const apiService = process.env.E2E_API_SERVICE ?? 'api';
const lbStatsUrl =
process.env.E2E_LB_STATS_URL ?? 'http://127.0.0.1:8404/stats;csv';
// PostgreSQL superuser of the stand (compose.yaml, .env); psql runs over the
// container's local socket, so no password is needed here.
const postgresUser = process.env.E2E_POSTGRES_USER ?? 'messenger';
const postgresDatabase = process.env.E2E_POSTGRES_DB ?? 'messenger';
const pgHbaPath =
process.env.E2E_PG_HBA_PATH ?? '/var/lib/postgresql/data/pg_hba.conf';
const pgHbaBackup = `${pgHbaPath}.lab3-partial-fault.bak`;
const password = 'outage-check:пароль🔐';
const runId = randomUUID().replaceAll('-', '').slice(0, 12);
// A request that reaches a stopped database waits for the pool connection
// timeout, so every outage step costs up to DATABASE_CONNECTION_TIMEOUT_MS.
const outagedRequestBudgetMs = 180_000;
const slotBudgetMs = 20_000;
const readyReads = 16;
const saturatingWrites = 60; // больше, чем replicas × DATABASE_POOL_MAX (2 × 10)
const execFileAsync = promisify(execFile);
type Result<T> = { body: T; text: string; response: Response };
type ReplicaProbe = {
instanceId: string | null;
requestId: string | null;
status: number;
text: string;
};
type LbSlot = { addr: string; check: string; name: string; status: string };
async function compose(...args: string[]): Promise<string> {
const result = await execFileAsync(
'docker',
['compose', '--env-file', composeEnvFile, ...args],
{ cwd: composeDir, timeout: 120_000 },
);
return result.stdout.trim();
}
async function docker(...args: string[]): Promise<string> {
const result = await execFileAsync('docker', args, {
cwd: composeDir,
timeout: 60_000,
});
return result.stdout.trim();
}
async function request<T = unknown>(
path: string,
init: RequestInit = {},
): Promise<Result<T>> {
const response = await fetch(`${baseUrl}${path}`, init);
const text = await response.text();
// With every backend out of rotation the answer is the LB's own HTML 503,
// so a body is not guaranteed to be JSON (it always is for API answers).
let body: T | undefined;
try {
body = text ? (JSON.parse(text) as T) : undefined;
} catch {
body = undefined;
}
return { body: body as T, text, response };
}
function json(
body: unknown,
headers: Record<string, string> = {},
): RequestInit {
return {
method: 'POST',
headers: { 'content-type': 'application/json', ...headers },
body: JSON.stringify(body),
};
}
function bearer(token: string): Record<string, string> {
return { authorization: `Bearer ${token}` };
}
// POST messages needs a unique Idempotency-Key in lab3+; outage checks only
// need the write path, not replay semantics.
function messageHeaders(token: string): Record<string, string> {
return {
...bearer(token),
'idempotency-key': randomUUID().replaceAll('-', ''),
};
}
function basic(username: string, secret: string): string {
return `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
}
// The dependencies recover asynchronously, so readiness is polled instead of
// asserted right after docker compose returns.
async function waitForReadiness(
expected: number,
timeoutMs = 90_000,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
let last: number | string = 'no response';
for (;;) {
try {
const probe = await request('/health/ready');
last = probe.response.status;
if (probe.response.status === expected) return;
} catch (error) {
last = error instanceof Error ? error.message : 'request failed';
}
if (Date.now() > deadline)
throw new Error(
`readiness never became ${expected} within ${timeoutMs}ms (last: ${String(last)})`,
);
await sleep(250);
}
}
async function waitFor(
condition: () => Promise<boolean>,
describe: () => Promise<string>,
timeoutMs: number,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
for (;;) {
if (await condition()) return;
if (Date.now() > deadline)
throw new Error(
`condition not reached within ${timeoutMs}ms: ${await describe()}`,
);
await sleep(250);
}
}
function expectBoundedError(
status: number,
text: string,
requestId: string | null,
): void {
expect(status).toBe(503);
const body = JSON.parse(text) as ApiErrorBody;
expect(body.error.code).toBe('unavailable');
expect(body.error.message).toBe('Service unavailable');
expect(body.error.request_id).toBe(requestId);
// A bounded error must not leak driver details, SQL text or a stack trace.
expect(text).not.toMatch(
/ECONNREFUSED|ECONNRESET|Connection terminated|pg\.|node_modules|at .*\(.+:\d+:\d+\)/u,
);
}
function expectBoundedOutage(result: Result<ApiErrorBody>): void {
expectBoundedError(
result.response.status,
result.text,
result.response.headers.get('x-request-id'),
);
}
// With every backend out of rotation the last hop is the load balancer itself:
// it answers with its own 503 page, so there is no API error body and no
// X-Instance-Id — the request never reached a replica.
function expectLbOutage(result: Result<unknown>): void {
expect(result.response.status).toBe(503);
expect(result.response.headers.get('x-instance-id')).toBeNull();
}
// What a client sees while a shared dependency is down: either the bounded 503
// of a replica that is still in rotation (identified by X-Instance-Id), or the
// LB's own 503 once every replica has left rotation. Never a success.
function expectUnavailableThroughLb(result: Result<ApiErrorBody>): void {
expect(result.response.status).toBe(503);
if (result.response.headers.get('x-instance-id') === null) return;
expectBoundedOutage(result);
}
async function historyOf(
chatId: string,
token: string,
): Promise<Result<Page<Message>>> {
return request<Page<Message>>(
`/api/v1/chats/${chatId}/messages?limit=100`,
{
headers: bearer(token),
},
);
}
async function apiContainers(): Promise<string[]> {
const stdout = await compose('ps', '-q', apiService);
return stdout
.split('\n')
.map((line) => line.trim())
.filter((line) => line.length > 0)
.sort();
}
async function containerIp(container: string): Promise<string> {
return docker(
'inspect',
'-f',
'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
container,
);
}
// The replicas publish no ports, so a single replica is probed from inside its
// own network namespace (loopback).
const probeScript = `
const params = JSON.parse(process.env.LAB3_PROBE);
const port = process.env.APP_PORT || 8080;
const response = await fetch('http://127.0.0.1:' + port + params.path, {
method: params.method || 'GET',
headers: params.headers || {},
body: params.body,
});
const text = await response.text();
console.log(JSON.stringify({
status: response.status,
text,
instanceId: response.headers.get('x-instance-id'),
requestId: response.headers.get('x-request-id'),
}));
`;
type ProbeInit = {
body?: string;
headers?: Record<string, string>;
method?: string;
};
async function probeReplica(
container: string,
path: string,
init: ProbeInit = {},
): Promise<ReplicaProbe> {
const stdout = await docker(
'exec',
'-e',
`LAB3_PROBE=${JSON.stringify({ path, ...init })}`,
container,
'node',
'--input-type=module',
'-e',
probeScript,
);
return JSON.parse(stdout) as ReplicaProbe;
}
async function waitForReplicaReadiness(
container: string,
expected: number,
timeoutMs: number,
): Promise<void> {
let last = 'no probe';
await waitFor(
async () => {
try {
const probe = await probeReplica(container, '/health/ready');
last = `${probe.status} ${probe.text}`;
return probe.status === expected;
} catch (error) {
last = error instanceof Error ? error.message : 'probe failed';
return false;
}
},
async () => last,
timeoutMs,
);
}
function parseCsvLine(line: string): string[] {
const values: string[] = [];
let current = '';
let quoted = false;
for (const character of line) {
if (character === '"') quoted = !quoted;
else if (character === ',' && !quoted) {
values.push(current);
current = '';
} else current += character;
}
values.push(current.replace(/\r$/u, ''));
return values;
}
// Resolved backend slots of the api backend: the unused slots of the
// server-template have no address and are ignored.
async function lbSlots(): Promise<LbSlot[]> {
const response = await fetch(lbStatsUrl);
const lines = (await response.text()).trim().split('\n');
const header = parseCsvLine(lines[0].replace(/^#\s*/u, '')).map((name) =>
name.trim(),
);
const column = (name: string): number => header.indexOf(name);
return lines
.slice(1)
.map(parseCsvLine)
.filter(
(columns) =>
columns[column('pxname')] === 'api' &&
/^api\d+$/u.test(columns[column('svname')] ?? '') &&
(columns[column('addr')] ?? '').length > 0,
)
.map((columns) => ({
addr: columns[column('addr')],
check: columns[column('check_status')],
name: columns[column('svname')],
status: columns[column('status')],
}));
}
async function waitForSlot(
predicate: (slots: LbSlot[]) => boolean,
describe: string,
timeoutMs = slotBudgetMs,
): Promise<LbSlot[]> {
let last = 'no stats';
await waitFor(
async () => {
try {
const slots = await lbSlots();
last = slots
.map(
(slot) =>
`${slot.name}@${slot.addr} ${slot.status}/${slot.check}`,
)
.join(', ');
return slots.length > 0 && predicate(slots);
} catch (error) {
last = error instanceof Error ? error.message : 'stats failed';
return false;
}
},
async () => `${describe} (last: ${last})`,
timeoutMs,
);
return lbSlots();
}
function upSlots(slots: LbSlot[]): LbSlot[] {
return slots.filter((slot) => slot.status === 'UP');
}
async function psql(query: string): Promise<string> {
return dockerExecService(
postgresService,
'psql',
'-U',
postgresUser,
'-d',
postgresDatabase,
'-tA',
'-c',
query,
);
}
// docker exec needs a container, while the suite is configured with Compose
// service names, so the container of the service is resolved first.
async function dockerExecService(
service: string,
...args: string[]
): Promise<string> {
const container = (await compose('ps', '-q', service))
.split('\n')[0]
?.trim();
if (!container) throw new Error(`no container for service ${service}`);
return docker('exec', container, ...args);
}
// Takes away the replica's own path to PostgreSQL without touching the shared
// database: a reject entry for the replica's address becomes the first matching
// line of pg_hba.conf, so its new connections fail while the server stays up
// for everyone else. The file is rewritten through the existing inode (`cat >
// file`) to preserve the owner and mode the server needs to reload it.
async function applyReplicaDatabaseReject(address: string): Promise<void> {
const script = [
`test -f ${pgHbaBackup} || cat ${pgHbaPath} > ${pgHbaBackup}`,
`printf 'host all all ${address}/32 reject\\n' > /tmp/lab3-pg-hba`,
`cat ${pgHbaPath} >> /tmp/lab3-pg-hba`,
`cat /tmp/lab3-pg-hba > ${pgHbaPath}`,
'rm -f /tmp/lab3-pg-hba',
].join('; ');
await dockerExecService(postgresService, 'sh', '-c', script);
await psql('select pg_reload_conf()');
await psql(
`select count(pg_terminate_backend(pid)) from pg_stat_activity where client_addr = '${address}'`,
);
}
// Idempotent restore: also runs from afterAll, so an assertion that failed
// mid-fault cannot leave the stand rejecting one replica.
async function restoreDatabaseAccess(): Promise<void> {
await dockerExecService(
postgresService,
'sh',
'-c',
`if [ -f ${pgHbaBackup} ]; then cat ${pgHbaBackup} > ${pgHbaPath}; rm -f ${pgHbaBackup}; fi`,
);
await psql('select pg_reload_conf()');
}
describe.skipIf(!enabled)('dependency outages', () => {
let user: User;
let token: string;
let chat: Chat;
let message: Message;
let replicas: string[];
beforeAll(async () => {
await compose('up', '-d');
await waitForReadiness(200);
const username = `outage_${runId}`;
const registered = await request<User>(
'/api/v1/users',
json({ username, display_name: 'Отказ зависимости', password }),
);
expect(registered.response.status).toBe(201);
user = registered.body;
const session = await request<SessionResponse>(
'/api/v1/auth/sessions',
{
method: 'POST',
headers: { authorization: basic(username, password) },
},
);
expect(session.response.status).toBe(201);
token = session.body.token;
const created = await request<Chat>(
'/api/v1/chats',
json(
{ title: `outage-${runId}`, member_ids: [user.id] },
bearer(token),
),
);
expect(created.response.status).toBe(201);
chat = created.body;
const sent = await request<Message>(
`/api/v1/chats/${chat.id}/messages`,
json({ text: 'Сообщение до отказа' }, messageHeaders(token)),
);
expect(sent.response.status).toBe(201);
message = sent.body;
replicas = await apiContainers();
expect(replicas.length).toBeGreaterThanOrEqual(2);
await waitForSlot(
(slots) => upSlots(slots).length === replicas.length,
'not every replica is UP',
30_000,
);
}, outagedRequestBudgetMs);
afterAll(async () => {
// Always restore the stand, even when an assertion failed mid-outage:
// first the services (pg_hba is only reachable in a running container),
// then the temporary per-replica rule, then rotation.
await compose('start', redisService);
await compose('start', postgresService);
await waitForReadiness(200);
await restoreDatabaseAccess();
const expected = (await apiContainers()).length;
await waitForSlot(
(slots) => upSlots(slots).length === expected,
'backends did not return to rotation',
30_000,
);
}, outagedRequestBudgetMs);
it(
'keeps every replica in rotation while the database pool is saturated',
async () => {
// A row lock on the chat parks every write in the shared pool; the
// readiness probe uses its own connection and must keep answering.
const lock = psql(
`begin; select id from chats where id = '${chat.id}' for update; select pg_sleep(4); commit;`,
);
await sleep(300);
const statuses = new Set<string>();
let sampling = true;
const sampler = (async () => {
while (sampling) {
for (const slot of await lbSlots()) statuses.add(slot.status);
await sleep(200);
}
})();
const writes = await Promise.all(
Array.from({ length: saturatingWrites }, () =>
request(
`/api/v1/chats/${chat.id}/messages`,
json({ text: 'saturation' }, messageHeaders(token)),
),
),
);
await lock;
await sleep(1000);
sampling = false;
await sampler;
expect([...statuses]).toEqual(['UP']);
expect(
writes
.map((write) => write.response.status)
.filter((status) => status !== 201),
).toEqual([]);
},
outagedRequestBudgetMs,
);
it(
'serves bounded 503 for the Redis outage and recovers the session',
async () => {
await compose('stop', redisService);
const replica = replicas[0];
expect((await probeReplica(replica, '/health/live')).status).toBe(
200,
);
await waitForReplicaReadiness(replica, 503, 30_000);
const readiness = await probeReplica(replica, '/health/ready');
expectBoundedError(
readiness.status,
readiness.text,
readiness.requestId,
);
const me = await probeReplica(replica, '/api/v1/users/me', {
headers: bearer(token),
});
expectBoundedError(me.status, me.text, me.requestId);
// A request without a session must stay unauthorized during an outage:
// the outage may not turn an anonymous caller into a trusted one.
const anonymous = await probeReplica(replica, '/api/v1/chats', {
headers: { 'x-user-id': user.id },
});
expect(anonymous.status).toBe(401);
expect(
(JSON.parse(anonymous.text) as ApiErrorBody).error.code,
).toBe('unauthorized');
expectUnavailableThroughLb(
await request<ApiErrorBody>('/api/v1/users/me', {
headers: bearer(token),
}),
);
await waitForSlot(
// 'UP 1/2' is still in rotation: wait for DOWN itself.
(slots) => slots.every((slot) => slot.status === 'DOWN'),
'not every slot left rotation',
);
expectLbOutage(
await request('/api/v1/users/me', { headers: bearer(token) }),
);
await compose('start', redisService);
await waitForReadiness(200);
await waitForSlot(
(slots) => upSlots(slots).length === replicas.length,
'backends did not return to rotation',
30_000,
);
// Sessions live in Redis with persistence, so the old token still works.
const recovered = await request<User>('/api/v1/users/me', {
headers: bearer(token),
});
expect(recovered.response.status).toBe(200);
expect(recovered.body.id).toBe(user.id);
const history = await historyOf(chat.id, token);
expect(history.response.status).toBe(200);
expect(history.body.items.map((item) => item.id)).toContain(
message.id,
);
},
outagedRequestBudgetMs,
);
it(
'serves bounded 503 for the PostgreSQL outage and keeps confirmed data',
async () => {
await compose('stop', postgresService);
const replica = replicas[0];
expect((await probeReplica(replica, '/health/live')).status).toBe(
200,
);
await waitForReplicaReadiness(replica, 503, 30_000);
const readiness = await probeReplica(replica, '/health/ready');
expectBoundedError(
readiness.status,
readiness.text,
readiness.requestId,
);
// Redis still answers, so these requests reach the database layer and
// must fail with the bounded 503, not an unhandled 500.
const me = await probeReplica(replica, '/api/v1/users/me', {
headers: bearer(token),
});
expectBoundedError(me.status, me.text, me.requestId);
const chats = await probeReplica(replica, '/api/v1/chats', {
headers: bearer(token),
});
expectBoundedError(chats.status, chats.text, chats.requestId);
const history = await probeReplica(
replica,
`/api/v1/chats/${chat.id}/messages?limit=100`,
{ headers: bearer(token) },
);
expectBoundedError(history.status, history.text, history.requestId);
const registration = await probeReplica(replica, '/api/v1/users', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
username: `outage_${runId}_late`,
display_name: 'Отказ зависимости',
password,
}),
});
expectBoundedError(
registration.status,
registration.text,
registration.requestId,
);
expectUnavailableThroughLb(
await request<ApiErrorBody>('/api/v1/users/me', {
headers: bearer(token),
}),
);
await waitForSlot(
// 'UP 1/2' is still in rotation: wait for DOWN itself.
(slots) => slots.every((slot) => slot.status === 'DOWN'),
'not every slot left rotation',
);
expectLbOutage(
await request('/api/v1/users/me', { headers: bearer(token) }),
);
await compose('start', postgresService);
await waitForReadiness(200);
await waitForSlot(
(slots) => upSlots(slots).length === replicas.length,
'backends did not return to rotation',
30_000,
);
const recovered = await request<User>('/api/v1/users/me', {
headers: bearer(token),
});
expect(recovered.response.status).toBe(200);
expect(recovered.body.id).toBe(user.id);
const after = await historyOf(chat.id, token);
expect(after.response.status).toBe(200);
expect(after.body.items.map((item) => item.id)).toContain(
message.id,
);
const sent = await request<Message>(
`/api/v1/chats/${chat.id}/messages`,
json({ text: 'Сообщение после отказа' }, messageHeaders(token)),
);
expect(sent.response.status).toBe(201);
},
outagedRequestBudgetMs,
);
it(
'excludes only the replica that lost its own database path',
async () => {
const selected = replicas[0];
const healthy = replicas[1];
const selectedIp = await containerIp(selected);
const selectedAddress = `${selectedIp}:8080`;
const selectedInstance = (
await probeReplica(selected, '/health/live')
).instanceId;
expect(selectedInstance).not.toBeNull();
const healthyInstance = (
await probeReplica(healthy, '/health/live')
).instanceId;
try {
await applyReplicaDatabaseReject(selectedIp);
const faultAt = Date.now();
await waitForReplicaReadiness(selected, 503, 30_000);
expect(
(await probeReplica(healthy, '/health/ready')).status,
).toBe(200);
// Only the affected slot leaves rotation. While `fall 2` counts the
// failed checks the slot passes through "UP 1/2 (L7STS)", so the
// wait is for DOWN itself, not for "no longer plain UP".
const slots = await waitForSlot(
(candidates) =>
upSlots(candidates).length === replicas.length - 1 &&
candidates.some(
(slot) =>
slot.addr === selectedAddress &&
slot.status === 'DOWN',
),
`selected slot ${selectedAddress} did not go down`,
30_000,
);
// inter 500 ms × fall 2 + timeout check 500 ms + polling step 250 ms.
expect(Date.now() - faultAt).toBeLessThan(3_000);
expect(
slots.find((slot) => slot.addr === selectedAddress)?.status,
).toBe('DOWN');
const served = new Set<string>();
for (let index = 0; index < readyReads; index += 1) {
const read = await historyOf(chat.id, token);
expect(read.response.status).toBe(200);
served.add(
read.response.headers.get('x-instance-id') ?? 'unknown',
);
}
expect(served.has(selectedInstance as string)).toBe(false);
expect(served.has(healthyInstance as string)).toBe(true);
} finally {
await restoreDatabaseAccess();
}
await waitForReplicaReadiness(selected, 200, 30_000);
await waitForSlot(
(slots) =>
upSlots(slots).length === replicas.length &&
slots.some(
(slot) =>
slot.addr === selectedAddress &&
slot.status === 'UP',
),
'restored replica did not return to rotation',
30_000,
);
let servedAgain = false;
for (
let index = 0;
index < readyReads * 2 && !servedAgain;
index += 1
) {
const read = await historyOf(chat.id, token);
expect(read.response.status).toBe(200);
servedAgain =
read.response.headers.get('x-instance-id') ===
selectedInstance;
}
expect(servedAgain).toBe(true);
},
outagedRequestBudgetMs,
);
});