Files
mtusi2026lab2/app/test/dependency-failures.e2e-spec.ts
T
2026-10-10 02:13:35 +03:00

291 lines
8.9 KiB
TypeScript

import { execFile } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { setTimeout as sleep } from 'node:timers/promises';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type { SessionResponse } from '../src/auth/auth.service.js';
import type { Chat } from '../src/chats/chat.js';
import type { ApiErrorBody } from '../src/common/errors/api-error.js';
import type { Page } from '../src/common/http/page.js';
import type { Message } from '../src/messages/message.js';
import type { User } from '../src/users/user.js';
// Automated counterpart of the manual Redis/PostgreSQL outage checks: the suite
// stops a dependency of the Compose stand, asserts a bounded 503 (never a 500,
// never a trusted anonymous request, never a leaked driver error) and then
// restarts it to prove recovery without data loss.
//
// It is opt-in because it needs control over the Docker Compose project:
// make test-dependencies
// cd app && E2E_DEPENDENCY_FAULTS=1 npm run test:dependencies
const enabled = process.env.E2E_DEPENDENCY_FAULTS === '1';
const baseUrl = (process.env.API_BASE_URL ?? 'http://localhost:8080').replace(
/\/+$/u,
'',
);
const composeDir =
process.env.E2E_COMPOSE_DIR ??
fileURLToPath(new URL('../..', import.meta.url));
const composeEnvFile = process.env.E2E_COMPOSE_ENV_FILE ?? '.env';
const redisService = process.env.E2E_REDIS_SERVICE ?? 'redis';
const postgresService = process.env.E2E_POSTGRES_SERVICE ?? 'postgres';
const password = 'outage-check:пароль🔐';
const runId = randomUUID().replaceAll('-', '').slice(0, 12);
// A request that reaches a stopped database waits for the pool connection
// timeout, so every outage step costs up to DATABASE_CONNECTION_TIMEOUT_MS.
const outagedRequestBudgetMs = 180_000;
const execFileAsync = promisify(execFile);
type Result<T> = { body: T; text: string; response: Response };
async function compose(...args: string[]): Promise<void> {
await execFileAsync(
'docker',
['compose', '--env-file', composeEnvFile, ...args],
{ cwd: composeDir, timeout: 120_000 },
);
}
async function request<T = unknown>(
path: string,
init: RequestInit = {},
): Promise<Result<T>> {
const response = await fetch(`${baseUrl}${path}`, init);
const text = await response.text();
return { body: (text ? JSON.parse(text) : undefined) as T, text, response };
}
function json(
body: unknown,
headers: Record<string, string> = {},
): RequestInit {
return {
method: 'POST',
headers: { 'content-type': 'application/json', ...headers },
body: JSON.stringify(body),
};
}
function bearer(token: string): Record<string, string> {
return { authorization: `Bearer ${token}` };
}
function basic(username: string, secret: string): string {
return `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
}
// The dependencies recover asynchronously, so readiness is polled instead of
// asserted right after docker compose returns.
async function waitForReadiness(
expected: number,
timeoutMs = 90_000,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
let last: number | string = 'no response';
for (;;) {
try {
const probe = await request('/health/ready');
last = probe.response.status;
if (probe.response.status === expected) return;
} catch (error) {
last = error instanceof Error ? error.message : 'request failed';
}
if (Date.now() > deadline)
throw new Error(
`readiness never became ${expected} within ${timeoutMs}ms (last: ${String(last)})`,
);
await sleep(250);
}
}
function expectBoundedOutage(result: Result<ApiErrorBody>): void {
expect(result.response.status).toBe(503);
expect(result.body.error.code).toBe('unavailable');
expect(result.body.error.message).toBe('Service unavailable');
expect(result.body.error.request_id).toBe(
result.response.headers.get('x-request-id'),
);
// A bounded error must not leak driver details, SQL text or a stack trace.
expect(result.text).not.toMatch(
/ECONNREFUSED|ECONNRESET|Connection terminated|pg\.|node_modules|at .*\(.+:\d+:\d+\)/u,
);
}
async function historyOf(
chatId: string,
token: string,
): Promise<Result<Page<Message>>> {
return request<Page<Message>>(`/api/v1/chats/${chatId}/messages?limit=100`, {
headers: bearer(token),
});
}
describe.skipIf(!enabled)('dependency outages', () => {
let user: User;
let token: string;
let chat: Chat;
let message: Message;
beforeAll(async () => {
await compose('up', '-d');
await waitForReadiness(200);
const username = `outage_${runId}`;
const registered = await request<User>(
'/api/v1/users',
json({ username, display_name: 'Отказ зависимости', password }),
);
expect(registered.response.status).toBe(201);
user = registered.body;
const session = await request<SessionResponse>(
'/api/v1/auth/sessions',
{
method: 'POST',
headers: { authorization: basic(username, password) },
},
);
expect(session.response.status).toBe(201);
token = session.body.token;
const created = await request<Chat>(
'/api/v1/chats',
json(
{ title: `outage-${runId}`, member_ids: [user.id] },
bearer(token),
),
);
expect(created.response.status).toBe(201);
chat = created.body;
const sent = await request<Message>(
`/api/v1/chats/${chat.id}/messages`,
json({ text: 'Сообщение до отказа' }, bearer(token)),
);
expect(sent.response.status).toBe(201);
message = sent.body;
}, outagedRequestBudgetMs);
afterAll(async () => {
// Always restore the stand, even when an assertion failed mid-outage.
await compose('start', redisService);
await compose('start', postgresService);
await waitForReadiness(200);
}, outagedRequestBudgetMs);
it(
'serves bounded 503 for the Redis outage and recovers the session',
async () => {
await compose('stop', redisService);
const live = await request('/health/live');
expect(live.response.status).toBe(200);
await waitForReadiness(503);
expectBoundedOutage(await request<ApiErrorBody>('/health/ready'));
expectBoundedOutage(
await request<ApiErrorBody>('/api/v1/users/me', {
headers: bearer(token),
}),
);
expectBoundedOutage(
await request<ApiErrorBody>('/api/v1/auth/sessions', {
method: 'POST',
headers: { authorization: basic(user.username, password) },
}),
);
// A request without a session must stay unauthorized during an outage:
// the outage may not turn an anonymous caller into a trusted one.
const anonymous = await request<ApiErrorBody>('/api/v1/chats', {
headers: { 'x-user-id': user.id },
});
expect(anonymous.response.status).toBe(401);
expect(anonymous.body.error.code).toBe('unauthorized');
await compose('start', redisService);
await waitForReadiness(200);
// Sessions live in Redis with persistence, so the old token still works.
const recovered = await request<User>('/api/v1/users/me', {
headers: bearer(token),
});
expect(recovered.response.status).toBe(200);
expect(recovered.body.id).toBe(user.id);
const history = await historyOf(chat.id, token);
expect(history.response.status).toBe(200);
expect(history.body.items.map((item) => item.id)).toContain(
message.id,
);
},
outagedRequestBudgetMs,
);
it(
'serves bounded 503 for the PostgreSQL outage and keeps confirmed data',
async () => {
await compose('stop', postgresService);
const live = await request('/health/live');
expect(live.response.status).toBe(200);
await waitForReadiness(503);
expectBoundedOutage(await request<ApiErrorBody>('/health/ready'));
// Redis still answers, so these requests reach the database layer.
expectBoundedOutage(
await request<ApiErrorBody>('/api/v1/users/me', {
headers: bearer(token),
}),
);
expectBoundedOutage(
await request<ApiErrorBody>('/api/v1/chats', {
headers: bearer(token),
}),
);
expectBoundedOutage(
await request<ApiErrorBody>(
`/api/v1/chats/${chat.id}/messages?limit=100`,
{ headers: bearer(token) },
),
);
expectBoundedOutage(
await request<ApiErrorBody>(
'/api/v1/users',
json({
username: `outage_${runId}_late`,
display_name: 'Отказ зависимости',
password,
}),
),
);
await compose('start', postgresService);
await waitForReadiness(200);
const recovered = await request<User>('/api/v1/users/me', {
headers: bearer(token),
});
expect(recovered.response.status).toBe(200);
expect(recovered.body.id).toBe(user.id);
const history = await historyOf(chat.id, token);
expect(history.response.status).toBe(200);
expect(history.body.items.map((item) => item.id)).toContain(
message.id,
);
const sent = await request<Message>(
`/api/v1/chats/${chat.id}/messages`,
json({ text: 'Сообщение после отказа' }, bearer(token)),
);
expect(sent.response.status).toBe(201);
},
outagedRequestBudgetMs,
);
});