forked from mtusi2026/mtusi2026lab2
291 lines
8.9 KiB
TypeScript
291 lines
8.9 KiB
TypeScript
import { execFile } from 'node:child_process';
|
|
import { randomUUID } from 'node:crypto';
|
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { promisify } from 'node:util';
|
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
|
import type { SessionResponse } from '../src/auth/auth.service.js';
|
|
import type { Chat } from '../src/chats/chat.js';
|
|
import type { ApiErrorBody } from '../src/common/errors/api-error.js';
|
|
import type { Page } from '../src/common/http/page.js';
|
|
import type { Message } from '../src/messages/message.js';
|
|
import type { User } from '../src/users/user.js';
|
|
|
|
// Automated counterpart of the manual Redis/PostgreSQL outage checks: the suite
|
|
// stops a dependency of the Compose stand, asserts a bounded 503 (never a 500,
|
|
// never a trusted anonymous request, never a leaked driver error) and then
|
|
// restarts it to prove recovery without data loss.
|
|
//
|
|
// It is opt-in because it needs control over the Docker Compose project:
|
|
// make test-dependencies
|
|
// cd app && E2E_DEPENDENCY_FAULTS=1 npm run test:dependencies
|
|
const enabled = process.env.E2E_DEPENDENCY_FAULTS === '1';
|
|
const baseUrl = (process.env.API_BASE_URL ?? 'http://localhost:8080').replace(
|
|
/\/+$/u,
|
|
'',
|
|
);
|
|
const composeDir =
|
|
process.env.E2E_COMPOSE_DIR ??
|
|
fileURLToPath(new URL('../..', import.meta.url));
|
|
const composeEnvFile = process.env.E2E_COMPOSE_ENV_FILE ?? '.env';
|
|
const redisService = process.env.E2E_REDIS_SERVICE ?? 'redis';
|
|
const postgresService = process.env.E2E_POSTGRES_SERVICE ?? 'postgres';
|
|
const password = 'outage-check:пароль🔐';
|
|
const runId = randomUUID().replaceAll('-', '').slice(0, 12);
|
|
// A request that reaches a stopped database waits for the pool connection
|
|
// timeout, so every outage step costs up to DATABASE_CONNECTION_TIMEOUT_MS.
|
|
const outagedRequestBudgetMs = 180_000;
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
|
|
type Result<T> = { body: T; text: string; response: Response };
|
|
|
|
async function compose(...args: string[]): Promise<void> {
|
|
await execFileAsync(
|
|
'docker',
|
|
['compose', '--env-file', composeEnvFile, ...args],
|
|
{ cwd: composeDir, timeout: 120_000 },
|
|
);
|
|
}
|
|
|
|
async function request<T = unknown>(
|
|
path: string,
|
|
init: RequestInit = {},
|
|
): Promise<Result<T>> {
|
|
const response = await fetch(`${baseUrl}${path}`, init);
|
|
const text = await response.text();
|
|
return { body: (text ? JSON.parse(text) : undefined) as T, text, response };
|
|
}
|
|
|
|
function json(
|
|
body: unknown,
|
|
headers: Record<string, string> = {},
|
|
): RequestInit {
|
|
return {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json', ...headers },
|
|
body: JSON.stringify(body),
|
|
};
|
|
}
|
|
|
|
function bearer(token: string): Record<string, string> {
|
|
return { authorization: `Bearer ${token}` };
|
|
}
|
|
|
|
function basic(username: string, secret: string): string {
|
|
return `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
|
|
}
|
|
|
|
// The dependencies recover asynchronously, so readiness is polled instead of
|
|
// asserted right after docker compose returns.
|
|
async function waitForReadiness(
|
|
expected: number,
|
|
timeoutMs = 90_000,
|
|
): Promise<void> {
|
|
const deadline = Date.now() + timeoutMs;
|
|
let last: number | string = 'no response';
|
|
for (;;) {
|
|
try {
|
|
const probe = await request('/health/ready');
|
|
last = probe.response.status;
|
|
if (probe.response.status === expected) return;
|
|
} catch (error) {
|
|
last = error instanceof Error ? error.message : 'request failed';
|
|
}
|
|
if (Date.now() > deadline)
|
|
throw new Error(
|
|
`readiness never became ${expected} within ${timeoutMs}ms (last: ${String(last)})`,
|
|
);
|
|
await sleep(250);
|
|
}
|
|
}
|
|
|
|
function expectBoundedOutage(result: Result<ApiErrorBody>): void {
|
|
expect(result.response.status).toBe(503);
|
|
expect(result.body.error.code).toBe('unavailable');
|
|
expect(result.body.error.message).toBe('Service unavailable');
|
|
expect(result.body.error.request_id).toBe(
|
|
result.response.headers.get('x-request-id'),
|
|
);
|
|
// A bounded error must not leak driver details, SQL text or a stack trace.
|
|
expect(result.text).not.toMatch(
|
|
/ECONNREFUSED|ECONNRESET|Connection terminated|pg\.|node_modules|at .*\(.+:\d+:\d+\)/u,
|
|
);
|
|
}
|
|
|
|
async function historyOf(
|
|
chatId: string,
|
|
token: string,
|
|
): Promise<Result<Page<Message>>> {
|
|
return request<Page<Message>>(`/api/v1/chats/${chatId}/messages?limit=100`, {
|
|
headers: bearer(token),
|
|
});
|
|
}
|
|
|
|
describe.skipIf(!enabled)('dependency outages', () => {
|
|
let user: User;
|
|
let token: string;
|
|
let chat: Chat;
|
|
let message: Message;
|
|
|
|
beforeAll(async () => {
|
|
await compose('up', '-d');
|
|
await waitForReadiness(200);
|
|
const username = `outage_${runId}`;
|
|
const registered = await request<User>(
|
|
'/api/v1/users',
|
|
json({ username, display_name: 'Отказ зависимости', password }),
|
|
);
|
|
expect(registered.response.status).toBe(201);
|
|
user = registered.body;
|
|
const session = await request<SessionResponse>(
|
|
'/api/v1/auth/sessions',
|
|
{
|
|
method: 'POST',
|
|
headers: { authorization: basic(username, password) },
|
|
},
|
|
);
|
|
expect(session.response.status).toBe(201);
|
|
token = session.body.token;
|
|
const created = await request<Chat>(
|
|
'/api/v1/chats',
|
|
json(
|
|
{ title: `outage-${runId}`, member_ids: [user.id] },
|
|
bearer(token),
|
|
),
|
|
);
|
|
expect(created.response.status).toBe(201);
|
|
chat = created.body;
|
|
const sent = await request<Message>(
|
|
`/api/v1/chats/${chat.id}/messages`,
|
|
json({ text: 'Сообщение до отказа' }, bearer(token)),
|
|
);
|
|
expect(sent.response.status).toBe(201);
|
|
message = sent.body;
|
|
}, outagedRequestBudgetMs);
|
|
|
|
afterAll(async () => {
|
|
// Always restore the stand, even when an assertion failed mid-outage.
|
|
await compose('start', redisService);
|
|
await compose('start', postgresService);
|
|
await waitForReadiness(200);
|
|
}, outagedRequestBudgetMs);
|
|
|
|
it(
|
|
'serves bounded 503 for the Redis outage and recovers the session',
|
|
async () => {
|
|
await compose('stop', redisService);
|
|
|
|
const live = await request('/health/live');
|
|
expect(live.response.status).toBe(200);
|
|
await waitForReadiness(503);
|
|
|
|
expectBoundedOutage(await request<ApiErrorBody>('/health/ready'));
|
|
|
|
expectBoundedOutage(
|
|
await request<ApiErrorBody>('/api/v1/users/me', {
|
|
headers: bearer(token),
|
|
}),
|
|
);
|
|
|
|
expectBoundedOutage(
|
|
await request<ApiErrorBody>('/api/v1/auth/sessions', {
|
|
method: 'POST',
|
|
headers: { authorization: basic(user.username, password) },
|
|
}),
|
|
);
|
|
|
|
// A request without a session must stay unauthorized during an outage:
|
|
// the outage may not turn an anonymous caller into a trusted one.
|
|
const anonymous = await request<ApiErrorBody>('/api/v1/chats', {
|
|
headers: { 'x-user-id': user.id },
|
|
});
|
|
expect(anonymous.response.status).toBe(401);
|
|
expect(anonymous.body.error.code).toBe('unauthorized');
|
|
|
|
await compose('start', redisService);
|
|
await waitForReadiness(200);
|
|
|
|
// Sessions live in Redis with persistence, so the old token still works.
|
|
const recovered = await request<User>('/api/v1/users/me', {
|
|
headers: bearer(token),
|
|
});
|
|
expect(recovered.response.status).toBe(200);
|
|
expect(recovered.body.id).toBe(user.id);
|
|
|
|
const history = await historyOf(chat.id, token);
|
|
expect(history.response.status).toBe(200);
|
|
expect(history.body.items.map((item) => item.id)).toContain(
|
|
message.id,
|
|
);
|
|
},
|
|
outagedRequestBudgetMs,
|
|
);
|
|
|
|
it(
|
|
'serves bounded 503 for the PostgreSQL outage and keeps confirmed data',
|
|
async () => {
|
|
await compose('stop', postgresService);
|
|
|
|
const live = await request('/health/live');
|
|
expect(live.response.status).toBe(200);
|
|
await waitForReadiness(503);
|
|
|
|
expectBoundedOutage(await request<ApiErrorBody>('/health/ready'));
|
|
|
|
// Redis still answers, so these requests reach the database layer.
|
|
expectBoundedOutage(
|
|
await request<ApiErrorBody>('/api/v1/users/me', {
|
|
headers: bearer(token),
|
|
}),
|
|
);
|
|
|
|
expectBoundedOutage(
|
|
await request<ApiErrorBody>('/api/v1/chats', {
|
|
headers: bearer(token),
|
|
}),
|
|
);
|
|
|
|
expectBoundedOutage(
|
|
await request<ApiErrorBody>(
|
|
`/api/v1/chats/${chat.id}/messages?limit=100`,
|
|
{ headers: bearer(token) },
|
|
),
|
|
);
|
|
|
|
expectBoundedOutage(
|
|
await request<ApiErrorBody>(
|
|
'/api/v1/users',
|
|
json({
|
|
username: `outage_${runId}_late`,
|
|
display_name: 'Отказ зависимости',
|
|
password,
|
|
}),
|
|
),
|
|
);
|
|
|
|
await compose('start', postgresService);
|
|
await waitForReadiness(200);
|
|
|
|
const recovered = await request<User>('/api/v1/users/me', {
|
|
headers: bearer(token),
|
|
});
|
|
expect(recovered.response.status).toBe(200);
|
|
expect(recovered.body.id).toBe(user.id);
|
|
|
|
const history = await historyOf(chat.id, token);
|
|
expect(history.response.status).toBe(200);
|
|
expect(history.body.items.map((item) => item.id)).toContain(
|
|
message.id,
|
|
);
|
|
|
|
const sent = await request<Message>(
|
|
`/api/v1/chats/${chat.id}/messages`,
|
|
json({ text: 'Сообщение после отказа' }, bearer(token)),
|
|
);
|
|
expect(sent.response.status).toBe(201);
|
|
},
|
|
outagedRequestBudgetMs,
|
|
);
|
|
});
|