import { execFile } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { setTimeout as sleep } from 'node:timers/promises'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import type { SessionResponse } from '../src/auth/auth.service.js'; import type { Chat } from '../src/chats/chat.js'; import type { ApiErrorBody } from '../src/common/errors/api-error.js'; import type { Page } from '../src/common/http/page.js'; import type { Message } from '../src/messages/message.js'; import type { User } from '../src/users/user.js'; // Automated counterpart of the manual Redis/PostgreSQL outage checks: the suite // stops a dependency of the Compose stand, asserts a bounded 503 (never a 500, // never a trusted anonymous request, never a leaked driver error) and then // restarts it to prove recovery without data loss. // // It is opt-in because it needs control over the Docker Compose project: // make test-dependencies // cd app && E2E_DEPENDENCY_FAULTS=1 npm run test:dependencies const enabled = process.env.E2E_DEPENDENCY_FAULTS === '1'; const baseUrl = (process.env.API_BASE_URL ?? 'http://localhost:8080').replace( /\/+$/u, '', ); const composeDir = process.env.E2E_COMPOSE_DIR ?? fileURLToPath(new URL('../..', import.meta.url)); const composeEnvFile = process.env.E2E_COMPOSE_ENV_FILE ?? '.env'; const redisService = process.env.E2E_REDIS_SERVICE ?? 'redis'; const postgresService = process.env.E2E_POSTGRES_SERVICE ?? 'postgres'; const password = 'outage-check:пароль🔐'; const runId = randomUUID().replaceAll('-', '').slice(0, 12); // A request that reaches a stopped database waits for the pool connection // timeout, so every outage step costs up to DATABASE_CONNECTION_TIMEOUT_MS. const outagedRequestBudgetMs = 180_000; const execFileAsync = promisify(execFile); type Result = { body: T; text: string; response: Response }; async function compose(...args: string[]): Promise { await execFileAsync( 'docker', ['compose', '--env-file', composeEnvFile, ...args], { cwd: composeDir, timeout: 120_000 }, ); } async function request( path: string, init: RequestInit = {}, ): Promise> { const response = await fetch(`${baseUrl}${path}`, init); const text = await response.text(); return { body: (text ? JSON.parse(text) : undefined) as T, text, response }; } function json( body: unknown, headers: Record = {}, ): RequestInit { return { method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body), }; } function bearer(token: string): Record { return { authorization: `Bearer ${token}` }; } function basic(username: string, secret: string): string { return `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`; } // The dependencies recover asynchronously, so readiness is polled instead of // asserted right after docker compose returns. async function waitForReadiness( expected: number, timeoutMs = 90_000, ): Promise { const deadline = Date.now() + timeoutMs; let last: number | string = 'no response'; for (;;) { try { const probe = await request('/health/ready'); last = probe.response.status; if (probe.response.status === expected) return; } catch (error) { last = error instanceof Error ? error.message : 'request failed'; } if (Date.now() > deadline) throw new Error( `readiness never became ${expected} within ${timeoutMs}ms (last: ${String(last)})`, ); await sleep(250); } } function expectBoundedOutage(result: Result): void { expect(result.response.status).toBe(503); expect(result.body.error.code).toBe('unavailable'); expect(result.body.error.message).toBe('Service unavailable'); expect(result.body.error.request_id).toBe( result.response.headers.get('x-request-id'), ); // A bounded error must not leak driver details, SQL text or a stack trace. expect(result.text).not.toMatch( /ECONNREFUSED|ECONNRESET|Connection terminated|pg\.|node_modules|at .*\(.+:\d+:\d+\)/u, ); } async function historyOf( chatId: string, token: string, ): Promise>> { return request>(`/api/v1/chats/${chatId}/messages?limit=100`, { headers: bearer(token), }); } describe.skipIf(!enabled)('dependency outages', () => { let user: User; let token: string; let chat: Chat; let message: Message; beforeAll(async () => { await compose('up', '-d'); await waitForReadiness(200); const username = `outage_${runId}`; const registered = await request( '/api/v1/users', json({ username, display_name: 'Отказ зависимости', password }), ); expect(registered.response.status).toBe(201); user = registered.body; const session = await request( '/api/v1/auth/sessions', { method: 'POST', headers: { authorization: basic(username, password) }, }, ); expect(session.response.status).toBe(201); token = session.body.token; const created = await request( '/api/v1/chats', json( { title: `outage-${runId}`, member_ids: [user.id] }, bearer(token), ), ); expect(created.response.status).toBe(201); chat = created.body; const sent = await request( `/api/v1/chats/${chat.id}/messages`, json({ text: 'Сообщение до отказа' }, bearer(token)), ); expect(sent.response.status).toBe(201); message = sent.body; }, outagedRequestBudgetMs); afterAll(async () => { // Always restore the stand, even when an assertion failed mid-outage. await compose('start', redisService); await compose('start', postgresService); await waitForReadiness(200); }, outagedRequestBudgetMs); it( 'serves bounded 503 for the Redis outage and recovers the session', async () => { await compose('stop', redisService); const live = await request('/health/live'); expect(live.response.status).toBe(200); await waitForReadiness(503); expectBoundedOutage(await request('/health/ready')); expectBoundedOutage( await request('/api/v1/users/me', { headers: bearer(token), }), ); expectBoundedOutage( await request('/api/v1/auth/sessions', { method: 'POST', headers: { authorization: basic(user.username, password) }, }), ); // A request without a session must stay unauthorized during an outage: // the outage may not turn an anonymous caller into a trusted one. const anonymous = await request('/api/v1/chats', { headers: { 'x-user-id': user.id }, }); expect(anonymous.response.status).toBe(401); expect(anonymous.body.error.code).toBe('unauthorized'); await compose('start', redisService); await waitForReadiness(200); // Sessions live in Redis with persistence, so the old token still works. const recovered = await request('/api/v1/users/me', { headers: bearer(token), }); expect(recovered.response.status).toBe(200); expect(recovered.body.id).toBe(user.id); const history = await historyOf(chat.id, token); expect(history.response.status).toBe(200); expect(history.body.items.map((item) => item.id)).toContain( message.id, ); }, outagedRequestBudgetMs, ); it( 'serves bounded 503 for the PostgreSQL outage and keeps confirmed data', async () => { await compose('stop', postgresService); const live = await request('/health/live'); expect(live.response.status).toBe(200); await waitForReadiness(503); expectBoundedOutage(await request('/health/ready')); // Redis still answers, so these requests reach the database layer. expectBoundedOutage( await request('/api/v1/users/me', { headers: bearer(token), }), ); expectBoundedOutage( await request('/api/v1/chats', { headers: bearer(token), }), ); expectBoundedOutage( await request( `/api/v1/chats/${chat.id}/messages?limit=100`, { headers: bearer(token) }, ), ); expectBoundedOutage( await request( '/api/v1/users', json({ username: `outage_${runId}_late`, display_name: 'Отказ зависимости', password, }), ), ); await compose('start', postgresService); await waitForReadiness(200); const recovered = await request('/api/v1/users/me', { headers: bearer(token), }); expect(recovered.response.status).toBe(200); expect(recovered.body.id).toBe(user.id); const history = await historyOf(chat.id, token); expect(history.response.status).toBe(200); expect(history.body.items.map((item) => item.id)).toContain( message.id, ); const sent = await request( `/api/v1/chats/${chat.id}/messages`, json({ text: 'Сообщение после отказа' }, bearer(token)), ); expect(sent.response.status).toBe(201); }, outagedRequestBudgetMs, ); });