import 'dotenv/config'; import assert from 'node:assert/strict'; import { randomBytes, randomUUID } from 'node:crypto'; import { createRequire } from 'node:module'; import { performance } from 'node:perf_hooks'; import test from 'node:test'; import { faultProxy } from './helpers/fault-proxy.mjs'; const require = createRequire(import.meta.url); const Redis = require('ioredis').default; const { NestFactory } = require('@nestjs/core'); const { AppModule } = require('../dist/app.module'); const { PrismaService } = require('../dist/prisma/prisma.service'); const { createFastifyAdapter, exposeRequestIdHeader, } = require('../dist/common/http/fastify-adapter'); const pause = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); test( 'Auth and dependency failures remain bounded and fail closed', { timeout: 120000 }, async (t) => { assert.ok( process.env.DATABASE_URL, 'Set DATABASE_URL and apply migrations', ); assert.ok( process.env.CACHE_URL, 'Set CACHE_URL to a running Redis instance', ); const original = { DATABASE_URL: process.env.DATABASE_URL, CACHE_URL: process.env.CACHE_URL, CACHE_PREFIX: process.env.CACHE_PREFIX, SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS, }; const prefix = `test:resilience:${randomUUID()}`; const db = await faultProxy(original.DATABASE_URL, 5432); const redis = await faultProxy(original.CACHE_URL, 6379); Object.assign(process.env, { DATABASE_URL: db.url, CACHE_URL: redis.url, CACHE_PREFIX: prefix, SESSION_TTL_SECONDS: '2', }); const observer = new Redis(original.CACHE_URL, { connectTimeout: 1000, commandTimeout: 1000, maxRetriesPerRequest: 0, }); observer.on('error', () => {}); let app; let prisma; let userId; let chatId; t.after(async () => { db.setMode('online'); redis.setMode('online'); try { if (prisma) { if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } }); if (userId) await prisma.user.deleteMany({ where: { id: userId } }); } let cursor = '0'; do { const [next, keys] = await observer.scan( cursor, 'MATCH', `${prefix}:*`, 'COUNT', 100, ); cursor = next; if (keys.length) await observer.del(...keys); } while (cursor !== '0'); } finally { observer.disconnect(); if (app) await app.close(); await db.close(); await redis.close(); for (const [key, value] of Object.entries(original)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } } }); await observer.ping(); app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false, }); exposeRequestIdHeader(app); await app.init(); const server = app.getHttpAdapter().getInstance(); await server.ready(); prisma = app.get(PrismaService); let token; const request = async ( method, url, { payload, authorization = token ? `Bearer ${token}` : undefined, headers = {}, budget = 8000, } = {}, ) => { const start = performance.now(); let timer; try { const response = await Promise.race([ server.inject({ method, url, headers: { ...headers, ...(authorization ? { authorization } : {}), }, ...(payload === undefined ? {} : { payload }), }), new Promise((_, reject) => { timer = setTimeout( () => reject(new Error(`${method} ${url} exceeded ${budget}ms`)), budget, ); }), ]); assert.ok( performance.now() - start < budget, 'request exceeded its deadline', ); return response; } finally { clearTimeout(timer); } }; const errorResponse = (response, status, code, challenge) => { assert.equal( response.statusCode, status, `${response.statusCode}: ${response.body}`, ); const body = response.json(); assert.deepEqual(Object.keys(body), ['error']); assert.deepEqual(Object.keys(body.error).sort(), [ 'code', 'message', 'request_id', ]); assert.equal(body.error.code, code); assert.equal(body.error.request_id, response.headers['x-request-id']); assert.ok(body.error.request_id); assert.ok(body.error.message.length <= 200); for (const secret of [ original.DATABASE_URL, original.CACHE_URL, token, ].filter(Boolean)) assert.ok(!response.body.includes(secret)); assert.ok( !/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test( response.body, ), 'error leaks implementation details', ); if (challenge) assert.equal(response.headers['www-authenticate'], challenge); }; const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`; const password = 'Synthetic-test-password:🔐123'; const basic = (secret) => `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`; const login = (secret = password) => request('POST', '/api/v1/auth/sessions', { authorization: basic(secret), }); const registered = await request('POST', '/api/v1/users', { payload: { username, display_name: 'Fault test', password }, budget: 15000, }); assert.equal(registered.statusCode, 201); userId = registered.json().id; await t.test('incorrect password is 401 with Basic challenge', async () => { const wrong = await login('wrong-password'); errorResponse( wrong, 401, 'unauthorized', 'Basic realm="messenger", charset="UTF-8"', ); assert.equal(await prisma.session.count({ where: { userId } }), 0); }); await t.test( 'SESSION_TTL_SECONDS=2 expires without renewing on reads', async () => { const response = await login(); assert.equal(response.statusCode, 201); token = response.json().token; const stored = await prisma.session.findFirstOrThrow({ where: { userId }, }); assert.ok( Math.abs( stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000, ) < 500, ); assert.equal( (await request('GET', '/api/v1/users/me')).statusCode, 200, ); await pause( Math.max(0, Date.parse(response.json().expires_at) - Date.now()) + 100, ); errorResponse( await request('GET', '/api/v1/users/me'), 401, 'unauthorized', 'Bearer realm="messenger"', ); errorResponse( await request('DELETE', '/api/v1/auth/sessions/current'), 401, 'unauthorized', 'Bearer realm="messenger"', ); }, ); await t.test( 'logout revokes immediately, including with a cached profile', async () => { const response = await login(); assert.equal(response.statusCode, 201); token = response.json().token; assert.equal( (await request('GET', '/api/v1/users/me')).statusCode, 200, ); const logout = await request('DELETE', '/api/v1/auth/sessions/current'); assert.equal(logout.statusCode, 204); assert.equal(logout.body, ''); errorResponse( await request('GET', '/api/v1/users/me'), 401, 'unauthorized', 'Bearer realm="messenger"', ); errorResponse( await request('DELETE', '/api/v1/auth/sessions/current'), 401, 'unauthorized', 'Bearer realm="messenger"', ); }, ); token = randomBytes(32).toString('base64url'); const { createHash } = await import('node:crypto'); await prisma.session.create({ data: { userId, tokenHash: createHash('sha256').update(token).digest('hex'), expiresAt: new Date(Date.now() + 600000), }, }); const chat = await request('POST', '/api/v1/chats', { payload: { title: 'Fault isolation', member_ids: [userId] }, }); assert.equal(chat.statusCode, 201); chatId = chat.json().id; const history = `/api/v1/chats/${chatId}/messages`; assert.equal( (await request('POST', history, { payload: { text: 'Before fault' } })) .statusCode, 201, ); assert.equal((await request('GET', history)).json().items.length, 1); const anonymousChecks = async () => { for (const authorization of ['', 'Bearer invalid', basic(password)]) { const headers = { 'x-user-id': userId }; errorResponse( await request('GET', history, { authorization, headers }), 401, 'unauthorized', 'Bearer realm="messenger"', ); errorResponse( await request('POST', history, { authorization, headers, payload: { text: 'Must not be stored' }, }), 401, 'unauthorized', 'Bearer realm="messenger"', ); } }; await t.test( 'Redis disconnect: SQL fallback and anonymous callers remain unauthorized', async () => { redis.setMode('cut'); await anonymousChecks(); assert.equal( (await request('GET', history, { budget: 3000 })).statusCode, 200, ); assert.equal( ( await request('POST', history, { payload: { text: 'Redis offline' }, budget: 3000, }) ).statusCode, 201, ); assert.equal( (await request('GET', history, { budget: 3000 })).json().items.length, 2, ); redis.setMode('online'); await pause(1300); }, ); await t.test('Redis hangs: command timeout bypasses cache', async () => { redis.setMode('hang'); try { assert.equal( (await request('GET', `${history}?limit=10`, { budget: 3000 })) .statusCode, 200, ); assert.equal( ( await request('POST', history, { payload: { text: 'Redis blackhole' }, budget: 3000, }) ).statusCode, 201, ); await anonymousChecks(); } finally { redis.setMode('cut'); redis.setMode('online'); } await pause(1300); }); await t.test( 'PostgreSQL disconnect: bounded 503, no cached authorization bypass', async () => { db.setMode('cut'); try { await anonymousChecks(); errorResponse( await request('GET', '/api/v1/users/me'), 503, 'unavailable', ); errorResponse(await request('GET', history), 503, 'unavailable'); errorResponse( await request('POST', history, { payload: { text: 'DB offline' } }), 503, 'unavailable', ); errorResponse( await request('DELETE', '/api/v1/auth/sessions/current'), 503, 'unavailable', ); errorResponse(await login(), 503, 'unavailable'); errorResponse( await request('GET', '/health/ready'), 503, 'unavailable', ); assert.equal((await request('GET', '/health/live')).statusCode, 200); } finally { db.setMode('online'); } assert.equal( (await request('GET', '/api/v1/users/me')).statusCode, 200, ); }, ); await t.test( 'PostgreSQL hangs on an established connection: client query timeout', async () => { assert.equal( (await request('GET', '/api/v1/users/me')).statusCode, 200, ); db.setMode('hang'); try { await anonymousChecks(); errorResponse(await request('GET', history), 503, 'unavailable'); } finally { db.setMode('cut'); db.setMode('online'); } assert.equal( (await request('GET', '/api/v1/users/me')).statusCode, 200, ); }, ); await t.test( 'both services unavailable: fail closed and recover confirmed data', async () => { db.setMode('cut'); redis.setMode('cut'); try { await anonymousChecks(); errorResponse(await request('GET', history), 503, 'unavailable'); errorResponse( await request('GET', history, { authorization: `Bearer ${randomBytes(32).toString('base64url')}`, }), 503, 'unavailable', ); } finally { db.setMode('online'); redis.setMode('online'); } const recovered = await request('GET', `${history}?limit=100`); assert.equal(recovered.statusCode, 200); assert.deepEqual( recovered.json().items.map((message) => message.text), ['Before fault', 'Redis offline', 'Redis blackhole'], ); assert.equal(await prisma.message.count({ where: { chatId } }), 3); assert.equal((await request('GET', '/health/ready')).statusCode, 200); }, ); }, );