Files
mtusi2026lab2-fork/test/resilience.mjs
T

442 lines
14 KiB
JavaScript

import 'dotenv/config';
import assert from 'node:assert/strict';
import { randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import { performance } from 'node:perf_hooks';
import test from 'node:test';
import { faultProxy } from './helpers/fault-proxy.mjs';
const require = createRequire(import.meta.url);
const Redis = require('ioredis').default;
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
const pause = (milliseconds) =>
new Promise((resolve) => setTimeout(resolve, milliseconds));
test(
'Auth and dependency failures remain bounded and fail closed',
{ timeout: 120000 },
async (t) => {
assert.ok(
process.env.DATABASE_URL,
'Set DATABASE_URL and apply migrations',
);
assert.ok(
process.env.CACHE_URL,
'Set CACHE_URL to a running Redis instance',
);
const original = {
DATABASE_URL: process.env.DATABASE_URL,
CACHE_URL: process.env.CACHE_URL,
CACHE_PREFIX: process.env.CACHE_PREFIX,
SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS,
};
const prefix = `test:resilience:${randomUUID()}`;
const db = await faultProxy(original.DATABASE_URL, 5432);
const redis = await faultProxy(original.CACHE_URL, 6379);
Object.assign(process.env, {
DATABASE_URL: db.url,
CACHE_URL: redis.url,
CACHE_PREFIX: prefix,
SESSION_TTL_SECONDS: '2',
});
const observer = new Redis(original.CACHE_URL, {
connectTimeout: 1000,
commandTimeout: 1000,
maxRetriesPerRequest: 0,
});
observer.on('error', () => {});
let app;
let prisma;
let userId;
let chatId;
t.after(async () => {
db.setMode('online');
redis.setMode('online');
try {
if (prisma) {
if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } });
if (userId) await prisma.user.deleteMany({ where: { id: userId } });
}
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
} finally {
observer.disconnect();
if (app) await app.close();
await db.close();
await redis.close();
for (const [key, value] of Object.entries(original)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
await observer.ping();
app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
exposeRequestIdHeader(app);
await app.init();
const server = app.getHttpAdapter().getInstance();
await server.ready();
prisma = app.get(PrismaService);
let token;
const request = async (
method,
url,
{
payload,
authorization = token ? `Bearer ${token}` : undefined,
headers = {},
budget = 8000,
} = {},
) => {
const start = performance.now();
let timer;
try {
const response = await Promise.race([
server.inject({
method,
url,
headers: {
...headers,
...(authorization ? { authorization } : {}),
},
...(payload === undefined ? {} : { payload }),
}),
new Promise((_, reject) => {
timer = setTimeout(
() => reject(new Error(`${method} ${url} exceeded ${budget}ms`)),
budget,
);
}),
]);
assert.ok(
performance.now() - start < budget,
'request exceeded its deadline',
);
return response;
} finally {
clearTimeout(timer);
}
};
const errorResponse = (response, status, code, challenge) => {
assert.equal(
response.statusCode,
status,
`${response.statusCode}: ${response.body}`,
);
const body = response.json();
assert.deepEqual(Object.keys(body), ['error']);
assert.deepEqual(Object.keys(body.error).sort(), [
'code',
'message',
'request_id',
]);
assert.equal(body.error.code, code);
assert.equal(body.error.request_id, response.headers['x-request-id']);
assert.ok(body.error.request_id);
assert.ok(body.error.message.length <= 200);
for (const secret of [
original.DATABASE_URL,
original.CACHE_URL,
token,
].filter(Boolean))
assert.ok(!response.body.includes(secret));
assert.ok(
!/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test(
response.body,
),
'error leaks implementation details',
);
if (challenge)
assert.equal(response.headers['www-authenticate'], challenge);
};
const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`;
const password = 'Synthetic-test-password:🔐123';
const basic = (secret) =>
`Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
const login = (secret = password) =>
request('POST', '/api/v1/auth/sessions', {
authorization: basic(secret),
});
const registered = await request('POST', '/api/v1/users', {
payload: { username, display_name: 'Fault test', password },
budget: 15000,
});
assert.equal(registered.statusCode, 201);
userId = registered.json().id;
await t.test('incorrect password is 401 with Basic challenge', async () => {
const wrong = await login('wrong-password');
errorResponse(
wrong,
401,
'unauthorized',
'Basic realm="messenger", charset="UTF-8"',
);
assert.equal(await prisma.session.count({ where: { userId } }), 0);
});
await t.test(
'SESSION_TTL_SECONDS=2 expires without renewing on reads',
async () => {
const response = await login();
assert.equal(response.statusCode, 201);
token = response.json().token;
const stored = await prisma.session.findFirstOrThrow({
where: { userId },
});
assert.ok(
Math.abs(
stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000,
) < 500,
);
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
await pause(
Math.max(0, Date.parse(response.json().expires_at) - Date.now()) +
100,
);
errorResponse(
await request('GET', '/api/v1/users/me'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
errorResponse(
await request('DELETE', '/api/v1/auth/sessions/current'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
},
);
await t.test(
'logout revokes immediately, including with a cached profile',
async () => {
const response = await login();
assert.equal(response.statusCode, 201);
token = response.json().token;
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
const logout = await request('DELETE', '/api/v1/auth/sessions/current');
assert.equal(logout.statusCode, 204);
assert.equal(logout.body, '');
errorResponse(
await request('GET', '/api/v1/users/me'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
errorResponse(
await request('DELETE', '/api/v1/auth/sessions/current'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
},
);
token = randomBytes(32).toString('base64url');
const { createHash } = await import('node:crypto');
await prisma.session.create({
data: {
userId,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 600000),
},
});
const chat = await request('POST', '/api/v1/chats', {
payload: { title: 'Fault isolation', member_ids: [userId] },
});
assert.equal(chat.statusCode, 201);
chatId = chat.json().id;
const history = `/api/v1/chats/${chatId}/messages`;
assert.equal(
(await request('POST', history, { payload: { text: 'Before fault' } }))
.statusCode,
201,
);
assert.equal((await request('GET', history)).json().items.length, 1);
const anonymousChecks = async () => {
for (const authorization of ['', 'Bearer invalid', basic(password)]) {
const headers = { 'x-user-id': userId };
errorResponse(
await request('GET', history, { authorization, headers }),
401,
'unauthorized',
'Bearer realm="messenger"',
);
errorResponse(
await request('POST', history, {
authorization,
headers,
payload: { text: 'Must not be stored' },
}),
401,
'unauthorized',
'Bearer realm="messenger"',
);
}
};
await t.test(
'Redis disconnect: SQL fallback and anonymous callers remain unauthorized',
async () => {
redis.setMode('cut');
await anonymousChecks();
assert.equal(
(await request('GET', history, { budget: 3000 })).statusCode,
200,
);
assert.equal(
(
await request('POST', history, {
payload: { text: 'Redis offline' },
budget: 3000,
})
).statusCode,
201,
);
assert.equal(
(await request('GET', history, { budget: 3000 })).json().items.length,
2,
);
redis.setMode('online');
await pause(1300);
},
);
await t.test('Redis hangs: command timeout bypasses cache', async () => {
redis.setMode('hang');
try {
assert.equal(
(await request('GET', `${history}?limit=10`, { budget: 3000 }))
.statusCode,
200,
);
assert.equal(
(
await request('POST', history, {
payload: { text: 'Redis blackhole' },
budget: 3000,
})
).statusCode,
201,
);
await anonymousChecks();
} finally {
redis.setMode('cut');
redis.setMode('online');
}
await pause(1300);
});
await t.test(
'PostgreSQL disconnect: bounded 503, no cached authorization bypass',
async () => {
db.setMode('cut');
try {
await anonymousChecks();
errorResponse(
await request('GET', '/api/v1/users/me'),
503,
'unavailable',
);
errorResponse(await request('GET', history), 503, 'unavailable');
errorResponse(
await request('POST', history, { payload: { text: 'DB offline' } }),
503,
'unavailable',
);
errorResponse(
await request('DELETE', '/api/v1/auth/sessions/current'),
503,
'unavailable',
);
errorResponse(await login(), 503, 'unavailable');
errorResponse(
await request('GET', '/health/ready'),
503,
'unavailable',
);
assert.equal((await request('GET', '/health/live')).statusCode, 200);
} finally {
db.setMode('online');
}
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
},
);
await t.test(
'PostgreSQL hangs on an established connection: client query timeout',
async () => {
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
db.setMode('hang');
try {
await anonymousChecks();
errorResponse(await request('GET', history), 503, 'unavailable');
} finally {
db.setMode('cut');
db.setMode('online');
}
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
},
);
await t.test(
'both services unavailable: fail closed and recover confirmed data',
async () => {
db.setMode('cut');
redis.setMode('cut');
try {
await anonymousChecks();
errorResponse(await request('GET', history), 503, 'unavailable');
errorResponse(
await request('GET', history, {
authorization: `Bearer ${randomBytes(32).toString('base64url')}`,
}),
503,
'unavailable',
);
} finally {
db.setMode('online');
redis.setMode('online');
}
const recovered = await request('GET', `${history}?limit=100`);
assert.equal(recovered.statusCode, 200);
assert.deepEqual(
recovered.json().items.map((message) => message.text),
['Before fault', 'Redis offline', 'Redis blackhole'],
);
assert.equal(await prisma.message.count({ where: { chatId } }), 3);
assert.equal((await request('GET', '/health/ready')).statusCode, 200);
},
);
},
);