Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
445 lines
14 KiB
JavaScript
445 lines
14 KiB
JavaScript
import 'dotenv/config';
|
|
import assert from 'node:assert/strict';
|
|
import { randomBytes, randomUUID } from 'node:crypto';
|
|
import { createRequire } from 'node:module';
|
|
import { performance } from 'node:perf_hooks';
|
|
import test from 'node:test';
|
|
import { faultProxy } from './helpers/fault-proxy.mjs';
|
|
|
|
const require = createRequire(import.meta.url);
|
|
const Redis = require('ioredis').default;
|
|
const { NestFactory } = require('@nestjs/core');
|
|
const { AppModule } = require('../dist/app.module');
|
|
const { PrismaService } = require('../dist/prisma/prisma.service');
|
|
const {
|
|
createFastifyAdapter,
|
|
exposeRequestIdHeader,
|
|
} = require('../dist/common/http/fastify-adapter');
|
|
const pause = (milliseconds) =>
|
|
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
|
|
test(
|
|
'Auth and dependency failures remain bounded and fail closed',
|
|
{ timeout: 120000 },
|
|
async (t) => {
|
|
assert.ok(
|
|
process.env.DATABASE_URL,
|
|
'Set DATABASE_URL and apply migrations',
|
|
);
|
|
assert.ok(
|
|
process.env.CACHE_URL,
|
|
'Set CACHE_URL to a running Redis instance',
|
|
);
|
|
const original = {
|
|
DATABASE_URL: process.env.DATABASE_URL,
|
|
CACHE_URL: process.env.CACHE_URL,
|
|
CACHE_PREFIX: process.env.CACHE_PREFIX,
|
|
SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS,
|
|
};
|
|
const prefix = `test:resilience:${randomUUID()}`;
|
|
const db = await faultProxy(original.DATABASE_URL, 5432);
|
|
const redis = await faultProxy(original.CACHE_URL, 6379);
|
|
Object.assign(process.env, {
|
|
DATABASE_URL: db.url,
|
|
CACHE_URL: redis.url,
|
|
CACHE_PREFIX: prefix,
|
|
SESSION_TTL_SECONDS: '2',
|
|
});
|
|
const observer = new Redis(original.CACHE_URL, {
|
|
connectTimeout: 1000,
|
|
commandTimeout: 1000,
|
|
maxRetriesPerRequest: 0,
|
|
});
|
|
observer.on('error', () => {});
|
|
let app;
|
|
let prisma;
|
|
let userId;
|
|
let chatId;
|
|
t.after(async () => {
|
|
db.setMode('online');
|
|
redis.setMode('online');
|
|
try {
|
|
if (prisma) {
|
|
if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } });
|
|
if (userId) await prisma.user.deleteMany({ where: { id: userId } });
|
|
}
|
|
let cursor = '0';
|
|
do {
|
|
const [next, keys] = await observer.scan(
|
|
cursor,
|
|
'MATCH',
|
|
`${prefix}:*`,
|
|
'COUNT',
|
|
100,
|
|
);
|
|
cursor = next;
|
|
if (keys.length) await observer.del(...keys);
|
|
} while (cursor !== '0');
|
|
} finally {
|
|
observer.disconnect();
|
|
if (app) await app.close();
|
|
await db.close();
|
|
await redis.close();
|
|
for (const [key, value] of Object.entries(original)) {
|
|
if (value === undefined) delete process.env[key];
|
|
else process.env[key] = value;
|
|
}
|
|
}
|
|
});
|
|
await observer.ping();
|
|
app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
|
logger: false,
|
|
abortOnError: false,
|
|
});
|
|
exposeRequestIdHeader(app);
|
|
await app.init();
|
|
const server = app.getHttpAdapter().getInstance();
|
|
await server.ready();
|
|
prisma = app.get(PrismaService);
|
|
let token;
|
|
const request = async (
|
|
method,
|
|
url,
|
|
{
|
|
payload,
|
|
authorization = token ? `Bearer ${token}` : undefined,
|
|
headers = {},
|
|
budget = 8000,
|
|
} = {},
|
|
) => {
|
|
const start = performance.now();
|
|
let timer;
|
|
try {
|
|
const response = await Promise.race([
|
|
server.inject({
|
|
method,
|
|
url,
|
|
headers: {
|
|
...headers,
|
|
...(method === 'POST' && /\/messages$/.test(url)
|
|
? { 'idempotency-key': randomUUID() }
|
|
: {}),
|
|
...(authorization ? { authorization } : {}),
|
|
},
|
|
...(payload === undefined ? {} : { payload }),
|
|
}),
|
|
new Promise((_, reject) => {
|
|
timer = setTimeout(
|
|
() => reject(new Error(`${method} ${url} exceeded ${budget}ms`)),
|
|
budget,
|
|
);
|
|
}),
|
|
]);
|
|
assert.ok(
|
|
performance.now() - start < budget,
|
|
'request exceeded its deadline',
|
|
);
|
|
return response;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
};
|
|
const errorResponse = (response, status, code, challenge) => {
|
|
assert.equal(
|
|
response.statusCode,
|
|
status,
|
|
`${response.statusCode}: ${response.body}`,
|
|
);
|
|
const body = response.json();
|
|
assert.deepEqual(Object.keys(body), ['error']);
|
|
assert.deepEqual(Object.keys(body.error).sort(), [
|
|
'code',
|
|
'message',
|
|
'request_id',
|
|
]);
|
|
assert.equal(body.error.code, code);
|
|
assert.equal(body.error.request_id, response.headers['x-request-id']);
|
|
assert.ok(body.error.request_id);
|
|
assert.ok(body.error.message.length <= 200);
|
|
for (const secret of [
|
|
original.DATABASE_URL,
|
|
original.CACHE_URL,
|
|
token,
|
|
].filter(Boolean))
|
|
assert.ok(!response.body.includes(secret));
|
|
assert.ok(
|
|
!/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test(
|
|
response.body,
|
|
),
|
|
'error leaks implementation details',
|
|
);
|
|
if (challenge)
|
|
assert.equal(response.headers['www-authenticate'], challenge);
|
|
};
|
|
const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`;
|
|
const password = 'Synthetic-test-password:🔐123';
|
|
const basic = (secret) =>
|
|
`Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
|
|
const login = (secret = password) =>
|
|
request('POST', '/api/v1/auth/sessions', {
|
|
authorization: basic(secret),
|
|
});
|
|
const registered = await request('POST', '/api/v1/users', {
|
|
payload: { username, display_name: 'Fault test', password },
|
|
budget: 15000,
|
|
});
|
|
assert.equal(registered.statusCode, 201);
|
|
userId = registered.json().id;
|
|
|
|
await t.test('incorrect password is 401 with Basic challenge', async () => {
|
|
const wrong = await login('wrong-password');
|
|
errorResponse(
|
|
wrong,
|
|
401,
|
|
'unauthorized',
|
|
'Basic realm="messenger", charset="UTF-8"',
|
|
);
|
|
assert.equal(await prisma.session.count({ where: { userId } }), 0);
|
|
});
|
|
await t.test(
|
|
'SESSION_TTL_SECONDS=2 expires without renewing on reads',
|
|
async () => {
|
|
const response = await login();
|
|
assert.equal(response.statusCode, 201);
|
|
token = response.json().token;
|
|
const stored = await prisma.session.findFirstOrThrow({
|
|
where: { userId },
|
|
});
|
|
assert.ok(
|
|
Math.abs(
|
|
stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000,
|
|
) < 500,
|
|
);
|
|
assert.equal(
|
|
(await request('GET', '/api/v1/users/me')).statusCode,
|
|
200,
|
|
);
|
|
await pause(
|
|
Math.max(0, Date.parse(response.json().expires_at) - Date.now()) +
|
|
100,
|
|
);
|
|
errorResponse(
|
|
await request('GET', '/api/v1/users/me'),
|
|
401,
|
|
'unauthorized',
|
|
'Bearer realm="messenger"',
|
|
);
|
|
errorResponse(
|
|
await request('DELETE', '/api/v1/auth/sessions/current'),
|
|
401,
|
|
'unauthorized',
|
|
'Bearer realm="messenger"',
|
|
);
|
|
},
|
|
);
|
|
await t.test(
|
|
'logout revokes immediately, including with a cached profile',
|
|
async () => {
|
|
const response = await login();
|
|
assert.equal(response.statusCode, 201);
|
|
token = response.json().token;
|
|
assert.equal(
|
|
(await request('GET', '/api/v1/users/me')).statusCode,
|
|
200,
|
|
);
|
|
const logout = await request('DELETE', '/api/v1/auth/sessions/current');
|
|
assert.equal(logout.statusCode, 204);
|
|
assert.equal(logout.body, '');
|
|
errorResponse(
|
|
await request('GET', '/api/v1/users/me'),
|
|
401,
|
|
'unauthorized',
|
|
'Bearer realm="messenger"',
|
|
);
|
|
errorResponse(
|
|
await request('DELETE', '/api/v1/auth/sessions/current'),
|
|
401,
|
|
'unauthorized',
|
|
'Bearer realm="messenger"',
|
|
);
|
|
},
|
|
);
|
|
token = randomBytes(32).toString('base64url');
|
|
const { createHash } = await import('node:crypto');
|
|
await prisma.session.create({
|
|
data: {
|
|
userId,
|
|
tokenHash: createHash('sha256').update(token).digest('hex'),
|
|
expiresAt: new Date(Date.now() + 600000),
|
|
},
|
|
});
|
|
const chat = await request('POST', '/api/v1/chats', {
|
|
payload: { title: 'Fault isolation', member_ids: [userId] },
|
|
});
|
|
assert.equal(chat.statusCode, 201);
|
|
chatId = chat.json().id;
|
|
const history = `/api/v1/chats/${chatId}/messages`;
|
|
assert.equal(
|
|
(await request('POST', history, { payload: { text: 'Before fault' } }))
|
|
.statusCode,
|
|
201,
|
|
);
|
|
assert.equal((await request('GET', history)).json().items.length, 1);
|
|
|
|
const anonymousChecks = async () => {
|
|
for (const authorization of ['', 'Bearer invalid', basic(password)]) {
|
|
const headers = { 'x-user-id': userId };
|
|
errorResponse(
|
|
await request('GET', history, { authorization, headers }),
|
|
401,
|
|
'unauthorized',
|
|
'Bearer realm="messenger"',
|
|
);
|
|
errorResponse(
|
|
await request('POST', history, {
|
|
authorization,
|
|
headers,
|
|
payload: { text: 'Must not be stored' },
|
|
}),
|
|
401,
|
|
'unauthorized',
|
|
'Bearer realm="messenger"',
|
|
);
|
|
}
|
|
};
|
|
await t.test(
|
|
'Redis disconnect: SQL fallback and anonymous callers remain unauthorized',
|
|
async () => {
|
|
redis.setMode('cut');
|
|
await anonymousChecks();
|
|
assert.equal(
|
|
(await request('GET', history, { budget: 3000 })).statusCode,
|
|
200,
|
|
);
|
|
assert.equal(
|
|
(
|
|
await request('POST', history, {
|
|
payload: { text: 'Redis offline' },
|
|
budget: 3000,
|
|
})
|
|
).statusCode,
|
|
201,
|
|
);
|
|
assert.equal(
|
|
(await request('GET', history, { budget: 3000 })).json().items.length,
|
|
2,
|
|
);
|
|
redis.setMode('online');
|
|
await pause(1300);
|
|
},
|
|
);
|
|
await t.test('Redis hangs: command timeout bypasses cache', async () => {
|
|
redis.setMode('hang');
|
|
try {
|
|
assert.equal(
|
|
(await request('GET', `${history}?limit=10`, { budget: 3000 }))
|
|
.statusCode,
|
|
200,
|
|
);
|
|
assert.equal(
|
|
(
|
|
await request('POST', history, {
|
|
payload: { text: 'Redis blackhole' },
|
|
budget: 3000,
|
|
})
|
|
).statusCode,
|
|
201,
|
|
);
|
|
await anonymousChecks();
|
|
} finally {
|
|
redis.setMode('cut');
|
|
redis.setMode('online');
|
|
}
|
|
await pause(1300);
|
|
});
|
|
await t.test(
|
|
'PostgreSQL disconnect: bounded 503, no cached authorization bypass',
|
|
async () => {
|
|
db.setMode('cut');
|
|
try {
|
|
await anonymousChecks();
|
|
errorResponse(
|
|
await request('GET', '/api/v1/users/me'),
|
|
503,
|
|
'unavailable',
|
|
);
|
|
errorResponse(await request('GET', history), 503, 'unavailable');
|
|
errorResponse(
|
|
await request('POST', history, { payload: { text: 'DB offline' } }),
|
|
503,
|
|
'unavailable',
|
|
);
|
|
errorResponse(
|
|
await request('DELETE', '/api/v1/auth/sessions/current'),
|
|
503,
|
|
'unavailable',
|
|
);
|
|
errorResponse(await login(), 503, 'unavailable');
|
|
errorResponse(
|
|
await request('GET', '/health/ready'),
|
|
503,
|
|
'unavailable',
|
|
);
|
|
assert.equal((await request('GET', '/health/live')).statusCode, 200);
|
|
} finally {
|
|
db.setMode('online');
|
|
}
|
|
assert.equal(
|
|
(await request('GET', '/api/v1/users/me')).statusCode,
|
|
200,
|
|
);
|
|
},
|
|
);
|
|
await t.test(
|
|
'PostgreSQL hangs on an established connection: client query timeout',
|
|
async () => {
|
|
assert.equal(
|
|
(await request('GET', '/api/v1/users/me')).statusCode,
|
|
200,
|
|
);
|
|
db.setMode('hang');
|
|
try {
|
|
await anonymousChecks();
|
|
errorResponse(await request('GET', history), 503, 'unavailable');
|
|
} finally {
|
|
db.setMode('cut');
|
|
db.setMode('online');
|
|
}
|
|
assert.equal(
|
|
(await request('GET', '/api/v1/users/me')).statusCode,
|
|
200,
|
|
);
|
|
},
|
|
);
|
|
await t.test(
|
|
'both services unavailable: fail closed and recover confirmed data',
|
|
async () => {
|
|
db.setMode('cut');
|
|
redis.setMode('cut');
|
|
try {
|
|
await anonymousChecks();
|
|
errorResponse(await request('GET', history), 503, 'unavailable');
|
|
errorResponse(
|
|
await request('GET', history, {
|
|
authorization: `Bearer ${randomBytes(32).toString('base64url')}`,
|
|
}),
|
|
503,
|
|
'unavailable',
|
|
);
|
|
} finally {
|
|
db.setMode('online');
|
|
redis.setMode('online');
|
|
}
|
|
const recovered = await request('GET', `${history}?limit=100`);
|
|
assert.equal(recovered.statusCode, 200);
|
|
assert.deepEqual(
|
|
recovered.json().items.map((message) => message.text),
|
|
['Before fault', 'Redis offline', 'Redis blackhole'],
|
|
);
|
|
assert.equal(await prisma.message.count({ where: { chatId } }), 3);
|
|
assert.equal((await request('GET', '/health/ready')).statusCode, 200);
|
|
},
|
|
);
|
|
},
|
|
);
|