forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,444 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import test from 'node:test';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
const pause = (milliseconds) =>
|
||||
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
test(
|
||||
'Auth and dependency failures remain bounded and fail closed',
|
||||
{ timeout: 120000 },
|
||||
async (t) => {
|
||||
assert.ok(
|
||||
process.env.DATABASE_URL,
|
||||
'Set DATABASE_URL and apply migrations',
|
||||
);
|
||||
assert.ok(
|
||||
process.env.CACHE_URL,
|
||||
'Set CACHE_URL to a running Redis instance',
|
||||
);
|
||||
const original = {
|
||||
DATABASE_URL: process.env.DATABASE_URL,
|
||||
CACHE_URL: process.env.CACHE_URL,
|
||||
CACHE_PREFIX: process.env.CACHE_PREFIX,
|
||||
SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS,
|
||||
};
|
||||
const prefix = `test:resilience:${randomUUID()}`;
|
||||
const db = await faultProxy(original.DATABASE_URL, 5432);
|
||||
const redis = await faultProxy(original.CACHE_URL, 6379);
|
||||
Object.assign(process.env, {
|
||||
DATABASE_URL: db.url,
|
||||
CACHE_URL: redis.url,
|
||||
CACHE_PREFIX: prefix,
|
||||
SESSION_TTL_SECONDS: '2',
|
||||
});
|
||||
const observer = new Redis(original.CACHE_URL, {
|
||||
connectTimeout: 1000,
|
||||
commandTimeout: 1000,
|
||||
maxRetriesPerRequest: 0,
|
||||
});
|
||||
observer.on('error', () => {});
|
||||
let app;
|
||||
let prisma;
|
||||
let userId;
|
||||
let chatId;
|
||||
t.after(async () => {
|
||||
db.setMode('online');
|
||||
redis.setMode('online');
|
||||
try {
|
||||
if (prisma) {
|
||||
if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } });
|
||||
if (userId) await prisma.user.deleteMany({ where: { id: userId } });
|
||||
}
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
observer.disconnect();
|
||||
if (app) await app.close();
|
||||
await db.close();
|
||||
await redis.close();
|
||||
for (const [key, value] of Object.entries(original)) {
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
await observer.ping();
|
||||
app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
const server = app.getHttpAdapter().getInstance();
|
||||
await server.ready();
|
||||
prisma = app.get(PrismaService);
|
||||
let token;
|
||||
const request = async (
|
||||
method,
|
||||
url,
|
||||
{
|
||||
payload,
|
||||
authorization = token ? `Bearer ${token}` : undefined,
|
||||
headers = {},
|
||||
budget = 8000,
|
||||
} = {},
|
||||
) => {
|
||||
const start = performance.now();
|
||||
let timer;
|
||||
try {
|
||||
const response = await Promise.race([
|
||||
server.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
...headers,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
...(authorization ? { authorization } : {}),
|
||||
},
|
||||
...(payload === undefined ? {} : { payload }),
|
||||
}),
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error(`${method} ${url} exceeded ${budget}ms`)),
|
||||
budget,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
assert.ok(
|
||||
performance.now() - start < budget,
|
||||
'request exceeded its deadline',
|
||||
);
|
||||
return response;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
const errorResponse = (response, status, code, challenge) => {
|
||||
assert.equal(
|
||||
response.statusCode,
|
||||
status,
|
||||
`${response.statusCode}: ${response.body}`,
|
||||
);
|
||||
const body = response.json();
|
||||
assert.deepEqual(Object.keys(body), ['error']);
|
||||
assert.deepEqual(Object.keys(body.error).sort(), [
|
||||
'code',
|
||||
'message',
|
||||
'request_id',
|
||||
]);
|
||||
assert.equal(body.error.code, code);
|
||||
assert.equal(body.error.request_id, response.headers['x-request-id']);
|
||||
assert.ok(body.error.request_id);
|
||||
assert.ok(body.error.message.length <= 200);
|
||||
for (const secret of [
|
||||
original.DATABASE_URL,
|
||||
original.CACHE_URL,
|
||||
token,
|
||||
].filter(Boolean))
|
||||
assert.ok(!response.body.includes(secret));
|
||||
assert.ok(
|
||||
!/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test(
|
||||
response.body,
|
||||
),
|
||||
'error leaks implementation details',
|
||||
);
|
||||
if (challenge)
|
||||
assert.equal(response.headers['www-authenticate'], challenge);
|
||||
};
|
||||
const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`;
|
||||
const password = 'Synthetic-test-password:🔐123';
|
||||
const basic = (secret) =>
|
||||
`Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
|
||||
const login = (secret = password) =>
|
||||
request('POST', '/api/v1/auth/sessions', {
|
||||
authorization: basic(secret),
|
||||
});
|
||||
const registered = await request('POST', '/api/v1/users', {
|
||||
payload: { username, display_name: 'Fault test', password },
|
||||
budget: 15000,
|
||||
});
|
||||
assert.equal(registered.statusCode, 201);
|
||||
userId = registered.json().id;
|
||||
|
||||
await t.test('incorrect password is 401 with Basic challenge', async () => {
|
||||
const wrong = await login('wrong-password');
|
||||
errorResponse(
|
||||
wrong,
|
||||
401,
|
||||
'unauthorized',
|
||||
'Basic realm="messenger", charset="UTF-8"',
|
||||
);
|
||||
assert.equal(await prisma.session.count({ where: { userId } }), 0);
|
||||
});
|
||||
await t.test(
|
||||
'SESSION_TTL_SECONDS=2 expires without renewing on reads',
|
||||
async () => {
|
||||
const response = await login();
|
||||
assert.equal(response.statusCode, 201);
|
||||
token = response.json().token;
|
||||
const stored = await prisma.session.findFirstOrThrow({
|
||||
where: { userId },
|
||||
});
|
||||
assert.ok(
|
||||
Math.abs(
|
||||
stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000,
|
||||
) < 500,
|
||||
);
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
await pause(
|
||||
Math.max(0, Date.parse(response.json().expires_at) - Date.now()) +
|
||||
100,
|
||||
);
|
||||
errorResponse(
|
||||
await request('GET', '/api/v1/users/me'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
errorResponse(
|
||||
await request('DELETE', '/api/v1/auth/sessions/current'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
},
|
||||
);
|
||||
await t.test(
|
||||
'logout revokes immediately, including with a cached profile',
|
||||
async () => {
|
||||
const response = await login();
|
||||
assert.equal(response.statusCode, 201);
|
||||
token = response.json().token;
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
const logout = await request('DELETE', '/api/v1/auth/sessions/current');
|
||||
assert.equal(logout.statusCode, 204);
|
||||
assert.equal(logout.body, '');
|
||||
errorResponse(
|
||||
await request('GET', '/api/v1/users/me'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
errorResponse(
|
||||
await request('DELETE', '/api/v1/auth/sessions/current'),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
},
|
||||
);
|
||||
token = randomBytes(32).toString('base64url');
|
||||
const { createHash } = await import('node:crypto');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
const chat = await request('POST', '/api/v1/chats', {
|
||||
payload: { title: 'Fault isolation', member_ids: [userId] },
|
||||
});
|
||||
assert.equal(chat.statusCode, 201);
|
||||
chatId = chat.json().id;
|
||||
const history = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal(
|
||||
(await request('POST', history, { payload: { text: 'Before fault' } }))
|
||||
.statusCode,
|
||||
201,
|
||||
);
|
||||
assert.equal((await request('GET', history)).json().items.length, 1);
|
||||
|
||||
const anonymousChecks = async () => {
|
||||
for (const authorization of ['', 'Bearer invalid', basic(password)]) {
|
||||
const headers = { 'x-user-id': userId };
|
||||
errorResponse(
|
||||
await request('GET', history, { authorization, headers }),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
errorResponse(
|
||||
await request('POST', history, {
|
||||
authorization,
|
||||
headers,
|
||||
payload: { text: 'Must not be stored' },
|
||||
}),
|
||||
401,
|
||||
'unauthorized',
|
||||
'Bearer realm="messenger"',
|
||||
);
|
||||
}
|
||||
};
|
||||
await t.test(
|
||||
'Redis disconnect: SQL fallback and anonymous callers remain unauthorized',
|
||||
async () => {
|
||||
redis.setMode('cut');
|
||||
await anonymousChecks();
|
||||
assert.equal(
|
||||
(await request('GET', history, { budget: 3000 })).statusCode,
|
||||
200,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await request('POST', history, {
|
||||
payload: { text: 'Redis offline' },
|
||||
budget: 3000,
|
||||
})
|
||||
).statusCode,
|
||||
201,
|
||||
);
|
||||
assert.equal(
|
||||
(await request('GET', history, { budget: 3000 })).json().items.length,
|
||||
2,
|
||||
);
|
||||
redis.setMode('online');
|
||||
await pause(1300);
|
||||
},
|
||||
);
|
||||
await t.test('Redis hangs: command timeout bypasses cache', async () => {
|
||||
redis.setMode('hang');
|
||||
try {
|
||||
assert.equal(
|
||||
(await request('GET', `${history}?limit=10`, { budget: 3000 }))
|
||||
.statusCode,
|
||||
200,
|
||||
);
|
||||
assert.equal(
|
||||
(
|
||||
await request('POST', history, {
|
||||
payload: { text: 'Redis blackhole' },
|
||||
budget: 3000,
|
||||
})
|
||||
).statusCode,
|
||||
201,
|
||||
);
|
||||
await anonymousChecks();
|
||||
} finally {
|
||||
redis.setMode('cut');
|
||||
redis.setMode('online');
|
||||
}
|
||||
await pause(1300);
|
||||
});
|
||||
await t.test(
|
||||
'PostgreSQL disconnect: bounded 503, no cached authorization bypass',
|
||||
async () => {
|
||||
db.setMode('cut');
|
||||
try {
|
||||
await anonymousChecks();
|
||||
errorResponse(
|
||||
await request('GET', '/api/v1/users/me'),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
errorResponse(await request('GET', history), 503, 'unavailable');
|
||||
errorResponse(
|
||||
await request('POST', history, { payload: { text: 'DB offline' } }),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
errorResponse(
|
||||
await request('DELETE', '/api/v1/auth/sessions/current'),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
errorResponse(await login(), 503, 'unavailable');
|
||||
errorResponse(
|
||||
await request('GET', '/health/ready'),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
assert.equal((await request('GET', '/health/live')).statusCode, 200);
|
||||
} finally {
|
||||
db.setMode('online');
|
||||
}
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
},
|
||||
);
|
||||
await t.test(
|
||||
'PostgreSQL hangs on an established connection: client query timeout',
|
||||
async () => {
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
db.setMode('hang');
|
||||
try {
|
||||
await anonymousChecks();
|
||||
errorResponse(await request('GET', history), 503, 'unavailable');
|
||||
} finally {
|
||||
db.setMode('cut');
|
||||
db.setMode('online');
|
||||
}
|
||||
assert.equal(
|
||||
(await request('GET', '/api/v1/users/me')).statusCode,
|
||||
200,
|
||||
);
|
||||
},
|
||||
);
|
||||
await t.test(
|
||||
'both services unavailable: fail closed and recover confirmed data',
|
||||
async () => {
|
||||
db.setMode('cut');
|
||||
redis.setMode('cut');
|
||||
try {
|
||||
await anonymousChecks();
|
||||
errorResponse(await request('GET', history), 503, 'unavailable');
|
||||
errorResponse(
|
||||
await request('GET', history, {
|
||||
authorization: `Bearer ${randomBytes(32).toString('base64url')}`,
|
||||
}),
|
||||
503,
|
||||
'unavailable',
|
||||
);
|
||||
} finally {
|
||||
db.setMode('online');
|
||||
redis.setMode('online');
|
||||
}
|
||||
const recovered = await request('GET', `${history}?limit=100`);
|
||||
assert.equal(recovered.statusCode, 200);
|
||||
assert.deepEqual(
|
||||
recovered.json().items.map((message) => message.text),
|
||||
['Before fault', 'Redis offline', 'Redis blackhole'],
|
||||
);
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 3);
|
||||
assert.equal((await request('GET', '/health/ready')).statusCode, 200);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
Reference in New Issue
Block a user