forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
import 'dotenv/config';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import test from 'node:test';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
import { messageEpoch } from './helpers/message-epoch.mjs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require('../dist/app.module');
|
||||
const { PrismaService } = require('../dist/prisma/prisma.service');
|
||||
const { CacheService } = require('../dist/cache/cache.service');
|
||||
const { ChatService } = require('../dist/Chat/chat.service');
|
||||
const {
|
||||
createFastifyAdapter,
|
||||
exposeRequestIdHeader,
|
||||
} = require('../dist/common/http/fastify-adapter');
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
test(
|
||||
'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback',
|
||||
{ timeout: 90000 },
|
||||
async (t) => {
|
||||
assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL);
|
||||
const oldUrl = process.env.CACHE_URL;
|
||||
const oldPrefix = process.env.CACHE_PREFIX;
|
||||
const prefix = `test:epochs:${randomUUID()}`;
|
||||
const proxy = await faultProxy(oldUrl, 6379);
|
||||
const observer = new Redis(oldUrl);
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
const apps = [];
|
||||
let prisma, userId;
|
||||
const chatIds = [];
|
||||
t.after(async () => {
|
||||
proxy.setMode('online');
|
||||
try {
|
||||
if (prisma) {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
}
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
} finally {
|
||||
for (const app of apps) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
process.env.CACHE_URL = oldUrl;
|
||||
if (oldPrefix === undefined) delete process.env.CACHE_PREFIX;
|
||||
else process.env.CACHE_PREFIX = oldPrefix;
|
||||
}
|
||||
});
|
||||
for (let i = 0; i < 2; i++) {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps.push(app);
|
||||
exposeRequestIdHeader(app);
|
||||
await app.init();
|
||||
await app.getHttpAdapter().getInstance().ready();
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
}
|
||||
prisma = apps[0].get(PrismaService);
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic epochs',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
const token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 600000),
|
||||
},
|
||||
});
|
||||
const call = async (replica, method, url, payload) => {
|
||||
const response = await apps[replica]
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
.inject({
|
||||
method,
|
||||
url,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(method === 'POST' && /\/messages$/.test(url)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(payload ? { payload } : {}),
|
||||
});
|
||||
return { status: response.statusCode, body: response.json() };
|
||||
};
|
||||
const createChat = async () => {
|
||||
const response = await call(0, 'POST', '/api/v1/chats', {
|
||||
title: 'Epoch test',
|
||||
member_ids: [userId],
|
||||
});
|
||||
assert.equal(response.status, 201);
|
||||
chatIds.push(response.body.id);
|
||||
return response.body.id;
|
||||
};
|
||||
assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const chatId = await createChat();
|
||||
proxy.setMode('online');
|
||||
const reconnect = async () => {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (apps.every((app) => app.get(CacheService).redis.status === 'ready'))
|
||||
return;
|
||||
await pause(100);
|
||||
}
|
||||
assert.fail('Redis did not reconnect');
|
||||
};
|
||||
await reconnect();
|
||||
assert.deepEqual(
|
||||
(await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id),
|
||||
[chatId],
|
||||
);
|
||||
assert.equal(
|
||||
(await prisma.user.findUniqueOrThrow({ where: { id: userId } }))
|
||||
.chatsEpoch,
|
||||
1n,
|
||||
);
|
||||
|
||||
const path = `/api/v1/chats/${chatId}/messages`;
|
||||
assert.equal((await call(1, 'GET', path)).body.items.length, 0);
|
||||
const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`);
|
||||
assert.ok(oldKeys.length);
|
||||
proxy.setMode('cut');
|
||||
await pause(100);
|
||||
const posted = await call(0, 'POST', path, {
|
||||
text: 'Write during Redis outage',
|
||||
});
|
||||
assert.equal(posted.status, 201);
|
||||
proxy.setMode('online');
|
||||
await reconnect();
|
||||
assert.equal(
|
||||
await observer.exists(...oldKeys),
|
||||
oldKeys.length,
|
||||
'old cache keys must still exist',
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await call(1, 'GET', path)).body.items.map((x) => x.id),
|
||||
[posted.body.id],
|
||||
);
|
||||
|
||||
const service = apps[0].get(ChatService);
|
||||
const original = service.prisma;
|
||||
service.prisma = original.$extends({
|
||||
query: {
|
||||
$queryRaw({ args, query }) {
|
||||
const index = args.values.indexOf('Must roll back');
|
||||
if (index !== -1) args.values[index] = 'x'.repeat(4001);
|
||||
return query(args);
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
assert.equal(
|
||||
(await call(0, 'POST', path, { text: 'Must roll back' })).status,
|
||||
500,
|
||||
);
|
||||
} finally {
|
||||
service.prisma = original;
|
||||
}
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
|
||||
assert.equal(await messageEpoch(prisma, chatId), 1n);
|
||||
|
||||
const writes = await Promise.all(
|
||||
Array.from({ length: 20 }, (_, i) =>
|
||||
call(i % 2, 'POST', path, { text: `Concurrent ${i}` }),
|
||||
),
|
||||
);
|
||||
assert.ok(writes.every((x) => x.status === 201));
|
||||
assert.equal(await prisma.message.count({ where: { chatId } }), 21);
|
||||
assert.equal(await messageEpoch(prisma, chatId), 21n);
|
||||
const history = (await call(1, 'GET', path)).body.items;
|
||||
assert.equal(new Set(history.map((x) => x.id)).size, 21);
|
||||
assert.ok(
|
||||
writes.every((x) => history.some((message) => message.id === x.body.id)),
|
||||
);
|
||||
},
|
||||
);
|
||||
Reference in New Issue
Block a user