Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
201 lines
6.6 KiB
JavaScript
201 lines
6.6 KiB
JavaScript
import 'dotenv/config';
|
|
import assert from 'node:assert/strict';
|
|
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
import { createRequire } from 'node:module';
|
|
import test from 'node:test';
|
|
import { faultProxy } from './helpers/fault-proxy.mjs';
|
|
import { messageEpoch } from './helpers/message-epoch.mjs';
|
|
|
|
const require = createRequire(import.meta.url);
|
|
const Redis = require('ioredis').default;
|
|
const { NestFactory } = require('@nestjs/core');
|
|
const { AppModule } = require('../dist/app.module');
|
|
const { PrismaService } = require('../dist/prisma/prisma.service');
|
|
const { CacheService } = require('../dist/cache/cache.service');
|
|
const { ChatService } = require('../dist/Chat/chat.service');
|
|
const {
|
|
createFastifyAdapter,
|
|
exposeRequestIdHeader,
|
|
} = require('../dist/common/http/fastify-adapter');
|
|
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
test(
|
|
'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback',
|
|
{ timeout: 90000 },
|
|
async (t) => {
|
|
assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL);
|
|
const oldUrl = process.env.CACHE_URL;
|
|
const oldPrefix = process.env.CACHE_PREFIX;
|
|
const prefix = `test:epochs:${randomUUID()}`;
|
|
const proxy = await faultProxy(oldUrl, 6379);
|
|
const observer = new Redis(oldUrl);
|
|
process.env.CACHE_URL = proxy.url;
|
|
process.env.CACHE_PREFIX = prefix;
|
|
const apps = [];
|
|
let prisma, userId;
|
|
const chatIds = [];
|
|
t.after(async () => {
|
|
proxy.setMode('online');
|
|
try {
|
|
if (prisma) {
|
|
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
|
if (userId) await prisma.user.delete({ where: { id: userId } });
|
|
}
|
|
let cursor = '0';
|
|
do {
|
|
const [next, keys] = await observer.scan(
|
|
cursor,
|
|
'MATCH',
|
|
`${prefix}:*`,
|
|
'COUNT',
|
|
100,
|
|
);
|
|
cursor = next;
|
|
if (keys.length) await observer.del(...keys);
|
|
} while (cursor !== '0');
|
|
} finally {
|
|
for (const app of apps) await app.close();
|
|
observer.disconnect();
|
|
await proxy.close();
|
|
process.env.CACHE_URL = oldUrl;
|
|
if (oldPrefix === undefined) delete process.env.CACHE_PREFIX;
|
|
else process.env.CACHE_PREFIX = oldPrefix;
|
|
}
|
|
});
|
|
for (let i = 0; i < 2; i++) {
|
|
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
|
logger: false,
|
|
abortOnError: false,
|
|
});
|
|
apps.push(app);
|
|
exposeRequestIdHeader(app);
|
|
await app.init();
|
|
await app.getHttpAdapter().getInstance().ready();
|
|
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
|
}
|
|
prisma = apps[0].get(PrismaService);
|
|
const user = await prisma.user.create({
|
|
data: {
|
|
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
|
displayName: 'Synthetic epochs',
|
|
},
|
|
});
|
|
userId = user.id;
|
|
const token = randomBytes(32).toString('base64url');
|
|
await prisma.session.create({
|
|
data: {
|
|
userId,
|
|
tokenHash: createHash('sha256').update(token).digest('hex'),
|
|
expiresAt: new Date(Date.now() + 600000),
|
|
},
|
|
});
|
|
const call = async (replica, method, url, payload) => {
|
|
const response = await apps[replica]
|
|
.getHttpAdapter()
|
|
.getInstance()
|
|
.inject({
|
|
method,
|
|
url,
|
|
headers: {
|
|
authorization: `Bearer ${token}`,
|
|
...(method === 'POST' && /\/messages$/.test(url)
|
|
? { 'idempotency-key': randomUUID() }
|
|
: {}),
|
|
},
|
|
...(payload ? { payload } : {}),
|
|
});
|
|
return { status: response.statusCode, body: response.json() };
|
|
};
|
|
const createChat = async () => {
|
|
const response = await call(0, 'POST', '/api/v1/chats', {
|
|
title: 'Epoch test',
|
|
member_ids: [userId],
|
|
});
|
|
assert.equal(response.status, 201);
|
|
chatIds.push(response.body.id);
|
|
return response.body.id;
|
|
};
|
|
assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0);
|
|
proxy.setMode('cut');
|
|
await pause(100);
|
|
const chatId = await createChat();
|
|
proxy.setMode('online');
|
|
const reconnect = async () => {
|
|
for (let i = 0; i < 60; i++) {
|
|
if (apps.every((app) => app.get(CacheService).redis.status === 'ready'))
|
|
return;
|
|
await pause(100);
|
|
}
|
|
assert.fail('Redis did not reconnect');
|
|
};
|
|
await reconnect();
|
|
assert.deepEqual(
|
|
(await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id),
|
|
[chatId],
|
|
);
|
|
assert.equal(
|
|
(await prisma.user.findUniqueOrThrow({ where: { id: userId } }))
|
|
.chatsEpoch,
|
|
1n,
|
|
);
|
|
|
|
const path = `/api/v1/chats/${chatId}/messages`;
|
|
assert.equal((await call(1, 'GET', path)).body.items.length, 0);
|
|
const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`);
|
|
assert.ok(oldKeys.length);
|
|
proxy.setMode('cut');
|
|
await pause(100);
|
|
const posted = await call(0, 'POST', path, {
|
|
text: 'Write during Redis outage',
|
|
});
|
|
assert.equal(posted.status, 201);
|
|
proxy.setMode('online');
|
|
await reconnect();
|
|
assert.equal(
|
|
await observer.exists(...oldKeys),
|
|
oldKeys.length,
|
|
'old cache keys must still exist',
|
|
);
|
|
assert.deepEqual(
|
|
(await call(1, 'GET', path)).body.items.map((x) => x.id),
|
|
[posted.body.id],
|
|
);
|
|
|
|
const service = apps[0].get(ChatService);
|
|
const original = service.prisma;
|
|
service.prisma = original.$extends({
|
|
query: {
|
|
$queryRaw({ args, query }) {
|
|
const index = args.values.indexOf('Must roll back');
|
|
if (index !== -1) args.values[index] = 'x'.repeat(4001);
|
|
return query(args);
|
|
},
|
|
},
|
|
});
|
|
try {
|
|
assert.equal(
|
|
(await call(0, 'POST', path, { text: 'Must roll back' })).status,
|
|
500,
|
|
);
|
|
} finally {
|
|
service.prisma = original;
|
|
}
|
|
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
|
|
assert.equal(await messageEpoch(prisma, chatId), 1n);
|
|
|
|
const writes = await Promise.all(
|
|
Array.from({ length: 20 }, (_, i) =>
|
|
call(i % 2, 'POST', path, { text: `Concurrent ${i}` }),
|
|
),
|
|
);
|
|
assert.ok(writes.every((x) => x.status === 201));
|
|
assert.equal(await prisma.message.count({ where: { chatId } }), 21);
|
|
assert.equal(await messageEpoch(prisma, chatId), 21n);
|
|
const history = (await call(1, 'GET', path)).body.items;
|
|
assert.equal(new Set(history.map((x) => x.id)).size, 21);
|
|
assert.ok(
|
|
writes.every((x) => history.some((message) => message.id === x.body.id)),
|
|
);
|
|
},
|
|
);
|