import 'dotenv/config'; import assert from 'node:assert/strict'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { createRequire } from 'node:module'; import test from 'node:test'; import { faultProxy } from './helpers/fault-proxy.mjs'; import { messageEpoch } from './helpers/message-epoch.mjs'; const require = createRequire(import.meta.url); const Redis = require('ioredis').default; const { NestFactory } = require('@nestjs/core'); const { AppModule } = require('../dist/app.module'); const { PrismaService } = require('../dist/prisma/prisma.service'); const { CacheService } = require('../dist/cache/cache.service'); const { ChatService } = require('../dist/Chat/chat.service'); const { createFastifyAdapter, exposeRequestIdHeader, } = require('../dist/common/http/fastify-adapter'); const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); test( 'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback', { timeout: 90000 }, async (t) => { assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL); const oldUrl = process.env.CACHE_URL; const oldPrefix = process.env.CACHE_PREFIX; const prefix = `test:epochs:${randomUUID()}`; const proxy = await faultProxy(oldUrl, 6379); const observer = new Redis(oldUrl); process.env.CACHE_URL = proxy.url; process.env.CACHE_PREFIX = prefix; const apps = []; let prisma, userId; const chatIds = []; t.after(async () => { proxy.setMode('online'); try { if (prisma) { await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); if (userId) await prisma.user.delete({ where: { id: userId } }); } let cursor = '0'; do { const [next, keys] = await observer.scan( cursor, 'MATCH', `${prefix}:*`, 'COUNT', 100, ); cursor = next; if (keys.length) await observer.del(...keys); } while (cursor !== '0'); } finally { for (const app of apps) await app.close(); observer.disconnect(); await proxy.close(); process.env.CACHE_URL = oldUrl; if (oldPrefix === undefined) delete process.env.CACHE_PREFIX; else process.env.CACHE_PREFIX = oldPrefix; } }); for (let i = 0; i < 2; i++) { const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false, }); apps.push(app); exposeRequestIdHeader(app); await app.init(); await app.getHttpAdapter().getInstance().ready(); require('nestjs-pino').PinoLogger.root.level = 'silent'; } prisma = apps[0].get(PrismaService); const user = await prisma.user.create({ data: { username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`, displayName: 'Synthetic epochs', }, }); userId = user.id; const token = randomBytes(32).toString('base64url'); await prisma.session.create({ data: { userId, tokenHash: createHash('sha256').update(token).digest('hex'), expiresAt: new Date(Date.now() + 600000), }, }); const call = async (replica, method, url, payload) => { const response = await apps[replica] .getHttpAdapter() .getInstance() .inject({ method, url, headers: { authorization: `Bearer ${token}`, ...(method === 'POST' && /\/messages$/.test(url) ? { 'idempotency-key': randomUUID() } : {}), }, ...(payload ? { payload } : {}), }); return { status: response.statusCode, body: response.json() }; }; const createChat = async () => { const response = await call(0, 'POST', '/api/v1/chats', { title: 'Epoch test', member_ids: [userId], }); assert.equal(response.status, 201); chatIds.push(response.body.id); return response.body.id; }; assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0); proxy.setMode('cut'); await pause(100); const chatId = await createChat(); proxy.setMode('online'); const reconnect = async () => { for (let i = 0; i < 60; i++) { if (apps.every((app) => app.get(CacheService).redis.status === 'ready')) return; await pause(100); } assert.fail('Redis did not reconnect'); }; await reconnect(); assert.deepEqual( (await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id), [chatId], ); assert.equal( (await prisma.user.findUniqueOrThrow({ where: { id: userId } })) .chatsEpoch, 1n, ); const path = `/api/v1/chats/${chatId}/messages`; assert.equal((await call(1, 'GET', path)).body.items.length, 0); const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`); assert.ok(oldKeys.length); proxy.setMode('cut'); await pause(100); const posted = await call(0, 'POST', path, { text: 'Write during Redis outage', }); assert.equal(posted.status, 201); proxy.setMode('online'); await reconnect(); assert.equal( await observer.exists(...oldKeys), oldKeys.length, 'old cache keys must still exist', ); assert.deepEqual( (await call(1, 'GET', path)).body.items.map((x) => x.id), [posted.body.id], ); const service = apps[0].get(ChatService); const original = service.prisma; service.prisma = original.$extends({ query: { $queryRaw({ args, query }) { const index = args.values.indexOf('Must roll back'); if (index !== -1) args.values[index] = 'x'.repeat(4001); return query(args); }, }, }); try { assert.equal( (await call(0, 'POST', path, { text: 'Must roll back' })).status, 500, ); } finally { service.prisma = original; } assert.equal(await prisma.message.count({ where: { chatId } }), 1); assert.equal(await messageEpoch(prisma, chatId), 1n); const writes = await Promise.all( Array.from({ length: 20 }, (_, i) => call(i % 2, 'POST', path, { text: `Concurrent ${i}` }), ), ); assert.ok(writes.every((x) => x.status === 201)); assert.equal(await prisma.message.count({ where: { chatId } }), 21); assert.equal(await messageEpoch(prisma, chatId), 21n); const history = (await call(1, 'GET', path)).body.items; assert.equal(new Set(history.map((x) => x.id)).size, 21); assert.ok( writes.every((x) => history.some((message) => message.id === x.body.id)), ); }, );