import assert from 'node:assert/strict'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { createRequire } from 'node:module'; import Module from 'node:module'; import { readFile, writeFile } from 'node:fs/promises'; import { performance } from 'node:perf_hooks'; import { faultProxy } from './helpers/fault-proxy.mjs'; const require = createRequire('/app/package.json'); const manifest = JSON.parse(await readFile('/variants/manifest.json', 'utf8')); const pg = require('pg'); let counts = {}; const originalQuery = pg.Client.prototype.query; pg.Client.prototype.query = function (query, ...args) { const sql = typeof query === 'string' ? query : query.text; const verb = sql.trim().split(/\s+/)[0].toUpperCase(); counts[verb] = (counts[verb] ?? 0) + 1; return originalQuery.call(this, query, ...args); }; const { NestFactory } = require('@nestjs/core'); const { AppModule } = require('/app/dist/app.module'); const { PrismaService } = require('/app/dist/prisma/prisma.service'); const { CacheService } = require('/app/dist/cache/cache.service'); const { ChatService } = require('/app/dist/Chat/chat.service'); const { createFastifyAdapter, exposeRequestIdHeader, } = require('/app/dist/common/http/fastify-adapter'); const exportedClasses = { 'Auth/auth.service.ts': 'AuthService', 'User/user.service.ts': 'UserService', 'User/user.guard.ts': 'UserGuard', 'Chat/chat.service.ts': 'ChatService', 'Chat/chat.controller.ts': 'ChatController', }; const originals = {}; const variants = {}; for (const [relative, name] of Object.entries(exportedClasses)) { const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`; originals[name] = require(filename)[name]; } for (const variant of manifest.variants) { variants[variant.name] = {}; for (const [relative, name] of Object.entries(exportedClasses)) { const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`; const source = await readFile( `/variants/${variant.name}/${relative.replace(/\.ts$/, '.js')}`, 'utf8', ); const loaded = new Module(filename); loaded.filename = filename; loaded.paths = Module._nodeModulePaths('/app'); loaded._compile(source, filename); variants[variant.name][name] = loaded.exports[name]; } } function activate(name) { for (const [className, replacement] of Object.entries(variants[name])) { const target = originals[className].prototype; for (const method of Object.getOwnPropertyNames(replacement.prototype)) { if (method === 'constructor') continue; const descriptor = Object.getOwnPropertyDescriptor( replacement.prototype, method, ); const previous = target[method]; if ( typeof descriptor.value === 'function' && typeof previous === 'function' ) { for (const key of Reflect.getMetadataKeys(previous)) Reflect.defineMetadata( key, Reflect.getMetadata(key, previous), descriptor.value, ); } Object.defineProperty(target, method, descriptor); } } } const Redis = require('ioredis').default; const prefix = `experiment:epoch-variants:${randomUUID()}`; const proxy = await faultProxy(process.env.CACHE_URL, 6379); const observer = new Redis(process.env.CACHE_URL); process.env.CACHE_URL = proxy.url; const apps = {}; const chatIds = []; const userIds = []; let prisma, token; const report = { recorded_at: new Date().toISOString(), methodology: { concurrency: 32, duration_seconds: 60, warmup_seconds: 10, repetitions: 3, seed_messages: 100000, writes_per_repeat: 4096, order: [ ['baseline', 'profile-join', 'access-join', 'striped'], ['striped', 'access-join', 'profile-join', 'baseline'], ['profile-join', 'striped', 'baseline', 'access-join'], ], requests: 'Real loopback HTTP; shared read fixtures, separate write chats and cache prefixes', resources: 'Four idle/active API instances and HTTP generator share one Node process; only one workload active at a time', sql: 'pg.Client.query; transaction-control commands excluded; no changes to database pool size', }, variants: manifest.variants, samples: [], correctness: {}, completed: false, }; const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); async function request(name, method, url, body, bearer = token) { const response = await fetch(`${apps[name].url}${url}`, { method, headers: { authorization: `Bearer ${bearer}`, ...(body ? { 'content-type': 'application/json' } : {}), }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(10000), }); const json = await response.json(); return { status: response.status, json }; } async function call(name, method, url, body) { const result = await request(name, method, url, body); assert.equal(result.status, method === 'POST' ? 201 : 200); return result.json; } async function clearCache() { let cursor = '0'; do { const [next, keys] = await observer.scan( cursor, 'MATCH', `${prefix}:*`, 'COUNT', 100, ); cursor = next; if (keys.length) await observer.del(...keys); } while (cursor !== '0'); } async function newChat(userId) { const chat = await prisma.chat.create({ data: { title: 'Variant benchmark', members: { create: { userId, position: 0 } }, }, }); chatIds.push(chat.id); return chat.id; } async function epoch(name, chatId) { if (name !== 'striped') return (await prisma.chat.findUniqueOrThrow({ where: { id: chatId } })) .messagesEpoch; const [row] = await prisma.$queryRawUnsafe( `SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM "${manifest.schema}".chat_message_epochs WHERE chat_id = $1::uuid`, chatId, ); return row.epoch; } async function measure(name, workload, repeat, operation, total) { counts = {}; const latencies = []; let errors = 0, index = 0; const cpu = process.cpuUsage(); const start = performance.now(); await Promise.all( Array.from({ length: 32 }, async () => { while ( total === undefined ? performance.now() - start < 60000 : index < total ) { const item = index++; const began = performance.now(); try { await operation(item); latencies.push(performance.now() - began); } catch { errors++; } } }), ); const seconds = (performance.now() - start) / 1000; const used = process.cpuUsage(cpu); latencies.sort((a, b) => a - b); const calls = { ...counts }; const queries = Object.entries(calls) .filter(([verb]) => !['BEGIN', 'COMMIT', 'ROLLBACK', 'SET'].includes(verb)) .reduce((sum, [, count]) => sum + count, 0); const sample = { variant: name, workload, repeat, successes: latencies.length, errors, elapsed_seconds: +seconds.toFixed(3), rps: +(latencies.length / seconds).toFixed(3), mean_latency_ms: +( latencies.reduce((sum, value) => sum + value, 0) / latencies.length ).toFixed(3), p95_ms: +latencies[Math.ceil(latencies.length * 0.95) - 1]?.toFixed(3), sql_queries: queries, sql_per_request: +(queries / (latencies.length + errors)).toFixed(4), sql_calls: calls, cpu_percent_one_core: +( (used.user + used.system) / (seconds * 10000) ).toFixed(2), rss_mib: +(process.memoryUsage().rss / 1048576).toFixed(2), }; report.samples.push(sample); console.log(JSON.stringify(sample)); assert.equal(errors, 0, 'HTTP workload errors'); } async function reconnect() { proxy.setMode('online'); for (let attempt = 0; attempt < 80; attempt++) { if ( Object.values(apps).every( ({ app }) => app.get(CacheService).redis.status === 'ready', ) ) return; await pause(100); } assert.fail('Redis reconnect timeout'); } async function correctness(name, userId, otherToken) { activate(name); await clearCache(); const replica = Object.keys(apps).find((candidate) => candidate !== name); const chatId = await newChat(userId); const url = `/api/v1/chats/${chatId}/messages`; assert.equal( ( await request( name, 'GET', url, undefined, randomBytes(32).toString('base64url'), ) ).status, 401, ); assert.equal( (await request(name, 'GET', url, undefined, otherToken)).status, 404, ); assert.equal( (await request(name, 'POST', url, { text: 'Forbidden' }, otherToken)) .status, 404, ); assert.equal( (await request(name, 'GET', '/api/v1/chats/invalid/messages')).status, 400, ); assert.equal( ( await request( name, 'GET', '/api/v1/chats/invalid/messages', undefined, randomBytes(32).toString('base64url'), ) ).status, 401, ); assert.equal((await call(replica, 'GET', url)).items.length, 0); const keys = await observer.keys(`${prefix}:${replica}:messages:${chatId}:*`); assert.ok(keys.length); proxy.setMode('cut'); await pause(100); const posted = await call(name, 'POST', url, { text: 'During outage' }); assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id); await reconnect(); assert.equal(await observer.exists(...keys), keys.length); assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id); const service = apps[name].app.get(ChatService); const actual = service.prisma; service.prisma = actual.$extends({ query: { $queryRaw({ args, query }) { const index = args.values.indexOf('Must roll back'); if (index !== -1) args.values[index] = 'x'.repeat(4001); return query(args); }, }, }); try { assert.equal( (await request(name, 'POST', url, { text: 'Must roll back' })).status, 500, ); } finally { service.prisma = actual; } assert.equal(await epoch(name, chatId), 1n); assert.equal(await prisma.message.count({ where: { chatId } }), 1); const raceChat = await newChat(userId); const raceUrl = `/api/v1/chats/${raceChat}/messages`; let began, release; const started = new Promise((resolve) => { began = resolve; }); const held = new Promise((resolve) => { release = resolve; }); service.prisma = actual.$extends({ query: { message: { async findMany({ args, query }) { const rows = await query(args); if (args.where?.chatId === raceChat) { began(); await held; } return rows; }, }, }, }); const pending = call(name, 'GET', raceUrl); try { await Promise.race([ started, pause(10000).then(() => { throw new Error('Race load timeout'); }), ]); const fresh = await call(replica, 'POST', raceUrl, { text: 'After old SELECT', }); release(); await pending; assert.equal((await call(name, 'GET', raceUrl)).items[0].id, fresh.id); } finally { release(); service.prisma = actual; } await clearCache(); assert.equal((await call(name, 'GET', url)).items[0].id, posted.id); report.correctness[name] = { access_statuses: true, redis_outage_and_retained_keys: true, rollback: true, delayed_old_fill: true, empty_cache: true, }; console.log(JSON.stringify({ correctness: name, result: 'PASS' })); } try { for (const variant of manifest.variants) { activate(variant.name); process.env.CACHE_PREFIX = `${prefix}:${variant.name}`; const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false, }); apps[variant.name] = { app }; exposeRequestIdHeader(app); await app.listen(0, '127.0.0.1'); require('nestjs-pino').PinoLogger.root.level = 'silent'; apps[variant.name].url = await app.getUrl(); } prisma = apps.baseline.app.get(PrismaService); const user = await prisma.user.create({ data: { username: `variant_${randomUUID().replaceAll('-', '').slice(0, 20)}`, displayName: 'Variant user', }, }); userIds.push(user.id); token = randomBytes(32).toString('base64url'); await prisma.session.create({ data: { userId: user.id, tokenHash: createHash('sha256').update(token).digest('hex'), expiresAt: new Date(Date.now() + 3600000), }, }); const other = await prisma.user.create({ data: { username: `other_${randomUUID().replaceAll('-', '').slice(0, 20)}`, displayName: 'Other user', }, }); userIds.push(other.id); const otherToken = randomBytes(32).toString('base64url'); await prisma.session.create({ data: { userId: other.id, tokenHash: createHash('sha256').update(otherToken).digest('hex'), expiresAt: new Date(Date.now() + 3600000), }, }); for (const name of Object.keys(apps)) await correctness(name, user.id, otherToken); const readChat = await newChat(user.id); for (let offset = 0; offset < 100000; offset += 1000) { await prisma.message.createMany({ data: Array.from({ length: 1000 }, (_, i) => ({ chatId: readChat, senderId: user.id, text: `Synthetic message ${offset + i}`, createdAt: new Date(1700000000000 + offset + i), })), }); } await prisma.chat.update({ where: { id: readChat }, data: { messagesEpoch: 100000n } }); await prisma.$executeRawUnsafe(`INSERT INTO "${manifest.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, readChat); for (const [index, order] of report.methodology.order.entries()) { for (const name of order) { activate(name); await clearCache(); const url = `/api/v1/chats/${readChat}/messages?limit=50`; const warmUntil = performance.now() + 10000; while (performance.now() < warmUntil) await call(name, 'GET', url); await measure(name, 'warm-history', index + 1, () => call(name, 'GET', url), ); const writeChat = await newChat(user.id); const writeUrl = `/api/v1/chats/${writeChat}/messages`; assert.equal((await call(name, 'GET', writeUrl)).items.length, 0); await measure( name, 'writes-single-chat', index + 1, (i) => call(name, 'POST', writeUrl, { text: `Concurrent ${i}` }), 4096, ); assert.equal( await prisma.message.count({ where: { chatId: writeChat } }), 4096, ); assert.equal(await epoch(name, writeChat), 4096n); assert.equal((await call(name, 'GET', writeUrl)).items.length, 50); } } report.completed = true; } finally { await reconnect(); if (prisma) { await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); await prisma.user.deleteMany({ where: { id: { in: userIds } } }); } await clearCache(); for (const { app } of Object.values(apps)) await app.close(); observer.disconnect(); await proxy.close(); await writeFile( '/results/epoch-variants.json', JSON.stringify(report, null, 2) + '\n', ); } console.log( JSON.stringify({ completed: report.completed, correctness: report.correctness, }), );