import assert from 'node:assert/strict'; import { createHash, randomBytes, randomUUID } from 'node:crypto'; import { createRequire } from 'node:module'; import { readFile, writeFile } from 'node:fs/promises'; import { performance } from 'node:perf_hooks'; import { faultProxy } from './helpers/fault-proxy.mjs'; import { messageEpoch } from './helpers/message-epoch.mjs'; const require = createRequire(`${process.cwd()}/package.json`); const pg = require('pg'); let counts = {}; const originalQuery = pg.Client.prototype.query; pg.Client.prototype.query = function (query, ...args) { const sql = typeof query === 'string' ? query : query.text; const verb = sql.trim().split(/\s+/)[0].toUpperCase(); const table = ['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) => new RegExp(`\\b${name}\\b`).test(sql), ) ?? 'other'; const label = `${verb}:${table}`; counts[label] = (counts[label] ?? 0) + 1; return originalQuery.call(this, query, ...args); }; const Redis = require('ioredis').default; const { NestFactory } = require('@nestjs/core'); const { AppModule } = require(`${process.cwd()}/dist/app.module`); const { PrismaService } = require( `${process.cwd()}/dist/prisma/prisma.service`, ); const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`); const { createFastifyAdapter, exposeRequestIdHeader } = require( `${process.cwd()}/dist/common/http/fastify-adapter`, ); const stage = process.argv[2]; const compareRedis = process.argv.includes('--compare-redis'); const historyControl = !process.argv.includes('--skip-history-control'); function integerOption(name, fallback) { const index = process.argv.indexOf(`--${name}`); const value = index === -1 ? fallback : Number(process.argv[index + 1]); assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`); return value; } const concurrency = integerOption('concurrency', 16); const duration = integerOption('duration-seconds', 10); const warmup = integerOption('warmup-seconds', 3); const repetitions = integerOption('repetitions', 3); const seedMessages = integerOption('seed-messages', 10000); const coldBurst = integerOption('cold-burst', 32); const writesPerRepeat = integerOption('writes-per-repeat', 512); assert.ok(['before', 'after'].includes(stage)); const prefix = `experiment:epochs:${randomUUID()}`; const proxy = await faultProxy(process.env.CACHE_URL, 6379); const observer = new Redis(process.env.CACHE_URL); process.env.CACHE_PREFIX = prefix; process.env.CACHE_URL = proxy.url; const apps = []; async function start() { const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false, }); apps.push(app); exposeRequestIdHeader(app); await app.listen(0, '127.0.0.1'); require('nestjs-pino').PinoLogger.root.level = 'silent'; return { app, url: await app.getUrl() }; } const first = await start(); const prisma = first.app.get(PrismaService); let userId; const chatIds = []; const report = { stage, recorded_at: new Date().toISOString(), node: process.version, methodology: { requests: 'real loopback HTTP; instrumented API process in measured backend image', sql: 'pg.Client.query calls; transaction control counted separately', concurrency, duration_seconds: duration, warmup_seconds: warmup, repetitions, seed_messages: seedMessages, cold_burst_requests: coldBurst, writes_per_repeat: writesPerRepeat, history_control: historyControl, resources: 'process CPU/RSS includes HTTP generator and instrumentation; not container CPU', limitations: 'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF', }, dist_sha256: {}, dependencies: {}, samples: [], correctness: {}, }; for (const name of [ '@nestjs/core', '@prisma/adapter-pg', 'pg', 'ioredis', 'fastify', ]) { try { report.dependencies[name] = require(`${name}/package.json`).version; } catch { report.dependencies[name] = 'package metadata not exported'; } } for (const name of [ 'Chat/chat.service.js', 'cache/cache.service.js', 'Auth/auth.service.js', 'User/user.guard.js', 'Chat/chat.controller.js', ]) { report.dist_sha256[name] = createHash('sha256') .update(await readFile(`${process.cwd()}/dist/${name}`)) .digest('hex'); } const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); async function clearCache() { let cursor = '0'; do { const [next, keys] = await observer.scan( cursor, 'MATCH', `${prefix}:*`, 'COUNT', 100, ); cursor = next; if (keys.length) await observer.del(...keys); } while (cursor !== '0'); } let token; async function request(method, path, body, target = first) { const response = await fetch(`${target.url}${path}`, { method, headers: { authorization: `Bearer ${token}`, ...(body ? { 'content-type': 'application/json' } : {}), }, ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(10000), }); const json = await response.json(); assert.equal( response.status, method === 'POST' ? 201 : 200, `HTTP ${response.status}`, ); return json; } async function newChat() { const chat = await prisma.chat.create({ data: { title: 'Synthetic epoch benchmark', members: { create: { userId, position: 0 } }, }, }); chatIds.push(chat.id); return chat.id; } async function measure( workload, repeat, operation, { duration: secondsToRun = duration, total, concurrency: clients = concurrency, } = {}, ) { counts = {}; const cpu = process.cpuUsage(); let peakRss = process.memoryUsage().rss; const timer = setInterval(() => { peakRss = Math.max(peakRss, process.memoryUsage().rss); }, 100); const latencies = []; let errors = 0, index = 0; const start = performance.now(); await Promise.all( Array.from({ length: clients }, async () => { while ( total === undefined ? performance.now() - start < secondsToRun * 1000 : index < total ) { const item = index++; const began = performance.now(); try { await operation(item); latencies.push(performance.now() - began); } catch { errors++; } } }), ); const seconds = (performance.now() - start) / 1000; clearInterval(timer); peakRss = Math.max(peakRss, process.memoryUsage().rss); const used = process.cpuUsage(cpu); latencies.sort((a, b) => a - b); const percentile = (p) => latencies.length ? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3) : null; const sql = Object.entries(counts) .filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key)) .reduce((sum, [, n]) => sum + n, 0); const sample = { workload, repeat, concurrency: clients, elapsed_seconds: +seconds.toFixed(3), successes: latencies.length, errors, error_ratio: errors / (latencies.length + errors), successful_rps: +(latencies.length / seconds).toFixed(3), latency_ms: { mean: latencies.length ? +( latencies.reduce((sum, value) => sum + value, 0) / latencies.length ).toFixed(3) : null, p50: percentile(0.5), p95: percentile(0.95), p99: percentile(0.99), }, sql_queries: sql, sql_per_request: +(sql / (latencies.length + errors)).toFixed(4), sql_calls: { ...counts }, process_cpu_percent_one_core: +( (used.user + used.system) / (seconds * 10000) ).toFixed(2), peak_process_rss_mib: +(peakRss / 1048576).toFixed(2), }; report.samples.push(sample); console.log(JSON.stringify(sample)); assert.equal(errors, 0, 'load errors recorded in report'); } try { const user = await prisma.user.create({ data: { username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`, displayName: 'Synthetic benchmark', }, }); userId = user.id; token = randomBytes(32).toString('base64url'); await prisma.session.create({ data: { userId, tokenHash: createHash('sha256').update(token).digest('hex'), expiresAt: new Date(Date.now() + 3600000), }, }); const readChat = await newChat(); for (let offset = 0; offset < seedMessages; offset += 1000) { await prisma.message.createMany({ data: Array.from( { length: Math.min(1000, seedMessages - offset) }, (_, i) => ({ chatId: readChat, senderId: userId, text: `Synthetic message ${offset + i}`, createdAt: new Date(1700000000000 + offset + i), }), ), }); } await prisma.chatMessageEpoch.createMany({ data: Array.from({ length: 16 }, (_, bucket) => ({ chatId: readChat, bucket, epoch: BigInt( Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0), ), })), }); const path = `/api/v1/chats/${readChat}/messages?limit=50`; if (compareRedis) { const cacheUrl = process.env.CACHE_URL; delete process.env.CACHE_URL; let withoutRedis; try { withoutRedis = await start(); } finally { process.env.CACHE_URL = cacheUrl; } assert.equal(withoutRedis.app.get(CacheService).redis, undefined); report.methodology.comparison = { modes: ['redis', 'no-redis'], no_redis: 'API started without CACHE_URL; all application Redis caches disabled', epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes', order: 'redis first in repeats 1/3, no-redis first in repeat 2', processes: 'both API instances and HTTP generator share one measured Node process', }; report.correctness.read_after_write = {}; for (let repeat = 1; repeat <= repetitions; repeat++) { const variants = [ { name: 'redis', target: first }, { name: 'no-redis', target: withoutRedis }, ]; if (repeat % 2 === 0) variants.reverse(); for (const { name, target } of variants) { await clearCache(); await measure( `cold-history-burst:${name}`, repeat, () => request('GET', path, undefined, target), { total: coldBurst, concurrency: coldBurst }, ); const warmUntil = performance.now() + warmup * 1000; while (performance.now() < warmUntil) await request('GET', path, undefined, target); await measure(`warm-history:${name}`, repeat, () => request('GET', path, undefined, target), ); const writeChat = await newChat(); const writePath = `/api/v1/chats/${writeChat}/messages`; assert.equal( (await request('GET', writePath, undefined, target)).items.length, 0, ); await measure( `writes-single-chat:${name}`, repeat, (i) => request( 'POST', writePath, { text: `Concurrent write ${i}` }, target, ), { total: writesPerRepeat }, ); assert.equal( await prisma.message.count({ where: { chatId: writeChat } }), writesPerRepeat, ); assert.equal( await messageEpoch(prisma, writeChat), BigInt(writesPerRepeat), ); const page = await request('GET', writePath, undefined, target); assert.equal(page.items.length, 50); assert.equal(new Set(page.items.map((message) => message.id)).size, 50); report.correctness.read_after_write[`${name}:${repeat}`] = true; } } } else { const cache = first.app.get(CacheService); const remember = cache.remember.bind(cache); for (let repeat = 1; repeat <= repetitions; repeat++) { await clearCache(); await request('GET', `/api/v1/chats/${readChat}`); await measure('cold-history-burst', repeat, () => request('GET', path), { total: coldBurst, concurrency: coldBurst, }); const warmUntil = performance.now() + warmup * 1000; while (performance.now() < warmUntil) await request('GET', path); await measure('warm-history', repeat, () => request('GET', path)); if (historyControl) { cache.remember = (namespace, parameters, ttl, load) => namespace.startsWith('messages:') ? load() : remember(namespace, parameters, ttl, load); const directUntil = performance.now() + warmup * 1000; while (performance.now() < directUntil) await request('GET', path); await measure('history-cache-bypassed', repeat, () => request('GET', path), ); cache.remember = remember; } const writeChat = await newChat(); await request('GET', `/api/v1/chats/${writeChat}`); await measure( 'writes-single-chat', repeat, (i) => request('POST', `/api/v1/chats/${writeChat}/messages`, { text: `Concurrent write ${i}`, }), { total: writesPerRepeat }, ); assert.equal( await prisma.message.count({ where: { chatId: writeChat } }), writesPerRepeat, ); } const second = await start(); const faultChat = await newChat(); const faultPath = `/api/v1/chats/${faultChat}/messages`; assert.equal( (await request('GET', faultPath, undefined, second)).items.length, 0, ); const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`); assert.ok(keysBefore.length); proxy.setMode('cut'); await sleep(100); const written = await request('POST', faultPath, { text: 'Committed while Redis is unavailable', }); const during = await request('GET', faultPath, undefined, second); proxy.setMode('online'); for (let i = 0; i < 60; i++) { if (apps.every((app) => app.get(CacheService).redis?.status === 'ready')) break; await sleep(100); } assert.ok( apps.every((app) => app.get(CacheService).redis?.status === 'ready'), ); const recovered = await request('GET', faultPath, undefined, second); report.correctness.redis_reconnect = { old_keys_retained: (await observer.exists(...keysBefore)) === keysBefore.length, read_after_write_during_outage: during.items.some( (x) => x.id === written.id, ), read_after_write_after_reconnect_other_replica: recovered.items.some( (x) => x.id === written.id, ), }; await clearCache(); const cold = await request('GET', faultPath, undefined, second); report.correctness.empty_cache_recovery = cold.items.some( (x) => x.id === written.id, ); const raceChat = await newChat(); const racePath = `/api/v1/chats/${raceChat}/messages`; const service = first.app.get( require(`${process.cwd()}/dist/Chat/chat.service`).ChatService, ); const actualPrisma = service.prisma; let release, began; const started = new Promise((resolve) => { began = resolve; }); const held = new Promise((resolve) => { release = resolve; }); service.prisma = actualPrisma.$extends({ query: { message: { async findMany({ args, query }) { const rows = await query(args); if (args.where?.chatId === raceChat) { began(); await held; } return rows; }, }, }, }); const pending = request('GET', racePath); let fresh; try { await Promise.race([ started, sleep(10000).then(() => { throw new Error('race load did not start'); }), ]); proxy.setMode('cut'); await sleep(100); fresh = await request( 'POST', racePath, { text: 'New message after old SELECT' }, second, ); proxy.setMode('online'); for (let i = 0; i < 60; i++) { if ( apps.every((app) => app.get(CacheService).redis?.status === 'ready') ) break; await sleep(100); } assert.ok( apps.every((app) => app.get(CacheService).redis?.status === 'ready'), ); } finally { release(); service.prisma = actualPrisma; } await pending; report.correctness.delayed_old_fill = ( await request('GET', racePath, undefined, second) ).items.some((x) => x.id === fresh.id); if (stage === 'after') { assert.equal( report.correctness.redis_reconnect .read_after_write_after_reconnect_other_replica, true, ); assert.equal(report.correctness.delayed_old_fill, true); assert.equal(report.correctness.empty_cache_recovery, true); } } report.completed = true; } finally { proxy.setMode('online'); try { await prisma.chat.deleteMany({ where: { id: { in: chatIds } } }); if (userId) await prisma.user.delete({ where: { id: userId } }); await clearCache(); report.cleanup = 'only experiment chats/user/cache prefix removed'; } finally { for (const app of apps) await app.close(); observer.disconnect(); await proxy.close(); await writeFile( process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`, JSON.stringify(report, null, 2) + '\n', ); } } console.log( JSON.stringify({ stage, correctness: report.correctness, completed: report.completed, }), );