Files
mtusi2026lab2-fork/test/cache-benchmark.mjs

539 lines
17 KiB
JavaScript

import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import { readFile, writeFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
import { faultProxy } from './helpers/fault-proxy.mjs';
import { messageEpoch } from './helpers/message-epoch.mjs';
const require = createRequire(`${process.cwd()}/package.json`);
const pg = require('pg');
let counts = {};
const originalQuery = pg.Client.prototype.query;
pg.Client.prototype.query = function (query, ...args) {
const sql = typeof query === 'string' ? query : query.text;
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
const table =
['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) =>
new RegExp(`\\b${name}\\b`).test(sql),
) ?? 'other';
const label = `${verb}:${table}`;
counts[label] = (counts[label] ?? 0) + 1;
return originalQuery.call(this, query, ...args);
};
const Redis = require('ioredis').default;
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
const { PrismaService } = require(
`${process.cwd()}/dist/prisma/prisma.service`,
);
const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`);
const { createFastifyAdapter, exposeRequestIdHeader } = require(
`${process.cwd()}/dist/common/http/fastify-adapter`,
);
const stage = process.argv[2];
const compareRedis = process.argv.includes('--compare-redis');
const historyControl = !process.argv.includes('--skip-history-control');
function integerOption(name, fallback) {
const index = process.argv.indexOf(`--${name}`);
const value = index === -1 ? fallback : Number(process.argv[index + 1]);
assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`);
return value;
}
const concurrency = integerOption('concurrency', 16);
const duration = integerOption('duration-seconds', 10);
const warmup = integerOption('warmup-seconds', 3);
const repetitions = integerOption('repetitions', 3);
const seedMessages = integerOption('seed-messages', 10000);
const coldBurst = integerOption('cold-burst', 32);
const writesPerRepeat = integerOption('writes-per-repeat', 512);
assert.ok(['before', 'after'].includes(stage));
const prefix = `experiment:epochs:${randomUUID()}`;
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
const observer = new Redis(process.env.CACHE_URL);
process.env.CACHE_PREFIX = prefix;
process.env.CACHE_URL = proxy.url;
const apps = [];
async function start() {
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
apps.push(app);
exposeRequestIdHeader(app);
await app.listen(0, '127.0.0.1');
require('nestjs-pino').PinoLogger.root.level = 'silent';
return { app, url: await app.getUrl() };
}
const first = await start();
const prisma = first.app.get(PrismaService);
let userId;
const chatIds = [];
const report = {
stage,
recorded_at: new Date().toISOString(),
node: process.version,
methodology: {
requests:
'real loopback HTTP; instrumented API process in measured backend image',
sql: 'pg.Client.query calls; transaction control counted separately',
concurrency,
duration_seconds: duration,
warmup_seconds: warmup,
repetitions,
seed_messages: seedMessages,
cold_burst_requests: coldBurst,
writes_per_repeat: writesPerRepeat,
history_control: historyControl,
resources:
'process CPU/RSS includes HTTP generator and instrumentation; not container CPU',
limitations:
'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF',
},
dist_sha256: {},
dependencies: {},
samples: [],
correctness: {},
};
for (const name of [
'@nestjs/core',
'@prisma/adapter-pg',
'pg',
'ioredis',
'fastify',
]) {
try {
report.dependencies[name] = require(`${name}/package.json`).version;
} catch {
report.dependencies[name] = 'package metadata not exported';
}
}
for (const name of [
'Chat/chat.service.js',
'cache/cache.service.js',
'Auth/auth.service.js',
'User/user.guard.js',
'Chat/chat.controller.js',
]) {
report.dist_sha256[name] = createHash('sha256')
.update(await readFile(`${process.cwd()}/dist/${name}`))
.digest('hex');
}
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function clearCache() {
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
}
let token;
async function request(method, path, body, target = first) {
const response = await fetch(`${target.url}${path}`, {
method,
headers: {
authorization: `Bearer ${token}`,
...(body ? { 'content-type': 'application/json' } : {}),
},
...(body ? { body: JSON.stringify(body) } : {}),
signal: AbortSignal.timeout(10000),
});
const json = await response.json();
assert.equal(
response.status,
method === 'POST' ? 201 : 200,
`HTTP ${response.status}`,
);
return json;
}
async function newChat() {
const chat = await prisma.chat.create({
data: {
title: 'Synthetic epoch benchmark',
members: { create: { userId, position: 0 } },
},
});
chatIds.push(chat.id);
return chat.id;
}
async function measure(
workload,
repeat,
operation,
{
duration: secondsToRun = duration,
total,
concurrency: clients = concurrency,
} = {},
) {
counts = {};
const cpu = process.cpuUsage();
let peakRss = process.memoryUsage().rss;
const timer = setInterval(() => {
peakRss = Math.max(peakRss, process.memoryUsage().rss);
}, 100);
const latencies = [];
let errors = 0,
index = 0;
const start = performance.now();
await Promise.all(
Array.from({ length: clients }, async () => {
while (
total === undefined
? performance.now() - start < secondsToRun * 1000
: index < total
) {
const item = index++;
const began = performance.now();
try {
await operation(item);
latencies.push(performance.now() - began);
} catch {
errors++;
}
}
}),
);
const seconds = (performance.now() - start) / 1000;
clearInterval(timer);
peakRss = Math.max(peakRss, process.memoryUsage().rss);
const used = process.cpuUsage(cpu);
latencies.sort((a, b) => a - b);
const percentile = (p) =>
latencies.length
? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3)
: null;
const sql = Object.entries(counts)
.filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key))
.reduce((sum, [, n]) => sum + n, 0);
const sample = {
workload,
repeat,
concurrency: clients,
elapsed_seconds: +seconds.toFixed(3),
successes: latencies.length,
errors,
error_ratio: errors / (latencies.length + errors),
successful_rps: +(latencies.length / seconds).toFixed(3),
latency_ms: {
mean: latencies.length
? +(
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
).toFixed(3)
: null,
p50: percentile(0.5),
p95: percentile(0.95),
p99: percentile(0.99),
},
sql_queries: sql,
sql_per_request: +(sql / (latencies.length + errors)).toFixed(4),
sql_calls: { ...counts },
process_cpu_percent_one_core: +(
(used.user + used.system) /
(seconds * 10000)
).toFixed(2),
peak_process_rss_mib: +(peakRss / 1048576).toFixed(2),
};
report.samples.push(sample);
console.log(JSON.stringify(sample));
assert.equal(errors, 0, 'load errors recorded in report');
}
try {
const user = await prisma.user.create({
data: {
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Synthetic benchmark',
},
});
userId = user.id;
token = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 3600000),
},
});
const readChat = await newChat();
for (let offset = 0; offset < seedMessages; offset += 1000) {
await prisma.message.createMany({
data: Array.from(
{ length: Math.min(1000, seedMessages - offset) },
(_, i) => ({
chatId: readChat,
senderId: userId,
text: `Synthetic message ${offset + i}`,
createdAt: new Date(1700000000000 + offset + i),
}),
),
});
}
await prisma.chatMessageEpoch.createMany({
data: Array.from({ length: 16 }, (_, bucket) => ({
chatId: readChat,
bucket,
epoch: BigInt(
Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0),
),
})),
});
const path = `/api/v1/chats/${readChat}/messages?limit=50`;
if (compareRedis) {
const cacheUrl = process.env.CACHE_URL;
delete process.env.CACHE_URL;
let withoutRedis;
try {
withoutRedis = await start();
} finally {
process.env.CACHE_URL = cacheUrl;
}
assert.equal(withoutRedis.app.get(CacheService).redis, undefined);
report.methodology.comparison = {
modes: ['redis', 'no-redis'],
no_redis:
'API started without CACHE_URL; all application Redis caches disabled',
epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes',
order: 'redis first in repeats 1/3, no-redis first in repeat 2',
processes:
'both API instances and HTTP generator share one measured Node process',
};
report.correctness.read_after_write = {};
for (let repeat = 1; repeat <= repetitions; repeat++) {
const variants = [
{ name: 'redis', target: first },
{ name: 'no-redis', target: withoutRedis },
];
if (repeat % 2 === 0) variants.reverse();
for (const { name, target } of variants) {
await clearCache();
await measure(
`cold-history-burst:${name}`,
repeat,
() => request('GET', path, undefined, target),
{ total: coldBurst, concurrency: coldBurst },
);
const warmUntil = performance.now() + warmup * 1000;
while (performance.now() < warmUntil)
await request('GET', path, undefined, target);
await measure(`warm-history:${name}`, repeat, () =>
request('GET', path, undefined, target),
);
const writeChat = await newChat();
const writePath = `/api/v1/chats/${writeChat}/messages`;
assert.equal(
(await request('GET', writePath, undefined, target)).items.length,
0,
);
await measure(
`writes-single-chat:${name}`,
repeat,
(i) =>
request(
'POST',
writePath,
{ text: `Concurrent write ${i}` },
target,
),
{ total: writesPerRepeat },
);
assert.equal(
await prisma.message.count({ where: { chatId: writeChat } }),
writesPerRepeat,
);
assert.equal(
await messageEpoch(prisma, writeChat),
BigInt(writesPerRepeat),
);
const page = await request('GET', writePath, undefined, target);
assert.equal(page.items.length, 50);
assert.equal(new Set(page.items.map((message) => message.id)).size, 50);
report.correctness.read_after_write[`${name}:${repeat}`] = true;
}
}
} else {
const cache = first.app.get(CacheService);
const remember = cache.remember.bind(cache);
for (let repeat = 1; repeat <= repetitions; repeat++) {
await clearCache();
await request('GET', `/api/v1/chats/${readChat}`);
await measure('cold-history-burst', repeat, () => request('GET', path), {
total: coldBurst,
concurrency: coldBurst,
});
const warmUntil = performance.now() + warmup * 1000;
while (performance.now() < warmUntil) await request('GET', path);
await measure('warm-history', repeat, () => request('GET', path));
if (historyControl) {
cache.remember = (namespace, parameters, ttl, load) =>
namespace.startsWith('messages:')
? load()
: remember(namespace, parameters, ttl, load);
const directUntil = performance.now() + warmup * 1000;
while (performance.now() < directUntil) await request('GET', path);
await measure('history-cache-bypassed', repeat, () =>
request('GET', path),
);
cache.remember = remember;
}
const writeChat = await newChat();
await request('GET', `/api/v1/chats/${writeChat}`);
await measure(
'writes-single-chat',
repeat,
(i) =>
request('POST', `/api/v1/chats/${writeChat}/messages`, {
text: `Concurrent write ${i}`,
}),
{ total: writesPerRepeat },
);
assert.equal(
await prisma.message.count({ where: { chatId: writeChat } }),
writesPerRepeat,
);
}
const second = await start();
const faultChat = await newChat();
const faultPath = `/api/v1/chats/${faultChat}/messages`;
assert.equal(
(await request('GET', faultPath, undefined, second)).items.length,
0,
);
const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`);
assert.ok(keysBefore.length);
proxy.setMode('cut');
await sleep(100);
const written = await request('POST', faultPath, {
text: 'Committed while Redis is unavailable',
});
const during = await request('GET', faultPath, undefined, second);
proxy.setMode('online');
for (let i = 0; i < 60; i++) {
if (apps.every((app) => app.get(CacheService).redis?.status === 'ready'))
break;
await sleep(100);
}
assert.ok(
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
);
const recovered = await request('GET', faultPath, undefined, second);
report.correctness.redis_reconnect = {
old_keys_retained:
(await observer.exists(...keysBefore)) === keysBefore.length,
read_after_write_during_outage: during.items.some(
(x) => x.id === written.id,
),
read_after_write_after_reconnect_other_replica: recovered.items.some(
(x) => x.id === written.id,
),
};
await clearCache();
const cold = await request('GET', faultPath, undefined, second);
report.correctness.empty_cache_recovery = cold.items.some(
(x) => x.id === written.id,
);
const raceChat = await newChat();
const racePath = `/api/v1/chats/${raceChat}/messages`;
const service = first.app.get(
require(`${process.cwd()}/dist/Chat/chat.service`).ChatService,
);
const actualPrisma = service.prisma;
let release, began;
const started = new Promise((resolve) => {
began = resolve;
});
const held = new Promise((resolve) => {
release = resolve;
});
service.prisma = actualPrisma.$extends({
query: {
message: {
async findMany({ args, query }) {
const rows = await query(args);
if (args.where?.chatId === raceChat) {
began();
await held;
}
return rows;
},
},
},
});
const pending = request('GET', racePath);
let fresh;
try {
await Promise.race([
started,
sleep(10000).then(() => {
throw new Error('race load did not start');
}),
]);
proxy.setMode('cut');
await sleep(100);
fresh = await request(
'POST',
racePath,
{ text: 'New message after old SELECT' },
second,
);
proxy.setMode('online');
for (let i = 0; i < 60; i++) {
if (
apps.every((app) => app.get(CacheService).redis?.status === 'ready')
)
break;
await sleep(100);
}
assert.ok(
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
);
} finally {
release();
service.prisma = actualPrisma;
}
await pending;
report.correctness.delayed_old_fill = (
await request('GET', racePath, undefined, second)
).items.some((x) => x.id === fresh.id);
if (stage === 'after') {
assert.equal(
report.correctness.redis_reconnect
.read_after_write_after_reconnect_other_replica,
true,
);
assert.equal(report.correctness.delayed_old_fill, true);
assert.equal(report.correctness.empty_cache_recovery, true);
}
}
report.completed = true;
} finally {
proxy.setMode('online');
try {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
if (userId) await prisma.user.delete({ where: { id: userId } });
await clearCache();
report.cleanup = 'only experiment chats/user/cache prefix removed';
} finally {
for (const app of apps) await app.close();
observer.disconnect();
await proxy.close();
await writeFile(
process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`,
JSON.stringify(report, null, 2) + '\n',
);
}
}
console.log(
JSON.stringify({
stage,
correctness: report.correctness,
completed: report.completed,
}),
);