Files
mtusi2026lab2-fork/test/epoch-variants-benchmark.mjs

476 lines
15 KiB
JavaScript

import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import Module from 'node:module';
import { readFile, writeFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
import { faultProxy } from './helpers/fault-proxy.mjs';
const require = createRequire('/app/package.json');
const manifest = JSON.parse(await readFile('/variants/manifest.json', 'utf8'));
const pg = require('pg');
let counts = {};
const originalQuery = pg.Client.prototype.query;
pg.Client.prototype.query = function (query, ...args) {
const sql = typeof query === 'string' ? query : query.text;
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
counts[verb] = (counts[verb] ?? 0) + 1;
return originalQuery.call(this, query, ...args);
};
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('/app/dist/app.module');
const { PrismaService } = require('/app/dist/prisma/prisma.service');
const { CacheService } = require('/app/dist/cache/cache.service');
const { ChatService } = require('/app/dist/Chat/chat.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('/app/dist/common/http/fastify-adapter');
const exportedClasses = {
'Auth/auth.service.ts': 'AuthService',
'User/user.service.ts': 'UserService',
'User/user.guard.ts': 'UserGuard',
'Chat/chat.service.ts': 'ChatService',
'Chat/chat.controller.ts': 'ChatController',
};
const originals = {};
const variants = {};
for (const [relative, name] of Object.entries(exportedClasses)) {
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
originals[name] = require(filename)[name];
}
for (const variant of manifest.variants) {
variants[variant.name] = {};
for (const [relative, name] of Object.entries(exportedClasses)) {
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
const source = await readFile(
`/variants/${variant.name}/${relative.replace(/\.ts$/, '.js')}`,
'utf8',
);
const loaded = new Module(filename);
loaded.filename = filename;
loaded.paths = Module._nodeModulePaths('/app');
loaded._compile(source, filename);
variants[variant.name][name] = loaded.exports[name];
}
}
function activate(name) {
for (const [className, replacement] of Object.entries(variants[name])) {
const target = originals[className].prototype;
for (const method of Object.getOwnPropertyNames(replacement.prototype)) {
if (method === 'constructor') continue;
const descriptor = Object.getOwnPropertyDescriptor(
replacement.prototype,
method,
);
const previous = target[method];
if (
typeof descriptor.value === 'function' &&
typeof previous === 'function'
) {
for (const key of Reflect.getMetadataKeys(previous))
Reflect.defineMetadata(
key,
Reflect.getMetadata(key, previous),
descriptor.value,
);
}
Object.defineProperty(target, method, descriptor);
}
}
}
const Redis = require('ioredis').default;
const prefix = `experiment:epoch-variants:${randomUUID()}`;
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
const observer = new Redis(process.env.CACHE_URL);
process.env.CACHE_URL = proxy.url;
const apps = {};
const chatIds = [];
const userIds = [];
let prisma, token;
const report = {
recorded_at: new Date().toISOString(),
methodology: {
concurrency: 32,
duration_seconds: 60,
warmup_seconds: 10,
repetitions: 3,
seed_messages: 100000,
writes_per_repeat: 4096,
order: [
['baseline', 'profile-join', 'access-join', 'striped'],
['striped', 'access-join', 'profile-join', 'baseline'],
['profile-join', 'striped', 'baseline', 'access-join'],
],
requests:
'Real loopback HTTP; shared read fixtures, separate write chats and cache prefixes',
resources:
'Four idle/active API instances and HTTP generator share one Node process; only one workload active at a time',
sql: 'pg.Client.query; transaction-control commands excluded; no changes to database pool size',
},
variants: manifest.variants,
samples: [],
correctness: {},
completed: false,
};
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function request(name, method, url, body, bearer = token) {
const response = await fetch(`${apps[name].url}${url}`, {
method,
headers: {
authorization: `Bearer ${bearer}`,
...(body ? { 'content-type': 'application/json' } : {}),
},
...(body ? { body: JSON.stringify(body) } : {}),
signal: AbortSignal.timeout(10000),
});
const json = await response.json();
return { status: response.status, json };
}
async function call(name, method, url, body) {
const result = await request(name, method, url, body);
assert.equal(result.status, method === 'POST' ? 201 : 200);
return result.json;
}
async function clearCache() {
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
}
async function newChat(userId) {
const chat = await prisma.chat.create({
data: {
title: 'Variant benchmark',
members: { create: { userId, position: 0 } },
},
});
chatIds.push(chat.id);
return chat.id;
}
async function epoch(name, chatId) {
if (name !== 'striped')
return (await prisma.chat.findUniqueOrThrow({ where: { id: chatId } }))
.messagesEpoch;
const [row] = await prisma.$queryRawUnsafe(
`SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM "${manifest.schema}".chat_message_epochs WHERE chat_id = $1::uuid`,
chatId,
);
return row.epoch;
}
async function measure(name, workload, repeat, operation, total) {
counts = {};
const latencies = [];
let errors = 0,
index = 0;
const cpu = process.cpuUsage();
const start = performance.now();
await Promise.all(
Array.from({ length: 32 }, async () => {
while (
total === undefined ? performance.now() - start < 60000 : index < total
) {
const item = index++;
const began = performance.now();
try {
await operation(item);
latencies.push(performance.now() - began);
} catch {
errors++;
}
}
}),
);
const seconds = (performance.now() - start) / 1000;
const used = process.cpuUsage(cpu);
latencies.sort((a, b) => a - b);
const calls = { ...counts };
const queries = Object.entries(calls)
.filter(([verb]) => !['BEGIN', 'COMMIT', 'ROLLBACK', 'SET'].includes(verb))
.reduce((sum, [, count]) => sum + count, 0);
const sample = {
variant: name,
workload,
repeat,
successes: latencies.length,
errors,
elapsed_seconds: +seconds.toFixed(3),
rps: +(latencies.length / seconds).toFixed(3),
mean_latency_ms: +(
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
).toFixed(3),
p95_ms: +latencies[Math.ceil(latencies.length * 0.95) - 1]?.toFixed(3),
sql_queries: queries,
sql_per_request: +(queries / (latencies.length + errors)).toFixed(4),
sql_calls: calls,
cpu_percent_one_core: +(
(used.user + used.system) /
(seconds * 10000)
).toFixed(2),
rss_mib: +(process.memoryUsage().rss / 1048576).toFixed(2),
};
report.samples.push(sample);
console.log(JSON.stringify(sample));
assert.equal(errors, 0, 'HTTP workload errors');
}
async function reconnect() {
proxy.setMode('online');
for (let attempt = 0; attempt < 80; attempt++) {
if (
Object.values(apps).every(
({ app }) => app.get(CacheService).redis.status === 'ready',
)
)
return;
await pause(100);
}
assert.fail('Redis reconnect timeout');
}
async function correctness(name, userId, otherToken) {
activate(name);
await clearCache();
const replica = Object.keys(apps).find((candidate) => candidate !== name);
const chatId = await newChat(userId);
const url = `/api/v1/chats/${chatId}/messages`;
assert.equal(
(
await request(
name,
'GET',
url,
undefined,
randomBytes(32).toString('base64url'),
)
).status,
401,
);
assert.equal(
(await request(name, 'GET', url, undefined, otherToken)).status,
404,
);
assert.equal(
(await request(name, 'POST', url, { text: 'Forbidden' }, otherToken))
.status,
404,
);
assert.equal(
(await request(name, 'GET', '/api/v1/chats/invalid/messages')).status,
400,
);
assert.equal(
(
await request(
name,
'GET',
'/api/v1/chats/invalid/messages',
undefined,
randomBytes(32).toString('base64url'),
)
).status,
401,
);
assert.equal((await call(replica, 'GET', url)).items.length, 0);
const keys = await observer.keys(`${prefix}:${replica}:messages:${chatId}:*`);
assert.ok(keys.length);
proxy.setMode('cut');
await pause(100);
const posted = await call(name, 'POST', url, { text: 'During outage' });
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
await reconnect();
assert.equal(await observer.exists(...keys), keys.length);
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
const service = apps[name].app.get(ChatService);
const actual = service.prisma;
service.prisma = actual.$extends({
query: {
$queryRaw({ args, query }) {
const index = args.values.indexOf('Must roll back');
if (index !== -1) args.values[index] = 'x'.repeat(4001);
return query(args);
},
},
});
try {
assert.equal(
(await request(name, 'POST', url, { text: 'Must roll back' })).status,
500,
);
} finally {
service.prisma = actual;
}
assert.equal(await epoch(name, chatId), 1n);
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
const raceChat = await newChat(userId);
const raceUrl = `/api/v1/chats/${raceChat}/messages`;
let began, release;
const started = new Promise((resolve) => {
began = resolve;
});
const held = new Promise((resolve) => {
release = resolve;
});
service.prisma = actual.$extends({
query: {
message: {
async findMany({ args, query }) {
const rows = await query(args);
if (args.where?.chatId === raceChat) {
began();
await held;
}
return rows;
},
},
},
});
const pending = call(name, 'GET', raceUrl);
try {
await Promise.race([
started,
pause(10000).then(() => {
throw new Error('Race load timeout');
}),
]);
const fresh = await call(replica, 'POST', raceUrl, {
text: 'After old SELECT',
});
release();
await pending;
assert.equal((await call(name, 'GET', raceUrl)).items[0].id, fresh.id);
} finally {
release();
service.prisma = actual;
}
await clearCache();
assert.equal((await call(name, 'GET', url)).items[0].id, posted.id);
report.correctness[name] = {
access_statuses: true,
redis_outage_and_retained_keys: true,
rollback: true,
delayed_old_fill: true,
empty_cache: true,
};
console.log(JSON.stringify({ correctness: name, result: 'PASS' }));
}
try {
for (const variant of manifest.variants) {
activate(variant.name);
process.env.CACHE_PREFIX = `${prefix}:${variant.name}`;
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
apps[variant.name] = { app };
exposeRequestIdHeader(app);
await app.listen(0, '127.0.0.1');
require('nestjs-pino').PinoLogger.root.level = 'silent';
apps[variant.name].url = await app.getUrl();
}
prisma = apps.baseline.app.get(PrismaService);
const user = await prisma.user.create({
data: {
username: `variant_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Variant user',
},
});
userIds.push(user.id);
token = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId: user.id,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 3600000),
},
});
const other = await prisma.user.create({
data: {
username: `other_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Other user',
},
});
userIds.push(other.id);
const otherToken = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId: other.id,
tokenHash: createHash('sha256').update(otherToken).digest('hex'),
expiresAt: new Date(Date.now() + 3600000),
},
});
for (const name of Object.keys(apps))
await correctness(name, user.id, otherToken);
const readChat = await newChat(user.id);
for (let offset = 0; offset < 100000; offset += 1000) {
await prisma.message.createMany({
data: Array.from({ length: 1000 }, (_, i) => ({
chatId: readChat,
senderId: user.id,
text: `Synthetic message ${offset + i}`,
createdAt: new Date(1700000000000 + offset + i),
})),
});
}
await prisma.chat.update({ where: { id: readChat }, data: { messagesEpoch: 100000n } });
await prisma.$executeRawUnsafe(`INSERT INTO "${manifest.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, readChat);
for (const [index, order] of report.methodology.order.entries()) {
for (const name of order) {
activate(name);
await clearCache();
const url = `/api/v1/chats/${readChat}/messages?limit=50`;
const warmUntil = performance.now() + 10000;
while (performance.now() < warmUntil) await call(name, 'GET', url);
await measure(name, 'warm-history', index + 1, () =>
call(name, 'GET', url),
);
const writeChat = await newChat(user.id);
const writeUrl = `/api/v1/chats/${writeChat}/messages`;
assert.equal((await call(name, 'GET', writeUrl)).items.length, 0);
await measure(
name,
'writes-single-chat',
index + 1,
(i) => call(name, 'POST', writeUrl, { text: `Concurrent ${i}` }),
4096,
);
assert.equal(
await prisma.message.count({ where: { chatId: writeChat } }),
4096,
);
assert.equal(await epoch(name, writeChat), 4096n);
assert.equal((await call(name, 'GET', writeUrl)).items.length, 50);
}
}
report.completed = true;
} finally {
await reconnect();
if (prisma) {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
}
await clearCache();
for (const { app } of Object.values(apps)) await app.close();
observer.disconnect();
await proxy.close();
await writeFile(
'/results/epoch-variants.json',
JSON.stringify(report, null, 2) + '\n',
);
}
console.log(
JSON.stringify({
completed: report.completed,
correctness: report.correctness,
}),
);