forked from mtusi2026/mtusi2026lab3
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
476 lines
15 KiB
JavaScript
476 lines
15 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
import { createRequire } from 'node:module';
|
|
import Module from 'node:module';
|
|
import { readFile, writeFile } from 'node:fs/promises';
|
|
import { performance } from 'node:perf_hooks';
|
|
import { faultProxy } from './helpers/fault-proxy.mjs';
|
|
|
|
const require = createRequire('/app/package.json');
|
|
const manifest = JSON.parse(await readFile('/variants/manifest.json', 'utf8'));
|
|
const pg = require('pg');
|
|
let counts = {};
|
|
const originalQuery = pg.Client.prototype.query;
|
|
pg.Client.prototype.query = function (query, ...args) {
|
|
const sql = typeof query === 'string' ? query : query.text;
|
|
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
|
|
counts[verb] = (counts[verb] ?? 0) + 1;
|
|
return originalQuery.call(this, query, ...args);
|
|
};
|
|
const { NestFactory } = require('@nestjs/core');
|
|
const { AppModule } = require('/app/dist/app.module');
|
|
const { PrismaService } = require('/app/dist/prisma/prisma.service');
|
|
const { CacheService } = require('/app/dist/cache/cache.service');
|
|
const { ChatService } = require('/app/dist/Chat/chat.service');
|
|
const {
|
|
createFastifyAdapter,
|
|
exposeRequestIdHeader,
|
|
} = require('/app/dist/common/http/fastify-adapter');
|
|
const exportedClasses = {
|
|
'Auth/auth.service.ts': 'AuthService',
|
|
'User/user.service.ts': 'UserService',
|
|
'User/user.guard.ts': 'UserGuard',
|
|
'Chat/chat.service.ts': 'ChatService',
|
|
'Chat/chat.controller.ts': 'ChatController',
|
|
};
|
|
const originals = {};
|
|
const variants = {};
|
|
for (const [relative, name] of Object.entries(exportedClasses)) {
|
|
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
|
|
originals[name] = require(filename)[name];
|
|
}
|
|
for (const variant of manifest.variants) {
|
|
variants[variant.name] = {};
|
|
for (const [relative, name] of Object.entries(exportedClasses)) {
|
|
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
|
|
const source = await readFile(
|
|
`/variants/${variant.name}/${relative.replace(/\.ts$/, '.js')}`,
|
|
'utf8',
|
|
);
|
|
const loaded = new Module(filename);
|
|
loaded.filename = filename;
|
|
loaded.paths = Module._nodeModulePaths('/app');
|
|
loaded._compile(source, filename);
|
|
variants[variant.name][name] = loaded.exports[name];
|
|
}
|
|
}
|
|
function activate(name) {
|
|
for (const [className, replacement] of Object.entries(variants[name])) {
|
|
const target = originals[className].prototype;
|
|
for (const method of Object.getOwnPropertyNames(replacement.prototype)) {
|
|
if (method === 'constructor') continue;
|
|
const descriptor = Object.getOwnPropertyDescriptor(
|
|
replacement.prototype,
|
|
method,
|
|
);
|
|
const previous = target[method];
|
|
if (
|
|
typeof descriptor.value === 'function' &&
|
|
typeof previous === 'function'
|
|
) {
|
|
for (const key of Reflect.getMetadataKeys(previous))
|
|
Reflect.defineMetadata(
|
|
key,
|
|
Reflect.getMetadata(key, previous),
|
|
descriptor.value,
|
|
);
|
|
}
|
|
Object.defineProperty(target, method, descriptor);
|
|
}
|
|
}
|
|
}
|
|
|
|
const Redis = require('ioredis').default;
|
|
const prefix = `experiment:epoch-variants:${randomUUID()}`;
|
|
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
|
|
const observer = new Redis(process.env.CACHE_URL);
|
|
process.env.CACHE_URL = proxy.url;
|
|
const apps = {};
|
|
const chatIds = [];
|
|
const userIds = [];
|
|
let prisma, token;
|
|
const report = {
|
|
recorded_at: new Date().toISOString(),
|
|
methodology: {
|
|
concurrency: 32,
|
|
duration_seconds: 60,
|
|
warmup_seconds: 10,
|
|
repetitions: 3,
|
|
seed_messages: 100000,
|
|
writes_per_repeat: 4096,
|
|
order: [
|
|
['baseline', 'profile-join', 'access-join', 'striped'],
|
|
['striped', 'access-join', 'profile-join', 'baseline'],
|
|
['profile-join', 'striped', 'baseline', 'access-join'],
|
|
],
|
|
requests:
|
|
'Real loopback HTTP; shared read fixtures, separate write chats and cache prefixes',
|
|
resources:
|
|
'Four idle/active API instances and HTTP generator share one Node process; only one workload active at a time',
|
|
sql: 'pg.Client.query; transaction-control commands excluded; no changes to database pool size',
|
|
},
|
|
variants: manifest.variants,
|
|
samples: [],
|
|
correctness: {},
|
|
completed: false,
|
|
};
|
|
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
async function request(name, method, url, body, bearer = token) {
|
|
const response = await fetch(`${apps[name].url}${url}`, {
|
|
method,
|
|
headers: {
|
|
authorization: `Bearer ${bearer}`,
|
|
...(body ? { 'content-type': 'application/json' } : {}),
|
|
},
|
|
...(body ? { body: JSON.stringify(body) } : {}),
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
const json = await response.json();
|
|
return { status: response.status, json };
|
|
}
|
|
async function call(name, method, url, body) {
|
|
const result = await request(name, method, url, body);
|
|
assert.equal(result.status, method === 'POST' ? 201 : 200);
|
|
return result.json;
|
|
}
|
|
async function clearCache() {
|
|
let cursor = '0';
|
|
do {
|
|
const [next, keys] = await observer.scan(
|
|
cursor,
|
|
'MATCH',
|
|
`${prefix}:*`,
|
|
'COUNT',
|
|
100,
|
|
);
|
|
cursor = next;
|
|
if (keys.length) await observer.del(...keys);
|
|
} while (cursor !== '0');
|
|
}
|
|
async function newChat(userId) {
|
|
const chat = await prisma.chat.create({
|
|
data: {
|
|
title: 'Variant benchmark',
|
|
members: { create: { userId, position: 0 } },
|
|
},
|
|
});
|
|
chatIds.push(chat.id);
|
|
return chat.id;
|
|
}
|
|
async function epoch(name, chatId) {
|
|
if (name !== 'striped')
|
|
return (await prisma.chat.findUniqueOrThrow({ where: { id: chatId } }))
|
|
.messagesEpoch;
|
|
const [row] = await prisma.$queryRawUnsafe(
|
|
`SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM "${manifest.schema}".chat_message_epochs WHERE chat_id = $1::uuid`,
|
|
chatId,
|
|
);
|
|
return row.epoch;
|
|
}
|
|
async function measure(name, workload, repeat, operation, total) {
|
|
counts = {};
|
|
const latencies = [];
|
|
let errors = 0,
|
|
index = 0;
|
|
const cpu = process.cpuUsage();
|
|
const start = performance.now();
|
|
await Promise.all(
|
|
Array.from({ length: 32 }, async () => {
|
|
while (
|
|
total === undefined ? performance.now() - start < 60000 : index < total
|
|
) {
|
|
const item = index++;
|
|
const began = performance.now();
|
|
try {
|
|
await operation(item);
|
|
latencies.push(performance.now() - began);
|
|
} catch {
|
|
errors++;
|
|
}
|
|
}
|
|
}),
|
|
);
|
|
const seconds = (performance.now() - start) / 1000;
|
|
const used = process.cpuUsage(cpu);
|
|
latencies.sort((a, b) => a - b);
|
|
const calls = { ...counts };
|
|
const queries = Object.entries(calls)
|
|
.filter(([verb]) => !['BEGIN', 'COMMIT', 'ROLLBACK', 'SET'].includes(verb))
|
|
.reduce((sum, [, count]) => sum + count, 0);
|
|
const sample = {
|
|
variant: name,
|
|
workload,
|
|
repeat,
|
|
successes: latencies.length,
|
|
errors,
|
|
elapsed_seconds: +seconds.toFixed(3),
|
|
rps: +(latencies.length / seconds).toFixed(3),
|
|
mean_latency_ms: +(
|
|
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
|
|
).toFixed(3),
|
|
p95_ms: +latencies[Math.ceil(latencies.length * 0.95) - 1]?.toFixed(3),
|
|
sql_queries: queries,
|
|
sql_per_request: +(queries / (latencies.length + errors)).toFixed(4),
|
|
sql_calls: calls,
|
|
cpu_percent_one_core: +(
|
|
(used.user + used.system) /
|
|
(seconds * 10000)
|
|
).toFixed(2),
|
|
rss_mib: +(process.memoryUsage().rss / 1048576).toFixed(2),
|
|
};
|
|
report.samples.push(sample);
|
|
console.log(JSON.stringify(sample));
|
|
assert.equal(errors, 0, 'HTTP workload errors');
|
|
}
|
|
async function reconnect() {
|
|
proxy.setMode('online');
|
|
for (let attempt = 0; attempt < 80; attempt++) {
|
|
if (
|
|
Object.values(apps).every(
|
|
({ app }) => app.get(CacheService).redis.status === 'ready',
|
|
)
|
|
)
|
|
return;
|
|
await pause(100);
|
|
}
|
|
assert.fail('Redis reconnect timeout');
|
|
}
|
|
async function correctness(name, userId, otherToken) {
|
|
activate(name);
|
|
await clearCache();
|
|
const replica = Object.keys(apps).find((candidate) => candidate !== name);
|
|
const chatId = await newChat(userId);
|
|
const url = `/api/v1/chats/${chatId}/messages`;
|
|
assert.equal(
|
|
(
|
|
await request(
|
|
name,
|
|
'GET',
|
|
url,
|
|
undefined,
|
|
randomBytes(32).toString('base64url'),
|
|
)
|
|
).status,
|
|
401,
|
|
);
|
|
assert.equal(
|
|
(await request(name, 'GET', url, undefined, otherToken)).status,
|
|
404,
|
|
);
|
|
assert.equal(
|
|
(await request(name, 'POST', url, { text: 'Forbidden' }, otherToken))
|
|
.status,
|
|
404,
|
|
);
|
|
assert.equal(
|
|
(await request(name, 'GET', '/api/v1/chats/invalid/messages')).status,
|
|
400,
|
|
);
|
|
assert.equal(
|
|
(
|
|
await request(
|
|
name,
|
|
'GET',
|
|
'/api/v1/chats/invalid/messages',
|
|
undefined,
|
|
randomBytes(32).toString('base64url'),
|
|
)
|
|
).status,
|
|
401,
|
|
);
|
|
assert.equal((await call(replica, 'GET', url)).items.length, 0);
|
|
const keys = await observer.keys(`${prefix}:${replica}:messages:${chatId}:*`);
|
|
assert.ok(keys.length);
|
|
proxy.setMode('cut');
|
|
await pause(100);
|
|
const posted = await call(name, 'POST', url, { text: 'During outage' });
|
|
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
|
|
await reconnect();
|
|
assert.equal(await observer.exists(...keys), keys.length);
|
|
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
|
|
const service = apps[name].app.get(ChatService);
|
|
const actual = service.prisma;
|
|
service.prisma = actual.$extends({
|
|
query: {
|
|
$queryRaw({ args, query }) {
|
|
const index = args.values.indexOf('Must roll back');
|
|
if (index !== -1) args.values[index] = 'x'.repeat(4001);
|
|
return query(args);
|
|
},
|
|
},
|
|
});
|
|
try {
|
|
assert.equal(
|
|
(await request(name, 'POST', url, { text: 'Must roll back' })).status,
|
|
500,
|
|
);
|
|
} finally {
|
|
service.prisma = actual;
|
|
}
|
|
assert.equal(await epoch(name, chatId), 1n);
|
|
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
|
|
const raceChat = await newChat(userId);
|
|
const raceUrl = `/api/v1/chats/${raceChat}/messages`;
|
|
let began, release;
|
|
const started = new Promise((resolve) => {
|
|
began = resolve;
|
|
});
|
|
const held = new Promise((resolve) => {
|
|
release = resolve;
|
|
});
|
|
service.prisma = actual.$extends({
|
|
query: {
|
|
message: {
|
|
async findMany({ args, query }) {
|
|
const rows = await query(args);
|
|
if (args.where?.chatId === raceChat) {
|
|
began();
|
|
await held;
|
|
}
|
|
return rows;
|
|
},
|
|
},
|
|
},
|
|
});
|
|
const pending = call(name, 'GET', raceUrl);
|
|
try {
|
|
await Promise.race([
|
|
started,
|
|
pause(10000).then(() => {
|
|
throw new Error('Race load timeout');
|
|
}),
|
|
]);
|
|
const fresh = await call(replica, 'POST', raceUrl, {
|
|
text: 'After old SELECT',
|
|
});
|
|
release();
|
|
await pending;
|
|
assert.equal((await call(name, 'GET', raceUrl)).items[0].id, fresh.id);
|
|
} finally {
|
|
release();
|
|
service.prisma = actual;
|
|
}
|
|
await clearCache();
|
|
assert.equal((await call(name, 'GET', url)).items[0].id, posted.id);
|
|
report.correctness[name] = {
|
|
access_statuses: true,
|
|
redis_outage_and_retained_keys: true,
|
|
rollback: true,
|
|
delayed_old_fill: true,
|
|
empty_cache: true,
|
|
};
|
|
console.log(JSON.stringify({ correctness: name, result: 'PASS' }));
|
|
}
|
|
|
|
try {
|
|
for (const variant of manifest.variants) {
|
|
activate(variant.name);
|
|
process.env.CACHE_PREFIX = `${prefix}:${variant.name}`;
|
|
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
|
logger: false,
|
|
abortOnError: false,
|
|
});
|
|
apps[variant.name] = { app };
|
|
exposeRequestIdHeader(app);
|
|
await app.listen(0, '127.0.0.1');
|
|
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
|
apps[variant.name].url = await app.getUrl();
|
|
}
|
|
prisma = apps.baseline.app.get(PrismaService);
|
|
const user = await prisma.user.create({
|
|
data: {
|
|
username: `variant_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
|
displayName: 'Variant user',
|
|
},
|
|
});
|
|
userIds.push(user.id);
|
|
token = randomBytes(32).toString('base64url');
|
|
await prisma.session.create({
|
|
data: {
|
|
userId: user.id,
|
|
tokenHash: createHash('sha256').update(token).digest('hex'),
|
|
expiresAt: new Date(Date.now() + 3600000),
|
|
},
|
|
});
|
|
const other = await prisma.user.create({
|
|
data: {
|
|
username: `other_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
|
displayName: 'Other user',
|
|
},
|
|
});
|
|
userIds.push(other.id);
|
|
const otherToken = randomBytes(32).toString('base64url');
|
|
await prisma.session.create({
|
|
data: {
|
|
userId: other.id,
|
|
tokenHash: createHash('sha256').update(otherToken).digest('hex'),
|
|
expiresAt: new Date(Date.now() + 3600000),
|
|
},
|
|
});
|
|
for (const name of Object.keys(apps))
|
|
await correctness(name, user.id, otherToken);
|
|
const readChat = await newChat(user.id);
|
|
for (let offset = 0; offset < 100000; offset += 1000) {
|
|
await prisma.message.createMany({
|
|
data: Array.from({ length: 1000 }, (_, i) => ({
|
|
chatId: readChat,
|
|
senderId: user.id,
|
|
text: `Synthetic message ${offset + i}`,
|
|
createdAt: new Date(1700000000000 + offset + i),
|
|
})),
|
|
});
|
|
}
|
|
await prisma.chat.update({ where: { id: readChat }, data: { messagesEpoch: 100000n } });
|
|
await prisma.$executeRawUnsafe(`INSERT INTO "${manifest.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, readChat);
|
|
for (const [index, order] of report.methodology.order.entries()) {
|
|
for (const name of order) {
|
|
activate(name);
|
|
await clearCache();
|
|
const url = `/api/v1/chats/${readChat}/messages?limit=50`;
|
|
const warmUntil = performance.now() + 10000;
|
|
while (performance.now() < warmUntil) await call(name, 'GET', url);
|
|
await measure(name, 'warm-history', index + 1, () =>
|
|
call(name, 'GET', url),
|
|
);
|
|
const writeChat = await newChat(user.id);
|
|
const writeUrl = `/api/v1/chats/${writeChat}/messages`;
|
|
assert.equal((await call(name, 'GET', writeUrl)).items.length, 0);
|
|
await measure(
|
|
name,
|
|
'writes-single-chat',
|
|
index + 1,
|
|
(i) => call(name, 'POST', writeUrl, { text: `Concurrent ${i}` }),
|
|
4096,
|
|
);
|
|
assert.equal(
|
|
await prisma.message.count({ where: { chatId: writeChat } }),
|
|
4096,
|
|
);
|
|
assert.equal(await epoch(name, writeChat), 4096n);
|
|
assert.equal((await call(name, 'GET', writeUrl)).items.length, 50);
|
|
}
|
|
}
|
|
report.completed = true;
|
|
} finally {
|
|
await reconnect();
|
|
if (prisma) {
|
|
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
|
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
|
|
}
|
|
await clearCache();
|
|
for (const { app } of Object.values(apps)) await app.close();
|
|
observer.disconnect();
|
|
await proxy.close();
|
|
await writeFile(
|
|
'/results/epoch-variants.json',
|
|
JSON.stringify(report, null, 2) + '\n',
|
|
);
|
|
}
|
|
console.log(
|
|
JSON.stringify({
|
|
completed: report.completed,
|
|
correctness: report.correctness,
|
|
}),
|
|
);
|