forked from mtusi2026/mtusi2026lab3
test: cover persistence failover and graceful draining
Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
@@ -0,0 +1,541 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
||||
import { createRequire } from 'node:module';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { faultProxy } from './helpers/fault-proxy.mjs';
|
||||
import { messageEpoch } from './helpers/message-epoch.mjs';
|
||||
|
||||
const require = createRequire(`${process.cwd()}/package.json`);
|
||||
const pg = require('pg');
|
||||
let counts = {};
|
||||
const originalQuery = pg.Client.prototype.query;
|
||||
pg.Client.prototype.query = function (query, ...args) {
|
||||
const sql = typeof query === 'string' ? query : query.text;
|
||||
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
|
||||
const table =
|
||||
['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) =>
|
||||
new RegExp(`\\b${name}\\b`).test(sql),
|
||||
) ?? 'other';
|
||||
const label = `${verb}:${table}`;
|
||||
counts[label] = (counts[label] ?? 0) + 1;
|
||||
return originalQuery.call(this, query, ...args);
|
||||
};
|
||||
const Redis = require('ioredis').default;
|
||||
const { NestFactory } = require('@nestjs/core');
|
||||
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
|
||||
const { PrismaService } = require(
|
||||
`${process.cwd()}/dist/prisma/prisma.service`,
|
||||
);
|
||||
const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`);
|
||||
const { createFastifyAdapter, exposeRequestIdHeader } = require(
|
||||
`${process.cwd()}/dist/common/http/fastify-adapter`,
|
||||
);
|
||||
const stage = process.argv[2];
|
||||
const compareRedis = process.argv.includes('--compare-redis');
|
||||
const historyControl = !process.argv.includes('--skip-history-control');
|
||||
function integerOption(name, fallback) {
|
||||
const index = process.argv.indexOf(`--${name}`);
|
||||
const value = index === -1 ? fallback : Number(process.argv[index + 1]);
|
||||
assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`);
|
||||
return value;
|
||||
}
|
||||
const concurrency = integerOption('concurrency', 16);
|
||||
const duration = integerOption('duration-seconds', 10);
|
||||
const warmup = integerOption('warmup-seconds', 3);
|
||||
const repetitions = integerOption('repetitions', 3);
|
||||
const seedMessages = integerOption('seed-messages', 10000);
|
||||
const coldBurst = integerOption('cold-burst', 32);
|
||||
const writesPerRepeat = integerOption('writes-per-repeat', 512);
|
||||
assert.ok(['before', 'after'].includes(stage));
|
||||
const prefix = `experiment:epochs:${randomUUID()}`;
|
||||
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
|
||||
const observer = new Redis(process.env.CACHE_URL);
|
||||
process.env.CACHE_PREFIX = prefix;
|
||||
process.env.CACHE_URL = proxy.url;
|
||||
const apps = [];
|
||||
async function start() {
|
||||
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
|
||||
logger: false,
|
||||
abortOnError: false,
|
||||
});
|
||||
apps.push(app);
|
||||
exposeRequestIdHeader(app);
|
||||
await app.listen(0, '127.0.0.1');
|
||||
require('nestjs-pino').PinoLogger.root.level = 'silent';
|
||||
return { app, url: await app.getUrl() };
|
||||
}
|
||||
const first = await start();
|
||||
const prisma = first.app.get(PrismaService);
|
||||
let userId;
|
||||
const chatIds = [];
|
||||
const report = {
|
||||
stage,
|
||||
recorded_at: new Date().toISOString(),
|
||||
node: process.version,
|
||||
methodology: {
|
||||
requests:
|
||||
'real loopback HTTP; instrumented API process in measured backend image',
|
||||
sql: 'pg.Client.query calls; transaction control counted separately',
|
||||
concurrency,
|
||||
duration_seconds: duration,
|
||||
warmup_seconds: warmup,
|
||||
repetitions,
|
||||
seed_messages: seedMessages,
|
||||
cold_burst_requests: coldBurst,
|
||||
writes_per_repeat: writesPerRepeat,
|
||||
history_control: historyControl,
|
||||
resources:
|
||||
'process CPU/RSS includes HTTP generator and instrumentation; not container CPU',
|
||||
limitations:
|
||||
'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF',
|
||||
},
|
||||
dist_sha256: {},
|
||||
dependencies: {},
|
||||
samples: [],
|
||||
correctness: {},
|
||||
};
|
||||
for (const name of [
|
||||
'@nestjs/core',
|
||||
'@prisma/adapter-pg',
|
||||
'pg',
|
||||
'ioredis',
|
||||
'fastify',
|
||||
]) {
|
||||
try {
|
||||
report.dependencies[name] = require(`${name}/package.json`).version;
|
||||
} catch {
|
||||
report.dependencies[name] = 'package metadata not exported';
|
||||
}
|
||||
}
|
||||
for (const name of [
|
||||
'Chat/chat.service.js',
|
||||
'cache/cache.service.js',
|
||||
'Auth/auth.service.js',
|
||||
'User/user.guard.js',
|
||||
'Chat/chat.controller.js',
|
||||
]) {
|
||||
report.dist_sha256[name] = createHash('sha256')
|
||||
.update(await readFile(`${process.cwd()}/dist/${name}`))
|
||||
.digest('hex');
|
||||
}
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
async function clearCache() {
|
||||
let cursor = '0';
|
||||
do {
|
||||
const [next, keys] = await observer.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}:*`,
|
||||
'COUNT',
|
||||
100,
|
||||
);
|
||||
cursor = next;
|
||||
if (keys.length) await observer.del(...keys);
|
||||
} while (cursor !== '0');
|
||||
}
|
||||
let token;
|
||||
async function request(method, path, body, target = first) {
|
||||
const response = await fetch(`${target.url}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
...(body ? { 'content-type': 'application/json' } : {}),
|
||||
...(method === 'POST' && /\/messages$/.test(path)
|
||||
? { 'idempotency-key': randomUUID() }
|
||||
: {}),
|
||||
},
|
||||
...(body ? { body: JSON.stringify(body) } : {}),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
const json = await response.json();
|
||||
assert.equal(
|
||||
response.status,
|
||||
method === 'POST' ? 201 : 200,
|
||||
`HTTP ${response.status}`,
|
||||
);
|
||||
return json;
|
||||
}
|
||||
async function newChat() {
|
||||
const chat = await prisma.chat.create({
|
||||
data: {
|
||||
title: 'Synthetic epoch benchmark',
|
||||
members: { create: { userId, position: 0 } },
|
||||
},
|
||||
});
|
||||
chatIds.push(chat.id);
|
||||
return chat.id;
|
||||
}
|
||||
async function measure(
|
||||
workload,
|
||||
repeat,
|
||||
operation,
|
||||
{
|
||||
duration: secondsToRun = duration,
|
||||
total,
|
||||
concurrency: clients = concurrency,
|
||||
} = {},
|
||||
) {
|
||||
counts = {};
|
||||
const cpu = process.cpuUsage();
|
||||
let peakRss = process.memoryUsage().rss;
|
||||
const timer = setInterval(() => {
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
}, 100);
|
||||
const latencies = [];
|
||||
let errors = 0,
|
||||
index = 0;
|
||||
const start = performance.now();
|
||||
await Promise.all(
|
||||
Array.from({ length: clients }, async () => {
|
||||
while (
|
||||
total === undefined
|
||||
? performance.now() - start < secondsToRun * 1000
|
||||
: index < total
|
||||
) {
|
||||
const item = index++;
|
||||
const began = performance.now();
|
||||
try {
|
||||
await operation(item);
|
||||
latencies.push(performance.now() - began);
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
const seconds = (performance.now() - start) / 1000;
|
||||
clearInterval(timer);
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
const used = process.cpuUsage(cpu);
|
||||
latencies.sort((a, b) => a - b);
|
||||
const percentile = (p) =>
|
||||
latencies.length
|
||||
? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3)
|
||||
: null;
|
||||
const sql = Object.entries(counts)
|
||||
.filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key))
|
||||
.reduce((sum, [, n]) => sum + n, 0);
|
||||
const sample = {
|
||||
workload,
|
||||
repeat,
|
||||
concurrency: clients,
|
||||
elapsed_seconds: +seconds.toFixed(3),
|
||||
successes: latencies.length,
|
||||
errors,
|
||||
error_ratio: errors / (latencies.length + errors),
|
||||
successful_rps: +(latencies.length / seconds).toFixed(3),
|
||||
latency_ms: {
|
||||
mean: latencies.length
|
||||
? +(
|
||||
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
|
||||
).toFixed(3)
|
||||
: null,
|
||||
p50: percentile(0.5),
|
||||
p95: percentile(0.95),
|
||||
p99: percentile(0.99),
|
||||
},
|
||||
sql_queries: sql,
|
||||
sql_per_request: +(sql / (latencies.length + errors)).toFixed(4),
|
||||
sql_calls: { ...counts },
|
||||
process_cpu_percent_one_core: +(
|
||||
(used.user + used.system) /
|
||||
(seconds * 10000)
|
||||
).toFixed(2),
|
||||
peak_process_rss_mib: +(peakRss / 1048576).toFixed(2),
|
||||
};
|
||||
report.samples.push(sample);
|
||||
console.log(JSON.stringify(sample));
|
||||
assert.equal(errors, 0, 'load errors recorded in report');
|
||||
}
|
||||
try {
|
||||
const user = await prisma.user.create({
|
||||
data: {
|
||||
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
|
||||
displayName: 'Synthetic benchmark',
|
||||
},
|
||||
});
|
||||
userId = user.id;
|
||||
token = randomBytes(32).toString('base64url');
|
||||
await prisma.session.create({
|
||||
data: {
|
||||
userId,
|
||||
tokenHash: createHash('sha256').update(token).digest('hex'),
|
||||
expiresAt: new Date(Date.now() + 3600000),
|
||||
},
|
||||
});
|
||||
const readChat = await newChat();
|
||||
for (let offset = 0; offset < seedMessages; offset += 1000) {
|
||||
await prisma.message.createMany({
|
||||
data: Array.from(
|
||||
{ length: Math.min(1000, seedMessages - offset) },
|
||||
(_, i) => ({
|
||||
chatId: readChat,
|
||||
senderId: userId,
|
||||
text: `Synthetic message ${offset + i}`,
|
||||
createdAt: new Date(1700000000000 + offset + i),
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
await prisma.chatMessageEpoch.createMany({
|
||||
data: Array.from({ length: 16 }, (_, bucket) => ({
|
||||
chatId: readChat,
|
||||
bucket,
|
||||
epoch: BigInt(
|
||||
Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0),
|
||||
),
|
||||
})),
|
||||
});
|
||||
const path = `/api/v1/chats/${readChat}/messages?limit=50`;
|
||||
if (compareRedis) {
|
||||
const cacheUrl = process.env.CACHE_URL;
|
||||
delete process.env.CACHE_URL;
|
||||
let withoutRedis;
|
||||
try {
|
||||
withoutRedis = await start();
|
||||
} finally {
|
||||
process.env.CACHE_URL = cacheUrl;
|
||||
}
|
||||
assert.equal(withoutRedis.app.get(CacheService).redis, undefined);
|
||||
report.methodology.comparison = {
|
||||
modes: ['redis', 'no-redis'],
|
||||
no_redis:
|
||||
'API started without CACHE_URL; all application Redis caches disabled',
|
||||
epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes',
|
||||
order: 'redis first in repeats 1/3, no-redis first in repeat 2',
|
||||
processes:
|
||||
'both API instances and HTTP generator share one measured Node process',
|
||||
};
|
||||
report.correctness.read_after_write = {};
|
||||
for (let repeat = 1; repeat <= repetitions; repeat++) {
|
||||
const variants = [
|
||||
{ name: 'redis', target: first },
|
||||
{ name: 'no-redis', target: withoutRedis },
|
||||
];
|
||||
if (repeat % 2 === 0) variants.reverse();
|
||||
for (const { name, target } of variants) {
|
||||
await clearCache();
|
||||
await measure(
|
||||
`cold-history-burst:${name}`,
|
||||
repeat,
|
||||
() => request('GET', path, undefined, target),
|
||||
{ total: coldBurst, concurrency: coldBurst },
|
||||
);
|
||||
const warmUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < warmUntil)
|
||||
await request('GET', path, undefined, target);
|
||||
await measure(`warm-history:${name}`, repeat, () =>
|
||||
request('GET', path, undefined, target),
|
||||
);
|
||||
const writeChat = await newChat();
|
||||
const writePath = `/api/v1/chats/${writeChat}/messages`;
|
||||
assert.equal(
|
||||
(await request('GET', writePath, undefined, target)).items.length,
|
||||
0,
|
||||
);
|
||||
await measure(
|
||||
`writes-single-chat:${name}`,
|
||||
repeat,
|
||||
(i) =>
|
||||
request(
|
||||
'POST',
|
||||
writePath,
|
||||
{ text: `Concurrent write ${i}` },
|
||||
target,
|
||||
),
|
||||
{ total: writesPerRepeat },
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
writesPerRepeat,
|
||||
);
|
||||
assert.equal(
|
||||
await messageEpoch(prisma, writeChat),
|
||||
BigInt(writesPerRepeat),
|
||||
);
|
||||
const page = await request('GET', writePath, undefined, target);
|
||||
assert.equal(page.items.length, 50);
|
||||
assert.equal(new Set(page.items.map((message) => message.id)).size, 50);
|
||||
report.correctness.read_after_write[`${name}:${repeat}`] = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const cache = first.app.get(CacheService);
|
||||
const remember = cache.remember.bind(cache);
|
||||
for (let repeat = 1; repeat <= repetitions; repeat++) {
|
||||
await clearCache();
|
||||
await request('GET', `/api/v1/chats/${readChat}`);
|
||||
await measure('cold-history-burst', repeat, () => request('GET', path), {
|
||||
total: coldBurst,
|
||||
concurrency: coldBurst,
|
||||
});
|
||||
const warmUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < warmUntil) await request('GET', path);
|
||||
await measure('warm-history', repeat, () => request('GET', path));
|
||||
if (historyControl) {
|
||||
cache.remember = (namespace, parameters, ttl, load) =>
|
||||
namespace.startsWith('messages:')
|
||||
? load()
|
||||
: remember(namespace, parameters, ttl, load);
|
||||
const directUntil = performance.now() + warmup * 1000;
|
||||
while (performance.now() < directUntil) await request('GET', path);
|
||||
await measure('history-cache-bypassed', repeat, () =>
|
||||
request('GET', path),
|
||||
);
|
||||
cache.remember = remember;
|
||||
}
|
||||
const writeChat = await newChat();
|
||||
await request('GET', `/api/v1/chats/${writeChat}`);
|
||||
await measure(
|
||||
'writes-single-chat',
|
||||
repeat,
|
||||
(i) =>
|
||||
request('POST', `/api/v1/chats/${writeChat}/messages`, {
|
||||
text: `Concurrent write ${i}`,
|
||||
}),
|
||||
{ total: writesPerRepeat },
|
||||
);
|
||||
assert.equal(
|
||||
await prisma.message.count({ where: { chatId: writeChat } }),
|
||||
writesPerRepeat,
|
||||
);
|
||||
}
|
||||
const second = await start();
|
||||
const faultChat = await newChat();
|
||||
const faultPath = `/api/v1/chats/${faultChat}/messages`;
|
||||
assert.equal(
|
||||
(await request('GET', faultPath, undefined, second)).items.length,
|
||||
0,
|
||||
);
|
||||
const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`);
|
||||
assert.ok(keysBefore.length);
|
||||
proxy.setMode('cut');
|
||||
await sleep(100);
|
||||
const written = await request('POST', faultPath, {
|
||||
text: 'Committed while Redis is unavailable',
|
||||
});
|
||||
const during = await request('GET', faultPath, undefined, second);
|
||||
proxy.setMode('online');
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (apps.every((app) => app.get(CacheService).redis?.status === 'ready'))
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
assert.ok(
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
|
||||
);
|
||||
const recovered = await request('GET', faultPath, undefined, second);
|
||||
report.correctness.redis_reconnect = {
|
||||
old_keys_retained:
|
||||
(await observer.exists(...keysBefore)) === keysBefore.length,
|
||||
read_after_write_during_outage: during.items.some(
|
||||
(x) => x.id === written.id,
|
||||
),
|
||||
read_after_write_after_reconnect_other_replica: recovered.items.some(
|
||||
(x) => x.id === written.id,
|
||||
),
|
||||
};
|
||||
await clearCache();
|
||||
const cold = await request('GET', faultPath, undefined, second);
|
||||
report.correctness.empty_cache_recovery = cold.items.some(
|
||||
(x) => x.id === written.id,
|
||||
);
|
||||
const raceChat = await newChat();
|
||||
const racePath = `/api/v1/chats/${raceChat}/messages`;
|
||||
const service = first.app.get(
|
||||
require(`${process.cwd()}/dist/Chat/chat.service`).ChatService,
|
||||
);
|
||||
const actualPrisma = service.prisma;
|
||||
let release, began;
|
||||
const started = new Promise((resolve) => {
|
||||
began = resolve;
|
||||
});
|
||||
const held = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
service.prisma = actualPrisma.$extends({
|
||||
query: {
|
||||
message: {
|
||||
async findMany({ args, query }) {
|
||||
const rows = await query(args);
|
||||
if (args.where?.chatId === raceChat) {
|
||||
began();
|
||||
await held;
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const pending = request('GET', racePath);
|
||||
let fresh;
|
||||
try {
|
||||
await Promise.race([
|
||||
started,
|
||||
sleep(10000).then(() => {
|
||||
throw new Error('race load did not start');
|
||||
}),
|
||||
]);
|
||||
proxy.setMode('cut');
|
||||
await sleep(100);
|
||||
fresh = await request(
|
||||
'POST',
|
||||
racePath,
|
||||
{ text: 'New message after old SELECT' },
|
||||
second,
|
||||
);
|
||||
proxy.setMode('online');
|
||||
for (let i = 0; i < 60; i++) {
|
||||
if (
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready')
|
||||
)
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
assert.ok(
|
||||
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
|
||||
);
|
||||
} finally {
|
||||
release();
|
||||
service.prisma = actualPrisma;
|
||||
}
|
||||
await pending;
|
||||
report.correctness.delayed_old_fill = (
|
||||
await request('GET', racePath, undefined, second)
|
||||
).items.some((x) => x.id === fresh.id);
|
||||
if (stage === 'after') {
|
||||
assert.equal(
|
||||
report.correctness.redis_reconnect
|
||||
.read_after_write_after_reconnect_other_replica,
|
||||
true,
|
||||
);
|
||||
assert.equal(report.correctness.delayed_old_fill, true);
|
||||
assert.equal(report.correctness.empty_cache_recovery, true);
|
||||
}
|
||||
}
|
||||
report.completed = true;
|
||||
} finally {
|
||||
proxy.setMode('online');
|
||||
try {
|
||||
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
|
||||
if (userId) await prisma.user.delete({ where: { id: userId } });
|
||||
await clearCache();
|
||||
report.cleanup = 'only experiment chats/user/cache prefix removed';
|
||||
} finally {
|
||||
for (const app of apps) await app.close();
|
||||
observer.disconnect();
|
||||
await proxy.close();
|
||||
await writeFile(
|
||||
process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`,
|
||||
JSON.stringify(report, null, 2) + '\n',
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
stage,
|
||||
correctness: report.correctness,
|
||||
completed: report.completed,
|
||||
}),
|
||||
);
|
||||
Reference in New Issue
Block a user