test: cover persistence failover and graceful draining

Add API integration checks, migration compatibility, rolling restart scenarios and a report verification collector.
This commit is contained in:
2026-10-29 11:47:46 +03:00
parent 676b38c18b
commit 5c8bcf0f80
37 changed files with 5538 additions and 0 deletions
+50
View File
@@ -0,0 +1,50 @@
import { createRequire } from 'node:module';
import { randomUUID } from 'node:crypto';
import { performance } from 'node:perf_hooks';
const require = createRequire(`${process.cwd()}/package.json`);
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
const { PrismaService } = require(`${process.cwd()}/dist/prisma/prisma.service`);
const { UserService } = require(`${process.cwd()}/dist/User/user.service`);
const { PasswordService } = require(`${process.cwd()}/dist/Auth/password.service`);
const { createFastifyAdapter } = require(`${process.cwd()}/dist/common/http/fastify-adapter`);
process.env.CACHE_URL = '';
const app = await NestFactory.create(AppModule, createFastifyAdapter(), { logger: false, abortOnError: false });
await app.init();
require('nestjs-pino').PinoLogger.root.level = 'silent';
const server = app.getHttpAdapter().getInstance();
await server.ready();
const prisma = app.get(PrismaService);
const passwords = app.get(PasswordService);
const originalHash = passwords.hash.bind(passwords);
const spans = [];
passwords.hash = async (password) => {
const start = performance.now();
const value = await originalHash(password);
spans.push({ phase: 'kdf', ms: Math.round(performance.now() - start) });
return value;
};
app.get(UserService).prisma = prisma.$extends({ query: { user: { async create({ args, query }) {
const start = performance.now();
const value = await query(args);
spans.push({ phase: 'sql_user_create', ms: Math.round(performance.now() - start) });
return value;
} } } });
const ids = [];
try {
const health = await server.inject({ method: 'GET', url: '/health/ready' });
console.log(JSON.stringify({ phase: 'health', status: health.statusCode }));
for (let sample = 1; sample <= 3; sample++) {
spans.length = 0;
const start = performance.now();
const response = await server.inject({ method: 'POST', url: '/api/v1/users', payload: {
username: `warm_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
display_name: 'Synthetic startup diagnostic', password: 'Synthetic-diagnostic-password-123',
} });
if (response.statusCode === 201) ids.push(response.json().id);
console.log(JSON.stringify({ sample, status: response.statusCode, total_ms: Math.round(performance.now() - start), spans }));
}
} finally {
await prisma.user.deleteMany({ where: { id: { in: ids } } });
await app.close();
}
+64
View File
@@ -0,0 +1,64 @@
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { createServer } from 'node:http';
import test from 'node:test';
test('scaling benchmark schedules reads and writes and reports distinct instances', async () => {
let sequence = 0;
const server = createServer(async (request, response) => {
for await (const _ of request) {
// Consume the request body before responding.
}
response.setHeader('Content-Type', 'application/json');
response.setHeader('X-Instance-Id', 'mock1');
let status = 200;
let body = {};
if (request.url === '/api/v1/users' && request.method === 'POST') {
status = 201;
body = { id: `user-${++sequence}` };
} else if (request.url === '/api/v1/auth/sessions') {
status = 201;
body = { token: 'synthetic-token' };
} else if (request.url === '/api/v1/chats' && request.method === 'POST') {
status = 201;
body = { id: 'chat-1' };
} else if (request.url.startsWith('/api/v1/chats/chat-1/messages')) {
if (request.method === 'POST') {
status = 201;
body = { id: `message-${++sequence}` };
} else {
body = { items: [] };
}
} else {
status = 404;
}
response.writeHead(status);
response.end(JSON.stringify(body));
});
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
try {
const port = server.address().port;
const args = [
'scripts/benchmark-scaling.mjs', '--base-url', `http://127.0.0.1:${port}`,
'--label', 'mock', '--rate', '20', '--duration-seconds', '2',
'--warmup-seconds', '1', '--seed-messages', '2',
'--read-percent', '50', '--expected-instances', '1',
];
const child = spawn(process.execPath, args, { cwd: process.cwd() });
let stdout = '';
let stderr = '';
child.stdout.on('data', (chunk) => { stdout += chunk; });
child.stderr.on('data', (chunk) => { stderr += chunk; });
const code = await new Promise((resolve) => child.on('close', resolve));
assert.equal(code, 0, stderr);
const result = JSON.parse(stdout);
assert.equal(result.measurement.scheduled_slots, 40);
assert.equal(result.measurement.client_dropped_slots, 0);
assert.equal(result.measurement.all.successful, 40);
assert.equal(result.measurement.all.instance_counts.mock1, 40);
assert.ok(result.measurement.read.successful > 0);
assert.ok(result.measurement.write.successful > 0);
} finally {
await new Promise((resolve) => server.close(resolve));
}
});
+541
View File
@@ -0,0 +1,541 @@
import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import { readFile, writeFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
import { faultProxy } from './helpers/fault-proxy.mjs';
import { messageEpoch } from './helpers/message-epoch.mjs';
const require = createRequire(`${process.cwd()}/package.json`);
const pg = require('pg');
let counts = {};
const originalQuery = pg.Client.prototype.query;
pg.Client.prototype.query = function (query, ...args) {
const sql = typeof query === 'string' ? query : query.text;
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
const table =
['messages', 'chat_members', 'sessions', 'users', 'chats'].find((name) =>
new RegExp(`\\b${name}\\b`).test(sql),
) ?? 'other';
const label = `${verb}:${table}`;
counts[label] = (counts[label] ?? 0) + 1;
return originalQuery.call(this, query, ...args);
};
const Redis = require('ioredis').default;
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require(`${process.cwd()}/dist/app.module`);
const { PrismaService } = require(
`${process.cwd()}/dist/prisma/prisma.service`,
);
const { CacheService } = require(`${process.cwd()}/dist/cache/cache.service`);
const { createFastifyAdapter, exposeRequestIdHeader } = require(
`${process.cwd()}/dist/common/http/fastify-adapter`,
);
const stage = process.argv[2];
const compareRedis = process.argv.includes('--compare-redis');
const historyControl = !process.argv.includes('--skip-history-control');
function integerOption(name, fallback) {
const index = process.argv.indexOf(`--${name}`);
const value = index === -1 ? fallback : Number(process.argv[index + 1]);
assert.ok(Number.isSafeInteger(value) && value > 0, `Invalid --${name}`);
return value;
}
const concurrency = integerOption('concurrency', 16);
const duration = integerOption('duration-seconds', 10);
const warmup = integerOption('warmup-seconds', 3);
const repetitions = integerOption('repetitions', 3);
const seedMessages = integerOption('seed-messages', 10000);
const coldBurst = integerOption('cold-burst', 32);
const writesPerRepeat = integerOption('writes-per-repeat', 512);
assert.ok(['before', 'after'].includes(stage));
const prefix = `experiment:epochs:${randomUUID()}`;
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
const observer = new Redis(process.env.CACHE_URL);
process.env.CACHE_PREFIX = prefix;
process.env.CACHE_URL = proxy.url;
const apps = [];
async function start() {
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
apps.push(app);
exposeRequestIdHeader(app);
await app.listen(0, '127.0.0.1');
require('nestjs-pino').PinoLogger.root.level = 'silent';
return { app, url: await app.getUrl() };
}
const first = await start();
const prisma = first.app.get(PrismaService);
let userId;
const chatIds = [];
const report = {
stage,
recorded_at: new Date().toISOString(),
node: process.version,
methodology: {
requests:
'real loopback HTTP; instrumented API process in measured backend image',
sql: 'pg.Client.query calls; transaction control counted separately',
concurrency,
duration_seconds: duration,
warmup_seconds: warmup,
repetitions,
seed_messages: seedMessages,
cold_burst_requests: coldBurst,
writes_per_repeat: writesPerRepeat,
history_control: historyControl,
resources:
'process CPU/RSS includes HTTP generator and instrumentation; not container CPU',
limitations:
'closed loop, client and API share process/CPU; other Docker workloads share host resources; no capacity/SLA claim; fixtures bypass password KDF',
},
dist_sha256: {},
dependencies: {},
samples: [],
correctness: {},
};
for (const name of [
'@nestjs/core',
'@prisma/adapter-pg',
'pg',
'ioredis',
'fastify',
]) {
try {
report.dependencies[name] = require(`${name}/package.json`).version;
} catch {
report.dependencies[name] = 'package metadata not exported';
}
}
for (const name of [
'Chat/chat.service.js',
'cache/cache.service.js',
'Auth/auth.service.js',
'User/user.guard.js',
'Chat/chat.controller.js',
]) {
report.dist_sha256[name] = createHash('sha256')
.update(await readFile(`${process.cwd()}/dist/${name}`))
.digest('hex');
}
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function clearCache() {
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
}
let token;
async function request(method, path, body, target = first) {
const response = await fetch(`${target.url}${path}`, {
method,
headers: {
authorization: `Bearer ${token}`,
...(body ? { 'content-type': 'application/json' } : {}),
...(method === 'POST' && /\/messages$/.test(path)
? { 'idempotency-key': randomUUID() }
: {}),
},
...(body ? { body: JSON.stringify(body) } : {}),
signal: AbortSignal.timeout(10000),
});
const json = await response.json();
assert.equal(
response.status,
method === 'POST' ? 201 : 200,
`HTTP ${response.status}`,
);
return json;
}
async function newChat() {
const chat = await prisma.chat.create({
data: {
title: 'Synthetic epoch benchmark',
members: { create: { userId, position: 0 } },
},
});
chatIds.push(chat.id);
return chat.id;
}
async function measure(
workload,
repeat,
operation,
{
duration: secondsToRun = duration,
total,
concurrency: clients = concurrency,
} = {},
) {
counts = {};
const cpu = process.cpuUsage();
let peakRss = process.memoryUsage().rss;
const timer = setInterval(() => {
peakRss = Math.max(peakRss, process.memoryUsage().rss);
}, 100);
const latencies = [];
let errors = 0,
index = 0;
const start = performance.now();
await Promise.all(
Array.from({ length: clients }, async () => {
while (
total === undefined
? performance.now() - start < secondsToRun * 1000
: index < total
) {
const item = index++;
const began = performance.now();
try {
await operation(item);
latencies.push(performance.now() - began);
} catch {
errors++;
}
}
}),
);
const seconds = (performance.now() - start) / 1000;
clearInterval(timer);
peakRss = Math.max(peakRss, process.memoryUsage().rss);
const used = process.cpuUsage(cpu);
latencies.sort((a, b) => a - b);
const percentile = (p) =>
latencies.length
? +latencies[Math.ceil(p * latencies.length) - 1].toFixed(3)
: null;
const sql = Object.entries(counts)
.filter(([key]) => !/^(BEGIN|COMMIT|ROLLBACK|SET):/.test(key))
.reduce((sum, [, n]) => sum + n, 0);
const sample = {
workload,
repeat,
concurrency: clients,
elapsed_seconds: +seconds.toFixed(3),
successes: latencies.length,
errors,
error_ratio: errors / (latencies.length + errors),
successful_rps: +(latencies.length / seconds).toFixed(3),
latency_ms: {
mean: latencies.length
? +(
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
).toFixed(3)
: null,
p50: percentile(0.5),
p95: percentile(0.95),
p99: percentile(0.99),
},
sql_queries: sql,
sql_per_request: +(sql / (latencies.length + errors)).toFixed(4),
sql_calls: { ...counts },
process_cpu_percent_one_core: +(
(used.user + used.system) /
(seconds * 10000)
).toFixed(2),
peak_process_rss_mib: +(peakRss / 1048576).toFixed(2),
};
report.samples.push(sample);
console.log(JSON.stringify(sample));
assert.equal(errors, 0, 'load errors recorded in report');
}
try {
const user = await prisma.user.create({
data: {
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Synthetic benchmark',
},
});
userId = user.id;
token = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 3600000),
},
});
const readChat = await newChat();
for (let offset = 0; offset < seedMessages; offset += 1000) {
await prisma.message.createMany({
data: Array.from(
{ length: Math.min(1000, seedMessages - offset) },
(_, i) => ({
chatId: readChat,
senderId: userId,
text: `Synthetic message ${offset + i}`,
createdAt: new Date(1700000000000 + offset + i),
}),
),
});
}
await prisma.chatMessageEpoch.createMany({
data: Array.from({ length: 16 }, (_, bucket) => ({
chatId: readChat,
bucket,
epoch: BigInt(
Math.floor(seedMessages / 16) + (bucket < seedMessages % 16 ? 1 : 0),
),
})),
});
const path = `/api/v1/chats/${readChat}/messages?limit=50`;
if (compareRedis) {
const cacheUrl = process.env.CACHE_URL;
delete process.env.CACHE_URL;
let withoutRedis;
try {
withoutRedis = await start();
} finally {
process.env.CACHE_URL = cacheUrl;
}
assert.equal(withoutRedis.app.get(CacheService).redis, undefined);
report.methodology.comparison = {
modes: ['redis', 'no-redis'],
no_redis:
'API started without CACHE_URL; all application Redis caches disabled',
epochs: 'PostgreSQL epochs and atomic CTE writes retained in both modes',
order: 'redis first in repeats 1/3, no-redis first in repeat 2',
processes:
'both API instances and HTTP generator share one measured Node process',
};
report.correctness.read_after_write = {};
for (let repeat = 1; repeat <= repetitions; repeat++) {
const variants = [
{ name: 'redis', target: first },
{ name: 'no-redis', target: withoutRedis },
];
if (repeat % 2 === 0) variants.reverse();
for (const { name, target } of variants) {
await clearCache();
await measure(
`cold-history-burst:${name}`,
repeat,
() => request('GET', path, undefined, target),
{ total: coldBurst, concurrency: coldBurst },
);
const warmUntil = performance.now() + warmup * 1000;
while (performance.now() < warmUntil)
await request('GET', path, undefined, target);
await measure(`warm-history:${name}`, repeat, () =>
request('GET', path, undefined, target),
);
const writeChat = await newChat();
const writePath = `/api/v1/chats/${writeChat}/messages`;
assert.equal(
(await request('GET', writePath, undefined, target)).items.length,
0,
);
await measure(
`writes-single-chat:${name}`,
repeat,
(i) =>
request(
'POST',
writePath,
{ text: `Concurrent write ${i}` },
target,
),
{ total: writesPerRepeat },
);
assert.equal(
await prisma.message.count({ where: { chatId: writeChat } }),
writesPerRepeat,
);
assert.equal(
await messageEpoch(prisma, writeChat),
BigInt(writesPerRepeat),
);
const page = await request('GET', writePath, undefined, target);
assert.equal(page.items.length, 50);
assert.equal(new Set(page.items.map((message) => message.id)).size, 50);
report.correctness.read_after_write[`${name}:${repeat}`] = true;
}
}
} else {
const cache = first.app.get(CacheService);
const remember = cache.remember.bind(cache);
for (let repeat = 1; repeat <= repetitions; repeat++) {
await clearCache();
await request('GET', `/api/v1/chats/${readChat}`);
await measure('cold-history-burst', repeat, () => request('GET', path), {
total: coldBurst,
concurrency: coldBurst,
});
const warmUntil = performance.now() + warmup * 1000;
while (performance.now() < warmUntil) await request('GET', path);
await measure('warm-history', repeat, () => request('GET', path));
if (historyControl) {
cache.remember = (namespace, parameters, ttl, load) =>
namespace.startsWith('messages:')
? load()
: remember(namespace, parameters, ttl, load);
const directUntil = performance.now() + warmup * 1000;
while (performance.now() < directUntil) await request('GET', path);
await measure('history-cache-bypassed', repeat, () =>
request('GET', path),
);
cache.remember = remember;
}
const writeChat = await newChat();
await request('GET', `/api/v1/chats/${writeChat}`);
await measure(
'writes-single-chat',
repeat,
(i) =>
request('POST', `/api/v1/chats/${writeChat}/messages`, {
text: `Concurrent write ${i}`,
}),
{ total: writesPerRepeat },
);
assert.equal(
await prisma.message.count({ where: { chatId: writeChat } }),
writesPerRepeat,
);
}
const second = await start();
const faultChat = await newChat();
const faultPath = `/api/v1/chats/${faultChat}/messages`;
assert.equal(
(await request('GET', faultPath, undefined, second)).items.length,
0,
);
const keysBefore = await observer.keys(`${prefix}:messages:${faultChat}:*`);
assert.ok(keysBefore.length);
proxy.setMode('cut');
await sleep(100);
const written = await request('POST', faultPath, {
text: 'Committed while Redis is unavailable',
});
const during = await request('GET', faultPath, undefined, second);
proxy.setMode('online');
for (let i = 0; i < 60; i++) {
if (apps.every((app) => app.get(CacheService).redis?.status === 'ready'))
break;
await sleep(100);
}
assert.ok(
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
);
const recovered = await request('GET', faultPath, undefined, second);
report.correctness.redis_reconnect = {
old_keys_retained:
(await observer.exists(...keysBefore)) === keysBefore.length,
read_after_write_during_outage: during.items.some(
(x) => x.id === written.id,
),
read_after_write_after_reconnect_other_replica: recovered.items.some(
(x) => x.id === written.id,
),
};
await clearCache();
const cold = await request('GET', faultPath, undefined, second);
report.correctness.empty_cache_recovery = cold.items.some(
(x) => x.id === written.id,
);
const raceChat = await newChat();
const racePath = `/api/v1/chats/${raceChat}/messages`;
const service = first.app.get(
require(`${process.cwd()}/dist/Chat/chat.service`).ChatService,
);
const actualPrisma = service.prisma;
let release, began;
const started = new Promise((resolve) => {
began = resolve;
});
const held = new Promise((resolve) => {
release = resolve;
});
service.prisma = actualPrisma.$extends({
query: {
message: {
async findMany({ args, query }) {
const rows = await query(args);
if (args.where?.chatId === raceChat) {
began();
await held;
}
return rows;
},
},
},
});
const pending = request('GET', racePath);
let fresh;
try {
await Promise.race([
started,
sleep(10000).then(() => {
throw new Error('race load did not start');
}),
]);
proxy.setMode('cut');
await sleep(100);
fresh = await request(
'POST',
racePath,
{ text: 'New message after old SELECT' },
second,
);
proxy.setMode('online');
for (let i = 0; i < 60; i++) {
if (
apps.every((app) => app.get(CacheService).redis?.status === 'ready')
)
break;
await sleep(100);
}
assert.ok(
apps.every((app) => app.get(CacheService).redis?.status === 'ready'),
);
} finally {
release();
service.prisma = actualPrisma;
}
await pending;
report.correctness.delayed_old_fill = (
await request('GET', racePath, undefined, second)
).items.some((x) => x.id === fresh.id);
if (stage === 'after') {
assert.equal(
report.correctness.redis_reconnect
.read_after_write_after_reconnect_other_replica,
true,
);
assert.equal(report.correctness.delayed_old_fill, true);
assert.equal(report.correctness.empty_cache_recovery, true);
}
}
report.completed = true;
} finally {
proxy.setMode('online');
try {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
if (userId) await prisma.user.delete({ where: { id: userId } });
await clearCache();
report.cleanup = 'only experiment chats/user/cache prefix removed';
} finally {
for (const app of apps) await app.close();
observer.disconnect();
await proxy.close();
await writeFile(
process.argv[3] ?? `/tmp/cache-epochs-${stage}.json`,
JSON.stringify(report, null, 2) + '\n',
);
}
}
console.log(
JSON.stringify({
stage,
correctness: report.correctness,
completed: report.completed,
}),
);
+25
View File
@@ -0,0 +1,25 @@
$ErrorActionPreference = 'Stop'
$dockerInfo = docker info --format '{{json .}}' | ConvertFrom-Json
if ($LASTEXITCODE -ne 0) { throw 'docker info failed' }
$processor = Get-CimInstance Win32_Processor | Select-Object -First 1
$computer = Get-CimInstance Win32_ComputerSystem
$containers = docker inspect mtusi2026lab2-fork-backend-1 mtusi2026lab2-fork-postgres-1 mtusi2026lab2-fork-redis-1 | ConvertFrom-Json
if ($LASTEXITCODE -ne 0) { throw 'docker inspect failed' }
$metadata = [ordered]@{
recorded_at = [DateTime]::UtcNow.ToString('o')
host_cpu = $processor.Name
host_physical_cores = $processor.NumberOfCores
host_logical_processors = $processor.NumberOfLogicalProcessors
host_ram_bytes = $computer.TotalPhysicalMemory
docker_version = $dockerInfo.ServerVersion
docker_cpu_count = $dockerInfo.NCPU
docker_ram_bytes = $dockerInfo.MemTotal
docker_kernel = $dockerInfo.KernelVersion
containers = @($containers | ForEach-Object {
[ordered]@{ name = $_.Name; image = $_.Image; memory_limit_bytes = $_.HostConfig.Memory; nano_cpus_limit = $_.HostConfig.NanoCpus; cpu_quota = $_.HostConfig.CpuQuota }
})
before_image = 'sha256:075170aed76a42aa98b435cfb918d83879259b7e17177275d5274e328198e119'
before_image_local_tag = 'lab2-cache-epochs:before'
}
$metadataPath = Join-Path (Get-Location).Path 'evidence/cache-epochs-environment.json'
[System.IO.File]::WriteAllText($metadataPath, ($metadata | ConvertTo-Json -Depth 6) + [Environment]::NewLine, [System.Text.UTF8Encoding]::new($false))
@@ -0,0 +1,91 @@
import assert from 'node:assert/strict';
import { readFile, writeFile } from 'node:fs/promises';
const reports = await Promise.all(
['after', 'optimized'].map(async (name) =>
JSON.parse(await readFile(`evidence/cache-epochs-${name}.json`, 'utf8')),
),
);
assert.ok(reports.every((report) => report.completed));
assert.deepEqual(reports[0].methodology, reports[1].methodology);
const median = (values) =>
[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
const labels = ['Отдельная транзакция', 'Один CTE'];
const lines = [
'# Оптимизация эпохи кэша в PostgreSQL',
'',
`Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`,
'',
'Сравнение сохранённого замера прежней реализации и нового замера после оптимизации. В обоих вариантах эпоха хранится в PostgreSQL; новый вариант объединяет проверку членства, UPDATE эпохи и INSERT сообщения одним запросом. При чтении истории убрано получение неиспользуемых метаданных чата.',
'',
'Медианы трёх повторов. Перцентили — медианы перцентилей повторов, без объединения выборок.',
'',
'| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | BEGIN/COMMIT на HTTP | CPU, % ядра | RSS, MiB | Ошибки |',
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
];
for (const workload of new Set(
reports[0].samples.map((sample) => sample.workload),
)) {
for (const [index, report] of reports.entries()) {
const samples = report.samples.filter(
(sample) => sample.workload === workload,
);
assert.equal(samples.length, 3);
const get = (select) => median(samples.map(select)).toFixed(3);
const controls = (sample) =>
Object.entries(sample.sql_calls)
.filter(([key]) => /^(BEGIN|COMMIT):/.test(key))
.reduce((total, [, count]) => total + count, 0) /
(sample.successes + sample.errors);
lines.push(
`| ${workload} | ${labels[index]} | ${get((sample) => sample.successful_rps)} | ${get((sample) => sample.latency_ms.p50)} | ${get((sample) => sample.latency_ms.p95)} | ${get((sample) => sample.latency_ms.p99)} | ${get((sample) => sample.sql_per_request)} | ${get(controls)} | ${get((sample) => sample.process_cpu_percent_one_core)} | ${get((sample) => sample.peak_process_rss_mib)} | ${samples.reduce((total, sample) => total + sample.errors, 0)} |`,
);
}
}
lines.push(
'',
'| Проверка | Отдельная транзакция | Один CTE |',
'| --- | --- | --- |',
);
for (const [label, select] of [
[
'Чтение после записи при отказе Redis',
(report) =>
report.correctness.redis_reconnect.read_after_write_during_outage,
],
[
'Чтение другой репликой после восстановления Redis',
(report) =>
report.correctness.redis_reconnect
.read_after_write_after_reconnect_other_replica,
],
[
'Старые ключи сохранились в Redis',
(report) => report.correctness.redis_reconnect.old_keys_retained,
],
[
'Восстановление с пустым кэшем',
(report) => report.correctness.empty_cache_recovery,
],
[
'Задержанная загрузка старой истории',
(report) => report.correctness.delayed_old_fill,
],
]) {
lines.push(
`| ${label} | ${reports.map((report) => (select(report) ? 'PASS' : 'FAIL')).join(' | ')} |`,
);
}
lines.push(
'',
'Методика: тот же `test/cache-benchmark.mjs`, 10 000 сообщений, limit=50, конкурентность 16, 3 с прогрева и 10 с чтения, 512 записей в новый чат, три повтора. Реальный HTTP по loopback в отдельных benchmark-контейнерах; общий PostgreSQL/Redis. Токены и пользователи создаются напрямую, хеширование паролей не измеряется. BEGIN/COMMIT исключены из SQL/HTTP; запрос WITH учитывается как один вызов pg.Client.query.',
'',
'CPU/RSS включают API, генератор HTTP и инструментацию. Docker-host разделяется с другими работающими сервисами. Старый и новый замеры сделаны в разное время; изменение фоновой нагрузки и состояния host ограничивает сравнение RPS/latency. Замеры не доказывают максимальную ёмкость или SLA. UPDATE общей строки эпохи по-прежнему сериализует записи одного чата.',
'',
'Сырые результаты: [до оптимизации](cache-epochs-after.json), [после оптимизации](cache-epochs-optimized.json). Хеши измеренных модулей сохранены в JSON. Исторические замеры до введения эпох остаются в `cache-epochs-summary.md`.',
);
await writeFile(
'evidence/cache-epochs-optimization-summary.md',
lines.join('\n') + '\n',
);
console.log(lines.join('\n'));
+200
View File
@@ -0,0 +1,200 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import test from 'node:test';
import { faultProxy } from './helpers/fault-proxy.mjs';
import { messageEpoch } from './helpers/message-epoch.mjs';
const require = createRequire(import.meta.url);
const Redis = require('ioredis').default;
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const { CacheService } = require('../dist/cache/cache.service');
const { ChatService } = require('../dist/Chat/chat.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
test(
'DB cache epochs: two replicas, retained Redis keys, concurrent writes and atomic rollback',
{ timeout: 90000 },
async (t) => {
assert.ok(process.env.CACHE_URL && process.env.DATABASE_URL);
const oldUrl = process.env.CACHE_URL;
const oldPrefix = process.env.CACHE_PREFIX;
const prefix = `test:epochs:${randomUUID()}`;
const proxy = await faultProxy(oldUrl, 6379);
const observer = new Redis(oldUrl);
process.env.CACHE_URL = proxy.url;
process.env.CACHE_PREFIX = prefix;
const apps = [];
let prisma, userId;
const chatIds = [];
t.after(async () => {
proxy.setMode('online');
try {
if (prisma) {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
if (userId) await prisma.user.delete({ where: { id: userId } });
}
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
} finally {
for (const app of apps) await app.close();
observer.disconnect();
await proxy.close();
process.env.CACHE_URL = oldUrl;
if (oldPrefix === undefined) delete process.env.CACHE_PREFIX;
else process.env.CACHE_PREFIX = oldPrefix;
}
});
for (let i = 0; i < 2; i++) {
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
apps.push(app);
exposeRequestIdHeader(app);
await app.init();
await app.getHttpAdapter().getInstance().ready();
require('nestjs-pino').PinoLogger.root.level = 'silent';
}
prisma = apps[0].get(PrismaService);
const user = await prisma.user.create({
data: {
username: `epoch_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Synthetic epochs',
},
});
userId = user.id;
const token = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 600000),
},
});
const call = async (replica, method, url, payload) => {
const response = await apps[replica]
.getHttpAdapter()
.getInstance()
.inject({
method,
url,
headers: {
authorization: `Bearer ${token}`,
...(method === 'POST' && /\/messages$/.test(url)
? { 'idempotency-key': randomUUID() }
: {}),
},
...(payload ? { payload } : {}),
});
return { status: response.statusCode, body: response.json() };
};
const createChat = async () => {
const response = await call(0, 'POST', '/api/v1/chats', {
title: 'Epoch test',
member_ids: [userId],
});
assert.equal(response.status, 201);
chatIds.push(response.body.id);
return response.body.id;
};
assert.equal((await call(1, 'GET', '/api/v1/chats')).body.items.length, 0);
proxy.setMode('cut');
await pause(100);
const chatId = await createChat();
proxy.setMode('online');
const reconnect = async () => {
for (let i = 0; i < 60; i++) {
if (apps.every((app) => app.get(CacheService).redis.status === 'ready'))
return;
await pause(100);
}
assert.fail('Redis did not reconnect');
};
await reconnect();
assert.deepEqual(
(await call(1, 'GET', '/api/v1/chats')).body.items.map((x) => x.id),
[chatId],
);
assert.equal(
(await prisma.user.findUniqueOrThrow({ where: { id: userId } }))
.chatsEpoch,
1n,
);
const path = `/api/v1/chats/${chatId}/messages`;
assert.equal((await call(1, 'GET', path)).body.items.length, 0);
const oldKeys = await observer.keys(`${prefix}:messages:${chatId}:*`);
assert.ok(oldKeys.length);
proxy.setMode('cut');
await pause(100);
const posted = await call(0, 'POST', path, {
text: 'Write during Redis outage',
});
assert.equal(posted.status, 201);
proxy.setMode('online');
await reconnect();
assert.equal(
await observer.exists(...oldKeys),
oldKeys.length,
'old cache keys must still exist',
);
assert.deepEqual(
(await call(1, 'GET', path)).body.items.map((x) => x.id),
[posted.body.id],
);
const service = apps[0].get(ChatService);
const original = service.prisma;
service.prisma = original.$extends({
query: {
$queryRaw({ args, query }) {
const index = args.values.indexOf('Must roll back');
if (index !== -1) args.values[index] = 'x'.repeat(4001);
return query(args);
},
},
});
try {
assert.equal(
(await call(0, 'POST', path, { text: 'Must roll back' })).status,
500,
);
} finally {
service.prisma = original;
}
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
assert.equal(await messageEpoch(prisma, chatId), 1n);
const writes = await Promise.all(
Array.from({ length: 20 }, (_, i) =>
call(i % 2, 'POST', path, { text: `Concurrent ${i}` }),
),
);
assert.ok(writes.every((x) => x.status === 201));
assert.equal(await prisma.message.count({ where: { chatId } }), 21);
assert.equal(await messageEpoch(prisma, chatId), 21n);
const history = (await call(1, 'GET', path)).body.items;
assert.equal(new Set(history.map((x) => x.id)).size, 21);
assert.ok(
writes.every((x) => history.some((message) => message.id === x.body.id)),
);
},
);
+73
View File
@@ -0,0 +1,73 @@
import assert from 'node:assert/strict';
import { readFile, writeFile } from 'node:fs/promises';
const before = JSON.parse(
await readFile('evidence/cache-epochs-before.json', 'utf8'),
);
const after = JSON.parse(
await readFile('evidence/cache-epochs-after.json', 'utf8'),
);
assert.ok(before.completed && after.completed);
assert.deepEqual(before.methodology, after.methodology);
const median = (values) =>
values.sort((a, b) => a - b)[Math.floor(values.length / 2)];
const lines = [
'# Эпохи кэша — сравнение 2026-10-13',
'',
'Медианы трёх повторов. p50/p95/p99 — медианы перцентилей отдельных повторов, не перцентили объединённой выборки.',
'',
'| Сценарий | Вариант | RPS | p50, мс | p95, мс | p99, мс | SQL/HTTP | CPU процесса, % ядра | Пиковый RSS, MiB | Ошибки |',
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
];
for (const workload of [...new Set(before.samples.map((x) => x.workload))]) {
for (const report of [before, after]) {
const samples = report.samples.filter((x) => x.workload === workload);
assert.equal(samples.length, 3);
const get = (fn) => median(samples.map(fn)).toFixed(3);
lines.push(
`| ${workload} | ${report.stage} | ${get((x) => x.successful_rps)} | ${get((x) => x.latency_ms.p50)} | ${get((x) => x.latency_ms.p95)} | ${get((x) => x.latency_ms.p99)} | ${get((x) => x.sql_per_request)} | ${get((x) => x.process_cpu_percent_one_core)} | ${get((x) => x.peak_process_rss_mib)} | ${samples.reduce((sum, x) => sum + x.errors, 0)} |`,
);
}
}
lines.push('', '| Проверка | До | После |', '| --- | --- | --- |');
for (const [label, select] of [
[
'Чтение после успешной записи при недоступном Redis',
(x) => x.correctness.redis_reconnect.read_after_write_during_outage,
],
[
'Чтение другой репликой после восстановления Redis со старыми ключами',
(x) =>
x.correctness.redis_reconnect
.read_after_write_after_reconnect_other_replica,
],
[
'Старые ключи действительно сохранились',
(x) => x.correctness.redis_reconnect.old_keys_retained,
],
['Восстановление с пустым кэшем', (x) => x.correctness.empty_cache_recovery],
[
'Задержанная загрузка старой истории не отравляет следующий GET',
(x) => x.correctness.delayed_old_fill,
],
])
lines.push(
`| ${label} | ${select(before) ? 'PASS' : 'FAIL'} | ${select(after) ? 'PASS' : 'FAIL'} |`,
);
lines.push(
'',
'Методика: `test/cache-benchmark.mjs`, сырой результат: `cache-epochs-before.json` и `cache-epochs-after.json`. Конфигурация: `cache-epochs-environment.json`.',
'',
'Реальный HTTP по loopback, 10 000 синтетических сообщений, первая страница limit=50. На повтор: холодная волна 32 запросов; прогрев 3 с; чтение с конкурентностью 16 в течение 10 с; прогрев 3 с и такой же контроль с отключённым кэшем истории; 512 записей в новый чат с конкурентностью 16. Три повтора. SQL считается на уровне pg.Client.query; BEGIN/COMMIT/ROLLBACK/SET вынесены в raw sql_calls и исключены из SQL/HTTP. Пароли не измеряются: пользователь и сессия создаются напрямую, токен остаётся в памяти.',
'',
'CPU/RSS относятся к процессу эксперимента, включающему API, генератор HTTP и инструментацию. Docker Desktop: 4 CPU, около 2.80 GiB RAM, отдельные лимиты сервисов не заданы. Замеры closed-loop, короткие и последовательные before → after; разброс и прогрев ограничивают выводы о latency/RPS. Это не тест максимальной ёмкости или SLA. Первый холодный повтор также включает прогрев runtime.',
'',
'Первичные короткие замеры (1 с прогрева, 5 с чтений, 128 записей) сохранены отдельно в cache-epochs-before-short.json и cache-epochs-after-short.json. Из-за большого разброса сделано одно повторное сравнение с увеличенным прогревом и нагрузкой; основные результаты получены в отдельных контейнерах benchmark из соответствующих образов. PostgreSQL/Redis и остальные контейнеры разделяют ресурсы Docker-host. Другие проекты на этом host не останавливались. Формулировки provenance в основных JSON уточнены после перехода от docker exec к отдельному контейнеру; численные измерения не менялись.',
'',
'До изменений использован фактически запущенный образ (локальный тег lab2-cache-epochs:before), после — пересобранный проект. Старый образ ещё использовал OR для курсорной пагинации; измеряемая первая страница не содержит курсора, поэтому этот путь в нагрузочном сравнении не выполняется. Хеши измеренных модулей записаны в JSON.',
'',
'Эпоха истории читается JOIN-ом вместе с проверкой членства; эпоха списка чатов — JOIN-ом с проверкой сессии. Для отдельного запроса дополнительного чтения эпохи нет. Однако raw JOIN проверки членства не использует прежний batching findUnique в Prisma: под конкуренцией суммарный SQL/HTTP может вырасти. Запись добавляет UPDATE чата и транзакционные команды; обновления одного чата сериализуются. Старые ключи недоступны новым читателям и удаляются по TTL.',
'',
'Корректность зависит от основного PostgreSQL и обновления эпох во всех путях изменения данных. Прямые SQL-изменения в обход приложения требуют увеличения соответствующей эпохи. После восстановления PostgreSQL из старого backup следует сменить CACHE_PREFIX или очистить кэш, чтобы исключить повторное использование эпох. Сохранность Docker-host и высокая доступность PostgreSQL не исследовались. Cache stampede не устраняется этим изменением.',
);
await writeFile('evidence/cache-epochs-summary.md', lines.join('\n') + '\n');
console.log(lines.slice(0, 14).join('\n'));
+117
View File
@@ -0,0 +1,117 @@
import assert from 'node:assert/strict';
import { readFile, writeFile } from 'node:fs/promises';
const resultPrefix = process.argv[2] ?? 'cache-strategy';
assert.match(resultPrefix, /^[a-z0-9-]+$/);
const reports = await Promise.all(
['full', 'no-redis-no-epochs'].map(async (name) =>
JSON.parse(await readFile(`evidence/${resultPrefix}-${name}.json`, 'utf8')),
),
);
const provenance = JSON.parse(
await readFile(`evidence/${resultPrefix}-experiment.json`, 'utf8'),
);
assert.ok(reports.every((report) => report.completed));
assert.deepEqual(reports[0].methodology, reports[1].methodology);
assert.equal(provenance.exit_code, 0);
assert.ok(provenance.sources_unchanged);
assert.ok(reports[1].correctness.epochs_unchanged);
assert.ok(reports[1].correctness.no_redis_client);
const median = (values) =>
[...values].sort((a, b) => a - b)[Math.floor(values.length / 2)];
const labels = ['Redis и эпохи', 'Без Redis и эпох'];
const aggregates = [];
const settings = reports[0].methodology;
const lines = [
'# Полная стратегия кэша и вариант без Redis и эпох',
'',
`Запуски UTC: ${reports.map((report) => report.recorded_at).join(' → ')}.`,
'',
'Сначала выполнен свежий прогон текущего образа с Redis, PostgreSQL-эпохами и записью сообщения через CTE. Затем шесть модулей приложения заменены временными версиями без поддержания эпох: убраны изменения счётчиков и чтения эпох в проверках сессии/членства. Объединённые проверки сессии, публичного профиля и доступа к истории сохранены. Сообщение вставляется одним INSERT ... SELECT с проверкой членства, без обновления счётчиков. Второй API запущен без CACHE_URL: все кэши приложения отключены. Исходники рабочего проекта и запущенный backend не изменены.',
'',
`Параметры обоих прогонов совпадают: ${settings.repetitions} повтора, конкурентность ${settings.concurrency}, ${settings.seed_messages} сообщений, limit=50, прогрев ${settings.warmup_seconds} с и чтение ${settings.duration_seconds} с, ${settings.writes_per_repeat} записей в новый чат. RPS = общее число успешных запросов / общее время повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее отдельных p95, не перцентиль объединённой выборки. SQL/HTTP исключает команды транзакций.`,
'',
'| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | Медиана RPS | Ошибки |',
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |',
];
for (const [workload, label] of [
['cold-history-burst', 'Холодная волна истории'],
['warm-history', 'Прогретая история'],
['history-cache-bypassed', 'Контроль без кэша истории'],
['writes-single-chat', 'Запись в один чат'],
]) {
if (
reports.every(
(report) =>
!report.samples.some((sample) => sample.workload === workload),
)
)
continue;
for (const [index, report] of reports.entries()) {
const samples = report.samples.filter(
(sample) => sample.workload === workload,
);
assert.equal(samples.length, settings.repetitions);
const sum = (select) =>
samples.reduce((total, sample) => total + select(sample), 0);
const successes = sum((sample) => sample.successes);
const errors = sum((sample) => sample.errors);
const aggregate = {
workload,
variant: labels[index],
successes,
errors,
mean_rps: successes / sum((sample) => sample.elapsed_seconds),
mean_latency_ms:
sum((sample) => sample.latency_ms.mean * sample.successes) / successes,
mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length,
sql_per_request:
sum((sample) => sample.sql_queries) / (successes + errors),
median_rps: median(samples.map((sample) => sample.successful_rps)),
repeat_rps: samples.map((sample) => sample.successful_rps),
};
aggregates.push(aggregate);
lines.push(
`| ${label} | ${labels[index]} | ${aggregate.mean_rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.median_rps.toFixed(3)} | ${errors} |`,
);
}
}
lines.push(
'',
`Учтено успешных HTTP-запросов: ${reports.map((report, index) => `${labels[index]} — ${report.samples.reduce((sum, sample) => sum + sample.successes, 0)}`).join('; ')}.`,
'',
...['warm-history', 'writes-single-chat'].map((workload) => {
const pair = aggregates.filter(
(aggregate) => aggregate.workload === workload,
);
return `${workload}: отношение среднего RPS полной стратегии к варианту без стратегии — ${(pair[0].mean_rps / pair[1].mean_rps).toFixed(3)}. RPS по повторам: ${pair.map((aggregate) => `${aggregate.variant}: ${aggregate.repeat_rps.join(' / ')}`).join('; ')}.`;
}),
'',
settings.history_control === false
? 'Дополнительный контроль с частично отключённым кэшем в этом прогоне пропущен. Сравниваются только полная стратегия и отсутствие Redis/поддержания эпох.'
: 'В контрольном сценарии у первого API отключён только кэш истории; кэш профиля продолжает работать. У второго API отключены все кэши. Поэтому этот контроль не измеряет отдельно чистую стоимость эпох.',
'',
`Проверки полной стратегии: чтение после записи при отказе Redis, чтение другой репликой после восстановления со старыми ключами, восстановление с пустым кэшем, задержанная загрузка старой истории. Проверки варианта без стратегии: приложение не создало Redis-клиент, эпохи синтетических пользователей/чатов остались равны 0 после всех записей, ${settings.writes_per_repeat} сообщений записаны в каждом новом чате, история читается после записи без повторяющихся ID.`,
'',
`Оба прогона используют один базовый образ; изменённые модули временной версии скомпилированы TypeScript без полной пересборки контейнера и подключены отдельными bind mounts. Хеши исходников и JS сохранены в [метаданных эксперимента](${resultPrefix}-experiment.json). Колонки эпох остаются в схеме; это исключает миграцию и позволяет сохранить рабочую БД. SQL-проверки профиля и членства во втором варианте не используют эпохи. Временные файлы удалены, хеши рабочих исходников проверены.`,
'',
'Ограничения: прогоны последовательные, Docker-host разделяется с другими сервисами; большой разброс ограничивает выводы о скорости. CPU/RSS процесса включают API, HTTP-генератор и инструментацию; CPU PostgreSQL отдельно не измерен. Синтетические сессии обходят KDF. Холодная волна означает пустой кэш приложения, а не холодный PostgreSQL. Отдельные цены Redis и эпох этим сравнением не разделяются: меняется вся стратегия. Данные рабочей БД и volumes сохранены, удалены только фикстуры эксперимента.',
'',
'Повторение:',
'',
'```powershell',
`docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/${resultPrefix}-full.json --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`,
`$env:EXPERIMENT_PREFIX='${resultPrefix}'`,
`node test/no-epochs-experiment.cjs --concurrency ${settings.concurrency} --duration-seconds ${settings.duration_seconds} --warmup-seconds ${settings.warmup_seconds} --repetitions ${settings.repetitions} --seed-messages ${settings.seed_messages} --cold-burst ${settings.cold_burst_requests} --writes-per-repeat ${settings.writes_per_repeat}${settings.history_control === false ? ' --skip-history-control' : ''}`,
`node test/cache-strategy-summary.mjs ${resultPrefix}`,
'```',
'',
`Сырые результаты: [полная стратегия](${resultPrefix}-full.json), [без Redis и эпох](${resultPrefix}-no-redis-no-epochs.json).`,
);
await writeFile(`evidence/${resultPrefix}-summary.md`, lines.join('\n') + '\n');
await writeFile(
`evidence/${resultPrefix}-averages.json`,
JSON.stringify(aggregates, null, 2) + '\n',
);
console.log(lines.slice(0, 18).join('\n'));
+292
View File
@@ -0,0 +1,292 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import test from 'node:test';
const require = createRequire(import.meta.url);
const Redis = require('ioredis').default;
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const { CacheService } = require('../dist/cache/cache.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
async function isolatedCache(t) {
assert.ok(process.env.CACHE_URL, 'Set CACHE_URL to a running Redis instance');
const previous = process.env.CACHE_PREFIX;
const prefix = `test:cache:${randomUUID()}`;
process.env.CACHE_PREFIX = prefix;
const observer = new Redis(process.env.CACHE_URL, {
maxRetriesPerRequest: 0,
connectTimeout: 1000,
commandTimeout: 1000,
});
observer.on('error', () => {});
await observer.ping();
t.after(async () => {
try {
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
} finally {
observer.disconnect();
if (previous === undefined) delete process.env.CACHE_PREFIX;
else process.env.CACHE_PREFIX = previous;
}
});
return { prefix, observer };
}
test('Redis cache: hits, expiry, invalidation race and unavailable-server fallback', async (t) => {
await isolatedCache(t);
const cache = new CacheService();
t.after(() => cache.onModuleDestroy());
await cache.onModuleInit();
let reads = 0;
const load = async () => ({ value: ++reads });
assert.deepEqual(await cache.remember('hits', 'page', 30, load), {
value: 1,
});
assert.deepEqual(await cache.remember('hits', 'page', 30, load), {
value: 1,
});
assert.equal(reads, 1);
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
value: 2,
});
await new Promise((resolve) => setTimeout(resolve, 1100));
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
value: 3,
});
await cache.invalidate('expiry');
assert.deepEqual(await cache.remember('expiry', 'page', 1, load), {
value: 4,
});
let resolveOld;
let began;
const started = new Promise((resolve) => {
began = resolve;
});
const pending = cache.remember('race', 'page', 30, () => {
began();
return new Promise((resolve) => {
resolveOld = resolve;
});
});
await started;
await cache.invalidate('race');
resolveOld({ value: 'old' });
await pending;
assert.deepEqual(
await cache.remember('race', 'page', 30, async () => ({ value: 'new' })),
{ value: 'new' },
);
assert.deepEqual(
await cache.remember('race', 'page', 30, async () => {
throw new Error('cache miss');
}),
{ value: 'new' },
);
await assert.rejects(
cache.remember('db-error', 'page', 30, async () => {
throw new Error('database failure');
}),
/database failure/,
);
const previous = process.env.CACHE_URL;
process.env.CACHE_URL = 'redis://127.0.0.1:1/0';
const offline = new CacheService();
process.env.CACHE_URL = previous;
t.after(() => offline.onModuleDestroy());
await offline.onModuleInit();
assert.deepEqual(
await offline.remember('offline', 'page', 30, async () => ({
value: 'database',
})),
{ value: 'database' },
);
await offline.invalidate('offline');
});
test('API cache: fewer SQL reads, write invalidation, permissions, logout and Redis fallback', async (t) => {
assert.ok(process.env.DATABASE_URL);
const { prefix, observer } = await isolatedCache(t);
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
exposeRequestIdHeader(app);
await app.init();
const server = app.getHttpAdapter().getInstance();
await server.ready();
const prisma = app.get(PrismaService);
const userIds = [];
const chatIds = [];
t.after(async () => {
try {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
} finally {
await app.close();
}
});
const actors = [];
for (let i = 0; i < 2; i++) {
const user = await prisma.user.create({
data: {
username: `cache_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Cache test',
},
});
userIds.push(user.id);
const token = randomBytes(32).toString('base64url');
const session = await prisma.session.create({
data: {
userId: user.id,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 600000),
},
});
actors.push({ id: user.id, token, sessionId: session.id });
}
const [alice, outsider] = actors;
const request = (method, url, payload, actor = alice) =>
server.inject({
method,
url,
headers: {
authorization: `Bearer ${actor.token}`,
...(method === 'POST' && /\/messages$/.test(url)
? { 'idempotency-key': randomUUID() }
: {}),
},
...(payload === undefined ? {} : { payload }),
});
const counts = new Map();
const tracked = prisma.$extends({
query: {
$allModels: {
$allOperations({ model, operation, args, query }) {
const key = `${model}:${operation}`;
counts.set(key, (counts.get(key) ?? 0) + 1);
return query(args);
},
},
},
});
for (const name of [
'User/user.service',
'Chat/chat.service',
'Auth/auth.service',
]) {
const exported = require(`../dist/${name}`);
const Service = Object.values(exported)[0];
app.get(Service).prisma = tracked;
}
const spy = (model, operation) => ({
mock: { callCount: () => counts.get(`${model}:${operation}`) ?? 0 },
});
const profileReads = spy('User', 'findUnique');
assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200);
assert.equal((await request('GET', '/api/v1/users/me')).statusCode, 200);
assert.equal(profileReads.mock.callCount(), 0);
const chatListReads = spy('Chat', 'findMany');
assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0);
assert.equal((await request('GET', '/api/v1/chats')).json().items.length, 0);
assert.equal(chatListReads.mock.callCount(), 1);
const created = await request('POST', '/api/v1/chats', {
title: 'Cached chat',
member_ids: [alice.id],
});
assert.equal(created.statusCode, 201);
const chatId = created.json().id;
chatIds.push(chatId);
assert.deepEqual(
(await request('GET', '/api/v1/chats')).json().items.map((x) => x.id),
[chatId],
);
assert.equal(chatListReads.mock.callCount(), 2);
const metadataReads = spy('Chat', 'findUnique');
const messageReads = spy('Message', 'findMany');
const path = `/api/v1/chats/${chatId}/messages`;
assert.equal((await request('GET', path)).json().items.length, 0);
assert.equal((await request('GET', path)).json().items.length, 0);
assert.equal(metadataReads.mock.callCount(), 0);
assert.equal(messageReads.mock.callCount(), 1);
const message = await request('POST', path, {
text: 'Invalidate cached empty history',
});
assert.equal(message.statusCode, 201);
assert.deepEqual(
(await request('GET', path)).json().items.map((x) => x.id),
[message.json().id],
);
assert.equal(messageReads.mock.callCount(), 2);
assert.equal(
(await request('GET', path, undefined, outsider)).statusCode,
404,
);
await prisma.chatMember.delete({
where: { chatId_userId: { chatId, userId: alice.id } },
});
assert.equal((await request('GET', path)).statusCode, 404);
await prisma.chatMember.create({
data: { chatId, userId: alice.id, position: 0 },
});
await prisma.session.update({
where: { id: outsider.sessionId },
data: { revokedAt: new Date() },
});
assert.equal(
(await request('GET', '/api/v1/users/me', undefined, outsider)).statusCode,
401,
);
const keys = [];
let cursor = '0';
do {
const [next, batch] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
keys.push(...batch);
} while (cursor !== '0');
assert.ok(keys.length >= 4, 'Redis must contain the cached responses');
for (const key of keys) {
const value = await observer.get(key);
assert.ok(
!value.includes('password_hash') && !value.includes('passwordHash'),
);
for (const actor of actors) assert.ok(!value.includes(actor.token));
}
app.get(CacheService).onModuleDestroy();
assert.equal((await request('GET', path)).statusCode, 200);
assert.equal(messageReads.mock.callCount(), 3);
assert.equal(
(await request('POST', path, { text: 'Write while Redis is unavailable' }))
.statusCode,
201,
);
assert.equal((await request('GET', path)).json().items.length, 2);
});
+257
View File
@@ -0,0 +1,257 @@
#!/usr/bin/env python3
"""Run the lab3 migration while a lab2 API uses an isolated temporary database."""
import base64
import json
from pathlib import Path
import secrets
import subprocess
import sys
import threading
import time
import urllib.error
from urllib.parse import quote
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
NETWORK = 'mtusi2026lab3-fork_default'
OLD_API = 'mtusi2026lab2-fork-backend:latest'
OLD_MIGRATE = 'mtusi2026lab2-fork-migrate:latest'
NEW_API = 'mtusi2026lab3-fork-api:latest'
NEW_MIGRATE = 'mtusi2026lab3-fork-migrate:latest'
def docker(*args, timeout=60):
result = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
text=True, timeout=timeout, check=False)
if result.returncode:
raise RuntimeError(f'docker {args[0]} failed with code {result.returncode}')
return result.stdout.strip()
def request(base, method, path, body=None, token=None, basic=None, key=None):
headers = {'Accept': 'application/json'}
if token:
headers['Authorization'] = 'Bearer ' + token
if basic:
headers['Authorization'] = 'Basic ' + base64.b64encode(
(basic[0] + ':' + basic[1]).encode()).decode()
if key:
headers['Idempotency-Key'] = key
data = None
if body is not None:
data = json.dumps(body).encode()
headers['Content-Type'] = 'application/json'
req = urllib.request.Request(base + path, data=data, headers=headers,
method=method)
try:
with urllib.request.urlopen(req, timeout=3) as response:
raw = response.read()
return response.status, json.loads(raw) if raw else None
except urllib.error.HTTPError as exc:
raise RuntimeError(f'{method} {path}: HTTP {exc.code}') from None
except (urllib.error.URLError, TimeoutError, OSError) as exc:
raise RuntimeError(f'{method} {path}: {type(exc).__name__}') from None
def wait_http(base):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
try:
if request(base, 'GET', '/health/ready')[0] == 200:
return
except RuntimeError:
pass
time.sleep(0.2)
raise RuntimeError('temporary API did not become ready')
def publish_port(name):
value = docker('port', name, '8080/tcp')
return 'http://127.0.0.1:' + value.rsplit(':', 1)[1]
def make_user(base, label):
username = 'compat_' + label + '_' + uuid.uuid4().hex[:8]
password = 'T3st-' + uuid.uuid4().hex
status, user = request(base, 'POST', '/api/v1/users',
{'username': username, 'display_name': label,
'password': password})
assert status == 201
status, session = request(base, 'POST', '/api/v1/auth/sessions',
basic=(username, password))
assert status == 201
return user, session['token']
def message(base, token, chat_id, text, key=None):
status, item = request(base, 'POST',
f'/api/v1/chats/{chat_id}/messages',
{'text': text}, token=token, key=key)
assert status == 201
return item
def history(base, token, chat_id):
items = []
cursor = None
for _ in range(100):
path = f'/api/v1/chats/{chat_id}/messages?limit=100'
if cursor:
path += '&cursor=' + quote(cursor, safe='')
status, page = request(base, 'GET', path, token=token)
assert status == 200
items.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return items
raise RuntimeError('history pagination exceeded safety bound')
def run():
suffix = uuid.uuid4().hex[:10]
names = {label: 'lab3-compat-' + suffix + '-' + label
for label in ('pg', 'redis', 'old', 'new')}
password = secrets.token_urlsafe(18)
db_url = f'postgresql://app:{password}@{names["pg"]}:5432/compat'
cache_url = f'redis://{names["redis"]}:6379/0'
created = []
try:
print('Starting isolated PostgreSQL and Redis...', flush=True)
docker('run', '--rm', '-d', '--name', names['pg'], '--network', NETWORK,
'--tmpfs', '/var/lib/postgresql/data:rw,mode=0700',
'-e', 'POSTGRES_DB=compat', '-e', 'POSTGRES_USER=app',
'-e', 'POSTGRES_PASSWORD=' + password,
'postgres:17.6-bookworm')
created.append(names['pg'])
docker('run', '--rm', '-d', '--name', names['redis'], '--network', NETWORK,
'redis:7.4.5-alpine', 'redis-server', '--save', '',
'--appendonly', 'no')
created.append(names['redis'])
deadline = time.monotonic() + 30
while True:
try:
docker('exec', names['pg'], 'pg_isready', '-U', 'app',
'-d', 'compat', timeout=5)
break
except RuntimeError:
if time.monotonic() >= deadline:
raise RuntimeError('temporary PostgreSQL did not start')
time.sleep(0.3)
docker('run', '--rm', '--network', NETWORK, '-e', 'DATABASE_URL=' + db_url,
OLD_MIGRATE, timeout=90)
print('Lab2 schema applied.', flush=True)
before_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
'compat', '-tAc',
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
assert before_table == 'f', 'new table exists before migration'
base_env = ['-e', 'DATABASE_URL=' + db_url,
'-e', 'CACHE_URL=' + cache_url,
'-e', 'CACHE_PREFIX=compat:' + suffix,
'-e', 'APP_HOST=0.0.0.0', '-e', 'APP_PORT=8080',
'-e', 'APP_ENV=lab', '-e', 'SESSION_STORE=postgres']
docker('run', '--rm', '-d', '--name', names['old'], '--network', NETWORK,
'-p', '127.0.0.1::8080', *base_env, OLD_API)
created.append(names['old'])
old_base = publish_port(names['old'])
wait_http(old_base)
print('Lab2 API ready; preparing synthetic chat.', flush=True)
alice, token = make_user(old_base, 'alice')
bob, _ = make_user(old_base, 'bob')
status, chat = request(old_base, 'POST', '/api/v1/chats',
{'title': 'Migration compatibility',
'member_ids': [alice['id'], bob['id']]},
token=token)
assert status == 201
pre = message(old_base, token, chat['id'], 'before migration')
stop = threading.Event()
events = []
def old_writer():
index = 0
while not stop.is_set():
at = time.monotonic()
try:
item = message(old_base, token, chat['id'],
f'during migration {index}')
events.append((at, True, item['id']))
except (RuntimeError, AssertionError):
events.append((at, False, None))
index += 1
stop.wait(0.3)
thread = threading.Thread(target=old_writer, daemon=True)
thread.start()
time.sleep(0.5)
started = time.monotonic()
try:
print('Applying lab3 migration while lab2 API writes...', flush=True)
docker('run', '--rm', '--network', NETWORK,
'-e', 'DATABASE_URL=' + db_url,
NEW_MIGRATE, timeout=90)
finally:
migrated = time.monotonic()
time.sleep(0.7)
stop.set()
thread.join(timeout=5)
after_table = docker('exec', names['pg'], 'psql', '-U', 'app', '-d',
'compat', '-tAc',
"SELECT to_regclass('public.message_idempotency') IS NOT NULL")
assert after_table == 't', 'new table was not created'
assert any(ok and at < started for at, ok, _ in events), 'no old writes before migration'
assert any(ok and at > migrated for at, ok, _ in events), 'no old writes after migration'
assert all(ok for _, ok, _ in events), 'old API write failed near migration'
after = message(old_base, token, chat['id'], 'after migration')
print('Lab2 API still reads and writes; starting lab3 API.', flush=True)
docker('run', '--rm', '-d', '--name', names['new'], '--network', NETWORK,
'-p', '127.0.0.1::8080', *base_env,
'-e', 'INSTANCE_ID=compat-new', NEW_API)
created.append(names['new'])
new_base = publish_port(names['new'])
wait_http(new_base)
print('Lab3 API ready; checking mixed-version reads and writes.', flush=True)
status, profile = request(new_base, 'GET', '/api/v1/users/me', token=token)
assert status == 200 and profile['id'] == alice['id'], 'session not shared'
key = uuid.uuid4().hex
new_message = message(new_base, token, chat['id'], 'from new version', key)
replay = message(new_base, token, chat['id'], 'from new version', key)
assert replay == new_message, 'new API idempotent replay changed response'
old_after_new = message(old_base, token, chat['id'], 'from old version')
for version, base in (('old', old_base), ('new', new_base)):
items = history(base, token, chat['id'])
ids = [item['id'] for item in items]
for label, item in (('pre', pre), ('after', after),
('new', new_message), ('old-after-new', old_after_new)):
assert ids.count(item['id']) == 1, f'{version} API cannot see {label} message'
result = {
'old_image': OLD_API, 'new_image': NEW_API,
'migration_image': NEW_MIGRATE,
'migration_seconds': round(migrated - started, 3),
'old_writes_near_migration': len(events),
'old_write_errors': sum(not ok for _, ok, _ in events),
'old_and_new_read_each_others_messages': True,
'existing_session_accepted_by_new_api': True,
'new_api_idempotent_replay': True,
'temporary_database': True,
}
print(json.dumps(result, indent=2))
print('PASS: additive migration compatible with running lab2 API')
finally:
print('Stopping isolated test containers...', flush=True)
for name in reversed(created):
try:
docker('stop', name, timeout=30)
except (RuntimeError, subprocess.TimeoutExpired):
print(f'WARNING: stop temporary container {name} manually',
file=sys.stderr)
if __name__ == '__main__':
try:
run()
except (RuntimeError, AssertionError, OSError, subprocess.TimeoutExpired) as exc:
print(f'FAIL: {exc}', file=sys.stderr)
raise SystemExit(1) from None
+170
View File
@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""Check readiness 200 -> 503 -> closed, liveness and LB withdrawal on SIGTERM."""
import argparse
import json
from pathlib import Path
import subprocess
import sys
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
PROBE = r'''
import json, sys, threading, time, urllib.error, urllib.request
url = sys.argv[1]
live_url = url.replace('/health/ready', '/health/live')
phase = 'before'
events = []
def commands():
global phase
for line in sys.stdin:
phase = line.strip()
if phase == 'end': break
def status(address):
try:
with urllib.request.urlopen(address, timeout=1) as response:
return str(response.status)
except urllib.error.HTTPError as error:
return str(error.code)
except (urllib.error.URLError, TimeoutError, OSError):
return 'connection_error'
threading.Thread(target=commands, daemon=True).start()
print('READY', flush=True)
while phase != 'end':
current = phase
ready = status(url)
live = status(live_url)
events.append({'phase': current, 'ready': ready, 'live': live})
time.sleep(0.05)
print(json.dumps(events), flush=True)
'''
def command(*args, timeout=45):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
timeout=timeout, check=False)
if result.returncode:
raise RuntimeError(f'{args[0]} {args[1]} failed (code {result.returncode})')
return result.stdout.strip()
def both_ready(base_url, expected):
seen = set()
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(base_url + '/health/ready', timeout=2) as response:
seen.add(response.headers['X-Instance-Id'])
except (urllib.error.URLError, TimeoutError, OSError):
pass
if seen == expected:
return
time.sleep(0.1)
raise RuntimeError(f'both replicas did not recover: {seen}')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
args = parser.parse_args()
ids = command('docker', 'compose', 'ps', '--all', '-q', 'api').splitlines()
if len(ids) != 2:
raise RuntimeError('shutdown probe requires two API replicas')
expected = {command('docker', 'inspect', '--format', '{{.Config.Hostname}}', item) for item in ids}
target = ids[0]
networks = json.loads(command('docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', target))
network, settings = next(iter(networks.items()))
address = settings['IPAddress']
nginx = command('docker', 'compose', 'ps', '-q', 'nginx')
both_ready(args.base_url, expected)
probe = subprocess.Popen(
['docker', 'run', '--rm', '-i', '--network', network,
'python:3.11-slim-bookworm', 'python', '-u', '-c', PROBE,
f'http://{address}:8080/health/ready'],
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True)
stop = None
def phase(value):
probe.stdin.write(value + '\n')
probe.stdin.flush()
try:
if probe.stdout.readline().strip() != 'READY':
raise RuntimeError('direct readiness probe did not start')
time.sleep(1)
phase('stopping')
started = time.monotonic()
stop = subprocess.Popen(['docker', 'stop', '--time', '30', target], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
withdrawn_at = None
withdrawn_while_running = False
while stop.poll() is None and time.monotonic() - started < 35:
upstreams = command('docker', 'exec', nginx, 'cat', '/etc/nginx/ready-upstreams.conf', timeout=10)
if f'server {address}:8080 ' not in upstreams:
withdrawn_at = time.monotonic() - started
withdrawn_while_running = command('docker', 'inspect', '--format', '{{.State.Running}}', target) == 'true'
if withdrawn_while_running:
phase('withdrawn')
break
time.sleep(0.1)
stop.communicate(timeout=40)
if stop.returncode:
raise RuntimeError('docker stop failed')
stop_seconds = time.monotonic() - started
exit_code = int(command('docker', 'inspect', '--format', '{{.State.ExitCode}}', target))
phase('stopped')
time.sleep(1)
phase('end')
output, _ = probe.communicate(timeout=15)
if probe.returncode:
raise RuntimeError('direct probe failed')
events = json.loads(output.strip())
summary = {name: {value: sum(e['phase'] == name and e['ready'] == value for e in events)
for value in sorted({e['ready'] for e in events if e['phase'] == name})}
for name in ('before', 'stopping', 'withdrawn', 'stopped')}
draining = [e for e in events if e['ready'] == '503']
first_503 = next((i for i, e in enumerate(events) if e['ready'] == '503'), None)
if summary['before'].get('200', 0) < 5:
raise RuntimeError('readiness was not healthy before SIGTERM')
if first_503 is None or any(e['ready'] == '200' for e in events[first_503:]):
raise RuntimeError('expected readiness 200 -> 503 with no late 200')
# These are separate requests; transport failures can straddle HTTP
# close or the client's deadline. Do not confuse them with unhealthy
# HTTP responses. Record them separately rather than hiding them.
if not draining or not any(e['live'] == '200' for e in draining):
raise RuntimeError('liveness must remain 200 while readiness is 503')
if any(e['live'] not in {'200', 'connection_error'} for e in draining):
raise RuntimeError('liveness returned an unhealthy HTTP status during draining')
if not withdrawn_while_running or not any(e['phase'] == 'withdrawn' and e['ready'] == '503' for e in events):
raise RuntimeError('LB did not withdraw the replica while its HTTP server was alive')
if summary['stopped'].get('200', 0) or summary['stopped'].get('503', 0) or not summary['stopped'].get('connection_error', 0):
raise RuntimeError('replica did not close HTTP after draining')
if exit_code == 137:
raise RuntimeError('container required SIGKILL instead of graceful shutdown')
print(json.dumps({'stop_seconds': round(stop_seconds, 3),
'lb_withdrawn_seconds': round(withdrawn_at, 3),
'lb_withdrawn_before_http_close': True,
'readiness_sequence': [200, 503, 'connection_error'],
'readiness_503_samples': len(draining),
'liveness_200_while_draining': sum(e['live'] == '200' for e in draining),
'liveness_connection_errors_while_draining': sum(e['live'] == 'connection_error' for e in draining),
'exit_code': exit_code, 'direct_ready_responses': summary}, indent=2))
print('PASS: readiness 200 -> 503 -> closed; liveness 200; LB withdrawal before HTTP close')
finally:
if probe.poll() is None:
probe.kill()
probe.communicate()
if stop is not None and stop.poll() is None:
stop.communicate(timeout=40)
command('docker', 'start', target)
both_ready(args.base_url, expected)
if __name__ == '__main__':
try:
main()
except (RuntimeError, OSError, subprocess.TimeoutExpired) as exc:
print(f'FAIL: {exc}', file=sys.stderr)
raise SystemExit(1) from None
+12
View File
@@ -0,0 +1,12 @@
services:
benchmark:
image: ${BENCHMARK_IMAGE:-mtusi2026lab2-fork-backend}
working_dir: /app
environment:
NODE_ENV: production
DATABASE_URL: postgresql://${POSTGRES_USER:-app}:${POSTGRES_PASSWORD:-local_dev_password}@postgres:5432/${POSTGRES_DB:-chat}
CACHE_URL: redis://redis:6379/0
volumes:
- ./test:/app/test:ro
- ./evidence:/results
command: ['node', 'test/cache-benchmark.mjs', '${BENCHMARK_STAGE:-after}', '/results/cache-epochs-${BENCHMARK_STAGE:-after}.json']
+5
View File
@@ -0,0 +1,5 @@
services:
api:
command: ['sh', '-c', 'INSTANCE_ID=$$(hostname) exec node --require /app/diagnostics/pool.cjs dist/main.js']
volumes:
- ./scripts/diagnostics/pool.cjs:/app/diagnostics/pool.cjs:ro
+35
View File
@@ -0,0 +1,35 @@
import importlib.util
from pathlib import Path
import unittest
import tempfile
spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).resolve().parents[1] / "scripts/run-scaling-diagnostics.py")
diagnostics = importlib.util.module_from_spec(spec)
spec.loader.exec_module(diagnostics)
class SummaryTest(unittest.TestCase):
def test_cumulative_counters_become_interval_rates(self):
summary = diagnostics.resource_summary([
{"time_ms": 1000, "cpu_usage_usec": 100, "cpu_nr_periods": 10,
"cpu_nr_throttled": 1, "cpu_throttled_usec": 100, "memory_bytes": 1024**2},
{"time_ms": 3000, "cpu_usage_usec": 1000100, "cpu_nr_periods": 30,
"cpu_nr_throttled": 6, "cpu_throttled_usec": 500100, "memory_bytes": 2*1024**2},
])
self.assertEqual(summary["mean_cpu_cores"], 0.5)
self.assertEqual(summary["throttled_period_fraction"], 0.25)
self.assertEqual(summary["throttled_seconds"], 0.5)
self.assertEqual(summary["peak_memory_mib"], 2)
def test_short_sample_does_not_invent_rates(self):
self.assertEqual(diagnostics.resource_summary([]), {"samples": 0})
def test_postgres_multiline_json_and_incomplete_tail(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "postgres.jsonl"
path.write_text('{"kind":"postgres",\n"activity":[\n{"state":"active"}\n]}\n{"kind":"cgroup"}\n{"unfinished":', encoding="utf-8")
self.assertEqual([r["kind"] for r in diagnostics.rows(path)], ["postgres", "cgroup"])
if __name__ == "__main__":
unittest.main()
+88
View File
@@ -0,0 +1,88 @@
import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
import { setImmediate, setTimeout } from 'node:timers/promises';
import test from 'node:test';
const require = createRequire(import.meta.url);
const { Module } = require('@nestjs/common');
const { NestFactory } = require('@nestjs/core');
const { AppController } = require('../dist/app.controller');
const { AppService } = require('../dist/app.service');
const { PrismaService } = require('../dist/prisma/prisma.service');
const { createFastifyAdapter } = require('../dist/common/http/fastify-adapter');
async function fixture(t, query = async () => []) {
const state = { queries: 0, destroyed: false };
const database = {
async $queryRaw() {
state.queries += 1;
return query();
},
onModuleDestroy() {
state.destroyed = true;
},
};
class FixtureModule {}
Module({
controllers: [AppController],
providers: [AppService, { provide: PrismaService, useValue: database }],
})(FixtureModule);
const adapter = createFastifyAdapter(30);
const app = await NestFactory.create(FixtureModule, adapter, { logger: false });
t.after(async () => {
state.release?.();
await app.close();
});
return { app, adapter, state, controller: app.get(AppController) };
}
test('draining keeps liveness and dependencies alive until active HTTP work completes', { timeout: 30000 }, async (t) => {
const { app, adapter, state, controller } = await fixture(t);
const entered = Promise.withResolvers();
const release = Promise.withResolvers();
// Always release the request, including when an assertion fails.
state.release = () => release.resolve();
adapter.getInstance().get('/slow-work', async () => {
entered.resolve();
await release.promise;
assert.equal(state.destroyed, false, 'database was destroyed during active work');
return { completed: true };
});
await app.listen(0, '127.0.0.1');
t.diagnostic('HTTP listener started');
const { port } = app.getHttpServer().address();
await controller.ready();
const response = fetch(`http://127.0.0.1:${port}/slow-work`, { signal: AbortSignal.timeout(5000) });
await entered.promise;
t.diagnostic('Slow request started');
let closed = false;
const closing = app.close('SIGTERM').then(() => { closed = true; });
await setImmediate();
assert.equal(adapter.isDraining, true);
assert.equal(adapter.beforeClose(), adapter.beforeClose(), 'shutdown must be idempotent');
await assert.rejects(controller.ready(), (error) => error.getStatus() === 503);
assert.equal(state.queries, 1, 'draining readiness must not acquire a DB connection');
assert.deepEqual(controller.live(), { status: 'ok' });
await setTimeout(80);
assert.equal(closed, false, 'server closed before the active request completed');
assert.equal(state.destroyed, false, 'dependencies closed before the active request');
release.resolve();
t.diagnostic('Slow request released');
assert.deepEqual(await (await response).json(), { completed: true });
await closing;
t.diagnostic('Application shutdown completed');
assert.equal(state.destroyed, true);
});
test('readiness already querying the DB cannot return 200 after draining begins', { timeout: 10000 }, async (t) => {
const query = Promise.withResolvers();
const { app, adapter, controller } = await fixture(t, () => query.promise);
await app.init();
const pending = assert.rejects(controller.ready(), (error) => error.getStatus() === 503);
const closing = app.close('SIGTERM');
await setImmediate();
assert.equal(adapter.isDraining, true);
query.resolve([]);
await pending;
await closing;
});
+475
View File
@@ -0,0 +1,475 @@
import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import Module from 'node:module';
import { readFile, writeFile } from 'node:fs/promises';
import { performance } from 'node:perf_hooks';
import { faultProxy } from './helpers/fault-proxy.mjs';
const require = createRequire('/app/package.json');
const manifest = JSON.parse(await readFile('/variants/manifest.json', 'utf8'));
const pg = require('pg');
let counts = {};
const originalQuery = pg.Client.prototype.query;
pg.Client.prototype.query = function (query, ...args) {
const sql = typeof query === 'string' ? query : query.text;
const verb = sql.trim().split(/\s+/)[0].toUpperCase();
counts[verb] = (counts[verb] ?? 0) + 1;
return originalQuery.call(this, query, ...args);
};
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('/app/dist/app.module');
const { PrismaService } = require('/app/dist/prisma/prisma.service');
const { CacheService } = require('/app/dist/cache/cache.service');
const { ChatService } = require('/app/dist/Chat/chat.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('/app/dist/common/http/fastify-adapter');
const exportedClasses = {
'Auth/auth.service.ts': 'AuthService',
'User/user.service.ts': 'UserService',
'User/user.guard.ts': 'UserGuard',
'Chat/chat.service.ts': 'ChatService',
'Chat/chat.controller.ts': 'ChatController',
};
const originals = {};
const variants = {};
for (const [relative, name] of Object.entries(exportedClasses)) {
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
originals[name] = require(filename)[name];
}
for (const variant of manifest.variants) {
variants[variant.name] = {};
for (const [relative, name] of Object.entries(exportedClasses)) {
const filename = `/app/dist/${relative.replace(/\.ts$/, '.js')}`;
const source = await readFile(
`/variants/${variant.name}/${relative.replace(/\.ts$/, '.js')}`,
'utf8',
);
const loaded = new Module(filename);
loaded.filename = filename;
loaded.paths = Module._nodeModulePaths('/app');
loaded._compile(source, filename);
variants[variant.name][name] = loaded.exports[name];
}
}
function activate(name) {
for (const [className, replacement] of Object.entries(variants[name])) {
const target = originals[className].prototype;
for (const method of Object.getOwnPropertyNames(replacement.prototype)) {
if (method === 'constructor') continue;
const descriptor = Object.getOwnPropertyDescriptor(
replacement.prototype,
method,
);
const previous = target[method];
if (
typeof descriptor.value === 'function' &&
typeof previous === 'function'
) {
for (const key of Reflect.getMetadataKeys(previous))
Reflect.defineMetadata(
key,
Reflect.getMetadata(key, previous),
descriptor.value,
);
}
Object.defineProperty(target, method, descriptor);
}
}
}
const Redis = require('ioredis').default;
const prefix = `experiment:epoch-variants:${randomUUID()}`;
const proxy = await faultProxy(process.env.CACHE_URL, 6379);
const observer = new Redis(process.env.CACHE_URL);
process.env.CACHE_URL = proxy.url;
const apps = {};
const chatIds = [];
const userIds = [];
let prisma, token;
const report = {
recorded_at: new Date().toISOString(),
methodology: {
concurrency: 32,
duration_seconds: 60,
warmup_seconds: 10,
repetitions: 3,
seed_messages: 100000,
writes_per_repeat: 4096,
order: [
['baseline', 'profile-join', 'access-join', 'striped'],
['striped', 'access-join', 'profile-join', 'baseline'],
['profile-join', 'striped', 'baseline', 'access-join'],
],
requests:
'Real loopback HTTP; shared read fixtures, separate write chats and cache prefixes',
resources:
'Four idle/active API instances and HTTP generator share one Node process; only one workload active at a time',
sql: 'pg.Client.query; transaction-control commands excluded; no changes to database pool size',
},
variants: manifest.variants,
samples: [],
correctness: {},
completed: false,
};
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function request(name, method, url, body, bearer = token) {
const response = await fetch(`${apps[name].url}${url}`, {
method,
headers: {
authorization: `Bearer ${bearer}`,
...(body ? { 'content-type': 'application/json' } : {}),
},
...(body ? { body: JSON.stringify(body) } : {}),
signal: AbortSignal.timeout(10000),
});
const json = await response.json();
return { status: response.status, json };
}
async function call(name, method, url, body) {
const result = await request(name, method, url, body);
assert.equal(result.status, method === 'POST' ? 201 : 200);
return result.json;
}
async function clearCache() {
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
}
async function newChat(userId) {
const chat = await prisma.chat.create({
data: {
title: 'Variant benchmark',
members: { create: { userId, position: 0 } },
},
});
chatIds.push(chat.id);
return chat.id;
}
async function epoch(name, chatId) {
if (name !== 'striped')
return (await prisma.chat.findUniqueOrThrow({ where: { id: chatId } }))
.messagesEpoch;
const [row] = await prisma.$queryRawUnsafe(
`SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM "${manifest.schema}".chat_message_epochs WHERE chat_id = $1::uuid`,
chatId,
);
return row.epoch;
}
async function measure(name, workload, repeat, operation, total) {
counts = {};
const latencies = [];
let errors = 0,
index = 0;
const cpu = process.cpuUsage();
const start = performance.now();
await Promise.all(
Array.from({ length: 32 }, async () => {
while (
total === undefined ? performance.now() - start < 60000 : index < total
) {
const item = index++;
const began = performance.now();
try {
await operation(item);
latencies.push(performance.now() - began);
} catch {
errors++;
}
}
}),
);
const seconds = (performance.now() - start) / 1000;
const used = process.cpuUsage(cpu);
latencies.sort((a, b) => a - b);
const calls = { ...counts };
const queries = Object.entries(calls)
.filter(([verb]) => !['BEGIN', 'COMMIT', 'ROLLBACK', 'SET'].includes(verb))
.reduce((sum, [, count]) => sum + count, 0);
const sample = {
variant: name,
workload,
repeat,
successes: latencies.length,
errors,
elapsed_seconds: +seconds.toFixed(3),
rps: +(latencies.length / seconds).toFixed(3),
mean_latency_ms: +(
latencies.reduce((sum, value) => sum + value, 0) / latencies.length
).toFixed(3),
p95_ms: +latencies[Math.ceil(latencies.length * 0.95) - 1]?.toFixed(3),
sql_queries: queries,
sql_per_request: +(queries / (latencies.length + errors)).toFixed(4),
sql_calls: calls,
cpu_percent_one_core: +(
(used.user + used.system) /
(seconds * 10000)
).toFixed(2),
rss_mib: +(process.memoryUsage().rss / 1048576).toFixed(2),
};
report.samples.push(sample);
console.log(JSON.stringify(sample));
assert.equal(errors, 0, 'HTTP workload errors');
}
async function reconnect() {
proxy.setMode('online');
for (let attempt = 0; attempt < 80; attempt++) {
if (
Object.values(apps).every(
({ app }) => app.get(CacheService).redis.status === 'ready',
)
)
return;
await pause(100);
}
assert.fail('Redis reconnect timeout');
}
async function correctness(name, userId, otherToken) {
activate(name);
await clearCache();
const replica = Object.keys(apps).find((candidate) => candidate !== name);
const chatId = await newChat(userId);
const url = `/api/v1/chats/${chatId}/messages`;
assert.equal(
(
await request(
name,
'GET',
url,
undefined,
randomBytes(32).toString('base64url'),
)
).status,
401,
);
assert.equal(
(await request(name, 'GET', url, undefined, otherToken)).status,
404,
);
assert.equal(
(await request(name, 'POST', url, { text: 'Forbidden' }, otherToken))
.status,
404,
);
assert.equal(
(await request(name, 'GET', '/api/v1/chats/invalid/messages')).status,
400,
);
assert.equal(
(
await request(
name,
'GET',
'/api/v1/chats/invalid/messages',
undefined,
randomBytes(32).toString('base64url'),
)
).status,
401,
);
assert.equal((await call(replica, 'GET', url)).items.length, 0);
const keys = await observer.keys(`${prefix}:${replica}:messages:${chatId}:*`);
assert.ok(keys.length);
proxy.setMode('cut');
await pause(100);
const posted = await call(name, 'POST', url, { text: 'During outage' });
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
await reconnect();
assert.equal(await observer.exists(...keys), keys.length);
assert.equal((await call(replica, 'GET', url)).items[0].id, posted.id);
const service = apps[name].app.get(ChatService);
const actual = service.prisma;
service.prisma = actual.$extends({
query: {
$queryRaw({ args, query }) {
const index = args.values.indexOf('Must roll back');
if (index !== -1) args.values[index] = 'x'.repeat(4001);
return query(args);
},
},
});
try {
assert.equal(
(await request(name, 'POST', url, { text: 'Must roll back' })).status,
500,
);
} finally {
service.prisma = actual;
}
assert.equal(await epoch(name, chatId), 1n);
assert.equal(await prisma.message.count({ where: { chatId } }), 1);
const raceChat = await newChat(userId);
const raceUrl = `/api/v1/chats/${raceChat}/messages`;
let began, release;
const started = new Promise((resolve) => {
began = resolve;
});
const held = new Promise((resolve) => {
release = resolve;
});
service.prisma = actual.$extends({
query: {
message: {
async findMany({ args, query }) {
const rows = await query(args);
if (args.where?.chatId === raceChat) {
began();
await held;
}
return rows;
},
},
},
});
const pending = call(name, 'GET', raceUrl);
try {
await Promise.race([
started,
pause(10000).then(() => {
throw new Error('Race load timeout');
}),
]);
const fresh = await call(replica, 'POST', raceUrl, {
text: 'After old SELECT',
});
release();
await pending;
assert.equal((await call(name, 'GET', raceUrl)).items[0].id, fresh.id);
} finally {
release();
service.prisma = actual;
}
await clearCache();
assert.equal((await call(name, 'GET', url)).items[0].id, posted.id);
report.correctness[name] = {
access_statuses: true,
redis_outage_and_retained_keys: true,
rollback: true,
delayed_old_fill: true,
empty_cache: true,
};
console.log(JSON.stringify({ correctness: name, result: 'PASS' }));
}
try {
for (const variant of manifest.variants) {
activate(variant.name);
process.env.CACHE_PREFIX = `${prefix}:${variant.name}`;
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
apps[variant.name] = { app };
exposeRequestIdHeader(app);
await app.listen(0, '127.0.0.1');
require('nestjs-pino').PinoLogger.root.level = 'silent';
apps[variant.name].url = await app.getUrl();
}
prisma = apps.baseline.app.get(PrismaService);
const user = await prisma.user.create({
data: {
username: `variant_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Variant user',
},
});
userIds.push(user.id);
token = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId: user.id,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 3600000),
},
});
const other = await prisma.user.create({
data: {
username: `other_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Other user',
},
});
userIds.push(other.id);
const otherToken = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId: other.id,
tokenHash: createHash('sha256').update(otherToken).digest('hex'),
expiresAt: new Date(Date.now() + 3600000),
},
});
for (const name of Object.keys(apps))
await correctness(name, user.id, otherToken);
const readChat = await newChat(user.id);
for (let offset = 0; offset < 100000; offset += 1000) {
await prisma.message.createMany({
data: Array.from({ length: 1000 }, (_, i) => ({
chatId: readChat,
senderId: user.id,
text: `Synthetic message ${offset + i}`,
createdAt: new Date(1700000000000 + offset + i),
})),
});
}
await prisma.chat.update({ where: { id: readChat }, data: { messagesEpoch: 100000n } });
await prisma.$executeRawUnsafe(`INSERT INTO "${manifest.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`, readChat);
for (const [index, order] of report.methodology.order.entries()) {
for (const name of order) {
activate(name);
await clearCache();
const url = `/api/v1/chats/${readChat}/messages?limit=50`;
const warmUntil = performance.now() + 10000;
while (performance.now() < warmUntil) await call(name, 'GET', url);
await measure(name, 'warm-history', index + 1, () =>
call(name, 'GET', url),
);
const writeChat = await newChat(user.id);
const writeUrl = `/api/v1/chats/${writeChat}/messages`;
assert.equal((await call(name, 'GET', writeUrl)).items.length, 0);
await measure(
name,
'writes-single-chat',
index + 1,
(i) => call(name, 'POST', writeUrl, { text: `Concurrent ${i}` }),
4096,
);
assert.equal(
await prisma.message.count({ where: { chatId: writeChat } }),
4096,
);
assert.equal(await epoch(name, writeChat), 4096n);
assert.equal((await call(name, 'GET', writeUrl)).items.length, 50);
}
}
report.completed = true;
} finally {
await reconnect();
if (prisma) {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
await prisma.user.deleteMany({ where: { id: { in: userIds } } });
}
await clearCache();
for (const { app } of Object.values(apps)) await app.close();
observer.disconnect();
await proxy.close();
await writeFile(
'/results/epoch-variants.json',
JSON.stringify(report, null, 2) + '\n',
);
}
console.log(
JSON.stringify({
completed: report.completed,
correctness: report.correctness,
}),
);
+261
View File
@@ -0,0 +1,261 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { randomUUID, createHash } = require('node:crypto');
const { spawn } = require('node:child_process');
const ts = require('typescript');
require('dotenv').config({ quiet: true });
const root = path.resolve(__dirname, '..');
const statePath = path.join(root, '.epoch-variants-state.json');
const files = [
'Auth/auth.service.ts',
'User/user.service.ts',
'User/user.guard.ts',
'Chat/chat.service.ts',
'Chat/chat.controller.ts',
];
const digest = (value) => createHash('sha256').update(value).digest('hex');
const command = process.argv[2];
async function main() {
let state;
if (command === 'init') {
assert.ok(!fs.existsSync(statePath));
assert.ok(
/messagesEpoch\s+BigInt/.test(
fs.readFileSync(path.join(root, 'prisma/schema.prisma'), 'utf8'),
),
'Historical comparison requires the pre-bucket schema. Original sources are saved in evidence/epoch-variants-sources.json',
);
state = {
directory: fs.mkdtempSync(path.join(os.tmpdir(), 'lab2-epoch-variants-')),
schema: 'epoch_variants_' + randomUUID().replaceAll('-', ''),
variants: [],
};
await database(async (client) => {
await client.query(`CREATE SCHEMA "${state.schema}"`);
await client.query(
`CREATE TABLE "${state.schema}".chat_message_epochs (chat_id uuid NOT NULL REFERENCES public.chats(id) ON DELETE CASCADE, bucket smallint NOT NULL CHECK (bucket BETWEEN 0 AND 15), epoch bigint NOT NULL DEFAULT 0, PRIMARY KEY (chat_id, bucket))`,
);
});
capture(state, 'baseline');
} else {
state = JSON.parse(fs.readFileSync(statePath, 'utf8'));
}
if (command === 'capture') capture(state, process.argv[3]);
if (command === 'capture-striped') capture(state, 'striped', true);
if (command === 'apply-striped') {
const sources = {};
for (const variant of state.variants) {
sources[variant.name] = Object.fromEntries(
files.map((relative) => [
relative,
fs.readFileSync(
path.join(state.directory, variant.name, relative),
'utf8',
),
]),
);
}
fs.writeFileSync(
path.join(root, 'evidence/epoch-variants-sources.json'),
JSON.stringify(sources, null, 2) + '\n',
);
for (const relative of ['Auth/auth.service.ts', 'Chat/chat.service.ts']) {
const source = sources.striped[relative].replaceAll(
`"${state.schema}".chat_message_epochs`,
'"public"."chat_message_epochs"',
);
fs.writeFileSync(path.join(root, 'src', relative), source);
}
console.log(
'Applied striped epochs to application sources; saved all source snapshots',
);
}
if (['init', 'capture', 'capture-striped'].includes(command)) {
fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
console.log(
'Captured:',
state.variants.map((variant) => variant.name).join(', '),
);
return;
}
if (command === 'run') {
assert.equal(state.variants.length, 4);
fs.writeFileSync(
path.join(state.directory, 'manifest.json'),
JSON.stringify(state, null, 2),
);
const override = path.join(state.directory, 'compose.json');
fs.writeFileSync(
override,
JSON.stringify({
services: {
benchmark: {
volumes: [
{
type: 'bind',
source: state.directory,
target: '/variants',
read_only: true,
},
],
},
},
}),
);
const args = [
'compose',
'-f',
'compose.yaml',
'-f',
'test/compose.benchmark.yaml',
'-f',
override,
'run',
'--rm',
'--no-deps',
'--pull',
'never',
'benchmark',
'node',
'test/epoch-variants-benchmark.mjs',
];
const code = await new Promise((resolve, reject) => {
const child = spawn('docker', args, { cwd: root, stdio: 'inherit' });
child.once('error', reject);
child.once('exit', resolve);
});
fs.writeFileSync(
path.join(root, 'evidence/epoch-variants-code.json'),
JSON.stringify(
{
variants: state.variants,
schema: state.schema,
exit_code: code,
implementation:
'TypeScript source snapshots; variant methods activated sequentially before HTTP workloads. Separate cache prefixes and API instances, shared fixtures and Node process.',
},
null,
2,
),
);
process.exitCode = code ?? 1;
}
if (command === 'seed-read-fixture') {
await database(async (client) => {
const result = await client.query(
`SELECT c.id FROM public.chats c JOIN public.messages m ON m.chat_id = c.id WHERE c.title = 'Variant benchmark' GROUP BY c.id HAVING COUNT(*) = 100000`,
);
assert.equal(result.rows.length, 1);
const chatId = result.rows[0].id;
await client.query(
'UPDATE public.chats SET messages_epoch = 100000 WHERE id = $1::uuid',
[chatId],
);
await client.query(
`INSERT INTO "${state.schema}".chat_message_epochs (chat_id, bucket, epoch) SELECT $1::uuid, bucket::smallint, 6250 FROM generate_series(0, 15) AS bucket ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = EXCLUDED.epoch`,
[chatId],
);
console.log('Seeded read fixture with 16 nonzero epoch buckets');
});
}
if (command === 'cleanup') {
await database((client) =>
client.query(`DROP SCHEMA "${state.schema}" CASCADE`),
);
const resolved = path.resolve(state.directory);
assert.equal(path.dirname(resolved), path.resolve(os.tmpdir()));
assert.ok(path.basename(resolved).startsWith('lab2-epoch-variants-'));
fs.rmSync(resolved, { recursive: true, force: true });
fs.unlinkSync(statePath);
console.log('Removed experiment schema and temporary source snapshots');
}
}
function capture(state, name, striped = false) {
assert.match(name, /^[a-z-]+$/);
assert.ok(!state.variants.some((variant) => variant.name === name));
const modules = {};
const table = `"${state.schema}".chat_message_epochs`;
for (const relative of files) {
let source = fs
.readFileSync(path.join(root, 'src', relative), 'utf8')
.replace(/\r\n/g, '\n');
if (
striped &&
['Auth/auth.service.ts', 'Chat/chat.service.ts'].includes(relative)
) {
source = source.replaceAll(
'c.messages_epoch',
`COALESCE((SELECT SUM(e.epoch) FROM ${table} AS e WHERE e.chat_id = c.id), 0)::bigint`,
);
if (relative === 'Chat/chat.service.ts') {
const start = source.indexOf(' const [message] =');
const end = source.indexOf(' if (!message)', start);
assert.ok(start !== -1 && end > start);
source =
source.slice(0, start) +
' const messageId = randomUUID();\n' +
' const bucket = parseInt(messageId.slice(0, 2), 16) % 16;\n' +
' const [message] = await this.prisma.$queryRaw<StoredMessage[]>(Prisma.sql`\n' +
` WITH updated AS (\n INSERT INTO ${table} AS e (chat_id, bucket, epoch)\n` +
' SELECT cm.chat_id, ${bucket}::smallint, 1 FROM chat_members AS cm\n' +
' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' +
' ON CONFLICT (chat_id, bucket) DO UPDATE SET epoch = e.epoch + 1\n' +
' RETURNING chat_id\n )\n' +
' INSERT INTO messages (id, chat_id, sender_id, text)\n' +
' SELECT ${messageId}::uuid, updated.chat_id, ${userId}::uuid, ${text} FROM updated\n' +
' RETURNING id, chat_id AS "chatId", sender_id AS "senderId", text, created_at AS "createdAt"\n' +
' `);\n' +
source.slice(end);
}
}
const compiled = ts.transpileModule(source, {
compilerOptions: {
target: ts.ScriptTarget.ES2023,
module: ts.ModuleKind.CommonJS,
experimentalDecorators: true,
emitDecoratorMetadata: true,
},
reportDiagnostics: true,
});
assert.ok(
!compiled.diagnostics.some(
(diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error,
),
);
const destination = path.join(
state.directory,
name,
relative.replace(/\.ts$/, '.js'),
);
fs.mkdirSync(path.dirname(destination), { recursive: true });
fs.writeFileSync(destination, compiled.outputText);
fs.writeFileSync(destination.replace(/\.js$/, '.ts'), source);
modules[relative] = {
source_sha256: digest(source),
js_sha256: digest(compiled.outputText),
};
}
state.variants.push({ name, modules, striped });
}
async function database(operation) {
const client = new (require('pg').Client)({
connectionString: process.env.DATABASE_URL,
});
await client.connect();
try {
return await operation(client);
} finally {
await client.end();
}
}
main().catch((error) => {
console.error(error.message);
process.exitCode = 1;
});
+72
View File
@@ -0,0 +1,72 @@
import assert from 'node:assert/strict';
import { readFile, writeFile } from 'node:fs/promises';
const report = JSON.parse(
await readFile('evidence/epoch-variants.json', 'utf8'),
);
assert.ok(report.completed);
const labels = {
baseline: 'Текущая схема',
'profile-join': 'Профиль из JOIN',
'access-join': 'Профиль + единая проверка доступа',
striped: 'Единая проверка + 16 счётчиков',
};
const aggregates = [];
const lines = [
'# Сравнение трёх оптимизаций эпох и текущей схемы',
'',
`Дата UTC: ${report.recorded_at}.`,
'',
'Все режимы используют Redis. Варианты последовательно добавляют оптимизации: профиль из запроса сессии; объединённую проверку сессии, членства и эпохи; 16 счётчиков истории, увеличиваемых независимо, с суммой как версией кэша. Порядок вариантов меняется между тремя повторами. Чтения: 100 000 сообщений, limit=50, конкурентность 32, прогрев 10 с, измерение 60 с. Запись: 4096 сообщений в отдельный новый чат на вариант/повтор. Только один workload активен одновременно.',
'',
'RPS рассчитан по общему числу успешных запросов и времени повторов. Средняя задержка взвешена по числу запросов. p95 — среднее p95 отдельных повторов, а не перцентиль объединённой выборки. SQL/HTTP исключает BEGIN/COMMIT/ROLLBACK/SET.',
'',
'| Сценарий | Вариант | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | RPS по повторам | Ошибки |',
'| --- | --- | ---: | ---: | ---: | ---: | --- | ---: |',
];
for (const workload of ['warm-history', 'writes-single-chat']) {
for (const [name, label] of Object.entries(labels)) {
const samples = report.samples.filter(
(sample) => sample.variant === name && sample.workload === workload,
);
assert.equal(samples.length, 3);
const sum = (select) =>
samples.reduce((total, sample) => total + select(sample), 0);
const successes = sum((sample) => sample.successes);
const errors = sum((sample) => sample.errors);
const aggregate = {
variant: name,
workload,
successes,
errors,
rps: successes / sum((sample) => sample.elapsed_seconds),
mean_latency_ms:
sum((sample) => sample.mean_latency_ms * sample.successes) / successes,
mean_p95_ms: sum((sample) => sample.p95_ms) / samples.length,
sql_per_request:
sum((sample) => sample.sql_queries) / (successes + errors),
repeat_rps: samples.map((sample) => sample.rps),
};
aggregates.push(aggregate);
lines.push(
`| ${workload} | ${label} | ${aggregate.rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.repeat_rps.map((value) => value.toFixed(1)).join(' / ')} | ${errors} |`,
);
assert.ok(Object.values(report.correctness[name]).every(Boolean));
}
}
lines.push(
'',
'Все варианты прошли проверки 401/404/400, доступа постороннего пользователя, чтения после записи при недоступном Redis и после восстановления со старыми ключами, отката счётчика при ошибке INSERT, задержанного заполнения старого кэша и работы с пустым кэшем. После каждого нагрузочного блока проверены число сообщений и значение эпохи 4096.',
'',
'Четыре API и HTTP-генератор работают в одном Node-процессе, каждый API имеет отдельный Redis-prefix и пул Prisma прежнего размера. Методы переключаются между скомпилированными снимками исходников перед каждым workload; скрипт сохраняет метаданные маршрутов. Idle-экземпляры присутствуют во всех режимах. CPU/RSS включает генератор и все экземпляры. Другие сервисы Docker-host продолжают работать. Сессии создаются напрямую, KDF не измеряется. Порядок меняется, но три повтора и фоновые нагрузки всё ещё ограничивают выводы об устойчивых коэффициентах ускорения.',
'',
'Распределённые счётчики находятся в отдельной временной схеме. Рабочая схема приложения не переключается на них. В нагрузочном чате 16 ненулевых счётчиков; стоимость их суммирования входит в запрос доступа. Первое чтение baseline выполнено до заполнения счётчиков временной схемы; данные истории неизменны, счётчики подготовлены до первого измерения striped. Это изменение подготовки фикстуры добавлено в скрипт для следующих запусков.',
'',
'Сырые результаты: [epoch-variants.json](epoch-variants.json). Хеши исходников вариантов: [epoch-variants-code.json](epoch-variants-code.json).',
);
await writeFile('evidence/epoch-variants-summary.md', lines.join('\n') + '\n');
await writeFile(
'evidence/epoch-variants-averages.json',
JSON.stringify(aggregates, null, 2) + '\n',
);
console.log(lines.slice(0, 19).join('\n'));
+195
View File
@@ -0,0 +1,195 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { createHash, randomUUID } from 'node:crypto';
import { mkdir, writeFile } from 'node:fs/promises';
import { createRequire } from 'node:module';
import { resolve } from 'node:path';
import { spawnSync } from 'node:child_process';
const require = createRequire(import.meta.url);
const { Client } = require('pg');
assert.ok(process.env.DATABASE_URL, 'Set DATABASE_URL for the project PostgreSQL');
const output = resolve('evidence');
const schema = `explain_history_${randomUUID().replaceAll('-', '')}`;
const table = `"${schema}"."messages"`;
const client = new Client({
connectionString: process.env.DATABASE_URL,
connectionTimeoutMillis: 5000,
query_timeout: 60000,
options: '-c statement_timeout=60000 -c lock_timeout=2000',
});
const seed = `
WITH base AS (
SELECT n, CASE WHEN (n - 1) % 6 < 2 THEN 0 ELSE (n - 1) % 6 - 1 END AS chat_no
FROM generate_series(1, 60000) AS n
), ranked AS (
SELECT n, chat_no, row_number() OVER (PARTITION BY chat_no ORDER BY n) AS chat_seq
FROM base
)
INSERT INTO ${table} (id, chat_id, sender_id, text, created_at)
SELECT md5('message:' || n)::uuid, md5('chat:' || chat_no)::uuid,
md5('sender:' || (n % 20))::uuid,
rpad('Synthetic message ' || n || ' ', 256, '.'),
'2026-01-01T00:00:00Z'::timestamptz + ((chat_seq - 1) / 10) * interval '1 second'
FROM ranked ORDER BY n`;
const columns = 'id, chat_id, sender_id, text, created_at';
const firstSql = `SELECT ${columns} FROM ${table}
WHERE chat_id = $1::uuid ORDER BY created_at ASC, id ASC LIMIT $2`;
const orSql = `SELECT ${columns} FROM ${table}
WHERE chat_id = $1::uuid
AND (created_at > $2::timestamptz OR (created_at = $2::timestamptz AND id > $3::uuid))
ORDER BY created_at ASC, id ASC LIMIT $4`;
const tupleSql = `SELECT ${columns} FROM ${table}
WHERE chat_id = $1::uuid AND (created_at, id) > ($2::timestamptz, $3::uuid)
ORDER BY created_at ASC, id ASC LIMIT $4`;
function nodes(plan) {
return [plan, ...(plan.Plans ?? []).flatMap(nodes)];
}
function median(values) {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.floor(sorted.length / 2)];
}
function inspectDocker() {
const template = '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"server_version":{{json .ServerVersion}},"os":{{json .OperatingSystem}},"arch":{{json .Architecture}}}';
const result = spawnSync('docker', ['info', '--format', template], {
encoding: 'utf8', timeout: 10000, windowsHide: true,
});
try {
if (result.status !== 0) return null;
const info = JSON.parse(result.stdout);
const ids = spawnSync('docker', ['compose', 'ps', '--quiet', 'postgres'], {
encoding: 'utf8', timeout: 10000, windowsHide: true,
});
if (ids.status === 0 && ids.stdout.trim()) {
const limitsTemplate = '{"image":{{json .Config.Image}},"memory_limit_bytes":{{.HostConfig.Memory}},"nano_cpus":{{.HostConfig.NanoCpus}},"cpu_quota":{{.HostConfig.CpuQuota}},"cpu_period":{{.HostConfig.CpuPeriod}},"cpuset_cpus":{{json .HostConfig.CpusetCpus}}}';
const limits = spawnSync('docker', ['inspect', '--format', limitsTemplate, ids.stdout.trim()], {
encoding: 'utf8', timeout: 10000, windowsHide: true,
});
if (limits.status === 0) info.postgres_container = JSON.parse(limits.stdout);
}
return info;
} catch {
return null;
}
}
await client.connect();
const runs = [];
const raw = [];
const signatures = new Map();
let metadata;
let queries;
let cleanup = false;
let sourceIndexes;
try {
sourceIndexes = (await client.query("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' AND tablename = 'messages' ORDER BY indexname")).rows;
await client.query('BEGIN');
await client.query(`CREATE SCHEMA "${schema}"`);
await client.query(`CREATE TABLE ${table} (LIKE public.messages INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING STORAGE)`);
await client.query(seed);
await client.query(`ANALYZE ${table}`);
const distribution = (await client.query(`SELECT chat_id, count(*)::integer AS messages FROM ${table} GROUP BY chat_id ORDER BY messages DESC, chat_id`)).rows;
assert.equal(distribution.reduce((sum, row) => sum + row.messages, 0), 60000);
const chatId = distribution.find((row) => row.messages === 20000).chat_id;
const cursor = async (offset) => (await client.query(
`SELECT id, created_at FROM ${table} WHERE chat_id = $1::uuid ORDER BY created_at, id OFFSET $2 LIMIT 1`,
[chatId, offset],
)).rows[0];
const middle = await cursor(9999);
const late = await cursor(17999);
queries = [
{ name: 'first_page', sql: firstSql, parameters: [chatId, 51] },
{ name: 'middle_page_or', sql: orSql, parameters: [chatId, middle.created_at.toISOString(), middle.id, 51] },
{ name: 'late_page_or', sql: orSql, parameters: [chatId, late.created_at.toISOString(), late.id, 51] },
{ name: 'late_page_tuple', sql: tupleSql, parameters: [chatId, late.created_at.toISOString(), late.id, 51] },
];
const settings = (await client.query(
'SELECT name, setting, unit FROM pg_settings WHERE name = ANY($1::text[]) ORDER BY name',
[['server_version', 'shared_buffers', 'work_mem', 'random_page_cost', 'effective_cache_size', 'max_parallel_workers_per_gather', 'jit', 'block_size']],
)).rows;
metadata = {
recorded_at: new Date().toISOString(),
command: 'node test/explain-history.mjs',
node_version: process.version,
docker: inspectDocker(),
settings,
source_table_indexes_before: sourceIndexes,
dataset: { total_messages: 60000, target_chat_messages: 20000, chats: 5, distribution, text_length: 256, timestamp_ties: 10, insertion_order: 'interleaved chats' },
method: { warmup_runs_per_query_per_phase: 1, measured_runs: 5, page_size: 50, fetched_rows: 51, cursor_positions: [10000, 18000], cache: 'warm PostgreSQL buffers; Redis is bypassed', cleanup: 'ROLLBACK the experiment schema; live tables and indexes unchanged', planner: 'default scan settings; no forced index or disabled sequential scans', schema_difference: 'copied columns/checks/defaults; foreign keys omitted equally in all phases', tuple_variant: 'current cursor SQL in ChatService.listMessages; OR is the pre-fix baseline' },
};
const phases = [
{ name: 'no_indexes', ddl: [] },
{ name: 'without_history_index', ddl: [`CREATE UNIQUE INDEX messages_pkey ON ${table} (id)`, `CREATE INDEX messages_sender_id_idx ON ${table} (sender_id)`] },
{ name: 'with_history_index', ddl: [`CREATE INDEX messages_chat_id_created_at_id_idx ON ${table} (chat_id, created_at, id)`] },
];
for (const phase of phases) {
for (const ddl of phase.ddl) await client.query(ddl);
await client.query(`ANALYZE ${table}`);
const indexes = (await client.query('SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = $1 ORDER BY indexname', [schema])).rows;
const phaseRuns = new Map(queries.map((query) => [query.name, []]));
for (const query of queries) {
const rows = (await client.query(query.sql, query.parameters)).rows;
assert.equal(rows.length, 51);
const signature = createHash('sha256').update(JSON.stringify(rows)).digest('hex');
if (signatures.has(query.name)) assert.equal(signature, signatures.get(query.name));
else signatures.set(query.name, signature);
if (query.name === 'late_page_tuple') assert.equal(signature, signatures.get('late_page_or'));
await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.parameters);
}
for (let repetition = 1; repetition <= 5; repetition++) {
for (const query of queries) {
const plan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${query.sql}`, query.parameters)).rows[0]['QUERY PLAN'][0];
phaseRuns.get(query.name).push(plan);
}
}
for (const query of queries) {
const plans = phaseRuns.get(query.name);
const medianMs = median(plans.map((plan) => plan['Execution Time']));
const representative = plans.find((plan) => plan['Execution Time'] === medianMs);
const flat = nodes(representative.Plan);
const scan = flat.find((node) => node['Node Type'].includes('Scan'));
runs.push({
phase: phase.name, query: query.name, indexes, sql: query.sql, parameters: query.parameters,
result_sha256: signatures.get(query.name), execution_ms: plans.map((plan) => plan['Execution Time']),
median_execution_ms: medianMs, representative_plan: representative,
scan_type: scan?.['Node Type'], index_name: scan?.['Index Name'] ?? null,
rows_removed_by_filter: scan?.['Rows Removed by Filter'] ?? 0,
scanned_rows: ((scan?.['Actual Rows'] ?? 0) + (scan?.['Rows Removed by Filter'] ?? 0)) * (scan?.['Actual Loops'] ?? 1),
sort: flat.some((node) => node['Node Type'] === 'Sort'),
shared_buffer_hits: representative.Plan['Shared Hit Blocks'] ?? 0,
shared_buffer_reads: representative.Plan['Shared Read Blocks'] ?? 0,
raw_plans: plans,
});
const textPlan = (await client.query(`EXPLAIN (ANALYZE, BUFFERS) ${query.sql}`, query.parameters)).rows.map((row) => row['QUERY PLAN']).join('\n');
raw.push(`=== ${phase.name} / ${query.name} ===\nIndexes: ${indexes.map((index) => index.indexname).join(', ') || '(none)'}\nSQL:\n${query.sql}\nParameters: ${JSON.stringify(query.parameters)}\n\n${textPlan}\n`);
console.log(`${phase.name} / ${query.name}: median ${medianMs.toFixed(3)} ms; ${scan?.['Node Type']}; scanned ${runs.at(-1).scanned_rows}; sort=${runs.at(-1).sort}`);
}
}
} finally {
try {
await client.query('ROLLBACK');
if (metadata) {
metadata.source_table_indexes_after = (await client.query("SELECT indexname, indexdef FROM pg_indexes WHERE schemaname = 'public' AND tablename = 'messages' ORDER BY indexname")).rows;
assert.deepEqual(metadata.source_table_indexes_after, sourceIndexes);
const leftovers = (await client.query('SELECT count(*)::integer AS count FROM pg_namespace WHERE nspname = $1', [schema])).rows[0].count;
assert.equal(leftovers, 0);
metadata.experiment_schema_absent = true;
}
cleanup = true;
} finally {
await client.end();
}
}
assert.equal(cleanup, true);
await mkdir(output, { recursive: true });
await writeFile(resolve(output, 'history-index-explain.json'), JSON.stringify({ ...metadata, cleanup_completed: cleanup, runs }, null, 2) + '\n');
await writeFile(resolve(output, 'history-index-plans.txt'), raw.join('\n'));
const labels = { no_indexes: 'Без индексов', without_history_index: 'PK + sender_id, без индекса истории', with_history_index: 'С индексом истории' };
const tableRows = runs.map((row) => `| ${labels[row.phase]} | ${row.query} | ${row.median_execution_ms.toFixed(3)} | ${row.scan_type} | ${row.scanned_rows} | ${row.sort ? 'да' : 'нет'} | ${row.shared_buffer_hits} / ${row.shared_buffer_reads} |`).join('\n');
const speedup = (query) => runs.find((row) => row.query === query && row.phase === 'no_indexes').median_execution_ms / runs.find((row) => row.query === query && row.phase === 'with_history_index').median_execution_ms;
const summary = `# EXPLAIN истории сообщений: с индексами и без\n\nДата UTC: ${metadata.recorded_at}. PostgreSQL ${metadata.settings.find((setting) => setting.name === 'server_version').setting}.\n\nКоманда: \`node test/explain-history.mjs\` (нужен \`DATABASE_URL\`; PostgreSQL проекта с применёнными миграциями).\n\n60 000 синтетических сообщений в пяти чатах; в целевом чате 20 000. Чаты перемежаются при вставке, текст 256 символов, по 10 сообщений имеют одинаковый created_at. Страница API: 50 сообщений + 1 для next_cursor. Курсоры после 10 000-го и 18 000-го сообщений. Запросы первой страницы и с OR соответствуют прежнему \`ChatService.listMessages\`; tuple — исправленное условие курсора, используемое в текущем API. Скрипт сравнивает прежний и исправленный запросы; сам код API не меняет.\n\n| Индексы | Запрос | Медиана, мс | Scan | Проверено строк | Sort | Buffers hit/read |\n| --- | --- | ---: | --- | ---: | --- | ---: |\n${tableRows}\n\nПервая страница с индексом быстрее варианта без индексов в ${speedup('first_page').toFixed(1)} раза на этом наборе и стенде. Само наличие PK и индекса sender_id не заменяет индекс истории. Составной B-tree \`(chat_id, created_at, id)\` соответствует равенству chat_id и порядку истории; наличие Sort и границы Index Cond видны в сохранённых планах. ORDER BY нужен и при наличии индекса.\n\nПоздняя страница с OR: ${runs.find((row) => row.phase === 'with_history_index' && row.query === 'late_page_or').scanned_rows} проверенных строк. Эквивалентное сравнение пары: ${runs.find((row) => row.phase === 'with_history_index' && row.query === 'late_page_tuple').scanned_rows} строк. Это показывает, удалось ли передать границу курсора в Index Cond или она осталась Filter. Совпадение результатов всех фаз и OR/tuple проверено SHA-256 от полных 51 строк.\n\nМетодика: отдельная таблица с колонками текущей public.messages в случайной схеме; одинаковые данные во всех фазах; ANALYZE после загрузки и изменений индексов; 1 прогрев + 5 измерений каждого запроса в каждой фазе, медиана PostgreSQL Execution Time. Планы не принуждались настройками enable_seqscan/enable_indexscan. Фаза с индексом сохраняет PK и sender_id. Все создание таблиц, данных и индексов выполнены внутри транзакции; ROLLBACK удалил экспериментальную схему. Рабочие таблицы и индексы не изменялись.\n\nОграничения: это прогретый SQL-тест без HTTP, Redis, аутентификации и конкуренции; не cold-cache benchmark, не измерение RPS и не доказательство latency всего API. FK не копировались, поскольку сравнивается чтение; это одинаково во всех фазах. Измерения зависят от ресурсов и нагрузки Docker-host. Лимиты и настройки PostgreSQL записаны в JSON; текстовые планы сняты отдельным запуском, поэтому их времена могут отличаться от медиан JSON.\n\nФайлы: [полные текстовые планы](history-index-plans.txt), [JSON, пять повторов, настройки и результаты](history-index-explain.json).\n`;
await writeFile(resolve(output, 'history-index-summary.md'), summary);
console.log(`Saved EXPLAIN evidence to ${output}; experiment schema rolled back.`);
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Check failover of a named Compose API replica without changing public tests."""
import argparse
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--target-index", type=int, choices=(1, 2), required=True)
args = parser.parse_args()
ids = subprocess.run(
["docker", "compose", "ps", "--all", "-q", "api"],
cwd=ROOT, capture_output=True, text=True, check=True,
).stdout.splitlines()
check(len(ids) == 2, "targeted failover requires two API replicas")
target = subprocess.run(
["docker", "inspect", "--format", "{{.Config.Hostname}}",
ids[args.target_index - 1]],
cwd=ROOT, capture_output=True, text=True, check=True,
).stdout.strip()
expected = {
subprocess.run(
["docker", "inspect", "--format", "{{.Config.Hostname}}", item],
cwd=ROOT, capture_output=True, text=True, check=True,
).stdout.strip()
for item in ids
}
client = Client(args.base_url, timeout=10)
alice, bob, _, chat = client.bootstrap()
before = client.message(alice, chat, "Before targeted failover")["json"]
observed = {
client.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
"headers"
]["X-Instance-Id"]
for _ in range(40)
}
check(observed == expected, "both replicas must answer before failure")
client.timeout = 2
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
env = dict(os.environ, TARGET_INSTANCE_ID=target)
started = time.monotonic()
result = subprocess.run(
[str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=10, check=False,
)
check(result.returncode == 0, "stop-one hook failed")
key = uuid.uuid4().hex
text = "Targeted failover " + key
failures = 0
while True:
try:
written = client.message(alice, chat, text, key=key)
break
except Failure:
failures += 1
check(time.monotonic() - started <= 10, "write recovery exceeded 10 seconds")
time.sleep(0.1)
recovery = time.monotonic() - started
check(recovery <= 10, "write recovery exceeded 10 seconds")
check(written["headers"]["X-Instance-Id"] != target,
"stopped replica answered the write")
reads = 0
first_read = None
while reads < 10:
try:
response = client.call("GET", "/api/v1/users/me", actor=alice,
schema="User")
except Failure:
failures += 1
check(time.monotonic() - started <= 10,
"read recovery exceeded 10 seconds")
continue
if first_read is None:
first_read = time.monotonic() - started
check(first_read <= 10, "read recovery exceeded 10 seconds")
check(response["headers"]["X-Instance-Id"] != target,
"stopped replica still serves traffic")
reads += 1
messages = client.call(
"GET", f"/api/v1/chats/{chat['id']}/messages?limit=100",
actor=bob, schema="MessagePage",
)["json"]["items"]
check(before in messages, "confirmed pre-failure message disappeared")
check(sum(message["text"] == text for message in messages) == 1,
"uncertain retry created a duplicate")
print(f"PASS: replica {args.target_index} stopped; write_recovery={recovery:.3f}s; "
f"read_recovery={first_read:.3f}s; transient_failures={failures}")
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
import argparse
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, run
from resilience import history
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
parser.add_argument('--ca-file')
parser.add_argument('--action-script', required=True)
args = parser.parse_args()
client = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = client.bootstrap()
original = client.message(alice, chat, 'Сохранность после полного перезапуска')['json']
check(history(client, bob, chat) == [original], 'pre-restart history mismatch')
hook = Path(args.action_script).resolve()
check(hook.is_file(), 'restart hook not found')
print('Restarting backend, PostgreSQL and Redis; hook output suppressed.', flush=True)
try:
proc = subprocess.run([str(hook)], cwd=ROOT, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=90)
except subprocess.TimeoutExpired:
raise Failure('restart hook exceeded 90 seconds') from None
check(proc.returncode == 0, 'restart hook failed')
deadline = time.monotonic() + 90
while True:
try:
client.call('GET', '/health/ready', schema='Health')
break
except Failure:
check(time.monotonic() < deadline, 'API did not recover within 90 seconds')
time.sleep(0.5)
for actor in (alice, bob, eve):
check(client.call('GET', '/api/v1/users/me', actor=actor, schema='User')['json']
== actor['user'], 'user or existing session changed after restart')
check(client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')['json']
== chat, 'chat or membership changed after restart')
check(history(client, bob, chat) == [original], 'confirmed message lost, changed or duplicated')
client.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve, schema='Error')
followup = client.message(bob, chat, 'Запись после восстановления')['json']
check(history(client, alice, chat) == [original, followup], 'post-restart write/read mismatch')
client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
client.call('GET', '/api/v1/users/me', 401, actor=alice, schema='Error')
print('PASS: users, chat, membership, confirmed message and all existing sessions survived; '
'access control, new writes and logout work after restart.')
if __name__ == '__main__':
run(main)
+332
View File
@@ -0,0 +1,332 @@
#!/usr/bin/env python3
"""Lab 3 grade 4: measured replica failover or sequential replacement."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
from resilience import history # noqa: E402
def compose(*args, timeout=60):
try:
result = subprocess.run(
["docker", "compose", "-f", "compose.yaml", "-f", "configs/two.yaml", *args],
cwd=ROOT, capture_output=True,
text=True, timeout=timeout, check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise Failure(f"docker compose {args[0]} failed: {type(exc).__name__}") from None
check(result.returncode == 0, f"docker compose {args[0]} failed; inspect Docker logs")
return result.stdout.strip()
def docker(*args, timeout=60):
try:
result = subprocess.run(
["docker", *args], cwd=ROOT, capture_output=True,
text=True, timeout=timeout, check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise Failure(f"docker {args[0]} failed: {type(exc).__name__}") from None
check(result.returncode == 0, f"docker {args[0]} failed; inspect Docker logs")
return result.stdout.strip()
def inspect(container_id, field):
try:
result = subprocess.run(
["docker", "inspect", "--format", field, container_id],
cwd=ROOT, capture_output=True, text=True, timeout=10, check=False,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise Failure(f"docker inspect failed: {type(exc).__name__}") from None
check(result.returncode == 0, "docker inspect failed")
return result.stdout.strip()
def containers(service):
ids = compose("ps", "--all", "-q", service).splitlines()
check(bool(ids), f"{service} container is missing")
return ids
def container(service):
ids = containers(service)
check(len(ids) == 1, f"expected one {service} container")
return ids[0]
def instance(container_id):
return inspect(container_id, "{{.Config.Hostname}}")
def healthy(container_id, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if inspect(container_id, "{{.State.Health.Status}}") == "healthy":
return
time.sleep(0.5)
raise Failure(f"API container did not become healthy in {timeout} seconds")
def both_observed(client, actor, expected, timeout=30):
seen = set()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
response = client.call("GET", "/api/v1/users/me", actor=actor,
schema="User")
seen.add(response["headers"]["X-Instance-Id"])
except Failure:
pass
if seen == expected:
return
time.sleep(0.1)
raise Failure(f"both replicas were not observed; saw {sorted(seen)}")
class Traffic:
def __init__(self, base_url, actor, chat, concurrency):
self.base_url = base_url
self.actor = actor
self.chat = chat
self.concurrency = concurrency
self.stop = threading.Event()
self.lock = threading.Lock()
self.events = []
self.confirmed = set()
self.attempted = set()
self.pool = ThreadPoolExecutor(max_workers=concurrency)
def start(self):
self.futures = [self.pool.submit(self.worker, i)
for i in range(self.concurrency)]
def worker(self, worker_id):
client = Client(self.base_url, timeout=2)
index = 0
pending = None
while not self.stop.is_set():
write = worker_id % 2 == 0
if write and pending is None:
key = uuid.uuid4().hex
pending = (key, f"grade4-{worker_id}-{index}-{key}")
index += 1
with self.lock:
self.attempted.add(pending[1])
try:
if write:
response = client.message(self.actor, self.chat,
pending[1], key=pending[0])
else:
response = client.call("GET", "/api/v1/users/me",
actor=self.actor, schema="User")
event = (time.monotonic(), "write" if write else "read", True,
response["headers"]["X-Instance-Id"])
if write:
with self.lock:
self.confirmed.add(pending[1])
pending = None
except Failure:
event = (time.monotonic(), "write" if write else "read", False, None)
with self.lock:
self.events.append(event)
self.stop.wait(0.05)
def finish(self):
self.stop.set()
try:
for future in self.futures:
future.result(timeout=5)
finally:
self.pool.shutdown(wait=True)
def snapshot(self):
with self.lock:
return list(self.events)
def metrics(events, started, ended, target, action_done, scenario):
window = [event for event in events if started <= event[0] <= ended]
check(bool(window), "no traffic was recorded during the action")
counts = {}
for kind in ("read", "write"):
records = [event for event in window if event[1] == kind]
successes = [event for event in records if event[2]]
check(bool(successes), f"no successful {kind}s during the action")
survivor = next((event[0] for event in successes
if event[0] >= started and event[3] != target), None)
check(survivor is not None, f"no {kind} served by surviving replica")
after_action = next((event[0] for event in successes
if event[0] >= action_done), None)
check(after_action is not None, f"no {kind} succeeded after the action")
counts[kind] = {
"successful": len(successes),
"errors": len(records) - len(successes),
"successful_rps": round(len(successes) / (ended - started), 3),
"first_survivor_seconds": round(survivor - started, 3),
"first_success_after_action_seconds": round(after_action - started, 3),
}
after = [event for event in window if event[0] >= action_done + 1 and event[2]]
check(bool(after), "no successful traffic after the action")
if scenario == "failover":
check(all(event[3] != target for event in after),
"stopped replica still served traffic after the action")
target_times = [event[0] for event in window
if event[2] and event[3] == target]
counts["last_target_response_seconds"] = (
round(max(target_times) - started, 3) if target_times else None
)
for kind in ("read", "write"):
check(counts[kind]["first_success_after_action_seconds"] <= 10,
f"{kind} did not recover within 10 seconds")
return counts
def verify_history(client, actor, chat, original, traffic):
messages = history(client, actor, chat)
counts = Counter(message["text"] for message in messages)
check(any(message == original for message in messages),
"confirmed pre-action message disappeared")
for text in traffic.attempted:
check(counts[text] <= 1, "an uncertain retry created a duplicate")
for text in traffic.confirmed:
check(counts[text] == 1, "confirmed write disappeared")
def resource_limits():
expected = {
"nginx": (500_000_000, 128 * 1024**2),
"migrate": (1_000_000_000, 512 * 1024**2),
"postgres": (1_000_000_000, 512 * 1024**2),
"redis": (500_000_000, 256 * 1024**2),
}
ids = containers("api")
check(len(ids) in (1, 2, 4), "expected 1, 2 or 4 API replicas")
api_limits = (2_000_000_000 // len(ids), 1024 * 1024**2 // len(ids))
observed = {"api": []}
for container_id in ids:
actual = (
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
int(inspect(container_id, "{{.HostConfig.Memory}}")),
)
check(actual == api_limits, "API resource limits differ from fixed budget")
observed["api"].append({"instance": instance(container_id),
"cpus": actual[0] / 1e9,
"memory_mib": actual[1] // 1024**2})
for service, limits in expected.items():
container_id = container(service)
actual = (
int(inspect(container_id, "{{.HostConfig.NanoCpus}}")),
int(inspect(container_id, "{{.HostConfig.Memory}}")),
)
check(actual == limits, f"{service} resource limits differ from compose.yaml")
observed[service] = {"cpus": actual[0] / 1e9,
"memory_mib": actual[1] // 1024**2}
return observed
def run(args):
check(2 <= args.concurrency <= 8, "concurrency must be 2..8")
check(3 <= args.observe_seconds <= 60,
"observe-seconds must be 3..60")
result = {"scenario": args.scenario, "resources": resource_limits(),
"actions": []}
if args.scenario == "resources":
return result
targets = containers("api")
check(len(targets) == 2, "failover and rolling scenarios require the two-replica profile")
client = Client(args.base_url, timeout=10)
alice, bob, _, chat = client.bootstrap()
original = client.message(alice, chat, "Before grade 4 actions")["json"]
both_observed(client, alice, {instance(item) for item in targets})
migrate_id = container("migrate")
traffic = Traffic(args.base_url, alice, chat, args.concurrency)
traffic.start()
try:
for before_id in targets:
target = instance(before_id)
healthy(before_id)
before_ids = set(containers("api"))
both_observed(client, alice, {instance(item) for item in before_ids})
time.sleep(1)
started = time.monotonic()
if args.scenario == "failover":
docker("stop", "--time", "30", before_id, timeout=40)
check(inspect(before_id, "{{.State.Status}}") == "exited",
"target API container was not stopped")
else:
docker("stop", "--time", "30", before_id, timeout=40)
docker("rm", before_id)
compose("up", "-d", "--no-build", "--no-deps",
"--scale", "api=2", "api", timeout=90)
replacements = set(containers("api")) - before_ids
check(len(replacements) == 1, "one API container was not replaced")
healthy(next(iter(replacements)))
action_done = time.monotonic()
time.sleep(args.observe_seconds)
ended = time.monotonic()
item = {"target": target,
"action_seconds": round(action_done - started, 3),
"window_seconds": round(ended - started, 3),
"traffic": metrics(traffic.snapshot(), started, ended,
target, action_done, args.scenario)}
if args.scenario == "failover":
docker("start", before_id)
healthy(before_id)
both_observed(client, alice,
{instance(item) for item in containers("api")})
result["actions"].append(item)
check(container("migrate") == migrate_id,
"migration container changed during API replacement")
finally:
traffic.finish()
# Restore both API replicas after a failed test; never remove volumes.
try:
compose("up", "-d", "--no-build", "--no-deps",
"--scale", "api=2", "api")
except Failure:
print("WARNING: restore API replicas manually", file=sys.stderr)
verify_history(client, bob, chat, original, traffic)
result["confirmed_writes"] = len(traffic.confirmed)
result["attempted_unique_writes"] = len(traffic.attempted)
result["migration_container_unchanged"] = True
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--scenario", choices=("resources", "failover", "rolling"),
required=True)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--concurrency", type=int, default=4)
parser.add_argument("--observe-seconds", type=float, default=10)
parser.add_argument("--output", help="optional JSON result path without tokens")
args = parser.parse_args()
try:
result = run(args)
output = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(output + "\n", encoding="utf-8")
print(output)
print("PASS: grade 4 scenario")
except (Failure, ValueError, OSError) as exc:
print(f"FAIL: {exc}", file=sys.stderr)
raise SystemExit(1) from None
if __name__ == "__main__":
main()
+56
View File
@@ -0,0 +1,56 @@
import assert from 'node:assert/strict';
import net from 'node:net';
export async function faultProxy(connectionUrl, defaultPort) {
const upstreamUrl = new URL(connectionUrl);
const host = upstreamUrl.hostname.replace(/^\[|\]$/g, '');
const port = Number(upstreamUrl.port || defaultPort);
let mode = 'online';
const sockets = new Set();
const track = (socket) => {
sockets.add(socket);
socket.on('error', () => {});
socket.on('close', () => sockets.delete(socket));
return socket;
};
const server = net.createServer((client) => {
track(client);
if (mode === 'cut') {
client.destroy();
return;
}
const upstream = track(net.connect({ host, port }));
client.on('data', (chunk) => {
if (mode === 'online' && !upstream.destroyed) upstream.write(chunk);
});
upstream.on('data', (chunk) => {
if (mode === 'online' && !client.destroyed) client.write(chunk);
});
client.on('close', () => upstream.destroy());
upstream.on('close', () => client.destroy());
});
await new Promise((resolve, reject) => {
server.once('error', reject);
server.listen(0, '127.0.0.1', () => {
server.off('error', reject);
resolve();
});
});
const relayUrl = new URL(connectionUrl);
relayUrl.hostname = '127.0.0.1';
relayUrl.port = String(server.address().port);
return {
url: relayUrl.toString(),
setMode(nextMode) {
assert.ok(['online', 'cut', 'hang'].includes(nextMode));
mode = nextMode;
if (mode === 'cut') {
for (const socket of sockets) socket.destroy();
}
},
async close() {
for (const socket of sockets) socket.destroy();
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
},
};
}
+7
View File
@@ -0,0 +1,7 @@
export async function messageEpoch(prisma, chatId) {
const [row] = await prisma.$queryRawUnsafe(
'SELECT COALESCE(SUM(epoch), 0)::bigint AS epoch FROM public.chat_message_epochs WHERE chat_id = $1::uuid',
chatId,
);
return row.epoch;
}
+146
View File
@@ -0,0 +1,146 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import test from 'node:test';
const require = createRequire(import.meta.url);
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
test(
'message idempotency is shared by two API instances',
{ timeout: 120000 },
async (t) => {
assert.ok(
process.env.DATABASE_URL,
'Set DATABASE_URL and apply migrations',
);
const apps = [];
let db;
let chatId;
const userIds = [];
t.after(async () => {
if (db && chatId) {
await db.$executeRawUnsafe(
'DELETE FROM message_idempotency WHERE chat_id = $1::uuid',
chatId,
);
await db.chat.deleteMany({ where: { id: chatId } });
}
if (db && userIds.length) {
await db.user.deleteMany({ where: { id: { in: userIds } } });
}
for (const app of apps) await app.close();
});
for (let index = 0; index < 2; index++) {
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
});
exposeRequestIdHeader(app);
await app.init();
await app.getHttpAdapter().getInstance().ready();
apps.push(app);
}
db = apps[0].get(PrismaService);
const servers = apps.map((app) => app.getHttpAdapter().getInstance());
const call = (index, method, url, token, payload, key) =>
servers[index].inject({
method,
url,
headers: {
...(token ? { authorization: `Bearer ${token}` } : {}),
...(key === undefined ? {} : { 'idempotency-key': key }),
},
...(payload === undefined ? {} : { payload }),
});
const register = async (label) => {
const username = `idem_${label}_${randomUUID().replaceAll('-', '').slice(0, 12)}`;
const password = `T3st-${randomUUID()}`;
const response = await call(0, 'POST', '/api/v1/users', undefined, {
username,
display_name: label,
password,
});
assert.equal(response.statusCode, 201, response.body);
const user = response.json();
userIds.push(user.id);
const login = await servers[0].inject({
method: 'POST',
url: '/api/v1/auth/sessions',
headers: {
authorization: `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`,
},
});
assert.equal(login.statusCode, 201, login.body);
return { user, token: login.json().token };
};
const owner = await register('owner');
const outsider = await register('outsider');
const created = await call(0, 'POST', '/api/v1/chats', owner.token, {
title: 'Idempotency test',
member_ids: [owner.user.id],
});
assert.equal(created.statusCode, 201, created.body);
chatId = created.json().id;
const path = `/api/v1/chats/${chatId}/messages`;
const key = randomUUID();
const body = { text: 'One message' };
assert.equal(
(await call(0, 'POST', path, owner.token, body)).statusCode,
400,
);
assert.equal(
(await call(0, 'POST', path, owner.token, body, 'bad key')).statusCode,
400,
);
const first = await call(0, 'POST', path, owner.token, body, key);
assert.equal(first.statusCode, 201, first.body);
const replay = await call(1, 'POST', path, owner.token, body, key);
assert.equal(replay.statusCode, 201, replay.body);
assert.deepEqual(replay.json(), first.json());
assert.equal(
(await call(1, 'POST', path, owner.token, { text: 'Changed' }, key))
.statusCode,
409,
);
assert.equal(
(await call(1, 'POST', path, outsider.token, body, key)).statusCode,
404,
);
const concurrentKey = randomUUID();
const concurrent = await Promise.all(
Array.from({ length: 12 }, (_, index) =>
call(
index % 2,
'POST',
path,
owner.token,
{ text: 'Concurrent' },
concurrentKey,
),
),
);
for (const response of concurrent) {
assert.equal(response.statusCode, 201, response.body);
assert.deepEqual(response.json(), concurrent[0].json());
}
assert.equal(await db.message.count({ where: { chatId } }), 2);
const [record] = await db.$queryRawUnsafe(
'SELECT expires_at AS "expiresAt" FROM message_idempotency WHERE user_id = $1::uuid AND chat_id = $2::uuid AND "key" = $3',
owner.user.id,
chatId,
concurrentKey,
);
assert.ok(record.expiresAt.getTime() - Date.now() >= 23 * 3600 * 1000);
},
);
+137
View File
@@ -0,0 +1,137 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { createHash, randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import test from 'node:test';
const require = createRequire(import.meta.url);
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
test('message keyset pagination: timestamp ties, chat isolation and deleted cursor row', async (t) => {
assert.ok(process.env.DATABASE_URL);
const previousCacheUrl = process.env.CACHE_URL;
process.env.CACHE_URL = '';
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
const prisma = app.get(PrismaService);
let userId;
const chatIds = [];
t.after(async () => {
try {
await prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
if (userId) await prisma.user.delete({ where: { id: userId } });
} finally {
await app.close();
if (previousCacheUrl === undefined) delete process.env.CACHE_URL;
else process.env.CACHE_URL = previousCacheUrl;
}
});
exposeRequestIdHeader(app);
await app.init();
const server = app.getHttpAdapter().getInstance();
await server.ready();
const user = await prisma.user.create({
data: {
username: `page_${randomUUID().replaceAll('-', '').slice(0, 20)}`,
displayName: 'Synthetic pagination test',
},
});
userId = user.id;
const token = randomBytes(32).toString('base64url');
await prisma.session.create({
data: {
userId,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 600000),
},
});
for (let i = 0; i < 2; i++) {
const chat = await prisma.chat.create({
data: {
title: `Pagination ${i}`,
members: { create: { userId, position: 0 } },
},
});
chatIds.push(chat.id);
}
const [chatId, otherChatId] = chatIds;
const rows = Array.from({ length: 140 }, (_, i) => ({
id: randomUUID(),
chatId,
senderId: userId,
text: `Message ${i}`,
createdAt: new Date(Date.UTC(2026, 0, 1) + Math.floor(i / 10) * 1000),
}));
await prisma.message.createMany({
data: [
...rows,
...rows.map((row) => ({
...row,
id: randomUUID(),
chatId: otherChatId,
text: 'Other chat',
})),
],
});
const expected = [...rows]
.sort(
(a, b) =>
a.createdAt - b.createdAt || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0),
)
.map((row) => ({
id: row.id,
chat_id: chatId,
sender_id: userId,
text: row.text,
created_at: row.createdAt.toISOString(),
}));
const request = (cursor, selectedChat = chatId) =>
server.inject({
method: 'GET',
url: `/api/v1/chats/${selectedChat}/messages?limit=7${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`,
headers: { authorization: `Bearer ${token}` },
});
const seen = [];
let cursor;
let firstCursor;
do {
const response = await request(cursor);
assert.equal(response.statusCode, 200);
const page = response.json();
assert.ok(page.items.length <= 7);
seen.push(...page.items);
if (!firstCursor) firstCursor = page.next_cursor;
cursor = page.next_cursor;
assert.ok(
seen.length <= rows.length,
'pagination must terminate without duplicates',
);
} while (cursor);
assert.deepEqual(seen, expected);
const anchor = JSON.parse(
Buffer.from(firstCursor, 'base64url').toString('utf8'),
);
await prisma.message.delete({ where: { id: anchor.id } });
const afterDeletion = await request(firstCursor);
assert.equal(afterDeletion.statusCode, 200);
assert.deepEqual(afterDeletion.json().items, expected.slice(7, 14));
assert.equal((await request(firstCursor, otherChatId)).statusCode, 400);
const pastEnd = Buffer.from(
JSON.stringify({
scope: `messages:${chatId}`,
id: randomUUID(),
created_at: '2027-01-01T00:00:00.000Z',
}),
).toString('base64url');
const empty = await request(pastEnd);
assert.equal(empty.statusCode, 200);
assert.deepEqual(empty.json(), { items: [], next_cursor: null });
});
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# A barrier makes sequential probes fail: all four requests must start together.
set -eu
sed '/^check_ready &/,$d' /source/start-nginx.sh > /tmp/readiness-under-test.sh
. /tmp/readiness-under-test.sh
mkdir -p /tmp/readiness-barrier
cycle=0
getent() {
for ip in 1 2 3 4; do printf '192.0.2.%s STREAM api\n' "$ip"; done
}
wget() {
for arg do
case "$arg" in http://*) address=${arg#http://}; address=${address%%:*} ;; esac
done
touch "/tmp/readiness-barrier/$address"
attempt=0
while [ "$attempt" -lt 50 ]; do
if [ "$(find /tmp/readiness-barrier -type f | wc -l)" -eq 4 ]; then return 0; fi
attempt=$((attempt + 1))
command sleep 0.01
done
return 1
}
nginx() { return 0; }
sleep() {
if [ "$cycle" -eq 2 ]; then
count=$(grep -c 'server 192.0.2.' "$upstreams" || true)
[ "$count" -eq 4 ] || { printf 'FAIL: readiness probes did not start in parallel\n'; exit 1; }
printf 'PASS: four concurrent probes admitted after two successful rounds\n'
exit 0
fi
rm -f /tmp/readiness-barrier/*
cycle=$((cycle + 1))
}
check_ready
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# Run in a disposable nginx container; never against a running load balancer.
set -eu
sed '/^check_ready &/,$d' /source/start-nginx.sh > /tmp/readiness-under-test.sh
. /tmp/readiness-under-test.sh
cycle=0
reloads=0
getent() { printf '192.0.2.1 STREAM api\n'; }
wget() {
case "$cycle" in
1|2|4|7|8) return 0 ;;
*) return 1 ;;
esac
}
nginx() {
if [ "$1" = '-s' ]; then reloads=$((reloads + 1)); fi
}
sleep() {
# Check the previous cycle before advancing to the next probe.
if [ "$cycle" -gt 0 ]; then
case "$cycle" in
2|3|4|5|8) expected=1 ;;
*) expected=0 ;;
esac
actual=0
if grep -q 'server 192.0.2.1:8080 ' "$upstreams"; then actual=1; fi
if [ "$actual" -ne "$expected" ]; then
printf 'FAIL: cycle %s, expected readiness %s, got %s\n' "$cycle" "$expected" "$actual" >&2
exit 1
fi
fi
cycle=$((cycle + 1))
if [ "$cycle" -eq 9 ]; then
[ "$reloads" -eq 3 ] || exit 1
printf 'PASS: admission, isolated failure, recovery, removal and re-admission; 3 reloads\n'
exit 0
fi
}
check_ready
+268
View File
@@ -0,0 +1,268 @@
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { createHash } = require('node:crypto');
const { spawn, execFileSync } = require('node:child_process');
const ts = require('typescript');
const root = path.resolve(__dirname, '..');
const resultPrefix = process.env.EXPERIMENT_PREFIX ?? 'cache-strategy';
assert.match(resultPrefix, /^[a-z0-9-]+$/);
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lab2-no-epochs-'));
const hashes = {};
const volumes = [];
const replace = (source, from, to) => {
assert.ok(source.includes(from), `Missing replacement: ${from}`);
assert.equal(
source.indexOf(from),
source.lastIndexOf(from),
`Ambiguous replacement: ${from}`,
);
return source.replace(from, to);
};
const section = (source, from, to, replacement) => {
const start = source.indexOf(from);
const end = source.indexOf(to, start);
assert.ok(start !== -1 && end > start, `Missing section: ${from}`);
return source.slice(0, start) + replacement + source.slice(end);
};
const digest = (value) => createHash('sha256').update(value).digest('hex');
const transforms = {
'Chat/chat.service.ts': (source) => {
source = section(
source,
' await tx.$executeRaw(',
' return created;',
'',
);
source = replace(source, ' epoch: string,\n', '');
source = replace(source, ' epoch,\n', '');
source = section(
source,
' const [membership] =',
' if (!membership)',
' const [membership] = await this.prisma.$queryRaw<{ id: string }[]>(Prisma.sql`\n' +
' SELECT cm.chat_id AS id FROM chat_members AS cm\n' +
' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' +
' `);\n',
);
source = section(
source,
' const [message] =',
' if (!message)',
' const [message] = await this.prisma.$queryRaw<StoredMessage[]>(Prisma.sql`\n' +
' INSERT INTO messages (id, chat_id, sender_id, text)\n' +
' SELECT ${randomUUID()}::uuid, cm.chat_id, ${userId}::uuid, ${text}\n' +
' FROM chat_members AS cm\n' +
' WHERE cm.chat_id = ${id}::uuid AND cm.user_id = ${userId}::uuid\n' +
' RETURNING id, chat_id AS "chatId", sender_id AS "senderId",\n' +
' text, created_at AS "createdAt"\n' +
' `);\n',
);
return replace(source, ' chat.messagesEpoch.toString(),\n', '');
},
'Auth/auth.service.ts': (source) => {
source = source.replace(
/COALESCE\(\(SELECT SUM\(e\.epoch\)[\s\S]*?0\)::bigint AS "messagesEpoch"/g,
'0::bigint AS "messagesEpoch"',
);
return source
.replaceAll('u.chats_epoch', '0::bigint')
.replaceAll('c.messages_epoch', '0::bigint');
},
'User/user.guard.ts': (source) => {
source = replace(source, ' currentChatsEpoch: string;\n', '');
return replace(
source,
' request.currentChatsEpoch = session.chatsEpoch;\n',
'',
);
},
'Chat/chat.controller.ts': (source) =>
replace(source, ' request.currentChatsEpoch,\n', ''),
'cache/cache.service.ts': (source) => {
source = replace(source, ' epoch?: string,\n', '');
return section(
source,
' if (epoch !== undefined)',
' const versionKey =',
'',
);
},
'User/user.service.ts': (source) => {
source = replace(
source,
'const user = await this.prisma.user.findUnique({ where: { id } });',
'const user = await this.prisma.user.findUnique({ where: { id }, select: { id: true, username: true, displayName: true, createdAt: true } });',
);
return source;
},
};
try {
for (const [relative, transform] of Object.entries(transforms)) {
const original = fs.readFileSync(path.join(root, 'src', relative), 'utf8');
const source = transform(original.replace(/\r\n/g, '\n'));
assert.ok(
!/messages_epoch|chats_epoch|chat_message_epochs/.test(source),
relative,
);
const compiled = ts.transpileModule(source, {
compilerOptions: {
target: ts.ScriptTarget.ES2023,
module: ts.ModuleKind.CommonJS,
experimentalDecorators: true,
emitDecoratorMetadata: true,
},
reportDiagnostics: true,
});
assert.equal(
compiled.diagnostics.filter(
(item) => item.category === ts.DiagnosticCategory.Error,
).length,
0,
);
const sourcePath = path.join(directory, relative);
const compiledPath = sourcePath.replace(/\.ts$/, '.js');
fs.mkdirSync(path.dirname(sourcePath), { recursive: true });
fs.writeFileSync(sourcePath, source);
fs.writeFileSync(compiledPath, compiled.outputText);
hashes[relative] = {
original_source: digest(original),
experiment_source: digest(source),
experiment_js: digest(compiled.outputText),
};
volumes.push({
type: 'bind',
source: compiledPath,
target: `/app/dist/${relative.replace(/\.ts$/, '.js')}`,
read_only: true,
});
}
let benchmark = fs
.readFileSync(path.join(root, 'test/cache-benchmark.mjs'), 'utf8')
.replace(/\r\n/g, '\n');
benchmark = replace(
benchmark,
'process.env.CACHE_URL = proxy.url;',
'delete process.env.CACHE_URL;',
);
benchmark = section(
benchmark,
' await prisma.chatMessageEpoch.createMany(',
' const path =',
'',
);
benchmark = section(
benchmark,
' const second = await start();',
' report.completed = true;',
' assert.equal(first.app.get(CacheService).redis, undefined);\n' +
' for (const chatId of chatIds) {\n' +
' assert.equal(await messageEpoch(prisma, chatId), 0n);\n' +
' }\n' +
' assert.equal((await prisma.user.findUniqueOrThrow({ where: { id: userId } })).chatsEpoch, 0n);\n' +
' for (const chatId of chatIds.slice(1)) {\n' +
' const page = await request("GET", `/api/v1/chats/${chatId}/messages`);\n' +
' assert.equal(page.items.length, 50);\n' +
' assert.equal(new Set(page.items.map(message => message.id)).size, 50);\n' +
' }\n' +
' report.correctness = { no_redis_client: true, epochs_unchanged: true, read_after_write: true };\n' +
' }\n',
);
const benchmarkPath = path.join(directory, 'cache-benchmark.mjs');
fs.writeFileSync(benchmarkPath, benchmark);
volumes.push({
type: 'bind',
source: benchmarkPath,
target: '/experiment/cache-benchmark.mjs',
read_only: true,
});
volumes.push({
type: 'bind',
source: path.join(root, 'test/helpers'),
target: '/experiment/helpers',
read_only: true,
});
const override = path.join(directory, 'compose.no-epochs.yaml');
fs.writeFileSync(
override,
JSON.stringify({ services: { benchmark: { volumes } } }, null, 2),
);
const args = [
'compose',
'-f',
'compose.yaml',
'-f',
'test/compose.benchmark.yaml',
'-f',
override,
'run',
'--rm',
'--no-deps',
'--pull',
'never',
'benchmark',
'node',
'/experiment/cache-benchmark.mjs',
'after',
`/results/${resultPrefix}-no-redis-no-epochs.json`,
...process.argv.slice(2),
];
const metadata = {
recorded_at: new Date().toISOString(),
image: execFileSync(
'docker',
['image', 'inspect', 'mtusi2026lab2-fork-backend', '--format', '{{.Id}}'],
{ encoding: 'utf8' },
).trim(),
modified_sources: hashes,
command: ['docker', ...args],
implementation:
'Separate temporary source copies compiled with TypeScript and mounted over six application modules. Workspace sources and running backend remain unchanged.',
schema:
'Epoch storage remains in the database; experiment API paths do not update counters or read authoritative epoch values.',
redis:
'Application starts without CACHE_URL. Observer/proxy are retained only for benchmark setup and cleanup.',
full_result: `${resultPrefix}-full.json`,
experiment_result: `${resultPrefix}-no-redis-no-epochs.json`,
};
const child = spawn('docker', args, {
cwd: root,
stdio: 'inherit',
env: process.env,
});
child.once('error', finish);
child.once('exit', (code, signal) => finish(null, code, signal));
let finished = false;
function finish(error, code, signal) {
if (finished) return;
finished = true;
metadata.exit_code = error ? 1 : code;
metadata.signal = signal;
metadata.sources_unchanged = Object.entries(hashes).every(
([relative, hash]) =>
digest(fs.readFileSync(path.join(root, 'src', relative), 'utf8')) ===
hash.original_source,
);
fs.writeFileSync(
path.join(root, `evidence/${resultPrefix}-experiment.json`),
JSON.stringify(metadata, null, 2) + '\n',
);
cleanup();
if (error) console.error(error.message);
process.exitCode = error ? 1 : (code ?? 1);
}
} catch (error) {
cleanup();
throw error;
}
function cleanup() {
const resolved = path.resolve(directory);
assert.equal(path.dirname(resolved), path.resolve(os.tmpdir()));
assert.ok(path.basename(resolved).startsWith('lab2-no-epochs-'));
fs.rmSync(resolved, { recursive: true, force: true });
}
+63
View File
@@ -0,0 +1,63 @@
import assert from 'node:assert/strict';
import { argon2 } from 'node:crypto';
import { createRequire } from 'node:module';
import test from 'node:test';
const require = createRequire(import.meta.url);
const { PasswordService } = require('../dist/Auth/password.service');
const password = '🔐'.repeat(40) + ':Синтетический пароль\0';
test('Argon2id only: PHC format, independent derivation, unique salts, Unicode and invalid hashes', async () => {
const service = new PasswordService();
const first = await service.hash(password);
const second = await service.hash(password);
assert.notEqual(
first,
second,
'same password must have independently salted hashes',
);
assert.match(
first,
/^\$argon2id\$v=19\$m=19456,t=2,p=1\$[A-Za-z0-9+/]{22}\$[A-Za-z0-9+/]{43}$/,
);
assert.ok(!first.includes(password));
const parts = first.split('$');
const independent = await new Promise((resolve, reject) =>
argon2(
'argon2id',
{
message: password,
nonce: Buffer.from(parts[4], 'base64'),
memory: 19456,
passes: 2,
parallelism: 1,
tagLength: 32,
},
(error, key) => (error ? reject(error) : resolve(key)),
),
);
assert.deepEqual(Buffer.from(parts[5], 'base64'), independent);
assert.equal(await service.verify(password, first), true);
assert.equal(
await service.verify(password.replace('🔐', '🔑'), first),
false,
);
assert.equal(
await service.verify(password.slice(0, -1), first),
false,
'NUL must not be silently truncated',
);
for (const invalid of [
null,
'',
'scrypt$131072$8$1$unsupported$unsupported',
'$argon2id$v=19$m=999999999,t=999999999,p=1$x$x',
first.replace('v=19', 'v=16'),
first.replace('argon2id', 'argon2i'),
first + '$extra',
first.replace(parts[5], 'a'),
first.replace(parts[4], parts[4] + '='),
]) {
assert.equal(await service.verify(password, invalid), false);
}
});
+349
View File
@@ -0,0 +1,349 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import test from 'node:test';
const require = createRequire(import.meta.url);
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
async function start() {
const app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
try {
exposeRequestIdHeader(app);
await app.init();
const server = app.getHttpAdapter().getInstance();
await server.ready();
return { app, server, prisma: app.get(PrismaService) };
} catch (error) {
await app.close();
throw error;
}
}
test('PostgreSQL storage: constraints, pagination, access and application restart', async (t) => {
assert.ok(
process.env.DATABASE_URL,
'Set DATABASE_URL and apply migrations before running this test',
);
let context = await start();
const userIds = [];
const chatIds = [];
t.after(async () => {
try {
await context.prisma.chat.deleteMany({ where: { id: { in: chatIds } } });
await context.prisma.user.deleteMany({ where: { id: { in: userIds } } });
} finally {
await context.app.close();
}
});
const suffix = randomUUID().replaceAll('-', '').slice(0, 16);
const password = '🔐'.repeat(40) + ':пароль с пробелами ';
const tokens = new Map();
const request = (method, url, payload, userId) =>
context.server.inject({
method,
url,
...(payload === undefined ? {} : { payload }),
headers: {
...(userId ? { authorization: `Bearer ${tokens.get(userId)}` } : {}),
...(method === 'POST' && /\/messages$/.test(url)
? { 'idempotency-key': randomUUID() }
: {}),
},
});
const registrations = await Promise.all(
Array.from({ length: 4 }, () =>
request('POST', '/api/v1/users', {
username: `race_${suffix}`,
display_name: 'Race',
password,
}),
),
);
for (const response of registrations) {
if (response.statusCode === 201) userIds.push(response.json().id);
}
assert.deepEqual(
registrations.map((response) => response.statusCode).sort(),
[201, 409, 409, 409],
);
const userId = userIds[0];
const outsider = await request('POST', '/api/v1/users', {
username: `other_${suffix}`,
display_name: 'Other',
password,
});
assert.equal(outsider.statusCode, 201);
userIds.push(outsider.json().id);
const login = (username, secret = password, payload) =>
context.server.inject({
method: 'POST',
url: '/api/v1/auth/sessions',
headers: {
authorization: `Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`,
},
...(payload === undefined ? {} : { payload }),
});
for (const user of [
registrations.find((r) => r.statusCode === 201).json(),
outsider.json(),
]) {
assert.deepEqual(Object.keys(user).sort(), [
'created_at',
'display_name',
'id',
'username',
]);
const response = await login(user.username);
assert.equal(response.statusCode, 201);
assert.equal(response.json().token_type, 'Bearer');
assert.match(response.json().token, /^[A-Za-z0-9_-]{43}$/);
assert.deepEqual(response.json().user, user);
tokens.set(user.id, response.json().token);
const stored = await context.prisma.user.findUniqueOrThrow({
where: { id: user.id },
});
assert.match(stored.passwordHash, /^\$argon2id\$v=19\$m=19456,t=2,p=1\$/);
assert.ok(!stored.passwordHash.includes(password));
}
const sessions = await context.prisma.session.findMany({ where: { userId } });
assert.equal(sessions.length, 1);
assert.match(sessions[0].tokenHash, /^[a-f0-9]{64}$/);
assert.notEqual(sessions[0].tokenHash, tokens.get(userId));
const wrong = await login(`race_${suffix}`, 'wrong-password');
const unknown = await login(`missing_${suffix}`, 'wrong-password');
assert.equal(wrong.statusCode, 401);
assert.equal(unknown.statusCode, 401);
assert.equal(wrong.json().error.message, unknown.json().error.message);
assert.equal(
wrong.headers['www-authenticate'],
'Basic realm="messenger", charset="UTF-8"',
);
assert.equal((await login(`race_${suffix}`, password, {})).statusCode, 400);
for (const invalid of [
undefined,
'short',
' '.repeat(12),
'🔐'.repeat(129),
]) {
assert.equal(
(
await request('POST', '/api/v1/users', {
username: `bad_${suffix}`,
display_name: 'Bad',
...(invalid === undefined ? {} : { password: invalid }),
})
).statusCode,
400,
);
}
for (const authorization of [
undefined,
'Bearer unknown',
`Basic ${Buffer.from(`race_${suffix}:${password}`).toString('base64')}`,
]) {
const response = await context.server.inject({
method: 'GET',
url: '/api/v1/users/me',
headers: {
'x-user-id': userId,
...(authorization ? { authorization } : {}),
},
});
assert.equal(response.statusCode, 401);
assert.equal(
response.headers['www-authenticate'],
'Bearer realm="messenger"',
);
}
const identity = await context.server.inject({
method: 'GET',
url: '/api/v1/users/me',
headers: {
authorization: `Bearer ${tokens.get(userId)}`,
'x-user-id': outsider.json().id,
},
});
assert.equal(identity.json().id, userId);
const ready = await request('GET', '/health/ready');
assert.equal(ready.statusCode, 200);
const invalidChat = await request(
'POST',
'/api/v1/chats',
{
title: `invalid_${suffix}`,
member_ids: [userId, randomUUID()],
},
userId,
);
assert.equal(invalidChat.statusCode, 400);
assert.equal(
await context.prisma.chat.count({ where: { title: `invalid_${suffix}` } }),
0,
);
const rolledBackId = randomUUID();
chatIds.push(rolledBackId);
await assert.rejects(
context.prisma.$transaction(async (tx) => {
await tx.chat.create({ data: { id: rolledBackId, title: 'Rollback' } });
await tx.chatMember.create({
data: { chatId: rolledBackId, userId: randomUUID(), position: 0 },
});
}),
{ code: 'P2003' },
);
assert.equal(
await context.prisma.chat.count({ where: { id: rolledBackId } }),
0,
);
const createdChat = await request(
'POST',
'/api/v1/chats',
{ title: 'Persisted chat', member_ids: [userId] },
userId,
);
assert.equal(createdChat.statusCode, 201);
const chatId = createdChat.json().id;
chatIds.push(chatId);
const forbidden = await request(
'GET',
`/api/v1/chats/${chatId}`,
undefined,
outsider.json().id,
);
assert.equal(forbidden.statusCode, 404);
const unauthorized = await request('GET', `/api/v1/chats/${chatId}`);
assert.equal(unauthorized.statusCode, 401);
const messageIds = [];
for (let i = 0; i < 5; i++) {
const message = await request(
'POST',
`/api/v1/chats/${chatId}/messages`,
{ text: `Message ${i}` },
userId,
);
assert.equal(message.statusCode, 201);
messageIds.push(message.json().id);
}
await context.prisma.message.updateMany({
where: { chatId },
data: { createdAt: new Date('2026-01-01T00:00:00.000Z') },
});
const first = await request(
'GET',
`/api/v1/chats/${chatId}/messages?limit=2`,
undefined,
userId,
);
assert.equal(first.statusCode, 200);
assert.equal(first.json().items.length, 2);
assert.ok(first.json().next_cursor);
await context.app.close();
context = await start();
const me = await request('GET', '/api/v1/users/me', undefined, userId);
assert.equal(me.statusCode, 200);
const chat = await request(
'GET',
`/api/v1/chats/${chatId}`,
undefined,
userId,
);
assert.equal(chat.statusCode, 200);
assert.deepEqual(chat.json().member_ids, [userId]);
const listed = await request(
'GET',
'/api/v1/chats?limit=1',
undefined,
userId,
);
assert.deepEqual(
listed.json().items.map((item) => item.id),
[chatId],
);
const seen = first.json().items.map((message) => message.id);
let cursor = first.json().next_cursor;
while (cursor) {
const page = await request(
'GET',
`/api/v1/chats/${chatId}/messages?limit=2&cursor=${cursor}`,
undefined,
userId,
);
assert.equal(page.statusCode, 200);
assert.ok(page.json().items.length <= 2);
seen.push(...page.json().items.map((message) => message.id));
cursor = page.json().next_cursor;
}
assert.deepEqual(seen, [...messageIds].sort());
assert.equal(new Set(seen).size, 5);
const logouts = await Promise.all(
Array.from({ length: 2 }, () =>
request('DELETE', '/api/v1/auth/sessions/current', undefined, userId),
),
);
assert.deepEqual(logouts.map((r) => r.statusCode).sort(), [204, 401]);
assert.equal(logouts.find((r) => r.statusCode === 204).body, '');
assert.equal(
(await request('GET', '/api/v1/users/me', undefined, userId)).statusCode,
401,
);
assert.equal(
(
await request(
'DELETE',
'/api/v1/auth/sessions/current',
undefined,
userId,
)
).statusCode,
401,
);
await context.app.close();
const originalTtl = process.env.SESSION_TTL_SECONDS;
process.env.SESSION_TTL_SECONDS = '2';
try {
context = await start();
} finally {
if (originalTtl === undefined) delete process.env.SESSION_TTL_SECONDS;
else process.env.SESSION_TTL_SECONDS = originalTtl;
}
const short = await login(`race_${suffix}`);
assert.equal(short.statusCode, 201);
tokens.set(userId, short.json().token);
assert.equal(
(await request('GET', '/api/v1/users/me', undefined, userId)).statusCode,
200,
);
await context.app.close();
context = await start();
await new Promise((resolve) =>
setTimeout(
resolve,
Math.max(0, Date.parse(short.json().expires_at) - Date.now()) + 100,
),
);
assert.equal(
(await request('GET', '/api/v1/users/me', undefined, userId)).statusCode,
401,
);
});
+63
View File
@@ -0,0 +1,63 @@
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import vm from 'node:vm';
import test from 'node:test';
test('pool observer preserves promise/callback results, errors and release', async () => {
let timer;
let now = 0;
const records = [];
const released = () => {};
const client = { release: released };
const failure = new Error('synthetic');
class FakePool {
totalCount = 2;
idleCount = 1;
waitingCount = 3;
fail = false;
connect(callback) {
now += 5;
if (callback) {
callback(this.fail ? failure : undefined, this.fail ? undefined : client, released);
return undefined;
}
return this.fail ? Promise.reject(failure) : Promise.resolve(client);
}
}
const modules = {
pg: { Pool: FakePool },
'node:fs': {
writeFileSync() {},
appendFileSync(path, value) { records.push(JSON.parse(value)); },
readFileSync(path) { return path.endsWith('memory.current') ? '1024' : 'usage_usec 10\n'; },
},
'node:perf_hooks': {
performance: { now: () => now },
monitorEventLoopDelay: () => ({ enable() {}, reset() {}, mean: 1, percentile: () => 1 }),
},
};
vm.runInNewContext(readFileSync('scripts/diagnostics/pool.cjs', 'utf8'), {
require: (name) => modules[name],
process: { pid: 1, env: { INSTANCE_ID: 'test' },
memoryUsage: () => ({ rss: 2048 }), cpuUsage: () => ({ user: 1, system: 2 }) },
setInterval(callback) { timer = callback; return { unref() {} }; },
Date,
});
const pool = new FakePool();
assert.equal(await pool.connect(), client);
assert.equal(pool.connect((error, value, release) => {
assert.equal(error, undefined);
assert.equal(value, client);
assert.equal(release, released);
}), undefined);
pool.fail = true;
await assert.rejects(pool.connect(), (error) => error === failure);
pool.connect((error) => assert.equal(error, failure));
now = 2000;
timer();
assert.equal(records[0].pools[0].acquisitions, 4);
assert.equal(records[0].pools[0].errors, 2);
assert.equal(records[0].pools[0].acquire_ms, 20);
assert.equal(records[0].pools[0].interval_max_waiting, 3);
assert.equal(records[0].pools[0].interval_max_busy, 1);
});
+85
View File
@@ -0,0 +1,85 @@
import assert from 'node:assert/strict';
import { readFile, writeFile } from 'node:fs/promises';
const report = JSON.parse(
await readFile('evidence/redis-comparison.json', 'utf8'),
);
assert.ok(report.completed);
assert.ok(Object.values(report.correctness.read_after_write).every(Boolean));
const lines = [
'# Redis и работа без кэша приложения',
'',
`Дата UTC: ${report.recorded_at}. Node.js ${report.node}.`,
'',
'Один прогон на одном образе: три повтора каждого режима с чередованием порядка. Второй экземпляр API запущен без CACHE_URL: отключены все кэши приложения, включая профиль пользователя. PostgreSQL-эпохи и запись одним CTE сохранены в обоих режимах. Контейнер Redis не останавливался; экземпляр без кэша не создаёт Redis-клиент.',
'',
'RPS = суммарное число успешных запросов / суммарное время трёх повторов. Средняя задержка взвешена по числу успешных запросов. p95 — среднее трёх отдельных p95, не перцентиль объединённой выборки. SQL/HTTP = суммарные SQL-вызовы / все HTTP-запросы; команды транзакций исключены.',
'',
'| Сценарий | Режим | Средний RPS | Средняя задержка, мс | Среднее p95, мс | SQL/HTTP | CPU, % ядра | Ошибки |',
'| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |',
];
const aggregates = [];
for (const [workload, label] of [
['cold-history-burst', 'Холодная волна истории'],
['warm-history', 'Прогретая история'],
['writes-single-chat', 'Запись в один чат'],
]) {
for (const mode of ['redis', 'no-redis']) {
const samples = report.samples.filter(
(sample) => sample.workload === `${workload}:${mode}`,
);
assert.equal(samples.length, 3);
const sum = (select) =>
samples.reduce((total, sample) => total + select(sample), 0);
const successes = sum((sample) => sample.successes);
const errors = sum((sample) => sample.errors);
const seconds = sum((sample) => sample.elapsed_seconds);
const aggregate = {
workload,
mode,
successes,
errors,
rps: successes / seconds,
mean_latency_ms:
sum((sample) => sample.latency_ms.mean * sample.successes) / successes,
mean_p95_ms: sum((sample) => sample.latency_ms.p95) / samples.length,
sql_per_request:
sum((sample) => sample.sql_queries) / (successes + errors),
cpu_percent_one_core:
sum(
(sample) =>
sample.process_cpu_percent_one_core * sample.elapsed_seconds,
) / seconds,
};
aggregates.push(aggregate);
lines.push(
`| ${label} | ${mode === 'redis' ? 'Redis' : 'Без Redis'} | ${aggregate.rps.toFixed(3)} | ${aggregate.mean_latency_ms.toFixed(3)} | ${aggregate.mean_p95_ms.toFixed(3)} | ${aggregate.sql_per_request.toFixed(3)} | ${aggregate.cpu_percent_one_core.toFixed(2)} | ${errors} |`,
);
}
}
lines.push(
'',
'В этом прогоне кэш полезен для прогретого чтения истории: выше пропускная способность, ниже средняя задержка и меньше SQL-вызовов. Для холодной волны и записи преимущество по времени не показано. Первый повтор с Redis значительно медленнее остальных; средние включают его, поэтому вывод о стабильной скорости записи ограничен. Это сравнение режимов кэша на текущей реализации, а не отдельное измерение цены PostgreSQL-эпох.',
'',
'Нагрузка: 10 000 сообщений, первая страница limit=50, конкурентность 16, 3 с прогрева и 10 с чтения на режим/повтор, холодная волна 32 запроса, 512 записей в новый чат на режим/повтор. Реальный HTTP по loopback; синтетические пользователи и сессии обходят хеширование паролей. Проверены 512 сообщений и эпоха 512 после каждого сценария записи, а также чтение истории после записи.',
'',
'Оба API и генератор HTTP работают в одном Node-процессе. CPU включает генератор, API и инструментацию; RSS общий и не показывает разницу памяти между режимами. PostgreSQL и Redis разделяют Docker-host с другими сервисами. Это короткая нагрузка, не предел производительности; холодные SQL-данные не имитируются. Уменьшение SQL/HTTP показывает снижение числа запросов, но не доказывает пропорциональное снижение CPU PostgreSQL. Цена самих эпох отдельно не измеряется.',
'',
'Команда повторения:',
'',
'```powershell',
'docker compose -f compose.yaml -f test/compose.benchmark.yaml run --rm --no-deps --pull never benchmark node test/cache-benchmark.mjs after /results/redis-comparison.json --compare-redis',
'node test/redis-comparison-summary.mjs',
'```',
'',
'Сырые результаты и хеши измеренных модулей: [redis-comparison.json](redis-comparison.json).',
);
await writeFile(
'evidence/redis-comparison-summary.md',
lines.join('\n') + '\n',
);
await writeFile(
'evidence/redis-comparison-averages.json',
JSON.stringify(aggregates, null, 2) + '\n',
);
console.log(lines.slice(0, 16).join('\n'));
+444
View File
@@ -0,0 +1,444 @@
import 'dotenv/config';
import assert from 'node:assert/strict';
import { randomBytes, randomUUID } from 'node:crypto';
import { createRequire } from 'node:module';
import { performance } from 'node:perf_hooks';
import test from 'node:test';
import { faultProxy } from './helpers/fault-proxy.mjs';
const require = createRequire(import.meta.url);
const Redis = require('ioredis').default;
const { NestFactory } = require('@nestjs/core');
const { AppModule } = require('../dist/app.module');
const { PrismaService } = require('../dist/prisma/prisma.service');
const {
createFastifyAdapter,
exposeRequestIdHeader,
} = require('../dist/common/http/fastify-adapter');
const pause = (milliseconds) =>
new Promise((resolve) => setTimeout(resolve, milliseconds));
test(
'Auth and dependency failures remain bounded and fail closed',
{ timeout: 120000 },
async (t) => {
assert.ok(
process.env.DATABASE_URL,
'Set DATABASE_URL and apply migrations',
);
assert.ok(
process.env.CACHE_URL,
'Set CACHE_URL to a running Redis instance',
);
const original = {
DATABASE_URL: process.env.DATABASE_URL,
CACHE_URL: process.env.CACHE_URL,
CACHE_PREFIX: process.env.CACHE_PREFIX,
SESSION_TTL_SECONDS: process.env.SESSION_TTL_SECONDS,
};
const prefix = `test:resilience:${randomUUID()}`;
const db = await faultProxy(original.DATABASE_URL, 5432);
const redis = await faultProxy(original.CACHE_URL, 6379);
Object.assign(process.env, {
DATABASE_URL: db.url,
CACHE_URL: redis.url,
CACHE_PREFIX: prefix,
SESSION_TTL_SECONDS: '2',
});
const observer = new Redis(original.CACHE_URL, {
connectTimeout: 1000,
commandTimeout: 1000,
maxRetriesPerRequest: 0,
});
observer.on('error', () => {});
let app;
let prisma;
let userId;
let chatId;
t.after(async () => {
db.setMode('online');
redis.setMode('online');
try {
if (prisma) {
if (chatId) await prisma.chat.deleteMany({ where: { id: chatId } });
if (userId) await prisma.user.deleteMany({ where: { id: userId } });
}
let cursor = '0';
do {
const [next, keys] = await observer.scan(
cursor,
'MATCH',
`${prefix}:*`,
'COUNT',
100,
);
cursor = next;
if (keys.length) await observer.del(...keys);
} while (cursor !== '0');
} finally {
observer.disconnect();
if (app) await app.close();
await db.close();
await redis.close();
for (const [key, value] of Object.entries(original)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
}
});
await observer.ping();
app = await NestFactory.create(AppModule, createFastifyAdapter(), {
logger: false,
abortOnError: false,
});
exposeRequestIdHeader(app);
await app.init();
const server = app.getHttpAdapter().getInstance();
await server.ready();
prisma = app.get(PrismaService);
let token;
const request = async (
method,
url,
{
payload,
authorization = token ? `Bearer ${token}` : undefined,
headers = {},
budget = 8000,
} = {},
) => {
const start = performance.now();
let timer;
try {
const response = await Promise.race([
server.inject({
method,
url,
headers: {
...headers,
...(method === 'POST' && /\/messages$/.test(url)
? { 'idempotency-key': randomUUID() }
: {}),
...(authorization ? { authorization } : {}),
},
...(payload === undefined ? {} : { payload }),
}),
new Promise((_, reject) => {
timer = setTimeout(
() => reject(new Error(`${method} ${url} exceeded ${budget}ms`)),
budget,
);
}),
]);
assert.ok(
performance.now() - start < budget,
'request exceeded its deadline',
);
return response;
} finally {
clearTimeout(timer);
}
};
const errorResponse = (response, status, code, challenge) => {
assert.equal(
response.statusCode,
status,
`${response.statusCode}: ${response.body}`,
);
const body = response.json();
assert.deepEqual(Object.keys(body), ['error']);
assert.deepEqual(Object.keys(body.error).sort(), [
'code',
'message',
'request_id',
]);
assert.equal(body.error.code, code);
assert.equal(body.error.request_id, response.headers['x-request-id']);
assert.ok(body.error.request_id);
assert.ok(body.error.message.length <= 200);
for (const secret of [
original.DATABASE_URL,
original.CACHE_URL,
token,
].filter(Boolean))
assert.ok(!response.body.includes(secret));
assert.ok(
!/Prisma|SELECT |INSERT |stack|ECONN|postgresql:\/\//i.test(
response.body,
),
'error leaks implementation details',
);
if (challenge)
assert.equal(response.headers['www-authenticate'], challenge);
};
const username = `fault_${randomUUID().replaceAll('-', '').slice(0, 20)}`;
const password = 'Synthetic-test-password:🔐123';
const basic = (secret) =>
`Basic ${Buffer.from(`${username}:${secret}`, 'utf8').toString('base64')}`;
const login = (secret = password) =>
request('POST', '/api/v1/auth/sessions', {
authorization: basic(secret),
});
const registered = await request('POST', '/api/v1/users', {
payload: { username, display_name: 'Fault test', password },
budget: 15000,
});
assert.equal(registered.statusCode, 201);
userId = registered.json().id;
await t.test('incorrect password is 401 with Basic challenge', async () => {
const wrong = await login('wrong-password');
errorResponse(
wrong,
401,
'unauthorized',
'Basic realm="messenger", charset="UTF-8"',
);
assert.equal(await prisma.session.count({ where: { userId } }), 0);
});
await t.test(
'SESSION_TTL_SECONDS=2 expires without renewing on reads',
async () => {
const response = await login();
assert.equal(response.statusCode, 201);
token = response.json().token;
const stored = await prisma.session.findFirstOrThrow({
where: { userId },
});
assert.ok(
Math.abs(
stored.expiresAt.getTime() - stored.createdAt.getTime() - 2000,
) < 500,
);
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
await pause(
Math.max(0, Date.parse(response.json().expires_at) - Date.now()) +
100,
);
errorResponse(
await request('GET', '/api/v1/users/me'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
errorResponse(
await request('DELETE', '/api/v1/auth/sessions/current'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
},
);
await t.test(
'logout revokes immediately, including with a cached profile',
async () => {
const response = await login();
assert.equal(response.statusCode, 201);
token = response.json().token;
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
const logout = await request('DELETE', '/api/v1/auth/sessions/current');
assert.equal(logout.statusCode, 204);
assert.equal(logout.body, '');
errorResponse(
await request('GET', '/api/v1/users/me'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
errorResponse(
await request('DELETE', '/api/v1/auth/sessions/current'),
401,
'unauthorized',
'Bearer realm="messenger"',
);
},
);
token = randomBytes(32).toString('base64url');
const { createHash } = await import('node:crypto');
await prisma.session.create({
data: {
userId,
tokenHash: createHash('sha256').update(token).digest('hex'),
expiresAt: new Date(Date.now() + 600000),
},
});
const chat = await request('POST', '/api/v1/chats', {
payload: { title: 'Fault isolation', member_ids: [userId] },
});
assert.equal(chat.statusCode, 201);
chatId = chat.json().id;
const history = `/api/v1/chats/${chatId}/messages`;
assert.equal(
(await request('POST', history, { payload: { text: 'Before fault' } }))
.statusCode,
201,
);
assert.equal((await request('GET', history)).json().items.length, 1);
const anonymousChecks = async () => {
for (const authorization of ['', 'Bearer invalid', basic(password)]) {
const headers = { 'x-user-id': userId };
errorResponse(
await request('GET', history, { authorization, headers }),
401,
'unauthorized',
'Bearer realm="messenger"',
);
errorResponse(
await request('POST', history, {
authorization,
headers,
payload: { text: 'Must not be stored' },
}),
401,
'unauthorized',
'Bearer realm="messenger"',
);
}
};
await t.test(
'Redis disconnect: SQL fallback and anonymous callers remain unauthorized',
async () => {
redis.setMode('cut');
await anonymousChecks();
assert.equal(
(await request('GET', history, { budget: 3000 })).statusCode,
200,
);
assert.equal(
(
await request('POST', history, {
payload: { text: 'Redis offline' },
budget: 3000,
})
).statusCode,
201,
);
assert.equal(
(await request('GET', history, { budget: 3000 })).json().items.length,
2,
);
redis.setMode('online');
await pause(1300);
},
);
await t.test('Redis hangs: command timeout bypasses cache', async () => {
redis.setMode('hang');
try {
assert.equal(
(await request('GET', `${history}?limit=10`, { budget: 3000 }))
.statusCode,
200,
);
assert.equal(
(
await request('POST', history, {
payload: { text: 'Redis blackhole' },
budget: 3000,
})
).statusCode,
201,
);
await anonymousChecks();
} finally {
redis.setMode('cut');
redis.setMode('online');
}
await pause(1300);
});
await t.test(
'PostgreSQL disconnect: bounded 503, no cached authorization bypass',
async () => {
db.setMode('cut');
try {
await anonymousChecks();
errorResponse(
await request('GET', '/api/v1/users/me'),
503,
'unavailable',
);
errorResponse(await request('GET', history), 503, 'unavailable');
errorResponse(
await request('POST', history, { payload: { text: 'DB offline' } }),
503,
'unavailable',
);
errorResponse(
await request('DELETE', '/api/v1/auth/sessions/current'),
503,
'unavailable',
);
errorResponse(await login(), 503, 'unavailable');
errorResponse(
await request('GET', '/health/ready'),
503,
'unavailable',
);
assert.equal((await request('GET', '/health/live')).statusCode, 200);
} finally {
db.setMode('online');
}
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
},
);
await t.test(
'PostgreSQL hangs on an established connection: client query timeout',
async () => {
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
db.setMode('hang');
try {
await anonymousChecks();
errorResponse(await request('GET', history), 503, 'unavailable');
} finally {
db.setMode('cut');
db.setMode('online');
}
assert.equal(
(await request('GET', '/api/v1/users/me')).statusCode,
200,
);
},
);
await t.test(
'both services unavailable: fail closed and recover confirmed data',
async () => {
db.setMode('cut');
redis.setMode('cut');
try {
await anonymousChecks();
errorResponse(await request('GET', history), 503, 'unavailable');
errorResponse(
await request('GET', history, {
authorization: `Bearer ${randomBytes(32).toString('base64url')}`,
}),
503,
'unavailable',
);
} finally {
db.setMode('online');
redis.setMode('online');
}
const recovered = await request('GET', `${history}?limit=100`);
assert.equal(recovered.statusCode, 200);
assert.deepEqual(
recovered.json().items.map((message) => message.text),
['Before fault', 'Redis offline', 'Redis blackhole'],
);
assert.equal(await prisma.message.count({ where: { chatId } }), 3);
assert.equal((await request('GET', '/health/ready')).statusCode, 200);
},
);
},
);
+29
View File
@@ -0,0 +1,29 @@
import importlib.util
from pathlib import Path
import unittest
spec = importlib.util.spec_from_file_location('saturation', Path(__file__).resolve().parents[1]/'scripts/run-saturation.py')
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
class ClassificationTest(unittest.TestCase):
def classify(self, attempted=1000, dropped=0, errors=0, p95=500):
return module.classify({'measurement': {'scheduled_slots': 1000,
'client_dropped_slots': dropped, 'all': {'attempted': attempted, 'errors': errors}},
'all_latency_ms': {'p95': p95}})['classification']
def test_delivered_healthy_workload_passes(self):
self.assertEqual(self.classify(), 'pass')
def test_latency_or_errors_break_service_criterion(self):
self.assertEqual(self.classify(p95=1500), 'slo_fail')
self.assertEqual(self.classify(errors=20), 'slo_fail')
def test_underfed_workload_is_not_a_server_capacity_measurement(self):
self.assertEqual(self.classify(attempted=800, dropped=200), 'generator_limited')
self.assertEqual(self.classify(attempted=800, dropped=200, p95=3000), 'generator_limited')
if __name__ == '__main__':
unittest.main()