Files
mtusi2026lab3-fork/scripts/benchmark-scaling.mjs
qrtx c1fec73ccc perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
2026-10-29 11:49:51 +03:00

403 lines
17 KiB
JavaScript

#!/usr/bin/env node
/**
* Synthetic lab3 scaling benchmark. Runs against an already configured stack.
* The client is outside the Docker resource budget; no automatic POST retries.
*/
import { execFile as execFileCallback } from 'node:child_process';
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
import { promisify } from 'node:util';
import { randomUUID } from 'node:crypto';
const execFile = promisify(execFileCallback);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
function options(argv) {
const defaults = {
'base-url': 'http://127.0.0.1:8080',
label: 'unlabelled',
rate: '100',
'duration-seconds': '30',
'warmup-seconds': '5',
'max-in-flight': '64',
'read-percent': '80',
'seed-messages': '100',
'history-limit': '50',
'timeout-ms': '5000',
'docker-stats-interval-seconds': '2',
};
const allowed = new Set([...Object.keys(defaults), 'output', 'docker-stats', 'expected-instances']);
const values = { ...defaults };
for (let i = 0; i < argv.length; i += 1) {
const name = argv[i];
if (name === '--help') {
console.log(`Usage: node scripts/benchmark-scaling.mjs [options]
--base-url URL Nginx URL (default: http://127.0.0.1:8080)
--label NAME Configuration label for later comparisons
--rate N Scheduled HTTP requests per second (default: 100)
--duration-seconds N Measured phase (default: 30)
--warmup-seconds N Read-only warmup (default: 5)
--max-in-flight N Client-side outstanding request cap (default: 64)
--read-percent N Percentage of history GETs (default: 80)
--seed-messages N Initial messages in a fresh chat (default: 100)
--history-limit N Messages per GET page (default: 50)
--timeout-ms N Per-request deadline (default: 5000)
--docker-stats Sample CPU/RAM of current Compose containers
--docker-stats-interval-seconds N Sampling interval (default: 2)
--expected-instances N Require N distinct X-Instance-Id values
--output PATH Save JSON result; stdout always prints summary
`);
process.exit(0);
}
if (!name.startsWith('--') || !allowed.has(name.slice(2))) {
throw new Error(`unknown option: ${name}`);
}
const key = name.slice(2);
if (key === 'docker-stats') {
values[key] = true;
continue;
}
const value = argv[++i];
if (value === undefined || value.startsWith('--')) {
throw new Error(`missing value for ${name}`);
}
values[key] = value;
}
const number = (key, min, max) => {
const value = Number(values[key]);
if (!Number.isFinite(value) || value < min || value > max) {
throw new Error(`${key} must be ${min}..${max}`);
}
return value;
};
const parsed = {
baseUrl: values['base-url'].replace(/\/$/, ''),
label: values.label,
rate: number('rate', 1, 1000),
durationSeconds: number('duration-seconds', 1, 3600),
warmupSeconds: number('warmup-seconds', 0, 600),
maxInFlight: number('max-in-flight', 1, 4096),
readPercent: number('read-percent', 0, 100),
seedMessages: number('seed-messages', 0, 10000),
historyLimit: number('history-limit', 1, 100),
timeoutMs: number('timeout-ms', 100, 120000),
dockerStats: Boolean(values['docker-stats']),
dockerStatsIntervalSeconds: number('docker-stats-interval-seconds', 0.5, 60),
expectedInstances: values['expected-instances'] === undefined
? null : Number(values['expected-instances']),
output: values.output,
};
if (!/^https?:\/\//.test(parsed.baseUrl)) {
throw new Error('base-url must begin with http:// or https://');
}
if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(parsed.label)) {
throw new Error('label must use letters, digits, dot, underscore or dash');
}
if (parsed.rate * parsed.durationSeconds > 200000) {
throw new Error('at most 200000 measured request slots are allowed per run');
}
for (const key of ['rate', 'maxInFlight', 'readPercent', 'seedMessages',
'historyLimit', 'timeoutMs']) {
if (!Number.isInteger(parsed[key])) throw new Error(`${key} must be an integer`);
}
if (parsed.expectedInstances !== null &&
(!Number.isInteger(parsed.expectedInstances) || parsed.expectedInstances < 1 ||
parsed.expectedInstances > 16)) {
throw new Error('expected-instances must be an integer from 1 to 16');
}
return parsed;
}
async function api(baseUrl, method, path, { body, authorization, key, timeoutMs } = {}) {
const headers = { Accept: 'application/json' };
if (authorization) headers.Authorization = authorization;
if (key) headers['Idempotency-Key'] = key;
if (body !== undefined) headers['Content-Type'] = 'application/json';
const response = await fetch(baseUrl + path, {
method,
headers,
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(timeoutMs ?? 10000),
});
const payload = await response.json().catch(() => null);
return { status: response.status, payload, instance: response.headers.get('x-instance-id') };
}
async function bootstrap(config) {
const suffix = randomUUID().replaceAll('-', '').slice(0, 12);
const password = `T3st-${randomUUID()}`;
const users = [];
for (const label of ['alice', 'bob']) {
const username = `bench_${label}_${suffix}`;
const created = await api(config.baseUrl, 'POST', '/api/v1/users', {
body: { username, display_name: `Benchmark ${label}`, password },
});
if (created.status !== 201 || !created.payload?.id) {
throw new Error(`setup: user registration returned ${created.status}`);
}
const basic = Buffer.from(`${username}:${password}`).toString('base64');
const session = await api(config.baseUrl, 'POST', '/api/v1/auth/sessions', {
authorization: `Basic ${basic}`,
});
if (session.status !== 201 || !session.payload?.token) {
throw new Error(`setup: session creation returned ${session.status}`);
}
users.push({ id: created.payload.id, token: session.payload.token });
}
const authorization = `Bearer ${users[0].token}`;
const chat = await api(config.baseUrl, 'POST', '/api/v1/chats', {
body: { title: 'Scaling benchmark', member_ids: users.map((user) => user.id) },
authorization,
});
if (chat.status !== 201 || !chat.payload?.id) {
throw new Error(`setup: chat creation returned ${chat.status}`);
}
const path = `/api/v1/chats/${chat.payload.id}/messages`;
for (let index = 0; index < config.seedMessages; index += 1) {
const seeded = await api(config.baseUrl, 'POST', path, {
body: { text: `seed-${suffix}-${index}` }, authorization, key: randomUUID(),
});
if (seeded.status !== 201 || !seeded.payload?.id) {
throw new Error(`setup: seed message ${index + 1} returned ${seeded.status}`);
}
}
return { authorization, path };
}
function percentile(sorted, fraction) {
if (!sorted.length) return null;
return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(3));
}
function summarize(events, durationSeconds, slots, dropped, late) {
const groups = {};
for (const kind of ['read', 'write', 'all']) {
const chosen = kind === 'all' ? events : events.filter((event) => event.kind === kind);
const successes = chosen.filter((event) => event.ok);
const latencies = successes.map((event) => event.latency_ms).sort((a, b) => a - b);
const statuses = {};
const errors = {};
const instances = {};
for (const event of chosen) {
const status = event.status === null ? 'network' : String(event.status);
statuses[status] = (statuses[status] ?? 0) + 1;
if (!event.ok) errors[event.error] = (errors[event.error] ?? 0) + 1;
if (event.instance) instances[event.instance] = (instances[event.instance] ?? 0) + 1;
}
groups[kind] = {
attempted: chosen.length,
successful: successes.length,
errors: chosen.length - successes.length,
successful_rps: Number((successes.length / durationSeconds).toFixed(3)),
status_counts: statuses,
error_counts: errors,
instance_counts: instances,
successful_latency_ms: {
p50: percentile(latencies, 0.5),
p95: percentile(latencies, 0.95),
p99: percentile(latencies, 0.99),
},
};
}
return { scheduled_slots: slots, client_dropped_slots: dropped,
scheduler_late_slots: late, ...groups };
}
async function runPhase(config, context, durationSeconds, readPercent) {
const slots = Math.floor(config.rate * durationSeconds);
const intervalMs = 1000 / config.rate;
const started = performance.now();
const startedEpochMs = Date.now();
const events = [];
const pending = new Set();
let dropped = 0;
let late = 0;
for (let sequence = 0; sequence < slots; sequence += 1) {
const scheduledAt = started + sequence * intervalMs;
const waitMs = scheduledAt - performance.now();
if (waitMs > 0) await sleep(waitMs);
const actualAt = performance.now();
if (actualAt - scheduledAt > intervalMs) {
late += 1;
dropped += 1;
continue;
}
if (pending.size >= config.maxInFlight) {
dropped += 1;
continue;
}
const kind = ((sequence * 37 + 17) % 100) < readPercent ? 'read' : 'write';
const task = (async () => {
let status = null;
let instance = null;
let error = null;
try {
const result = kind === 'read'
? await api(config.baseUrl, 'GET', `${context.path}?limit=${config.historyLimit}`, {
authorization: context.authorization, timeoutMs: config.timeoutMs,
})
: await api(config.baseUrl, 'POST', context.path, {
authorization: context.authorization, timeoutMs: config.timeoutMs,
key: randomUUID(), body: { text: `bench-${sequence}-${randomUUID()}` },
});
status = result.status;
instance = result.instance;
if (status !== (kind === 'read' ? 200 : 201)) error = `http_${status}`;
else if (!instance) error = 'missing_instance_id';
else if (kind === 'read' && !Array.isArray(result.payload?.items)) error = 'invalid_page';
else if (kind === 'write' && !result.payload?.id) error = 'invalid_message';
} catch (caught) {
error = caught?.name === 'TimeoutError' ? 'timeout' : 'network_or_parse';
}
events.push({ sequence, kind, ok: error === null, status, error, instance,
scheduled_ms: Number((scheduledAt - started).toFixed(3)),
start_lag_ms: Number((actualAt - scheduledAt).toFixed(3)),
latency_ms: Number((performance.now() - actualAt).toFixed(3)) });
})();
pending.add(task);
void task.finally(() => pending.delete(task));
}
const scheduledEnd = started + durationSeconds * 1000;
if (performance.now() < scheduledEnd) await sleep(scheduledEnd - performance.now());
await Promise.all(pending);
return { events, started_epoch_ms: startedEpochMs,
scheduled_end_epoch_ms: startedEpochMs + durationSeconds * 1000,
completed_epoch_ms: Date.now(),
summary: summarize(events, durationSeconds, slots, dropped, late),
drain_seconds: Number(((performance.now() - scheduledEnd) / 1000).toFixed(3)) };
}
function bytesToMiB(value) {
const match = /^([\d.]+)\s*([KMGT]?i?B)$/i.exec(value.trim());
if (!match) return null;
const scale = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12,
KIB: 1024, MIB: 1024 ** 2, GIB: 1024 ** 3, TIB: 1024 ** 4 };
return Number((Number(match[1]) * scale[match[2].toUpperCase()] / 1024 ** 2).toFixed(3));
}
async function dockerSampler(intervalSeconds) {
const { stdout } = await execFile('docker', ['compose', 'ps', '-q'], {
timeout: 15000, maxBuffer: 1024 * 1024,
});
const ids = stdout.trim().split(/\s+/).filter(Boolean);
if (!ids.length) throw new Error('docker-stats: no running Compose containers');
let stopped = false;
const samples = [];
const errors = [];
const task = (async () => {
while (!stopped) {
const started = performance.now();
try {
const result = await execFile('docker',
['stats', '--no-stream', '--format', '{{json .}}', ...ids],
{ timeout: 15000, maxBuffer: 1024 * 1024 });
for (const line of result.stdout.trim().split(/\r?\n/).filter(Boolean)) {
const row = JSON.parse(line);
samples.push({ time_ms: Number(started.toFixed(3)), container: row.Name,
cpu_percent: Number(row.CPUPerc.replace('%', '')),
memory_mib: bytesToMiB(row.MemUsage.split('/')[0]),
memory_limit_mib: bytesToMiB(row.MemUsage.split('/')[1]),
pids: Number(row.PIDs) });
}
} catch (error) {
errors.push(error?.code || error?.name || 'docker_stats_error');
}
const waitMs = Math.max(0, intervalSeconds * 1000 - (performance.now() - started));
if (!stopped && waitMs) await sleep(waitMs);
}
})();
return { stop: async () => { stopped = true; await task; return { samples, errors }; } };
}
async function main() {
const config = options(process.argv.slice(2));
const context = await bootstrap(config);
const warmup = config.warmupSeconds
? await runPhase(config, context, config.warmupSeconds, 100)
: null;
const sampler = config.dockerStats
? await dockerSampler(config.dockerStatsIntervalSeconds)
: null;
const delay = monitorEventLoopDelay({ resolution: 20 });
delay.enable();
const cpuBefore = process.cpuUsage();
const memoryBefore = process.memoryUsage().rss;
let measured;
let dockerStats = null;
try {
measured = await runPhase(config, context, config.durationSeconds, config.readPercent);
} finally {
if (sampler) dockerStats = await sampler.stop();
delay.disable();
}
const cpu = process.cpuUsage(cpuBefore);
const result = {
label: config.label,
base_url: config.baseUrl,
profile: {
arrival_model: 'scheduled open loop; late or over-capacity slots dropped',
offered_rps: config.rate,
duration_seconds: config.durationSeconds,
warmup_seconds: config.warmupSeconds,
warmup_profile: 'read only',
read_percent: config.readPercent,
write_percent: 100 - config.readPercent,
initial_messages: config.seedMessages,
history_limit: config.historyLimit,
max_in_flight: config.maxInFlight,
timeout_ms: config.timeoutMs,
automatic_retries: false,
expected_instances: config.expectedInstances,
},
warmup: warmup?.summary ?? null,
measurement: measured.summary,
drain_seconds: measured.drain_seconds,
timing: {
measurement_started_epoch_ms: measured.started_epoch_ms,
measurement_scheduled_end_epoch_ms: measured.scheduled_end_epoch_ms,
measurement_completed_epoch_ms: measured.completed_epoch_ms,
},
client: {
cpu_seconds: Number(((cpu.user + cpu.system) / 1e6).toFixed(3)),
rss_before_mib: Number((memoryBefore / 1024 ** 2).toFixed(3)),
rss_after_mib: Number((process.memoryUsage().rss / 1024 ** 2).toFixed(3)),
event_loop_delay_ms: {
mean: Number((delay.mean / 1e6).toFixed(3)),
p99: Number((delay.percentile(99) / 1e6).toFixed(3)),
},
},
docker_stats: dockerStats,
request_samples: measured.events,
notes: [
'Setup and read-only warmup are excluded from measured RPS.',
'RPS uses the scheduled measurement window; requests may complete during drain.',
'HTTP failures, timeouts and client-dropped slots are reported separately.',
'One chat is used per run; writes increase its history during measurement.',
'The Node client runs outside Docker CPU/RAM limits and can itself saturate.',
'A timed-out POST might have committed; this benchmark never retries it.',
],
};
const json = JSON.stringify(result, null, 2) + '\n';
if (config.output) {
await mkdir(dirname(config.output), { recursive: true });
await writeFile(config.output, json, 'utf8');
}
console.log(JSON.stringify({ label: result.label, profile: result.profile,
measurement: result.measurement, drain_seconds: result.drain_seconds,
client: result.client, docker_stats_samples: dockerStats?.samples.length ?? 0,
output: config.output ?? null }, null, 2));
const observed = Object.keys(result.measurement.all.instance_counts).length;
if (config.expectedInstances !== null && observed !== config.expectedInstances) {
console.error(`expected ${config.expectedInstances} API instances, observed ${observed}`);
process.exitCode = 1;
}
if (!result.measurement.all.successful) process.exitCode = 1;
}
main().catch((error) => {
console.error(`benchmark failed: ${error?.message ?? String(error)}`);
process.exitCode = 1;
});