forked from mtusi2026/mtusi2026lab3
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
403 lines
17 KiB
JavaScript
403 lines
17 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Synthetic lab3 scaling benchmark. Runs against an already configured stack.
|
|
* The client is outside the Docker resource budget; no automatic POST retries.
|
|
*/
|
|
|
|
import { execFile as execFileCallback } from 'node:child_process';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { dirname } from 'node:path';
|
|
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
|
import { promisify } from 'node:util';
|
|
import { randomUUID } from 'node:crypto';
|
|
|
|
const execFile = promisify(execFileCallback);
|
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
function options(argv) {
|
|
const defaults = {
|
|
'base-url': 'http://127.0.0.1:8080',
|
|
label: 'unlabelled',
|
|
rate: '100',
|
|
'duration-seconds': '30',
|
|
'warmup-seconds': '5',
|
|
'max-in-flight': '64',
|
|
'read-percent': '80',
|
|
'seed-messages': '100',
|
|
'history-limit': '50',
|
|
'timeout-ms': '5000',
|
|
'docker-stats-interval-seconds': '2',
|
|
};
|
|
const allowed = new Set([...Object.keys(defaults), 'output', 'docker-stats', 'expected-instances']);
|
|
const values = { ...defaults };
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const name = argv[i];
|
|
if (name === '--help') {
|
|
console.log(`Usage: node scripts/benchmark-scaling.mjs [options]
|
|
--base-url URL Nginx URL (default: http://127.0.0.1:8080)
|
|
--label NAME Configuration label for later comparisons
|
|
--rate N Scheduled HTTP requests per second (default: 100)
|
|
--duration-seconds N Measured phase (default: 30)
|
|
--warmup-seconds N Read-only warmup (default: 5)
|
|
--max-in-flight N Client-side outstanding request cap (default: 64)
|
|
--read-percent N Percentage of history GETs (default: 80)
|
|
--seed-messages N Initial messages in a fresh chat (default: 100)
|
|
--history-limit N Messages per GET page (default: 50)
|
|
--timeout-ms N Per-request deadline (default: 5000)
|
|
--docker-stats Sample CPU/RAM of current Compose containers
|
|
--docker-stats-interval-seconds N Sampling interval (default: 2)
|
|
--expected-instances N Require N distinct X-Instance-Id values
|
|
--output PATH Save JSON result; stdout always prints summary
|
|
`);
|
|
process.exit(0);
|
|
}
|
|
if (!name.startsWith('--') || !allowed.has(name.slice(2))) {
|
|
throw new Error(`unknown option: ${name}`);
|
|
}
|
|
const key = name.slice(2);
|
|
if (key === 'docker-stats') {
|
|
values[key] = true;
|
|
continue;
|
|
}
|
|
const value = argv[++i];
|
|
if (value === undefined || value.startsWith('--')) {
|
|
throw new Error(`missing value for ${name}`);
|
|
}
|
|
values[key] = value;
|
|
}
|
|
const number = (key, min, max) => {
|
|
const value = Number(values[key]);
|
|
if (!Number.isFinite(value) || value < min || value > max) {
|
|
throw new Error(`${key} must be ${min}..${max}`);
|
|
}
|
|
return value;
|
|
};
|
|
const parsed = {
|
|
baseUrl: values['base-url'].replace(/\/$/, ''),
|
|
label: values.label,
|
|
rate: number('rate', 1, 1000),
|
|
durationSeconds: number('duration-seconds', 1, 3600),
|
|
warmupSeconds: number('warmup-seconds', 0, 600),
|
|
maxInFlight: number('max-in-flight', 1, 4096),
|
|
readPercent: number('read-percent', 0, 100),
|
|
seedMessages: number('seed-messages', 0, 10000),
|
|
historyLimit: number('history-limit', 1, 100),
|
|
timeoutMs: number('timeout-ms', 100, 120000),
|
|
dockerStats: Boolean(values['docker-stats']),
|
|
dockerStatsIntervalSeconds: number('docker-stats-interval-seconds', 0.5, 60),
|
|
expectedInstances: values['expected-instances'] === undefined
|
|
? null : Number(values['expected-instances']),
|
|
output: values.output,
|
|
};
|
|
if (!/^https?:\/\//.test(parsed.baseUrl)) {
|
|
throw new Error('base-url must begin with http:// or https://');
|
|
}
|
|
if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(parsed.label)) {
|
|
throw new Error('label must use letters, digits, dot, underscore or dash');
|
|
}
|
|
if (parsed.rate * parsed.durationSeconds > 200000) {
|
|
throw new Error('at most 200000 measured request slots are allowed per run');
|
|
}
|
|
for (const key of ['rate', 'maxInFlight', 'readPercent', 'seedMessages',
|
|
'historyLimit', 'timeoutMs']) {
|
|
if (!Number.isInteger(parsed[key])) throw new Error(`${key} must be an integer`);
|
|
}
|
|
if (parsed.expectedInstances !== null &&
|
|
(!Number.isInteger(parsed.expectedInstances) || parsed.expectedInstances < 1 ||
|
|
parsed.expectedInstances > 16)) {
|
|
throw new Error('expected-instances must be an integer from 1 to 16');
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
async function api(baseUrl, method, path, { body, authorization, key, timeoutMs } = {}) {
|
|
const headers = { Accept: 'application/json' };
|
|
if (authorization) headers.Authorization = authorization;
|
|
if (key) headers['Idempotency-Key'] = key;
|
|
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
|
const response = await fetch(baseUrl + path, {
|
|
method,
|
|
headers,
|
|
body: body === undefined ? undefined : JSON.stringify(body),
|
|
signal: AbortSignal.timeout(timeoutMs ?? 10000),
|
|
});
|
|
const payload = await response.json().catch(() => null);
|
|
return { status: response.status, payload, instance: response.headers.get('x-instance-id') };
|
|
}
|
|
|
|
async function bootstrap(config) {
|
|
const suffix = randomUUID().replaceAll('-', '').slice(0, 12);
|
|
const password = `T3st-${randomUUID()}`;
|
|
const users = [];
|
|
for (const label of ['alice', 'bob']) {
|
|
const username = `bench_${label}_${suffix}`;
|
|
const created = await api(config.baseUrl, 'POST', '/api/v1/users', {
|
|
body: { username, display_name: `Benchmark ${label}`, password },
|
|
});
|
|
if (created.status !== 201 || !created.payload?.id) {
|
|
throw new Error(`setup: user registration returned ${created.status}`);
|
|
}
|
|
const basic = Buffer.from(`${username}:${password}`).toString('base64');
|
|
const session = await api(config.baseUrl, 'POST', '/api/v1/auth/sessions', {
|
|
authorization: `Basic ${basic}`,
|
|
});
|
|
if (session.status !== 201 || !session.payload?.token) {
|
|
throw new Error(`setup: session creation returned ${session.status}`);
|
|
}
|
|
users.push({ id: created.payload.id, token: session.payload.token });
|
|
}
|
|
const authorization = `Bearer ${users[0].token}`;
|
|
const chat = await api(config.baseUrl, 'POST', '/api/v1/chats', {
|
|
body: { title: 'Scaling benchmark', member_ids: users.map((user) => user.id) },
|
|
authorization,
|
|
});
|
|
if (chat.status !== 201 || !chat.payload?.id) {
|
|
throw new Error(`setup: chat creation returned ${chat.status}`);
|
|
}
|
|
const path = `/api/v1/chats/${chat.payload.id}/messages`;
|
|
for (let index = 0; index < config.seedMessages; index += 1) {
|
|
const seeded = await api(config.baseUrl, 'POST', path, {
|
|
body: { text: `seed-${suffix}-${index}` }, authorization, key: randomUUID(),
|
|
});
|
|
if (seeded.status !== 201 || !seeded.payload?.id) {
|
|
throw new Error(`setup: seed message ${index + 1} returned ${seeded.status}`);
|
|
}
|
|
}
|
|
return { authorization, path };
|
|
}
|
|
|
|
function percentile(sorted, fraction) {
|
|
if (!sorted.length) return null;
|
|
return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(3));
|
|
}
|
|
|
|
function summarize(events, durationSeconds, slots, dropped, late) {
|
|
const groups = {};
|
|
for (const kind of ['read', 'write', 'all']) {
|
|
const chosen = kind === 'all' ? events : events.filter((event) => event.kind === kind);
|
|
const successes = chosen.filter((event) => event.ok);
|
|
const latencies = successes.map((event) => event.latency_ms).sort((a, b) => a - b);
|
|
const statuses = {};
|
|
const errors = {};
|
|
const instances = {};
|
|
for (const event of chosen) {
|
|
const status = event.status === null ? 'network' : String(event.status);
|
|
statuses[status] = (statuses[status] ?? 0) + 1;
|
|
if (!event.ok) errors[event.error] = (errors[event.error] ?? 0) + 1;
|
|
if (event.instance) instances[event.instance] = (instances[event.instance] ?? 0) + 1;
|
|
}
|
|
groups[kind] = {
|
|
attempted: chosen.length,
|
|
successful: successes.length,
|
|
errors: chosen.length - successes.length,
|
|
successful_rps: Number((successes.length / durationSeconds).toFixed(3)),
|
|
status_counts: statuses,
|
|
error_counts: errors,
|
|
instance_counts: instances,
|
|
successful_latency_ms: {
|
|
p50: percentile(latencies, 0.5),
|
|
p95: percentile(latencies, 0.95),
|
|
p99: percentile(latencies, 0.99),
|
|
},
|
|
};
|
|
}
|
|
return { scheduled_slots: slots, client_dropped_slots: dropped,
|
|
scheduler_late_slots: late, ...groups };
|
|
}
|
|
|
|
async function runPhase(config, context, durationSeconds, readPercent) {
|
|
const slots = Math.floor(config.rate * durationSeconds);
|
|
const intervalMs = 1000 / config.rate;
|
|
const started = performance.now();
|
|
const startedEpochMs = Date.now();
|
|
const events = [];
|
|
const pending = new Set();
|
|
let dropped = 0;
|
|
let late = 0;
|
|
for (let sequence = 0; sequence < slots; sequence += 1) {
|
|
const scheduledAt = started + sequence * intervalMs;
|
|
const waitMs = scheduledAt - performance.now();
|
|
if (waitMs > 0) await sleep(waitMs);
|
|
const actualAt = performance.now();
|
|
if (actualAt - scheduledAt > intervalMs) {
|
|
late += 1;
|
|
dropped += 1;
|
|
continue;
|
|
}
|
|
if (pending.size >= config.maxInFlight) {
|
|
dropped += 1;
|
|
continue;
|
|
}
|
|
const kind = ((sequence * 37 + 17) % 100) < readPercent ? 'read' : 'write';
|
|
const task = (async () => {
|
|
let status = null;
|
|
let instance = null;
|
|
let error = null;
|
|
try {
|
|
const result = kind === 'read'
|
|
? await api(config.baseUrl, 'GET', `${context.path}?limit=${config.historyLimit}`, {
|
|
authorization: context.authorization, timeoutMs: config.timeoutMs,
|
|
})
|
|
: await api(config.baseUrl, 'POST', context.path, {
|
|
authorization: context.authorization, timeoutMs: config.timeoutMs,
|
|
key: randomUUID(), body: { text: `bench-${sequence}-${randomUUID()}` },
|
|
});
|
|
status = result.status;
|
|
instance = result.instance;
|
|
if (status !== (kind === 'read' ? 200 : 201)) error = `http_${status}`;
|
|
else if (!instance) error = 'missing_instance_id';
|
|
else if (kind === 'read' && !Array.isArray(result.payload?.items)) error = 'invalid_page';
|
|
else if (kind === 'write' && !result.payload?.id) error = 'invalid_message';
|
|
} catch (caught) {
|
|
error = caught?.name === 'TimeoutError' ? 'timeout' : 'network_or_parse';
|
|
}
|
|
events.push({ sequence, kind, ok: error === null, status, error, instance,
|
|
scheduled_ms: Number((scheduledAt - started).toFixed(3)),
|
|
start_lag_ms: Number((actualAt - scheduledAt).toFixed(3)),
|
|
latency_ms: Number((performance.now() - actualAt).toFixed(3)) });
|
|
})();
|
|
pending.add(task);
|
|
void task.finally(() => pending.delete(task));
|
|
}
|
|
const scheduledEnd = started + durationSeconds * 1000;
|
|
if (performance.now() < scheduledEnd) await sleep(scheduledEnd - performance.now());
|
|
await Promise.all(pending);
|
|
return { events, started_epoch_ms: startedEpochMs,
|
|
scheduled_end_epoch_ms: startedEpochMs + durationSeconds * 1000,
|
|
completed_epoch_ms: Date.now(),
|
|
summary: summarize(events, durationSeconds, slots, dropped, late),
|
|
drain_seconds: Number(((performance.now() - scheduledEnd) / 1000).toFixed(3)) };
|
|
}
|
|
|
|
function bytesToMiB(value) {
|
|
const match = /^([\d.]+)\s*([KMGT]?i?B)$/i.exec(value.trim());
|
|
if (!match) return null;
|
|
const scale = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12,
|
|
KIB: 1024, MIB: 1024 ** 2, GIB: 1024 ** 3, TIB: 1024 ** 4 };
|
|
return Number((Number(match[1]) * scale[match[2].toUpperCase()] / 1024 ** 2).toFixed(3));
|
|
}
|
|
|
|
async function dockerSampler(intervalSeconds) {
|
|
const { stdout } = await execFile('docker', ['compose', 'ps', '-q'], {
|
|
timeout: 15000, maxBuffer: 1024 * 1024,
|
|
});
|
|
const ids = stdout.trim().split(/\s+/).filter(Boolean);
|
|
if (!ids.length) throw new Error('docker-stats: no running Compose containers');
|
|
let stopped = false;
|
|
const samples = [];
|
|
const errors = [];
|
|
const task = (async () => {
|
|
while (!stopped) {
|
|
const started = performance.now();
|
|
try {
|
|
const result = await execFile('docker',
|
|
['stats', '--no-stream', '--format', '{{json .}}', ...ids],
|
|
{ timeout: 15000, maxBuffer: 1024 * 1024 });
|
|
for (const line of result.stdout.trim().split(/\r?\n/).filter(Boolean)) {
|
|
const row = JSON.parse(line);
|
|
samples.push({ time_ms: Number(started.toFixed(3)), container: row.Name,
|
|
cpu_percent: Number(row.CPUPerc.replace('%', '')),
|
|
memory_mib: bytesToMiB(row.MemUsage.split('/')[0]),
|
|
memory_limit_mib: bytesToMiB(row.MemUsage.split('/')[1]),
|
|
pids: Number(row.PIDs) });
|
|
}
|
|
} catch (error) {
|
|
errors.push(error?.code || error?.name || 'docker_stats_error');
|
|
}
|
|
const waitMs = Math.max(0, intervalSeconds * 1000 - (performance.now() - started));
|
|
if (!stopped && waitMs) await sleep(waitMs);
|
|
}
|
|
})();
|
|
return { stop: async () => { stopped = true; await task; return { samples, errors }; } };
|
|
}
|
|
|
|
async function main() {
|
|
const config = options(process.argv.slice(2));
|
|
const context = await bootstrap(config);
|
|
const warmup = config.warmupSeconds
|
|
? await runPhase(config, context, config.warmupSeconds, 100)
|
|
: null;
|
|
const sampler = config.dockerStats
|
|
? await dockerSampler(config.dockerStatsIntervalSeconds)
|
|
: null;
|
|
const delay = monitorEventLoopDelay({ resolution: 20 });
|
|
delay.enable();
|
|
const cpuBefore = process.cpuUsage();
|
|
const memoryBefore = process.memoryUsage().rss;
|
|
let measured;
|
|
let dockerStats = null;
|
|
try {
|
|
measured = await runPhase(config, context, config.durationSeconds, config.readPercent);
|
|
} finally {
|
|
if (sampler) dockerStats = await sampler.stop();
|
|
delay.disable();
|
|
}
|
|
const cpu = process.cpuUsage(cpuBefore);
|
|
const result = {
|
|
label: config.label,
|
|
base_url: config.baseUrl,
|
|
profile: {
|
|
arrival_model: 'scheduled open loop; late or over-capacity slots dropped',
|
|
offered_rps: config.rate,
|
|
duration_seconds: config.durationSeconds,
|
|
warmup_seconds: config.warmupSeconds,
|
|
warmup_profile: 'read only',
|
|
read_percent: config.readPercent,
|
|
write_percent: 100 - config.readPercent,
|
|
initial_messages: config.seedMessages,
|
|
history_limit: config.historyLimit,
|
|
max_in_flight: config.maxInFlight,
|
|
timeout_ms: config.timeoutMs,
|
|
automatic_retries: false,
|
|
expected_instances: config.expectedInstances,
|
|
},
|
|
warmup: warmup?.summary ?? null,
|
|
measurement: measured.summary,
|
|
drain_seconds: measured.drain_seconds,
|
|
timing: {
|
|
measurement_started_epoch_ms: measured.started_epoch_ms,
|
|
measurement_scheduled_end_epoch_ms: measured.scheduled_end_epoch_ms,
|
|
measurement_completed_epoch_ms: measured.completed_epoch_ms,
|
|
},
|
|
client: {
|
|
cpu_seconds: Number(((cpu.user + cpu.system) / 1e6).toFixed(3)),
|
|
rss_before_mib: Number((memoryBefore / 1024 ** 2).toFixed(3)),
|
|
rss_after_mib: Number((process.memoryUsage().rss / 1024 ** 2).toFixed(3)),
|
|
event_loop_delay_ms: {
|
|
mean: Number((delay.mean / 1e6).toFixed(3)),
|
|
p99: Number((delay.percentile(99) / 1e6).toFixed(3)),
|
|
},
|
|
},
|
|
docker_stats: dockerStats,
|
|
request_samples: measured.events,
|
|
notes: [
|
|
'Setup and read-only warmup are excluded from measured RPS.',
|
|
'RPS uses the scheduled measurement window; requests may complete during drain.',
|
|
'HTTP failures, timeouts and client-dropped slots are reported separately.',
|
|
'One chat is used per run; writes increase its history during measurement.',
|
|
'The Node client runs outside Docker CPU/RAM limits and can itself saturate.',
|
|
'A timed-out POST might have committed; this benchmark never retries it.',
|
|
],
|
|
};
|
|
const json = JSON.stringify(result, null, 2) + '\n';
|
|
if (config.output) {
|
|
await mkdir(dirname(config.output), { recursive: true });
|
|
await writeFile(config.output, json, 'utf8');
|
|
}
|
|
console.log(JSON.stringify({ label: result.label, profile: result.profile,
|
|
measurement: result.measurement, drain_seconds: result.drain_seconds,
|
|
client: result.client, docker_stats_samples: dockerStats?.samples.length ?? 0,
|
|
output: config.output ?? null }, null, 2));
|
|
const observed = Object.keys(result.measurement.all.instance_counts).length;
|
|
if (config.expectedInstances !== null && observed !== config.expectedInstances) {
|
|
console.error(`expected ${config.expectedInstances} API instances, observed ${observed}`);
|
|
process.exitCode = 1;
|
|
}
|
|
if (!result.measurement.all.successful) process.exitCode = 1;
|
|
}
|
|
|
|
main().catch((error) => {
|
|
console.error(`benchmark failed: ${error?.message ?? String(error)}`);
|
|
process.exitCode = 1;
|
|
});
|