#!/usr/bin/env node /** * Synthetic lab3 scaling benchmark. Runs against an already configured stack. * The client is outside the Docker resource budget; no automatic POST retries. */ import { execFile as execFileCallback } from 'node:child_process'; import { mkdir, writeFile } from 'node:fs/promises'; import { dirname } from 'node:path'; import { monitorEventLoopDelay, performance } from 'node:perf_hooks'; import { promisify } from 'node:util'; import { randomUUID } from 'node:crypto'; const execFile = promisify(execFileCallback); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); function options(argv) { const defaults = { 'base-url': 'http://127.0.0.1:8080', label: 'unlabelled', rate: '100', 'duration-seconds': '30', 'warmup-seconds': '5', 'max-in-flight': '64', 'read-percent': '80', 'seed-messages': '100', 'history-limit': '50', 'timeout-ms': '5000', 'docker-stats-interval-seconds': '2', }; const allowed = new Set([...Object.keys(defaults), 'output', 'docker-stats', 'expected-instances']); const values = { ...defaults }; for (let i = 0; i < argv.length; i += 1) { const name = argv[i]; if (name === '--help') { console.log(`Usage: node scripts/benchmark-scaling.mjs [options] --base-url URL Nginx URL (default: http://127.0.0.1:8080) --label NAME Configuration label for later comparisons --rate N Scheduled HTTP requests per second (default: 100) --duration-seconds N Measured phase (default: 30) --warmup-seconds N Read-only warmup (default: 5) --max-in-flight N Client-side outstanding request cap (default: 64) --read-percent N Percentage of history GETs (default: 80) --seed-messages N Initial messages in a fresh chat (default: 100) --history-limit N Messages per GET page (default: 50) --timeout-ms N Per-request deadline (default: 5000) --docker-stats Sample CPU/RAM of current Compose containers --docker-stats-interval-seconds N Sampling interval (default: 2) --expected-instances N Require N distinct X-Instance-Id values --output PATH Save JSON result; stdout always prints summary `); process.exit(0); } if (!name.startsWith('--') || !allowed.has(name.slice(2))) { throw new Error(`unknown option: ${name}`); } const key = name.slice(2); if (key === 'docker-stats') { values[key] = true; continue; } const value = argv[++i]; if (value === undefined || value.startsWith('--')) { throw new Error(`missing value for ${name}`); } values[key] = value; } const number = (key, min, max) => { const value = Number(values[key]); if (!Number.isFinite(value) || value < min || value > max) { throw new Error(`${key} must be ${min}..${max}`); } return value; }; const parsed = { baseUrl: values['base-url'].replace(/\/$/, ''), label: values.label, rate: number('rate', 1, 1000), durationSeconds: number('duration-seconds', 1, 3600), warmupSeconds: number('warmup-seconds', 0, 600), maxInFlight: number('max-in-flight', 1, 4096), readPercent: number('read-percent', 0, 100), seedMessages: number('seed-messages', 0, 10000), historyLimit: number('history-limit', 1, 100), timeoutMs: number('timeout-ms', 100, 120000), dockerStats: Boolean(values['docker-stats']), dockerStatsIntervalSeconds: number('docker-stats-interval-seconds', 0.5, 60), expectedInstances: values['expected-instances'] === undefined ? null : Number(values['expected-instances']), output: values.output, }; if (!/^https?:\/\//.test(parsed.baseUrl)) { throw new Error('base-url must begin with http:// or https://'); } if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(parsed.label)) { throw new Error('label must use letters, digits, dot, underscore or dash'); } if (parsed.rate * parsed.durationSeconds > 200000) { throw new Error('at most 200000 measured request slots are allowed per run'); } for (const key of ['rate', 'maxInFlight', 'readPercent', 'seedMessages', 'historyLimit', 'timeoutMs']) { if (!Number.isInteger(parsed[key])) throw new Error(`${key} must be an integer`); } if (parsed.expectedInstances !== null && (!Number.isInteger(parsed.expectedInstances) || parsed.expectedInstances < 1 || parsed.expectedInstances > 16)) { throw new Error('expected-instances must be an integer from 1 to 16'); } return parsed; } async function api(baseUrl, method, path, { body, authorization, key, timeoutMs } = {}) { const headers = { Accept: 'application/json' }; if (authorization) headers.Authorization = authorization; if (key) headers['Idempotency-Key'] = key; if (body !== undefined) headers['Content-Type'] = 'application/json'; const response = await fetch(baseUrl + path, { method, headers, body: body === undefined ? undefined : JSON.stringify(body), signal: AbortSignal.timeout(timeoutMs ?? 10000), }); const payload = await response.json().catch(() => null); return { status: response.status, payload, instance: response.headers.get('x-instance-id') }; } async function bootstrap(config) { const suffix = randomUUID().replaceAll('-', '').slice(0, 12); const password = `T3st-${randomUUID()}`; const users = []; for (const label of ['alice', 'bob']) { const username = `bench_${label}_${suffix}`; const created = await api(config.baseUrl, 'POST', '/api/v1/users', { body: { username, display_name: `Benchmark ${label}`, password }, }); if (created.status !== 201 || !created.payload?.id) { throw new Error(`setup: user registration returned ${created.status}`); } const basic = Buffer.from(`${username}:${password}`).toString('base64'); const session = await api(config.baseUrl, 'POST', '/api/v1/auth/sessions', { authorization: `Basic ${basic}`, }); if (session.status !== 201 || !session.payload?.token) { throw new Error(`setup: session creation returned ${session.status}`); } users.push({ id: created.payload.id, token: session.payload.token }); } const authorization = `Bearer ${users[0].token}`; const chat = await api(config.baseUrl, 'POST', '/api/v1/chats', { body: { title: 'Scaling benchmark', member_ids: users.map((user) => user.id) }, authorization, }); if (chat.status !== 201 || !chat.payload?.id) { throw new Error(`setup: chat creation returned ${chat.status}`); } const path = `/api/v1/chats/${chat.payload.id}/messages`; for (let index = 0; index < config.seedMessages; index += 1) { const seeded = await api(config.baseUrl, 'POST', path, { body: { text: `seed-${suffix}-${index}` }, authorization, key: randomUUID(), }); if (seeded.status !== 201 || !seeded.payload?.id) { throw new Error(`setup: seed message ${index + 1} returned ${seeded.status}`); } } return { authorization, path }; } function percentile(sorted, fraction) { if (!sorted.length) return null; return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(3)); } function summarize(events, durationSeconds, slots, dropped, late) { const groups = {}; for (const kind of ['read', 'write', 'all']) { const chosen = kind === 'all' ? events : events.filter((event) => event.kind === kind); const successes = chosen.filter((event) => event.ok); const latencies = successes.map((event) => event.latency_ms).sort((a, b) => a - b); const statuses = {}; const errors = {}; const instances = {}; for (const event of chosen) { const status = event.status === null ? 'network' : String(event.status); statuses[status] = (statuses[status] ?? 0) + 1; if (!event.ok) errors[event.error] = (errors[event.error] ?? 0) + 1; if (event.instance) instances[event.instance] = (instances[event.instance] ?? 0) + 1; } groups[kind] = { attempted: chosen.length, successful: successes.length, errors: chosen.length - successes.length, successful_rps: Number((successes.length / durationSeconds).toFixed(3)), status_counts: statuses, error_counts: errors, instance_counts: instances, successful_latency_ms: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95), p99: percentile(latencies, 0.99), }, }; } return { scheduled_slots: slots, client_dropped_slots: dropped, scheduler_late_slots: late, ...groups }; } async function runPhase(config, context, durationSeconds, readPercent) { const slots = Math.floor(config.rate * durationSeconds); const intervalMs = 1000 / config.rate; const started = performance.now(); const startedEpochMs = Date.now(); const events = []; const pending = new Set(); let dropped = 0; let late = 0; for (let sequence = 0; sequence < slots; sequence += 1) { const scheduledAt = started + sequence * intervalMs; const waitMs = scheduledAt - performance.now(); if (waitMs > 0) await sleep(waitMs); const actualAt = performance.now(); if (actualAt - scheduledAt > intervalMs) { late += 1; dropped += 1; continue; } if (pending.size >= config.maxInFlight) { dropped += 1; continue; } const kind = ((sequence * 37 + 17) % 100) < readPercent ? 'read' : 'write'; const task = (async () => { let status = null; let instance = null; let error = null; try { const result = kind === 'read' ? await api(config.baseUrl, 'GET', `${context.path}?limit=${config.historyLimit}`, { authorization: context.authorization, timeoutMs: config.timeoutMs, }) : await api(config.baseUrl, 'POST', context.path, { authorization: context.authorization, timeoutMs: config.timeoutMs, key: randomUUID(), body: { text: `bench-${sequence}-${randomUUID()}` }, }); status = result.status; instance = result.instance; if (status !== (kind === 'read' ? 200 : 201)) error = `http_${status}`; else if (!instance) error = 'missing_instance_id'; else if (kind === 'read' && !Array.isArray(result.payload?.items)) error = 'invalid_page'; else if (kind === 'write' && !result.payload?.id) error = 'invalid_message'; } catch (caught) { error = caught?.name === 'TimeoutError' ? 'timeout' : 'network_or_parse'; } events.push({ sequence, kind, ok: error === null, status, error, instance, scheduled_ms: Number((scheduledAt - started).toFixed(3)), start_lag_ms: Number((actualAt - scheduledAt).toFixed(3)), latency_ms: Number((performance.now() - actualAt).toFixed(3)) }); })(); pending.add(task); void task.finally(() => pending.delete(task)); } const scheduledEnd = started + durationSeconds * 1000; if (performance.now() < scheduledEnd) await sleep(scheduledEnd - performance.now()); await Promise.all(pending); return { events, started_epoch_ms: startedEpochMs, scheduled_end_epoch_ms: startedEpochMs + durationSeconds * 1000, completed_epoch_ms: Date.now(), summary: summarize(events, durationSeconds, slots, dropped, late), drain_seconds: Number(((performance.now() - scheduledEnd) / 1000).toFixed(3)) }; } function bytesToMiB(value) { const match = /^([\d.]+)\s*([KMGT]?i?B)$/i.exec(value.trim()); if (!match) return null; const scale = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12, KIB: 1024, MIB: 1024 ** 2, GIB: 1024 ** 3, TIB: 1024 ** 4 }; return Number((Number(match[1]) * scale[match[2].toUpperCase()] / 1024 ** 2).toFixed(3)); } async function dockerSampler(intervalSeconds) { const { stdout } = await execFile('docker', ['compose', 'ps', '-q'], { timeout: 15000, maxBuffer: 1024 * 1024, }); const ids = stdout.trim().split(/\s+/).filter(Boolean); if (!ids.length) throw new Error('docker-stats: no running Compose containers'); let stopped = false; const samples = []; const errors = []; const task = (async () => { while (!stopped) { const started = performance.now(); try { const result = await execFile('docker', ['stats', '--no-stream', '--format', '{{json .}}', ...ids], { timeout: 15000, maxBuffer: 1024 * 1024 }); for (const line of result.stdout.trim().split(/\r?\n/).filter(Boolean)) { const row = JSON.parse(line); samples.push({ time_ms: Number(started.toFixed(3)), container: row.Name, cpu_percent: Number(row.CPUPerc.replace('%', '')), memory_mib: bytesToMiB(row.MemUsage.split('/')[0]), memory_limit_mib: bytesToMiB(row.MemUsage.split('/')[1]), pids: Number(row.PIDs) }); } } catch (error) { errors.push(error?.code || error?.name || 'docker_stats_error'); } const waitMs = Math.max(0, intervalSeconds * 1000 - (performance.now() - started)); if (!stopped && waitMs) await sleep(waitMs); } })(); return { stop: async () => { stopped = true; await task; return { samples, errors }; } }; } async function main() { const config = options(process.argv.slice(2)); const context = await bootstrap(config); const warmup = config.warmupSeconds ? await runPhase(config, context, config.warmupSeconds, 100) : null; const sampler = config.dockerStats ? await dockerSampler(config.dockerStatsIntervalSeconds) : null; const delay = monitorEventLoopDelay({ resolution: 20 }); delay.enable(); const cpuBefore = process.cpuUsage(); const memoryBefore = process.memoryUsage().rss; let measured; let dockerStats = null; try { measured = await runPhase(config, context, config.durationSeconds, config.readPercent); } finally { if (sampler) dockerStats = await sampler.stop(); delay.disable(); } const cpu = process.cpuUsage(cpuBefore); const result = { label: config.label, base_url: config.baseUrl, profile: { arrival_model: 'scheduled open loop; late or over-capacity slots dropped', offered_rps: config.rate, duration_seconds: config.durationSeconds, warmup_seconds: config.warmupSeconds, warmup_profile: 'read only', read_percent: config.readPercent, write_percent: 100 - config.readPercent, initial_messages: config.seedMessages, history_limit: config.historyLimit, max_in_flight: config.maxInFlight, timeout_ms: config.timeoutMs, automatic_retries: false, expected_instances: config.expectedInstances, }, warmup: warmup?.summary ?? null, measurement: measured.summary, drain_seconds: measured.drain_seconds, timing: { measurement_started_epoch_ms: measured.started_epoch_ms, measurement_scheduled_end_epoch_ms: measured.scheduled_end_epoch_ms, measurement_completed_epoch_ms: measured.completed_epoch_ms, }, client: { cpu_seconds: Number(((cpu.user + cpu.system) / 1e6).toFixed(3)), rss_before_mib: Number((memoryBefore / 1024 ** 2).toFixed(3)), rss_after_mib: Number((process.memoryUsage().rss / 1024 ** 2).toFixed(3)), event_loop_delay_ms: { mean: Number((delay.mean / 1e6).toFixed(3)), p99: Number((delay.percentile(99) / 1e6).toFixed(3)), }, }, docker_stats: dockerStats, request_samples: measured.events, notes: [ 'Setup and read-only warmup are excluded from measured RPS.', 'RPS uses the scheduled measurement window; requests may complete during drain.', 'HTTP failures, timeouts and client-dropped slots are reported separately.', 'One chat is used per run; writes increase its history during measurement.', 'The Node client runs outside Docker CPU/RAM limits and can itself saturate.', 'A timed-out POST might have committed; this benchmark never retries it.', ], }; const json = JSON.stringify(result, null, 2) + '\n'; if (config.output) { await mkdir(dirname(config.output), { recursive: true }); await writeFile(config.output, json, 'utf8'); } console.log(JSON.stringify({ label: result.label, profile: result.profile, measurement: result.measurement, drain_seconds: result.drain_seconds, client: result.client, docker_stats_samples: dockerStats?.samples.length ?? 0, output: config.output ?? null }, null, 2)); const observed = Object.keys(result.measurement.all.instance_counts).length; if (config.expectedInstances !== null && observed !== config.expectedInstances) { console.error(`expected ${config.expectedInstances} API instances, observed ${observed}`); process.exitCode = 1; } if (!result.measurement.all.successful) process.exitCode = 1; } main().catch((error) => { console.error(`benchmark failed: ${error?.message ?? String(error)}`); process.exitCode = 1; });