forked from mtusi2026/mtusi2026lab3
perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compatibility entry point for the consolidated report figures."""
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
if __name__ == '__main__':
|
||||
helper = Path(__file__).with_name('report-evidence.py')
|
||||
sys.argv = [str(helper), '--plot', *sys.argv[1:]]
|
||||
runpy.run_path(str(helper), run_name='__main__')
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check saved source snapshots, run completion and actual container budgets."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('directory', type=Path)
|
||||
args = parser.parse_args()
|
||||
root = args.directory
|
||||
manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8'))
|
||||
sources = {name: hashlib.sha256((root / 'sources' / name).read_bytes()).hexdigest() == digest
|
||||
for name, digest in manifest['source_sha256'].items()}
|
||||
budgets = {}
|
||||
images = set()
|
||||
for profile, containers in manifest['containers'].items():
|
||||
apis = [c for c in containers if '-api-' in c['name']]
|
||||
images.update(c['image'] for c in apis)
|
||||
budgets[profile] = {'replicas': len(apis), 'cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
|
||||
'ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
|
||||
'total_pool': sum(int(c['pool_max']) for c in apis)}
|
||||
runs = {}
|
||||
groups = Counter()
|
||||
for label in manifest['runs']:
|
||||
path = root / label
|
||||
summary = json.loads((path / 'summary.json').read_text(encoding='utf-8'))
|
||||
execution = json.loads((path / 'execution.json').read_text(encoding='utf-8'))
|
||||
profile = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
|
||||
expected = budgets[profile]['replicas']
|
||||
rate = summary['profile']['offered_rps']
|
||||
if summary['profile'].get('phase') == 'confirmation':
|
||||
groups[f'{profile}/r{rate}'] += 1
|
||||
runs[label] = {'returncode': execution['returncode'],
|
||||
'diagnostics_complete': summary.get('diagnostics_complete', False),
|
||||
'replicas_match': summary['profile']['expected_instances'] == expected,
|
||||
'criterion': summary.get('criterion')}
|
||||
requested = manifest.get('requested_confirmation_rates')
|
||||
complete_groups = all(count == manifest['repetitions'] for count in groups.values())
|
||||
if requested:
|
||||
complete_groups = complete_groups and all(
|
||||
groups[f'{profile}/r{rate}'] == manifest['repetitions']
|
||||
for profile in manifest['profiles'] for rate in requested)
|
||||
checks = {'finished': bool(manifest.get('finished')), 'source_snapshots_match': all(sources.values()),
|
||||
'one_api_image': len(images) == 1, 'all_runs_complete': bool(runs) and all(
|
||||
r['returncode'] == 0 and r['diagnostics_complete'] and r['replicas_match'] for r in runs.values()),
|
||||
'confirmation_groups_complete': bool(groups) and complete_groups,
|
||||
'pool_budget_matches': all(b['total_pool'] == manifest['pool_budget'] for b in budgets.values())}
|
||||
audit = {'checks_pass': all(checks.values()), 'checks': checks, 'budgets': budgets,
|
||||
'source_snapshots': sources, 'confirmation_groups': dict(groups), 'runs': runs}
|
||||
(root / 'audit.json').write_text(json.dumps(audit, indent=2) + '\n', encoding='utf-8')
|
||||
print(json.dumps({'checks_pass': audit['checks_pass'], 'checks': checks, 'budgets': budgets,
|
||||
'runs': len(runs)}, indent=2))
|
||||
if not audit['checks_pass']:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Synthetic lab3 scaling benchmark. Runs against an already configured stack.
|
||||
* The client is outside the Docker resource budget; no automatic POST retries.
|
||||
*/
|
||||
|
||||
import { execFile as execFileCallback } from 'node:child_process';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
||||
import { promisify } from 'node:util';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function options(argv) {
|
||||
const defaults = {
|
||||
'base-url': 'http://127.0.0.1:8080',
|
||||
label: 'unlabelled',
|
||||
rate: '100',
|
||||
'duration-seconds': '30',
|
||||
'warmup-seconds': '5',
|
||||
'max-in-flight': '64',
|
||||
'read-percent': '80',
|
||||
'seed-messages': '100',
|
||||
'history-limit': '50',
|
||||
'timeout-ms': '5000',
|
||||
'docker-stats-interval-seconds': '2',
|
||||
};
|
||||
const allowed = new Set([...Object.keys(defaults), 'output', 'docker-stats', 'expected-instances']);
|
||||
const values = { ...defaults };
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const name = argv[i];
|
||||
if (name === '--help') {
|
||||
console.log(`Usage: node scripts/benchmark-scaling.mjs [options]
|
||||
--base-url URL Nginx URL (default: http://127.0.0.1:8080)
|
||||
--label NAME Configuration label for later comparisons
|
||||
--rate N Scheduled HTTP requests per second (default: 100)
|
||||
--duration-seconds N Measured phase (default: 30)
|
||||
--warmup-seconds N Read-only warmup (default: 5)
|
||||
--max-in-flight N Client-side outstanding request cap (default: 64)
|
||||
--read-percent N Percentage of history GETs (default: 80)
|
||||
--seed-messages N Initial messages in a fresh chat (default: 100)
|
||||
--history-limit N Messages per GET page (default: 50)
|
||||
--timeout-ms N Per-request deadline (default: 5000)
|
||||
--docker-stats Sample CPU/RAM of current Compose containers
|
||||
--docker-stats-interval-seconds N Sampling interval (default: 2)
|
||||
--expected-instances N Require N distinct X-Instance-Id values
|
||||
--output PATH Save JSON result; stdout always prints summary
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
if (!name.startsWith('--') || !allowed.has(name.slice(2))) {
|
||||
throw new Error(`unknown option: ${name}`);
|
||||
}
|
||||
const key = name.slice(2);
|
||||
if (key === 'docker-stats') {
|
||||
values[key] = true;
|
||||
continue;
|
||||
}
|
||||
const value = argv[++i];
|
||||
if (value === undefined || value.startsWith('--')) {
|
||||
throw new Error(`missing value for ${name}`);
|
||||
}
|
||||
values[key] = value;
|
||||
}
|
||||
const number = (key, min, max) => {
|
||||
const value = Number(values[key]);
|
||||
if (!Number.isFinite(value) || value < min || value > max) {
|
||||
throw new Error(`${key} must be ${min}..${max}`);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const parsed = {
|
||||
baseUrl: values['base-url'].replace(/\/$/, ''),
|
||||
label: values.label,
|
||||
rate: number('rate', 1, 1000),
|
||||
durationSeconds: number('duration-seconds', 1, 3600),
|
||||
warmupSeconds: number('warmup-seconds', 0, 600),
|
||||
maxInFlight: number('max-in-flight', 1, 4096),
|
||||
readPercent: number('read-percent', 0, 100),
|
||||
seedMessages: number('seed-messages', 0, 10000),
|
||||
historyLimit: number('history-limit', 1, 100),
|
||||
timeoutMs: number('timeout-ms', 100, 120000),
|
||||
dockerStats: Boolean(values['docker-stats']),
|
||||
dockerStatsIntervalSeconds: number('docker-stats-interval-seconds', 0.5, 60),
|
||||
expectedInstances: values['expected-instances'] === undefined
|
||||
? null : Number(values['expected-instances']),
|
||||
output: values.output,
|
||||
};
|
||||
if (!/^https?:\/\//.test(parsed.baseUrl)) {
|
||||
throw new Error('base-url must begin with http:// or https://');
|
||||
}
|
||||
if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(parsed.label)) {
|
||||
throw new Error('label must use letters, digits, dot, underscore or dash');
|
||||
}
|
||||
if (parsed.rate * parsed.durationSeconds > 200000) {
|
||||
throw new Error('at most 200000 measured request slots are allowed per run');
|
||||
}
|
||||
for (const key of ['rate', 'maxInFlight', 'readPercent', 'seedMessages',
|
||||
'historyLimit', 'timeoutMs']) {
|
||||
if (!Number.isInteger(parsed[key])) throw new Error(`${key} must be an integer`);
|
||||
}
|
||||
if (parsed.expectedInstances !== null &&
|
||||
(!Number.isInteger(parsed.expectedInstances) || parsed.expectedInstances < 1 ||
|
||||
parsed.expectedInstances > 16)) {
|
||||
throw new Error('expected-instances must be an integer from 1 to 16');
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function api(baseUrl, method, path, { body, authorization, key, timeoutMs } = {}) {
|
||||
const headers = { Accept: 'application/json' };
|
||||
if (authorization) headers.Authorization = authorization;
|
||||
if (key) headers['Idempotency-Key'] = key;
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||
const response = await fetch(baseUrl + path, {
|
||||
method,
|
||||
headers,
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(timeoutMs ?? 10000),
|
||||
});
|
||||
const payload = await response.json().catch(() => null);
|
||||
return { status: response.status, payload, instance: response.headers.get('x-instance-id') };
|
||||
}
|
||||
|
||||
async function bootstrap(config) {
|
||||
const suffix = randomUUID().replaceAll('-', '').slice(0, 12);
|
||||
const password = `T3st-${randomUUID()}`;
|
||||
const users = [];
|
||||
for (const label of ['alice', 'bob']) {
|
||||
const username = `bench_${label}_${suffix}`;
|
||||
const created = await api(config.baseUrl, 'POST', '/api/v1/users', {
|
||||
body: { username, display_name: `Benchmark ${label}`, password },
|
||||
});
|
||||
if (created.status !== 201 || !created.payload?.id) {
|
||||
throw new Error(`setup: user registration returned ${created.status}`);
|
||||
}
|
||||
const basic = Buffer.from(`${username}:${password}`).toString('base64');
|
||||
const session = await api(config.baseUrl, 'POST', '/api/v1/auth/sessions', {
|
||||
authorization: `Basic ${basic}`,
|
||||
});
|
||||
if (session.status !== 201 || !session.payload?.token) {
|
||||
throw new Error(`setup: session creation returned ${session.status}`);
|
||||
}
|
||||
users.push({ id: created.payload.id, token: session.payload.token });
|
||||
}
|
||||
const authorization = `Bearer ${users[0].token}`;
|
||||
const chat = await api(config.baseUrl, 'POST', '/api/v1/chats', {
|
||||
body: { title: 'Scaling benchmark', member_ids: users.map((user) => user.id) },
|
||||
authorization,
|
||||
});
|
||||
if (chat.status !== 201 || !chat.payload?.id) {
|
||||
throw new Error(`setup: chat creation returned ${chat.status}`);
|
||||
}
|
||||
const path = `/api/v1/chats/${chat.payload.id}/messages`;
|
||||
for (let index = 0; index < config.seedMessages; index += 1) {
|
||||
const seeded = await api(config.baseUrl, 'POST', path, {
|
||||
body: { text: `seed-${suffix}-${index}` }, authorization, key: randomUUID(),
|
||||
});
|
||||
if (seeded.status !== 201 || !seeded.payload?.id) {
|
||||
throw new Error(`setup: seed message ${index + 1} returned ${seeded.status}`);
|
||||
}
|
||||
}
|
||||
return { authorization, path };
|
||||
}
|
||||
|
||||
function percentile(sorted, fraction) {
|
||||
if (!sorted.length) return null;
|
||||
return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(3));
|
||||
}
|
||||
|
||||
function summarize(events, durationSeconds, slots, dropped, late) {
|
||||
const groups = {};
|
||||
for (const kind of ['read', 'write', 'all']) {
|
||||
const chosen = kind === 'all' ? events : events.filter((event) => event.kind === kind);
|
||||
const successes = chosen.filter((event) => event.ok);
|
||||
const latencies = successes.map((event) => event.latency_ms).sort((a, b) => a - b);
|
||||
const statuses = {};
|
||||
const errors = {};
|
||||
const instances = {};
|
||||
for (const event of chosen) {
|
||||
const status = event.status === null ? 'network' : String(event.status);
|
||||
statuses[status] = (statuses[status] ?? 0) + 1;
|
||||
if (!event.ok) errors[event.error] = (errors[event.error] ?? 0) + 1;
|
||||
if (event.instance) instances[event.instance] = (instances[event.instance] ?? 0) + 1;
|
||||
}
|
||||
groups[kind] = {
|
||||
attempted: chosen.length,
|
||||
successful: successes.length,
|
||||
errors: chosen.length - successes.length,
|
||||
successful_rps: Number((successes.length / durationSeconds).toFixed(3)),
|
||||
status_counts: statuses,
|
||||
error_counts: errors,
|
||||
instance_counts: instances,
|
||||
successful_latency_ms: {
|
||||
p50: percentile(latencies, 0.5),
|
||||
p95: percentile(latencies, 0.95),
|
||||
p99: percentile(latencies, 0.99),
|
||||
},
|
||||
};
|
||||
}
|
||||
return { scheduled_slots: slots, client_dropped_slots: dropped,
|
||||
scheduler_late_slots: late, ...groups };
|
||||
}
|
||||
|
||||
async function runPhase(config, context, durationSeconds, readPercent) {
|
||||
const slots = Math.floor(config.rate * durationSeconds);
|
||||
const intervalMs = 1000 / config.rate;
|
||||
const started = performance.now();
|
||||
const startedEpochMs = Date.now();
|
||||
const events = [];
|
||||
const pending = new Set();
|
||||
let dropped = 0;
|
||||
let late = 0;
|
||||
for (let sequence = 0; sequence < slots; sequence += 1) {
|
||||
const scheduledAt = started + sequence * intervalMs;
|
||||
const waitMs = scheduledAt - performance.now();
|
||||
if (waitMs > 0) await sleep(waitMs);
|
||||
const actualAt = performance.now();
|
||||
if (actualAt - scheduledAt > intervalMs) {
|
||||
late += 1;
|
||||
dropped += 1;
|
||||
continue;
|
||||
}
|
||||
if (pending.size >= config.maxInFlight) {
|
||||
dropped += 1;
|
||||
continue;
|
||||
}
|
||||
const kind = ((sequence * 37 + 17) % 100) < readPercent ? 'read' : 'write';
|
||||
const task = (async () => {
|
||||
let status = null;
|
||||
let instance = null;
|
||||
let error = null;
|
||||
try {
|
||||
const result = kind === 'read'
|
||||
? await api(config.baseUrl, 'GET', `${context.path}?limit=${config.historyLimit}`, {
|
||||
authorization: context.authorization, timeoutMs: config.timeoutMs,
|
||||
})
|
||||
: await api(config.baseUrl, 'POST', context.path, {
|
||||
authorization: context.authorization, timeoutMs: config.timeoutMs,
|
||||
key: randomUUID(), body: { text: `bench-${sequence}-${randomUUID()}` },
|
||||
});
|
||||
status = result.status;
|
||||
instance = result.instance;
|
||||
if (status !== (kind === 'read' ? 200 : 201)) error = `http_${status}`;
|
||||
else if (!instance) error = 'missing_instance_id';
|
||||
else if (kind === 'read' && !Array.isArray(result.payload?.items)) error = 'invalid_page';
|
||||
else if (kind === 'write' && !result.payload?.id) error = 'invalid_message';
|
||||
} catch (caught) {
|
||||
error = caught?.name === 'TimeoutError' ? 'timeout' : 'network_or_parse';
|
||||
}
|
||||
events.push({ sequence, kind, ok: error === null, status, error, instance,
|
||||
scheduled_ms: Number((scheduledAt - started).toFixed(3)),
|
||||
start_lag_ms: Number((actualAt - scheduledAt).toFixed(3)),
|
||||
latency_ms: Number((performance.now() - actualAt).toFixed(3)) });
|
||||
})();
|
||||
pending.add(task);
|
||||
void task.finally(() => pending.delete(task));
|
||||
}
|
||||
const scheduledEnd = started + durationSeconds * 1000;
|
||||
if (performance.now() < scheduledEnd) await sleep(scheduledEnd - performance.now());
|
||||
await Promise.all(pending);
|
||||
return { events, started_epoch_ms: startedEpochMs,
|
||||
scheduled_end_epoch_ms: startedEpochMs + durationSeconds * 1000,
|
||||
completed_epoch_ms: Date.now(),
|
||||
summary: summarize(events, durationSeconds, slots, dropped, late),
|
||||
drain_seconds: Number(((performance.now() - scheduledEnd) / 1000).toFixed(3)) };
|
||||
}
|
||||
|
||||
function bytesToMiB(value) {
|
||||
const match = /^([\d.]+)\s*([KMGT]?i?B)$/i.exec(value.trim());
|
||||
if (!match) return null;
|
||||
const scale = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12,
|
||||
KIB: 1024, MIB: 1024 ** 2, GIB: 1024 ** 3, TIB: 1024 ** 4 };
|
||||
return Number((Number(match[1]) * scale[match[2].toUpperCase()] / 1024 ** 2).toFixed(3));
|
||||
}
|
||||
|
||||
async function dockerSampler(intervalSeconds) {
|
||||
const { stdout } = await execFile('docker', ['compose', 'ps', '-q'], {
|
||||
timeout: 15000, maxBuffer: 1024 * 1024,
|
||||
});
|
||||
const ids = stdout.trim().split(/\s+/).filter(Boolean);
|
||||
if (!ids.length) throw new Error('docker-stats: no running Compose containers');
|
||||
let stopped = false;
|
||||
const samples = [];
|
||||
const errors = [];
|
||||
const task = (async () => {
|
||||
while (!stopped) {
|
||||
const started = performance.now();
|
||||
try {
|
||||
const result = await execFile('docker',
|
||||
['stats', '--no-stream', '--format', '{{json .}}', ...ids],
|
||||
{ timeout: 15000, maxBuffer: 1024 * 1024 });
|
||||
for (const line of result.stdout.trim().split(/\r?\n/).filter(Boolean)) {
|
||||
const row = JSON.parse(line);
|
||||
samples.push({ time_ms: Number(started.toFixed(3)), container: row.Name,
|
||||
cpu_percent: Number(row.CPUPerc.replace('%', '')),
|
||||
memory_mib: bytesToMiB(row.MemUsage.split('/')[0]),
|
||||
memory_limit_mib: bytesToMiB(row.MemUsage.split('/')[1]),
|
||||
pids: Number(row.PIDs) });
|
||||
}
|
||||
} catch (error) {
|
||||
errors.push(error?.code || error?.name || 'docker_stats_error');
|
||||
}
|
||||
const waitMs = Math.max(0, intervalSeconds * 1000 - (performance.now() - started));
|
||||
if (!stopped && waitMs) await sleep(waitMs);
|
||||
}
|
||||
})();
|
||||
return { stop: async () => { stopped = true; await task; return { samples, errors }; } };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const config = options(process.argv.slice(2));
|
||||
const context = await bootstrap(config);
|
||||
const warmup = config.warmupSeconds
|
||||
? await runPhase(config, context, config.warmupSeconds, 100)
|
||||
: null;
|
||||
const sampler = config.dockerStats
|
||||
? await dockerSampler(config.dockerStatsIntervalSeconds)
|
||||
: null;
|
||||
const delay = monitorEventLoopDelay({ resolution: 20 });
|
||||
delay.enable();
|
||||
const cpuBefore = process.cpuUsage();
|
||||
const memoryBefore = process.memoryUsage().rss;
|
||||
let measured;
|
||||
let dockerStats = null;
|
||||
try {
|
||||
measured = await runPhase(config, context, config.durationSeconds, config.readPercent);
|
||||
} finally {
|
||||
if (sampler) dockerStats = await sampler.stop();
|
||||
delay.disable();
|
||||
}
|
||||
const cpu = process.cpuUsage(cpuBefore);
|
||||
const result = {
|
||||
label: config.label,
|
||||
base_url: config.baseUrl,
|
||||
profile: {
|
||||
arrival_model: 'scheduled open loop; late or over-capacity slots dropped',
|
||||
offered_rps: config.rate,
|
||||
duration_seconds: config.durationSeconds,
|
||||
warmup_seconds: config.warmupSeconds,
|
||||
warmup_profile: 'read only',
|
||||
read_percent: config.readPercent,
|
||||
write_percent: 100 - config.readPercent,
|
||||
initial_messages: config.seedMessages,
|
||||
history_limit: config.historyLimit,
|
||||
max_in_flight: config.maxInFlight,
|
||||
timeout_ms: config.timeoutMs,
|
||||
automatic_retries: false,
|
||||
expected_instances: config.expectedInstances,
|
||||
},
|
||||
warmup: warmup?.summary ?? null,
|
||||
measurement: measured.summary,
|
||||
drain_seconds: measured.drain_seconds,
|
||||
timing: {
|
||||
measurement_started_epoch_ms: measured.started_epoch_ms,
|
||||
measurement_scheduled_end_epoch_ms: measured.scheduled_end_epoch_ms,
|
||||
measurement_completed_epoch_ms: measured.completed_epoch_ms,
|
||||
},
|
||||
client: {
|
||||
cpu_seconds: Number(((cpu.user + cpu.system) / 1e6).toFixed(3)),
|
||||
rss_before_mib: Number((memoryBefore / 1024 ** 2).toFixed(3)),
|
||||
rss_after_mib: Number((process.memoryUsage().rss / 1024 ** 2).toFixed(3)),
|
||||
event_loop_delay_ms: {
|
||||
mean: Number((delay.mean / 1e6).toFixed(3)),
|
||||
p99: Number((delay.percentile(99) / 1e6).toFixed(3)),
|
||||
},
|
||||
},
|
||||
docker_stats: dockerStats,
|
||||
request_samples: measured.events,
|
||||
notes: [
|
||||
'Setup and read-only warmup are excluded from measured RPS.',
|
||||
'RPS uses the scheduled measurement window; requests may complete during drain.',
|
||||
'HTTP failures, timeouts and client-dropped slots are reported separately.',
|
||||
'One chat is used per run; writes increase its history during measurement.',
|
||||
'The Node client runs outside Docker CPU/RAM limits and can itself saturate.',
|
||||
'A timed-out POST might have committed; this benchmark never retries it.',
|
||||
],
|
||||
};
|
||||
const json = JSON.stringify(result, null, 2) + '\n';
|
||||
if (config.output) {
|
||||
await mkdir(dirname(config.output), { recursive: true });
|
||||
await writeFile(config.output, json, 'utf8');
|
||||
}
|
||||
console.log(JSON.stringify({ label: result.label, profile: result.profile,
|
||||
measurement: result.measurement, drain_seconds: result.drain_seconds,
|
||||
client: result.client, docker_stats_samples: dockerStats?.samples.length ?? 0,
|
||||
output: config.output ?? null }, null, 2));
|
||||
const observed = Object.keys(result.measurement.all.instance_counts).length;
|
||||
if (config.expectedInstances !== null && observed !== config.expectedInstances) {
|
||||
console.error(`expected ${config.expectedInstances} API instances, observed ${observed}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
if (!result.measurement.all.successful) process.exitCode = 1;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`benchmark failed: ${error?.message ?? String(error)}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure idle PostgreSQL CPU with/without the observer after a scaling sweep."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.check_output(["docker", *args], cwd=ROOT, text=True, encoding="utf-8", timeout=30)
|
||||
|
||||
|
||||
def snapshot(container):
|
||||
values = {k: int(v) for k, v in (line.split() for line in docker("exec", container, "cat", "/sys/fs/cgroup/cpu.stat").splitlines())}
|
||||
return {"time": time.time(), "cpu": values}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--seconds", type=int, default=30)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
container = docker("compose", "ps", "-q", "postgres").strip()
|
||||
results = []
|
||||
for phase in ("without-before", "with-observer", "without-after"):
|
||||
child = None
|
||||
files = []
|
||||
stop = f"/tmp/scaling-calibration-{uuid.uuid4().hex}.stop"
|
||||
try:
|
||||
if phase == "with-observer":
|
||||
files = [(args.output / "postgres.jsonl").open("wb"), (args.output / "postgres.stderr").open("wb")]
|
||||
child = subprocess.Popen(["docker", "exec", "-i", container, "sh", "-s", "postgres", stop],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=files[0], stderr=files[1])
|
||||
child.stdin.write((ROOT / "scripts/diagnostics/observer.sh").read_bytes())
|
||||
child.stdin.close()
|
||||
time.sleep(3)
|
||||
before = snapshot(container)
|
||||
time.sleep(args.seconds)
|
||||
after = snapshot(container)
|
||||
elapsed = after["time"] - before["time"]
|
||||
row = {"phase": phase, "before": before, "after": after,
|
||||
"mean_cpu_cores": (after["cpu"]["usage_usec"]-before["cpu"]["usage_usec"])/elapsed/1e6}
|
||||
results.append(row)
|
||||
print(f"{phase}: CPU={row['mean_cpu_cores']:.4f} cores", flush=True)
|
||||
finally:
|
||||
if child is not None:
|
||||
docker("exec", container, "touch", stop)
|
||||
child.wait(timeout=15)
|
||||
docker("cp", f"{container}:{stop}.pg.jsonl", str(args.output / "postgres-sql.jsonl"))
|
||||
docker("exec", container, "rm", "-f", stop + ".pg.jsonl")
|
||||
for file in files:
|
||||
file.close()
|
||||
overhead = results[1]["mean_cpu_cores"] - (results[0]["mean_cpu_cores"] + results[2]["mean_cpu_cores"])/2
|
||||
report = {"phases": results, "estimated_observer_cpu_cores": overhead,
|
||||
"notes": ["Idle calibration with the running stack's normal readiness probes; no HTTP load generator.",
|
||||
"Difference includes host variation, monitor clients and monitoring SQL; not a correction factor for loaded runs."]}
|
||||
(args.output / "summary.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||
print(f"Estimated observer CPU={overhead:.4f} cores", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
# Диагностическая серия масштабирования
|
||||
|
||||
Из корня репозитория, при работающем Docker и собранном образе API:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/run-scaling-diagnostics.py --client container --output .artifacts/scaling/diagnostics-r100
|
||||
```
|
||||
|
||||
Значения по умолчанию: профили one/two/four, только 100 запланированных
|
||||
запросов/с, три повтора на профиль, 15 с прогрева и 60 с измерения, 80% чтений истории и 20%
|
||||
записей. Пул берётся из Compose: по умолчанию 15 на реплику для one/two,
|
||||
8 для four; фактические значения сохраняются в `manifest.json`.
|
||||
В каждом прогоне — новый чат и 30 исходных
|
||||
сообщений. Всего 9 прогонов: 11 минут 15 секунд прогрева и измерений плюс время запуска
|
||||
контейнеров и сбора данных (ориентировочно около 15 минут, не жёсткий тайм-аут). Меньшие нагрузки
|
||||
по умолчанию исключены; при необходимости их можно явно задать через `--rates`.
|
||||
|
||||
Нагрузка для профиля перестаёт повышаться, если минимум в двух из трёх повторов
|
||||
встречается один из признаков: больше 5% слотов пропущено из-за опоздания генератора,
|
||||
больше 5% пропущено из-за лимита одновременно выполняемых запросов, больше 1% ошибок
|
||||
среди отправленных запросов или p95 успешных ответов превышает 1000 мс. Это границы
|
||||
эксперимента, а не установленный SLA и не доказательство насыщения сервера.
|
||||
|
||||
Для полной матрицы всех запрошенных ступеней добавьте `--full-matrix`. Тогда
|
||||
превышения порогов записываются в `threshold_crossings` и не прерывают профиль.
|
||||
Точки с большими пропусками генератора не доказывают максимальную ёмкость API.
|
||||
Ошибки запуска, отсутствие результата или неполные диагностические данные всё
|
||||
ещё останавливают серию с восстановлением обычного профиля two.
|
||||
|
||||
Сбор инструментированного пула включается только через `test/compose.diagnostics.yaml`.
|
||||
Preload сохраняет callback/Promise-интерфейсы pg.Pool и раз в секунду пишет
|
||||
`/tmp/scaling-pool.jsonl` внутри API. Очередь и занятые соединения наблюдаются каждые
|
||||
100 мс; краткие очереди могут не попасть в выборку, поэтому дополнительно измеряется
|
||||
время каждого получения соединения. Оно включает создание новых соединений и
|
||||
задержки event loop, а не только ожидание очереди. Cчётчики приобретений накопительные,
|
||||
max-показатели относятся к интервалу записи.
|
||||
|
||||
PostgreSQL, Nginx и Redis наблюдаются через один постоянный docker exec на сервис
|
||||
с паузой 2 с между выборками. Наблюдатель версии 3 выполняет SQL через одно
|
||||
постоянное psql-соединение (`\watch 2`); результат находится в `postgres-sql.jsonl`.
|
||||
Сохраняются cgroup CPU/memory, ожидания активных
|
||||
запросов PostgreSQL, число простаивающих транзакций, состав upstream Nginx и события
|
||||
reload. Тексты SQL, токены, пароли и сообщения не записываются. Наблюдатели потребляют
|
||||
часть ресурсов тех же контейнеров; это одинаковая методика для всех профилей.
|
||||
|
||||
В каталоге прогона:
|
||||
|
||||
- `benchmark.json`: исходные HTTP-замеры и границы измеряемого окна в epoch ms;
|
||||
- `api-*.jsonl`: пул, event loop и cgroup API (включают также прогрев и предыдущие
|
||||
повторы того же профиля; summary выбирает только текущее измеряемое окно);
|
||||
- `postgres/nginx/redis.jsonl`: наблюдения остальных сервисов;
|
||||
- `postgres-sql.jsonl`: SQL-снимки постоянного psql-наблюдателя версии 3;
|
||||
- `*.stderr`, `nginx-events.log`: ошибки наблюдателей и события Nginx;
|
||||
- `summary.json`: показатели в измеряемом окне; скорости CPU рассчитаны из
|
||||
приращений счётчиков, память — максимум по выборкам;
|
||||
- `execution.json`: команда и временные границы прогона.
|
||||
|
||||
`manifest.json` в корне фиксирует настройки, причины остановки нагрузки, ID образов,
|
||||
хеши сборщиков, окружение Docker и лимиты контейнеров. После завершения или исключения runner восстанавливает профиль
|
||||
two без диагностического preload. При принудительном завершении процесса это можно
|
||||
сделать вручную:
|
||||
|
||||
```powershell
|
||||
docker compose -f compose.yaml -f configs/two.yaml up -d --no-build --no-deps --wait api
|
||||
```
|
||||
|
||||
Проверки сборщика:
|
||||
|
||||
```powershell
|
||||
node --test test/pool-diagnostics.mjs test/benchmark-scaling.mjs
|
||||
python -X utf8 test/diagnostic-summary.py
|
||||
```
|
||||
|
||||
Для отдельной проверки влияния Windows-клиента и публикации порта можно запустить
|
||||
тот же генератор в Linux-контейнере:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/run-scaling-diagnostics.py --client container --rates 100 --repetitions 3 --output .artifacts/scaling/diagnostics-linux-client-r100
|
||||
```
|
||||
|
||||
Клиент использует тот же собранный образ (только Node и benchmark-скрипт), не имеет
|
||||
лимита CPU/RAM и обращается к `http://nginx` по сети Compose. Это другой путь HTTP:
|
||||
без Windows port forwarding, но с конкуренцией за ресурсы той же Docker VM. Такой
|
||||
контроль помогает проверить влияние клиентского пути, но не заменяет внешний
|
||||
генератор на отдельной машине. Не запускайте его одновременно с основной серией.
|
||||
|
||||
После нагрузки можно оценить накладные расходы наблюдателя PostgreSQL на простое:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/calibrate-scaling-observer.py --output .artifacts/scaling/observer-calibration
|
||||
```
|
||||
|
||||
Выполняются три фазы по 30 с: без наблюдателя, с наблюдателем и снова без него.
|
||||
Это оценка собственных расходов сборщика, а не поправочный коэффициент к нагрузке.
|
||||
|
||||
После завершения серии соберите таблицу и графики. Matplotlib можно запустить
|
||||
через временное окружение uv, не меняя зависимости API:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/summarize-scaling-diagnostics.py .artifacts/scaling/diagnostics-pg1cpu-full-20261028
|
||||
uv run --no-project --with "matplotlib>=3.9,<4" python scripts/plot-scaling-diagnostics.py .artifacts/scaling/diagnostics-pg1cpu-full-20261028
|
||||
```
|
||||
|
||||
Графики `performance`, `resources`, `delivered-load` сохраняются в PNG и SVG.
|
||||
Линии показывают медианы повторов, затенение — диапазон min/max, а не доверительный
|
||||
интервал. Отдельный график задержек и ошибок по фактически отправленным RPS помогает
|
||||
учесть пропуски генератора. Превышение номинальной нагрузки не означает, что сервер
|
||||
фактически получил столько запросов.
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/bin/sh
|
||||
# One persistent docker exec per service; stdout contains only numeric diagnostics.
|
||||
set -eu
|
||||
role=$1
|
||||
stop=$2
|
||||
watch_pid=''
|
||||
cleanup() {
|
||||
if [ -n "$watch_pid" ]; then
|
||||
kill "$watch_pid" 2>/dev/null || true
|
||||
wait "$watch_pid" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 0' TERM INT
|
||||
printf '{"kind":"observer","version":3,"time_ms":%s000}\n' "$(date +%s)"
|
||||
if [ "$role" = postgres ]; then
|
||||
PGAPPNAME=scaling-observer psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" > "${stop}.pg.jsonl" <<'SQL' &
|
||||
SELECT json_build_object(
|
||||
'kind', 'postgres', 'time_ms', (extract(epoch FROM clock_timestamp())*1000)::bigint,
|
||||
'activity', (SELECT coalesce(json_agg(a), '[]'::json) FROM (
|
||||
SELECT state, wait_event_type, wait_event, count(*) AS count,
|
||||
max(extract(epoch FROM clock_timestamp()-query_start)) FILTER (WHERE state = 'active') AS max_active_seconds
|
||||
FROM pg_stat_activity WHERE datname=current_database() AND backend_type='client backend'
|
||||
AND application_name <> 'scaling-observer'
|
||||
GROUP BY state,wait_event_type,wait_event
|
||||
) a),
|
||||
'db', (SELECT row_to_json(d) FROM (
|
||||
SELECT xact_commit,xact_rollback,blks_read,blks_hit,temp_files,temp_bytes,deadlocks,stats_reset
|
||||
FROM pg_stat_database WHERE datname=current_database()
|
||||
) d)
|
||||
)::jsonb
|
||||
\watch 2
|
||||
SQL
|
||||
watch_pid=$!
|
||||
fi
|
||||
while [ ! -f "$stop" ]; do
|
||||
stamp="$(date +%s)000"
|
||||
awk -v stamp="$stamp" '
|
||||
BEGIN {printf "{\"kind\":\"cgroup\",\"time_ms\":%s", stamp}
|
||||
FILENAME ~ /cpu.stat$/ {printf ",\"cpu_%s\":%s", $1, $2}
|
||||
FILENAME ~ /memory.events$/ {printf ",\"memory_%s\":%s", $1, $2}
|
||||
FILENAME ~ /memory.current$/ {printf ",\"memory_bytes\":%s", $1}
|
||||
END {print "}"}
|
||||
' /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.events /sys/fs/cgroup/memory.current
|
||||
if [ "$role" = postgres ]; then
|
||||
awk -v stamp="$stamp" '
|
||||
BEGIN {printf "{\"kind\":\"host\",\"time_ms\":%s", stamp}
|
||||
FILENAME ~ /pressure\// {
|
||||
name=FILENAME; sub(/^.*\//,"",name)
|
||||
for (i=2;i<=NF;i++) {
|
||||
split($i,value,"=")
|
||||
if (value[1] == "total") printf ",\"%s_%s_total_usec\":%s",name,$1,value[2]
|
||||
}
|
||||
}
|
||||
FILENAME ~ /meminfo$/ && ($1 == "MemAvailable:" || $1 == "SwapFree:" || $1 == "SwapTotal:") {
|
||||
name=$1; sub(/:$/,"",name); printf ",\"%s_kib\":%s",name,$2
|
||||
}
|
||||
END {print "}"}
|
||||
' /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io /proc/meminfo
|
||||
fi
|
||||
if [ "$role" = nginx ]; then
|
||||
awk -v stamp="$stamp" '
|
||||
BEGIN {printf "{\"kind\":\"upstream\",\"time_ms\":%s,\"servers\":[", stamp; sep=""}
|
||||
/^server .*:8080 / {printf "%s\"%s\"", sep, $2; sep=","}
|
||||
END {print "]}"}
|
||||
' /etc/nginx/ready-upstreams.conf
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
rm -f "$stop"
|
||||
@@ -0,0 +1,73 @@
|
||||
// Loaded only by test/compose.diagnostics.yaml. No HTTP endpoint or credentials.
|
||||
const { Pool } = require('pg');
|
||||
const fs = require('node:fs');
|
||||
const { performance, monitorEventLoopDelay } = require('node:perf_hooks');
|
||||
|
||||
const states = new Map();
|
||||
const originalConnect = Pool.prototype.connect;
|
||||
Pool.prototype.connect = function (...args) {
|
||||
let state = states.get(this);
|
||||
if (!state) {
|
||||
state = { acquisitions: 0, errors: 0, acquire_ms: 0, interval_max_ms: 0,
|
||||
interval_max_waiting: 0, interval_max_busy: 0 };
|
||||
states.set(this, state);
|
||||
}
|
||||
const started = performance.now();
|
||||
const finish = (error) => {
|
||||
const elapsed = performance.now() - started;
|
||||
state.acquisitions += 1;
|
||||
state.errors += Number(Boolean(error));
|
||||
state.acquire_ms += elapsed;
|
||||
state.interval_max_ms = Math.max(state.interval_max_ms, elapsed);
|
||||
};
|
||||
if (typeof args[0] === 'function') {
|
||||
const callback = args[0];
|
||||
args[0] = (error, client, release) => {
|
||||
finish(error);
|
||||
callback(error, client, release);
|
||||
};
|
||||
return originalConnect.apply(this, args);
|
||||
}
|
||||
return originalConnect.apply(this, args).then(
|
||||
(client) => { finish(null); return client; },
|
||||
(error) => { finish(error); throw error; },
|
||||
);
|
||||
};
|
||||
|
||||
const output = '/tmp/scaling-pool.jsonl';
|
||||
fs.writeFileSync(output, '');
|
||||
const delay = monitorEventLoopDelay({ resolution: 20 });
|
||||
delay.enable();
|
||||
function counters(path) {
|
||||
try {
|
||||
return Object.fromEntries(fs.readFileSync(path, 'utf8').trim().split('\n')
|
||||
.map((line) => { const [key, value] = line.split(/\s+/); return [key, Number(value)]; }));
|
||||
} catch { return null; }
|
||||
}
|
||||
let lastWrite = performance.now();
|
||||
const timer = setInterval(() => {
|
||||
for (const [pool, state] of states) {
|
||||
state.interval_max_waiting = Math.max(state.interval_max_waiting, pool.waitingCount);
|
||||
state.interval_max_busy = Math.max(state.interval_max_busy, pool.totalCount - pool.idleCount);
|
||||
}
|
||||
if (performance.now() - lastWrite < 1000) return;
|
||||
lastWrite = performance.now();
|
||||
const pools = [...states].map(([pool, state]) => ({
|
||||
...state, total: pool.totalCount, idle: pool.idleCount, waiting: pool.waitingCount,
|
||||
}));
|
||||
let memory = null;
|
||||
try { memory = Number(fs.readFileSync('/sys/fs/cgroup/memory.current', 'utf8')); } catch {}
|
||||
fs.appendFileSync(output, JSON.stringify({ time_ms: Date.now(), pid: process.pid,
|
||||
instance: process.env.INSTANCE_ID, pools, cpu: counters('/sys/fs/cgroup/cpu.stat'),
|
||||
memory_bytes: memory, memory_events: counters('/sys/fs/cgroup/memory.events'),
|
||||
rss_bytes: process.memoryUsage().rss, process_cpu: process.cpuUsage(),
|
||||
event_loop_delay_ms: { mean: delay.mean / 1e6, p99: delay.percentile(99) / 1e6 },
|
||||
}) + '\n');
|
||||
delay.reset();
|
||||
for (const state of states.values()) {
|
||||
state.interval_max_ms = 0;
|
||||
state.interval_max_waiting = 0;
|
||||
state.interval_max_busy = 0;
|
||||
}
|
||||
}, 100);
|
||||
timer.unref();
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One read-only snapshot of SQL classes, never SQL text or parameters."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
SQL = """
|
||||
SELECT jsonb_build_object('time_ms',(extract(epoch FROM clock_timestamp())*1000)::bigint,
|
||||
'activity', (SELECT coalesce(jsonb_agg(a),'[]'::jsonb) FROM (
|
||||
SELECT state,wait_event_type,wait_event,
|
||||
CASE
|
||||
WHEN query LIKE '%INSERT INTO "public"."chat_message_epochs"%' THEN 'message_and_epoch_write'
|
||||
WHEN query LIKE '%message_idempotency%' THEN 'idempotency'
|
||||
WHEN lower(ltrim(query)) LIKE 'commit%' THEN 'commit'
|
||||
WHEN query LIKE '%token_hash%' THEN 'authentication'
|
||||
WHEN query LIKE '%chat_message_epochs%' THEN 'epoch_read'
|
||||
WHEN query LIKE '%messages%' THEN 'message_read'
|
||||
ELSE 'other'
|
||||
END AS operation,
|
||||
count(*) AS count,
|
||||
max(extract(epoch FROM clock_timestamp()-query_start)) AS max_query_seconds,
|
||||
max(cardinality(pg_blocking_pids(pid))) AS max_blockers
|
||||
FROM pg_stat_activity
|
||||
WHERE datname=current_database() AND backend_type='client backend'
|
||||
AND application_name <> 'scaling-observer' AND state='active'
|
||||
GROUP BY state,wait_event_type,wait_event,operation
|
||||
) a));
|
||||
"""
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "-e", "PGAPPNAME=scaling-observer",
|
||||
"postgres", "psql", "-X", "-qAt", "-U", "app", "-d", "chat", "-c", SQL],
|
||||
capture_output=True, text=True, encoding="utf-8", timeout=30, check=True)
|
||||
data = json.loads(result.stdout)
|
||||
args.output.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(data))
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot completed diagnostic runs using matplotlib; reads results.csv and manifest.json."""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--phase", help="Only plot this CSV phase, e.g. confirmation")
|
||||
parser.add_argument("--all-latency", action="store_true", help="Use all-request p95, including failures")
|
||||
parser.add_argument("--valid-delivery-only", action="store_true", help="Exclude runs with unverified arrival rate")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
with (args.directory / "results.csv").open(encoding="utf-8") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
if args.phase:
|
||||
rows = [row for row in rows if row.get("phase") == args.phase]
|
||||
if args.valid_delivery_only:
|
||||
rows = [row for row in rows if row.get("delivery_valid") in ("True", "true", "1")]
|
||||
if not rows:
|
||||
parser.error("no results")
|
||||
for row in rows:
|
||||
row['configuration'] = row.get('configuration') or next(
|
||||
(profile for profile in manifest.get('profiles', []) if row['label'].startswith(profile + '-r')),
|
||||
str(row['replicas']))
|
||||
for key in ("replicas", "offered_rps"):
|
||||
row[key] = int(row[key])
|
||||
for key in ("successful_rps", "attempted_rps", "p95_ms", "drop_percent", "acquisition_mean_ms",
|
||||
"pool_max_waiting", "api_cpu_cores", "api_max_throttle_percent",
|
||||
"pg_cpu_cores", "pg_throttle_percent"):
|
||||
row[key] = float(row[key] or 'nan')
|
||||
if args.all_latency:
|
||||
if not row.get("all_request_p95_ms"):
|
||||
parser.error("all-request latency is missing from this dataset")
|
||||
row["p95_ms"] = float(row["all_request_p95_ms"])
|
||||
summary = json.loads((args.directory / row["label"] / "summary.json").read_text(encoding="utf-8"))
|
||||
m = summary["measurement"]["all"]
|
||||
row["error_percent"] = 100*m["errors"]/max(m["attempted"], 1)
|
||||
pg = next(c for group in manifest["containers"].values() for c in group if "-postgres-" in c["name"])
|
||||
pg_cpus = pg["nano_cpus"]/1e9
|
||||
memory_mib = pg["memory_bytes"]/1024**2
|
||||
configurations = sorted({r['configuration'] for r in rows})
|
||||
palette = ['#2563eb', '#059669', '#d97706', '#9333ea', '#db2777', '#0891b2', '#64748b']
|
||||
colors = {configuration: palette[i % len(palette)] for i, configuration in enumerate(configurations)}
|
||||
budgets = {}
|
||||
labels = {}
|
||||
for configuration in configurations:
|
||||
apis = [c for c in manifest.get('containers', {}).get(configuration, []) if '-api-' in c['name']]
|
||||
replicas = next(r['replicas'] for r in rows if r['configuration'] == configuration)
|
||||
if apis:
|
||||
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
|
||||
memory = sum(c['memory_bytes'] for c in apis) / 1024**3
|
||||
budgets[configuration] = (cpu, memory)
|
||||
labels[configuration] = f'{replicas} API ({cpu:g} CPU / {memory:g} GiB)'
|
||||
else:
|
||||
labels[configuration] = f'{configuration}: {replicas} API'
|
||||
budget_description = '; '.join(f'{cpu:g} CPU / {memory:g} GiB'
|
||||
for cpu, memory in sorted(set(budgets.values()))) or 'see manifest'
|
||||
|
||||
def draw(ax, field, title, ylabel, log=False):
|
||||
for configuration in configurations:
|
||||
selected = [r for r in rows if r['configuration'] == configuration]
|
||||
rates = sorted({r["offered_rps"] for r in selected})
|
||||
samples = [[r[field] for r in selected if r["offered_rps"] == rate] for rate in rates]
|
||||
values = [statistics.median(v) for v in samples]
|
||||
ax.plot(rates, values, marker="o", color=colors[configuration], label=labels[configuration])
|
||||
ax.fill_between(rates, [min(v) for v in samples], [max(v) for v in samples],
|
||||
color=colors[configuration], alpha=0.12)
|
||||
ax.set(title=title, xlabel="Scheduled requests/s", ylabel=ylabel)
|
||||
ax.grid(alpha=0.25)
|
||||
if log:
|
||||
ax.set_yscale("log")
|
||||
else:
|
||||
ax.set_ylim(bottom=0)
|
||||
|
||||
def save(fig, name, caption):
|
||||
if args.valid_delivery_only:
|
||||
caption += "\nUnderfed runs excluded; sample counts can differ between points."
|
||||
fig.text(0.5, 0.015, caption, ha="center", fontsize=9)
|
||||
fig.tight_layout(rect=(0, 0.055, 1, 0.94))
|
||||
for extension in ("png", "svg"):
|
||||
fig.savefig(args.directory / f"{name}.{extension}", dpi=160, bbox_inches="tight", pad_inches=0.25)
|
||||
plt.close(fig)
|
||||
|
||||
plt.rcParams.update({"font.size": 10, "axes.titlesize": 11})
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
fig.suptitle(f"PostgreSQL {pg_cpus:g} CPU / {memory_mib:g} MiB; API budgets: {budget_description}", fontsize=12)
|
||||
draw(axes[0, 0], "successful_rps", "Successful throughput", "Successful requests/s")
|
||||
rates = sorted({r["offered_rps"] for r in rows})
|
||||
axes[0, 0].plot(rates, rates, color="#475569", linestyle="--", alpha=0.7, label="Scheduled")
|
||||
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], max(rates)*1.05))
|
||||
axes[0, 0].legend()
|
||||
draw(axes[0, 1], "p95_ms", "Latency of all requests" if args.all_latency else "Latency of successful requests", "p95, ms (log scale)", log=True)
|
||||
if args.all_latency:
|
||||
axes[0, 1].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
|
||||
axes[0, 1].legend(fontsize=8)
|
||||
draw(axes[1, 0], "error_percent", "Errors among attempted requests", "Errors, %")
|
||||
axes[1, 0].axhline(1, color="#64748b", linestyle="--", linewidth=1)
|
||||
axes[1, 0].set_ylim(0, max(axes[1, 0].get_ylim()[1], 1.05))
|
||||
draw(axes[1, 1], "drop_percent", "Slots dropped by client", "Dropped scheduled slots, %")
|
||||
drop_limit = 2 if args.valid_delivery_only else 5
|
||||
axes[1, 1].axhline(drop_limit, color="#64748b", linestyle="--", linewidth=1)
|
||||
axes[1, 1].set_ylim(0, max(axes[1, 1].get_ylim()[1], drop_limit*1.05))
|
||||
save(fig, "performance", "Lines: medians of repeats; shading: min-max. High client-drop points do not establish server capacity.")
|
||||
|
||||
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
|
||||
fig.suptitle("Resource and connection-pool diagnostics", fontsize=14)
|
||||
draw(axes[0, 0], "pg_cpu_cores", "PostgreSQL CPU usage", "CPU cores")
|
||||
axes[0, 0].axhline(pg_cpus, color="#64748b", linestyle="--", linewidth=1, label="CPU limit")
|
||||
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], pg_cpus*1.05))
|
||||
draw(axes[0, 1], "api_cpu_cores", "Total API CPU usage", "CPU cores")
|
||||
for configuration, (cpu, _) in budgets.items():
|
||||
axes[0, 1].axhline(cpu, color=colors[configuration], linestyle='--', linewidth=1,
|
||||
label=f'{configuration} limit: {cpu:g} CPU')
|
||||
if budgets:
|
||||
axes[0, 1].set_ylim(0, max(axes[0, 1].get_ylim()[1], max(cpu for cpu, _ in budgets.values()) * 1.05))
|
||||
axes[0, 1].legend(fontsize=8)
|
||||
draw(axes[0, 2], "pg_throttle_percent", "PostgreSQL CPU throttling", "Throttled periods, %")
|
||||
draw(axes[1, 0], "api_max_throttle_percent", "Most throttled API replica", "Throttled periods, %")
|
||||
draw(axes[1, 1], "acquisition_mean_ms", "Mean connection acquisition", "Acquisition, ms (log scale)", log=True)
|
||||
draw(axes[1, 2], "pool_max_waiting", "Largest observed pool queue", "Waiting requests")
|
||||
axes[0, 0].legend()
|
||||
save(fig, "resources", "Lines: medians of repeats; shading: min-max. Queue sampled every 100 ms; acquisition includes connection setup.")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
|
||||
fig.suptitle("Latency and errors against the workload actually delivered", fontsize=14)
|
||||
for configuration in configurations:
|
||||
selected = [r for r in rows if r['configuration'] == configuration]
|
||||
rates = sorted({r["offered_rps"] for r in selected})
|
||||
groups = [[r for r in selected if r["offered_rps"] == rate] for rate in rates]
|
||||
x = [statistics.median(r["attempted_rps"] for r in group) for group in groups]
|
||||
for axis, field, title, ylabel in (
|
||||
(axes[0], "p95_ms", "Latency", "p95, ms (log scale)"),
|
||||
(axes[1], "error_percent", "Errors", "Errors among attempts, %"),
|
||||
):
|
||||
y = [statistics.median(r[field] for r in group) for group in groups]
|
||||
axis.plot(x, y, marker="o", color=colors[configuration], label=labels[configuration])
|
||||
for rate, xp, yp in zip(rates, x, y):
|
||||
axis.annotate(str(rate), (xp, yp), xytext=(4, 4), textcoords="offset points", fontsize=8)
|
||||
axis.set(title=title, xlabel="Actually attempted requests/s", ylabel=ylabel)
|
||||
axis.grid(alpha=0.25)
|
||||
axes[0].set_yscale("log")
|
||||
if args.all_latency:
|
||||
axes[0].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
|
||||
axes[1].set_ylim(bottom=0)
|
||||
axes[0].legend()
|
||||
save(fig, "delivered-load", "Points: medians of repeats. Small labels: scheduled RPS. Client shares the Docker VM; no server-capacity claim.")
|
||||
print(f"Wrote performance/resources/delivered-load PNG and SVG for {len(rows)} runs to {args.directory}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compatibility entry point for the consolidated report figures."""
|
||||
import runpy
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
if __name__ == '__main__':
|
||||
helper = Path(__file__).with_name('report-evidence.py')
|
||||
sys.argv = [str(helper), '--plot', *sys.argv[1:]]
|
||||
runpy.run_path(str(helper), run_name='__main__')
|
||||
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep report evidence in one JSON, one CSV, two figures and checks.json."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERIES = [
|
||||
('api-cpu', None, 'api-small-growth-20261029'),
|
||||
('pg-cpu', 'pg0.5', 'diagnostics-v3-linux-20261028'),
|
||||
('pg-cpu', 'pg1', 'diagnostics-pg1cpu-linux-20261028'),
|
||||
('pool-limit', 'pool60', 'pool15-four-r100-short-20261028'),
|
||||
('pool-limit', 'pool32', 'pool8-four-r100-short-20261028'),
|
||||
('pg-upper-cpu', 'pg1', 'api-small-db-control-20261029/pg1'),
|
||||
('pg-upper-cpu', 'pg2', 'api-small-db-control-20261029/pg2'),
|
||||
]
|
||||
|
||||
|
||||
def digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def flatten(bundle):
|
||||
rows = []
|
||||
for run in bundle['runs']:
|
||||
profile, measurement = run['profile'], run['measurement']
|
||||
all_requests = measurement['all']
|
||||
diagnostics = run['diagnostics']
|
||||
resources, pools = diagnostics['resources'], diagnostics['pools']
|
||||
manifest = bundle['manifests'][run['manifest']]
|
||||
containers = manifest['containers'][run['configuration']]
|
||||
apis = [c for c in containers if '-api-' in c['name']]
|
||||
pg = next(c for c in containers if '-postgres-' in c['name'])
|
||||
seconds, rate = profile['duration_seconds'], profile['offered_rps']
|
||||
acquired = sum(p['acquisitions'] for p in pools.values())
|
||||
fraction = all_requests['attempted'] / (seconds * rate)
|
||||
latency = all_requests['successful_latency_ms']
|
||||
all_latency = run.get('all_latency_ms') or {}
|
||||
rows.append({
|
||||
'dataset': run['dataset'], 'variant': run['variant'], 'label': run['label'],
|
||||
'replicas': len(apis), 'api_cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
|
||||
'api_ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
|
||||
'pool_total': sum(int(c['pool_max']) for c in apis),
|
||||
'pg_cpu': pg['nano_cpus'] / 1e9, 'pg_ram_mib': pg['memory_bytes'] / 1024**2,
|
||||
'offered_rps': rate, 'duration_seconds': seconds,
|
||||
'warmup_seconds': profile.get('warmup_seconds', manifest['warmup_seconds']),
|
||||
'attempted': all_requests['attempted'], 'successful': all_requests['successful'],
|
||||
'errors': all_requests['errors'], 'successful_rps': all_requests['successful'] / seconds,
|
||||
'attempted_rps': all_requests['attempted'] / seconds,
|
||||
'delivery_fraction': fraction, 'delivery_valid': .98 <= fraction <= 1.02,
|
||||
'p50_ms': latency['p50'], 'p95_ms': latency['p95'], 'p99_ms': latency['p99'],
|
||||
'all_request_p95_ms': all_latency.get('p95'),
|
||||
'api_cpu_cores': sum(r.get('mean_cpu_cores', 0) for name, r in resources.items() if name.startswith('api-')),
|
||||
'api_peak_memory_mib_sum': sum(r.get('peak_memory_mib', 0) for name, r in resources.items() if name.startswith('api-')),
|
||||
'api_max_throttle_percent': max((p.get('throttled_period_fraction', 0) for name, p in resources.items() if name.startswith('api-')), default=0) * 100,
|
||||
'pg_cpu_cores': resources['postgres'].get('mean_cpu_cores'),
|
||||
'pg_peak_memory_mib': resources['postgres'].get('peak_memory_mib'),
|
||||
'pg_throttle_percent': resources['postgres'].get('throttled_period_fraction', 0) * 100,
|
||||
'host_cpu_pressure_percent': diagnostics['host'].get('cpu_some_fraction', 0) * 100,
|
||||
'acquisition_mean_ms': sum(p['mean_acquisition_ms'] * p['acquisitions'] for p in pools.values()) / max(acquired, 1),
|
||||
'acquisition_errors': sum(p['acquisition_errors'] for p in pools.values()),
|
||||
'pool_max_waiting': max((p['max_waiting'] for p in pools.values()), default=0),
|
||||
'nginx_min_ready': diagnostics['nginx']['min_ready'],
|
||||
'diagnostics_complete': diagnostics['diagnostics_complete'],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def collect(source, output):
|
||||
manifests, runs = {}, []
|
||||
for dataset, variant, directory in SERIES:
|
||||
path = source / directory
|
||||
manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8'))
|
||||
manifests[directory] = manifest
|
||||
for label in manifest['runs']:
|
||||
location = path / label
|
||||
benchmark = json.loads((location / 'benchmark.json').read_text(encoding='utf-8'))
|
||||
summary = json.loads((location / 'summary.json').read_text(encoding='utf-8'))
|
||||
configuration = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
|
||||
runs.append({
|
||||
'dataset': dataset, 'variant': variant or configuration, 'configuration': configuration,
|
||||
'manifest': directory, 'label': label,
|
||||
'profile': benchmark['profile'], 'measurement': benchmark['measurement'],
|
||||
'timing': benchmark.get('timing'), 'client': benchmark.get('client'),
|
||||
'all_latency_ms': benchmark.get('all_latency_ms'),
|
||||
'criterion': summary.get('criterion'),
|
||||
'diagnostics': {key: summary[key] for key in
|
||||
('resources', 'pools', 'host', 'postgres', 'nginx', 'limits_reached', 'diagnostics_complete')},
|
||||
'source_hashes': {'benchmark': digest(location / 'benchmark.json'),
|
||||
'summary': digest(location / 'summary.json')},
|
||||
})
|
||||
sources = sorted([p for p in (ROOT / 'src').rglob('*.ts') if 'generated' not in p.parts]
|
||||
+ list((ROOT / 'prisma').rglob('*.sql'))
|
||||
+ list((ROOT / 'configs').glob('*.yaml'))
|
||||
+ [ROOT / p for p in ('compose.yaml', 'Dockerfile', 'infra/nginx.conf', 'infra/start-nginx.sh', 'pnpm-lock.yaml')])
|
||||
host_path = source / 'diagnostics-pg1cpu-full-20261028/host-environment.json'
|
||||
host = json.loads(host_path.read_text(encoding='utf-8'))
|
||||
bundle = {
|
||||
'schema_version': 1, 'recorded_at': datetime.now(timezone.utc).isoformat(),
|
||||
'git_base': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
|
||||
'revision_note': 'Implementation is in the working tree; git_base identifies the scaffold, not a committed implementation.',
|
||||
'retained_data': 'Per-run HTTP aggregates, profiles, CPU/RAM/pool/SQL-state diagnostics, original manifests and source hashes. Request samples, stdout/stderr, traces and duplicate documentation omitted.',
|
||||
'hardware': {'processor': host['processor'], 'windows_total_visible_kib': host['windows_memory_kib']['total_visible']},
|
||||
'application_source_sha256': {p.relative_to(ROOT).as_posix(): digest(p) for p in sources},
|
||||
'manifests': manifests, 'runs': runs,
|
||||
}
|
||||
(output / 'measurements.json').write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
return bundle
|
||||
|
||||
|
||||
def verify(bundle, rows):
|
||||
counts = Counter(r['dataset'] for r in rows)
|
||||
assert counts == {'api-cpu': 36, 'pg-cpu': 6, 'pool-limit': 6, 'pg-upper-cpu': 6}, counts
|
||||
assert len({(r['dataset'], r['variant'], r['label']) for r in rows}) == 54
|
||||
assert all(r['attempted'] == r['successful'] + r['errors'] for r in rows)
|
||||
assert all(r['diagnostics_complete'] for r in rows)
|
||||
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
|
||||
assert all(r['pool_total'] == 32 and r['api_ram_mib'] == 1024 and r['pg_cpu'] == 1 for r in matrix)
|
||||
assert all(n == 3 for n in Counter((r['variant'], r['offered_rps']) for r in matrix).values())
|
||||
for name, expected in [('pg-cpu', {'pg0.5': .5, 'pg1': 1}),
|
||||
('pg-upper-cpu', {'pg1': 1, 'pg2': 2})]:
|
||||
assert all(r['pg_cpu'] == expected[r['variant']] for r in rows if r['dataset'] == name)
|
||||
assert {r['pool_total'] for r in rows if r['dataset'] == 'pool-limit'} == {32, 60}
|
||||
assert len(bundle['manifests']) == 7
|
||||
|
||||
|
||||
def plot(rows, output):
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
for cpu, color in ((1, '#9333ea'), (2, '#059669')):
|
||||
for ax, rate, field, title, ylabel in (
|
||||
(axes[0, 0], 40, 'all_request_p95_ms', '40 scheduled RPS: all-request latency', 'Median p95, ms'),
|
||||
(axes[0, 1], 80, 'successful_rps', '80 scheduled RPS: successful throughput', 'Median successful requests/s'),
|
||||
(axes[1, 0], 40, 'api_max_throttle_percent', '40 RPS: most throttled API', 'Median throttled periods, %'),
|
||||
(axes[1, 1], 80, 'delivery_fraction', '80 RPS: actual load delivery', 'Median started / scheduled, %'),
|
||||
):
|
||||
groups = [[r for r in matrix if r['api_cpu'] == cpu and r['replicas'] == n and r['offered_rps'] == rate] for n in (1, 2, 4)]
|
||||
scale = 100 if field == 'delivery_fraction' else 1
|
||||
values = [statistics.median(r[field] for r in g) * scale for g in groups]
|
||||
ax.plot([1, 2, 4], values, marker='o', color=color, label=f'API total {cpu} CPU / 1 GiB')
|
||||
ax.fill_between([1, 2, 4], [min(r[field] for r in g) * scale for g in groups],
|
||||
[max(r[field] for r in g) * scale for g in groups], color=color, alpha=.1)
|
||||
ax.set(title=title, xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
|
||||
ax.grid(alpha=.25)
|
||||
ax.legend(fontsize=8)
|
||||
axes[0, 0].axhline(1000, color='#64748b', linestyle='--')
|
||||
axes[1, 1].axhline(98, color='#64748b', linestyle='--')
|
||||
fig.suptitle('Fixed budgets and CPU growth; PG 1 CPU / 512 MiB; total pool 32', fontsize=13)
|
||||
fig.text(.5, .012, 'All repeats shown, including underfed runs. Lines: medians; bands: min/max, not confidence intervals. No measured plateau claim.', ha='center', fontsize=8)
|
||||
fig.tight_layout(rect=(0, .04, 1, .95))
|
||||
fig.savefig(output / 'scaling.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
panels = [
|
||||
('api-cpu', ['two-small', 'two'], 'all_request_p95_ms', ['API 1 CPU', 'API 2 CPU'], '2 API, 40 RPS: CPU increase', 'All-request p95, ms', 40),
|
||||
('pg-cpu', ['pg0.5', 'pg1'], 'p95_ms', ['PG 0.5 CPU', 'PG 1 CPU'], '4 API, 60 RPS: database quota', 'Successful-request p95, ms', 60),
|
||||
('pool-limit', ['pool60', 'pool32'], 'successful_rps', ['60 connections', '32 connections'], '4 API, 100 scheduled RPS: pool limit', 'Successful requests/s', 100),
|
||||
('pg-upper-cpu', ['pg1', 'pg2'], 'all_request_p95_ms', ['PG 1 CPU', 'PG 2 CPU'], '2 API, 80 RPS: further PG CPU increase', 'All-request p95, ms', 80),
|
||||
]
|
||||
for ax, (dataset, variants, field, labels, title, ylabel, rate) in zip(axes.flat, panels):
|
||||
for i, variant in enumerate(variants):
|
||||
group = [r for r in rows if r['dataset'] == dataset and r['variant'] == variant and r['offered_rps'] == rate]
|
||||
assert len(group) == 3
|
||||
color = '#9333ea' if i == 0 else '#059669'
|
||||
for j, r in enumerate(group):
|
||||
# Older Node series use a different scheduler: show their observations without a retroactive k6 classification.
|
||||
underfed = dataset in {'api-cpu', 'pg-upper-cpu'} and not r['delivery_valid']
|
||||
ax.scatter(i + (j - 1) * .08, r[field], color=color, marker='x' if underfed else 'o', s=55)
|
||||
value = (sum(r['successful'] for r in group) / sum(r['duration_seconds'] for r in group)
|
||||
if dataset == 'pool-limit' else statistics.median(r[field] for r in group))
|
||||
ax.hlines(value, i - .17, i + .17, color='#0f172a', linewidth=2)
|
||||
ax.annotate(f'{value:.0f}' if value > 100 else f'{value:.2f}', (i, value),
|
||||
xytext=(12, 8), textcoords='offset points', fontsize=9)
|
||||
ax.set(title=title, ylabel=ylabel, xticks=[0, 1], xticklabels=labels, xlim=(-.4, 1.4))
|
||||
ax.grid(axis='y', alpha=.25)
|
||||
if field == 'all_request_p95_ms':
|
||||
ax.axhline(1000, color='#64748b', linestyle='--', linewidth=1)
|
||||
maximum = max(r[field] for r in rows if r['dataset'] == dataset and r['variant'] in variants and r['offered_rps'] == rate)
|
||||
ax.set_ylim(0, maximum * 1.2)
|
||||
fig.suptitle('Separate experiments: compare variants within each panel', fontsize=13)
|
||||
fig.text(.5, .012, 'Dots: individual runs; black bars: medians (pool panel: aggregate successful RPS). Crosses: k6 underdelivery. See REPORT.md for scheduler differences.', ha='center', fontsize=8)
|
||||
fig.tight_layout(rect=(0, .04, 1, .95))
|
||||
fig.savefig(output / 'improvements.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--collect', action='store_true', help='Import the selected raw experiment directories before cleanup')
|
||||
parser.add_argument('--source-root', type=Path, default=ROOT / '.artifacts/scaling')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence')
|
||||
parser.add_argument('--input', type=Path, default=ROOT / 'evidence/measurements.json')
|
||||
parser.add_argument('--plot', action='store_true')
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
bundle = collect(args.source_root, args.output) if args.collect else json.loads(args.input.read_text(encoding='utf-8'))
|
||||
rows = flatten(bundle)
|
||||
verify(bundle, rows)
|
||||
with (args.output / 'scaling.csv').open('w', encoding='utf-8', newline='') as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=rows[0].keys())
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
if args.plot:
|
||||
plot(rows, args.output)
|
||||
print(f'Verified {len(rows)} runs from {len(bundle["manifests"])} source series; compact evidence in {args.output}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bracket an empirical SLO boundary with k6; preserve separate probe/confirmation data."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROFILE_REPLICAS = {'one': 1, 'two': 2, 'four': 4, 'four-large': 4,
|
||||
'one-small': 1, 'two-small': 2, 'four-small': 4}
|
||||
spec = importlib.util.spec_from_file_location('diagnostics', ROOT/'scripts/run-scaling-diagnostics.py')
|
||||
diag = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(diag)
|
||||
|
||||
|
||||
def call(args, timeout=180, env=None):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=timeout, env=env)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'{args[0]} {args[1]}: {result.stderr[-1800:]}')
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def watch(container, role, directory):
|
||||
stop = '/tmp/saturation-' + uuid.uuid4().hex + '.stop'
|
||||
output = (directory/f'{role}.jsonl').open('wb')
|
||||
error = (directory/f'{role}.stderr').open('wb')
|
||||
process = subprocess.Popen(['docker', 'exec', '-i', container, 'sh', '-s', role, stop],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=error)
|
||||
process.stdin.write((ROOT/'scripts/diagnostics/observer.sh').read_bytes())
|
||||
process.stdin.close()
|
||||
return container, role, stop, process, output, error
|
||||
|
||||
|
||||
def stop_watch(worker, directory):
|
||||
container, role, stop, process, output, error = worker
|
||||
try:
|
||||
call(['docker', 'exec', container, 'touch', stop], timeout=15)
|
||||
process.wait(timeout=15)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
output.close()
|
||||
error.close()
|
||||
if role == 'postgres':
|
||||
call(['docker', 'cp', f'{container}:{stop}.pg.jsonl', str(directory/'postgres-sql.jsonl')])
|
||||
call(['docker', 'exec', container, 'rm', '-f', stop+'.pg.jsonl'])
|
||||
|
||||
|
||||
def classify(benchmark):
|
||||
m = benchmark['measurement']
|
||||
count = m['all']['attempted']
|
||||
ratio = count/max(m['scheduled_slots'], 1)
|
||||
error_fraction = m['all']['errors']/max(count, 1)
|
||||
p95 = benchmark['all_latency_ms']['p95']
|
||||
delivered = 0.98 <= ratio <= 1.02 and m['client_dropped_slots']/max(m['scheduled_slots'], 1) <= 0.02
|
||||
slo = count > 0 and error_fraction <= 0.01 and p95 is not None and p95 <= 1000
|
||||
return {'delivery_fraction': ratio, 'error_fraction': error_fraction,
|
||||
'all_request_p95_ms': p95, 'delivery_valid': delivered, 'slo_pass': slo,
|
||||
'classification': 'pass' if delivered and slo else 'slo_fail' if delivered else 'generator_limited'}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', required=True, type=Path)
|
||||
parser.add_argument('--image', default='grafana/k6:1.3.0')
|
||||
parser.add_argument('--profiles', nargs='+', choices=list(PROFILE_REPLICAS), default=['one', 'two', 'four'])
|
||||
parser.add_argument('--probe-seconds', type=int, default=15)
|
||||
parser.add_argument('--confirm-seconds', type=int, default=30)
|
||||
parser.add_argument('--warmup-seconds', type=int, default=10,
|
||||
help='Mixed-load warmup before each confirmation run')
|
||||
parser.add_argument('--calibration-only', action='store_true')
|
||||
parser.add_argument('--confirm-rates', nargs='+', type=int, help='Confirm these rates directly, skipping coarse search')
|
||||
parser.add_argument('--no-expand', action='store_true',
|
||||
help='With --confirm-rates, run only the requested rates for a controlled comparison')
|
||||
args = parser.parse_args()
|
||||
if args.no_expand and not args.confirm_rates:
|
||||
parser.error('--no-expand requires --confirm-rates')
|
||||
if args.warmup_seconds < 1:
|
||||
parser.error('--warmup-seconds must be positive')
|
||||
out = args.output.resolve()
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
manifest = {'started': datetime.now(timezone.utc).isoformat(), 'profiles': args.profiles,
|
||||
'client': 'container', 'engine': 'k6', 'duration_seconds': args.confirm_seconds,
|
||||
'warmup_seconds': args.warmup_seconds, 'probe_duration_seconds': args.probe_seconds, 'probe_warmup_seconds': 5,
|
||||
'repetitions': 3, 'rates': [], 'containers': {}, 'runs': [], 'boundaries': {},
|
||||
'stopped_profiles': {}, 'observer_version': 3,
|
||||
'criterion': 'delivery 98..102%, dropped iterations <=2%, errors <=1%, all-request p95 <=1000 ms',
|
||||
'pool_budget': 32, 'notes': ['Local empirical SLO boundary, not universal maximum throughput.',
|
||||
'Native sequential warmup/measurement scenarios at target-rate 80/20; custom counters belong only to measurement.',
|
||||
'Generator shares Docker VM CPU; calibration does not eliminate competition during API load.',
|
||||
'Probe durations differ; use confirmation rows for reported bounds.'],
|
||||
'source_sha256': {p: hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in
|
||||
['scripts/saturation.k6.js', 'scripts/saturation-mock.cjs', 'scripts/run-saturation.py',
|
||||
'scripts/run-scaling-diagnostics.py', 'scripts/diagnostics/pool.cjs', 'scripts/diagnostics/observer.sh',
|
||||
'compose.yaml', 'test/compose.diagnostics.yaml',
|
||||
*[f'configs/{profile}.yaml' for profile in args.profiles]]}}
|
||||
save = lambda: (out/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n', encoding='utf-8')
|
||||
manifest['requested_confirmation_rates'] = args.confirm_rates
|
||||
manifest['expand_confirmation_rates'] = not args.no_expand
|
||||
for name in manifest['source_sha256']:
|
||||
target = out/'sources'/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes((ROOT/name).read_bytes())
|
||||
save()
|
||||
nginx = json.loads(call(['docker', 'inspect', call(diag.compose('two', False)+['ps', '-q', 'nginx'])]))[0]
|
||||
network = next(iter(nginx['NetworkSettings']['Networks']))
|
||||
api_id = call(diag.compose('two', False)+['ps', '-q', 'api']).split()[0]
|
||||
api_image = json.loads(call(['docker', 'inspect', api_id]))[0]['Image']
|
||||
suffix = uuid.uuid4().hex
|
||||
generator = 'lab3-saturation-client-' + suffix
|
||||
mock = 'lab3-saturation-mock-' + suffix
|
||||
generator_watch = None
|
||||
counter = 0
|
||||
manifest['generator_container'] = generator
|
||||
manifest['mock_container'] = mock
|
||||
save()
|
||||
|
||||
def run(profile, rate, phase, seconds, warm, expected, base='http://nginx'):
|
||||
nonlocal counter
|
||||
if (out/'CANCEL').exists():
|
||||
raise InterruptedError('Cancellation requested; restoring ordinary profile')
|
||||
counter += 1
|
||||
label = f'{profile}-r{rate}-{phase}-{counter}'
|
||||
directory = out/label
|
||||
directory.mkdir()
|
||||
print(f'START {label} ({seconds}s)', flush=True)
|
||||
envs = {'BASE_URL': base, 'RATE': str(rate), 'SECONDS': str(seconds), 'WARMUP': str(warm),
|
||||
'EXPECTED': str(expected), 'LABEL': label, 'PHASE': phase,
|
||||
'OUTPUT': f'/results/{label}/benchmark.json', 'CALIBRATION': str(int(profile == 'calibration'))}
|
||||
command = ['docker', 'exec']
|
||||
for k, v in envs.items():
|
||||
command += ['-e', f'{k}={v}']
|
||||
command += [generator, 'k6', 'run', '--quiet', '/scripts/saturation.k6.js']
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=seconds+warm+150)
|
||||
(directory/'execution.json').write_text(json.dumps({'returncode': result.returncode,
|
||||
'started': started, 'ended': datetime.now(timezone.utc).isoformat(), 'command': command}, indent=2)+'\n')
|
||||
(directory/'k6.stdout').write_text(result.stdout, encoding='utf-8')
|
||||
(directory/'k6.stderr').write_text(result.stderr, encoding='utf-8')
|
||||
if result.returncode or not (directory/'benchmark.json').exists():
|
||||
raise RuntimeError(f'{label}: k6 execution failed: {result.stderr[-1000:]}')
|
||||
b = json.loads((directory/'benchmark.json').read_text(encoding='utf-8'))
|
||||
if not b['timing']['measurement_started_epoch_ms']:
|
||||
raise RuntimeError(f'{label}: no measured requests')
|
||||
verdict = classify(b)
|
||||
(directory/'criterion.json').write_text(json.dumps(verdict, indent=2)+'\n')
|
||||
print(f"DONE {label}: {verdict['classification']} delivered={verdict['delivery_fraction']:.3f} "
|
||||
f"RPS={b['measurement']['all']['successful_rps']:.2f} p95={verdict['all_request_p95_ms']:.0f}ms "
|
||||
f"errors={b['measurement']['all']['errors']}", flush=True)
|
||||
if profile != 'calibration':
|
||||
manifest['runs'].append(label)
|
||||
manifest['rates'] = sorted(set(manifest['rates']+[rate]))
|
||||
save()
|
||||
time.sleep(1 if verdict['classification'] == 'pass' else 5)
|
||||
return label, verdict
|
||||
|
||||
try:
|
||||
call(['docker', 'run', '-d', '--name', generator, '--network', network,
|
||||
'-v', f'{ROOT / "scripts"}:/scripts:ro', '-v', f'{out}:/results',
|
||||
'--entrypoint', 'sh', args.image, '-c', 'while :; do sleep 3600; done'])
|
||||
manifest['generator_image'] = json.loads(call(['docker', 'inspect', generator]))[0]['Image']
|
||||
manifest['generator_version'] = call(['docker', 'exec', generator, 'k6', 'version'])
|
||||
manifest['docker'] = json.loads(call(['docker', 'info', '--format',
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}}}']))
|
||||
generator_watch = watch(generator, 'generator', out)
|
||||
call(['docker', 'run', '-d', '--name', mock, '--network', network,
|
||||
'-v', f'{ROOT / "scripts/saturation-mock.cjs"}:/mock.cjs:ro',
|
||||
'--entrypoint', 'node', api_image, '/mock.cjs'])
|
||||
calibrations = []
|
||||
for calibration_rate in [400, 200, 100]:
|
||||
label, calibration = run('calibration', calibration_rate, 'calibration', 10, 3, 1, f'http://{mock}:8080')
|
||||
calibrations.append({'label': label, 'rate': calibration_rate, **calibration})
|
||||
if calibration['classification'] == 'pass': break
|
||||
manifest['calibration_attempts'] = calibrations
|
||||
manifest['calibration'] = {'label': label, **calibration}
|
||||
low_label, low_calibration = run('calibration', 40, 'calibration-low-rate', 10, 3, 1, f'http://{mock}:8080')
|
||||
manifest['calibration_low_rate'] = {'label': low_label, **low_calibration}
|
||||
save()
|
||||
call(['docker', 'rm', '-f', mock])
|
||||
if calibration['classification'] != 'pass' or low_calibration['classification'] != 'pass':
|
||||
raise RuntimeError('Generator calibration failed; do not claim API saturation')
|
||||
if args.calibration_only:
|
||||
return
|
||||
for profile in args.profiles:
|
||||
profile_started = datetime.now(timezone.utc).isoformat()
|
||||
n = PROFILE_REPLICAS[profile]
|
||||
print(f'SWITCH {profile}, pool={32//n} per replica', flush=True)
|
||||
env = {**os.environ, 'DB_POOL_MAX': str(32//n)}
|
||||
call(diag.compose(profile)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], env=env)
|
||||
time.sleep(6)
|
||||
containers = {role: call(diag.compose(profile)+['ps', '-q', role]).split() for role in ['api', 'postgres', 'nginx', 'redis']}
|
||||
details = json.loads(call(['docker', 'inspect', *[c for group in containers.values() for c in group]]))
|
||||
manifest['containers'][profile] = [{'id': c['Id'], 'name': c['Name'], 'image': c['Image'],
|
||||
'nano_cpus': c['HostConfig']['NanoCpus'], 'memory_bytes': c['HostConfig']['Memory'],
|
||||
'pool_max': next((e.split('=', 1)[1] for e in c['Config']['Env'] if e.startswith('DB_POOL_MAX=')), None)} for c in details]
|
||||
profile_dir = out/f'observers-{profile}'
|
||||
profile_dir.mkdir()
|
||||
workers = []
|
||||
first = len(manifest['runs'])
|
||||
save()
|
||||
try:
|
||||
for role in ['postgres', 'nginx', 'redis']:
|
||||
workers.append(watch(containers[role][0], role, profile_dir))
|
||||
low = None
|
||||
high = None
|
||||
for rate in ([] if args.confirm_rates else [40, 80, 120, 160, 240, 320, 480]):
|
||||
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
|
||||
if v['classification'] == 'pass': low = rate
|
||||
else:
|
||||
high = rate
|
||||
break
|
||||
if low is None and not args.confirm_rates:
|
||||
for rate in [20, 10]:
|
||||
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
|
||||
if v['classification'] == 'pass': low = rate; break
|
||||
high = rate
|
||||
if low is not None and high is not None:
|
||||
for _ in range(2):
|
||||
if high-low <= 20: break
|
||||
middle = ((low+high)//20)*10
|
||||
_, v = run(profile, middle, 'probe', args.probe_seconds, 5, n)
|
||||
if v['classification'] == 'pass': low = middle
|
||||
else: high = middle
|
||||
confirmations = {}
|
||||
for repeat in range(3):
|
||||
for rate in sorted(set(args.confirm_rates or [r for r in [low, high] if r is not None])):
|
||||
_, v = run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)
|
||||
confirmations.setdefault(rate, []).append(v)
|
||||
for _ in range(0 if args.no_expand else 2):
|
||||
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
|
||||
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
|
||||
if good and bad and max(good) < min(bad): break
|
||||
if any(v['classification'] == 'generator_limited' for vv in confirmations.values() for v in vv): break
|
||||
rate = max(10, min(confirmations)-20) if not good else max(confirmations)+20
|
||||
if rate in confirmations: break
|
||||
confirmations[rate] = [run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)[1] for _ in range(3)]
|
||||
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
|
||||
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
|
||||
manifest['boundaries'][profile] = {'confirmed_good': max(good) if good else None,
|
||||
'confirmed_bad': min(bad) if bad else None, 'confirmations': confirmations,
|
||||
'bracketed': bool(good and bad and max(good) < min(bad))}
|
||||
print(f"BOUNDARY {profile}: {manifest['boundaries'][profile]['confirmed_good']} .. "
|
||||
f"{manifest['boundaries'][profile]['confirmed_bad']} RPS", flush=True)
|
||||
finally:
|
||||
for worker in workers: stop_watch(worker, profile_dir)
|
||||
log_result = subprocess.run(['docker', 'logs', '--timestamps', '--since', profile_started,
|
||||
containers['nginx'][0]], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace', timeout=30)
|
||||
events = [line for line in (log_result.stdout+'\n'+log_result.stderr).splitlines()
|
||||
if any(term in line for term in ['worker_connections', 'no live upstreams',
|
||||
'upstream timed out', 'download timed out', 'signal process started'])]
|
||||
(profile_dir/'nginx-events.log').write_text('\n'.join(events)+'\n', encoding='utf-8')
|
||||
for index, container in enumerate(containers['api'], 1):
|
||||
call(['docker', 'cp', f'{container}:/tmp/scaling-pool.jsonl', str(profile_dir/f'api-{index}.jsonl')])
|
||||
for label in manifest['runs'][first:]:
|
||||
directory = out/label
|
||||
for p in profile_dir.glob('*.jsonl'):
|
||||
(directory/p.name).write_bytes(p.read_bytes())
|
||||
(directory/'generator.jsonl').write_bytes((out/'generator.jsonl').read_bytes())
|
||||
b = json.loads((directory/'benchmark.json').read_text())
|
||||
summary = diag.summarize(directory, b)
|
||||
samples = [r for r in diag.rows(directory/'generator.jsonl') if r['kind'] == 'cgroup'
|
||||
and b['timing']['measurement_started_epoch_ms'] <= r['time_ms'] <= b['timing']['measurement_scheduled_end_epoch_ms']]
|
||||
summary['resources']['generator'] = diag.resource_summary(samples)
|
||||
summary['all_latency_ms'] = b['all_latency_ms']
|
||||
summary['criterion'] = json.loads((directory/'criterion.json').read_text())
|
||||
summary['diagnostics_complete'] = len(summary['pools']) == n and all(v['samples'] >= 2 for v in summary['resources'].values())
|
||||
(directory/'summary.json').write_text(json.dumps(summary, indent=2)+'\n')
|
||||
save()
|
||||
finally:
|
||||
if generator_watch: stop_watch(generator_watch, out)
|
||||
for name in [mock, generator]:
|
||||
subprocess.run(['docker', 'rm', '-f', name], cwd=ROOT, capture_output=True, timeout=30)
|
||||
if not args.calibration_only:
|
||||
print('RESTORE normal two profile, original pool setting', flush=True)
|
||||
call(diag.compose('two', False)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'])
|
||||
manifest['finished'] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a bounded scaling sweep and collect diagnostics without HTTP metric endpoints."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import hashlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, timeout=180):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"{args[0]} {args[1]} failed: {result.stderr[-1500:]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def compose(profile, diagnostics=True):
|
||||
args = ["docker", "compose", "-f", "compose.yaml", "-f", f"configs/{profile}.yaml"]
|
||||
if diagnostics:
|
||||
args += ["-f", "test/compose.diagnostics.yaml"]
|
||||
return args
|
||||
|
||||
|
||||
def rows(path):
|
||||
data = []
|
||||
if path.exists():
|
||||
text = path.read_text(encoding="utf-8")
|
||||
decoder = json.JSONDecoder()
|
||||
position = 0
|
||||
while position < len(text):
|
||||
if text[position].isspace():
|
||||
position += 1
|
||||
continue
|
||||
try:
|
||||
value, position = decoder.raw_decode(text, position)
|
||||
data.append(value)
|
||||
except json.JSONDecodeError:
|
||||
break # Last record can be in progress when a running observer is copied.
|
||||
return data
|
||||
|
||||
|
||||
def delta(first, last, name):
|
||||
return max(0, last.get(name, 0) - first.get(name, 0))
|
||||
|
||||
|
||||
def resource_summary(samples):
|
||||
if len(samples) < 2:
|
||||
return {"samples": len(samples)}
|
||||
first, last = samples[0], samples[-1]
|
||||
elapsed = (last["time_ms"] - first["time_ms"]) / 1000
|
||||
periods = delta(first, last, "cpu_nr_periods")
|
||||
return {
|
||||
"samples": len(samples), "coverage_seconds": elapsed,
|
||||
"max_sample_gap_seconds": max((b["time_ms"] - a["time_ms"]) / 1000 for a, b in zip(samples, samples[1:])),
|
||||
"mean_cpu_cores": delta(first, last, "cpu_usage_usec") / max(elapsed * 1e6, 1),
|
||||
"throttled_seconds": delta(first, last, "cpu_throttled_usec") / 1e6,
|
||||
"throttled_period_fraction": delta(first, last, "cpu_nr_throttled") / max(periods, 1),
|
||||
"peak_memory_mib": max(s.get("memory_bytes", 0) or 0 for s in samples) / 1024**2,
|
||||
"oom_kills": delta(first, last, "memory_oom_kill"),
|
||||
}
|
||||
|
||||
|
||||
def summarize(directory, benchmark):
|
||||
start = benchmark["timing"]["measurement_started_epoch_ms"]
|
||||
end = benchmark["timing"]["measurement_scheduled_end_epoch_ms"]
|
||||
within = lambda data: [r for r in data if start <= r["time_ms"] <= end]
|
||||
resources = {}
|
||||
pools = {}
|
||||
for path in sorted(directory.glob("api-*.jsonl")):
|
||||
data = within(rows(path))
|
||||
flattened = [{**r, **{f"cpu_{k}": v for k, v in (r.get("cpu") or {}).items()},
|
||||
**{f"memory_{k}": v for k, v in (r.get("memory_events") or {}).items()}}
|
||||
for r in data]
|
||||
resources[path.stem] = resource_summary(flattened)
|
||||
if len(data) < 2:
|
||||
continue
|
||||
first, last = data[0]["pools"], data[-1]["pools"]
|
||||
acquisitions = sum(p["acquisitions"] for p in last) - sum(p["acquisitions"] for p in first)
|
||||
acquire_ms = sum(p["acquire_ms"] for p in last) - sum(p["acquire_ms"] for p in first)
|
||||
intervals = [p for r in data for p in r["pools"]]
|
||||
totals = [sum(p["total"] for p in r["pools"]) for r in data]
|
||||
pools[path.stem] = {
|
||||
"samples": len(data), "acquisitions": acquisitions,
|
||||
"acquisition_errors": sum(p["errors"] for p in last) - sum(p["errors"] for p in first),
|
||||
"mean_acquisition_ms": acquire_ms / max(acquisitions, 1),
|
||||
"max_acquisition_ms": max((p["interval_max_ms"] for p in intervals), default=0),
|
||||
"max_waiting": max((p["interval_max_waiting"] for p in intervals), default=0),
|
||||
"queued_interval_fraction": sum(p["interval_max_waiting"] > 0 for p in intervals) / max(len(intervals), 1),
|
||||
"max_total": max((p["total"] for p in intervals), default=0),
|
||||
"observed_connection_increases": sum(max(0, b-a) for a, b in zip(totals, totals[1:])),
|
||||
"observed_connection_decreases": sum(max(0, a-b) for a, b in zip(totals, totals[1:])),
|
||||
"event_loop_p99_max_ms": max(r["event_loop_delay_ms"]["p99"] for r in data),
|
||||
}
|
||||
pg = within(rows(directory / "postgres.jsonl"))
|
||||
pg_samples = [r for r in pg if r["kind"] == "postgres"]
|
||||
pg_samples += within(rows(directory / "postgres-sql.jsonl"))
|
||||
pg_samples.sort(key=lambda r: r["time_ms"])
|
||||
host_samples = [r for r in pg if r["kind"] == "host"]
|
||||
host = {"samples": len(host_samples)}
|
||||
if len(host_samples) > 1:
|
||||
host_seconds = (host_samples[-1]["time_ms"] - host_samples[0]["time_ms"])/1000
|
||||
host.update({key + "_fraction": delta(host_samples[0], host_samples[-1], key + "_total_usec") / max(host_seconds*1e6, 1)
|
||||
for key in ("cpu_some", "memory_some", "memory_full", "io_some", "io_full")})
|
||||
host["min_available_mib"] = min(r["MemAvailable_kib"] for r in host_samples)/1024
|
||||
host["max_swap_used_mib"] = max(r["SwapTotal_kib"] - r["SwapFree_kib"] for r in host_samples)/1024
|
||||
waits = {}
|
||||
max_active = max_idle_tx = 0
|
||||
for sample in pg_samples:
|
||||
max_active = max(max_active, sum(a["count"] for a in sample["activity"] if a["state"] == "active"))
|
||||
max_idle_tx = max(max_idle_tx, sum(a["count"] for a in sample["activity"] if a["state"] == "idle in transaction"))
|
||||
for a in sample["activity"]:
|
||||
if a["state"] == "active":
|
||||
key = f"{a['wait_event_type'] or 'no_wait_event'}/{a['wait_event'] or '-'}"
|
||||
waits[key] = waits.get(key, 0) + a["count"]
|
||||
for role in ("postgres", "nginx", "redis"):
|
||||
resources[role] = resource_summary([r for r in within(rows(directory / f"{role}.jsonl")) if r["kind"] == "cgroup"])
|
||||
upstream = [r for r in within(rows(directory / "nginx.jsonl")) if r["kind"] == "upstream"]
|
||||
changes = sum(a["servers"] != b["servers"] for a, b in zip(upstream, upstream[1:]))
|
||||
m = benchmark["measurement"]
|
||||
return {
|
||||
"label": benchmark["label"], "profile": benchmark["profile"],
|
||||
"measurement": m, "client": benchmark["client"],
|
||||
"resources": resources, "pools": pools,
|
||||
"host": host,
|
||||
"postgres": {"samples": len(pg_samples), "max_active": max_active,
|
||||
"max_idle_in_transaction": max_idle_tx, "active_wait_observations": waits},
|
||||
"nginx": {"samples": len(upstream), "upstream_changes": changes,
|
||||
"min_ready": min((len(r["servers"]) for r in upstream), default=None),
|
||||
"max_ready": max((len(r["servers"]) for r in upstream), default=None)},
|
||||
"limits_reached": {
|
||||
"generator_late": m["scheduler_late_slots"] / m["scheduled_slots"] > 0.05,
|
||||
"client_capacity": (m["client_dropped_slots"] - m["scheduler_late_slots"]) / m["scheduled_slots"] > 0.05,
|
||||
"request_errors": m["all"]["errors"] / max(m["all"]["attempted"], 1) > 0.01,
|
||||
"latency": (m["all"]["successful_latency_ms"]["p95"] or 0) > 1000,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_one(config, profile, rate, repeat, containers):
|
||||
label = f"{profile}-r{rate}-repeat{repeat}"
|
||||
directory = config.output / label
|
||||
directory.mkdir(parents=True, exist_ok=False)
|
||||
workers = []
|
||||
stop = f"/tmp/scaling-observer-{uuid.uuid4().hex}.stop"
|
||||
observer = (ROOT / "scripts/diagnostics/observer.sh").read_bytes()
|
||||
client_container = None
|
||||
try:
|
||||
for role in ("postgres", "nginx", "redis"):
|
||||
output = (directory / f"{role}.jsonl").open("wb")
|
||||
errors = (directory / f"{role}.stderr").open("wb")
|
||||
child = subprocess.Popen(["docker", "exec", "-i", containers[role][0],
|
||||
"sh", "-s", role, stop], cwd=ROOT,
|
||||
stdin=subprocess.PIPE, stdout=output, stderr=errors)
|
||||
workers.append((role, child, output, errors))
|
||||
child.stdin.write(observer)
|
||||
child.stdin.close()
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
target = directory / "benchmark.json"
|
||||
args = ["node", "scripts/benchmark-scaling.mjs", "--base-url", config.base_url,
|
||||
"--label", label, "--rate", str(rate), "--duration-seconds", str(config.duration_seconds),
|
||||
"--warmup-seconds", str(config.warmup_seconds), "--seed-messages", "30",
|
||||
"--expected-instances", str({"one": 1, "two": 2, "four": 4}[profile]),
|
||||
"--output", str(target)]
|
||||
if config.client == "container":
|
||||
nginx = json.loads(command(["docker", "inspect", containers["nginx"][0]]))[0]
|
||||
api = json.loads(command(["docker", "inspect", containers["api"][0]]))[0]
|
||||
networks = set(nginx["NetworkSettings"]["Networks"]) & set(api["NetworkSettings"]["Networks"])
|
||||
if len(networks) != 1:
|
||||
raise RuntimeError("expected one common API/nginx network")
|
||||
client_container = "lab3-diagnostic-client-" + uuid.uuid4().hex
|
||||
args[args.index("--base-url") + 1] = config.container_url
|
||||
args[args.index("--output") + 1] = "/tmp/scaling-benchmark.json"
|
||||
args[1] = "/benchmark.mjs"
|
||||
args = ["docker", "run", "--name", client_container, "--no-healthcheck",
|
||||
"--network", next(iter(networks)),
|
||||
"-v", f"{ROOT / 'scripts/benchmark-scaling.mjs'}:/benchmark.mjs:ro",
|
||||
"--entrypoint", "node", api["Image"], *args[1:]]
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace",
|
||||
timeout=config.duration_seconds + config.warmup_seconds + 120)
|
||||
(directory / "benchmark.stderr").write_text(result.stderr, encoding="utf-8")
|
||||
if client_container is not None:
|
||||
command(["docker", "cp", f"{client_container}:/tmp/scaling-benchmark.json", str(target)])
|
||||
if not target.exists():
|
||||
raise RuntimeError(f"{label}: benchmark did not produce results: {result.stderr}")
|
||||
benchmark = json.loads(target.read_text(encoding="utf-8"))
|
||||
for index, container in enumerate(containers["api"], 1):
|
||||
command(["docker", "cp", f"{container}:/tmp/scaling-pool.jsonl", str(directory / f"api-{index}.jsonl")])
|
||||
ended = datetime.now(timezone.utc).isoformat()
|
||||
# Docker puts nginx stderr notices on CLI stderr; collect both streams.
|
||||
log_result = subprocess.run(["docker", "logs", "--timestamps", "--since", started,
|
||||
"--until", ended, containers["nginx"][0]], cwd=ROOT,
|
||||
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30)
|
||||
diagnostic_lines = [line for line in (log_result.stdout + log_result.stderr).splitlines()
|
||||
if any(key in line for key in ("signal process started", "download timed out",
|
||||
"no live upstreams", "upstream timed out"))]
|
||||
(directory / "nginx-events.log").write_text("\n".join(diagnostic_lines), encoding="utf-8")
|
||||
(directory / "execution.json").write_text(json.dumps({"returncode": result.returncode,
|
||||
"started": started, "ended": ended, "command": args}, indent=2), encoding="utf-8")
|
||||
finally:
|
||||
if client_container is not None:
|
||||
# Only the uniquely named disposable generator created by this run.
|
||||
subprocess.run(["docker", "rm", "-f", client_container], cwd=ROOT, capture_output=True, timeout=30)
|
||||
for role, child, output, errors in workers:
|
||||
try:
|
||||
command(["docker", "exec", containers[role][0], "touch", stop], timeout=15)
|
||||
child.wait(timeout=15)
|
||||
except (RuntimeError, subprocess.TimeoutExpired):
|
||||
child.terminate()
|
||||
child.wait(timeout=10)
|
||||
output.close()
|
||||
errors.close()
|
||||
if role == "postgres":
|
||||
command(["docker", "cp", f"{containers[role][0]}:{stop}.pg.jsonl", str(directory / "postgres-sql.jsonl")])
|
||||
command(["docker", "exec", containers[role][0], "rm", "-f", stop + ".pg.jsonl"])
|
||||
summary = summarize(directory, benchmark)
|
||||
summary["diagnostics_complete"] = (
|
||||
len(summary["pools"]) == len(containers["api"])
|
||||
and summary["postgres"]["samples"] > 0
|
||||
and summary["nginx"]["samples"] > 0
|
||||
and all(v["samples"] > 0 for v in summary["resources"].values())
|
||||
)
|
||||
(directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
||||
if not summary["diagnostics_complete"]:
|
||||
raise RuntimeError(f"{label}: diagnostics incomplete; inspect observer stderr files")
|
||||
m = summary["measurement"]
|
||||
print(f"DONE {label}: RPS={m['all']['successful_rps']} p95={m['all']['successful_latency_ms']['p95']}ms "
|
||||
f"errors={m['all']['errors']} drops={m['client_dropped_slots']}/{m['scheduled_slots']} "
|
||||
f"pool_wait_max={max((p['max_waiting'] for p in summary['pools'].values()), default=0)}", flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
|
||||
parser.add_argument("--client", choices=("host", "container"), default="host")
|
||||
parser.add_argument("--container-url", default="http://nginx")
|
||||
parser.add_argument("--profiles", nargs="+", choices=("one", "two", "four"), default=["one", "two", "four"])
|
||||
parser.add_argument("--rates", nargs="+", type=int, default=[100], help="Scheduled requests/s (default: 100 only)")
|
||||
parser.add_argument("--duration-seconds", type=int, default=60, help="Measured phase per run (default: 60 seconds)")
|
||||
parser.add_argument("--warmup-seconds", type=int, default=15, help="Read-only warmup per run (default: 15 seconds)")
|
||||
parser.add_argument("--repetitions", type=int, default=3)
|
||||
parser.add_argument("--full-matrix", action="store_true", help="Run all requested rates; record threshold crossings instead of stopping a profile")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
config = parser.parse_args()
|
||||
if config.repetitions < 1 or config.duration_seconds < 1 or config.warmup_seconds < 0 or any(r < 1 or r > 1000 for r in config.rates):
|
||||
parser.error("invalid duration, repetitions or rates")
|
||||
config.output = config.output.resolve()
|
||||
config.output.mkdir(parents=True, exist_ok=False)
|
||||
manifest = {"started": datetime.now(timezone.utc).isoformat(), "profiles": config.profiles,
|
||||
"client": config.client,
|
||||
"rates": config.rates, "duration_seconds": config.duration_seconds,
|
||||
"warmup_seconds": config.warmup_seconds, "repetitions": config.repetitions,
|
||||
"full_matrix": config.full_matrix,
|
||||
"stop_rule": ("record only; continue all requested rates after >=2 repeats exceed thresholds: " if config.full_matrix
|
||||
else "stop a profile after >=2 repeats at one rate exceed any threshold: ") +
|
||||
"scheduler late >5%, capacity drops >5%, request errors >1%, success p95 >1000ms",
|
||||
"sampling": "API queue every 100ms; API JSON/cgroup about 1s; other services and PG about 2s",
|
||||
"observer_version": 3,
|
||||
"source_sha256": {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in (
|
||||
"scripts/diagnostics/pool.cjs", "scripts/diagnostics/observer.sh",
|
||||
"scripts/benchmark-scaling.mjs", "scripts/run-scaling-diagnostics.py")},
|
||||
"notes": ["All profiles use the same instrumentation; actual per-replica pool limits are recorded in containers.",
|
||||
"A container client uses the nginx service directly; it shares Docker VM CPU and bypasses Windows port forwarding.",
|
||||
"Acquisition latency includes opening new connections; it is not SQL execution time.",
|
||||
"Observers consume resources inside the existing container budgets.",
|
||||
"Instrumentation stores no tokens, passwords, message text or SQL parameters."],
|
||||
"containers": {}, "environments": {}, "runs": [], "stopped_profiles": {}, "threshold_crossings": {}}
|
||||
manifest_path = config.output / "manifest.json"
|
||||
save = lambda: manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
||||
save()
|
||||
try:
|
||||
for profile in config.profiles:
|
||||
print(f"SWITCH {profile}", flush=True)
|
||||
command(compose(profile) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
|
||||
time.sleep(6) # Allow readiness's two successes and nginx reload to settle.
|
||||
containers = {}
|
||||
for role in ("api", "postgres", "nginx", "redis"):
|
||||
containers[role] = command(compose(profile) + ["ps", "-q", role]).split()
|
||||
if not containers[role]:
|
||||
raise RuntimeError(f"no running {role}")
|
||||
details = json.loads(command(["docker", "inspect", *[c for ids in containers.values() for c in ids]]))
|
||||
running = [json.loads(line) for line in command(["docker", "ps", "--format", "{{json .}}"]).splitlines()]
|
||||
manifest["environments"][profile] = {
|
||||
"docker": json.loads(command(["docker", "info", "--format",
|
||||
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}},"kernel":{{json .KernelVersion}}}'])),
|
||||
"running_containers": [{"name": row.get("Names"), "image": row.get("Image")} for row in running],
|
||||
}
|
||||
manifest["containers"][profile] = [{"id": c["Id"], "name": c["Name"], "image": c["Image"],
|
||||
"nano_cpus": c["HostConfig"]["NanoCpus"], "memory_bytes": c["HostConfig"]["Memory"],
|
||||
"pool_max": next((e.split("=", 1)[1] for e in c["Config"]["Env"] if e.startswith("DB_POOL_MAX=")), None)} for c in details]
|
||||
save()
|
||||
for rate in config.rates:
|
||||
batch = []
|
||||
for repeat in range(1, config.repetitions + 1):
|
||||
print(f"START {profile} rate={rate} repeat={repeat}", flush=True)
|
||||
summary = run_one(config, profile, rate, repeat, containers)
|
||||
batch.append(summary)
|
||||
manifest["runs"].append(summary["label"])
|
||||
save()
|
||||
reasons = [key for key in batch[0]["limits_reached"]
|
||||
if sum(r["limits_reached"][key] for r in batch) >= min(2, config.repetitions)]
|
||||
if reasons:
|
||||
manifest["threshold_crossings"].setdefault(profile, []).append({"rate": rate, "reasons": reasons})
|
||||
if not config.full_matrix:
|
||||
manifest["stopped_profiles"][profile] = {"rate": rate, "reasons": reasons}
|
||||
print(f"{'CONTINUE (full matrix)' if config.full_matrix else 'STOP'} {profile} at {rate}: {', '.join(reasons)}", flush=True)
|
||||
save()
|
||||
if not config.full_matrix:
|
||||
break
|
||||
finally:
|
||||
print("RESTORE two (without diagnostic preload)", flush=True)
|
||||
command(compose("two", diagnostics=False) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
|
||||
manifest["finished"] = datetime.now(timezone.utc).isoformat()
|
||||
save()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
// Generator calibration only: no application or database state.
|
||||
const http = require('node:http');
|
||||
const item = { id: '00000000-0000-4000-8000-000000000001',
|
||||
chat_id: '00000000-0000-4000-8000-000000000002',
|
||||
sender_id: '00000000-0000-4000-8000-000000000003',
|
||||
text: 'Synthetic benchmark payload '.repeat(3), created_at: '2026-01-01T00:00:00.000Z' };
|
||||
const page = JSON.stringify({ items: Array.from({ length: 50 }, () => item), next_cursor: null });
|
||||
const keys = new Set();
|
||||
http.createServer(async (req, res) => {
|
||||
for await (const chunk of req) { /* consume POST body */ }
|
||||
if (req.method === 'POST') {
|
||||
const key = req.headers['idempotency-key'];
|
||||
if (!key || keys.has(key)) {
|
||||
res.writeHead(409, { 'Content-Type': 'application/json', 'X-Instance-Id': 'calibration' });
|
||||
res.end(JSON.stringify({ error: 'calibration detected a repeated write key' }));
|
||||
return;
|
||||
}
|
||||
keys.add(key);
|
||||
}
|
||||
res.writeHead(req.method === 'GET' ? 200 : 201,
|
||||
{ 'Content-Type': 'application/json', 'X-Instance-Id': 'calibration' });
|
||||
res.end(req.method === 'GET' ? page : JSON.stringify(item));
|
||||
}).listen(8080, '0.0.0.0');
|
||||
@@ -0,0 +1,166 @@
|
||||
import http from 'k6/http';
|
||||
import encoding from 'k6/encoding';
|
||||
import execution from 'k6/execution';
|
||||
import { check, fail, sleep } from 'k6';
|
||||
import { Counter, Trend } from 'k6/metrics';
|
||||
|
||||
const base = __ENV.BASE_URL || 'http://nginx';
|
||||
const rate = Number(__ENV.RATE || 100);
|
||||
const seconds = Number(__ENV.SECONDS || 30);
|
||||
const warmup = Number(__ENV.WARMUP || 10);
|
||||
const expected = Number(__ENV.EXPECTED || 1);
|
||||
const calibration = __ENV.CALIBRATION === '1';
|
||||
const attempts = new Counter('bench_attempts');
|
||||
const successes = new Counter('bench_successes');
|
||||
const failures = new Counter('bench_failures');
|
||||
const latency = new Trend('bench_latency', true);
|
||||
const successLatency = new Trend('bench_success_latency', true);
|
||||
const epoch = new Trend('bench_start_epoch');
|
||||
const operations = Object.fromEntries(['read', 'write'].map((op) => [op, {
|
||||
attempts: new Counter(`bench_${op}_attempts`),
|
||||
successes: new Counter(`bench_${op}_successes`),
|
||||
failures: new Counter(`bench_${op}_failures`),
|
||||
latency: new Trend(`bench_${op}_latency`, true),
|
||||
}]));
|
||||
const statuses = Object.fromEntries(['200', '201', 'other', 'network'].map((key) => [key, new Counter(`bench_status_${key}`)]));
|
||||
|
||||
const arrival = { executor: 'constant-arrival-rate', rate, timeUnit: '1s',
|
||||
preAllocatedVUs: 128, maxVUs: 128 };
|
||||
export const options = {
|
||||
scenarios: {
|
||||
warmup: { ...arrival, exec: 'warmupLoad', duration: `${warmup}s`, gracefulStop: '5s' },
|
||||
measurement: { ...arrival, exec: 'default', startTime: `${warmup}s`,
|
||||
duration: `${seconds}s`, gracefulStop: '10s' },
|
||||
},
|
||||
summaryTrendStats: ['min', 'med', 'avg', 'max', 'p(95)', 'p(99)'],
|
||||
setupTimeout: '120s',
|
||||
};
|
||||
|
||||
function request(method, path, token, body, key) {
|
||||
const headers = { Accept: 'application/json' };
|
||||
if (token) headers.Authorization = token;
|
||||
if (key) headers['Idempotency-Key'] = key;
|
||||
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||
return http.request(method, base + path, body === undefined ? null : JSON.stringify(body), {
|
||||
headers, timeout: '5s', tags: { name: path.includes('/messages') ? 'messages' : 'setup' },
|
||||
});
|
||||
}
|
||||
|
||||
function required(response, status, field) {
|
||||
let data;
|
||||
try { data = response.json(); } catch { fail(`setup: invalid JSON, status ${response.status}`); }
|
||||
if (response.status !== status || !data?.[field]) fail(`setup: expected ${status}/${field}, got ${response.status}`);
|
||||
return data;
|
||||
}
|
||||
|
||||
export function setup() {
|
||||
const salt = `${Date.now().toString(36)}-${Math.floor(Math.random() * 1e9)}`;
|
||||
let token = 'Bearer calibration';
|
||||
let path = '/messages';
|
||||
if (!calibration) {
|
||||
const users = ['alice', 'bob'].map((label) => {
|
||||
const username = `bench_${label}_${salt.replaceAll('-', '')}`;
|
||||
const password = `T3st-${salt}-Aa1!`;
|
||||
const user = required(request('POST', '/api/v1/users', null,
|
||||
{ username, display_name: `Benchmark ${label}`, password }), 201, 'id');
|
||||
const auth = `Basic ${encoding.b64encode(`${username}:${password}`)}`;
|
||||
const session = required(request('POST', '/api/v1/auth/sessions', auth), 201, 'token');
|
||||
return { id: user.id, token: session.token };
|
||||
});
|
||||
token = `Bearer ${users[0].token}`;
|
||||
const chat = required(request('POST', '/api/v1/chats', token,
|
||||
{ title: 'Saturation benchmark', member_ids: users.map((u) => u.id) }), 201, 'id');
|
||||
path = `/api/v1/chats/${chat.id}/messages`;
|
||||
for (let i = 0; i < 30; i++) required(request('POST', path, token,
|
||||
{ text: `seed-${salt}-${i}` }, `seed-${salt}-${i}`), 201, 'id');
|
||||
}
|
||||
const seen = new Set();
|
||||
const until = Date.now() + 20000;
|
||||
do {
|
||||
const response = request('GET', `${path}?limit=50`, token);
|
||||
if (response.status !== 200) fail(`warmup status ${response.status}`);
|
||||
const id = response.headers['X-Instance-Id'];
|
||||
if (id) seen.add(id);
|
||||
if (seen.size === expected) break;
|
||||
sleep(0.02);
|
||||
} while (Date.now() < until);
|
||||
if (seen.size !== expected) fail(`warmup observed ${seen.size} instances, expected ${expected}`);
|
||||
return { token, path, salt };
|
||||
}
|
||||
|
||||
export function warmupLoad(data) { iteration(data, false); }
|
||||
export default function (data) { iteration(data, true); }
|
||||
|
||||
function iteration(data, measuring) {
|
||||
if (measuring && execution.scenario.iterationInTest === 0) epoch.add(Date.now());
|
||||
const sequence = execution.scenario.iterationInTest;
|
||||
const identity = `${execution.scenario.name}-${sequence}`;
|
||||
const op = ((sequence * 37 + 17) % 100) < 80 ? 'read' : 'write';
|
||||
if (measuring) { attempts.add(1); operations[op].attempts.add(1); }
|
||||
const response = op === 'read'
|
||||
? request('GET', `${data.path}?limit=50`, data.token)
|
||||
: request('POST', data.path, data.token, { text: `bench-${identity}-${data.salt}` }, `${data.salt}-${identity}`);
|
||||
if (!measuring) return;
|
||||
let payload;
|
||||
try { payload = response.json(); } catch { payload = null; }
|
||||
const instance = response.headers['X-Instance-Id'];
|
||||
const ok = response.status === (op === 'read' ? 200 : 201) && Boolean(instance)
|
||||
&& (op === 'read' ? Array.isArray(payload?.items) : Boolean(payload?.id));
|
||||
const elapsed = response.timings.duration + response.timings.blocked;
|
||||
latency.add(elapsed);
|
||||
successes.add(ok ? 1 : 0);
|
||||
failures.add(ok ? 0 : 1);
|
||||
operations[op].successes.add(ok ? 1 : 0);
|
||||
operations[op].failures.add(ok ? 0 : 1);
|
||||
if (ok) { successLatency.add(elapsed); operations[op].latency.add(elapsed); }
|
||||
statuses[String(response.status)]?.add(1);
|
||||
if (![200, 201].includes(response.status)) statuses[response.status === 0 ? 'network' : 'other'].add(1);
|
||||
if (instance) check(response, { [`instance:${instance}`]: () => true });
|
||||
}
|
||||
|
||||
export function handleSummary(data) {
|
||||
const count = (name) => data.metrics[name]?.values.count || 0;
|
||||
const times = (name) => ({ p50: data.metrics[name]?.values.med ?? null,
|
||||
p95: data.metrics[name]?.values['p(95)'] ?? null,
|
||||
p99: data.metrics[name]?.values['p(99)'] ?? null });
|
||||
const start = data.metrics.bench_start_epoch?.values.min;
|
||||
const totalDropped = count('dropped_iterations');
|
||||
const dropped = Math.max(0, rate * seconds - count('bench_attempts'));
|
||||
const instances = Object.fromEntries((data.root_group.checks || []).filter((c) => c.name.startsWith('instance:'))
|
||||
.map((c) => [c.name.slice(9), c.passes]));
|
||||
const group = (op) => {
|
||||
const prefix = op === 'all' ? 'bench' : `bench_${op}`;
|
||||
return { attempted: count(`${prefix}_attempts`), successful: count(`${prefix}_successes`),
|
||||
errors: count(`${prefix}_failures`), successful_rps: count(`${prefix}_successes`) / seconds,
|
||||
successful_latency_ms: times(op === 'all' ? 'bench_success_latency' : `${prefix}_latency`),
|
||||
status_counts: {}, error_counts: {}, instance_counts: op === 'all' ? instances : {} };
|
||||
};
|
||||
const result = {
|
||||
label: __ENV.LABEL, profile: { arrival_model: 'k6 constant-arrival-rate; one HTTP request per iteration',
|
||||
offered_rps: rate, duration_seconds: seconds, warmup_seconds: warmup, warmup_profile: 'same 80/20 workload at target rate',
|
||||
read_percent: 80, write_percent: 20, initial_messages: calibration ? 0 : 30, history_limit: 50,
|
||||
max_in_flight: 128, timeout_ms: 5000, automatic_retries: false, expected_instances: expected,
|
||||
phase: __ENV.PHASE || 'probe' },
|
||||
measurement: { scheduled_slots: rate * seconds, client_dropped_slots: dropped,
|
||||
scheduler_late_slots: 0, read: group('read'), write: group('write'), all: group('all') },
|
||||
timing: { measurement_started_epoch_ms: start,
|
||||
measurement_scheduled_end_epoch_ms: start + seconds * 1000,
|
||||
measurement_completed_epoch_ms: Date.now() },
|
||||
client: { engine: 'k6', scheduler_lateness_measured: false,
|
||||
raw_dropped_iterations_including_warmup: totalDropped,
|
||||
peak_vus: data.metrics.vus?.values.max, max_vus: 128 },
|
||||
all_latency_ms: times('bench_latency'), raw_metrics: data.metrics,
|
||||
notes: ['Setup and in-scenario mixed warmup excluded from custom bench metrics.',
|
||||
'Native warmup and measurement scenarios; no Date.now gating of measured requests.',
|
||||
'Latency uses native HTTP duration plus blocked time, not subtraction of wall-clock timestamps.',
|
||||
'Measurement missing slots are planned minus actual starts; raw k6 dropped iterations also include warmup.',
|
||||
'scheduler_late_slots is a compatibility placeholder, not a k6 measurement.',
|
||||
'Dropped iterations and actual delivered fraction must be checked.',
|
||||
'No credentials or message bodies are exported.'],
|
||||
};
|
||||
result.measurement.all.status_counts = Object.fromEntries(['200', '201', 'other', 'network']
|
||||
.map((status) => [status, count(`bench_status_${status}`)]));
|
||||
return { [__ENV.OUTPUT]: JSON.stringify(result, null, 2),
|
||||
stdout: JSON.stringify({ label: result.label, attempted: result.measurement.all.attempted,
|
||||
errors: result.measurement.all.errors, p95: result.all_latency_ms.p95, dropped }) + '\n' };
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create CSV and Markdown from completed diagnostic runs (can run during a sweep)."""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import importlib.util
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--refresh", action="store_true", help="Recompute summaries from raw records")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
results = []
|
||||
raw_summaries = []
|
||||
if args.refresh:
|
||||
spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).with_name("run-scaling-diagnostics.py"))
|
||||
diagnostics = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(diagnostics)
|
||||
for label in manifest["runs"]:
|
||||
path = args.directory / label / "summary.json"
|
||||
if not path.exists():
|
||||
continue
|
||||
s = json.loads(path.read_text(encoding="utf-8"))
|
||||
if args.refresh:
|
||||
benchmark = json.loads((path.parent / "benchmark.json").read_text(encoding="utf-8"))
|
||||
refreshed = diagnostics.summarize(path.parent, benchmark)
|
||||
refreshed["diagnostics_complete"] = s.get("diagnostics_complete", False)
|
||||
s = refreshed
|
||||
path.write_text(json.dumps(s, indent=2) + "\n", encoding="utf-8")
|
||||
raw_summaries.append(s)
|
||||
m, p, r = s["measurement"], s["pools"], s["resources"]
|
||||
acquisitions = sum(v["acquisitions"] for v in p.values())
|
||||
results.append({
|
||||
"label": label, "replicas": s["profile"]["expected_instances"],
|
||||
"configuration": next((profile for profile in manifest.get("profiles", [])
|
||||
if label.startswith(profile + "-r")), str(s["profile"]["expected_instances"])),
|
||||
"phase": s["profile"].get("phase", "measurement"),
|
||||
"delivery_valid": s.get("criterion", {}).get("delivery_valid"),
|
||||
"duration_seconds": s["profile"]["duration_seconds"],
|
||||
"offered_rps": s["profile"]["offered_rps"], "successful_rps": m["all"]["successful_rps"],
|
||||
"attempted_rps": m["all"]["attempted"] / s["profile"]["duration_seconds"],
|
||||
"p50_ms": m["all"]["successful_latency_ms"]["p50"],
|
||||
"p95_ms": m["all"]["successful_latency_ms"]["p95"],
|
||||
"all_request_p95_ms": s.get("all_latency_ms", {}).get("p95"),
|
||||
"p99_ms": m["all"]["successful_latency_ms"]["p99"],
|
||||
"errors": m["all"]["errors"], "dropped": m["client_dropped_slots"],
|
||||
"drop_percent": 100*m["client_dropped_slots"]/m["scheduled_slots"],
|
||||
"late_percent": 100*m["scheduler_late_slots"]/m["scheduled_slots"],
|
||||
"acquisition_mean_ms": sum(v["mean_acquisition_ms"]*v["acquisitions"] for v in p.values()) / max(acquisitions, 1),
|
||||
"acquisition_max_ms": max((v["max_acquisition_ms"] for v in p.values()), default=0),
|
||||
"pool_max_waiting": max((v["max_waiting"] for v in p.values()), default=0),
|
||||
"pool_observed_connection_increases": sum(v.get("observed_connection_increases", 0) for v in p.values()),
|
||||
"pool_observed_connection_decreases": sum(v.get("observed_connection_decreases", 0) for v in p.values()),
|
||||
"api_cpu_cores": sum(v.get("mean_cpu_cores", 0) for k, v in r.items() if k.startswith("api-")),
|
||||
"api_max_throttle_percent": 100*max((v.get("throttled_period_fraction", 0) for k, v in r.items() if k.startswith("api-")), default=0),
|
||||
"pg_cpu_cores": r["postgres"].get("mean_cpu_cores"),
|
||||
"pg_throttle_percent": 100*r["postgres"].get("throttled_period_fraction", 0),
|
||||
"nginx_cpu_cores": r["nginx"].get("mean_cpu_cores"),
|
||||
"redis_cpu_cores": r["redis"].get("mean_cpu_cores"),
|
||||
"host_cpu_pressure_percent": 100*s.get("host", {}).get("cpu_some_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
|
||||
"host_memory_pressure_percent": 100*s.get("host", {}).get("memory_full_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
|
||||
"host_io_pressure_percent": 100*s.get("host", {}).get("io_full_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
|
||||
"host_min_available_mib": s.get("host", {}).get("min_available_mib"),
|
||||
"host_max_swap_used_mib": s.get("host", {}).get("max_swap_used_mib"),
|
||||
"pg_max_active": s["postgres"]["max_active"],
|
||||
"pg_max_idle_tx": s["postgres"]["max_idle_in_transaction"],
|
||||
"pg_active_wait_observations": json.dumps(s["postgres"]["active_wait_observations"], sort_keys=True),
|
||||
"nginx_min_ready": s["nginx"]["min_ready"], "nginx_changes": s["nginx"]["upstream_changes"],
|
||||
"min_resource_coverage_seconds": min(v.get("coverage_seconds", 0) for v in r.values()),
|
||||
})
|
||||
if not results:
|
||||
print("No completed runs yet")
|
||||
return
|
||||
with (args.directory / "results.csv").open("w", encoding="utf-8", newline="") as output:
|
||||
writer = csv.DictWriter(output, fieldnames=results[0].keys())
|
||||
writer.writeheader()
|
||||
writer.writerows(results)
|
||||
pg_limits = next((c for containers in manifest.get("containers", {}).values()
|
||||
for c in containers if "-postgres-" in c.get("name", "")), None)
|
||||
pg_description = (f"PostgreSQL — {pg_limits['nano_cpus']/1e9:g} CPU/{pg_limits['memory_bytes']/1024**2:g} МиБ."
|
||||
if pg_limits else "Лимиты PostgreSQL не записаны в manifest.")
|
||||
pool_descriptions = [f"{profile}: " + "/".join(sorted({str(c['pool_max']) for c in containers
|
||||
if "-api-" in c.get("name", "") and c.get("pool_max") is not None}))
|
||||
for profile, containers in manifest.get("containers", {}).items()]
|
||||
pool_description = "; ".join(pool_descriptions) or "см. manifest"
|
||||
api_budgets = []
|
||||
for profile, containers in manifest.get("containers", {}).items():
|
||||
apis = [c for c in containers if "-api-" in c.get("name", "")]
|
||||
if apis:
|
||||
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
|
||||
memory = sum(c['memory_bytes'] for c in apis) / 1024**2
|
||||
api_budgets.append(f"{profile}: {cpu:g} CPU/{memory:g} МиБ")
|
||||
api_description = "; ".join(api_budgets) or "см. manifest"
|
||||
probe_description = (f"Разведочные прогоны: {manifest['probe_duration_seconds']} с после "
|
||||
f"{manifest['probe_warmup_seconds']} с прогрева. Указанная ниже длительность "
|
||||
"относится к подтверждающим прогонам. " if "probe_duration_seconds" in manifest else "")
|
||||
lines = ["# Диагностическая серия масштабирования", "",
|
||||
probe_description + f"Завершено прогонов: {len(results)}. Длительность измерения — {manifest['duration_seconds']} с, "
|
||||
f"прогрев — {manifest['warmup_seconds']} с. Пул на реплику — {pool_description}. Общий бюджет API — {api_description}; " + pg_description, "",
|
||||
("Клиент: Linux-контейнер через внутренний адрес Nginx; разделяет ресурсы Docker VM." if manifest.get("client") == "container"
|
||||
else "Клиент: Windows-хост через опубликованный порт Nginx."),
|
||||
f"Версия SQL-наблюдателя: {manifest.get('observer_version', '1/2, первоначальная серия')}. "
|
||||
"Серия с версиями 1/2 имеет повышенные накладные расходы наблюдения; см. калибровку в итоговой интерпретации.", "",
|
||||
"Методика и повторение: [scripts/diagnostics/README.md](../../../scripts/diagnostics/README.md). "
|
||||
"Исходные HTTP-данные, временные ряды и summary каждого прогона находятся в его подкаталоге. "
|
||||
"Полная таблица метрик: [results.csv](results.csv). Все настройки и причины остановки нагрузки: [manifest.json](manifest.json).", "",
|
||||
"Показатели ниже — результаты каждого прогона. CPU выражен в долях одного ядра; "
|
||||
"throttle — доля периодов CPU, в которых cgroup ограничивала выполнение. Получение соединения "
|
||||
"включает установку новых соединений и задержки event loop. Очередь наблюдалась каждые 100 мс. "
|
||||
"Наблюдатели имеют собственные накладные расходы; результаты не являются ненаблюдаемой максимальной ёмкостью API.", "",
|
||||
"| Прогон | Успешные RPS | p95, мс | Ошибки | Пропуски, % | Пул: среднее / max, мс | Max очередь | CPU API / PG | Throttle API max / PG, % | Upstream min / изменений |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
|
||||
for r in results:
|
||||
lines.append(f"| [{r['label']}]({r['label']}/summary.json) | {r['successful_rps']:.2f} | "
|
||||
f"{r['p95_ms']:.0f} | {r['errors']} | {r['drop_percent']:.1f} | "
|
||||
f"{r['acquisition_mean_ms']:.2f} / {r['acquisition_max_ms']:.0f} | {r['pool_max_waiting']} | "
|
||||
f"{r['api_cpu_cores']:.2f} / {r['pg_cpu_cores'] or 0:.2f} | "
|
||||
f"{r['api_max_throttle_percent']:.1f} / {r['pg_throttle_percent']:.1f} | "
|
||||
f"{r['nginx_min_ready']} / {r['nginx_changes']} |")
|
||||
lines += ["", "## Повторы на одном уровне нагрузки", "",
|
||||
"Здесь показаны медианы результатов отдельных прогонов. Медиана p95 не является p95 объединённых запросов.", "",
|
||||
"| Профиль | Задано RPS | Повторов | Медиана отправленных RPS | Медиана успешных RPS | Медиана p95, мс | Всего ошибок | Медиана пропусков, % |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
|
||||
for key in sorted({(r["configuration"], r["offered_rps"]) for r in results}):
|
||||
group = [r for r in results if (r["configuration"], r["offered_rps"]) == key]
|
||||
lines.append(f"| {key[0]} | {key[1]} | {len(group)} | "
|
||||
f"{statistics.median(r['attempted_rps'] for r in group):.2f} | "
|
||||
f"{statistics.median(r['successful_rps'] for r in group):.2f} | "
|
||||
f"{statistics.median(r['p95_ms'] for r in group):.0f} | "
|
||||
f"{sum(r['errors'] for r in group)} | "
|
||||
f"{statistics.median(r['drop_percent'] for r in group):.1f} |")
|
||||
lines += ["", "## Границы серии", ""]
|
||||
for profile, stop in manifest["stopped_profiles"].items():
|
||||
lines.append(f"- {profile}: повышение остановлено после {stop['rate']} запросов/с: {', '.join(stop['reasons'])}.")
|
||||
if manifest.get("full_matrix"):
|
||||
lines.append("Режим полной матрицы: превышения порогов не останавливают повышение нагрузки; все перечисленные точки запланированы к выполнению.")
|
||||
for profile, crossings in manifest.get("threshold_crossings", {}).items():
|
||||
for crossing in crossings:
|
||||
lines.append(f"- {profile}, {crossing['rate']} запросов/с: {', '.join(crossing['reasons'])}.")
|
||||
lines += ["", "Превышение порога задержки или пропусков не доказывает насыщение конкретного компонента. "
|
||||
"Для интерпретации необходимо сопоставить очередь пула, ограничения CPU, ожидания PostgreSQL и изменения upstream. "
|
||||
"SQL-тексты, токены и пароли в этой серии не сохраняются.", ""]
|
||||
if (args.directory / "findings.md").exists():
|
||||
lines += ["## Итоговая интерпретация", "",
|
||||
"[Выводы и графики](findings.md). Проверка полноты серии: [audit.json](audit.json).", ""]
|
||||
(args.directory / "README.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"Wrote {len(results)} runs to {args.directory / 'results.csv'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user