perf(bench): add scaling experiments and saturation diagnostics

Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
This commit is contained in:
2026-10-29 11:49:51 +03:00
parent 5c8bcf0f80
commit c1fec73ccc
16 changed files with 2180 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env python3
"""Compatibility entry point for the consolidated report figures."""
import runpy
from pathlib import Path
import sys
if __name__ == '__main__':
helper = Path(__file__).with_name('report-evidence.py')
sys.argv = [str(helper), '--plot', *sys.argv[1:]]
runpy.run_path(str(helper), run_name='__main__')
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Check saved source snapshots, run completion and actual container budgets."""
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('directory', type=Path)
args = parser.parse_args()
root = args.directory
manifest = json.loads((root / 'manifest.json').read_text(encoding='utf-8'))
sources = {name: hashlib.sha256((root / 'sources' / name).read_bytes()).hexdigest() == digest
for name, digest in manifest['source_sha256'].items()}
budgets = {}
images = set()
for profile, containers in manifest['containers'].items():
apis = [c for c in containers if '-api-' in c['name']]
images.update(c['image'] for c in apis)
budgets[profile] = {'replicas': len(apis), 'cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
'ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
'total_pool': sum(int(c['pool_max']) for c in apis)}
runs = {}
groups = Counter()
for label in manifest['runs']:
path = root / label
summary = json.loads((path / 'summary.json').read_text(encoding='utf-8'))
execution = json.loads((path / 'execution.json').read_text(encoding='utf-8'))
profile = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
expected = budgets[profile]['replicas']
rate = summary['profile']['offered_rps']
if summary['profile'].get('phase') == 'confirmation':
groups[f'{profile}/r{rate}'] += 1
runs[label] = {'returncode': execution['returncode'],
'diagnostics_complete': summary.get('diagnostics_complete', False),
'replicas_match': summary['profile']['expected_instances'] == expected,
'criterion': summary.get('criterion')}
requested = manifest.get('requested_confirmation_rates')
complete_groups = all(count == manifest['repetitions'] for count in groups.values())
if requested:
complete_groups = complete_groups and all(
groups[f'{profile}/r{rate}'] == manifest['repetitions']
for profile in manifest['profiles'] for rate in requested)
checks = {'finished': bool(manifest.get('finished')), 'source_snapshots_match': all(sources.values()),
'one_api_image': len(images) == 1, 'all_runs_complete': bool(runs) and all(
r['returncode'] == 0 and r['diagnostics_complete'] and r['replicas_match'] for r in runs.values()),
'confirmation_groups_complete': bool(groups) and complete_groups,
'pool_budget_matches': all(b['total_pool'] == manifest['pool_budget'] for b in budgets.values())}
audit = {'checks_pass': all(checks.values()), 'checks': checks, 'budgets': budgets,
'source_snapshots': sources, 'confirmation_groups': dict(groups), 'runs': runs}
(root / 'audit.json').write_text(json.dumps(audit, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'checks_pass': audit['checks_pass'], 'checks': checks, 'budgets': budgets,
'runs': len(runs)}, indent=2))
if not audit['checks_pass']:
raise SystemExit(1)
if __name__ == '__main__':
main()
+402
View File
@@ -0,0 +1,402 @@
#!/usr/bin/env node
/**
* Synthetic lab3 scaling benchmark. Runs against an already configured stack.
* The client is outside the Docker resource budget; no automatic POST retries.
*/
import { execFile as execFileCallback } from 'node:child_process';
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
import { promisify } from 'node:util';
import { randomUUID } from 'node:crypto';
const execFile = promisify(execFileCallback);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
function options(argv) {
const defaults = {
'base-url': 'http://127.0.0.1:8080',
label: 'unlabelled',
rate: '100',
'duration-seconds': '30',
'warmup-seconds': '5',
'max-in-flight': '64',
'read-percent': '80',
'seed-messages': '100',
'history-limit': '50',
'timeout-ms': '5000',
'docker-stats-interval-seconds': '2',
};
const allowed = new Set([...Object.keys(defaults), 'output', 'docker-stats', 'expected-instances']);
const values = { ...defaults };
for (let i = 0; i < argv.length; i += 1) {
const name = argv[i];
if (name === '--help') {
console.log(`Usage: node scripts/benchmark-scaling.mjs [options]
--base-url URL Nginx URL (default: http://127.0.0.1:8080)
--label NAME Configuration label for later comparisons
--rate N Scheduled HTTP requests per second (default: 100)
--duration-seconds N Measured phase (default: 30)
--warmup-seconds N Read-only warmup (default: 5)
--max-in-flight N Client-side outstanding request cap (default: 64)
--read-percent N Percentage of history GETs (default: 80)
--seed-messages N Initial messages in a fresh chat (default: 100)
--history-limit N Messages per GET page (default: 50)
--timeout-ms N Per-request deadline (default: 5000)
--docker-stats Sample CPU/RAM of current Compose containers
--docker-stats-interval-seconds N Sampling interval (default: 2)
--expected-instances N Require N distinct X-Instance-Id values
--output PATH Save JSON result; stdout always prints summary
`);
process.exit(0);
}
if (!name.startsWith('--') || !allowed.has(name.slice(2))) {
throw new Error(`unknown option: ${name}`);
}
const key = name.slice(2);
if (key === 'docker-stats') {
values[key] = true;
continue;
}
const value = argv[++i];
if (value === undefined || value.startsWith('--')) {
throw new Error(`missing value for ${name}`);
}
values[key] = value;
}
const number = (key, min, max) => {
const value = Number(values[key]);
if (!Number.isFinite(value) || value < min || value > max) {
throw new Error(`${key} must be ${min}..${max}`);
}
return value;
};
const parsed = {
baseUrl: values['base-url'].replace(/\/$/, ''),
label: values.label,
rate: number('rate', 1, 1000),
durationSeconds: number('duration-seconds', 1, 3600),
warmupSeconds: number('warmup-seconds', 0, 600),
maxInFlight: number('max-in-flight', 1, 4096),
readPercent: number('read-percent', 0, 100),
seedMessages: number('seed-messages', 0, 10000),
historyLimit: number('history-limit', 1, 100),
timeoutMs: number('timeout-ms', 100, 120000),
dockerStats: Boolean(values['docker-stats']),
dockerStatsIntervalSeconds: number('docker-stats-interval-seconds', 0.5, 60),
expectedInstances: values['expected-instances'] === undefined
? null : Number(values['expected-instances']),
output: values.output,
};
if (!/^https?:\/\//.test(parsed.baseUrl)) {
throw new Error('base-url must begin with http:// or https://');
}
if (!/^[a-zA-Z0-9_.-]{1,80}$/.test(parsed.label)) {
throw new Error('label must use letters, digits, dot, underscore or dash');
}
if (parsed.rate * parsed.durationSeconds > 200000) {
throw new Error('at most 200000 measured request slots are allowed per run');
}
for (const key of ['rate', 'maxInFlight', 'readPercent', 'seedMessages',
'historyLimit', 'timeoutMs']) {
if (!Number.isInteger(parsed[key])) throw new Error(`${key} must be an integer`);
}
if (parsed.expectedInstances !== null &&
(!Number.isInteger(parsed.expectedInstances) || parsed.expectedInstances < 1 ||
parsed.expectedInstances > 16)) {
throw new Error('expected-instances must be an integer from 1 to 16');
}
return parsed;
}
async function api(baseUrl, method, path, { body, authorization, key, timeoutMs } = {}) {
const headers = { Accept: 'application/json' };
if (authorization) headers.Authorization = authorization;
if (key) headers['Idempotency-Key'] = key;
if (body !== undefined) headers['Content-Type'] = 'application/json';
const response = await fetch(baseUrl + path, {
method,
headers,
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(timeoutMs ?? 10000),
});
const payload = await response.json().catch(() => null);
return { status: response.status, payload, instance: response.headers.get('x-instance-id') };
}
async function bootstrap(config) {
const suffix = randomUUID().replaceAll('-', '').slice(0, 12);
const password = `T3st-${randomUUID()}`;
const users = [];
for (const label of ['alice', 'bob']) {
const username = `bench_${label}_${suffix}`;
const created = await api(config.baseUrl, 'POST', '/api/v1/users', {
body: { username, display_name: `Benchmark ${label}`, password },
});
if (created.status !== 201 || !created.payload?.id) {
throw new Error(`setup: user registration returned ${created.status}`);
}
const basic = Buffer.from(`${username}:${password}`).toString('base64');
const session = await api(config.baseUrl, 'POST', '/api/v1/auth/sessions', {
authorization: `Basic ${basic}`,
});
if (session.status !== 201 || !session.payload?.token) {
throw new Error(`setup: session creation returned ${session.status}`);
}
users.push({ id: created.payload.id, token: session.payload.token });
}
const authorization = `Bearer ${users[0].token}`;
const chat = await api(config.baseUrl, 'POST', '/api/v1/chats', {
body: { title: 'Scaling benchmark', member_ids: users.map((user) => user.id) },
authorization,
});
if (chat.status !== 201 || !chat.payload?.id) {
throw new Error(`setup: chat creation returned ${chat.status}`);
}
const path = `/api/v1/chats/${chat.payload.id}/messages`;
for (let index = 0; index < config.seedMessages; index += 1) {
const seeded = await api(config.baseUrl, 'POST', path, {
body: { text: `seed-${suffix}-${index}` }, authorization, key: randomUUID(),
});
if (seeded.status !== 201 || !seeded.payload?.id) {
throw new Error(`setup: seed message ${index + 1} returned ${seeded.status}`);
}
}
return { authorization, path };
}
function percentile(sorted, fraction) {
if (!sorted.length) return null;
return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(3));
}
function summarize(events, durationSeconds, slots, dropped, late) {
const groups = {};
for (const kind of ['read', 'write', 'all']) {
const chosen = kind === 'all' ? events : events.filter((event) => event.kind === kind);
const successes = chosen.filter((event) => event.ok);
const latencies = successes.map((event) => event.latency_ms).sort((a, b) => a - b);
const statuses = {};
const errors = {};
const instances = {};
for (const event of chosen) {
const status = event.status === null ? 'network' : String(event.status);
statuses[status] = (statuses[status] ?? 0) + 1;
if (!event.ok) errors[event.error] = (errors[event.error] ?? 0) + 1;
if (event.instance) instances[event.instance] = (instances[event.instance] ?? 0) + 1;
}
groups[kind] = {
attempted: chosen.length,
successful: successes.length,
errors: chosen.length - successes.length,
successful_rps: Number((successes.length / durationSeconds).toFixed(3)),
status_counts: statuses,
error_counts: errors,
instance_counts: instances,
successful_latency_ms: {
p50: percentile(latencies, 0.5),
p95: percentile(latencies, 0.95),
p99: percentile(latencies, 0.99),
},
};
}
return { scheduled_slots: slots, client_dropped_slots: dropped,
scheduler_late_slots: late, ...groups };
}
async function runPhase(config, context, durationSeconds, readPercent) {
const slots = Math.floor(config.rate * durationSeconds);
const intervalMs = 1000 / config.rate;
const started = performance.now();
const startedEpochMs = Date.now();
const events = [];
const pending = new Set();
let dropped = 0;
let late = 0;
for (let sequence = 0; sequence < slots; sequence += 1) {
const scheduledAt = started + sequence * intervalMs;
const waitMs = scheduledAt - performance.now();
if (waitMs > 0) await sleep(waitMs);
const actualAt = performance.now();
if (actualAt - scheduledAt > intervalMs) {
late += 1;
dropped += 1;
continue;
}
if (pending.size >= config.maxInFlight) {
dropped += 1;
continue;
}
const kind = ((sequence * 37 + 17) % 100) < readPercent ? 'read' : 'write';
const task = (async () => {
let status = null;
let instance = null;
let error = null;
try {
const result = kind === 'read'
? await api(config.baseUrl, 'GET', `${context.path}?limit=${config.historyLimit}`, {
authorization: context.authorization, timeoutMs: config.timeoutMs,
})
: await api(config.baseUrl, 'POST', context.path, {
authorization: context.authorization, timeoutMs: config.timeoutMs,
key: randomUUID(), body: { text: `bench-${sequence}-${randomUUID()}` },
});
status = result.status;
instance = result.instance;
if (status !== (kind === 'read' ? 200 : 201)) error = `http_${status}`;
else if (!instance) error = 'missing_instance_id';
else if (kind === 'read' && !Array.isArray(result.payload?.items)) error = 'invalid_page';
else if (kind === 'write' && !result.payload?.id) error = 'invalid_message';
} catch (caught) {
error = caught?.name === 'TimeoutError' ? 'timeout' : 'network_or_parse';
}
events.push({ sequence, kind, ok: error === null, status, error, instance,
scheduled_ms: Number((scheduledAt - started).toFixed(3)),
start_lag_ms: Number((actualAt - scheduledAt).toFixed(3)),
latency_ms: Number((performance.now() - actualAt).toFixed(3)) });
})();
pending.add(task);
void task.finally(() => pending.delete(task));
}
const scheduledEnd = started + durationSeconds * 1000;
if (performance.now() < scheduledEnd) await sleep(scheduledEnd - performance.now());
await Promise.all(pending);
return { events, started_epoch_ms: startedEpochMs,
scheduled_end_epoch_ms: startedEpochMs + durationSeconds * 1000,
completed_epoch_ms: Date.now(),
summary: summarize(events, durationSeconds, slots, dropped, late),
drain_seconds: Number(((performance.now() - scheduledEnd) / 1000).toFixed(3)) };
}
function bytesToMiB(value) {
const match = /^([\d.]+)\s*([KMGT]?i?B)$/i.exec(value.trim());
if (!match) return null;
const scale = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12,
KIB: 1024, MIB: 1024 ** 2, GIB: 1024 ** 3, TIB: 1024 ** 4 };
return Number((Number(match[1]) * scale[match[2].toUpperCase()] / 1024 ** 2).toFixed(3));
}
async function dockerSampler(intervalSeconds) {
const { stdout } = await execFile('docker', ['compose', 'ps', '-q'], {
timeout: 15000, maxBuffer: 1024 * 1024,
});
const ids = stdout.trim().split(/\s+/).filter(Boolean);
if (!ids.length) throw new Error('docker-stats: no running Compose containers');
let stopped = false;
const samples = [];
const errors = [];
const task = (async () => {
while (!stopped) {
const started = performance.now();
try {
const result = await execFile('docker',
['stats', '--no-stream', '--format', '{{json .}}', ...ids],
{ timeout: 15000, maxBuffer: 1024 * 1024 });
for (const line of result.stdout.trim().split(/\r?\n/).filter(Boolean)) {
const row = JSON.parse(line);
samples.push({ time_ms: Number(started.toFixed(3)), container: row.Name,
cpu_percent: Number(row.CPUPerc.replace('%', '')),
memory_mib: bytesToMiB(row.MemUsage.split('/')[0]),
memory_limit_mib: bytesToMiB(row.MemUsage.split('/')[1]),
pids: Number(row.PIDs) });
}
} catch (error) {
errors.push(error?.code || error?.name || 'docker_stats_error');
}
const waitMs = Math.max(0, intervalSeconds * 1000 - (performance.now() - started));
if (!stopped && waitMs) await sleep(waitMs);
}
})();
return { stop: async () => { stopped = true; await task; return { samples, errors }; } };
}
async function main() {
const config = options(process.argv.slice(2));
const context = await bootstrap(config);
const warmup = config.warmupSeconds
? await runPhase(config, context, config.warmupSeconds, 100)
: null;
const sampler = config.dockerStats
? await dockerSampler(config.dockerStatsIntervalSeconds)
: null;
const delay = monitorEventLoopDelay({ resolution: 20 });
delay.enable();
const cpuBefore = process.cpuUsage();
const memoryBefore = process.memoryUsage().rss;
let measured;
let dockerStats = null;
try {
measured = await runPhase(config, context, config.durationSeconds, config.readPercent);
} finally {
if (sampler) dockerStats = await sampler.stop();
delay.disable();
}
const cpu = process.cpuUsage(cpuBefore);
const result = {
label: config.label,
base_url: config.baseUrl,
profile: {
arrival_model: 'scheduled open loop; late or over-capacity slots dropped',
offered_rps: config.rate,
duration_seconds: config.durationSeconds,
warmup_seconds: config.warmupSeconds,
warmup_profile: 'read only',
read_percent: config.readPercent,
write_percent: 100 - config.readPercent,
initial_messages: config.seedMessages,
history_limit: config.historyLimit,
max_in_flight: config.maxInFlight,
timeout_ms: config.timeoutMs,
automatic_retries: false,
expected_instances: config.expectedInstances,
},
warmup: warmup?.summary ?? null,
measurement: measured.summary,
drain_seconds: measured.drain_seconds,
timing: {
measurement_started_epoch_ms: measured.started_epoch_ms,
measurement_scheduled_end_epoch_ms: measured.scheduled_end_epoch_ms,
measurement_completed_epoch_ms: measured.completed_epoch_ms,
},
client: {
cpu_seconds: Number(((cpu.user + cpu.system) / 1e6).toFixed(3)),
rss_before_mib: Number((memoryBefore / 1024 ** 2).toFixed(3)),
rss_after_mib: Number((process.memoryUsage().rss / 1024 ** 2).toFixed(3)),
event_loop_delay_ms: {
mean: Number((delay.mean / 1e6).toFixed(3)),
p99: Number((delay.percentile(99) / 1e6).toFixed(3)),
},
},
docker_stats: dockerStats,
request_samples: measured.events,
notes: [
'Setup and read-only warmup are excluded from measured RPS.',
'RPS uses the scheduled measurement window; requests may complete during drain.',
'HTTP failures, timeouts and client-dropped slots are reported separately.',
'One chat is used per run; writes increase its history during measurement.',
'The Node client runs outside Docker CPU/RAM limits and can itself saturate.',
'A timed-out POST might have committed; this benchmark never retries it.',
],
};
const json = JSON.stringify(result, null, 2) + '\n';
if (config.output) {
await mkdir(dirname(config.output), { recursive: true });
await writeFile(config.output, json, 'utf8');
}
console.log(JSON.stringify({ label: result.label, profile: result.profile,
measurement: result.measurement, drain_seconds: result.drain_seconds,
client: result.client, docker_stats_samples: dockerStats?.samples.length ?? 0,
output: config.output ?? null }, null, 2));
const observed = Object.keys(result.measurement.all.instance_counts).length;
if (config.expectedInstances !== null && observed !== config.expectedInstances) {
console.error(`expected ${config.expectedInstances} API instances, observed ${observed}`);
process.exitCode = 1;
}
if (!result.measurement.all.successful) process.exitCode = 1;
}
main().catch((error) => {
console.error(`benchmark failed: ${error?.message ?? String(error)}`);
process.exitCode = 1;
});
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""Measure idle PostgreSQL CPU with/without the observer after a scaling sweep."""
import argparse
import json
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.check_output(["docker", *args], cwd=ROOT, text=True, encoding="utf-8", timeout=30)
def snapshot(container):
values = {k: int(v) for k, v in (line.split() for line in docker("exec", container, "cat", "/sys/fs/cgroup/cpu.stat").splitlines())}
return {"time": time.time(), "cpu": values}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--seconds", type=int, default=30)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=False)
container = docker("compose", "ps", "-q", "postgres").strip()
results = []
for phase in ("without-before", "with-observer", "without-after"):
child = None
files = []
stop = f"/tmp/scaling-calibration-{uuid.uuid4().hex}.stop"
try:
if phase == "with-observer":
files = [(args.output / "postgres.jsonl").open("wb"), (args.output / "postgres.stderr").open("wb")]
child = subprocess.Popen(["docker", "exec", "-i", container, "sh", "-s", "postgres", stop],
cwd=ROOT, stdin=subprocess.PIPE, stdout=files[0], stderr=files[1])
child.stdin.write((ROOT / "scripts/diagnostics/observer.sh").read_bytes())
child.stdin.close()
time.sleep(3)
before = snapshot(container)
time.sleep(args.seconds)
after = snapshot(container)
elapsed = after["time"] - before["time"]
row = {"phase": phase, "before": before, "after": after,
"mean_cpu_cores": (after["cpu"]["usage_usec"]-before["cpu"]["usage_usec"])/elapsed/1e6}
results.append(row)
print(f"{phase}: CPU={row['mean_cpu_cores']:.4f} cores", flush=True)
finally:
if child is not None:
docker("exec", container, "touch", stop)
child.wait(timeout=15)
docker("cp", f"{container}:{stop}.pg.jsonl", str(args.output / "postgres-sql.jsonl"))
docker("exec", container, "rm", "-f", stop + ".pg.jsonl")
for file in files:
file.close()
overhead = results[1]["mean_cpu_cores"] - (results[0]["mean_cpu_cores"] + results[2]["mean_cpu_cores"])/2
report = {"phases": results, "estimated_observer_cpu_cores": overhead,
"notes": ["Idle calibration with the running stack's normal readiness probes; no HTTP load generator.",
"Difference includes host variation, monitor clients and monitoring SQL; not a correction factor for loaded runs."]}
(args.output / "summary.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
print(f"Estimated observer CPU={overhead:.4f} cores", flush=True)
if __name__ == "__main__":
main()
+108
View File
@@ -0,0 +1,108 @@
# Диагностическая серия масштабирования
Из корня репозитория, при работающем Docker и собранном образе API:
```powershell
python -X utf8 scripts/run-scaling-diagnostics.py --client container --output .artifacts/scaling/diagnostics-r100
```
Значения по умолчанию: профили one/two/four, только 100 запланированных
запросов/с, три повтора на профиль, 15 с прогрева и 60 с измерения, 80% чтений истории и 20%
записей. Пул берётся из Compose: по умолчанию 15 на реплику для one/two,
8 для four; фактические значения сохраняются в `manifest.json`.
В каждом прогоне — новый чат и 30 исходных
сообщений. Всего 9 прогонов: 11 минут 15 секунд прогрева и измерений плюс время запуска
контейнеров и сбора данных (ориентировочно около 15 минут, не жёсткий тайм-аут). Меньшие нагрузки
по умолчанию исключены; при необходимости их можно явно задать через `--rates`.
Нагрузка для профиля перестаёт повышаться, если минимум в двух из трёх повторов
встречается один из признаков: больше 5% слотов пропущено из-за опоздания генератора,
больше 5% пропущено из-за лимита одновременно выполняемых запросов, больше 1% ошибок
среди отправленных запросов или p95 успешных ответов превышает 1000 мс. Это границы
эксперимента, а не установленный SLA и не доказательство насыщения сервера.
Для полной матрицы всех запрошенных ступеней добавьте `--full-matrix`. Тогда
превышения порогов записываются в `threshold_crossings` и не прерывают профиль.
Точки с большими пропусками генератора не доказывают максимальную ёмкость API.
Ошибки запуска, отсутствие результата или неполные диагностические данные всё
ещё останавливают серию с восстановлением обычного профиля two.
Сбор инструментированного пула включается только через `test/compose.diagnostics.yaml`.
Preload сохраняет callback/Promise-интерфейсы pg.Pool и раз в секунду пишет
`/tmp/scaling-pool.jsonl` внутри API. Очередь и занятые соединения наблюдаются каждые
100 мс; краткие очереди могут не попасть в выборку, поэтому дополнительно измеряется
время каждого получения соединения. Оно включает создание новых соединений и
задержки event loop, а не только ожидание очереди. Cчётчики приобретений накопительные,
max-показатели относятся к интервалу записи.
PostgreSQL, Nginx и Redis наблюдаются через один постоянный docker exec на сервис
с паузой 2 с между выборками. Наблюдатель версии 3 выполняет SQL через одно
постоянное psql-соединение (`\watch 2`); результат находится в `postgres-sql.jsonl`.
Сохраняются cgroup CPU/memory, ожидания активных
запросов PostgreSQL, число простаивающих транзакций, состав upstream Nginx и события
reload. Тексты SQL, токены, пароли и сообщения не записываются. Наблюдатели потребляют
часть ресурсов тех же контейнеров; это одинаковая методика для всех профилей.
В каталоге прогона:
- `benchmark.json`: исходные HTTP-замеры и границы измеряемого окна в epoch ms;
- `api-*.jsonl`: пул, event loop и cgroup API (включают также прогрев и предыдущие
повторы того же профиля; summary выбирает только текущее измеряемое окно);
- `postgres/nginx/redis.jsonl`: наблюдения остальных сервисов;
- `postgres-sql.jsonl`: SQL-снимки постоянного psql-наблюдателя версии 3;
- `*.stderr`, `nginx-events.log`: ошибки наблюдателей и события Nginx;
- `summary.json`: показатели в измеряемом окне; скорости CPU рассчитаны из
приращений счётчиков, память — максимум по выборкам;
- `execution.json`: команда и временные границы прогона.
`manifest.json` в корне фиксирует настройки, причины остановки нагрузки, ID образов,
хеши сборщиков, окружение Docker и лимиты контейнеров. После завершения или исключения runner восстанавливает профиль
two без диагностического preload. При принудительном завершении процесса это можно
сделать вручную:
```powershell
docker compose -f compose.yaml -f configs/two.yaml up -d --no-build --no-deps --wait api
```
Проверки сборщика:
```powershell
node --test test/pool-diagnostics.mjs test/benchmark-scaling.mjs
python -X utf8 test/diagnostic-summary.py
```
Для отдельной проверки влияния Windows-клиента и публикации порта можно запустить
тот же генератор в Linux-контейнере:
```powershell
python -X utf8 scripts/run-scaling-diagnostics.py --client container --rates 100 --repetitions 3 --output .artifacts/scaling/diagnostics-linux-client-r100
```
Клиент использует тот же собранный образ (только Node и benchmark-скрипт), не имеет
лимита CPU/RAM и обращается к `http://nginx` по сети Compose. Это другой путь HTTP:
без Windows port forwarding, но с конкуренцией за ресурсы той же Docker VM. Такой
контроль помогает проверить влияние клиентского пути, но не заменяет внешний
генератор на отдельной машине. Не запускайте его одновременно с основной серией.
После нагрузки можно оценить накладные расходы наблюдателя PostgreSQL на простое:
```powershell
python -X utf8 scripts/calibrate-scaling-observer.py --output .artifacts/scaling/observer-calibration
```
Выполняются три фазы по 30 с: без наблюдателя, с наблюдателем и снова без него.
Это оценка собственных расходов сборщика, а не поправочный коэффициент к нагрузке.
После завершения серии соберите таблицу и графики. Matplotlib можно запустить
через временное окружение uv, не меняя зависимости API:
```powershell
python -X utf8 scripts/summarize-scaling-diagnostics.py .artifacts/scaling/diagnostics-pg1cpu-full-20261028
uv run --no-project --with "matplotlib>=3.9,<4" python scripts/plot-scaling-diagnostics.py .artifacts/scaling/diagnostics-pg1cpu-full-20261028
```
Графики `performance`, `resources`, `delivered-load` сохраняются в PNG и SVG.
Линии показывают медианы повторов, затенение — диапазон min/max, а не доверительный
интервал. Отдельный график задержек и ошибок по фактически отправленным RPS помогает
учесть пропуски генератора. Превышение номинальной нагрузки не означает, что сервер
фактически получил столько запросов.
+70
View File
@@ -0,0 +1,70 @@
#!/bin/sh
# One persistent docker exec per service; stdout contains only numeric diagnostics.
set -eu
role=$1
stop=$2
watch_pid=''
cleanup() {
if [ -n "$watch_pid" ]; then
kill "$watch_pid" 2>/dev/null || true
wait "$watch_pid" 2>/dev/null || true
fi
}
trap cleanup EXIT
trap 'exit 0' TERM INT
printf '{"kind":"observer","version":3,"time_ms":%s000}\n' "$(date +%s)"
if [ "$role" = postgres ]; then
PGAPPNAME=scaling-observer psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" > "${stop}.pg.jsonl" <<'SQL' &
SELECT json_build_object(
'kind', 'postgres', 'time_ms', (extract(epoch FROM clock_timestamp())*1000)::bigint,
'activity', (SELECT coalesce(json_agg(a), '[]'::json) FROM (
SELECT state, wait_event_type, wait_event, count(*) AS count,
max(extract(epoch FROM clock_timestamp()-query_start)) FILTER (WHERE state = 'active') AS max_active_seconds
FROM pg_stat_activity WHERE datname=current_database() AND backend_type='client backend'
AND application_name <> 'scaling-observer'
GROUP BY state,wait_event_type,wait_event
) a),
'db', (SELECT row_to_json(d) FROM (
SELECT xact_commit,xact_rollback,blks_read,blks_hit,temp_files,temp_bytes,deadlocks,stats_reset
FROM pg_stat_database WHERE datname=current_database()
) d)
)::jsonb
\watch 2
SQL
watch_pid=$!
fi
while [ ! -f "$stop" ]; do
stamp="$(date +%s)000"
awk -v stamp="$stamp" '
BEGIN {printf "{\"kind\":\"cgroup\",\"time_ms\":%s", stamp}
FILENAME ~ /cpu.stat$/ {printf ",\"cpu_%s\":%s", $1, $2}
FILENAME ~ /memory.events$/ {printf ",\"memory_%s\":%s", $1, $2}
FILENAME ~ /memory.current$/ {printf ",\"memory_bytes\":%s", $1}
END {print "}"}
' /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/memory.events /sys/fs/cgroup/memory.current
if [ "$role" = postgres ]; then
awk -v stamp="$stamp" '
BEGIN {printf "{\"kind\":\"host\",\"time_ms\":%s", stamp}
FILENAME ~ /pressure\// {
name=FILENAME; sub(/^.*\//,"",name)
for (i=2;i<=NF;i++) {
split($i,value,"=")
if (value[1] == "total") printf ",\"%s_%s_total_usec\":%s",name,$1,value[2]
}
}
FILENAME ~ /meminfo$/ && ($1 == "MemAvailable:" || $1 == "SwapFree:" || $1 == "SwapTotal:") {
name=$1; sub(/:$/,"",name); printf ",\"%s_kib\":%s",name,$2
}
END {print "}"}
' /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io /proc/meminfo
fi
if [ "$role" = nginx ]; then
awk -v stamp="$stamp" '
BEGIN {printf "{\"kind\":\"upstream\",\"time_ms\":%s,\"servers\":[", stamp; sep=""}
/^server .*:8080 / {printf "%s\"%s\"", sep, $2; sep=","}
END {print "]}"}
' /etc/nginx/ready-upstreams.conf
fi
sleep 2
done
rm -f "$stop"
+73
View File
@@ -0,0 +1,73 @@
// Loaded only by test/compose.diagnostics.yaml. No HTTP endpoint or credentials.
const { Pool } = require('pg');
const fs = require('node:fs');
const { performance, monitorEventLoopDelay } = require('node:perf_hooks');
const states = new Map();
const originalConnect = Pool.prototype.connect;
Pool.prototype.connect = function (...args) {
let state = states.get(this);
if (!state) {
state = { acquisitions: 0, errors: 0, acquire_ms: 0, interval_max_ms: 0,
interval_max_waiting: 0, interval_max_busy: 0 };
states.set(this, state);
}
const started = performance.now();
const finish = (error) => {
const elapsed = performance.now() - started;
state.acquisitions += 1;
state.errors += Number(Boolean(error));
state.acquire_ms += elapsed;
state.interval_max_ms = Math.max(state.interval_max_ms, elapsed);
};
if (typeof args[0] === 'function') {
const callback = args[0];
args[0] = (error, client, release) => {
finish(error);
callback(error, client, release);
};
return originalConnect.apply(this, args);
}
return originalConnect.apply(this, args).then(
(client) => { finish(null); return client; },
(error) => { finish(error); throw error; },
);
};
const output = '/tmp/scaling-pool.jsonl';
fs.writeFileSync(output, '');
const delay = monitorEventLoopDelay({ resolution: 20 });
delay.enable();
function counters(path) {
try {
return Object.fromEntries(fs.readFileSync(path, 'utf8').trim().split('\n')
.map((line) => { const [key, value] = line.split(/\s+/); return [key, Number(value)]; }));
} catch { return null; }
}
let lastWrite = performance.now();
const timer = setInterval(() => {
for (const [pool, state] of states) {
state.interval_max_waiting = Math.max(state.interval_max_waiting, pool.waitingCount);
state.interval_max_busy = Math.max(state.interval_max_busy, pool.totalCount - pool.idleCount);
}
if (performance.now() - lastWrite < 1000) return;
lastWrite = performance.now();
const pools = [...states].map(([pool, state]) => ({
...state, total: pool.totalCount, idle: pool.idleCount, waiting: pool.waitingCount,
}));
let memory = null;
try { memory = Number(fs.readFileSync('/sys/fs/cgroup/memory.current', 'utf8')); } catch {}
fs.appendFileSync(output, JSON.stringify({ time_ms: Date.now(), pid: process.pid,
instance: process.env.INSTANCE_ID, pools, cpu: counters('/sys/fs/cgroup/cpu.stat'),
memory_bytes: memory, memory_events: counters('/sys/fs/cgroup/memory.events'),
rss_bytes: process.memoryUsage().rss, process_cpu: process.cpuUsage(),
event_loop_delay_ms: { mean: delay.mean / 1e6, p99: delay.percentile(99) / 1e6 },
}) + '\n');
delay.reset();
for (const state of states.values()) {
state.interval_max_ms = 0;
state.interval_max_waiting = 0;
state.interval_max_busy = 0;
}
}, 100);
timer.unref();
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env python3
"""One read-only snapshot of SQL classes, never SQL text or parameters."""
import argparse
import json
from pathlib import Path
import subprocess
SQL = """
SELECT jsonb_build_object('time_ms',(extract(epoch FROM clock_timestamp())*1000)::bigint,
'activity', (SELECT coalesce(jsonb_agg(a),'[]'::jsonb) FROM (
SELECT state,wait_event_type,wait_event,
CASE
WHEN query LIKE '%INSERT INTO "public"."chat_message_epochs"%' THEN 'message_and_epoch_write'
WHEN query LIKE '%message_idempotency%' THEN 'idempotency'
WHEN lower(ltrim(query)) LIKE 'commit%' THEN 'commit'
WHEN query LIKE '%token_hash%' THEN 'authentication'
WHEN query LIKE '%chat_message_epochs%' THEN 'epoch_read'
WHEN query LIKE '%messages%' THEN 'message_read'
ELSE 'other'
END AS operation,
count(*) AS count,
max(extract(epoch FROM clock_timestamp()-query_start)) AS max_query_seconds,
max(cardinality(pg_blocking_pids(pid))) AS max_blockers
FROM pg_stat_activity
WHERE datname=current_database() AND backend_type='client backend'
AND application_name <> 'scaling-observer' AND state='active'
GROUP BY state,wait_event_type,wait_event,operation
) a));
"""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = subprocess.run(["docker", "compose", "exec", "-T", "-e", "PGAPPNAME=scaling-observer",
"postgres", "psql", "-X", "-qAt", "-U", "app", "-d", "chat", "-c", SQL],
capture_output=True, text=True, encoding="utf-8", timeout=30, check=True)
data = json.loads(result.stdout)
args.output.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
print(json.dumps(data))
+159
View File
@@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""Plot completed diagnostic runs using matplotlib; reads results.csv and manifest.json."""
import argparse
import csv
import json
from pathlib import Path
import statistics
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
parser.add_argument("--phase", help="Only plot this CSV phase, e.g. confirmation")
parser.add_argument("--all-latency", action="store_true", help="Use all-request p95, including failures")
parser.add_argument("--valid-delivery-only", action="store_true", help="Exclude runs with unverified arrival rate")
args = parser.parse_args()
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
with (args.directory / "results.csv").open(encoding="utf-8") as stream:
rows = list(csv.DictReader(stream))
if args.phase:
rows = [row for row in rows if row.get("phase") == args.phase]
if args.valid_delivery_only:
rows = [row for row in rows if row.get("delivery_valid") in ("True", "true", "1")]
if not rows:
parser.error("no results")
for row in rows:
row['configuration'] = row.get('configuration') or next(
(profile for profile in manifest.get('profiles', []) if row['label'].startswith(profile + '-r')),
str(row['replicas']))
for key in ("replicas", "offered_rps"):
row[key] = int(row[key])
for key in ("successful_rps", "attempted_rps", "p95_ms", "drop_percent", "acquisition_mean_ms",
"pool_max_waiting", "api_cpu_cores", "api_max_throttle_percent",
"pg_cpu_cores", "pg_throttle_percent"):
row[key] = float(row[key] or 'nan')
if args.all_latency:
if not row.get("all_request_p95_ms"):
parser.error("all-request latency is missing from this dataset")
row["p95_ms"] = float(row["all_request_p95_ms"])
summary = json.loads((args.directory / row["label"] / "summary.json").read_text(encoding="utf-8"))
m = summary["measurement"]["all"]
row["error_percent"] = 100*m["errors"]/max(m["attempted"], 1)
pg = next(c for group in manifest["containers"].values() for c in group if "-postgres-" in c["name"])
pg_cpus = pg["nano_cpus"]/1e9
memory_mib = pg["memory_bytes"]/1024**2
configurations = sorted({r['configuration'] for r in rows})
palette = ['#2563eb', '#059669', '#d97706', '#9333ea', '#db2777', '#0891b2', '#64748b']
colors = {configuration: palette[i % len(palette)] for i, configuration in enumerate(configurations)}
budgets = {}
labels = {}
for configuration in configurations:
apis = [c for c in manifest.get('containers', {}).get(configuration, []) if '-api-' in c['name']]
replicas = next(r['replicas'] for r in rows if r['configuration'] == configuration)
if apis:
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
memory = sum(c['memory_bytes'] for c in apis) / 1024**3
budgets[configuration] = (cpu, memory)
labels[configuration] = f'{replicas} API ({cpu:g} CPU / {memory:g} GiB)'
else:
labels[configuration] = f'{configuration}: {replicas} API'
budget_description = '; '.join(f'{cpu:g} CPU / {memory:g} GiB'
for cpu, memory in sorted(set(budgets.values()))) or 'see manifest'
def draw(ax, field, title, ylabel, log=False):
for configuration in configurations:
selected = [r for r in rows if r['configuration'] == configuration]
rates = sorted({r["offered_rps"] for r in selected})
samples = [[r[field] for r in selected if r["offered_rps"] == rate] for rate in rates]
values = [statistics.median(v) for v in samples]
ax.plot(rates, values, marker="o", color=colors[configuration], label=labels[configuration])
ax.fill_between(rates, [min(v) for v in samples], [max(v) for v in samples],
color=colors[configuration], alpha=0.12)
ax.set(title=title, xlabel="Scheduled requests/s", ylabel=ylabel)
ax.grid(alpha=0.25)
if log:
ax.set_yscale("log")
else:
ax.set_ylim(bottom=0)
def save(fig, name, caption):
if args.valid_delivery_only:
caption += "\nUnderfed runs excluded; sample counts can differ between points."
fig.text(0.5, 0.015, caption, ha="center", fontsize=9)
fig.tight_layout(rect=(0, 0.055, 1, 0.94))
for extension in ("png", "svg"):
fig.savefig(args.directory / f"{name}.{extension}", dpi=160, bbox_inches="tight", pad_inches=0.25)
plt.close(fig)
plt.rcParams.update({"font.size": 10, "axes.titlesize": 11})
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
fig.suptitle(f"PostgreSQL {pg_cpus:g} CPU / {memory_mib:g} MiB; API budgets: {budget_description}", fontsize=12)
draw(axes[0, 0], "successful_rps", "Successful throughput", "Successful requests/s")
rates = sorted({r["offered_rps"] for r in rows})
axes[0, 0].plot(rates, rates, color="#475569", linestyle="--", alpha=0.7, label="Scheduled")
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], max(rates)*1.05))
axes[0, 0].legend()
draw(axes[0, 1], "p95_ms", "Latency of all requests" if args.all_latency else "Latency of successful requests", "p95, ms (log scale)", log=True)
if args.all_latency:
axes[0, 1].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
axes[0, 1].legend(fontsize=8)
draw(axes[1, 0], "error_percent", "Errors among attempted requests", "Errors, %")
axes[1, 0].axhline(1, color="#64748b", linestyle="--", linewidth=1)
axes[1, 0].set_ylim(0, max(axes[1, 0].get_ylim()[1], 1.05))
draw(axes[1, 1], "drop_percent", "Slots dropped by client", "Dropped scheduled slots, %")
drop_limit = 2 if args.valid_delivery_only else 5
axes[1, 1].axhline(drop_limit, color="#64748b", linestyle="--", linewidth=1)
axes[1, 1].set_ylim(0, max(axes[1, 1].get_ylim()[1], drop_limit*1.05))
save(fig, "performance", "Lines: medians of repeats; shading: min-max. High client-drop points do not establish server capacity.")
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
fig.suptitle("Resource and connection-pool diagnostics", fontsize=14)
draw(axes[0, 0], "pg_cpu_cores", "PostgreSQL CPU usage", "CPU cores")
axes[0, 0].axhline(pg_cpus, color="#64748b", linestyle="--", linewidth=1, label="CPU limit")
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], pg_cpus*1.05))
draw(axes[0, 1], "api_cpu_cores", "Total API CPU usage", "CPU cores")
for configuration, (cpu, _) in budgets.items():
axes[0, 1].axhline(cpu, color=colors[configuration], linestyle='--', linewidth=1,
label=f'{configuration} limit: {cpu:g} CPU')
if budgets:
axes[0, 1].set_ylim(0, max(axes[0, 1].get_ylim()[1], max(cpu for cpu, _ in budgets.values()) * 1.05))
axes[0, 1].legend(fontsize=8)
draw(axes[0, 2], "pg_throttle_percent", "PostgreSQL CPU throttling", "Throttled periods, %")
draw(axes[1, 0], "api_max_throttle_percent", "Most throttled API replica", "Throttled periods, %")
draw(axes[1, 1], "acquisition_mean_ms", "Mean connection acquisition", "Acquisition, ms (log scale)", log=True)
draw(axes[1, 2], "pool_max_waiting", "Largest observed pool queue", "Waiting requests")
axes[0, 0].legend()
save(fig, "resources", "Lines: medians of repeats; shading: min-max. Queue sampled every 100 ms; acquisition includes connection setup.")
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
fig.suptitle("Latency and errors against the workload actually delivered", fontsize=14)
for configuration in configurations:
selected = [r for r in rows if r['configuration'] == configuration]
rates = sorted({r["offered_rps"] for r in selected})
groups = [[r for r in selected if r["offered_rps"] == rate] for rate in rates]
x = [statistics.median(r["attempted_rps"] for r in group) for group in groups]
for axis, field, title, ylabel in (
(axes[0], "p95_ms", "Latency", "p95, ms (log scale)"),
(axes[1], "error_percent", "Errors", "Errors among attempts, %"),
):
y = [statistics.median(r[field] for r in group) for group in groups]
axis.plot(x, y, marker="o", color=colors[configuration], label=labels[configuration])
for rate, xp, yp in zip(rates, x, y):
axis.annotate(str(rate), (xp, yp), xytext=(4, 4), textcoords="offset points", fontsize=8)
axis.set(title=title, xlabel="Actually attempted requests/s", ylabel=ylabel)
axis.grid(alpha=0.25)
axes[0].set_yscale("log")
if args.all_latency:
axes[0].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
axes[1].set_ylim(bottom=0)
axes[0].legend()
save(fig, "delivered-load", "Points: medians of repeats. Small labels: scheduled RPS. Client shares the Docker VM; no server-capacity claim.")
print(f"Wrote performance/resources/delivered-load PNG and SVG for {len(rows)} runs to {args.directory}")
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env python3
"""Compatibility entry point for the consolidated report figures."""
import runpy
from pathlib import Path
import sys
if __name__ == '__main__':
helper = Path(__file__).with_name('report-evidence.py')
sys.argv = [str(helper), '--plot', *sys.argv[1:]]
runpy.run_path(str(helper), run_name='__main__')
+220
View File
@@ -0,0 +1,220 @@
#!/usr/bin/env python3
"""Keep report evidence in one JSON, one CSV, two figures and checks.json."""
import argparse
from collections import Counter
import csv
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import statistics
import subprocess
ROOT = Path(__file__).resolve().parents[1]
SERIES = [
('api-cpu', None, 'api-small-growth-20261029'),
('pg-cpu', 'pg0.5', 'diagnostics-v3-linux-20261028'),
('pg-cpu', 'pg1', 'diagnostics-pg1cpu-linux-20261028'),
('pool-limit', 'pool60', 'pool15-four-r100-short-20261028'),
('pool-limit', 'pool32', 'pool8-four-r100-short-20261028'),
('pg-upper-cpu', 'pg1', 'api-small-db-control-20261029/pg1'),
('pg-upper-cpu', 'pg2', 'api-small-db-control-20261029/pg2'),
]
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def flatten(bundle):
rows = []
for run in bundle['runs']:
profile, measurement = run['profile'], run['measurement']
all_requests = measurement['all']
diagnostics = run['diagnostics']
resources, pools = diagnostics['resources'], diagnostics['pools']
manifest = bundle['manifests'][run['manifest']]
containers = manifest['containers'][run['configuration']]
apis = [c for c in containers if '-api-' in c['name']]
pg = next(c for c in containers if '-postgres-' in c['name'])
seconds, rate = profile['duration_seconds'], profile['offered_rps']
acquired = sum(p['acquisitions'] for p in pools.values())
fraction = all_requests['attempted'] / (seconds * rate)
latency = all_requests['successful_latency_ms']
all_latency = run.get('all_latency_ms') or {}
rows.append({
'dataset': run['dataset'], 'variant': run['variant'], 'label': run['label'],
'replicas': len(apis), 'api_cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
'api_ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
'pool_total': sum(int(c['pool_max']) for c in apis),
'pg_cpu': pg['nano_cpus'] / 1e9, 'pg_ram_mib': pg['memory_bytes'] / 1024**2,
'offered_rps': rate, 'duration_seconds': seconds,
'warmup_seconds': profile.get('warmup_seconds', manifest['warmup_seconds']),
'attempted': all_requests['attempted'], 'successful': all_requests['successful'],
'errors': all_requests['errors'], 'successful_rps': all_requests['successful'] / seconds,
'attempted_rps': all_requests['attempted'] / seconds,
'delivery_fraction': fraction, 'delivery_valid': .98 <= fraction <= 1.02,
'p50_ms': latency['p50'], 'p95_ms': latency['p95'], 'p99_ms': latency['p99'],
'all_request_p95_ms': all_latency.get('p95'),
'api_cpu_cores': sum(r.get('mean_cpu_cores', 0) for name, r in resources.items() if name.startswith('api-')),
'api_peak_memory_mib_sum': sum(r.get('peak_memory_mib', 0) for name, r in resources.items() if name.startswith('api-')),
'api_max_throttle_percent': max((p.get('throttled_period_fraction', 0) for name, p in resources.items() if name.startswith('api-')), default=0) * 100,
'pg_cpu_cores': resources['postgres'].get('mean_cpu_cores'),
'pg_peak_memory_mib': resources['postgres'].get('peak_memory_mib'),
'pg_throttle_percent': resources['postgres'].get('throttled_period_fraction', 0) * 100,
'host_cpu_pressure_percent': diagnostics['host'].get('cpu_some_fraction', 0) * 100,
'acquisition_mean_ms': sum(p['mean_acquisition_ms'] * p['acquisitions'] for p in pools.values()) / max(acquired, 1),
'acquisition_errors': sum(p['acquisition_errors'] for p in pools.values()),
'pool_max_waiting': max((p['max_waiting'] for p in pools.values()), default=0),
'nginx_min_ready': diagnostics['nginx']['min_ready'],
'diagnostics_complete': diagnostics['diagnostics_complete'],
})
return rows
def collect(source, output):
manifests, runs = {}, []
for dataset, variant, directory in SERIES:
path = source / directory
manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8'))
manifests[directory] = manifest
for label in manifest['runs']:
location = path / label
benchmark = json.loads((location / 'benchmark.json').read_text(encoding='utf-8'))
summary = json.loads((location / 'summary.json').read_text(encoding='utf-8'))
configuration = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
runs.append({
'dataset': dataset, 'variant': variant or configuration, 'configuration': configuration,
'manifest': directory, 'label': label,
'profile': benchmark['profile'], 'measurement': benchmark['measurement'],
'timing': benchmark.get('timing'), 'client': benchmark.get('client'),
'all_latency_ms': benchmark.get('all_latency_ms'),
'criterion': summary.get('criterion'),
'diagnostics': {key: summary[key] for key in
('resources', 'pools', 'host', 'postgres', 'nginx', 'limits_reached', 'diagnostics_complete')},
'source_hashes': {'benchmark': digest(location / 'benchmark.json'),
'summary': digest(location / 'summary.json')},
})
sources = sorted([p for p in (ROOT / 'src').rglob('*.ts') if 'generated' not in p.parts]
+ list((ROOT / 'prisma').rglob('*.sql'))
+ list((ROOT / 'configs').glob('*.yaml'))
+ [ROOT / p for p in ('compose.yaml', 'Dockerfile', 'infra/nginx.conf', 'infra/start-nginx.sh', 'pnpm-lock.yaml')])
host_path = source / 'diagnostics-pg1cpu-full-20261028/host-environment.json'
host = json.loads(host_path.read_text(encoding='utf-8'))
bundle = {
'schema_version': 1, 'recorded_at': datetime.now(timezone.utc).isoformat(),
'git_base': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
'revision_note': 'Implementation is in the working tree; git_base identifies the scaffold, not a committed implementation.',
'retained_data': 'Per-run HTTP aggregates, profiles, CPU/RAM/pool/SQL-state diagnostics, original manifests and source hashes. Request samples, stdout/stderr, traces and duplicate documentation omitted.',
'hardware': {'processor': host['processor'], 'windows_total_visible_kib': host['windows_memory_kib']['total_visible']},
'application_source_sha256': {p.relative_to(ROOT).as_posix(): digest(p) for p in sources},
'manifests': manifests, 'runs': runs,
}
(output / 'measurements.json').write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
return bundle
def verify(bundle, rows):
counts = Counter(r['dataset'] for r in rows)
assert counts == {'api-cpu': 36, 'pg-cpu': 6, 'pool-limit': 6, 'pg-upper-cpu': 6}, counts
assert len({(r['dataset'], r['variant'], r['label']) for r in rows}) == 54
assert all(r['attempted'] == r['successful'] + r['errors'] for r in rows)
assert all(r['diagnostics_complete'] for r in rows)
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
assert all(r['pool_total'] == 32 and r['api_ram_mib'] == 1024 and r['pg_cpu'] == 1 for r in matrix)
assert all(n == 3 for n in Counter((r['variant'], r['offered_rps']) for r in matrix).values())
for name, expected in [('pg-cpu', {'pg0.5': .5, 'pg1': 1}),
('pg-upper-cpu', {'pg1': 1, 'pg2': 2})]:
assert all(r['pg_cpu'] == expected[r['variant']] for r in rows if r['dataset'] == name)
assert {r['pool_total'] for r in rows if r['dataset'] == 'pool-limit'} == {32, 60}
assert len(bundle['manifests']) == 7
def plot(rows, output):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
for cpu, color in ((1, '#9333ea'), (2, '#059669')):
for ax, rate, field, title, ylabel in (
(axes[0, 0], 40, 'all_request_p95_ms', '40 scheduled RPS: all-request latency', 'Median p95, ms'),
(axes[0, 1], 80, 'successful_rps', '80 scheduled RPS: successful throughput', 'Median successful requests/s'),
(axes[1, 0], 40, 'api_max_throttle_percent', '40 RPS: most throttled API', 'Median throttled periods, %'),
(axes[1, 1], 80, 'delivery_fraction', '80 RPS: actual load delivery', 'Median started / scheduled, %'),
):
groups = [[r for r in matrix if r['api_cpu'] == cpu and r['replicas'] == n and r['offered_rps'] == rate] for n in (1, 2, 4)]
scale = 100 if field == 'delivery_fraction' else 1
values = [statistics.median(r[field] for r in g) * scale for g in groups]
ax.plot([1, 2, 4], values, marker='o', color=color, label=f'API total {cpu} CPU / 1 GiB')
ax.fill_between([1, 2, 4], [min(r[field] for r in g) * scale for g in groups],
[max(r[field] for r in g) * scale for g in groups], color=color, alpha=.1)
ax.set(title=title, xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
ax.grid(alpha=.25)
ax.legend(fontsize=8)
axes[0, 0].axhline(1000, color='#64748b', linestyle='--')
axes[1, 1].axhline(98, color='#64748b', linestyle='--')
fig.suptitle('Fixed budgets and CPU growth; PG 1 CPU / 512 MiB; total pool 32', fontsize=13)
fig.text(.5, .012, 'All repeats shown, including underfed runs. Lines: medians; bands: min/max, not confidence intervals. No measured plateau claim.', ha='center', fontsize=8)
fig.tight_layout(rect=(0, .04, 1, .95))
fig.savefig(output / 'scaling.png', dpi=160)
plt.close(fig)
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
panels = [
('api-cpu', ['two-small', 'two'], 'all_request_p95_ms', ['API 1 CPU', 'API 2 CPU'], '2 API, 40 RPS: CPU increase', 'All-request p95, ms', 40),
('pg-cpu', ['pg0.5', 'pg1'], 'p95_ms', ['PG 0.5 CPU', 'PG 1 CPU'], '4 API, 60 RPS: database quota', 'Successful-request p95, ms', 60),
('pool-limit', ['pool60', 'pool32'], 'successful_rps', ['60 connections', '32 connections'], '4 API, 100 scheduled RPS: pool limit', 'Successful requests/s', 100),
('pg-upper-cpu', ['pg1', 'pg2'], 'all_request_p95_ms', ['PG 1 CPU', 'PG 2 CPU'], '2 API, 80 RPS: further PG CPU increase', 'All-request p95, ms', 80),
]
for ax, (dataset, variants, field, labels, title, ylabel, rate) in zip(axes.flat, panels):
for i, variant in enumerate(variants):
group = [r for r in rows if r['dataset'] == dataset and r['variant'] == variant and r['offered_rps'] == rate]
assert len(group) == 3
color = '#9333ea' if i == 0 else '#059669'
for j, r in enumerate(group):
# Older Node series use a different scheduler: show their observations without a retroactive k6 classification.
underfed = dataset in {'api-cpu', 'pg-upper-cpu'} and not r['delivery_valid']
ax.scatter(i + (j - 1) * .08, r[field], color=color, marker='x' if underfed else 'o', s=55)
value = (sum(r['successful'] for r in group) / sum(r['duration_seconds'] for r in group)
if dataset == 'pool-limit' else statistics.median(r[field] for r in group))
ax.hlines(value, i - .17, i + .17, color='#0f172a', linewidth=2)
ax.annotate(f'{value:.0f}' if value > 100 else f'{value:.2f}', (i, value),
xytext=(12, 8), textcoords='offset points', fontsize=9)
ax.set(title=title, ylabel=ylabel, xticks=[0, 1], xticklabels=labels, xlim=(-.4, 1.4))
ax.grid(axis='y', alpha=.25)
if field == 'all_request_p95_ms':
ax.axhline(1000, color='#64748b', linestyle='--', linewidth=1)
maximum = max(r[field] for r in rows if r['dataset'] == dataset and r['variant'] in variants and r['offered_rps'] == rate)
ax.set_ylim(0, maximum * 1.2)
fig.suptitle('Separate experiments: compare variants within each panel', fontsize=13)
fig.text(.5, .012, 'Dots: individual runs; black bars: medians (pool panel: aggregate successful RPS). Crosses: k6 underdelivery. See REPORT.md for scheduler differences.', ha='center', fontsize=8)
fig.tight_layout(rect=(0, .04, 1, .95))
fig.savefig(output / 'improvements.png', dpi=160)
plt.close(fig)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--collect', action='store_true', help='Import the selected raw experiment directories before cleanup')
parser.add_argument('--source-root', type=Path, default=ROOT / '.artifacts/scaling')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence')
parser.add_argument('--input', type=Path, default=ROOT / 'evidence/measurements.json')
parser.add_argument('--plot', action='store_true')
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
bundle = collect(args.source_root, args.output) if args.collect else json.loads(args.input.read_text(encoding='utf-8'))
rows = flatten(bundle)
verify(bundle, rows)
with (args.output / 'scaling.csv').open('w', encoding='utf-8', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=rows[0].keys())
writer.writeheader()
writer.writerows(rows)
if args.plot:
plot(rows, args.output)
print(f'Verified {len(rows)} runs from {len(bundle["manifests"])} source series; compact evidence in {args.output}')
if __name__ == '__main__':
main()
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python3
"""Bracket an empirical SLO boundary with k6; preserve separate probe/confirmation data."""
import argparse
from datetime import datetime, timezone
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
PROFILE_REPLICAS = {'one': 1, 'two': 2, 'four': 4, 'four-large': 4,
'one-small': 1, 'two-small': 2, 'four-small': 4}
spec = importlib.util.spec_from_file_location('diagnostics', ROOT/'scripts/run-scaling-diagnostics.py')
diag = importlib.util.module_from_spec(spec)
spec.loader.exec_module(diag)
def call(args, timeout=180, env=None):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=timeout, env=env)
if result.returncode:
raise RuntimeError(f'{args[0]} {args[1]}: {result.stderr[-1800:]}')
return result.stdout.strip()
def watch(container, role, directory):
stop = '/tmp/saturation-' + uuid.uuid4().hex + '.stop'
output = (directory/f'{role}.jsonl').open('wb')
error = (directory/f'{role}.stderr').open('wb')
process = subprocess.Popen(['docker', 'exec', '-i', container, 'sh', '-s', role, stop],
cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=error)
process.stdin.write((ROOT/'scripts/diagnostics/observer.sh').read_bytes())
process.stdin.close()
return container, role, stop, process, output, error
def stop_watch(worker, directory):
container, role, stop, process, output, error = worker
try:
call(['docker', 'exec', container, 'touch', stop], timeout=15)
process.wait(timeout=15)
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=10)
output.close()
error.close()
if role == 'postgres':
call(['docker', 'cp', f'{container}:{stop}.pg.jsonl', str(directory/'postgres-sql.jsonl')])
call(['docker', 'exec', container, 'rm', '-f', stop+'.pg.jsonl'])
def classify(benchmark):
m = benchmark['measurement']
count = m['all']['attempted']
ratio = count/max(m['scheduled_slots'], 1)
error_fraction = m['all']['errors']/max(count, 1)
p95 = benchmark['all_latency_ms']['p95']
delivered = 0.98 <= ratio <= 1.02 and m['client_dropped_slots']/max(m['scheduled_slots'], 1) <= 0.02
slo = count > 0 and error_fraction <= 0.01 and p95 is not None and p95 <= 1000
return {'delivery_fraction': ratio, 'error_fraction': error_fraction,
'all_request_p95_ms': p95, 'delivery_valid': delivered, 'slo_pass': slo,
'classification': 'pass' if delivered and slo else 'slo_fail' if delivered else 'generator_limited'}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', required=True, type=Path)
parser.add_argument('--image', default='grafana/k6:1.3.0')
parser.add_argument('--profiles', nargs='+', choices=list(PROFILE_REPLICAS), default=['one', 'two', 'four'])
parser.add_argument('--probe-seconds', type=int, default=15)
parser.add_argument('--confirm-seconds', type=int, default=30)
parser.add_argument('--warmup-seconds', type=int, default=10,
help='Mixed-load warmup before each confirmation run')
parser.add_argument('--calibration-only', action='store_true')
parser.add_argument('--confirm-rates', nargs='+', type=int, help='Confirm these rates directly, skipping coarse search')
parser.add_argument('--no-expand', action='store_true',
help='With --confirm-rates, run only the requested rates for a controlled comparison')
args = parser.parse_args()
if args.no_expand and not args.confirm_rates:
parser.error('--no-expand requires --confirm-rates')
if args.warmup_seconds < 1:
parser.error('--warmup-seconds must be positive')
out = args.output.resolve()
out.mkdir(parents=True, exist_ok=False)
manifest = {'started': datetime.now(timezone.utc).isoformat(), 'profiles': args.profiles,
'client': 'container', 'engine': 'k6', 'duration_seconds': args.confirm_seconds,
'warmup_seconds': args.warmup_seconds, 'probe_duration_seconds': args.probe_seconds, 'probe_warmup_seconds': 5,
'repetitions': 3, 'rates': [], 'containers': {}, 'runs': [], 'boundaries': {},
'stopped_profiles': {}, 'observer_version': 3,
'criterion': 'delivery 98..102%, dropped iterations <=2%, errors <=1%, all-request p95 <=1000 ms',
'pool_budget': 32, 'notes': ['Local empirical SLO boundary, not universal maximum throughput.',
'Native sequential warmup/measurement scenarios at target-rate 80/20; custom counters belong only to measurement.',
'Generator shares Docker VM CPU; calibration does not eliminate competition during API load.',
'Probe durations differ; use confirmation rows for reported bounds.'],
'source_sha256': {p: hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in
['scripts/saturation.k6.js', 'scripts/saturation-mock.cjs', 'scripts/run-saturation.py',
'scripts/run-scaling-diagnostics.py', 'scripts/diagnostics/pool.cjs', 'scripts/diagnostics/observer.sh',
'compose.yaml', 'test/compose.diagnostics.yaml',
*[f'configs/{profile}.yaml' for profile in args.profiles]]}}
save = lambda: (out/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n', encoding='utf-8')
manifest['requested_confirmation_rates'] = args.confirm_rates
manifest['expand_confirmation_rates'] = not args.no_expand
for name in manifest['source_sha256']:
target = out/'sources'/name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes((ROOT/name).read_bytes())
save()
nginx = json.loads(call(['docker', 'inspect', call(diag.compose('two', False)+['ps', '-q', 'nginx'])]))[0]
network = next(iter(nginx['NetworkSettings']['Networks']))
api_id = call(diag.compose('two', False)+['ps', '-q', 'api']).split()[0]
api_image = json.loads(call(['docker', 'inspect', api_id]))[0]['Image']
suffix = uuid.uuid4().hex
generator = 'lab3-saturation-client-' + suffix
mock = 'lab3-saturation-mock-' + suffix
generator_watch = None
counter = 0
manifest['generator_container'] = generator
manifest['mock_container'] = mock
save()
def run(profile, rate, phase, seconds, warm, expected, base='http://nginx'):
nonlocal counter
if (out/'CANCEL').exists():
raise InterruptedError('Cancellation requested; restoring ordinary profile')
counter += 1
label = f'{profile}-r{rate}-{phase}-{counter}'
directory = out/label
directory.mkdir()
print(f'START {label} ({seconds}s)', flush=True)
envs = {'BASE_URL': base, 'RATE': str(rate), 'SECONDS': str(seconds), 'WARMUP': str(warm),
'EXPECTED': str(expected), 'LABEL': label, 'PHASE': phase,
'OUTPUT': f'/results/{label}/benchmark.json', 'CALIBRATION': str(int(profile == 'calibration'))}
command = ['docker', 'exec']
for k, v in envs.items():
command += ['-e', f'{k}={v}']
command += [generator, 'k6', 'run', '--quiet', '/scripts/saturation.k6.js']
started = datetime.now(timezone.utc).isoformat()
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=seconds+warm+150)
(directory/'execution.json').write_text(json.dumps({'returncode': result.returncode,
'started': started, 'ended': datetime.now(timezone.utc).isoformat(), 'command': command}, indent=2)+'\n')
(directory/'k6.stdout').write_text(result.stdout, encoding='utf-8')
(directory/'k6.stderr').write_text(result.stderr, encoding='utf-8')
if result.returncode or not (directory/'benchmark.json').exists():
raise RuntimeError(f'{label}: k6 execution failed: {result.stderr[-1000:]}')
b = json.loads((directory/'benchmark.json').read_text(encoding='utf-8'))
if not b['timing']['measurement_started_epoch_ms']:
raise RuntimeError(f'{label}: no measured requests')
verdict = classify(b)
(directory/'criterion.json').write_text(json.dumps(verdict, indent=2)+'\n')
print(f"DONE {label}: {verdict['classification']} delivered={verdict['delivery_fraction']:.3f} "
f"RPS={b['measurement']['all']['successful_rps']:.2f} p95={verdict['all_request_p95_ms']:.0f}ms "
f"errors={b['measurement']['all']['errors']}", flush=True)
if profile != 'calibration':
manifest['runs'].append(label)
manifest['rates'] = sorted(set(manifest['rates']+[rate]))
save()
time.sleep(1 if verdict['classification'] == 'pass' else 5)
return label, verdict
try:
call(['docker', 'run', '-d', '--name', generator, '--network', network,
'-v', f'{ROOT / "scripts"}:/scripts:ro', '-v', f'{out}:/results',
'--entrypoint', 'sh', args.image, '-c', 'while :; do sleep 3600; done'])
manifest['generator_image'] = json.loads(call(['docker', 'inspect', generator]))[0]['Image']
manifest['generator_version'] = call(['docker', 'exec', generator, 'k6', 'version'])
manifest['docker'] = json.loads(call(['docker', 'info', '--format',
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}}}']))
generator_watch = watch(generator, 'generator', out)
call(['docker', 'run', '-d', '--name', mock, '--network', network,
'-v', f'{ROOT / "scripts/saturation-mock.cjs"}:/mock.cjs:ro',
'--entrypoint', 'node', api_image, '/mock.cjs'])
calibrations = []
for calibration_rate in [400, 200, 100]:
label, calibration = run('calibration', calibration_rate, 'calibration', 10, 3, 1, f'http://{mock}:8080')
calibrations.append({'label': label, 'rate': calibration_rate, **calibration})
if calibration['classification'] == 'pass': break
manifest['calibration_attempts'] = calibrations
manifest['calibration'] = {'label': label, **calibration}
low_label, low_calibration = run('calibration', 40, 'calibration-low-rate', 10, 3, 1, f'http://{mock}:8080')
manifest['calibration_low_rate'] = {'label': low_label, **low_calibration}
save()
call(['docker', 'rm', '-f', mock])
if calibration['classification'] != 'pass' or low_calibration['classification'] != 'pass':
raise RuntimeError('Generator calibration failed; do not claim API saturation')
if args.calibration_only:
return
for profile in args.profiles:
profile_started = datetime.now(timezone.utc).isoformat()
n = PROFILE_REPLICAS[profile]
print(f'SWITCH {profile}, pool={32//n} per replica', flush=True)
env = {**os.environ, 'DB_POOL_MAX': str(32//n)}
call(diag.compose(profile)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], env=env)
time.sleep(6)
containers = {role: call(diag.compose(profile)+['ps', '-q', role]).split() for role in ['api', 'postgres', 'nginx', 'redis']}
details = json.loads(call(['docker', 'inspect', *[c for group in containers.values() for c in group]]))
manifest['containers'][profile] = [{'id': c['Id'], 'name': c['Name'], 'image': c['Image'],
'nano_cpus': c['HostConfig']['NanoCpus'], 'memory_bytes': c['HostConfig']['Memory'],
'pool_max': next((e.split('=', 1)[1] for e in c['Config']['Env'] if e.startswith('DB_POOL_MAX=')), None)} for c in details]
profile_dir = out/f'observers-{profile}'
profile_dir.mkdir()
workers = []
first = len(manifest['runs'])
save()
try:
for role in ['postgres', 'nginx', 'redis']:
workers.append(watch(containers[role][0], role, profile_dir))
low = None
high = None
for rate in ([] if args.confirm_rates else [40, 80, 120, 160, 240, 320, 480]):
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
if v['classification'] == 'pass': low = rate
else:
high = rate
break
if low is None and not args.confirm_rates:
for rate in [20, 10]:
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
if v['classification'] == 'pass': low = rate; break
high = rate
if low is not None and high is not None:
for _ in range(2):
if high-low <= 20: break
middle = ((low+high)//20)*10
_, v = run(profile, middle, 'probe', args.probe_seconds, 5, n)
if v['classification'] == 'pass': low = middle
else: high = middle
confirmations = {}
for repeat in range(3):
for rate in sorted(set(args.confirm_rates or [r for r in [low, high] if r is not None])):
_, v = run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)
confirmations.setdefault(rate, []).append(v)
for _ in range(0 if args.no_expand else 2):
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
if good and bad and max(good) < min(bad): break
if any(v['classification'] == 'generator_limited' for vv in confirmations.values() for v in vv): break
rate = max(10, min(confirmations)-20) if not good else max(confirmations)+20
if rate in confirmations: break
confirmations[rate] = [run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)[1] for _ in range(3)]
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
manifest['boundaries'][profile] = {'confirmed_good': max(good) if good else None,
'confirmed_bad': min(bad) if bad else None, 'confirmations': confirmations,
'bracketed': bool(good and bad and max(good) < min(bad))}
print(f"BOUNDARY {profile}: {manifest['boundaries'][profile]['confirmed_good']} .. "
f"{manifest['boundaries'][profile]['confirmed_bad']} RPS", flush=True)
finally:
for worker in workers: stop_watch(worker, profile_dir)
log_result = subprocess.run(['docker', 'logs', '--timestamps', '--since', profile_started,
containers['nginx'][0]], cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=30)
events = [line for line in (log_result.stdout+'\n'+log_result.stderr).splitlines()
if any(term in line for term in ['worker_connections', 'no live upstreams',
'upstream timed out', 'download timed out', 'signal process started'])]
(profile_dir/'nginx-events.log').write_text('\n'.join(events)+'\n', encoding='utf-8')
for index, container in enumerate(containers['api'], 1):
call(['docker', 'cp', f'{container}:/tmp/scaling-pool.jsonl', str(profile_dir/f'api-{index}.jsonl')])
for label in manifest['runs'][first:]:
directory = out/label
for p in profile_dir.glob('*.jsonl'):
(directory/p.name).write_bytes(p.read_bytes())
(directory/'generator.jsonl').write_bytes((out/'generator.jsonl').read_bytes())
b = json.loads((directory/'benchmark.json').read_text())
summary = diag.summarize(directory, b)
samples = [r for r in diag.rows(directory/'generator.jsonl') if r['kind'] == 'cgroup'
and b['timing']['measurement_started_epoch_ms'] <= r['time_ms'] <= b['timing']['measurement_scheduled_end_epoch_ms']]
summary['resources']['generator'] = diag.resource_summary(samples)
summary['all_latency_ms'] = b['all_latency_ms']
summary['criterion'] = json.loads((directory/'criterion.json').read_text())
summary['diagnostics_complete'] = len(summary['pools']) == n and all(v['samples'] >= 2 for v in summary['resources'].values())
(directory/'summary.json').write_text(json.dumps(summary, indent=2)+'\n')
save()
finally:
if generator_watch: stop_watch(generator_watch, out)
for name in [mock, generator]:
subprocess.run(['docker', 'rm', '-f', name], cwd=ROOT, capture_output=True, timeout=30)
if not args.calibration_only:
print('RESTORE normal two profile, original pool setting', flush=True)
call(diag.compose('two', False)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'])
manifest['finished'] = datetime.now(timezone.utc).isoformat()
save()
if __name__ == '__main__':
main()
+325
View File
@@ -0,0 +1,325 @@
#!/usr/bin/env python3
"""Run a bounded scaling sweep and collect diagnostics without HTTP metric endpoints."""
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import subprocess
import time
import uuid
import hashlib
ROOT = Path(__file__).resolve().parents[1]
def command(args, timeout=180):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout)
if result.returncode:
raise RuntimeError(f"{args[0]} {args[1]} failed: {result.stderr[-1500:]}")
return result.stdout
def compose(profile, diagnostics=True):
args = ["docker", "compose", "-f", "compose.yaml", "-f", f"configs/{profile}.yaml"]
if diagnostics:
args += ["-f", "test/compose.diagnostics.yaml"]
return args
def rows(path):
data = []
if path.exists():
text = path.read_text(encoding="utf-8")
decoder = json.JSONDecoder()
position = 0
while position < len(text):
if text[position].isspace():
position += 1
continue
try:
value, position = decoder.raw_decode(text, position)
data.append(value)
except json.JSONDecodeError:
break # Last record can be in progress when a running observer is copied.
return data
def delta(first, last, name):
return max(0, last.get(name, 0) - first.get(name, 0))
def resource_summary(samples):
if len(samples) < 2:
return {"samples": len(samples)}
first, last = samples[0], samples[-1]
elapsed = (last["time_ms"] - first["time_ms"]) / 1000
periods = delta(first, last, "cpu_nr_periods")
return {
"samples": len(samples), "coverage_seconds": elapsed,
"max_sample_gap_seconds": max((b["time_ms"] - a["time_ms"]) / 1000 for a, b in zip(samples, samples[1:])),
"mean_cpu_cores": delta(first, last, "cpu_usage_usec") / max(elapsed * 1e6, 1),
"throttled_seconds": delta(first, last, "cpu_throttled_usec") / 1e6,
"throttled_period_fraction": delta(first, last, "cpu_nr_throttled") / max(periods, 1),
"peak_memory_mib": max(s.get("memory_bytes", 0) or 0 for s in samples) / 1024**2,
"oom_kills": delta(first, last, "memory_oom_kill"),
}
def summarize(directory, benchmark):
start = benchmark["timing"]["measurement_started_epoch_ms"]
end = benchmark["timing"]["measurement_scheduled_end_epoch_ms"]
within = lambda data: [r for r in data if start <= r["time_ms"] <= end]
resources = {}
pools = {}
for path in sorted(directory.glob("api-*.jsonl")):
data = within(rows(path))
flattened = [{**r, **{f"cpu_{k}": v for k, v in (r.get("cpu") or {}).items()},
**{f"memory_{k}": v for k, v in (r.get("memory_events") or {}).items()}}
for r in data]
resources[path.stem] = resource_summary(flattened)
if len(data) < 2:
continue
first, last = data[0]["pools"], data[-1]["pools"]
acquisitions = sum(p["acquisitions"] for p in last) - sum(p["acquisitions"] for p in first)
acquire_ms = sum(p["acquire_ms"] for p in last) - sum(p["acquire_ms"] for p in first)
intervals = [p for r in data for p in r["pools"]]
totals = [sum(p["total"] for p in r["pools"]) for r in data]
pools[path.stem] = {
"samples": len(data), "acquisitions": acquisitions,
"acquisition_errors": sum(p["errors"] for p in last) - sum(p["errors"] for p in first),
"mean_acquisition_ms": acquire_ms / max(acquisitions, 1),
"max_acquisition_ms": max((p["interval_max_ms"] for p in intervals), default=0),
"max_waiting": max((p["interval_max_waiting"] for p in intervals), default=0),
"queued_interval_fraction": sum(p["interval_max_waiting"] > 0 for p in intervals) / max(len(intervals), 1),
"max_total": max((p["total"] for p in intervals), default=0),
"observed_connection_increases": sum(max(0, b-a) for a, b in zip(totals, totals[1:])),
"observed_connection_decreases": sum(max(0, a-b) for a, b in zip(totals, totals[1:])),
"event_loop_p99_max_ms": max(r["event_loop_delay_ms"]["p99"] for r in data),
}
pg = within(rows(directory / "postgres.jsonl"))
pg_samples = [r for r in pg if r["kind"] == "postgres"]
pg_samples += within(rows(directory / "postgres-sql.jsonl"))
pg_samples.sort(key=lambda r: r["time_ms"])
host_samples = [r for r in pg if r["kind"] == "host"]
host = {"samples": len(host_samples)}
if len(host_samples) > 1:
host_seconds = (host_samples[-1]["time_ms"] - host_samples[0]["time_ms"])/1000
host.update({key + "_fraction": delta(host_samples[0], host_samples[-1], key + "_total_usec") / max(host_seconds*1e6, 1)
for key in ("cpu_some", "memory_some", "memory_full", "io_some", "io_full")})
host["min_available_mib"] = min(r["MemAvailable_kib"] for r in host_samples)/1024
host["max_swap_used_mib"] = max(r["SwapTotal_kib"] - r["SwapFree_kib"] for r in host_samples)/1024
waits = {}
max_active = max_idle_tx = 0
for sample in pg_samples:
max_active = max(max_active, sum(a["count"] for a in sample["activity"] if a["state"] == "active"))
max_idle_tx = max(max_idle_tx, sum(a["count"] for a in sample["activity"] if a["state"] == "idle in transaction"))
for a in sample["activity"]:
if a["state"] == "active":
key = f"{a['wait_event_type'] or 'no_wait_event'}/{a['wait_event'] or '-'}"
waits[key] = waits.get(key, 0) + a["count"]
for role in ("postgres", "nginx", "redis"):
resources[role] = resource_summary([r for r in within(rows(directory / f"{role}.jsonl")) if r["kind"] == "cgroup"])
upstream = [r for r in within(rows(directory / "nginx.jsonl")) if r["kind"] == "upstream"]
changes = sum(a["servers"] != b["servers"] for a, b in zip(upstream, upstream[1:]))
m = benchmark["measurement"]
return {
"label": benchmark["label"], "profile": benchmark["profile"],
"measurement": m, "client": benchmark["client"],
"resources": resources, "pools": pools,
"host": host,
"postgres": {"samples": len(pg_samples), "max_active": max_active,
"max_idle_in_transaction": max_idle_tx, "active_wait_observations": waits},
"nginx": {"samples": len(upstream), "upstream_changes": changes,
"min_ready": min((len(r["servers"]) for r in upstream), default=None),
"max_ready": max((len(r["servers"]) for r in upstream), default=None)},
"limits_reached": {
"generator_late": m["scheduler_late_slots"] / m["scheduled_slots"] > 0.05,
"client_capacity": (m["client_dropped_slots"] - m["scheduler_late_slots"]) / m["scheduled_slots"] > 0.05,
"request_errors": m["all"]["errors"] / max(m["all"]["attempted"], 1) > 0.01,
"latency": (m["all"]["successful_latency_ms"]["p95"] or 0) > 1000,
},
}
def run_one(config, profile, rate, repeat, containers):
label = f"{profile}-r{rate}-repeat{repeat}"
directory = config.output / label
directory.mkdir(parents=True, exist_ok=False)
workers = []
stop = f"/tmp/scaling-observer-{uuid.uuid4().hex}.stop"
observer = (ROOT / "scripts/diagnostics/observer.sh").read_bytes()
client_container = None
try:
for role in ("postgres", "nginx", "redis"):
output = (directory / f"{role}.jsonl").open("wb")
errors = (directory / f"{role}.stderr").open("wb")
child = subprocess.Popen(["docker", "exec", "-i", containers[role][0],
"sh", "-s", role, stop], cwd=ROOT,
stdin=subprocess.PIPE, stdout=output, stderr=errors)
workers.append((role, child, output, errors))
child.stdin.write(observer)
child.stdin.close()
started = datetime.now(timezone.utc).isoformat()
target = directory / "benchmark.json"
args = ["node", "scripts/benchmark-scaling.mjs", "--base-url", config.base_url,
"--label", label, "--rate", str(rate), "--duration-seconds", str(config.duration_seconds),
"--warmup-seconds", str(config.warmup_seconds), "--seed-messages", "30",
"--expected-instances", str({"one": 1, "two": 2, "four": 4}[profile]),
"--output", str(target)]
if config.client == "container":
nginx = json.loads(command(["docker", "inspect", containers["nginx"][0]]))[0]
api = json.loads(command(["docker", "inspect", containers["api"][0]]))[0]
networks = set(nginx["NetworkSettings"]["Networks"]) & set(api["NetworkSettings"]["Networks"])
if len(networks) != 1:
raise RuntimeError("expected one common API/nginx network")
client_container = "lab3-diagnostic-client-" + uuid.uuid4().hex
args[args.index("--base-url") + 1] = config.container_url
args[args.index("--output") + 1] = "/tmp/scaling-benchmark.json"
args[1] = "/benchmark.mjs"
args = ["docker", "run", "--name", client_container, "--no-healthcheck",
"--network", next(iter(networks)),
"-v", f"{ROOT / 'scripts/benchmark-scaling.mjs'}:/benchmark.mjs:ro",
"--entrypoint", "node", api["Image"], *args[1:]]
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=config.duration_seconds + config.warmup_seconds + 120)
(directory / "benchmark.stderr").write_text(result.stderr, encoding="utf-8")
if client_container is not None:
command(["docker", "cp", f"{client_container}:/tmp/scaling-benchmark.json", str(target)])
if not target.exists():
raise RuntimeError(f"{label}: benchmark did not produce results: {result.stderr}")
benchmark = json.loads(target.read_text(encoding="utf-8"))
for index, container in enumerate(containers["api"], 1):
command(["docker", "cp", f"{container}:/tmp/scaling-pool.jsonl", str(directory / f"api-{index}.jsonl")])
ended = datetime.now(timezone.utc).isoformat()
# Docker puts nginx stderr notices on CLI stderr; collect both streams.
log_result = subprocess.run(["docker", "logs", "--timestamps", "--since", started,
"--until", ended, containers["nginx"][0]], cwd=ROOT,
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30)
diagnostic_lines = [line for line in (log_result.stdout + log_result.stderr).splitlines()
if any(key in line for key in ("signal process started", "download timed out",
"no live upstreams", "upstream timed out"))]
(directory / "nginx-events.log").write_text("\n".join(diagnostic_lines), encoding="utf-8")
(directory / "execution.json").write_text(json.dumps({"returncode": result.returncode,
"started": started, "ended": ended, "command": args}, indent=2), encoding="utf-8")
finally:
if client_container is not None:
# Only the uniquely named disposable generator created by this run.
subprocess.run(["docker", "rm", "-f", client_container], cwd=ROOT, capture_output=True, timeout=30)
for role, child, output, errors in workers:
try:
command(["docker", "exec", containers[role][0], "touch", stop], timeout=15)
child.wait(timeout=15)
except (RuntimeError, subprocess.TimeoutExpired):
child.terminate()
child.wait(timeout=10)
output.close()
errors.close()
if role == "postgres":
command(["docker", "cp", f"{containers[role][0]}:{stop}.pg.jsonl", str(directory / "postgres-sql.jsonl")])
command(["docker", "exec", containers[role][0], "rm", "-f", stop + ".pg.jsonl"])
summary = summarize(directory, benchmark)
summary["diagnostics_complete"] = (
len(summary["pools"]) == len(containers["api"])
and summary["postgres"]["samples"] > 0
and summary["nginx"]["samples"] > 0
and all(v["samples"] > 0 for v in summary["resources"].values())
)
(directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
if not summary["diagnostics_complete"]:
raise RuntimeError(f"{label}: diagnostics incomplete; inspect observer stderr files")
m = summary["measurement"]
print(f"DONE {label}: RPS={m['all']['successful_rps']} p95={m['all']['successful_latency_ms']['p95']}ms "
f"errors={m['all']['errors']} drops={m['client_dropped_slots']}/{m['scheduled_slots']} "
f"pool_wait_max={max((p['max_waiting'] for p in summary['pools'].values()), default=0)}", flush=True)
return summary
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
parser.add_argument("--client", choices=("host", "container"), default="host")
parser.add_argument("--container-url", default="http://nginx")
parser.add_argument("--profiles", nargs="+", choices=("one", "two", "four"), default=["one", "two", "four"])
parser.add_argument("--rates", nargs="+", type=int, default=[100], help="Scheduled requests/s (default: 100 only)")
parser.add_argument("--duration-seconds", type=int, default=60, help="Measured phase per run (default: 60 seconds)")
parser.add_argument("--warmup-seconds", type=int, default=15, help="Read-only warmup per run (default: 15 seconds)")
parser.add_argument("--repetitions", type=int, default=3)
parser.add_argument("--full-matrix", action="store_true", help="Run all requested rates; record threshold crossings instead of stopping a profile")
parser.add_argument("--output", type=Path, required=True)
config = parser.parse_args()
if config.repetitions < 1 or config.duration_seconds < 1 or config.warmup_seconds < 0 or any(r < 1 or r > 1000 for r in config.rates):
parser.error("invalid duration, repetitions or rates")
config.output = config.output.resolve()
config.output.mkdir(parents=True, exist_ok=False)
manifest = {"started": datetime.now(timezone.utc).isoformat(), "profiles": config.profiles,
"client": config.client,
"rates": config.rates, "duration_seconds": config.duration_seconds,
"warmup_seconds": config.warmup_seconds, "repetitions": config.repetitions,
"full_matrix": config.full_matrix,
"stop_rule": ("record only; continue all requested rates after >=2 repeats exceed thresholds: " if config.full_matrix
else "stop a profile after >=2 repeats at one rate exceed any threshold: ") +
"scheduler late >5%, capacity drops >5%, request errors >1%, success p95 >1000ms",
"sampling": "API queue every 100ms; API JSON/cgroup about 1s; other services and PG about 2s",
"observer_version": 3,
"source_sha256": {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in (
"scripts/diagnostics/pool.cjs", "scripts/diagnostics/observer.sh",
"scripts/benchmark-scaling.mjs", "scripts/run-scaling-diagnostics.py")},
"notes": ["All profiles use the same instrumentation; actual per-replica pool limits are recorded in containers.",
"A container client uses the nginx service directly; it shares Docker VM CPU and bypasses Windows port forwarding.",
"Acquisition latency includes opening new connections; it is not SQL execution time.",
"Observers consume resources inside the existing container budgets.",
"Instrumentation stores no tokens, passwords, message text or SQL parameters."],
"containers": {}, "environments": {}, "runs": [], "stopped_profiles": {}, "threshold_crossings": {}}
manifest_path = config.output / "manifest.json"
save = lambda: manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
save()
try:
for profile in config.profiles:
print(f"SWITCH {profile}", flush=True)
command(compose(profile) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
time.sleep(6) # Allow readiness's two successes and nginx reload to settle.
containers = {}
for role in ("api", "postgres", "nginx", "redis"):
containers[role] = command(compose(profile) + ["ps", "-q", role]).split()
if not containers[role]:
raise RuntimeError(f"no running {role}")
details = json.loads(command(["docker", "inspect", *[c for ids in containers.values() for c in ids]]))
running = [json.loads(line) for line in command(["docker", "ps", "--format", "{{json .}}"]).splitlines()]
manifest["environments"][profile] = {
"docker": json.loads(command(["docker", "info", "--format",
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}},"kernel":{{json .KernelVersion}}}'])),
"running_containers": [{"name": row.get("Names"), "image": row.get("Image")} for row in running],
}
manifest["containers"][profile] = [{"id": c["Id"], "name": c["Name"], "image": c["Image"],
"nano_cpus": c["HostConfig"]["NanoCpus"], "memory_bytes": c["HostConfig"]["Memory"],
"pool_max": next((e.split("=", 1)[1] for e in c["Config"]["Env"] if e.startswith("DB_POOL_MAX=")), None)} for c in details]
save()
for rate in config.rates:
batch = []
for repeat in range(1, config.repetitions + 1):
print(f"START {profile} rate={rate} repeat={repeat}", flush=True)
summary = run_one(config, profile, rate, repeat, containers)
batch.append(summary)
manifest["runs"].append(summary["label"])
save()
reasons = [key for key in batch[0]["limits_reached"]
if sum(r["limits_reached"][key] for r in batch) >= min(2, config.repetitions)]
if reasons:
manifest["threshold_crossings"].setdefault(profile, []).append({"rate": rate, "reasons": reasons})
if not config.full_matrix:
manifest["stopped_profiles"][profile] = {"rate": rate, "reasons": reasons}
print(f"{'CONTINUE (full matrix)' if config.full_matrix else 'STOP'} {profile} at {rate}: {', '.join(reasons)}", flush=True)
save()
if not config.full_matrix:
break
finally:
print("RESTORE two (without diagnostic preload)", flush=True)
command(compose("two", diagnostics=False) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
manifest["finished"] = datetime.now(timezone.utc).isoformat()
save()
if __name__ == "__main__":
main()
+23
View File
@@ -0,0 +1,23 @@
// Generator calibration only: no application or database state.
const http = require('node:http');
const item = { id: '00000000-0000-4000-8000-000000000001',
chat_id: '00000000-0000-4000-8000-000000000002',
sender_id: '00000000-0000-4000-8000-000000000003',
text: 'Synthetic benchmark payload '.repeat(3), created_at: '2026-01-01T00:00:00.000Z' };
const page = JSON.stringify({ items: Array.from({ length: 50 }, () => item), next_cursor: null });
const keys = new Set();
http.createServer(async (req, res) => {
for await (const chunk of req) { /* consume POST body */ }
if (req.method === 'POST') {
const key = req.headers['idempotency-key'];
if (!key || keys.has(key)) {
res.writeHead(409, { 'Content-Type': 'application/json', 'X-Instance-Id': 'calibration' });
res.end(JSON.stringify({ error: 'calibration detected a repeated write key' }));
return;
}
keys.add(key);
}
res.writeHead(req.method === 'GET' ? 200 : 201,
{ 'Content-Type': 'application/json', 'X-Instance-Id': 'calibration' });
res.end(req.method === 'GET' ? page : JSON.stringify(item));
}).listen(8080, '0.0.0.0');
+166
View File
@@ -0,0 +1,166 @@
import http from 'k6/http';
import encoding from 'k6/encoding';
import execution from 'k6/execution';
import { check, fail, sleep } from 'k6';
import { Counter, Trend } from 'k6/metrics';
const base = __ENV.BASE_URL || 'http://nginx';
const rate = Number(__ENV.RATE || 100);
const seconds = Number(__ENV.SECONDS || 30);
const warmup = Number(__ENV.WARMUP || 10);
const expected = Number(__ENV.EXPECTED || 1);
const calibration = __ENV.CALIBRATION === '1';
const attempts = new Counter('bench_attempts');
const successes = new Counter('bench_successes');
const failures = new Counter('bench_failures');
const latency = new Trend('bench_latency', true);
const successLatency = new Trend('bench_success_latency', true);
const epoch = new Trend('bench_start_epoch');
const operations = Object.fromEntries(['read', 'write'].map((op) => [op, {
attempts: new Counter(`bench_${op}_attempts`),
successes: new Counter(`bench_${op}_successes`),
failures: new Counter(`bench_${op}_failures`),
latency: new Trend(`bench_${op}_latency`, true),
}]));
const statuses = Object.fromEntries(['200', '201', 'other', 'network'].map((key) => [key, new Counter(`bench_status_${key}`)]));
const arrival = { executor: 'constant-arrival-rate', rate, timeUnit: '1s',
preAllocatedVUs: 128, maxVUs: 128 };
export const options = {
scenarios: {
warmup: { ...arrival, exec: 'warmupLoad', duration: `${warmup}s`, gracefulStop: '5s' },
measurement: { ...arrival, exec: 'default', startTime: `${warmup}s`,
duration: `${seconds}s`, gracefulStop: '10s' },
},
summaryTrendStats: ['min', 'med', 'avg', 'max', 'p(95)', 'p(99)'],
setupTimeout: '120s',
};
function request(method, path, token, body, key) {
const headers = { Accept: 'application/json' };
if (token) headers.Authorization = token;
if (key) headers['Idempotency-Key'] = key;
if (body !== undefined) headers['Content-Type'] = 'application/json';
return http.request(method, base + path, body === undefined ? null : JSON.stringify(body), {
headers, timeout: '5s', tags: { name: path.includes('/messages') ? 'messages' : 'setup' },
});
}
function required(response, status, field) {
let data;
try { data = response.json(); } catch { fail(`setup: invalid JSON, status ${response.status}`); }
if (response.status !== status || !data?.[field]) fail(`setup: expected ${status}/${field}, got ${response.status}`);
return data;
}
export function setup() {
const salt = `${Date.now().toString(36)}-${Math.floor(Math.random() * 1e9)}`;
let token = 'Bearer calibration';
let path = '/messages';
if (!calibration) {
const users = ['alice', 'bob'].map((label) => {
const username = `bench_${label}_${salt.replaceAll('-', '')}`;
const password = `T3st-${salt}-Aa1!`;
const user = required(request('POST', '/api/v1/users', null,
{ username, display_name: `Benchmark ${label}`, password }), 201, 'id');
const auth = `Basic ${encoding.b64encode(`${username}:${password}`)}`;
const session = required(request('POST', '/api/v1/auth/sessions', auth), 201, 'token');
return { id: user.id, token: session.token };
});
token = `Bearer ${users[0].token}`;
const chat = required(request('POST', '/api/v1/chats', token,
{ title: 'Saturation benchmark', member_ids: users.map((u) => u.id) }), 201, 'id');
path = `/api/v1/chats/${chat.id}/messages`;
for (let i = 0; i < 30; i++) required(request('POST', path, token,
{ text: `seed-${salt}-${i}` }, `seed-${salt}-${i}`), 201, 'id');
}
const seen = new Set();
const until = Date.now() + 20000;
do {
const response = request('GET', `${path}?limit=50`, token);
if (response.status !== 200) fail(`warmup status ${response.status}`);
const id = response.headers['X-Instance-Id'];
if (id) seen.add(id);
if (seen.size === expected) break;
sleep(0.02);
} while (Date.now() < until);
if (seen.size !== expected) fail(`warmup observed ${seen.size} instances, expected ${expected}`);
return { token, path, salt };
}
export function warmupLoad(data) { iteration(data, false); }
export default function (data) { iteration(data, true); }
function iteration(data, measuring) {
if (measuring && execution.scenario.iterationInTest === 0) epoch.add(Date.now());
const sequence = execution.scenario.iterationInTest;
const identity = `${execution.scenario.name}-${sequence}`;
const op = ((sequence * 37 + 17) % 100) < 80 ? 'read' : 'write';
if (measuring) { attempts.add(1); operations[op].attempts.add(1); }
const response = op === 'read'
? request('GET', `${data.path}?limit=50`, data.token)
: request('POST', data.path, data.token, { text: `bench-${identity}-${data.salt}` }, `${data.salt}-${identity}`);
if (!measuring) return;
let payload;
try { payload = response.json(); } catch { payload = null; }
const instance = response.headers['X-Instance-Id'];
const ok = response.status === (op === 'read' ? 200 : 201) && Boolean(instance)
&& (op === 'read' ? Array.isArray(payload?.items) : Boolean(payload?.id));
const elapsed = response.timings.duration + response.timings.blocked;
latency.add(elapsed);
successes.add(ok ? 1 : 0);
failures.add(ok ? 0 : 1);
operations[op].successes.add(ok ? 1 : 0);
operations[op].failures.add(ok ? 0 : 1);
if (ok) { successLatency.add(elapsed); operations[op].latency.add(elapsed); }
statuses[String(response.status)]?.add(1);
if (![200, 201].includes(response.status)) statuses[response.status === 0 ? 'network' : 'other'].add(1);
if (instance) check(response, { [`instance:${instance}`]: () => true });
}
export function handleSummary(data) {
const count = (name) => data.metrics[name]?.values.count || 0;
const times = (name) => ({ p50: data.metrics[name]?.values.med ?? null,
p95: data.metrics[name]?.values['p(95)'] ?? null,
p99: data.metrics[name]?.values['p(99)'] ?? null });
const start = data.metrics.bench_start_epoch?.values.min;
const totalDropped = count('dropped_iterations');
const dropped = Math.max(0, rate * seconds - count('bench_attempts'));
const instances = Object.fromEntries((data.root_group.checks || []).filter((c) => c.name.startsWith('instance:'))
.map((c) => [c.name.slice(9), c.passes]));
const group = (op) => {
const prefix = op === 'all' ? 'bench' : `bench_${op}`;
return { attempted: count(`${prefix}_attempts`), successful: count(`${prefix}_successes`),
errors: count(`${prefix}_failures`), successful_rps: count(`${prefix}_successes`) / seconds,
successful_latency_ms: times(op === 'all' ? 'bench_success_latency' : `${prefix}_latency`),
status_counts: {}, error_counts: {}, instance_counts: op === 'all' ? instances : {} };
};
const result = {
label: __ENV.LABEL, profile: { arrival_model: 'k6 constant-arrival-rate; one HTTP request per iteration',
offered_rps: rate, duration_seconds: seconds, warmup_seconds: warmup, warmup_profile: 'same 80/20 workload at target rate',
read_percent: 80, write_percent: 20, initial_messages: calibration ? 0 : 30, history_limit: 50,
max_in_flight: 128, timeout_ms: 5000, automatic_retries: false, expected_instances: expected,
phase: __ENV.PHASE || 'probe' },
measurement: { scheduled_slots: rate * seconds, client_dropped_slots: dropped,
scheduler_late_slots: 0, read: group('read'), write: group('write'), all: group('all') },
timing: { measurement_started_epoch_ms: start,
measurement_scheduled_end_epoch_ms: start + seconds * 1000,
measurement_completed_epoch_ms: Date.now() },
client: { engine: 'k6', scheduler_lateness_measured: false,
raw_dropped_iterations_including_warmup: totalDropped,
peak_vus: data.metrics.vus?.values.max, max_vus: 128 },
all_latency_ms: times('bench_latency'), raw_metrics: data.metrics,
notes: ['Setup and in-scenario mixed warmup excluded from custom bench metrics.',
'Native warmup and measurement scenarios; no Date.now gating of measured requests.',
'Latency uses native HTTP duration plus blocked time, not subtraction of wall-clock timestamps.',
'Measurement missing slots are planned minus actual starts; raw k6 dropped iterations also include warmup.',
'scheduler_late_slots is a compatibility placeholder, not a k6 measurement.',
'Dropped iterations and actual delivered fraction must be checked.',
'No credentials or message bodies are exported.'],
};
result.measurement.all.status_counts = Object.fromEntries(['200', '201', 'other', 'network']
.map((status) => [status, count(`bench_status_${status}`)]));
return { [__ENV.OUTPUT]: JSON.stringify(result, null, 2),
stdout: JSON.stringify({ label: result.label, attempted: result.measurement.all.attempted,
errors: result.measurement.all.errors, p95: result.all_latency_ms.p95, dropped }) + '\n' };
}
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Create CSV and Markdown from completed diagnostic runs (can run during a sweep)."""
import argparse
import csv
import json
from pathlib import Path
import statistics
import importlib.util
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
parser.add_argument("--refresh", action="store_true", help="Recompute summaries from raw records")
args = parser.parse_args()
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
results = []
raw_summaries = []
if args.refresh:
spec = importlib.util.spec_from_file_location("diagnostics", Path(__file__).with_name("run-scaling-diagnostics.py"))
diagnostics = importlib.util.module_from_spec(spec)
spec.loader.exec_module(diagnostics)
for label in manifest["runs"]:
path = args.directory / label / "summary.json"
if not path.exists():
continue
s = json.loads(path.read_text(encoding="utf-8"))
if args.refresh:
benchmark = json.loads((path.parent / "benchmark.json").read_text(encoding="utf-8"))
refreshed = diagnostics.summarize(path.parent, benchmark)
refreshed["diagnostics_complete"] = s.get("diagnostics_complete", False)
s = refreshed
path.write_text(json.dumps(s, indent=2) + "\n", encoding="utf-8")
raw_summaries.append(s)
m, p, r = s["measurement"], s["pools"], s["resources"]
acquisitions = sum(v["acquisitions"] for v in p.values())
results.append({
"label": label, "replicas": s["profile"]["expected_instances"],
"configuration": next((profile for profile in manifest.get("profiles", [])
if label.startswith(profile + "-r")), str(s["profile"]["expected_instances"])),
"phase": s["profile"].get("phase", "measurement"),
"delivery_valid": s.get("criterion", {}).get("delivery_valid"),
"duration_seconds": s["profile"]["duration_seconds"],
"offered_rps": s["profile"]["offered_rps"], "successful_rps": m["all"]["successful_rps"],
"attempted_rps": m["all"]["attempted"] / s["profile"]["duration_seconds"],
"p50_ms": m["all"]["successful_latency_ms"]["p50"],
"p95_ms": m["all"]["successful_latency_ms"]["p95"],
"all_request_p95_ms": s.get("all_latency_ms", {}).get("p95"),
"p99_ms": m["all"]["successful_latency_ms"]["p99"],
"errors": m["all"]["errors"], "dropped": m["client_dropped_slots"],
"drop_percent": 100*m["client_dropped_slots"]/m["scheduled_slots"],
"late_percent": 100*m["scheduler_late_slots"]/m["scheduled_slots"],
"acquisition_mean_ms": sum(v["mean_acquisition_ms"]*v["acquisitions"] for v in p.values()) / max(acquisitions, 1),
"acquisition_max_ms": max((v["max_acquisition_ms"] for v in p.values()), default=0),
"pool_max_waiting": max((v["max_waiting"] for v in p.values()), default=0),
"pool_observed_connection_increases": sum(v.get("observed_connection_increases", 0) for v in p.values()),
"pool_observed_connection_decreases": sum(v.get("observed_connection_decreases", 0) for v in p.values()),
"api_cpu_cores": sum(v.get("mean_cpu_cores", 0) for k, v in r.items() if k.startswith("api-")),
"api_max_throttle_percent": 100*max((v.get("throttled_period_fraction", 0) for k, v in r.items() if k.startswith("api-")), default=0),
"pg_cpu_cores": r["postgres"].get("mean_cpu_cores"),
"pg_throttle_percent": 100*r["postgres"].get("throttled_period_fraction", 0),
"nginx_cpu_cores": r["nginx"].get("mean_cpu_cores"),
"redis_cpu_cores": r["redis"].get("mean_cpu_cores"),
"host_cpu_pressure_percent": 100*s.get("host", {}).get("cpu_some_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
"host_memory_pressure_percent": 100*s.get("host", {}).get("memory_full_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
"host_io_pressure_percent": 100*s.get("host", {}).get("io_full_fraction", 0) if s.get("host", {}).get("samples", 0) > 1 else None,
"host_min_available_mib": s.get("host", {}).get("min_available_mib"),
"host_max_swap_used_mib": s.get("host", {}).get("max_swap_used_mib"),
"pg_max_active": s["postgres"]["max_active"],
"pg_max_idle_tx": s["postgres"]["max_idle_in_transaction"],
"pg_active_wait_observations": json.dumps(s["postgres"]["active_wait_observations"], sort_keys=True),
"nginx_min_ready": s["nginx"]["min_ready"], "nginx_changes": s["nginx"]["upstream_changes"],
"min_resource_coverage_seconds": min(v.get("coverage_seconds", 0) for v in r.values()),
})
if not results:
print("No completed runs yet")
return
with (args.directory / "results.csv").open("w", encoding="utf-8", newline="") as output:
writer = csv.DictWriter(output, fieldnames=results[0].keys())
writer.writeheader()
writer.writerows(results)
pg_limits = next((c for containers in manifest.get("containers", {}).values()
for c in containers if "-postgres-" in c.get("name", "")), None)
pg_description = (f"PostgreSQL — {pg_limits['nano_cpus']/1e9:g} CPU/{pg_limits['memory_bytes']/1024**2:g} МиБ."
if pg_limits else "Лимиты PostgreSQL не записаны в manifest.")
pool_descriptions = [f"{profile}: " + "/".join(sorted({str(c['pool_max']) for c in containers
if "-api-" in c.get("name", "") and c.get("pool_max") is not None}))
for profile, containers in manifest.get("containers", {}).items()]
pool_description = "; ".join(pool_descriptions) or "см. manifest"
api_budgets = []
for profile, containers in manifest.get("containers", {}).items():
apis = [c for c in containers if "-api-" in c.get("name", "")]
if apis:
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
memory = sum(c['memory_bytes'] for c in apis) / 1024**2
api_budgets.append(f"{profile}: {cpu:g} CPU/{memory:g} МиБ")
api_description = "; ".join(api_budgets) or "см. manifest"
probe_description = (f"Разведочные прогоны: {manifest['probe_duration_seconds']} с после "
f"{manifest['probe_warmup_seconds']} с прогрева. Указанная ниже длительность "
"относится к подтверждающим прогонам. " if "probe_duration_seconds" in manifest else "")
lines = ["# Диагностическая серия масштабирования", "",
probe_description + f"Завершено прогонов: {len(results)}. Длительность измерения — {manifest['duration_seconds']} с, "
f"прогрев — {manifest['warmup_seconds']} с. Пул на реплику — {pool_description}. Общий бюджет API — {api_description}; " + pg_description, "",
("Клиент: Linux-контейнер через внутренний адрес Nginx; разделяет ресурсы Docker VM." if manifest.get("client") == "container"
else "Клиент: Windows-хост через опубликованный порт Nginx."),
f"Версия SQL-наблюдателя: {manifest.get('observer_version', '1/2, первоначальная серия')}. "
"Серия с версиями 1/2 имеет повышенные накладные расходы наблюдения; см. калибровку в итоговой интерпретации.", "",
"Методика и повторение: [scripts/diagnostics/README.md](../../../scripts/diagnostics/README.md). "
"Исходные HTTP-данные, временные ряды и summary каждого прогона находятся в его подкаталоге. "
"Полная таблица метрик: [results.csv](results.csv). Все настройки и причины остановки нагрузки: [manifest.json](manifest.json).", "",
"Показатели ниже — результаты каждого прогона. CPU выражен в долях одного ядра; "
"throttle — доля периодов CPU, в которых cgroup ограничивала выполнение. Получение соединения "
"включает установку новых соединений и задержки event loop. Очередь наблюдалась каждые 100 мс. "
"Наблюдатели имеют собственные накладные расходы; результаты не являются ненаблюдаемой максимальной ёмкостью API.", "",
"| Прогон | Успешные RPS | p95, мс | Ошибки | Пропуски, % | Пул: среднее / max, мс | Max очередь | CPU API / PG | Throttle API max / PG, % | Upstream min / изменений |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
for r in results:
lines.append(f"| [{r['label']}]({r['label']}/summary.json) | {r['successful_rps']:.2f} | "
f"{r['p95_ms']:.0f} | {r['errors']} | {r['drop_percent']:.1f} | "
f"{r['acquisition_mean_ms']:.2f} / {r['acquisition_max_ms']:.0f} | {r['pool_max_waiting']} | "
f"{r['api_cpu_cores']:.2f} / {r['pg_cpu_cores'] or 0:.2f} | "
f"{r['api_max_throttle_percent']:.1f} / {r['pg_throttle_percent']:.1f} | "
f"{r['nginx_min_ready']} / {r['nginx_changes']} |")
lines += ["", "## Повторы на одном уровне нагрузки", "",
"Здесь показаны медианы результатов отдельных прогонов. Медиана p95 не является p95 объединённых запросов.", "",
"| Профиль | Задано RPS | Повторов | Медиана отправленных RPS | Медиана успешных RPS | Медиана p95, мс | Всего ошибок | Медиана пропусков, % |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |"]
for key in sorted({(r["configuration"], r["offered_rps"]) for r in results}):
group = [r for r in results if (r["configuration"], r["offered_rps"]) == key]
lines.append(f"| {key[0]} | {key[1]} | {len(group)} | "
f"{statistics.median(r['attempted_rps'] for r in group):.2f} | "
f"{statistics.median(r['successful_rps'] for r in group):.2f} | "
f"{statistics.median(r['p95_ms'] for r in group):.0f} | "
f"{sum(r['errors'] for r in group)} | "
f"{statistics.median(r['drop_percent'] for r in group):.1f} |")
lines += ["", "## Границы серии", ""]
for profile, stop in manifest["stopped_profiles"].items():
lines.append(f"- {profile}: повышение остановлено после {stop['rate']} запросов/с: {', '.join(stop['reasons'])}.")
if manifest.get("full_matrix"):
lines.append("Режим полной матрицы: превышения порогов не останавливают повышение нагрузки; все перечисленные точки запланированы к выполнению.")
for profile, crossings in manifest.get("threshold_crossings", {}).items():
for crossing in crossings:
lines.append(f"- {profile}, {crossing['rate']} запросов/с: {', '.join(crossing['reasons'])}.")
lines += ["", "Превышение порога задержки или пропусков не доказывает насыщение конкретного компонента. "
"Для интерпретации необходимо сопоставить очередь пула, ограничения CPU, ожидания PostgreSQL и изменения upstream. "
"SQL-тексты, токены и пароли в этой серии не сохраняются.", ""]
if (args.directory / "findings.md").exists():
lines += ["## Итоговая интерпретация", "",
"[Выводы и графики](findings.md). Проверка полноты серии: [audit.json](audit.json).", ""]
(args.directory / "README.md").write_text("\n".join(lines), encoding="utf-8")
print(f"Wrote {len(results)} runs to {args.directory / 'results.csv'}")
if __name__ == "__main__":
main()